diff --git a/.gitignore b/.gitignore index 61d8642b..9d61bef3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ Thumbs.db # Python virtual-envs & tooling .venv*/ +venv/ .python-version __pycache__/ *.egg-info/ @@ -38,3 +39,6 @@ tmp/ # Project-specific files history.txt digest.txt + +# Environment variables +.env diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f38e4d1f..f2d0072a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -117,6 +117,7 @@ repos: 'fastapi[standard]>=0.109.1', httpx, pathspec>=0.12.1, + prometheus-client, pydantic, pytest-asyncio, pytest-mock, @@ -138,6 +139,7 @@ repos: 'fastapi[standard]>=0.109.1', httpx, pathspec>=0.12.1, + prometheus-client, pydantic, pytest-asyncio, pytest-mock, diff --git a/Dockerfile b/Dockerfile index 90ae4134..1fb2357f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,4 +43,5 @@ RUN set -eux; \ USER appuser EXPOSE 8000 +EXPOSE 9090 CMD ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/pyproject.toml b/pyproject.toml index fb78abab..e70e7caa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "tiktoken>=0.7.0", # Support for o200k_base encoding "typing_extensions>= 4.0.0; python_version < '3.10'", "uvicorn>=0.11.7", # Minimum safe release (https://osv.dev/vulnerability/PYSEC-2020-150) + "prometheus-client", ] license = {file = "LICENSE"} diff --git a/requirements.txt b/requirements.txt index f9f9b50a..bfec694e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ click>=8.0.0 fastapi[standard]>=0.109.1 # Vulnerable to https://osv.dev/vulnerability/PYSEC-2024-38 httpx pathspec>=0.12.1 +prometheus-client pydantic python-dotenv slowapi diff --git a/src/server/main.py b/src/server/main.py index 09904256..5bdd2e15 100644 --- a/src/server/main.py +++ b/src/server/main.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import threading from pathlib import Path from dotenv import load_dotenv @@ -12,6 +13,7 @@ from slowapi.errors import RateLimitExceeded from starlette.middleware.trustedhost import TrustedHostMiddleware +from server.metrics_server import start_metrics_server from server.routers import dynamic, index, ingest from server.server_config import templates from server.server_utils import lifespan, limiter, rate_limit_exception_handler @@ -26,6 +28,17 @@ # Register the custom exception handler for rate limits app.add_exception_handler(RateLimitExceeded, rate_limit_exception_handler) +# Start metrics server in a separate thread if enabled +if os.getenv("GITINGEST_METRICS_ENABLED", "false").lower() == "true": + metrics_host = os.getenv("GITINGEST_METRICS_HOST", "127.0.0.1") + metrics_port = int(os.getenv("GITINGEST_METRICS_PORT", "9090")) + metrics_thread = threading.Thread( + target=start_metrics_server, + args=(metrics_host, metrics_port), + daemon=True, + ) + metrics_thread.start() + # Mount static files dynamically to serve CSS, JS, and other static assets static_dir = Path(__file__).parent.parent / "static" diff --git a/src/server/metrics_server.py b/src/server/metrics_server.py new file mode 100644 index 00000000..1de3d022 --- /dev/null +++ b/src/server/metrics_server.py @@ -0,0 +1,57 @@ +"""Prometheus metrics server running on a separate port.""" + +import logging + +import uvicorn +from fastapi import FastAPI +from fastapi.responses import HTMLResponse +from prometheus_client import REGISTRY, generate_latest + +# Create a logger for this module +logger = logging.getLogger(__name__) + +# Create a separate FastAPI app for metrics +metrics_app = FastAPI( + title="Gitingest Metrics", + description="Prometheus metrics for Gitingest", + docs_url=None, + redoc_url=None, +) + + +@metrics_app.get("/metrics") +async def metrics() -> HTMLResponse: + """Serve Prometheus metrics without authentication. + + This endpoint is only accessible from the local network. + + Returns + ------- + HTMLResponse + Prometheus metrics in text format + + """ + return HTMLResponse( + content=generate_latest(REGISTRY), + status_code=200, + media_type="text/plain", + ) + + +def start_metrics_server(host: str = "127.0.0.1", port: int = 9090) -> None: + """Start the metrics server on a separate port. + + Parameters + ---------- + host : str + The host to bind to (default: 127.0.0.1 for local network only) + port : int + The port to bind to (default: 9090) + + Returns + ------- + None + + """ + logger.info("Starting metrics server on %s:%s", host, port) + uvicorn.run(metrics_app, host=host, port=port) diff --git a/src/server/routers/ingest.py b/src/server/routers/ingest.py index 117161bf..514db272 100644 --- a/src/server/routers/ingest.py +++ b/src/server/routers/ingest.py @@ -2,6 +2,7 @@ from fastapi import APIRouter, HTTPException, Request, status from fastapi.responses import FileResponse, JSONResponse +from prometheus_client import Counter from gitingest.config import TMP_BASE_PATH from server.models import IngestRequest @@ -9,6 +10,8 @@ from server.server_config import MAX_DISPLAY_SIZE from server.server_utils import limiter +ingest_counter = Counter("gitingest_ingest_total", "Number of ingests", ["status", "url"]) + router = APIRouter() @@ -33,13 +36,16 @@ async def api_ingest( - **JSONResponse**: Success response with ingestion results or error response with appropriate HTTP status code """ - return await _perform_ingestion( + response = await _perform_ingestion( input_text=ingest_request.input_text, max_file_size=ingest_request.max_file_size, pattern_type=ingest_request.pattern_type, pattern=ingest_request.pattern, token=ingest_request.token, ) + # limit URL to 255 characters + ingest_counter.labels(status=response.status_code, url=ingest_request.input_text[:255]).inc() + return response @router.get("/api/{user}/{repository}", responses=COMMON_INGEST_RESPONSES) @@ -72,13 +78,16 @@ async def api_ingest_get( **Returns** - **JSONResponse**: Success response with ingestion results or error response with appropriate HTTP status code """ - return await _perform_ingestion( + response = await _perform_ingestion( input_text=f"{user}/{repository}", max_file_size=max_file_size, pattern_type=pattern_type, pattern=pattern, token=token or None, ) + # limit URL to 255 characters + ingest_counter.labels(status=response.status_code, url=f"{user}/{repository}"[:255]).inc() + return response @router.get("/api/download/file/{ingest_id}", response_class=FileResponse) diff --git a/src/server/server_config.py b/src/server/server_config.py index 99ef5c91..0257db8b 100644 --- a/src/server/server_config.py +++ b/src/server/server_config.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + from fastapi.templating import Jinja2Templates MAX_DISPLAY_SIZE: int = 300_000 @@ -19,4 +21,7 @@ {"name": "ApiAnalytics", "url": "https://github.com/tom-draper/api-analytics"}, ] -templates = Jinja2Templates(directory="server/templates") + +# Use absolute path to templates directory +templates_dir = Path(__file__).parent / "templates" +templates = Jinja2Templates(directory=templates_dir)