dcc-backend-common 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcc_backend_common/__init__.py +0 -0
- dcc_backend_common/config/__init__.py +3 -0
- dcc_backend_common/config/app_config.py +78 -0
- dcc_backend_common/fastapi_error_handling/__init__.py +12 -0
- dcc_backend_common/fastapi_error_handling/error_codes.py +12 -0
- dcc_backend_common/fastapi_error_handling/error_exception.py +62 -0
- dcc_backend_common/fastapi_error_handling/error_handler.py +42 -0
- dcc_backend_common/fastapi_health_probes/__init__.py +3 -0
- dcc_backend_common/fastapi_health_probes/router.py +98 -0
- dcc_backend_common/logger/__init__.py +3 -0
- dcc_backend_common/logger/logger.py +127 -0
- dcc_backend_common/py.typed +0 -0
- dcc_backend_common-0.0.1.dist-info/METADATA +270 -0
- dcc_backend_common-0.0.1.dist-info/RECORD +16 -0
- dcc_backend_common-0.0.1.dist-info/WHEEL +4 -0
- dcc_backend_common-0.0.1.dist-info/licenses/LICENSE +21 -0
|
File without changes
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import override
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, Field
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class AppConfigError(ValueError):
|
|
8
|
+
"""Exception raised for errors in the application configuration."""
|
|
9
|
+
|
|
10
|
+
def __init__(self, variable_name: str) -> None:
|
|
11
|
+
super().__init__(f"Configuration variable '{variable_name}' is not set or invalid.")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_env_or_throw(env_name: str) -> str:
|
|
15
|
+
value = os.getenv(env_name)
|
|
16
|
+
if value is None:
|
|
17
|
+
raise AppConfigError(env_name)
|
|
18
|
+
return value
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def log_secret(secret: str | None) -> str:
|
|
22
|
+
return "****" if secret is not None and len(secret) > 0 else "None"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class AbstractAppConfig(BaseModel):
|
|
26
|
+
"""Abstract base class for application configurations."""
|
|
27
|
+
|
|
28
|
+
@classmethod
|
|
29
|
+
def from_env(cls) -> "AbstractAppConfig":
|
|
30
|
+
raise NotImplementedError("Subclasses must implement from_env.")
|
|
31
|
+
|
|
32
|
+
@override
|
|
33
|
+
def __str__(self) -> str:
|
|
34
|
+
raise NotImplementedError("Subclasses must implement __str__.")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class AppConfig(AbstractAppConfig):
|
|
38
|
+
client_url: str = Field(description="The URL for the client application")
|
|
39
|
+
hmac_secret: str = Field(description="The secret key for HMAC authentication")
|
|
40
|
+
openai_api_key: str = Field(description="The API key for authenticating with OpenAI")
|
|
41
|
+
llm_url: str = Field(description="The URL for the LLM API")
|
|
42
|
+
docling_url: str = Field(description="The URL for the Docling service")
|
|
43
|
+
whisper_url: str = Field(description="The URL for the Whisper API")
|
|
44
|
+
ocr_url: str = Field(description="The URL for the OCR API")
|
|
45
|
+
|
|
46
|
+
@classmethod
|
|
47
|
+
@override
|
|
48
|
+
def from_env(cls) -> "AppConfig":
|
|
49
|
+
client_url: str = get_env_or_throw("CLIENT_URL")
|
|
50
|
+
hmac_secret: str = get_env_or_throw("HMAC_SECRET")
|
|
51
|
+
openai_api_key: str = get_env_or_throw("OPENAI_API_KEY")
|
|
52
|
+
llm_url: str = get_env_or_throw("LLM_URL")
|
|
53
|
+
docling_url: str = get_env_or_throw("DOCLING_URL")
|
|
54
|
+
whisper_url: str = get_env_or_throw("WHISPER_URL")
|
|
55
|
+
ocr_url: str = get_env_or_throw("OCR_URL")
|
|
56
|
+
return cls(
|
|
57
|
+
client_url=client_url,
|
|
58
|
+
hmac_secret=hmac_secret,
|
|
59
|
+
openai_api_key=openai_api_key,
|
|
60
|
+
llm_url=llm_url,
|
|
61
|
+
docling_url=docling_url,
|
|
62
|
+
whisper_url=whisper_url,
|
|
63
|
+
ocr_url=ocr_url,
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
@override
|
|
67
|
+
def __str__(self) -> str:
|
|
68
|
+
return f"""
|
|
69
|
+
AppConfig(
|
|
70
|
+
client_url={self.client_url},
|
|
71
|
+
hmac_secret={log_secret(self.hmac_secret)},
|
|
72
|
+
openai_api_key={log_secret(self.openai_api_key)},
|
|
73
|
+
llm_url={self.llm_url},
|
|
74
|
+
docling_url={self.docling_url},
|
|
75
|
+
whisper_url={self.whisper_url},
|
|
76
|
+
ocr_url={self.ocr_url},
|
|
77
|
+
)
|
|
78
|
+
"""
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from .error_codes import ApiErrorCodes
|
|
2
|
+
from .error_exception import ApiErrorException, ErrorResponse, api_error_exception, construct_api_error_exception
|
|
3
|
+
from .error_handler import inject_api_error_handler
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"ApiErrorCodes",
|
|
7
|
+
"ApiErrorException",
|
|
8
|
+
"ErrorResponse",
|
|
9
|
+
"api_error_exception",
|
|
10
|
+
"construct_api_error_exception",
|
|
11
|
+
"inject_api_error_handler",
|
|
12
|
+
]
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from enum import StrEnum
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ApiErrorCodes(StrEnum):
|
|
5
|
+
UNEXPECTED_ERROR = "unexpected_error"
|
|
6
|
+
SERVICE_UNAVAILABLE = "service_unavailable"
|
|
7
|
+
INVALID_REQUEST = "invalid_request"
|
|
8
|
+
AUTHENTICATION_FAILED = "authentication_failed"
|
|
9
|
+
PERMISSION_DENIED = "permission_denied"
|
|
10
|
+
RESOURCE_NOT_FOUND = "resource_not_found"
|
|
11
|
+
RATE_LIMIT_EXCEEDED = "rate_limit_exceeded"
|
|
12
|
+
VALIDATION_ERROR = "validation_error"
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
from typing import TypedDict
|
|
2
|
+
|
|
3
|
+
from fastapi import status
|
|
4
|
+
from fastapi.exceptions import HTTPException
|
|
5
|
+
|
|
6
|
+
from dcc_backend_common.fastapi_error_handling.error_codes import ApiErrorCodes
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ErrorResponse(TypedDict, total=False):
|
|
10
|
+
errorId: str | ApiErrorCodes
|
|
11
|
+
status: int | None # default to 500 if not provided
|
|
12
|
+
debugMessage: str | None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ApiErrorException(Exception):
|
|
16
|
+
def __init__(self, error_response: ErrorResponse):
|
|
17
|
+
if "status" not in error_response:
|
|
18
|
+
error_response["status"] = status.HTTP_500_INTERNAL_SERVER_ERROR
|
|
19
|
+
self.error_response = error_response
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def api_error_exception(
|
|
23
|
+
errorId: str = ApiErrorCodes.UNEXPECTED_ERROR,
|
|
24
|
+
status: int = status.HTTP_500_INTERNAL_SERVER_ERROR,
|
|
25
|
+
debugMessage: str | None = None,
|
|
26
|
+
):
|
|
27
|
+
return ApiErrorException(
|
|
28
|
+
error_response={
|
|
29
|
+
"errorId": errorId,
|
|
30
|
+
"status": status,
|
|
31
|
+
"debugMessage": debugMessage,
|
|
32
|
+
}
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def construct_api_error_exception(
|
|
37
|
+
exception: Exception,
|
|
38
|
+
error_id: str | ApiErrorCodes = ApiErrorCodes.UNEXPECTED_ERROR,
|
|
39
|
+
status_code: int | None = None,
|
|
40
|
+
) -> ApiErrorException:
|
|
41
|
+
"""
|
|
42
|
+
Constructs an ApiErrorException from the given exception, error ID, and status code.
|
|
43
|
+
|
|
44
|
+
Parameters:
|
|
45
|
+
exception (Exception): The original exception that occurred.
|
|
46
|
+
error_id (str | ApiErrorCodes): A string or enum value representing the error ID
|
|
47
|
+
status_code (int | None): An optional HTTP status code for the error response.
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
ApiErrorException: An exception containing the structured error response.
|
|
51
|
+
"""
|
|
52
|
+
error_response: ErrorResponse = {"errorId": error_id, "status": status_code}
|
|
53
|
+
|
|
54
|
+
if not status_code and isinstance(exception, HTTPException):
|
|
55
|
+
error_response["status"] = exception.status_code
|
|
56
|
+
|
|
57
|
+
debug_message = str(exception) if exception is not None else None
|
|
58
|
+
|
|
59
|
+
if debug_message is not None:
|
|
60
|
+
error_response["debugMessage"] = debug_message
|
|
61
|
+
|
|
62
|
+
return ApiErrorException(error_response)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
from fastapi import FastAPI, Request, Response, status
|
|
2
|
+
from fastapi.responses import JSONResponse
|
|
3
|
+
|
|
4
|
+
from dcc_backend_common.fastapi_error_handling.error_codes import ApiErrorCodes
|
|
5
|
+
from dcc_backend_common.fastapi_error_handling.error_exception import ApiErrorException
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def inject_api_error_handler(app: FastAPI):
|
|
9
|
+
app.add_exception_handler(ApiErrorException, api_error_handler)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def api_error_handler(request: Request, exc: Exception) -> Response:
|
|
13
|
+
"""
|
|
14
|
+
Convert exceptions raised during request handling into a JSON HTTP response.
|
|
15
|
+
|
|
16
|
+
If `exc` is an `ApiErrorException`, returns its `error_response` payload and status code.
|
|
17
|
+
Otherwise returns a 500 response with `errorId` set to `UNEXPECTED_ERROR`, `status` 500, and a `debugMessage` containing the exception string.
|
|
18
|
+
|
|
19
|
+
Parameters:
|
|
20
|
+
request (Request): The incoming HTTP request that triggered the exception.
|
|
21
|
+
exc (Exception): The exception raised during request processing.
|
|
22
|
+
|
|
23
|
+
Returns:
|
|
24
|
+
Response: A JSON HTTP response describing the error.
|
|
25
|
+
"""
|
|
26
|
+
if isinstance(exc, ApiErrorException):
|
|
27
|
+
status_code = (
|
|
28
|
+
exc.error_response.get("status", status.HTTP_500_INTERNAL_SERVER_ERROR)
|
|
29
|
+
or status.HTTP_500_INTERNAL_SERVER_ERROR
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
return JSONResponse(
|
|
33
|
+
status_code=status_code,
|
|
34
|
+
media_type="application/json",
|
|
35
|
+
content=exc.error_response,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
return JSONResponse(
|
|
39
|
+
status_code=500,
|
|
40
|
+
media_type="application/json",
|
|
41
|
+
content={"errorId": ApiErrorCodes.UNEXPECTED_ERROR, "status": 500, "debugMessage": str(exc)},
|
|
42
|
+
)
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import time
|
|
3
|
+
from datetime import UTC, datetime
|
|
4
|
+
from http import HTTPStatus
|
|
5
|
+
from typing import Any, TypedDict
|
|
6
|
+
|
|
7
|
+
import aiohttp
|
|
8
|
+
from fastapi import APIRouter, HTTPException, Response
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ServiceDependency(TypedDict):
|
|
12
|
+
name: str
|
|
13
|
+
health_check_url: str
|
|
14
|
+
api_key: str | None
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def health_probe_router(service_dependencies: list[ServiceDependency]) -> APIRouter:
|
|
18
|
+
router = APIRouter(prefix="/health")
|
|
19
|
+
|
|
20
|
+
START_TIME = time.time()
|
|
21
|
+
|
|
22
|
+
# Disable logging for health check endpoints
|
|
23
|
+
class EndpointFilter(logging.Filter):
|
|
24
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
25
|
+
# Endpoints to exclude from logging
|
|
26
|
+
skip_paths = {"/health"}
|
|
27
|
+
|
|
28
|
+
# Extract the request path from the log message
|
|
29
|
+
return all(skip_path not in record.getMessage() for skip_path in skip_paths)
|
|
30
|
+
|
|
31
|
+
# Configure the filter
|
|
32
|
+
logging.getLogger("uvicorn.access").addFilter(EndpointFilter())
|
|
33
|
+
|
|
34
|
+
@router.get("/liveness")
|
|
35
|
+
async def liveness_probe():
|
|
36
|
+
"""
|
|
37
|
+
Liveness Probe
|
|
38
|
+
* Purpose: Checks if the application process is running and not deadlocked.
|
|
39
|
+
* K8s Action: If this fails, the container is KILLED and RESTARTED.
|
|
40
|
+
* Rule: Keep it simple. Do NOT check databases here.
|
|
41
|
+
"""
|
|
42
|
+
return {"status": "up", "uptime_seconds": time.time() - START_TIME}
|
|
43
|
+
|
|
44
|
+
@router.get("/readiness")
|
|
45
|
+
async def readiness_probe(response: Response):
|
|
46
|
+
"""
|
|
47
|
+
Readiness Probe
|
|
48
|
+
* Purpose: Checks if the app is ready to handle user requests (e.g., external APIs).
|
|
49
|
+
* K8s Action: If this fails, traffic stops sending to this pod.
|
|
50
|
+
* Rule: Check critical dependencies here.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
health_check: dict[str, Any] = {
|
|
54
|
+
"status": "ready",
|
|
55
|
+
"checks": {service["name"]: "unknown" for service in service_dependencies},
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
timeout = aiohttp.ClientTimeout(total=5.0)
|
|
60
|
+
|
|
61
|
+
for service in service_dependencies:
|
|
62
|
+
async with aiohttp.ClientSession(
|
|
63
|
+
timeout=timeout,
|
|
64
|
+
headers={"Authorization": f"Bearer {service['api_key']}"} if service["api_key"] else {},
|
|
65
|
+
) as session:
|
|
66
|
+
try:
|
|
67
|
+
async with session.get(service["health_check_url"]) as svc_response:
|
|
68
|
+
if svc_response.status == 200:
|
|
69
|
+
health_check["checks"][service["name"]] = "healthy"
|
|
70
|
+
else:
|
|
71
|
+
health_check["checks"][service["name"]] = f"unhealthy (status: {svc_response.status})"
|
|
72
|
+
raise HTTPException(
|
|
73
|
+
status_code=HTTPStatus.SERVICE_UNAVAILABLE,
|
|
74
|
+
detail=f"{service['name']} returned status {svc_response.status}",
|
|
75
|
+
)
|
|
76
|
+
except aiohttp.ClientError as e:
|
|
77
|
+
health_check["checks"][service["name"]] = f"error: {e!s}"
|
|
78
|
+
raise
|
|
79
|
+
|
|
80
|
+
except Exception as e:
|
|
81
|
+
# If a critical dependency fails, we must return a 503.
|
|
82
|
+
# This tells K8s to stop sending traffic to this specific pod.
|
|
83
|
+
response.status_code = HTTPStatus.SERVICE_UNAVAILABLE
|
|
84
|
+
return {"status": "unhealthy", "checks": health_check["checks"], "error": str(e)}
|
|
85
|
+
else:
|
|
86
|
+
return health_check
|
|
87
|
+
|
|
88
|
+
@router.get("/startup")
|
|
89
|
+
async def startup_probe():
|
|
90
|
+
"""
|
|
91
|
+
Startup Probe
|
|
92
|
+
* Purpose: Checks if the application has finished initialization.
|
|
93
|
+
* K8s Action: Blocks Liveness/Readiness probes until this returns 200.
|
|
94
|
+
* Rule: Useful for apps that need to load large ML models or caches on boot.
|
|
95
|
+
"""
|
|
96
|
+
return {"status": "started", "timestamp": datetime.now(UTC).isoformat()}
|
|
97
|
+
|
|
98
|
+
return router
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import os
|
|
3
|
+
import time
|
|
4
|
+
import uuid
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import structlog
|
|
9
|
+
import structlog.processors
|
|
10
|
+
from structlog.processors import CallsiteParameter
|
|
11
|
+
from structlog.stdlib import BoundLogger
|
|
12
|
+
from structlog.types import EventDict, Processor
|
|
13
|
+
|
|
14
|
+
from dcc_backend_common.config import get_env_or_throw
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# Standard library logging setup
|
|
18
|
+
def setup_stdlib_logging() -> None:
|
|
19
|
+
"""Configure standard library logging to work with structlog."""
|
|
20
|
+
log_level = os.getenv("LOG_LEVEL", "INFO").upper()
|
|
21
|
+
|
|
22
|
+
level = getattr(logging, log_level, logging.INFO)
|
|
23
|
+
|
|
24
|
+
# Create a handler for console output
|
|
25
|
+
handler = logging.StreamHandler()
|
|
26
|
+
|
|
27
|
+
# Configure root logger
|
|
28
|
+
root_logger = logging.getLogger()
|
|
29
|
+
root_logger.setLevel(level)
|
|
30
|
+
root_logger.addHandler(handler)
|
|
31
|
+
|
|
32
|
+
# Disable propagation for libraries that are too verbose
|
|
33
|
+
for logger_name in ["uvicorn.access"]:
|
|
34
|
+
lib_logger = logging.getLogger(logger_name)
|
|
35
|
+
lib_logger.propagate = False
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def add_request_id(logger: BoundLogger, method_name: str, event_dict: EventDict) -> Mapping[str, Any]:
|
|
39
|
+
"""
|
|
40
|
+
Add a request ID to the log context if it doesn't exist.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
logger: The logger instance
|
|
44
|
+
method_name: The name of the logging method
|
|
45
|
+
event_dict: The event dictionary
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
The updated event dictionary
|
|
49
|
+
"""
|
|
50
|
+
if "request_id" not in event_dict:
|
|
51
|
+
event_dict["request_id"] = str(uuid.uuid4())
|
|
52
|
+
return event_dict
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def add_timestamp(logger: BoundLogger, method_name: str, event_dict: EventDict) -> Mapping[str, Any]:
|
|
56
|
+
"""
|
|
57
|
+
Add an ISO-8601 timestamp to the log entry.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
logger: The logger instance
|
|
61
|
+
method_name: The name of the logging method
|
|
62
|
+
event_dict: The event dictionary
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
The updated event dictionary
|
|
66
|
+
"""
|
|
67
|
+
event_dict["timestamp"] = time.strftime("%Y-%m-%dT%H:%M:%S%z")
|
|
68
|
+
return event_dict
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def init_logger() -> None:
|
|
72
|
+
"""
|
|
73
|
+
Initialize the logger configuration based on environment.
|
|
74
|
+
Uses JSON renderer in production environment for compatibility with fluentbit.
|
|
75
|
+
Adds the module name as context to the logger.
|
|
76
|
+
"""
|
|
77
|
+
# Set up standard library logging first
|
|
78
|
+
setup_stdlib_logging()
|
|
79
|
+
|
|
80
|
+
# Define processors list for structlog
|
|
81
|
+
processors: list[Processor] = [
|
|
82
|
+
structlog.stdlib.filter_by_level,
|
|
83
|
+
structlog.processors.add_log_level,
|
|
84
|
+
structlog.processors.StackInfoRenderer(),
|
|
85
|
+
add_timestamp,
|
|
86
|
+
add_request_id,
|
|
87
|
+
structlog.processors.CallsiteParameterAdder(
|
|
88
|
+
parameters=[
|
|
89
|
+
CallsiteParameter.MODULE,
|
|
90
|
+
CallsiteParameter.FUNC_NAME,
|
|
91
|
+
CallsiteParameter.LINENO,
|
|
92
|
+
]
|
|
93
|
+
),
|
|
94
|
+
structlog.processors.UnicodeDecoder(),
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
# Use different renderers for development vs production
|
|
98
|
+
if get_env_or_throw("IS_PROD").lower() == "true":
|
|
99
|
+
# JSON renderer for production to be fluentbit compatible
|
|
100
|
+
processors.append(structlog.processors.JSONRenderer())
|
|
101
|
+
else:
|
|
102
|
+
# For development, use a colored console renderer
|
|
103
|
+
processors.append(structlog.dev.ConsoleRenderer(colors=True))
|
|
104
|
+
|
|
105
|
+
# Configure structlog
|
|
106
|
+
structlog.configure(
|
|
107
|
+
processors=processors,
|
|
108
|
+
context_class=dict,
|
|
109
|
+
logger_factory=structlog.stdlib.LoggerFactory(),
|
|
110
|
+
wrapper_class=structlog.stdlib.BoundLogger,
|
|
111
|
+
cache_logger_on_first_use=True,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def get_logger(name: str | None = None) -> BoundLogger:
|
|
116
|
+
"""
|
|
117
|
+
Get a structured logger instance.
|
|
118
|
+
|
|
119
|
+
Args:
|
|
120
|
+
name: Optional name for the logger, typically the module name
|
|
121
|
+
|
|
122
|
+
Returns:
|
|
123
|
+
A bound logger instance for structured logging
|
|
124
|
+
"""
|
|
125
|
+
if name:
|
|
126
|
+
return structlog.get_logger(name) # type: ignore[no-any-return]
|
|
127
|
+
return structlog.get_logger() # type: ignore[no-any-return]
|
|
File without changes
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dcc-backend-common
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Common python code used in out backend services.
|
|
5
|
+
Project-URL: Homepage, https://github.com/dcc-bs
|
|
6
|
+
Project-URL: Repository, https://github.com/DCC-BS/backend-common
|
|
7
|
+
Project-URL: Documentation, https://DCC-BS.github.io/documentation/
|
|
8
|
+
Author-email: Data Competence Center Basel-Stadt <dcc@bs.ch>, Tobias Bollinger <tobias.bollinger@bs.ch>, Yanick Schraner <yanick.schraner@bs.ch>
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: python
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Requires-Python: <4.0,>=3.12
|
|
21
|
+
Requires-Dist: datasets>=4.4.1
|
|
22
|
+
Requires-Dist: dspy>=3.0.4
|
|
23
|
+
Requires-Dist: jiwer>=4.0.0
|
|
24
|
+
Requires-Dist: pydantic>=2.12.5
|
|
25
|
+
Requires-Dist: python-dotenv>=1.0.1
|
|
26
|
+
Requires-Dist: structlog>=25.1.0
|
|
27
|
+
Provides-Extra: fastapi
|
|
28
|
+
Requires-Dist: aiohttp>=3.13.2; extra == 'fastapi'
|
|
29
|
+
Requires-Dist: fastapi<1.0,>=0.115; extra == 'fastapi'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# dcc-backend-common
|
|
33
|
+
|
|
34
|
+
[](https://img.shields.io/github/commit-activity/m/DCC-BS/backend-common)
|
|
35
|
+
[](https://img.shields.io/github/license/DCC-BS/backend-common)
|
|
36
|
+
|
|
37
|
+
Common utilities and components for backend services developed by the Data Competence Center Basel-Stadt.
|
|
38
|
+
|
|
39
|
+
## Overview
|
|
40
|
+
|
|
41
|
+
`dcc-backend-common` is a Python library that provides shared functionality for backend services, including:
|
|
42
|
+
|
|
43
|
+
- **FastAPI Health Probes**: Kubernetes-ready health check endpoints (liveness, readiness, startup)
|
|
44
|
+
- **Structured Logging**: Integration with `structlog` for consistent logging across services
|
|
45
|
+
- **Configuration Management**: Environment-based configuration with `python-dotenv`
|
|
46
|
+
- **DSPy Utilities**: Helpers for DSPy modules, streaming listeners, metrics, and dataset preparation
|
|
47
|
+
|
|
48
|
+
## Installation
|
|
49
|
+
|
|
50
|
+
### Basic Installation (uv)
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
uv add dcc-backend-common
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### With FastAPI Support
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
uv add "dcc-backend-common[fastapi]"
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Requirements
|
|
63
|
+
|
|
64
|
+
- Python 3.12 or higher
|
|
65
|
+
- Dependencies:
|
|
66
|
+
- `dspy>=3.0.4`
|
|
67
|
+
- `python-dotenv>=1.0.1`
|
|
68
|
+
- `structlog>=25.1.0`
|
|
69
|
+
|
|
70
|
+
### Optional Dependencies
|
|
71
|
+
|
|
72
|
+
- FastAPI extras: `aiohttp>=3.13.2`, `fastapi>=0.115,<1.0`
|
|
73
|
+
|
|
74
|
+
## Features
|
|
75
|
+
|
|
76
|
+
### FastAPI Health Probes
|
|
77
|
+
|
|
78
|
+
The library provides Kubernetes-ready health check endpoints that follow best practices for container orchestration:
|
|
79
|
+
|
|
80
|
+
#### Example Usage
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from fastapi import FastAPI
|
|
84
|
+
from dcc_backend_common.fastapi_health_probes import health_probe_router
|
|
85
|
+
|
|
86
|
+
app = FastAPI()
|
|
87
|
+
|
|
88
|
+
# Define external service dependencies
|
|
89
|
+
service_dependencies = [
|
|
90
|
+
{
|
|
91
|
+
"name": "database",
|
|
92
|
+
"health_check_url": "http://postgres:5432/health",
|
|
93
|
+
"api_key": None # Optional API key for authenticated health checks
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
"name": "external-api",
|
|
97
|
+
"health_check_url": "https://api.example.com/health",
|
|
98
|
+
"api_key": "your-api-key-here"
|
|
99
|
+
}
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
# Include health probe router
|
|
103
|
+
app.include_router(health_probe_router(service_dependencies))
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
#### Available Endpoints
|
|
107
|
+
|
|
108
|
+
##### 1. Liveness Probe (`GET /health/liveness`)
|
|
109
|
+
|
|
110
|
+
- **Purpose**: Checks if the application process is running and not deadlocked
|
|
111
|
+
- **Kubernetes Action**: If this fails, the container is killed and restarted
|
|
112
|
+
- **Response**: Returns uptime in seconds
|
|
113
|
+
- **Rule**: Keep it simple. Do NOT check databases or external dependencies here
|
|
114
|
+
|
|
115
|
+
```json
|
|
116
|
+
{
|
|
117
|
+
"status": "up",
|
|
118
|
+
"uptime_seconds": 123.45
|
|
119
|
+
}
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
##### 2. Readiness Probe (`GET /health/readiness`)
|
|
123
|
+
|
|
124
|
+
- **Purpose**: Checks if the app is ready to handle user requests
|
|
125
|
+
- **Kubernetes Action**: If this fails, traffic stops being sent to this pod
|
|
126
|
+
- **Response**: Returns status of all configured service dependencies
|
|
127
|
+
- **Rule**: Check critical dependencies here (databases, external APIs, etc.)
|
|
128
|
+
|
|
129
|
+
```json
|
|
130
|
+
{
|
|
131
|
+
"status": "ready",
|
|
132
|
+
"checks": {
|
|
133
|
+
"database": "healthy",
|
|
134
|
+
"external-api": "healthy"
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
If a dependency fails:
|
|
140
|
+
|
|
141
|
+
```json
|
|
142
|
+
{
|
|
143
|
+
"status": "unhealthy",
|
|
144
|
+
"checks": {
|
|
145
|
+
"database": "error: Connection refused",
|
|
146
|
+
"external-api": "unhealthy (status: 503)"
|
|
147
|
+
},
|
|
148
|
+
"error": "Service unavailable"
|
|
149
|
+
}
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
##### 3. Startup Probe (`GET /health/startup`)
|
|
153
|
+
|
|
154
|
+
- **Purpose**: Checks if the application has finished initialization
|
|
155
|
+
- **Kubernetes Action**: Blocks liveness/readiness probes until this returns 200
|
|
156
|
+
- **Response**: Returns startup timestamp
|
|
157
|
+
- **Rule**: Useful for apps that need to load large ML models or caches on boot
|
|
158
|
+
|
|
159
|
+
```json
|
|
160
|
+
{
|
|
161
|
+
"status": "started",
|
|
162
|
+
"timestamp": "2025-12-04T10:30:00.000000+00:00"
|
|
163
|
+
}
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
#### Features
|
|
167
|
+
|
|
168
|
+
- **Automatic Logging Suppression**: Health check endpoints are automatically excluded from access logs to reduce noise
|
|
169
|
+
- **Dependency Health Checks**: Readiness probe checks external service dependencies with configurable timeouts (5 seconds default)
|
|
170
|
+
- **Authentication Support**: Optional API key support for authenticated health checks
|
|
171
|
+
- **Kubernetes-Ready**: HTTP status codes follow Kubernetes conventions (200 = healthy, 503 = unhealthy)
|
|
172
|
+
|
|
173
|
+
### Structured Logging
|
|
174
|
+
|
|
175
|
+
- Initialize structured logging with `init_logger()`, which auto-selects JSON output in production (`IS_PROD=true`) and colored console output otherwise.
|
|
176
|
+
- Retrieve loggers via `get_logger(__name__)`. A `request_id` and timestamp are added automatically.
|
|
177
|
+
|
|
178
|
+
### Application Configuration
|
|
179
|
+
|
|
180
|
+
Load strongly-typed configuration from environment variables:
|
|
181
|
+
|
|
182
|
+
```python
|
|
183
|
+
from dcc_backend_common.config.app_config import AppConfig
|
|
184
|
+
|
|
185
|
+
config = AppConfig.from_env()
|
|
186
|
+
print(config) # secrets are redacted in __str__
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Required variables: `CLIENT_URL`, `HMAC_SECRET`, `OPENAI_API_KEY`, `LLM_URL`, `DOCLING_URL`, `WHISPER_URL`, `OCR_URL`. Missing values raise `AppConfigError`.
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
## Development
|
|
193
|
+
|
|
194
|
+
### Setup
|
|
195
|
+
|
|
196
|
+
1. Clone the repository:
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
git clone https://github.com/DCC-BS/backend-common.git
|
|
200
|
+
cd backend-common
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
2. Install development dependencies:
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
uv sync --group dev --extra fastapi # include FastAPI extras for local dev
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### Running Tests
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
uv run pytest
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### Code Quality
|
|
216
|
+
|
|
217
|
+
This project uses:
|
|
218
|
+
|
|
219
|
+
- **Ruff**: For linting and formatting
|
|
220
|
+
- **Pre-commit**: For automated code quality checks
|
|
221
|
+
- **Tox**: For testing across multiple Python versions
|
|
222
|
+
|
|
223
|
+
Run linting:
|
|
224
|
+
|
|
225
|
+
```bash
|
|
226
|
+
uv run ruff check .
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
Run pre-commit hooks:
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
uv run pre-commit run --all-files
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
## Releasing
|
|
236
|
+
|
|
237
|
+
This project uses GitHub Actions for automated releases to PyPI.
|
|
238
|
+
|
|
239
|
+
To release a new version:
|
|
240
|
+
|
|
241
|
+
1. **Update the version**: Update the `version` field in `pyproject.toml`.
|
|
242
|
+
2. **Commit and push**: Commit the version change and push it to the `main` branch.
|
|
243
|
+
3. **Trigger the workflow**:
|
|
244
|
+
* Navigate to the **Actions** tab in the GitHub repository.
|
|
245
|
+
* Select the **Publish to PyPI** workflow.
|
|
246
|
+
* Click **Run workflow**.
|
|
247
|
+
4. **Automated steps**: The workflow will:
|
|
248
|
+
* Automatically detect the version using `uv version --short`.
|
|
249
|
+
* Create and push a git tag (e.g., `v0.1.0`).
|
|
250
|
+
* Build the package with `uv build`.
|
|
251
|
+
* Publish to PyPI using Trusted Publishing.
|
|
252
|
+
|
|
253
|
+
## Contributing
|
|
254
|
+
|
|
255
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to contribute to this project.
|
|
256
|
+
|
|
257
|
+
## License
|
|
258
|
+
|
|
259
|
+
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
260
|
+
|
|
261
|
+
## Authors
|
|
262
|
+
|
|
263
|
+
- **Data Competence Center Basel-Stadt** - [dcc@bs.ch](mailto:dcc@bs.ch)
|
|
264
|
+
- **Tobias Bollinger** - [tobias.bollinger@bs.ch](mailto:tobias.bollinger@bs.ch)
|
|
265
|
+
|
|
266
|
+
## Links
|
|
267
|
+
|
|
268
|
+
- **Homepage**: https://DCC-BS.github.io/backend-common/
|
|
269
|
+
- **Repository**: https://github.com/DCC-BS/backend-common
|
|
270
|
+
- **Documentation**: https://DCC-BS.github.io/backend-common/
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
dcc_backend_common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
dcc_backend_common/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
dcc_backend_common/config/__init__.py,sha256=sqUsKzWO-H7S6OY6XOiDYd4mjduHM3IxFSlmz2JNnk8,197
|
|
4
|
+
dcc_backend_common/config/app_config.py,sha256=SfosCdyGWtt6TTLnwX75YMyC1A2tP9_4q9ODpIYgVHs,2697
|
|
5
|
+
dcc_backend_common/fastapi_error_handling/__init__.py,sha256=ZOVVHyZ7xhNlTe7SVmHPGiOx0_wf0oXfP8xeX1oipW8,383
|
|
6
|
+
dcc_backend_common/fastapi_error_handling/error_codes.py,sha256=0RTxeKaZefaCJQ63AxqGFUgA1JeUOhzKBjQKwU6-8P8,419
|
|
7
|
+
dcc_backend_common/fastapi_error_handling/error_exception.py,sha256=wAQVRt1sdL4Zg3eAGbC78zJy2kPzDeNoMeR2um0zP3s,2022
|
|
8
|
+
dcc_backend_common/fastapi_error_handling/error_handler.py,sha256=poVjt-5qtU5XB6WNv9n6hQ_jCS7pyWxC319ibI2Q7Rk,1611
|
|
9
|
+
dcc_backend_common/fastapi_health_probes/__init__.py,sha256=3qzn5MHQC0MR01arNFf2ckjkIbw_9hDpCd04d7La3jc,75
|
|
10
|
+
dcc_backend_common/fastapi_health_probes/router.py,sha256=RUHiCOLnDXr5uOmEUsJCXpA0Lg-UUmoHRt6KU1KXF4o,3880
|
|
11
|
+
dcc_backend_common/logger/__init__.py,sha256=dSAG_ktfLo9Fdqsuixg2_9-e_h3jMpuFFm2qPZ58-2Q,85
|
|
12
|
+
dcc_backend_common/logger/logger.py,sha256=hdNoBbqiimOdjhBkQLIimmaPgTPL4LprWo04TSD-7X8,3867
|
|
13
|
+
dcc_backend_common-0.0.1.dist-info/METADATA,sha256=13DevsUe-S7Y3O_MjmT1tHKjG7cH7r1ZLoqpkjyJaMs,7941
|
|
14
|
+
dcc_backend_common-0.0.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
15
|
+
dcc_backend_common-0.0.1.dist-info/licenses/LICENSE,sha256=qgoMkVVKejeBUg8jcD2ZjwydZ4pcIGNztLXJ2OXtJX0,1091
|
|
16
|
+
dcc_backend_common-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Data Competence Center Basel-Stadt
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|