karrio-server 2025.5rc35__py3-none-any.whl → 2025.5.1__py3-none-any.whl

karrio/server/VERSION

@@ -1 +1 @@
-2025.5rc35
+2025.5.1

karrio/server/settings/apm.py

@@ -35,33 +35,192 @@ if POSTHOG_KEY:
     posthog.host = POSTHOG_HOST
 
 
-#  Sentry
+# =============================================================================
+# Sentry Configuration
+# =============================================================================
+#
+# Sentry provides error tracking, performance monitoring, and distributed
+# tracing for Karrio. When SENTRY_DSN is configured, the following features
+# are enabled:
+#
+# 1. Error Tracking: All unhandled exceptions are captured with full context
+# 2. Performance Monitoring: API endpoints and gateway operations are traced
+# 3. Distributed Tracing: Carrier API calls are linked to parent transactions
+# 4. Metrics: Custom metrics for carrier operations (rates, shipments, etc.)
+# 5. Logging Integration: ERROR/CRITICAL logs are sent to Sentry
+#
+# Environment Variables:
+#   SENTRY_DSN                  - Sentry Data Source Name (required to enable)
+#   SENTRY_ENVIRONMENT          - Environment name (default: from ENV or "production")
+#   SENTRY_RELEASE              - Release version (default: VERSION)
+#   SENTRY_TRACES_SAMPLE_RATE   - Transaction sampling rate 0.0-1.0 (default: 1.0)
+#   SENTRY_PROFILES_SAMPLE_RATE - Profile sampling rate 0.0-1.0 (default: 1.0)
+#   SENTRY_SEND_PII             - Send personally identifiable information (default: true)
+#   SENTRY_DEBUG                - Enable Sentry debug mode (default: false)
+#
+# =============================================================================
+
 sentry_sdk.utils.MAX_STRING_LENGTH = 4096
 SENTRY_DSN = config("SENTRY_DSN", default=None)
+SENTRY_ENVIRONMENT = config("SENTRY_ENVIRONMENT", default=config("ENV", default="production"))
+SENTRY_RELEASE = config("SENTRY_RELEASE", default=config("VERSION", default=None))
+SENTRY_TRACES_SAMPLE_RATE = config("SENTRY_TRACES_SAMPLE_RATE", default=1.0, cast=float)
+SENTRY_PROFILES_SAMPLE_RATE = config("SENTRY_PROFILES_SAMPLE_RATE", default=1.0, cast=float)
+SENTRY_SEND_PII = config("SENTRY_SEND_PII", default=True, cast=bool)
+SENTRY_DEBUG = config("SENTRY_DEBUG", default=False, cast=bool)
+
+
+def _sentry_before_send(event, hint):
+    """Pre-process events before sending to Sentry.
+
+    This hook allows us to:
+    - Scrub sensitive data (API keys, tokens, passwords)
+    - Add custom tags
+    - Filter out certain events
+    """
+    # Scrub sensitive data from request bodies
+    if "request" in event:
+        request_data = event["request"]
+
+        # Scrub headers
+        if "headers" in request_data:
+            sensitive_headers = ["authorization", "x-api-key", "cookie", "x-csrf-token"]
+            for header in sensitive_headers:
+                if header in request_data["headers"]:
+                    request_data["headers"][header] = "[Filtered]"
+
+        # Scrub POST data
+        if "data" in request_data and isinstance(request_data["data"], dict):
+            sensitive_fields = [
+                "password", "secret", "token", "api_key", "apikey",
+                "access_token", "refresh_token", "client_secret",
+                "account_number", "meter_number", "license_key",
+            ]
+            for field in sensitive_fields:
+                for key in list(request_data["data"].keys()):
+                    if field.lower() in key.lower():
+                        request_data["data"][key] = "[Filtered]"
+
+    return event
+
+
+def _sentry_before_send_transaction(event, hint):
+    """Pre-process transactions before sending to Sentry.
+
+    This hook allows us to:
+    - Filter out noisy transactions (health checks, static files)
+    - Add custom tags
+    """
+    transaction_name = event.get("transaction", "")
+
+    # Filter out health check and monitoring endpoints
+    noisy_endpoints = [
+        "/health",
+        "/ready",
+        "/live",
+        "/_health",
+        "/favicon.ico",
+        "/static/",
+        "/robots.txt",
+    ]
+
+    for endpoint in noisy_endpoints:
+        if transaction_name.startswith(endpoint):
+            return None  # Drop this transaction
+
+    return event
+
 
 if SENTRY_DSN:
     # Build integrations list
-    integrations = [DjangoIntegration()]
+    integrations = [
+        DjangoIntegration(
+            transaction_style="url",  # Use URL patterns for transaction names
+            middleware_spans=True,    # Create spans for middleware
+            signals_spans=True,       # Create spans for Django signals
+        ),
+    ]
+
+    # Add PostHog integration if available
     if POSTHOG_KEY and PostHogIntegration is not None:
         integrations.append(PostHogIntegration())
 
+    # Try to add Redis integration if Redis is configured
+    try:
+        from sentry_sdk.integrations.redis import RedisIntegration
+        if config("REDIS_URL", default=None) or config("REDIS_HOST", default=None):
+            integrations.append(RedisIntegration())
+    except ImportError:
+        pass
+
+    # Try to add Huey integration for background tasks
+    try:
+        from sentry_sdk.integrations.huey import HueyIntegration
+        integrations.append(HueyIntegration())
+    except ImportError:
+        pass
+
+    # Try to add httpx integration for async HTTP clients
+    try:
+        from sentry_sdk.integrations.httpx import HttpxIntegration
+        integrations.append(HttpxIntegration())
+    except Exception:
+        pass  # httpx may not be installed
+
+    # Try to add Strawberry GraphQL integration
+    try:
+        from sentry_sdk.integrations.strawberry import StrawberryIntegration
+        integrations.append(StrawberryIntegration(async_execution=False))
+    except Exception:
+        pass  # strawberry integration may not be available
+
     sentry_sdk.init(
         dsn=SENTRY_DSN,
         integrations=integrations,
-        # Set traces_sample_rate to 1.0 to capture 100%
-        # of transactions for tracing.
-        traces_sample_rate=1.0,
-        # Set profile_session_sample_rate to 1.0 to profile 100%
-        # of profile sessions.
-        profile_session_sample_rate=1.0,
-        # Set profile_lifecycle to "trace" to automatically
-        # run the profiler on when there is an active transaction
+
+        # Environment and release tracking
+        environment=SENTRY_ENVIRONMENT,
+        release=SENTRY_RELEASE,
+
+        # Performance monitoring
+        traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE,
+        profile_session_sample_rate=SENTRY_PROFILES_SAMPLE_RATE,
         profile_lifecycle="trace",
-        # If you wish to associate users to errors (assuming you are using
-        # django.contrib.auth) you may enable sending PII data.
-        send_default_pii=True,
-        # Enable sending logs to Sentry
+
+        # Privacy settings
+        send_default_pii=SENTRY_SEND_PII,
+
+        # Logging integration
         enable_logs=True,
+
+        # Debug mode
+        debug=SENTRY_DEBUG,
+
+        # Event processing hooks
+        before_send=_sentry_before_send,
+        before_send_transaction=_sentry_before_send_transaction,
+
+        # Additional options
+        max_breadcrumbs=50,  # Keep last 50 breadcrumbs for context
+        attach_stacktrace=True,  # Attach stack traces to messages
+        include_source_context=True,  # Include source code in stack traces
+        include_local_variables=True,  # Include local variables in stack traces
+
+        # Set custom tags
+        _experiments={
+            "record_sql_params": True,  # Record SQL query parameters
+        },
+    )
+
+    # Set default tags that will be applied to all events
+    sentry_sdk.set_tag("service", "karrio-api")
+    sentry_sdk.set_tag("framework", "django")
+
+    import logging
+    logger = logging.getLogger(__name__)
+    logger.info(
+        f"Sentry initialized: env={SENTRY_ENVIRONMENT}, "
+        f"traces_sample_rate={SENTRY_TRACES_SAMPLE_RATE}"
     )
 
 
@@ -191,3 +350,141 @@ if OTEL_ENABLED and OTEL_EXPORTER_OTLP_ENDPOINT:
     # Log that OpenTelemetry is enabled
     logger = logging.getLogger(__name__)
     logger.info(f"OpenTelemetry enabled: Service={OTEL_SERVICE_NAME}, Endpoint={OTEL_EXPORTER_OTLP_ENDPOINT}")
+
+
+# =============================================================================
+# Datadog Configuration
+# =============================================================================
+#
+# Datadog provides full-stack observability including APM, infrastructure
+# monitoring, logs, and more. When DD_TRACE_ENABLED is set, the following
+# features are enabled:
+#
+# 1. APM Tracing: Distributed tracing for all requests and background tasks
+# 2. Metrics: Custom metrics via DogStatsD
+# 3. Logs: Automatic trace ID injection into logs
+# 4. Profiling: Continuous profiling (optional)
+#
+# Environment Variables:
+#   DD_TRACE_ENABLED        - Enable Datadog tracing (default: false)
+#   DD_SERVICE              - Service name (default: karrio-api)
+#   DD_ENV                  - Environment name (default: from ENV or "production")
+#   DD_VERSION              - Application version (default: VERSION)
+#   DD_AGENT_HOST           - Datadog agent host (default: localhost)
+#   DD_TRACE_AGENT_PORT     - Datadog agent port (default: 8126)
+#   DD_DOGSTATSD_PORT       - DogStatsD port for metrics (default: 8125)
+#   DD_TRACE_SAMPLE_RATE    - Sampling rate 0.0-1.0 (default: 1.0)
+#   DD_PROFILING_ENABLED    - Enable continuous profiling (default: false)
+#   DD_LOGS_INJECTION       - Inject trace IDs into logs (default: true)
+#
+# =============================================================================
+
+DD_TRACE_ENABLED = config("DD_TRACE_ENABLED", default=False, cast=bool)
+DATADOG_ENABLED = config("DATADOG_ENABLED", default=False, cast=bool)  # Alias
+
+# Use either DD_TRACE_ENABLED or DATADOG_ENABLED
+_datadog_enabled = DD_TRACE_ENABLED or DATADOG_ENABLED
+
+if _datadog_enabled:
+    # Datadog configuration
+    DD_SERVICE = config("DD_SERVICE", default="karrio-api")
+    DD_ENV = config("DD_ENV", default=config("ENV", default="production"))
+    DD_VERSION = config("DD_VERSION", default=config("VERSION", default="unknown"))
+    DD_AGENT_HOST = config("DD_AGENT_HOST", default="localhost")
+    DD_TRACE_AGENT_PORT = config("DD_TRACE_AGENT_PORT", default=8126, cast=int)
+    DD_DOGSTATSD_PORT = config("DD_DOGSTATSD_PORT", default=8125, cast=int)
+    DD_TRACE_SAMPLE_RATE = config("DD_TRACE_SAMPLE_RATE", default=1.0, cast=float)
+    DD_PROFILING_ENABLED = config("DD_PROFILING_ENABLED", default=False, cast=bool)
+    DD_LOGS_INJECTION = config("DD_LOGS_INJECTION", default=True, cast=bool)
+
+    try:
+        import ddtrace
+        from ddtrace import config as dd_config, tracer, patch_all
+
+        # Configure tracer
+        ddtrace.config.service = DD_SERVICE
+        ddtrace.config.env = DD_ENV
+        ddtrace.config.version = DD_VERSION
+
+        # Configure Django integration
+        dd_config.django["service_name"] = DD_SERVICE
+        dd_config.django["cache_service_name"] = f"{DD_SERVICE}-cache"
+        dd_config.django["database_service_name"] = f"{DD_SERVICE}-db"
+        dd_config.django["trace_query_string"] = True
+        dd_config.django["analytics_enabled"] = True
+
+        # Configure trace sampling
+        tracer.configure(
+            hostname=DD_AGENT_HOST,
+            port=DD_TRACE_AGENT_PORT,
+        )
+
+        # Set global sample rate
+        from ddtrace.sampler import DatadogSampler
+        tracer.configure(sampler=DatadogSampler(default_sample_rate=DD_TRACE_SAMPLE_RATE))
+
+        # Enable log injection
+        if DD_LOGS_INJECTION:
+            ddtrace.patch(logging=True)
+
+        # Patch all supported libraries
+        patch_all(
+            django=True,
+            redis=True,
+            psycopg=True,
+            requests=True,
+            httpx=True,
+            logging=DD_LOGS_INJECTION,
+        )
+
+        # Patch Huey for background task tracing
+        try:
+            from ddtrace import patch
+            patch(huey=True)
+        except Exception:
+            pass  # Huey integration may not be available in all ddtrace versions
+
+        # Enable profiling if configured
+        if DD_PROFILING_ENABLED:
+            try:
+                import ddtrace.profiling.auto  # noqa: F401
+            except ImportError:
+                pass
+
+        # Configure DogStatsD for metrics
+        try:
+            from datadog import initialize, statsd
+
+            initialize(
+                statsd_host=DD_AGENT_HOST,
+                statsd_port=DD_DOGSTATSD_PORT,
+            )
+
+            # Set default tags for all metrics
+            statsd.constant_tags = [
+                f"service:{DD_SERVICE}",
+                f"env:{DD_ENV}",
+                f"version:{DD_VERSION}",
+            ]
+
+        except ImportError:
+            pass  # datadog package not installed, metrics won't work
+
+        import logging
+        logger = logging.getLogger(__name__)
+        logger.info(
+            f"Datadog APM initialized: service={DD_SERVICE}, env={DD_ENV}, "
+            f"agent={DD_AGENT_HOST}:{DD_TRACE_AGENT_PORT}"
+        )
+
+    except ImportError:
+        import logging
+        logger = logging.getLogger(__name__)
+        logger.warning(
+            "Datadog tracing enabled but ddtrace package not installed. "
+            "Install with: pip install ddtrace"
+        )
+    except Exception as e:
+        import logging
+        logger = logging.getLogger(__name__)
+        logger.warning(f"Failed to initialize Datadog APM: {e}")

karrio/server/settings/constance.py

@@ -1,4 +1,4 @@
-""" Dynamic configuration editable on runtime powered by django-constance."""
+"""Dynamic configuration editable on runtime powered by django-constance."""
 
 from decouple import config
 import karrio.references as ref
@@ -22,6 +22,7 @@ GOOGLE_CLOUD_API_KEY = config("GOOGLE_CLOUD_API_KEY", default="")
 CANADAPOST_ADDRESS_COMPLETE_API_KEY = config(
     "CANADAPOST_ADDRESS_COMPLETE_API_KEY", default=""
 )
+
 # data retention env in days
 ORDER_DATA_RETENTION = config("ORDER_DATA_RETENTION", default=183, cast=int)
 TRACKER_DATA_RETENTION = config("TRACKER_DATA_RETENTION", default=183, cast=int)
@@ -29,7 +30,9 @@ SHIPMENT_DATA_RETENTION = config("SHIPMENT_DATA_RETENTION", default=183, cast=in
 API_LOGS_DATA_RETENTION = config("API_LOGS_DATA_RETENTION", default=92, cast=int)
 
 # registry config
-ENABLE_ALL_PLUGINS_BY_DEFAULT = config("ENABLE_ALL_PLUGINS_BY_DEFAULT", default=True if base.DEBUG else False, cast=bool)
+ENABLE_ALL_PLUGINS_BY_DEFAULT = config(
+    "ENABLE_ALL_PLUGINS_BY_DEFAULT", default=True if base.DEBUG else False, cast=bool
+)
 
 # Create feature flags config only for modules that exist
 FEATURE_FLAGS_CONFIG = {
@@ -161,8 +164,21 @@ PLUGIN_REGISTRY = {
             config(f"{ext.upper()}_ENABLED", default=True, cast=bool),
             f"{metadata.get('label')} plugin",
             bool,
-        ) for ext, metadata in ref.PLUGIN_METADATA.items()
-    }
+        )
+        for ext, metadata in ref.PLUGIN_METADATA.items()
+    },
+}
+
+# Collect plugin system configs from ref.SYSTEM_CONFIGS
+# Format: Dict[str, Tuple[default_value, description, type]]
+PLUGIN_SYSTEM_CONFIG = {
+    key: (config(key, default=default_value, cast=value_type), description, value_type)
+    for key, (default_value, description, value_type) in ref.SYSTEM_CONFIGS.items()
+}
+PLUGIN_SYSTEM_CONFIG_FIELDSETS = {
+    f"{metadata.get('label')} Config": tuple(metadata.get("system_config", {}).keys())
+    for _, metadata in ref.PLUGIN_METADATA.items()
+    if metadata.get("system_config")
 }
 
 
@@ -218,6 +234,7 @@ CONSTANCE_CONFIG = {
     ),
     **{k: v for k, v in FEATURE_FLAGS_CONFIG.items() if v is not None},
     **PLUGIN_REGISTRY,
+    **PLUGIN_SYSTEM_CONFIG,
 }
 
 CONSTANCE_CONFIG_FIELDSETS = {
@@ -241,5 +258,12 @@ CONSTANCE_CONFIG_FIELDSETS = {
     ),
     "Feature Flags": tuple(FEATURE_FLAGS_FIELDSET),
     "Registry Config": ("ENABLE_ALL_PLUGINS_BY_DEFAULT",),
-    "Registry Plugins": tuple([k for k in PLUGIN_REGISTRY.keys() if not k in ("ENABLE_ALL_PLUGINS_BY_DEFAULT",)]),
+    "Registry Plugins": tuple(
+        [
+            k
+            for k in PLUGIN_REGISTRY.keys()
+            if not k in ("ENABLE_ALL_PLUGINS_BY_DEFAULT",)
+        ]
+    ),
+    **PLUGIN_SYSTEM_CONFIG_FIELDSETS,
 }