monkeybrain-runtime 1.0.0__py3-none-any.whl

services/common/event_reducers.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+from services.common.compliance import sha256_hex
+
+
+REDUCED_STATE_COLLECTION = "event_reduced_state"
+REDUCER_AUDIT_COLLECTION = "event_reducer_audits"
+CANONICAL_SNAPSHOT_COLLECTION = "canonical_state_snapshots"
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _compact_subject(subject: str | None) -> str:
+    return str(subject or "unknown").strip() or "unknown"
+
+
+def _event_hash(event: dict[str, Any], subject: str | None) -> str:
+    stable_event = {
+        key: value
+        for key, value in event.items()
+        if key not in {"reducer", "broadcast", "websocket_delivery"}
+    }
+    return sha256_hex({"subject": subject, "event": stable_event})
+
+
+def _event_id(event: dict[str, Any], subject: str | None) -> str:
+    return str(event.get("event_id") or event.get("id") or f"event-{_event_hash(event, subject)[:24]}")
+
+
+def _event_time(event: dict[str, Any]) -> Any:
+    return (
+        event.get("occurred_at")
+        or event.get("created_at")
+        or event.get("updated_at")
+        or event.get("timestamp")
+        or _now()
+    )
+
+
+def _aggregate_parts(event: dict[str, Any], subject: str | None) -> tuple[str, str, str]:
+    event_type = str(event.get("event_type") or _compact_subject(subject).rsplit(".", 1)[-1])
+    collection = str(event.get("target_collection") or event.get("collection") or event.get("source_collection") or "")
+    record_id = str(
+        event.get("target_id")
+        or event.get("record_id")
+        or event.get("source_id")
+        or event.get("approval_id")
+        or ""
+    )
+    payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
+    if not record_id and isinstance(payload, dict):
+        record_id = str(
+            payload.get("id")
+            or payload.get("work_order_id")
+            or payload.get("document_id")
+            or payload.get("equipment_id")
+            or payload.get("machine_id")
+            or payload.get("transaction_id")
+            or ""
+        )
+    if not collection:
+        collection = str(event.get("target_entity") or event.get("source") or event_type)
+    if not record_id:
+        record_id = _event_id(event, subject)
+    return event_type, collection, record_id
+
+
+def _state_from_event(event: dict[str, Any]) -> tuple[dict[str, Any], str]:
+    operation = str(event.get("operation") or "").lower()
+    if event.get("event_type") == "cdc-change":
+        if operation == "delete":
+            before = event.get("before") if isinstance(event.get("before"), dict) else {}
+            return {**before, "reducer_record_status": "deleted"}, "delete"
+        after = event.get("after") if isinstance(event.get("after"), dict) else {}
+        if after:
+            return after, operation or "upsert"
+    if isinstance(event.get("payload"), dict):
+        return dict(event["payload"]), operation or "upsert"
+    return {
+        key: value
+        for key, value in event.items()
+        if key not in {"before", "after", "filter", "update", "result"}
+    }, operation or "event"
+
+
+def canonical_snapshot_for_state(
+    *,
+    aggregate_key: str,
+    state: dict[str, Any],
+    source_event_hash: str,
+    event_type: str,
+    subject: str | None,
+) -> dict[str, Any]:
+    snapshot = {
+        "snapshot_id": f"canonical-snapshot-{uuid4().hex}",
+        "aggregate_key": aggregate_key,
+        "event_type": event_type,
+        "subject": subject,
+        "state": state,
+        "source_event_hash": source_event_hash,
+        "created_at": _now(),
+    }
+    snapshot["state_hash"] = sha256_hex(state)
+    snapshot["snapshot_hash"] = sha256_hex(snapshot)
+    return snapshot
+
+
+async def reduce_event_to_state(
+    db: AsyncIOMotorDatabase,
+    event: dict[str, Any],
+    *,
+    subject: str | None = None,
+) -> dict[str, Any]:
+    event_type, collection, record_id = _aggregate_parts(event, subject)
+    aggregate_key = f"{collection}:{record_id}"
+    event_hash = _event_hash(event, subject)
+    existing_audit = await db[REDUCER_AUDIT_COLLECTION].find_one({"event_hash": event_hash}, {"_id": 0})
+    if existing_audit:
+        return {**existing_audit, "idempotent_replay": True}
+
+    reduced_state, operation = _state_from_event(event)
+    prior = await db[REDUCED_STATE_COLLECTION].find_one({"aggregate_key": aggregate_key}, {"_id": 0})
+    previous_state = prior.get("state") if isinstance(prior, dict) and isinstance(prior.get("state"), dict) else {}
+    if operation == "delete":
+        state = {**previous_state, **reduced_state}
+    else:
+        state = {**previous_state, **reduced_state}
+        state.pop("_id", None)
+        state["reducer_record_status"] = "active"
+
+    state_hash = sha256_hex(state)
+    snapshot = canonical_snapshot_for_state(
+        aggregate_key=aggregate_key,
+        state=state,
+        source_event_hash=event_hash,
+        event_type=event_type,
+        subject=subject,
+    )
+    await db[CANONICAL_SNAPSHOT_COLLECTION].insert_one(snapshot.copy())
+
+    now = _now()
+    state_document = {
+        "aggregate_key": aggregate_key,
+        "collection": collection,
+        "record_id": record_id,
+        "event_type": event_type,
+        "subject": subject,
+        "state": state,
+        "state_hash": state_hash,
+        "last_event_id": _event_id(event, subject),
+        "last_event_hash": event_hash,
+        "last_event_at": _event_time(event),
+        "snapshot_id": snapshot["snapshot_id"],
+        "snapshot_hash": snapshot["snapshot_hash"],
+        "updated_at": now,
+    }
+    await db[REDUCED_STATE_COLLECTION].update_one(
+        {"aggregate_key": aggregate_key},
+        {
+            "$set": state_document,
+            "$setOnInsert": {"created_at": now},
+            "$inc": {"version": 1},
+        },
+        upsert=True,
+    )
+    stored = await db[REDUCED_STATE_COLLECTION].find_one({"aggregate_key": aggregate_key}, {"_id": 0})
+    audit = {
+        "reducer_audit_id": f"event-reducer-audit-{uuid4().hex}",
+        "event_id": _event_id(event, subject),
+        "event_hash": event_hash,
+        "event_type": event_type,
+        "subject": subject,
+        "aggregate_key": aggregate_key,
+        "collection": collection,
+        "record_id": record_id,
+        "operation": operation,
+        "previous_state_hash": prior.get("state_hash") if isinstance(prior, dict) else None,
+        "state_hash": state_hash,
+        "snapshot_id": snapshot["snapshot_id"],
+        "snapshot_hash": snapshot["snapshot_hash"],
+        "reduced_state_version": stored.get("version") if stored else 1,
+        "created_at": now,
+    }
+    audit["audit_hash"] = sha256_hex(audit)
+    await db[REDUCER_AUDIT_COLLECTION].insert_one(audit)
+    return {**audit, "idempotent_replay": False}

services/common/event_types.py

@@ -0,0 +1,51 @@
+import re
+from typing import Any
+
+
+DEFAULT_EVENT_TYPE = "generic"
+
+
+def slugify_event_type(value: Any) -> str:
+    text = str(value or "").strip().lower()
+    text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
+    return text or DEFAULT_EVENT_TYPE
+
+
+def measurement_suffix(value: Any) -> str:
+    return slugify_event_type(value).replace("-", "_")
+
+
+def infer_event_type(event: dict[str, Any] | None) -> str:
+    if not isinstance(event, dict):
+        return DEFAULT_EVENT_TYPE
+
+    payload = event.get("payload")
+    payload_dict = payload if isinstance(payload, dict) else {}
+
+    for key in ("event_type", "eventType", "category", "type"):
+        value = event.get(key)
+        if value:
+            return slugify_event_type(value)
+
+    for key in ("event_type", "eventType", "category", "type"):
+        value = payload_dict.get(key)
+        if value:
+            return slugify_event_type(value)
+
+    direction = event.get("direction") or payload_dict.get("direction")
+    provider = event.get("provider") or payload_dict.get("provider")
+    if direction and provider:
+        return slugify_event_type(f"{provider}-{direction}")
+    if direction:
+        return slugify_event_type(direction)
+
+    return DEFAULT_EVENT_TYPE
+
+
+
+def configured_event_type_slugs(value: str) -> list[str]:
+    return [
+        slugify_event_type(event_type)
+        for event_type in str(value or "").split(",")
+        if event_type.strip()
+    ]

services/common/integration_ingestion.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from typing import Any, Literal
+from uuid import uuid4
+
+from fastapi import HTTPException, status
+from motor.motor_asyncio import AsyncIOMotorDatabase
+
+from services.auth.helpers import nats_store
+from services.common.data_transformation import RAW_INTEGRATION_SUBJECT
+from services.common.ontology_registry import load_integration_adapter
+
+
+EXTERNAL_INGESTION_AUDIT_COLLECTION = "external_ingestion_audits"
+IngestionMode = Literal["spark_mini_batch", "kafka_realtime", "flink_realtime", "batch"]
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _payload_hash(payload: dict[str, Any]) -> str:
+    canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str)
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _source_object_names(adapter: dict[str, Any]) -> set[str]:
+    return {str(mapping.get("source_object")) for mapping in adapter.get("mappings") or [] if mapping.get("source_object")}
+
+
+def validate_ingestion_mapping(adapter_id: str, source_object: str) -> dict[str, Any]:
+    try:
+        adapter = load_integration_adapter(adapter_id)
+    except FileNotFoundError as exc:
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Unknown integration adapter '{adapter_id}'.") from exc
+    source_objects = _source_object_names(adapter)
+    if source_object not in source_objects:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+            detail=f"Adapter '{adapter_id}' does not define a direct transformation for source object '{source_object}'.",
+        )
+    return adapter
+
+
+def _normalize_records(records: Any) -> list[dict[str, Any]]:
+    if isinstance(records, dict):
+        records = [records]
+    if not isinstance(records, list) or not records:
+        raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail="Ingestion payload requires at least one record.")
+    normalized: list[dict[str, Any]] = []
+    for index, record in enumerate(records):
+        if not isinstance(record, dict):
+            raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Record at index {index} must be a JSON object.")
+        normalized.append(record)
+    return normalized
+
+
+def _ingestion_source(mode: IngestionMode, source: dict[str, Any] | None) -> dict[str, Any]:
+    details = dict(source or {})
+    details.setdefault("mode", mode)
+    if mode == "spark_mini_batch":
+        details.setdefault("engine", "spark")
+        details.setdefault("cadence", "mini_batch")
+    elif mode == "kafka_realtime":
+        details.setdefault("engine", "kafka")
+        details.setdefault("cadence", "real_time")
+    elif mode == "flink_realtime":
+        details.setdefault("engine", "flink")
+        details.setdefault("cadence", "real_time")
+    else:
+        details.setdefault("engine", "batch")
+        details.setdefault("cadence", "batch")
+    return details
+
+
+def build_raw_integration_event(
+    *,
+    adapter_id: str,
+    source_object: str,
+    payload: dict[str, Any],
+    ingestion_mode: IngestionMode,
+    ingestion_batch_id: str,
+    record_index: int,
+    source: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    return {
+        "event_id": f"external-ingest-{uuid4().hex[:12]}",
+        "event_type": "integration-raw",
+        "source": "external-data-ingestion",
+        "adapter_id": adapter_id,
+        "source_object": source_object,
+        "ingestion_mode": ingestion_mode,
+        "ingestion_batch_id": ingestion_batch_id,
+        "record_index": record_index,
+        "payload_hash": _payload_hash(payload),
+        "payload": payload,
+        "upstream": _ingestion_source(ingestion_mode, source),
+        "received_at": _now(),
+    }
+
+
+async def publish_external_ingestion_records(
+    db: AsyncIOMotorDatabase,
+    *,
+    adapter_id: str,
+    source_object: str,
+    records: Any,
+    ingestion_mode: IngestionMode,
+    source: dict[str, Any] | None = None,
+    requested_by: str | None = None,
+) -> dict[str, Any]:
+    validate_ingestion_mapping(adapter_id, source_object)
+    normalized_records = _normalize_records(records)
+    ingestion_batch_id = f"ing_{uuid4().hex[:16]}"
+    events = [
+        build_raw_integration_event(
+            adapter_id=adapter_id,
+            source_object=source_object,
+            payload=record,
+            ingestion_mode=ingestion_mode,
+            ingestion_batch_id=ingestion_batch_id,
+            record_index=index,
+            source=source,
+        )
+        for index, record in enumerate(normalized_records)
+    ]
+
+    published = 0
+    errors: list[dict[str, Any]] = []
+    for event in events:
+        try:
+            await nats_store.publish_event(nats_store._event_bytes(event), subject=RAW_INTEGRATION_SUBJECT)
+            published += 1
+        except Exception as exc:
+            errors.append({"event_id": event["event_id"], "record_index": event["record_index"], "error": str(exc)})
+
+    audit = {
+        "ingestion_batch_id": ingestion_batch_id,
+        "event_type": "external-ingestion-queued",
+        "adapter_id": adapter_id,
+        "source_object": source_object,
+        "ingestion_mode": ingestion_mode,
+        "record_count": len(events),
+        "published_count": published,
+        "failed_count": len(errors),
+        "subject": RAW_INTEGRATION_SUBJECT,
+        "source": _ingestion_source(ingestion_mode, source),
+        "requested_by": requested_by,
+        "event_ids": [event["event_id"] for event in events],
+        "errors": errors,
+        "created_at": _now(),
+    }
+    await db[EXTERNAL_INGESTION_AUDIT_COLLECTION].insert_one(audit)
+    return {
+        "queued": published == len(events),
+        "subject": RAW_INTEGRATION_SUBJECT,
+        "ingestion_batch_id": ingestion_batch_id,
+        "adapter_id": adapter_id,
+        "source_object": source_object,
+        "ingestion_mode": ingestion_mode,
+        "record_count": len(events),
+        "published_count": published,
+        "failed_count": len(errors),
+        "event_ids": audit["event_ids"],
+        "errors": errors,
+    }

services/common/logging.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import contextvars
+import json
+import logging
+import logging.handlers
+import os
+import sys
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from fastapi import FastAPI, Request
+
+
+_request_id: contextvars.ContextVar[str] = contextvars.ContextVar(
+    "request_id",
+    default="-",
+)
+
+
+class RequestContextFilter(logging.Filter):
+    def __init__(self, service_name: str) -> None:
+        super().__init__()
+        self.service_name = service_name
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.service_name = self.service_name
+        record.request_id = _request_id.get()
+        return True
+
+
+class JsonFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "@timestamp": datetime.fromtimestamp(record.created, timezone.utc).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "service": getattr(record, "service_name", "-"),
+            "request_id": getattr(record, "request_id", "-"),
+            "module": record.module,
+            "function": record.funcName,
+            "line": record.lineno,
+            "process": record.process,
+            "thread": record.threadName,
+        }
+        for key in (
+            "http_method",
+            "http_path",
+            "http_status",
+            "duration_ms",
+            "client_host",
+            "event",
+            "trace_id",
+            "trace_type",
+            "started_at",
+            "finished_at",
+            "method",
+            "path",
+            "query",
+            "status_code",
+            "user_agent",
+            "reasoning_trace_id",
+            "reasoning_stage",
+            "reasoning_summary",
+            "input_summary",
+            "output_summary",
+            "tool_calls",
+            "metadata",
+        ):
+            if hasattr(record, key):
+                payload[key] = getattr(record, key)
+        if record.exc_info:
+            payload["exception"] = self.formatException(record.exc_info)
+        return json.dumps(payload, default=str, separators=(",", ":"))
+
+
+class LogstashTCPHandler(logging.handlers.SocketHandler):
+    """Send newline-delimited JSON logs to Logstash TCP input."""
+
+    def makePickle(self, record: logging.LogRecord) -> bytes:
+        return (self.format(record) + "\n").encode("utf-8")
+
+
+def _env_int(name: str, default: int) -> int:
+    try:
+        return int(os.getenv(name, str(default)))
+    except ValueError:
+        return default
+
+
+def _build_handlers(service_name: str) -> list[logging.Handler]:
+    mode = os.getenv("LOG_MODE", os.getenv("LOG_BACKEND", "console")).strip().lower()
+    level = os.getenv("LOG_LEVEL", "INFO").upper()
+
+    if mode == "elk":
+        host = os.getenv("ELK_HOST", os.getenv("LOGSTASH_HOST", "localhost"))
+        port = _env_int("ELK_PORT", _env_int("LOGSTASH_PORT", 5000))
+        handler: logging.Handler = LogstashTCPHandler(host, port)
+        handler.setFormatter(JsonFormatter())
+        handler.setLevel(level)
+        return [handler]
+
+    log_dir = Path(os.getenv("LOG_DIR", "logs"))
+    log_dir.mkdir(parents=True, exist_ok=True)
+    file_handler = logging.handlers.RotatingFileHandler(
+        log_dir / f"{service_name}.log",
+        maxBytes=_env_int("LOG_MAX_BYTES", 10 * 1024 * 1024),
+        backupCount=_env_int("LOG_BACKUP_COUNT", 5),
+        encoding="utf-8",
+    )
+    file_handler.setLevel(level)
+    file_handler.setFormatter(
+        logging.Formatter(
+            "%(asctime)s %(levelname)s [%(service_name)s] "
+            "[request_id=%(request_id)s] %(name)s: %(message)s"
+        )
+    )
+
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(level)
+    stream_handler.setFormatter(file_handler.formatter)
+    return [file_handler, stream_handler]
+
+
+def configure_service_logging(service_name: str) -> logging.Logger:
+    level = os.getenv("LOG_LEVEL", "INFO").upper()
+    handlers = _build_handlers(service_name)
+    context_filter = RequestContextFilter(service_name)
+
+    root = logging.getLogger()
+    root.handlers = handlers
+    root.setLevel(level)
+
+    for handler in handlers:
+        handler.addFilter(context_filter)
+
+    for logger_name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
+        logger = logging.getLogger(logger_name)
+        logger.handlers = handlers
+        logger.propagate = False
+        logger.setLevel(level)
+
+    logger = logging.getLogger(service_name)
+    logger.info(
+        "Logging configured",
+        extra={
+            "event": "logging_configured",
+            "log_mode": os.getenv("LOG_MODE", os.getenv("LOG_BACKEND", "elk")),
+        },
+    )
+    return logger
+
+
+def install_request_logging(app: FastAPI, service_name: str) -> None:
+    logger = logging.getLogger(f"{service_name}.requests")
+
+    @app.middleware("http")
+    async def request_logging_middleware(request: Request, call_next):
+        request_id = request.headers.get("x-request-id") or str(uuid.uuid4())
+        token = _request_id.set(request_id)
+        start = time.perf_counter()
+        client_host = request.client.host if request.client else "-"
+        try:
+            response = await call_next(request)
+            duration_ms = round((time.perf_counter() - start) * 1000, 2)
+            logger.info(
+                "%s %s completed with %s in %.2fms",
+                request.method,
+                request.url.path,
+                response.status_code,
+                duration_ms,
+                extra={
+                    "event": "http_request",
+                    "http_method": request.method,
+                    "http_path": request.url.path,
+                    "http_status": response.status_code,
+                    "duration_ms": duration_ms,
+                    "client_host": client_host,
+                },
+            )
+            response.headers["x-request-id"] = request_id
+            return response
+        except Exception:
+            duration_ms = round((time.perf_counter() - start) * 1000, 2)
+            logger.exception(
+                "%s %s failed in %.2fms",
+                request.method,
+                request.url.path,
+                duration_ms,
+                extra={
+                    "event": "http_request_error",
+                    "http_method": request.method,
+                    "http_path": request.url.path,
+                    "duration_ms": duration_ms,
+                    "client_host": client_host,
+                },
+            )
+            raise
+        finally:
+            _request_id.reset(token)

services/common/models/__init__.py

@@ -0,0 +1,2 @@
+"""Shared model primitives used across services."""
+

services/common/models/databricks.py

@@ -0,0 +1,25 @@
+from typing import Literal
+
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field
+
+class DatabricksConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    host: str = Field(default="", description="Databricks workspace URL, for example https://<workspace>.azuredatabricks.net.")
+    token: str = Field(default="", description="Databricks personal access token or service principal token.")
+    cluster_id: str = Field(default="", description="Existing Databricks cluster ID.")
+    catalog: str = "hive_metastore"
+    schema_name: str = Field(
+        default="default",
+        validation_alias=AliasChoices("schema", "schema_name"),
+        serialization_alias="schema",
+    )
+
+class IngestRequest(BaseModel):
+    table_name: str
+    mode: Literal["overwrite", "append", "error", "errorifexists", "ignore"] = "overwrite"
+    databricks_config: DatabricksConfig
+
+class QueryRequest(BaseModel):
+    sql: str
+    databricks_config: DatabricksConfig