schematalog-core 0.1.0__py3-none-any.whl

schematalog/common/__init__.py

@@ -0,0 +1 @@
+"""Layer-neutral utilities shared across the application (no dependency on other layers)."""

schematalog/common/avroform/__init__.py

@@ -0,0 +1,15 @@
+"""Convert between JSON Schema and Apache Avro schemas.
+
+A small, dependency-free converter covering the common subset of JSON Schema:
+objects/records, primitives, arrays, enums, nullable fields (unions), nested
+records, string formats that map to Avro logical types, and internal ``$ref``
+pointers (``#/$defs/...``, inlined before conversion). Constructs with no clean Avro
+equivalent (``oneOf``/``anyOf``/``allOf``, external or recursive ``$ref``) raise
+:class:`AvroConversionError` rather than emitting an invalid schema.
+"""
+
+from .exceptions import AvroConversionError
+from .to_avro import to_avro
+from .to_json_schema import to_json_schema
+
+__all__ = ["AvroConversionError", "to_avro", "to_json_schema"]

schematalog/common/avroform/exceptions.py

@@ -0,0 +1,2 @@
+class AvroConversionError(Exception):
+    """Raised when a schema cannot be converted between JSON Schema and Avro."""

schematalog/common/avroform/to_avro.py

@@ -0,0 +1,208 @@
+"""JSON Schema -> Avro schema."""
+
+import re
+from typing import Any
+
+from .exceptions import AvroConversionError
+
+AvroType = str | dict[str, Any] | list[Any]
+
+# JSON Schema primitive `type` -> Avro primitive type.
+_PRIMITIVES = {
+    "boolean": "boolean",
+    "integer": "long",
+    "number": "double",
+    "null": "null",
+}
+
+# JSON Schema string `format` -> (Avro base type, Avro logicalType).
+_STRING_FORMATS = {
+    "date": ("int", "date"),
+    "time": ("int", "time-millis"),
+    "date-time": ("long", "timestamp-millis"),
+    "uuid": ("string", "uuid"),
+}
+
+# JSON Schema keywords we cannot faithfully express in Avro. (`$ref` is handled
+# separately: internal pointers are inlined before conversion, see `_resolve_refs`.)
+_UNSUPPORTED = ("oneOf", "anyOf", "allOf", "not")
+
+
+def to_avro(
+    json_schema: dict[str, Any], *, name: str = "Record", namespace: str = ""
+) -> AvroType:
+    """Convert a JSON Schema into an Avro schema.
+
+    Args:
+        json_schema: The JSON Schema to convert.
+        name: Name for the root Avro record/enum (a schema ``title`` overrides it).
+        namespace: Optional Avro namespace for named types.
+
+    Returns:
+        The Avro schema as a JSON-compatible structure.
+
+    Raises:
+        AvroConversionError: If the schema uses unsupported constructs.
+    """
+    return _convert(_resolve_refs(json_schema, json_schema, ()), name, namespace, set())
+
+
+def _resolve_refs(schema: Any, root: dict, stack: tuple[str, ...]) -> Any:
+    """Inline internal ``$ref`` pointers (``#/...``) against the document root.
+
+    Avro has no reference mechanism, so a fragment ref is expanded in place before
+    conversion - e.g. ``{"$ref": "#/$defs/country"}`` becomes the ``country`` subschema.
+    External refs (anything not ``#/...``) and cyclic refs cannot be expressed and raise.
+    """
+    if isinstance(schema, list):
+        return [_resolve_refs(item, root, stack) for item in schema]
+    if not isinstance(schema, dict):
+        return schema
+    ref = schema.get("$ref")
+    if isinstance(ref, str):
+        return _resolve_ref(schema, ref, root, stack)
+    return {key: _resolve_refs(value, root, stack) for key, value in schema.items()}
+
+
+def _resolve_ref(schema: dict, ref: str, root: dict, stack: tuple[str, ...]) -> Any:
+    """Expand a single ``$ref`` node, overlaying any sibling keys onto the target."""
+    if not ref.startswith("#/"):
+        raise AvroConversionError(f"Cannot resolve external $ref: {ref!r}.")
+    if ref in stack:
+        raise AvroConversionError(f"Cannot express recursive $ref in Avro: {ref!r}.")
+    target = _resolve_refs(_deref(ref, root), root, (*stack, ref))
+    # JSON Schema allows keys beside `$ref`; overlay them onto the resolved object.
+    siblings = {key: value for key, value in schema.items() if key != "$ref"}
+    if siblings and isinstance(target, dict):
+        return {**target, **_resolve_refs(siblings, root, stack)}
+    return target
+
+
+def _deref(ref: str, root: dict) -> Any:
+    """Resolve a ``#/a/b`` JSON Pointer against ``root`` (with ~0/~1 unescaping)."""
+    node: Any = root
+    for raw in ref[2:].split("/"):  # drop the leading '#/'
+        token = raw.replace("~1", "/").replace("~0", "~")
+        if not isinstance(node, dict) or token not in node:
+            raise AvroConversionError(f"Cannot resolve $ref: {ref!r}.")
+        node = node[token]
+    return node
+
+
+def _convert(schema: Any, name: str, namespace: str, seen: set[str]) -> AvroType:
+    if not isinstance(schema, dict):
+        raise AvroConversionError(f"Expected a schema object, got {type(schema).__name__}.")
+    for keyword in _UNSUPPORTED:
+        if keyword in schema:
+            raise AvroConversionError(f"Unsupported JSON Schema keyword: {keyword!r}.")
+
+    if "enum" in schema:
+        return _enum(schema, name, namespace, seen)
+
+    json_type = schema.get("type")
+    if isinstance(json_type, list):
+        return _union(json_type, schema, name, namespace, seen)
+    if json_type == "object":
+        return _object(schema, name, namespace, seen)
+    if json_type == "array":
+        return _array(schema, name, namespace, seen)
+    if json_type == "string":
+        return _string(schema)
+    if json_type in _PRIMITIVES:
+        return _PRIMITIVES[json_type]
+    raise AvroConversionError(f"Unsupported or missing JSON Schema type: {json_type!r}.")
+
+
+def _object(schema: dict, name: str, namespace: str, seen: set[str]) -> AvroType:
+    additional = schema.get("additionalProperties")
+    if not schema.get("properties") and isinstance(additional, dict):
+        return {"type": "map", "values": _convert(additional, name, namespace, seen)}
+
+    record_name = _unique(_pascal(schema.get("title") or name), seen)
+    required = set(schema.get("required", []))
+    fields = []
+    for prop_name, prop_schema in schema.get("properties", {}).items():
+        field_type = _convert(prop_schema, prop_name, namespace, seen)
+        field: dict[str, Any] = {"name": prop_name, "type": field_type}
+        if isinstance(prop_schema, dict) and prop_schema.get("description"):
+            field["doc"] = prop_schema["description"]
+        if prop_name not in required:
+            field["type"] = _nullable(field_type)
+            field["default"] = None
+        fields.append(field)
+
+    record: dict[str, Any] = {"type": "record", "name": record_name, "fields": fields}
+    if namespace:
+        record["namespace"] = namespace
+    if schema.get("description"):
+        record["doc"] = schema["description"]
+    return record
+
+
+def _array(schema: dict, name: str, namespace: str, seen: set[str]) -> AvroType:
+    items = schema.get("items")
+    if not isinstance(items, dict):
+        raise AvroConversionError("Array schema must declare an 'items' object.")
+    return {"type": "array", "items": _convert(items, f"{_pascal(name)}Item", namespace, seen)}
+
+
+def _string(schema: dict) -> AvroType:
+    fmt = schema.get("format")
+    if isinstance(fmt, str) and fmt in _STRING_FORMATS:
+        base, logical = _STRING_FORMATS[fmt]
+        return {"type": base, "logicalType": logical}
+    return "string"
+
+
+def _enum(schema: dict, name: str, namespace: str, seen: set[str]) -> AvroType:
+    symbols = schema["enum"]
+    valid = all(isinstance(s, str) and _is_avro_name(s) for s in symbols)
+    if valid and len(set(symbols)) == len(symbols):
+        enum: dict[str, Any] = {
+            "type": "enum",
+            "name": _unique(_pascal(name), seen),
+            "symbols": list(symbols),
+        }
+        if namespace:
+            enum["namespace"] = namespace
+        return enum
+    # Enums Avro can't represent (non-string or non-identifier values) degrade to string.
+    return "string"
+
+
+def _union(types: list, schema: dict, name: str, namespace: str, seen: set[str]) -> AvroType:
+    base = {key: value for key, value in schema.items() if key not in ("type", "enum")}
+    parts: list[AvroType] = []
+    for json_type in types:
+        member = _convert({**base, "type": json_type}, name, namespace, seen)
+        if member not in parts:
+            parts.append(member)
+    # Avro convention: null first so a `null` default is valid.
+    if "null" in parts:
+        parts = ["null", *(part for part in parts if part != "null")]
+    return parts
+
+
+def _nullable(avro_type: AvroType) -> AvroType:
+    if isinstance(avro_type, list):
+        return avro_type if "null" in avro_type else ["null", *avro_type]
+    return ["null", avro_type]
+
+
+def _pascal(value: str) -> str:
+    parts = re.split(r"[^0-9a-zA-Z]+", value)
+    name = "".join(part[:1].upper() + part[1:] for part in parts if part)
+    return name or "Record"
+
+
+def _unique(name: str, seen: set[str]) -> str:
+    candidate, index = name, 1
+    while candidate in seen:
+        index += 1
+        candidate = f"{name}{index}"
+    seen.add(candidate)
+    return candidate
+
+
+def _is_avro_name(value: str) -> bool:
+    return bool(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value))

schematalog/common/avroform/to_json_schema.py

@@ -0,0 +1,124 @@
+"""Avro schema -> JSON Schema."""
+
+from typing import Any
+
+from .exceptions import AvroConversionError
+
+JsonSchema = dict[str, Any]
+
+# Avro primitive type -> JSON Schema.
+_PRIMITIVES = {
+    "string": {"type": "string"},
+    "bytes": {"type": "string"},
+    "int": {"type": "integer"},
+    "long": {"type": "integer"},
+    "float": {"type": "number"},
+    "double": {"type": "number"},
+    "boolean": {"type": "boolean"},
+    "null": {"type": "null"},
+}
+
+# Avro logicalType -> JSON Schema.
+_LOGICAL_TYPES = {
+    "date": {"type": "string", "format": "date"},
+    "time-millis": {"type": "string", "format": "time"},
+    "timestamp-millis": {"type": "string", "format": "date-time"},
+    "uuid": {"type": "string", "format": "uuid"},
+}
+
+
+def to_json_schema(avro_schema: Any) -> JsonSchema:
+    """Convert an Avro schema into a JSON Schema."""
+    return _convert(avro_schema)
+
+
+def _convert(avro: Any) -> JsonSchema:
+    if isinstance(avro, str):
+        return _primitive(avro)
+    if isinstance(avro, list):
+        return _union(avro)
+    if isinstance(avro, dict):
+        return _named(avro)
+    raise AvroConversionError(f"Unexpected Avro schema node: {avro!r}.")
+
+
+def _primitive(name: str) -> JsonSchema:
+    try:
+        return dict(_PRIMITIVES[name])
+    except KeyError:
+        raise AvroConversionError(f"Unknown Avro type: {name!r}.") from None
+
+
+def _named(avro: dict) -> JsonSchema:
+    logical = avro.get("logicalType")
+    if logical in _LOGICAL_TYPES:
+        return dict(_LOGICAL_TYPES[logical])
+
+    avro_type = avro.get("type")
+    if avro_type == "record":
+        return _record(avro)
+    if avro_type == "enum":
+        return {"type": "string", "enum": list(avro["symbols"])}
+    if avro_type == "array":
+        return {"type": "array", "items": _convert(avro["items"])}
+    if avro_type == "map":
+        return {"type": "object", "additionalProperties": _convert(avro["values"])}
+    if avro_type == "fixed":
+        return {"type": "string"}
+    if isinstance(avro_type, str):
+        return _primitive(avro_type)
+    if isinstance(avro_type, (list, dict)):
+        return _convert(avro_type)
+    raise AvroConversionError(f"Unsupported Avro schema: {avro!r}.")
+
+
+def _record(avro: dict) -> JsonSchema:
+    properties: dict[str, Any] = {}
+    required: list[str] = []
+    for field in avro.get("fields", []):
+        field_type = field["type"]
+        optional = _is_nullable(field_type) or "default" in field
+        prop = _convert(_strip_null(field_type))
+        if field.get("doc"):
+            prop = {**prop, "description": field["doc"]}
+        properties[field["name"]] = prop
+        if not optional:
+            required.append(field["name"])
+
+    result: JsonSchema = {"type": "object", "properties": properties}
+    if required:
+        result["required"] = required
+    if avro.get("name"):
+        result["title"] = avro["name"]
+    if avro.get("doc"):
+        result["description"] = avro["doc"]
+    return result
+
+
+def _union(members: list) -> JsonSchema:
+    non_null = [member for member in members if member != "null"]
+    nullable = "null" in members
+    if len(non_null) == 1:
+        schema = _convert(non_null[0])
+        return _with_null(schema) if nullable else schema
+    options = [_convert(member) for member in non_null]
+    if nullable:
+        options.append({"type": "null"})
+    return {"anyOf": options}
+
+
+def _with_null(schema: JsonSchema) -> JsonSchema:
+    if set(schema) == {"type"} and isinstance(schema["type"], str):
+        return {"type": [schema["type"], "null"]}
+    return {"anyOf": [schema, {"type": "null"}]}
+
+
+def _is_nullable(field_type: Any) -> bool:
+    return isinstance(field_type, list) and "null" in field_type
+
+
+def _strip_null(field_type: Any) -> Any:
+    if not isinstance(field_type, list):
+        return field_type
+    non_null = [member for member in field_type if member != "null"]
+    return non_null[0] if len(non_null) == 1 else non_null

schematalog/common/logging.py

@@ -0,0 +1,141 @@
+"""Structured logging: unclogger (JSON via structlog) plus a redaction safety net.
+
+This is the one place logging is configured. The policy:
+
+- `get_logger(name)` returns an `unclogger.Unclogger` that emits JSON; use it for
+  *our* events (presentation/application, and domain where a rule needs a trace).
+  We deliberately do not reroute uvicorn/FastAPI's own access logs through here -
+  they already log requests; duplicating them adds noise, not signal.
+- `LogContext` is a typed mapping of the identifier fields that may be bound to the
+  context. The type checker rejects unknown keys, so a misspelled or contents-bearing
+  field is a type error rather than a silent leak.
+- `bind_context` / `clear_context` bind those identifiers (request id) at a
+  request entry point so every downstream log inherits them without
+  the call site repeating them.
+- A `sanitary.StructlogSanitizer` processor redacts credential-shaped keys and values
+  as a last-resort net. It is not a license to log secrets - it is the backstop for
+  when one slips through.
+
+`common` is layer-neutral, so `configure_logging` takes `debug` as an argument rather
+than importing `wiring.config`; the composition root passes `settings.DEBUG`.
+"""
+
+import logging
+import re
+from typing import Final, TypedDict, Unpack
+
+from sanitary import StructlogSanitizer
+import unclogger
+
+REDACTED: Final = "[REDACTED]"
+
+# Value patterns are the primary net: they catch a credential-shaped *value* even
+# under an innocuous field name. Matched against string values.
+_FORBIDDEN_VALUE_PATTERNS: Final = (
+    re.compile(r"\bBearer\s+[A-Za-z0-9._~+/=-]{8,}", re.IGNORECASE),  # bearer token
+    re.compile(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+"),  # JWT (PropelAuth)
+    re.compile(r"://[^/\s:@]+:[^/\s@]+@"),  # credentials in a URL/DSN userinfo
+)
+
+# Field names are the secondary fallback for a sensitive value logged under an obvious
+# name. Matched case-insensitively by exact name (not substring).
+_FORBIDDEN_KEYS: Final = frozenset(
+    {
+        "password",
+        "secret",
+        "token",
+        "access_token",
+        "session_token",
+        "pending_token",
+        "api_key",
+        "apikey",
+        "authorization",
+        "cookie",
+        "set-cookie",
+        "credentials",
+    }
+)
+
+# Noisy third-party loggers pinned to WARNING; raise only when actively debugging.
+# Uvicorn is deliberately absent - we leave its access/error logs as-is.
+_THIRD_PARTY_LOGGERS: Final = (
+    "sqlalchemy",
+    "sqlalchemy.engine",
+    "asyncio",
+)
+
+# `unknown_objects="deny"` is the runtime backstop: any object reaching the sanitizer
+# without a `__sanitary_context__` hook is masked wholesale rather than walked via
+# `vars()` (which would leak its every attribute). Scalars pass through untouched.
+_sanitizer: Final = StructlogSanitizer(
+    keys=_FORBIDDEN_KEYS,
+    patterns=_FORBIDDEN_VALUE_PATTERNS,
+    replacement=REDACTED,
+    message=REDACTED,
+    unknown_objects="deny",
+)
+
+
+class LogContext(TypedDict, total=False):
+    """Identifier fields that may be bound to the logging context.
+
+    Only identifiers belong here, never contents. Binding via `bind_context` is
+    type-checked against this mapping, so an unknown or misspelled key is a type
+    error rather than a silently-leaked field.
+
+    Deliberately minimal (request correlation only): richer per-request attributes
+    and spans are left to the planned OpenTelemetry instrumentation rather than
+    hand-bound here.
+    """
+
+    request_id: str
+    method: str
+    path: str
+
+
+def bind_context(**fields: Unpack[LogContext]) -> None:
+    """Bind identifier fields to the logging context for all downstream logs.
+
+    Call at a request entry point so every log line emitted while handling the
+    request inherits the identifiers without repeating them.
+    """
+    unclogger.context_bind(**fields)
+
+
+def clear_context(*keys: str) -> None:
+    """Clear bound context fields, or all of them if no keys are named."""
+    unclogger.context_clear(*keys)
+
+
+_configured = False
+
+
+def configure_logging(*, debug: bool = False) -> None:
+    """Configure process-wide structured logging. Idempotent; safe to call repeatedly.
+
+    Registers the redaction processor, pins noisy third-party loggers to WARNING, and
+    sets the root level from `debug`. Does not touch uvicorn's loggers.
+
+    Args:
+        debug: When true, emit DEBUG-level logs; otherwise INFO.
+    """
+    global _configured
+    if _configured:
+        return
+    unclogger.add_processors(_sanitizer)
+    for name in _THIRD_PARTY_LOGGERS:
+        logging.getLogger(name).setLevel(logging.WARNING)
+    unclogger.set_level(logging.DEBUG if debug else logging.INFO)
+    _configured = True
+
+
+def get_logger(name: str) -> unclogger.Unclogger:
+    """Return a structured logger.
+
+    Args:
+        name: The logger name; by convention `__name__`.
+
+    Returns:
+        An `Unclogger` usable like a standard library logger, emitting JSON.
+    """
+    return unclogger.get_logger(name)

schematalog/common/models.py

@@ -0,0 +1,12 @@
+from pydantic import BaseModel, ConfigDict
+
+
+class FrozenModel(BaseModel):
+    """Base class for immutable, frozen Pydantic models.
+
+    Layer-neutral implementation base. Domain layers wrap this with semantic
+    aliases (e.g. `ValueObject` in `domain/schema.py`); presentation DTOs can
+    inherit directly. Mirrors gapmap's `common.models.FrozenModel`.
+    """
+
+    model_config = ConfigDict(frozen=True)

schematalog/common/validation.py

@@ -0,0 +1,80 @@
+from jsonschema.exceptions import SchemaError
+from jsonschema.validators import validator_for
+
+# Accepted JSON Schema metaschemas, ordered oldest to newest. `infer_metaschema`
+# tries them newest-first (i.e. reversed), so the order matters.
+META_SCHEMAS: list[str] = [
+    "https://json-schema.org/draft-04/schema",
+    "https://json-schema.org/draft-06/schema",
+    "https://json-schema.org/draft-07/schema",
+    "https://json-schema.org/draft/2019-09/schema",
+    "https://json-schema.org/draft/2020-12/schema",
+]
+
+
+class IncompatibleSchemaError(Exception):
+    """Raised if a schema does not conform to any of the accepted meta schemas."""
+
+
+def preprocess_schema(schema: dict) -> dict:
+    schema = validate_metaschema(schema)
+    schema = convert_openapi_nullable(schema)
+    schema.pop("$id", None)
+    return schema
+
+
+def validate_metaschema(schema: dict) -> dict:
+    """Validate schema against declared metaschema.
+
+    If the schema does not declare the `$schema` keyword, infers the metaschema
+    and inserts the keyword.
+    """
+    if "$schema" in schema:
+        validator = validator_for(schema)
+        try:
+            validator.check_schema(schema)
+        except SchemaError:
+            return infer_metaschema(schema)
+        else:
+            return schema
+    else:
+        return infer_metaschema(schema)
+
+
+def infer_metaschema(schema: dict) -> dict:
+    """Infers and declares the correct metaschema for the schema.
+
+    The metaschema is inferred by checking the schema against each
+    of the supported metaschema URLs, starting with the latest.
+    The first valid metaschema is inserted as the value of the
+    `$schema` keyword.
+    """
+    for metaschema_url in reversed(META_SCHEMAS):
+        schema["$schema"] = metaschema_url
+        try:
+            validator_for(schema).check_schema(schema)
+        except SchemaError:
+            pass
+        else:
+            return schema
+    raise IncompatibleSchemaError
+
+
+def convert_openapi_nullable(schema: dict) -> dict:
+    """Converts the OpenAPI-style nullable to a null type.
+
+    Prior to version 3.1, OpenAPI did not recognise the `null` type,
+    instead using a separate keyword `nullable = true` to declare
+    values that can be `null`. This functions removes this keyword
+    and adds `"null"` to the list of declared types for that field.
+
+    If the schema contains `properties`, it is applied recursively.
+    """
+    if schema.pop("nullable", ...) is True:
+        if isinstance(schema["type"], str):
+            schema["type"] = [schema["type"]]
+        if "null" not in schema["type"]:
+            schema["type"].append("null")
+    for prop_name, prop in schema.get("properties", {}).items():
+        schema["properties"][prop_name] = convert_openapi_nullable(prop)
+    return schema

schematalog/domain/__init__.py

@@ -0,0 +1,8 @@
+"""The domain: what a schema is, and the contract a storage backend implements.
+
+`__version__` is this distribution's own. It is deliberately separate from the
+application's: the contract a backend codes against changes on its own schedule, and a
+backend pinning it should not be dragged along by a UI release.
+"""
+
+__version__ = "0.1.0"

schematalog/domain/exceptions.py

@@ -0,0 +1,13 @@
+class SchemaConflictError(Exception):
+    """Signifies that the schema version being stored already exists."""
+
+    def __init__(self, schema):
+        super().__init__(f"Conflict: Schema `{schema.name} v{schema.version}` already exists.")
+
+
+class UnknownSchemaError(Exception):
+    """Signifies that the schema version being requested does not exist."""
+
+    def __init__(self, schema_name, version=""):
+        version = f" v{version}" if version else ""
+        super().__init__(f"Unknown schema: `{schema_name}{version}`.")