contractforge-aws 0.1.0__py3-none-any.whl

contractforge_aws/__init__.py

@@ -0,0 +1,126 @@
+"""Public API for the ContractForge AWS adapter."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _version
+
+from contractforge_aws.adapter import AWSAdapter
+from contractforge_aws.api import (
+    plan_aws_contract,
+    render_aws_annotations_evidence_sql,
+    render_aws_annotations_plan,
+    render_aws_contract,
+    render_aws_deployment_manifest,
+    render_aws_glue_job_cloudformation,
+    render_aws_glue_job_definition,
+    render_aws_glue_job_iam_policy,
+    render_aws_glue_job_terraform,
+    render_aws_lake_formation_evidence_sql,
+    render_aws_lake_formation_plan,
+    render_aws_native_passthrough_plan,
+    render_aws_operations_evidence_sql,
+    render_aws_operations_json,
+    render_aws_operational_cost_query,
+    render_aws_quality_dqdl,
+)
+from contractforge_aws.runtime import (
+    AthenaQueryResult,
+    AthenaSqlRunner,
+    audit_evidence_tables,
+    apply_aws_annotations_contract,
+    apply_aws_annotations_plan,
+    apply_aws_lake_formation_contract,
+    apply_aws_lake_formation_plan,
+    create_or_update_schedule_payload,
+    create_or_update_state_machine_payload,
+    ensure_aws_evidence_tables,
+    deploy_aws_contract_to_glue,
+    get_aws_glue_job_run_status,
+    get_state_machine_execution_status,
+    publish_aws_contract_artifacts_to_s3,
+    record_aws_operations_contract,
+    reconcile_aws_glue_job_run_evidence,
+    register_aws_glue_job,
+    register_aws_glue_job_definition_payload,
+    render_aws_glue_job_run_evidence_sql,
+    start_aws_glue_job_run,
+    start_state_machine_execution,
+    wait_aws_glue_job_run,
+    wait_state_machine_execution,
+)
+from contractforge_aws.orchestration import (
+    render_eventbridge_scheduler_payload,
+    render_stepfunctions_state_machine_definition,
+    render_stepfunctions_state_machine_payload,
+)
+from contractforge_aws.capabilities import AWS_SUBTARGET_GLUE_ICEBERG, glue_iceberg_capabilities
+from contractforge_aws.cost import CostModel, render_operational_cost_query
+from contractforge_aws.environment import AWSEnvironment
+from contractforge_aws.lineage import (
+    build_openlineage_event,
+    openlineage_namespace,
+    render_openlineage_insert_sql,
+)
+from contractforge_aws.subtargets import list_aws_subtargets
+
+try:
+    __version__ = _version("contractforge-aws")
+except PackageNotFoundError:  # pragma: no cover - editable/source tree without installed metadata
+    __version__ = "0.1.0"
+
+__all__ = [
+    "AWSAdapter",
+    "AWSEnvironment",
+    "AWS_SUBTARGET_GLUE_ICEBERG",
+    "AthenaQueryResult",
+    "AthenaSqlRunner",
+    "CostModel",
+    "__version__",
+    "apply_aws_annotations_contract",
+    "apply_aws_annotations_plan",
+    "audit_evidence_tables",
+    "create_or_update_schedule_payload",
+    "create_or_update_state_machine_payload",
+    "ensure_aws_evidence_tables",
+    "deploy_aws_contract_to_glue",
+    "get_aws_glue_job_run_status",
+    "get_state_machine_execution_status",
+    "glue_iceberg_capabilities",
+    "build_openlineage_event",
+    "apply_aws_lake_formation_contract",
+    "apply_aws_lake_formation_plan",
+    "list_aws_subtargets",
+    "openlineage_namespace",
+    "plan_aws_contract",
+    "publish_aws_contract_artifacts_to_s3",
+    "record_aws_operations_contract",
+    "reconcile_aws_glue_job_run_evidence",
+    "register_aws_glue_job",
+    "register_aws_glue_job_definition_payload",
+    "render_aws_annotations_evidence_sql",
+    "render_aws_annotations_plan",
+    "render_aws_contract",
+    "render_aws_deployment_manifest",
+    "render_aws_glue_job_cloudformation",
+    "render_aws_glue_job_definition",
+    "render_aws_glue_job_iam_policy",
+    "render_aws_glue_job_terraform",
+    "render_aws_glue_job_run_evidence_sql",
+    "render_aws_lake_formation_evidence_sql",
+    "render_aws_lake_formation_plan",
+    "render_aws_native_passthrough_plan",
+    "render_aws_operations_evidence_sql",
+    "render_aws_operations_json",
+    "render_aws_operational_cost_query",
+    "render_aws_quality_dqdl",
+    "render_operational_cost_query",
+    "render_openlineage_insert_sql",
+    "render_eventbridge_scheduler_payload",
+    "render_stepfunctions_state_machine_definition",
+    "render_stepfunctions_state_machine_payload",
+    "start_aws_glue_job_run",
+    "start_state_machine_execution",
+    "wait_aws_glue_job_run",
+    "wait_state_machine_execution",
+]

contractforge_aws/adapter.py

@@ -0,0 +1,62 @@
+"""ContractForge adapter implementation for AWS targets."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from contractforge_core.adapters import RenderedArtifacts
+from contractforge_core.capabilities import PlatformCapabilities
+from contractforge_core.planner import ExecutionPlan, PlanningResult, plan_contract
+from contractforge_core.semantic import SemanticContract
+from contractforge_aws.capabilities import AWS_SUBTARGET_GLUE_ICEBERG, glue_iceberg_capabilities
+from contractforge_aws.contract_extensions import aws_extension_warnings
+from contractforge_aws.diagnostics import aws_planning_warnings, unsupported_source_blockers
+from contractforge_aws.environment import AWSEnvironment
+from contractforge_aws.rendering import render_aws_review_artifacts
+
+
+@dataclass(frozen=True)
+class AWSAdapter:
+    """AWS adapter for Glue/Iceberg planning and artifact rendering.
+
+    AWS SDK calls remain optional runtime helpers. The base adapter path stays
+    deterministic and SDK-free.
+    """
+
+    declared_capabilities: PlatformCapabilities
+    environment: AWSEnvironment = AWSEnvironment()
+    name: str = AWS_SUBTARGET_GLUE_ICEBERG
+
+    @classmethod
+    def glue_iceberg(cls, environment: dict[str, Any] | None = None) -> "AWSAdapter":
+        return cls(glue_iceberg_capabilities(), environment=AWSEnvironment.from_contract(environment))
+
+    def capabilities(self) -> PlatformCapabilities:
+        return self.declared_capabilities
+
+    def plan(self, contract: SemanticContract) -> PlanningResult:
+        source_blockers = unsupported_source_blockers(contract)
+        if source_blockers:
+            return PlanningResult(status="UNSUPPORTED", plan=None, blockers=source_blockers)
+
+        result = plan_contract(contract, self.capabilities())
+        aws_warnings = aws_planning_warnings(contract) + aws_extension_warnings(contract)
+        if not aws_warnings:
+            return result
+        warnings = result.warnings + aws_warnings
+        if result.status == "SUPPORTED":
+            return PlanningResult(status="SUPPORTED_WITH_WARNINGS", plan=result.plan, warnings=warnings)
+        return PlanningResult(status=result.status, plan=result.plan, blockers=result.blockers, warnings=warnings)
+
+    def render(self, plan: ExecutionPlan) -> RenderedArtifacts:
+        return render_aws_review_artifacts(plan=plan, planning=None, environment=self.environment)
+
+    def render_contract(self, contract: SemanticContract) -> RenderedArtifacts:
+        planning = self.plan(contract)
+        return render_aws_review_artifacts(
+            plan=planning.plan,
+            planning=planning,
+            contract=contract,
+            environment=self.environment,
+        )

contractforge_aws/annotations/__init__.py

@@ -0,0 +1,13 @@
+"""AWS Glue Catalog annotation planning."""
+
+from contractforge_aws.annotations.rendering import (
+    annotations_plan,
+    render_annotations_evidence_sql,
+    render_annotations_plan,
+)
+
+__all__ = [
+    "annotations_plan",
+    "render_annotations_evidence_sql",
+    "render_annotations_plan",
+]

contractforge_aws/annotations/api.py

@@ -0,0 +1,54 @@
+"""Runtime-facing AWS annotation helpers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from contractforge_core.contracts import semantic_contract_from_mapping
+from contractforge_aws.capabilities import AWS_SUBTARGET_GLUE_ICEBERG
+from contractforge_aws.annotations.rendering import render_annotations_plan
+from contractforge_aws.annotations.runtime import (
+    GlueCatalogAnnotationApplyResult,
+    apply_glue_catalog_annotations_plan,
+)
+from contractforge_aws.subtargets import validate_aws_subtarget
+
+
+def apply_aws_annotations_plan(
+    plan: str | dict[str, Any],
+    *,
+    glue_client: Any | None = None,
+    catalog_id: str | None = None,
+    skip_archive: bool = True,
+) -> GlueCatalogAnnotationApplyResult:
+    return apply_glue_catalog_annotations_plan(
+        plan,
+        glue_client=glue_client,
+        catalog_id=catalog_id,
+        skip_archive=skip_archive,
+    )
+
+
+def apply_aws_annotations_contract(
+    contract: dict[str, Any],
+    *,
+    subtarget: str = AWS_SUBTARGET_GLUE_ICEBERG,
+    glue_client: Any | None = None,
+    catalog_id: str | None = None,
+    skip_archive: bool = True,
+) -> GlueCatalogAnnotationApplyResult:
+    validate_aws_subtarget(subtarget)
+    semantic = semantic_contract_from_mapping(contract)
+    plan = render_annotations_plan(semantic)
+    if not plan:
+        target = semantic.target
+        return GlueCatalogAnnotationApplyResult(target.namespace or "default", target.name, "NOOP")
+    return apply_glue_catalog_annotations_plan(
+        plan,
+        glue_client=glue_client,
+        catalog_id=catalog_id,
+        skip_archive=skip_archive,
+    )
+
+
+__all__ = ["apply_aws_annotations_contract", "apply_aws_annotations_plan"]

contractforge_aws/annotations/rendering.py

@@ -0,0 +1,209 @@
+"""Render AWS Glue Catalog annotation plans and evidence."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import date, datetime
+from functools import singledispatch
+from typing import Any, Callable
+
+from contractforge_core.security import redact_value
+from contractforge_core.semantic import SemanticContract
+from contractforge_aws.evidence.ddl import evidence_table_names
+from contractforge_aws.rendering.names import glue_database_name, glue_table_name, iceberg_table_name
+
+
+@dataclass(frozen=True)
+class _TagExtractor:
+    prefix: str
+    extract: Callable[[dict[str, Any]], dict[str, str]]
+
+
+_TABLE_TAG_EXTRACTORS: tuple[_TagExtractor, ...] = (
+    _TagExtractor("", lambda data: _str_map(data.get("tags"))),
+    _TagExtractor("alias_", lambda data: _indexed_tags(data.get("aliases"))),
+    _TagExtractor("deprecated_", lambda data: _deprecated_tags(data.get("deprecated"))),
+)
+_COLUMN_TAG_EXTRACTORS: tuple[_TagExtractor, ...] = (
+    *_TABLE_TAG_EXTRACTORS,
+    _TagExtractor("pii_", lambda data: _pii_tags(data.get("pii"))),
+)
+
+
+def render_annotations_plan(contract: SemanticContract) -> str:
+    plan = annotations_plan(contract)
+    if not plan["changes"]:
+        return ""
+    return json.dumps(plan, indent=2, sort_keys=True)
+
+
+def annotations_plan(contract: SemanticContract) -> dict[str, Any]:
+    annotations = contract.governance.annotations if contract.governance else None
+    changes = _annotation_changes(annotations if isinstance(annotations, dict) else {})
+    return {
+        "target": iceberg_table_name(contract),
+        "resource": {"DatabaseName": glue_database_name(contract), "Name": glue_table_name(contract)},
+        "status": "PLANNED" if changes else "NOOP",
+        "apply_operation": "glue:UpdateTable",
+        "note": "Application requires reading the current Glue table definition and submitting a full TableInput.",
+        "changes": changes,
+    }
+
+
+def render_annotations_evidence_sql(contract: SemanticContract, *, database: str, run_id: str = "${run_id}", captured_at_utc: datetime | None = None) -> str:
+    rows = _annotation_evidence_rows(contract, run_id=run_id, captured_at_utc=captured_at_utc)
+    if not rows:
+        return "-- No annotation intent declared.\n"
+    table = evidence_table_names(database)["annotations"]
+    return "\n".join(_insert(table, row) for row in rows) + "\n"
+
+
+def _annotation_changes(annotations: dict[str, Any]) -> list[dict[str, Any]]:
+    table = _mapping(annotations.get("table"))
+    rows = _table_changes(table)
+    rows.extend(_column_changes(_mapping(annotations.get("columns"))))
+    return rows
+
+
+def _table_changes(table: dict[str, Any]) -> list[dict[str, Any]]:
+    rows = []
+    description = table.get("description")
+    if description:
+        rows.append(_change("table", "description", None, "Description", str(description)))
+    for key, value in _tags(table, _TABLE_TAG_EXTRACTORS).items():
+        rows.append(_change("table", "parameter", None, f"Parameters.{key}", value, key=key))
+    return rows
+
+
+def _column_changes(columns: dict[str, Any]) -> list[dict[str, Any]]:
+    rows = []
+    for column, config in columns.items():
+        data = _mapping(config)
+        description = data.get("description")
+        if description:
+            rows.append(
+                _change("column", "description", str(column), "StorageDescriptor.Columns[].Comment", str(description))
+            )
+        for key, value in _tags(data, _COLUMN_TAG_EXTRACTORS).items():
+            rows.append(_change("column", "parameter", str(column), "StorageDescriptor.Columns[].Parameters." + key, value, key=key))
+    return rows
+
+
+def _change(scope: str, kind: str, column: str | None, glue_path: str, value: str, *, key: str | None = None) -> dict[str, Any]:
+    return {
+        "annotation_scope": scope,
+        "annotation_type": kind,
+        "column_name": column,
+        "key": key or kind,
+        "value": value,
+        "glue_path": glue_path,
+        "status": "PLANNED",
+    }
+
+
+def _annotation_evidence_rows(contract: SemanticContract, *, run_id: str, captured_at_utc: datetime | None) -> list[dict[str, Any]]:
+    captured = captured_at_utc or datetime(1970, 1, 1, 0, 0, 0)
+    target = iceberg_table_name(contract)
+    return [
+        {
+            "run_id": run_id,
+            "target_table": target,
+            "annotation_scope": change["annotation_scope"],
+            "annotation_type": change["annotation_type"],
+            "column_name": change["column_name"],
+            "key": change["key"],
+            "value": change["value"],
+            "status": change["status"],
+            "applied_sql": "glue:UpdateTable",
+            "annotation_ts_utc": captured,
+            "annotation_date": captured.date(),
+            "framework_version": "contractforge-aws",
+            "ctrl_schema_version": 1,
+        }
+        for change in annotations_plan(contract)["changes"]
+    ]
+
+
+def _tags(data: dict[str, Any], extractors: tuple[_TagExtractor, ...]) -> dict[str, str]:
+    tags: dict[str, str] = {}
+    for extractor in extractors:
+        tags.update({f"{extractor.prefix}{key}": value for key, value in extractor.extract(data).items()})
+    return tags
+
+
+def _mapping(value: object) -> dict[str, Any]:
+    return dict(value) if isinstance(value, dict) else {}
+
+
+def _str_map(value: object) -> dict[str, str]:
+    return {str(key): _tag_value(item) for key, item in _mapping(value).items()}
+
+
+def _indexed_tags(value: object) -> dict[str, str]:
+    return {str(idx): item for idx, item in enumerate(_as_list(value), start=1)}
+
+
+def _deprecated_tags(value: object) -> dict[str, str]:
+    deprecated = _mapping(value)
+    if not deprecated:
+        return {}
+    return {"enabled": "true", **{key: str(deprecated[key]) for key in ("since", "replacement", "removal_date") if deprecated.get(key)}}
+
+
+def _pii_tags(value: object) -> dict[str, str]:
+    pii = _mapping(value)
+    if not pii:
+        return {}
+    return {
+        "enabled": _tag_value(pii.get("enabled", True)),
+        "type": str(pii.get("type", "unknown")),
+        "sensitivity": str(pii.get("sensitivity", "internal")),
+    }
+
+
+def _as_list(value: object) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [item.strip() for item in value.split("|") if item.strip()]
+    return [str(item).strip() for item in value if str(item).strip()]  # type: ignore[union-attr]
+
+
+def _tag_value(value: object) -> str:
+    return str(value).lower() if isinstance(value, bool) else str(value)
+
+
+def _insert(table: str, columns: dict[str, Any]) -> str:
+    filtered = {key: value for key, value in columns.items() if value is not None}
+    names = ", ".join(_quote_identifier(name) for name in filtered)
+    values = ", ".join(_literal(value) for value in filtered.values())
+    return f"INSERT INTO {table} ({names}) VALUES ({values});"
+
+
+@singledispatch
+def _literal(value: Any) -> str:
+    return _string(str(redact_value(value)))
+
+
+@_literal.register(int)
+def _literal_int(value: int) -> str:
+    return str(value)
+
+
+@_literal.register(datetime)
+def _literal_datetime(value: datetime) -> str:
+    return f"TIMESTAMP {_string(value.strftime('%Y-%m-%d %H:%M:%S'))}"
+
+
+@_literal.register(date)
+def _literal_date(value: date) -> str:
+    return f"DATE {_string(value.strftime('%Y-%m-%d'))}"
+
+
+def _string(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _quote_identifier(value: str) -> str:
+    return f"`{str(value).replace('`', '``')}`"

contractforge_aws/annotations/runtime.py

@@ -0,0 +1,168 @@
+"""Optional AWS Glue Catalog annotation apply helpers."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from contractforge_aws.runtime.dependencies import require_boto3
+
+
+@dataclass(frozen=True)
+class GlueCatalogAnnotationApplyResult:
+    database: str
+    table: str
+    status: str
+    applied: int = 0
+    skipped: int = 0
+
+
+@dataclass(frozen=True)
+class AnnotationChangeHandler:
+    scope: str
+    kind: str
+    apply: Callable[[dict[str, Any], dict[str, Any]], None]
+
+    @property
+    def key(self) -> tuple[str, str]:
+        return self.scope, self.kind
+
+
+def apply_glue_catalog_annotations_plan(
+    plan: str | dict[str, Any],
+    *,
+    glue_client: Any | None = None,
+    catalog_id: str | None = None,
+    skip_archive: bool = True,
+) -> GlueCatalogAnnotationApplyResult:
+    payload = _plan_payload(plan)
+    resource = _resource(payload)
+    changes = _changes(payload)
+    if not changes:
+        return GlueCatalogAnnotationApplyResult(resource["DatabaseName"], resource["Name"], "NOOP")
+
+    client = glue_client or require_boto3().client("glue")
+    get_args = _catalog_args(catalog_id, DatabaseName=resource["DatabaseName"], Name=resource["Name"])
+    table = client.get_table(**get_args)["Table"]
+    table_input = _table_input(table)
+    applied = _apply_changes(table_input, changes)
+    update_args = _catalog_args(
+        catalog_id,
+        DatabaseName=resource["DatabaseName"],
+        Name=resource["Name"],
+        TableInput=table_input,
+        SkipArchive=skip_archive,
+    )
+    client.update_table(**update_args)
+    return GlueCatalogAnnotationApplyResult(resource["DatabaseName"], resource["Name"], "SUCCESS", applied=applied)
+
+
+def _plan_payload(plan: str | dict[str, Any]) -> dict[str, Any]:
+    if isinstance(plan, str):
+        loaded = json.loads(plan)
+        if not isinstance(loaded, dict):
+            raise ValueError("Glue Catalog annotation plan JSON must decode to an object")
+        return loaded
+    return dict(plan)
+
+
+def _resource(plan: dict[str, Any]) -> dict[str, str]:
+    resource = plan.get("resource")
+    if not isinstance(resource, dict):
+        raise ValueError("Glue Catalog annotation plan requires resource")
+    database = str(resource.get("DatabaseName") or "").strip()
+    table = str(resource.get("Name") or "").strip()
+    if not database or not table:
+        raise ValueError("Glue Catalog annotation plan requires resource.DatabaseName and resource.Name")
+    return {"DatabaseName": database, "Name": table}
+
+
+def _changes(plan: dict[str, Any]) -> list[dict[str, Any]]:
+    changes = plan.get("changes")
+    if not isinstance(changes, list):
+        return []
+    return [dict(item) for item in changes if isinstance(item, dict)]
+
+
+def _table_input(table: dict[str, Any]) -> dict[str, Any]:
+    return {key: table[key] for key in _TABLE_INPUT_KEYS if key in table and table[key] is not None}
+
+
+def _apply_changes(table_input: dict[str, Any], changes: list[dict[str, Any]]) -> int:
+    applied = 0
+    for change in changes:
+        handler = _CHANGE_HANDLERS.get((str(change.get("annotation_scope")), str(change.get("annotation_type"))))
+        if handler is not None:
+            handler.apply(table_input, change)
+            applied += 1
+    return applied
+
+
+def _apply_table_description(table_input: dict[str, Any], change: dict[str, Any]) -> None:
+    table_input["Description"] = str(change.get("value") or "")
+
+
+def _apply_table_parameter(table_input: dict[str, Any], change: dict[str, Any]) -> None:
+    _parameters(table_input)[str(change["key"])] = str(change.get("value") or "")
+
+
+def _apply_column_description(table_input: dict[str, Any], change: dict[str, Any]) -> None:
+    _column(table_input, str(change["column_name"]))["Comment"] = str(change.get("value") or "")
+
+
+def _apply_column_parameter(table_input: dict[str, Any], change: dict[str, Any]) -> None:
+    column = _column(table_input, str(change["column_name"]))
+    column.setdefault("Parameters", {})[str(change["key"])] = str(change.get("value") or "")
+
+
+def _parameters(table_input: dict[str, Any]) -> dict[str, str]:
+    value = table_input.setdefault("Parameters", {})
+    if not isinstance(value, dict):
+        raise ValueError("Glue table Parameters must be an object")
+    return value
+
+
+def _column(table_input: dict[str, Any], name: str) -> dict[str, Any]:
+    columns = table_input.get("StorageDescriptor", {}).get("Columns")
+    if not isinstance(columns, list):
+        raise ValueError("Glue table StorageDescriptor.Columns is required for column annotations")
+    for column in columns:
+        if isinstance(column, dict) and column.get("Name") == name:
+            return column
+    raise ValueError(f"Glue table does not contain column {name!r}")
+
+
+def _catalog_args(catalog_id: str | None, **kwargs: Any) -> dict[str, Any]:
+    payload = dict(kwargs)
+    if catalog_id:
+        payload["CatalogId"] = catalog_id
+    return payload
+
+
+_TABLE_INPUT_KEYS = (
+    "Name",
+    "Description",
+    "Owner",
+    "LastAccessTime",
+    "LastAnalyzedTime",
+    "Retention",
+    "StorageDescriptor",
+    "PartitionKeys",
+    "ViewOriginalText",
+    "ViewExpandedText",
+    "TableType",
+    "Parameters",
+    "TargetTable",
+)
+
+
+_CHANGE_HANDLERS = {
+    handler.key: handler
+    for handler in (
+        AnnotationChangeHandler("table", "description", _apply_table_description),
+        AnnotationChangeHandler("table", "parameter", _apply_table_parameter),
+        AnnotationChangeHandler("column", "description", _apply_column_description),
+        AnnotationChangeHandler("column", "parameter", _apply_column_parameter),
+    )
+}