contractforge-core 0.1.0__py3-none-any.whl

contractforge_core/__init__.py

@@ -0,0 +1,135 @@
+"""Platform-neutral semantic core for contract-first ingestion."""
+
+from importlib.metadata import PackageNotFoundError, version as _pkg_version
+
+try:
+    __version__ = _pkg_version("contractforge-core")
+except PackageNotFoundError:  # pragma: no cover - editable installs without metadata
+    __version__ = "0.0.0+unknown"
+
+from contractforge_core.capabilities import CapabilityEvidence, NativeCapability, PlatformCapabilities, capability
+from contractforge_core.connectors import (
+    diagnose_source_connectors,
+    list_source_connector_details,
+    source_connector_details,
+)
+from contractforge_core.contracts import (
+    AccessContractModel,
+    AccessGrantContractModel,
+    AnnotationsContractModel,
+    ColumnAnnotationsContractModel,
+    ColumnMaskContractModel,
+    ContractBundle,
+    ConnectorSourceContract,
+    DeduplicateContractModel,
+    DeprecatedContractModel,
+    ExecutionCatchupContractModel,
+    ExecutionContractModel,
+    ExecutionWindowContractModel,
+    GenericSourceContract,
+    OperationsContractModel,
+    PiiContractModel,
+    QualityExpressionContractModel,
+    QualityRulesContractModel,
+    RowFilterContractModel,
+    ShapeArrayContractModel,
+    ShapeColumnContractModel,
+    ShapeContractModel,
+    ShapeFlattenContractModel,
+    ShapeJsonContractModel,
+    ShapeZipArraysContractModel,
+    StandardizeColumnContractModel,
+    TableAnnotationsContractModel,
+    TransformContractModel,
+    contract_model_schemas,
+    load_contract_bundle,
+    semantic_contract_from_mapping,
+    target_full_table_name,
+    target_schema_name,
+    validate_plan_shape,
+    yaml_schema,
+)
+from contractforge_core.errors import ContractForgeExecutionError, raise_for_failure_result
+from contractforge_core.execution import ExecutionWindow
+from contractforge_core.naming import DerivedNames, NamingConfig, derive_names, normalize_identifier, normalize_slug
+from contractforge_core.planner.result import PlanningResult, PlanningStatus
+from contractforge_core.portability import classify_write_mode
+from contractforge_core.project import (
+    ProjectScheduleIntent,
+    StandardCron,
+    adapter_scheduling,
+    parse_standard_cron,
+    project_schedule_intent,
+    quartz_cron_expression,
+)
+from contractforge_core.security import redact_secrets, redact_text, redact_value
+from contractforge_core.semantic.models import SemanticContract
+from contractforge_core.watermark import decode_watermark_value, encode_watermark_values, extract_watermark_field_value
+
+__all__ = [
+    "AccessContractModel",
+    "AccessGrantContractModel",
+    "AnnotationsContractModel",
+    "PlatformCapabilities",
+    "PlanningResult",
+    "PlanningStatus",
+    "ProjectScheduleIntent",
+    "StandardCron",
+    "SemanticContract",
+    "ContractForgeExecutionError",
+    "ContractBundle",
+    "CapabilityEvidence",
+    "ColumnAnnotationsContractModel",
+    "ColumnMaskContractModel",
+    "ConnectorSourceContract",
+    "DerivedNames",
+    "DeduplicateContractModel",
+    "DeprecatedContractModel",
+    "ExecutionCatchupContractModel",
+    "ExecutionContractModel",
+    "ExecutionWindow",
+    "ExecutionWindowContractModel",
+    "GenericSourceContract",
+    "NamingConfig",
+    "NativeCapability",
+    "OperationsContractModel",
+    "PiiContractModel",
+    "QualityExpressionContractModel",
+    "QualityRulesContractModel",
+    "RowFilterContractModel",
+    "ShapeArrayContractModel",
+    "ShapeColumnContractModel",
+    "ShapeContractModel",
+    "ShapeFlattenContractModel",
+    "ShapeJsonContractModel",
+    "ShapeZipArraysContractModel",
+    "StandardizeColumnContractModel",
+    "TableAnnotationsContractModel",
+    "TransformContractModel",
+    "capability",
+    "adapter_scheduling",
+    "classify_write_mode",
+    "contract_model_schemas",
+    "diagnose_source_connectors",
+    "derive_names",
+    "list_source_connector_details",
+    "load_contract_bundle",
+    "normalize_identifier",
+    "normalize_slug",
+    "project_schedule_intent",
+    "parse_standard_cron",
+    "quartz_cron_expression",
+    "redact_text",
+    "redact_secrets",
+    "redact_value",
+    "decode_watermark_value",
+    "encode_watermark_values",
+    "extract_watermark_field_value",
+    "raise_for_failure_result",
+    "semantic_contract_from_mapping",
+    "source_connector_details",
+    "target_full_table_name",
+    "target_schema_name",
+    "validate_plan_shape",
+    "yaml_schema",
+]

contractforge_core/adapters/__init__.py

@@ -0,0 +1,15 @@
+from contractforge_core.adapters.base import (
+    CapabilitiesAdapter,
+    PlatformAdapter,
+    RenderedArtifacts,
+    append_only_adapter,
+    full_feature_adapter,
+)
+
+__all__ = [
+    "CapabilitiesAdapter",
+    "PlatformAdapter",
+    "RenderedArtifacts",
+    "append_only_adapter",
+    "full_feature_adapter",
+]

contractforge_core/adapters/base/__init__.py

@@ -0,0 +1,10 @@
+from contractforge_core.adapters.base.generic import CapabilitiesAdapter, append_only_adapter, full_feature_adapter
+from contractforge_core.adapters.base.protocol import PlatformAdapter, RenderedArtifacts
+
+__all__ = [
+    "CapabilitiesAdapter",
+    "PlatformAdapter",
+    "RenderedArtifacts",
+    "append_only_adapter",
+    "full_feature_adapter",
+]

contractforge_core/adapters/base/generic.py

@@ -0,0 +1,62 @@
+"""Generic in-memory adapters useful for tests and dry planning."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from contractforge_core.adapters.base.protocol import RenderedArtifacts
+from contractforge_core.capabilities.models import PlatformCapabilities
+from contractforge_core.planner import ExecutionPlan, PlanningResult, plan_contract
+from contractforge_core.semantic.models import SemanticContract
+
+
+@dataclass(frozen=True)
+class CapabilitiesAdapter:
+    """Small adapter backed only by a capability declaration."""
+
+    name: str
+    declared_capabilities: PlatformCapabilities
+
+    def capabilities(self) -> PlatformCapabilities:
+        return self.declared_capabilities
+
+    def plan(self, contract: SemanticContract) -> PlanningResult:
+        return plan_contract(contract, self.declared_capabilities)
+
+    def render(self, plan: ExecutionPlan) -> RenderedArtifacts:
+        lines = [
+            f"# Execution plan for {plan.platform}",
+            "",
+            "| Step | Intent |",
+            "| --- | --- |",
+        ]
+        lines.extend(f"| {step.name} | {step.intent} |" for step in plan.steps)
+        return RenderedArtifacts(artifacts={"review.md": "\n".join(lines) + "\n"})
+
+
+def full_feature_adapter() -> CapabilitiesAdapter:
+    capabilities = PlatformCapabilities(
+        platform="full-feature-generic",
+        supports_append=True,
+        supports_overwrite=True,
+        supports_merge=True,
+        supports_hash_diff=True,
+        supports_scd2=True,
+        supports_snapshot_soft_delete=True,
+        supports_schema_evolution=True,
+        supports_row_filters=True,
+        supports_column_masks=True,
+        supports_available_now_streaming=True,
+        evidence_stores=("audit_tables",),
+    )
+    return CapabilitiesAdapter(name=capabilities.platform, declared_capabilities=capabilities)
+
+
+def append_only_adapter() -> CapabilitiesAdapter:
+    capabilities = PlatformCapabilities(
+        platform="append-only-generic",
+        supports_append=True,
+        evidence_stores=("audit_files",),
+    )
+    return CapabilitiesAdapter(name=capabilities.platform, declared_capabilities=capabilities)
+

contractforge_core/adapters/base/protocol.py

@@ -0,0 +1,29 @@
+"""Base adapter protocol for platform-specific implementations."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+from contractforge_core.capabilities.models import PlatformCapabilities
+from contractforge_core.planner.result import ExecutionPlan, PlanningResult
+from contractforge_core.semantic.models import SemanticContract
+
+
+@dataclass(frozen=True)
+class RenderedArtifacts:
+    artifacts: dict[str, str]
+
+
+class PlatformAdapter(Protocol):
+    name: str
+
+    def capabilities(self) -> PlatformCapabilities:
+        ...
+
+    def plan(self, contract: SemanticContract) -> PlanningResult:
+        ...
+
+    def render(self, plan: ExecutionPlan) -> RenderedArtifacts:
+        ...
+

contractforge_core/capabilities/__init__.py

@@ -0,0 +1,4 @@
+from contractforge_core.capabilities.models import PlatformCapabilities
+from contractforge_core.capabilities.native import CapabilityEvidence, CapabilityStatus, NativeCapability, capability
+
+__all__ = ["CapabilityEvidence", "CapabilityStatus", "NativeCapability", "PlatformCapabilities", "capability"]

contractforge_core/capabilities/models.py

@@ -0,0 +1,29 @@
+"""Platform capability declarations consumed by the semantic planner."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class PlatformCapabilities:
+    platform: str
+    supports_append: bool = False
+    supports_overwrite: bool = False
+    supports_merge: bool = False
+    supports_hash_diff: bool = False
+    supports_scd2: bool = False
+    supports_snapshot_soft_delete: bool = False
+    supports_schema_evolution: bool = False
+    supports_row_filters: bool = False
+    supports_column_masks: bool = False
+    supports_available_now_streaming: bool = False
+    supports_required_columns_quality: bool = True
+    supports_unique_key_quality: bool = True
+    supports_max_null_ratio_quality: bool = True
+    supports_expression_quality: bool = False
+    supports_shape: bool = False
+    supports_transform: bool = False
+    evidence_stores: tuple[str, ...] = ()
+    review_required_semantics: tuple[str, ...] = ()
+    supported_custom_write_modes: tuple[str, ...] = ()

contractforge_core/capabilities/native.py

@@ -0,0 +1,53 @@
+"""Platform-neutral native capability evidence models."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Literal
+
+CapabilityStatus = Literal["supported", "unsupported", "unknown"]
+
+
+@dataclass(frozen=True)
+class CapabilityEvidence:
+    source: str
+    message: str
+    value: str | None = None
+
+    def as_dict(self) -> dict[str, str]:
+        payload = {"source": self.source, "message": self.message, "value": self.value}
+        return {key: value for key, value in payload.items() if value is not None}
+
+
+@dataclass(frozen=True)
+class NativeCapability:
+    name: str
+    status: CapabilityStatus
+    reason: str
+    evidence: tuple[CapabilityEvidence, ...] = ()
+    requires: tuple[str, ...] = ()
+
+    @property
+    def supported(self) -> bool:
+        return self.status == "supported"
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "status": self.status,
+            "supported": self.supported,
+            "reason": self.reason,
+            "requires": list(self.requires),
+            "evidence": [item.as_dict() for item in self.evidence],
+        }
+
+
+def capability(
+    name: str,
+    status: CapabilityStatus,
+    reason: str,
+    *,
+    evidence: tuple[CapabilityEvidence, ...] = (),
+    requires: tuple[str, ...] = (),
+) -> NativeCapability:
+    return NativeCapability(name=name, status=status, reason=reason, evidence=evidence, requires=requires)

contractforge_core/cli.py

@@ -0,0 +1,96 @@
+"""Core ContractForge CLI for platform-neutral contract utilities."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+from typing import Any
+
+from contractforge_core.cli_connectors import handle_connector_command
+from contractforge_core.cli_contracts import handle_contract_command
+from contractforge_core.config import VALID_SCHEMA_POLICIES, VALID_WRITE_MODES
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="contractforge")
+    sub = parser.add_subparsers(dest="command", required=True)
+    _add_contract_parsers(sub)
+    _add_connector_parser(sub)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return _dispatch(args)
+    except Exception as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 1
+
+
+def _dispatch(args: argparse.Namespace) -> int:
+    contract_result = handle_contract_command(args)
+    if contract_result is not None:
+        return contract_result
+    connector_result = handle_connector_command(args)
+    if connector_result is not None:
+        return connector_result
+    raise ValueError(f"unsupported command: {args.command}")
+
+
+def _add_contract_parsers(subparsers: Any) -> None:
+    validate = subparsers.add_parser("validate", help="Validate contract files without executing a platform")
+    validate.add_argument("paths", nargs="+", type=Path)
+    validate.add_argument("--indent", type=int, default=2)
+    validate_bundle = subparsers.add_parser("validate-bundle", help="Validate split contract bundles")
+    validate_bundle.add_argument("paths", nargs="+", type=Path)
+    validate_bundle.add_argument("--indent", type=int, default=2)
+    validate_project = subparsers.add_parser("validate-project", help="Discover and validate contracts recursively")
+    validate_project.add_argument("paths", nargs="+", type=Path)
+    validate_project.add_argument("--indent", type=int, default=2)
+    schema = subparsers.add_parser("schema", help="Print generated core contract JSON Schemas")
+    schema.add_argument("--indent", type=int, default=2)
+    init = subparsers.add_parser("init", help="Generate a starter split ContractForge contract")
+    init.add_argument("--output", required=True, type=Path)
+    init.add_argument("--source", required=True)
+    init.add_argument("--target-table", required=True)
+    init.add_argument("--catalog", default="main")
+    init.add_argument("--layer", default="bronze")
+    init.add_argument("--target-schema")
+    init.add_argument("--adapter", default="generic")
+    init.add_argument("--mode", default="scd0_append", choices=sorted(VALID_WRITE_MODES))
+    init.add_argument("--schema-policy", default="additive_only", choices=sorted(VALID_SCHEMA_POLICIES))
+    init.add_argument("--merge-keys")
+    init.add_argument("--hash-keys")
+    init.add_argument("--watermark-columns")
+    init.add_argument("--description")
+    init.add_argument("--domain")
+    init.add_argument("--owner")
+    init.add_argument("--technical-owner")
+    init.add_argument("--support-group")
+    init.add_argument("--criticality", default="medium")
+    init.add_argument("--expected-frequency", default="daily")
+    init.add_argument("--freshness-sla-minutes", type=int, default=1440)
+    init.add_argument("--runbook-url")
+    init.add_argument("--access-principal")
+    init.add_argument("--force", action="store_true")
+    init.add_argument("--indent", type=int, default=2)
+
+
+def _add_connector_parser(subparsers: Any) -> None:
+    parser = subparsers.add_parser("connectors", help="List or inspect source connector portability")
+    sub = parser.add_subparsers(dest="connector_command", required=True)
+    list_parser = sub.add_parser("list", help="List known portable source types")
+    list_parser.add_argument("--indent", type=int, default=2)
+    show = sub.add_parser("show", help="Show connector details")
+    show.add_argument("names", nargs="+")
+    show.add_argument("--indent", type=int, default=2)
+    doctor = sub.add_parser("doctor", help="Diagnose connector portability without opening external connections")
+    doctor.add_argument("names", nargs="*")
+    doctor.add_argument("--indent", type=int, default=2)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

contractforge_core/cli_connectors.py

@@ -0,0 +1,29 @@
+"""Connector catalog commands for the core ContractForge CLI."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from contractforge_core.connectors import (
+    diagnose_source_connectors,
+    list_source_connector_details,
+    source_connector_details,
+)
+
+
+def handle_connector_command(args: Any) -> int | None:
+    if args.command != "connectors":
+        return None
+    if args.connector_command == "list":
+        return _print(list_source_connector_details(), args.indent)
+    if args.connector_command == "show":
+        return _print([source_connector_details(name) for name in args.names], args.indent)
+    if args.connector_command == "doctor":
+        return _print({"status": "SUCCESS", "items": diagnose_source_connectors(args.names)}, args.indent)
+    raise ValueError(f"unsupported connectors command: {args.connector_command}")
+
+
+def _print(payload: object, indent: int) -> int:
+    print(json.dumps(payload, indent=indent, sort_keys=True, default=str))
+    return 0

contractforge_core/cli_contracts.py

@@ -0,0 +1,178 @@
+"""Contract-oriented core CLI commands."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from contractforge_core.cli_init import init_contract
+from contractforge_core.cli_io import yaml_load
+from contractforge_core.contracts import (
+    contract_model_schemas,
+    load_contract_bundle,
+    semantic_contract_from_mapping,
+    validate_source_semantics,
+)
+
+CONTRACT_SUFFIXES = {".json", ".yaml", ".yml"}
+SPLIT_MARKERS = (".annotations.", ".operations.", ".access.", ".environment.")
+
+
+def handle_contract_command(args: Any) -> int | None:
+    if args.command == "validate":
+        return validate_contracts(args.paths, indent=args.indent)
+    if args.command == "validate-bundle":
+        return validate_bundles(args.paths, indent=args.indent)
+    if args.command == "validate-project":
+        return validate_project(args.paths, indent=args.indent)
+    if args.command == "schema":
+        return _print(contract_model_schemas(), args.indent)
+    if args.command == "init":
+        return init_contract(args)
+    return None
+
+
+def validate_contracts(paths: list[Path], *, indent: int) -> int:
+    items = []
+    for path in paths:
+        try:
+            contract = _load_mapping(path)
+            semantic = semantic_contract_from_mapping(contract)
+            _validate_source_if_mapping(contract.get("source"))
+            items.append({"path": str(path), "kind": "contract", "status": "SUCCESS", "target": semantic.target.name, "mode": semantic.write.mode})
+        except Exception as exc:
+            items.append({"path": str(path), "kind": "contract", "status": "FAILED", "error": str(exc)})
+    return _report(items, indent)
+
+
+def validate_bundles(paths: list[Path], *, indent: int) -> int:
+    items = []
+    for path in paths:
+        try:
+            bundle = _load_bundle(path)
+            _validate_source_if_mapping(bundle.contract.get("source"))
+            items.append(
+                {
+                    "path": str(path),
+                    "kind": "bundle",
+                    "status": "SUCCESS",
+                    "target": bundle.semantic.target.name,
+                    "mode": bundle.semantic.write.mode,
+                    "split_files": dict(bundle.metadata.get("paths", {})),
+                }
+            )
+        except Exception as exc:
+            items.append({"path": str(path), "kind": "bundle", "status": "FAILED", "error": str(exc)})
+    return _report(items, indent)
+
+
+def validate_project(paths: list[Path], *, indent: int) -> int:
+    items = []
+    for root in paths:
+        discovered = _discover_contracts(root)
+        if not discovered:
+            items.append({"path": str(root), "kind": "project", "status": "FAILED", "error": "no contracts found"})
+            continue
+        for kind, path in discovered:
+            items.extend(_validate_bundle_for_project(path) if kind == "bundle" else _validate_single_for_project(path))
+    return _report(items, indent)
+
+
+def _validate_bundle_for_project(path: Path) -> list[dict[str, Any]]:
+    try:
+        bundle = _load_bundle(path)
+        _validate_source_if_mapping(bundle.contract.get("source"))
+        return [{"path": str(path), "kind": "bundle", "status": "SUCCESS", "target": bundle.semantic.target.name, "mode": bundle.semantic.write.mode}]
+    except Exception as exc:
+        return [{"path": str(path), "kind": "bundle", "status": "FAILED", "error": str(exc)}]
+
+
+def _validate_single_for_project(path: Path) -> list[dict[str, Any]]:
+    try:
+        contract = _load_mapping(path)
+        semantic = semantic_contract_from_mapping(contract)
+        _validate_source_if_mapping(contract.get("source"))
+        return [{"path": str(path), "kind": "contract", "status": "SUCCESS", "target": semantic.target.name}]
+    except Exception as exc:
+        return [{"path": str(path), "kind": "contract", "status": "FAILED", "error": str(exc)}]
+
+
+def _load_bundle(path: Path):
+    return load_contract_bundle(_bundle_base(path))
+
+
+def _discover_contracts(root: Path) -> list[tuple[str, Path]]:
+    if root.is_file():
+        if _is_split_ingestion(root):
+            return [("bundle", root)]
+        return [("contract", root)] if _is_standalone_contract(root) else []
+    found = []
+    for path in sorted(root.rglob("*")):
+        if _is_discovery_ignored(path):
+            continue
+        if _is_split_ingestion(path):
+            found.append(("bundle", path))
+        elif _is_standalone_contract(path):
+            found.append(("contract", path))
+    return found
+
+
+def _report(items: list[dict[str, Any]], indent: int) -> int:
+    failed = [item for item in items if item["status"] == "FAILED"]
+    payload = {"status": "FAILED" if failed else "SUCCESS", "total": len(items), "succeeded": len(items) - len(failed), "failed": len(failed), "items": items}
+    _print(payload, indent)
+    return 1 if failed else 0
+
+
+def _load_mapping(path: Path) -> dict[str, Any]:
+    text = path.read_text(encoding="utf-8")
+    payload = json.loads(text) if path.suffix.lower() == ".json" else yaml_load(text)
+    if not isinstance(payload, dict):
+        raise ValueError(f"{path} must contain an object")
+    return payload
+
+
+def _validate_source_if_mapping(source: object) -> None:
+    if isinstance(source, dict):
+        validate_source_semantics(source)
+
+
+def _print(payload: object, indent: int) -> int:
+    print(json.dumps(payload, indent=indent, sort_keys=True, default=str))
+    return 0
+
+
+def _base_output(path: Path) -> Path:
+    name = path.name
+    for suffix in (".ingestion.yaml", ".ingestion.yml", ".ingestion.json"):
+        if name.endswith(suffix):
+            return path.with_name(name[: -len(suffix)])
+    return path
+
+
+def _bundle_base(path: Path) -> Path:
+    return _base_output(path) if _is_split_ingestion(path) else path
+
+
+def _is_contract_file(path: Path) -> bool:
+    return path.is_file() and path.suffix.lower() in CONTRACT_SUFFIXES
+
+
+def _is_split_ingestion(path: Path) -> bool:
+    return _is_contract_file(path) and ".ingestion." in path.name
+
+
+def _is_standalone_contract(path: Path) -> bool:
+    if not _is_contract_file(path) or ".ingestion." in path.name or any(marker in path.name for marker in SPLIT_MARKERS):
+        return False
+    try:
+        payload = _load_mapping(path)
+    except Exception:
+        return False
+    return isinstance(payload.get("source"), dict) and isinstance(payload.get("target"), dict)
+
+
+def _is_discovery_ignored(path: Path) -> bool:
+    ignored_dirs = {".databricks", ".git", ".venv", "__pycache__", "node_modules", "build", "dist"}
+    return any(part in ignored_dirs for part in path.parts)