contractforge-core 0.1.0__py3-none-any.whl

contractforge_core/cli_init.py

@@ -0,0 +1,117 @@
+"""Starter contract generation for the core ContractForge CLI."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from contractforge_core.cli_io import yaml_dump
+
+
+def init_contract(args: Any) -> int:
+    output = _base_output(args.output)
+    files = {
+        output.with_suffix(".ingestion.yaml"): _init_ingestion(args),
+        output.with_suffix(".annotations.yaml"): _init_annotations(args),
+        output.with_suffix(".operations.yaml"): _init_operations(args),
+        output.with_suffix(".access.yaml"): _init_access(args),
+        output.with_suffix(".environment.yaml"): _init_environment(args),
+    }
+    written = []
+    for path, payload in files.items():
+        _write_mapping(path, payload, force=args.force)
+        written.append(str(path))
+    return _print({"status": "SUCCESS", "written": written}, args.indent)
+
+
+def _init_ingestion(args: Any) -> dict[str, Any]:
+    merge_keys = _csv(args.merge_keys)
+    hash_keys = _csv(args.hash_keys) or merge_keys
+    if args.mode in {"scd1_upsert", "scd2_historical", "snapshot_soft_delete"} and not merge_keys:
+        raise ValueError(f"--merge-keys is required for mode={args.mode}")
+    if args.mode == "scd1_hash_diff" and not hash_keys:
+        raise ValueError("--hash-keys or --merge-keys is required for mode=scd1_hash_diff")
+    contract: dict[str, Any] = {
+        "source": {"type": "table", "table": args.source},
+        "target": {"catalog": args.catalog, "schema": args.target_schema or args.layer, "table": args.target_table},
+        "layer": args.layer,
+        "mode": args.mode,
+        "schema_policy": args.schema_policy,
+    }
+    if merge_keys:
+        contract["merge_keys"] = merge_keys
+        contract["quality_rules"] = {"not_null": merge_keys}
+    if args.mode == "scd1_hash_diff" and hash_keys:
+        contract["hash_keys"] = hash_keys
+    watermarks = _csv(args.watermark_columns)
+    if watermarks:
+        contract["watermark_columns"] = watermarks
+    return contract
+
+
+def _init_annotations(args: Any) -> dict[str, Any]:
+    return {
+        "target": _target(args),
+        "table": {"description": args.description or f"TODO: describe {args.target_table}", "tags": {"domain": args.domain or "TODO"}},
+        "columns": {},
+    }
+
+
+def _init_operations(args: Any) -> dict[str, Any]:
+    return {
+        "target": _target(args),
+        "ownership": {
+            "business_owner": args.owner or "TODO",
+            "technical_owner": args.technical_owner or "data-platform",
+            "support_group": args.support_group or "data-platform",
+        },
+        "operations": {
+            "criticality": args.criticality,
+            "expected_frequency": args.expected_frequency,
+            "freshness_sla_minutes": args.freshness_sla_minutes,
+            "alert_on_failure": True,
+            "alert_on_quality_fail": True,
+            "runbook_url": args.runbook_url or "TODO",
+        },
+    }
+
+
+def _init_access(args: Any) -> dict[str, Any]:
+    return {
+        "target": _target(args),
+        "access_policy": {"mode": "validate_only", "on_drift": "warn", "revoke_unmanaged": False},
+        "grants": [{"principal": args.access_principal or "data-engineers", "privileges": ["SELECT"]}],
+    }
+
+
+def _init_environment(args: Any) -> dict[str, Any]:
+    return {"name": "dev", "adapter": args.adapter, "evidence": {"catalog": args.catalog, "schema": "ops"}}
+
+
+def _target(args: Any) -> dict[str, str]:
+    return {"catalog": args.catalog, "schema": args.target_schema or args.layer, "table": args.target_table}
+
+
+def _write_mapping(path: Path, payload: dict[str, Any], *, force: bool) -> None:
+    if path.exists() and not force:
+        raise FileExistsError(f"{path} already exists; use --force to overwrite it")
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(yaml_dump(payload), encoding="utf-8")
+
+
+def _print(payload: object, indent: int) -> int:
+    print(json.dumps(payload, indent=indent, sort_keys=True, default=str))
+    return 0
+
+
+def _csv(value: str | None) -> list[str]:
+    return [item.strip() for item in str(value or "").split(",") if item.strip()]
+
+
+def _base_output(path: Path) -> Path:
+    name = path.name
+    for suffix in (".ingestion.yaml", ".ingestion.yml", ".ingestion.json"):
+        if name.endswith(suffix):
+            return path.with_name(name[: -len(suffix)])
+    return path

contractforge_core/cli_io.py

@@ -0,0 +1,22 @@
+"""Small JSON/YAML helpers for core CLI modules."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def yaml_load(text: str) -> Any:
+    try:
+        import yaml  # type: ignore
+    except Exception as exc:  # pragma: no cover
+        raise RuntimeError("YAML support requires PyYAML; use JSON files or install PyYAML") from exc
+    return yaml.safe_load(text)
+
+
+def yaml_dump(payload: dict[str, Any]) -> str:
+    try:
+        import yaml  # type: ignore
+    except Exception:  # pragma: no cover
+        return json.dumps(payload, indent=2, sort_keys=False) + "\n"
+    return yaml.safe_dump(payload, allow_unicode=True, sort_keys=False)

contractforge_core/config.py

@@ -0,0 +1,164 @@
+"""Platform-neutral constants and shared type aliases."""
+
+from __future__ import annotations
+
+from typing import Any, Literal, Union
+
+FRAMEWORK_VERSION = "1.0.0"
+CTRL_SCHEMA_VERSION = 1
+
+Layer = str
+
+WriteMode = Literal[
+    "scd0_append",
+    "scd0_overwrite",
+    "scd1_upsert",
+    "scd1_hash_diff",
+    "scd2_historical",
+    "snapshot_soft_delete",
+]
+WriteEngine = Literal[
+    "auto",
+    "core_managed",
+    "native_merge",
+    "native_cdc",
+]
+WriteEngineFallbackPolicy = Literal["fail", "fallback_to_core", "preview_only"]
+SchemaPolicy = Literal["permissive", "additive_only", "strict"]
+QualityFailAction = Literal["fail", "warn", "quarantine"]
+QualityRuleSeverity = Literal["warn", "quarantine", "abort"]
+SCD2LateArrivingPolicy = Literal["apply", "ignore", "reject"]
+GovernanceFailurePolicy = Literal["fail", "warn", "ignore"]
+AccessMode = Literal["apply", "validate_only", "ignore"]
+AccessDriftPolicy = Literal["fail", "warn", "reconcile"]
+IdempotencyPolicy = Literal["always_run", "skip_if_success", "fail_if_success", "rerun_if_failed"]
+Source = Union[str, dict[str, Any]]
+
+CUSTOM_WRITE_MODE_PREFIX = "custom:"
+
+VALID_WRITE_MODES = {
+    "scd0_append",
+    "scd0_overwrite",
+    "scd1_upsert",
+    "scd1_hash_diff",
+    "scd2_historical",
+    "snapshot_soft_delete",
+}
+VALID_WRITE_ENGINES = {"auto", "core_managed", "native_merge", "native_cdc"}
+VALID_WRITE_ENGINE_FALLBACK_POLICIES = {"fail", "fallback_to_core", "preview_only"}
+VALID_SCHEMA_POLICIES = {"permissive", "additive_only", "strict"}
+VALID_QUALITY_FAIL_ACTIONS = {"fail", "warn", "quarantine"}
+VALID_QUALITY_RULE_SEVERITIES = {"warn", "quarantine", "abort"}
+VALID_SCD2_LATE_ARRIVING_POLICIES = {"apply", "ignore", "reject"}
+VALID_GOVERNANCE_FAILURE_POLICIES = {"fail", "warn", "ignore"}
+VALID_ACCESS_MODES = {"apply", "validate_only", "ignore"}
+VALID_ACCESS_DRIFT_POLICIES = {"fail", "warn", "reconcile"}
+VALID_CRITICALITY_LEVELS = {"low", "medium", "high", "critical"}
+VALID_EXPECTED_FREQUENCIES = {"hourly", "daily", "weekly", "monthly", "ad_hoc"}
+VALID_SENSITIVITY_LEVELS = {"public", "internal", "restricted", "confidential"}
+VALID_PII_TYPES = {
+    "address",
+    "bank_account",
+    "birth_date",
+    "credit_card",
+    "device_id",
+    "document",
+    "email",
+    "financial",
+    "health",
+    "ip_address",
+    "name",
+    "national_id",
+    "other",
+    "phone",
+    "ssn",
+    "tax_id",
+    "unknown",
+}
+VALID_ACCESS_PRIVILEGES = {
+    "ALL PRIVILEGES",
+    "APPLY TAG",
+    "CREATE",
+    "CREATE FUNCTION",
+    "CREATE MODEL",
+    "CREATE TABLE",
+    "CREATE VOLUME",
+    "EXECUTE",
+    "MANAGE",
+    "MODIFY",
+    "READ FILES",
+    "READ VOLUME",
+    "REFRESH",
+    "SELECT",
+    "USAGE",
+    "WRITE FILES",
+    "WRITE VOLUME",
+}
+VALID_IDEMPOTENCY_POLICIES = {"always_run", "skip_if_success", "fail_if_success", "rerun_if_failed"}
+VALID_EXPLAIN_FORMATS = {"simple", "extended", "codegen", "cost", "formatted"}
+
+VALID_SOURCE_TYPES = {"connector"}
+VALID_SOURCE_CONNECTORS = {
+    "adls",
+    "avro",
+    "azure_blob",
+    "bigquery_jdbc",
+    "blob",
+    "csv",
+    "db2",
+    "delta",
+    "delta_share",
+    "delta_table",
+    "eventhubs_available_now",
+    "eventhubs_bounded",
+    "gcs",
+    "http_csv",
+    "http_file",
+    "http_json",
+    "http_text",
+    "iceberg_table",
+    "incremental_files",
+    "jdbc",
+    "json",
+    "kafka_available_now",
+    "kafka_bounded",
+    "mariadb",
+    "mysql",
+    "native_passthrough",
+    "object_storage",
+    "oracle",
+    "orc",
+    "parquet",
+    "postgres",
+    "redshift",
+    "rest_api",
+    "s3",
+    "snowflake_jdbc",
+    "sql",
+    "sqlserver",
+    "table",
+    "text",
+    "view",
+    "xml",
+}
+VALID_OBJECT_STORAGE_PROVIDERS = {"adls", "azure_blob", "gcs", "s3"}
+VALID_FILE_CONNECTOR_FORMATS = {"avro", "csv", "delta", "json", "jsonl", "ndjson", "orc", "parquet", "text", "xml"}
+VALID_HTTP_FILE_FORMATS = {"csv", "json", "jsonl", "ndjson", "text"}
+VALID_SOURCE_TRIGGERS = {"available_now"}
+ARRAY_MODES = {"explode", "explode_outer", "first", "keep", "size", "to_json"}
+MAX_INLINE_ACCEPTED_VALUES = 1000
+CONTROL_COLUMNS = {
+    "ingestion_ts_utc",
+    "__run_id",
+    "row_hash",
+    "valid_from",
+    "valid_to",
+    "is_current",
+    "is_active",
+    "deleted_at",
+    "changed_columns",
+}
+
+
+def is_valid_write_mode(value: str) -> bool:
+    return value in VALID_WRITE_MODES or value.startswith(CUSTOM_WRITE_MODE_PREFIX)

contractforge_core/connectors/__init__.py

@@ -0,0 +1,140 @@
+"""Platform-neutral connector helpers."""
+
+from contractforge_core.connectors.http_files import (
+    HTTP_FILE_TYPES,
+    cleanup_http_file_downloads,
+    download_http_file,
+    http_file_format,
+    http_file_headers,
+    http_file_params,
+    http_file_reader_options,
+    http_file_url,
+    is_http_file_source,
+    read_http_file_payload,
+)
+from contractforge_core.connectors.catalog import (
+    CATALOG_SOURCE_TYPES,
+    LogicalTableReference,
+    TableRefResolver,
+    catalog_source_query,
+    catalog_source_table_or_path,
+    has_table_reference_placeholders,
+    is_catalog_source,
+    parse_logical_table_reference,
+    render_table_reference_placeholders,
+    source_logical_table_reference,
+)
+from contractforge_core.connectors.files import (
+    FILE_SOURCE_TYPES,
+    OBJECT_STORAGE_TYPES,
+    file_reader_options,
+    file_source_format,
+    is_file_source,
+    normalize_file_format,
+    object_storage_provider,
+)
+from contractforge_core.connectors.streams import (
+    AVAILABLE_NOW_STREAM_TYPES,
+    BOUNDED_STREAM_TYPES,
+    STREAM_SOURCE_TYPES,
+    eventhubs_bounded_options,
+    is_available_now_stream_source,
+    is_bounded_stream_source,
+    is_eventhubs_stream_source,
+    is_kafka_stream_source,
+    kafka_bounded_options,
+    stream_source_format,
+)
+from contractforge_core.connectors.sharing import delta_share_options, is_delta_share_source
+from contractforge_core.connectors.databases import JDBC_CONNECTORS, jdbc_common_options, validate_jdbc_source
+from contractforge_core.connectors.native_passthrough import (
+    is_native_passthrough_source,
+    native_passthrough_descriptor,
+    redact_secret_fields,
+)
+from contractforge_core.connectors.metadata import (
+    diagnose_source_connectors,
+    list_source_connector_details,
+    source_capabilities,
+    source_connector_details,
+    source_metadata_from_contract,
+    source_metadata_from_mapping,
+    source_provider,
+)
+from contractforge_core.connectors.databases import (
+    generate_rds_iam_auth_token,
+    infer_aws_region_from_rds_host,
+    parse_jdbc_host_port,
+    rds_iam_review_options,
+)
+from contractforge_core.connectors.api.rest import (
+    REST_API_CONNECTORS,
+    is_rest_api_connector,
+    read_rest_api_records,
+    rest_api_descriptor,
+    rest_request_headers,
+)
+
+__all__ = [
+    "HTTP_FILE_TYPES",
+    "cleanup_http_file_downloads",
+    "download_http_file",
+    "read_rest_api_records",
+    "rest_request_headers",
+    "JDBC_CONNECTORS",
+    "CATALOG_SOURCE_TYPES",
+    "LogicalTableReference",
+    "TableRefResolver",
+    "AVAILABLE_NOW_STREAM_TYPES",
+    "BOUNDED_STREAM_TYPES",
+    "STREAM_SOURCE_TYPES",
+    "FILE_SOURCE_TYPES",
+    "OBJECT_STORAGE_TYPES",
+    "REST_API_CONNECTORS",
+    "catalog_source_query",
+    "catalog_source_table_or_path",
+    "delta_share_options",
+    "diagnose_source_connectors",
+    "eventhubs_bounded_options",
+    "file_reader_options",
+    "file_source_format",
+    "generate_rds_iam_auth_token",
+    "has_table_reference_placeholders",
+    "http_file_format",
+    "http_file_headers",
+    "http_file_params",
+    "http_file_reader_options",
+    "http_file_url",
+    "read_http_file_payload",
+    "infer_aws_region_from_rds_host",
+    "is_http_file_source",
+    "is_available_now_stream_source",
+    "is_bounded_stream_source",
+    "is_catalog_source",
+    "is_delta_share_source",
+    "is_eventhubs_stream_source",
+    "is_file_source",
+    "is_kafka_stream_source",
+    "stream_source_format",
+    "jdbc_common_options",
+    "list_source_connector_details",
+    "is_native_passthrough_source",
+    "is_rest_api_connector",
+    "kafka_bounded_options",
+    "native_passthrough_descriptor",
+    "parse_jdbc_host_port",
+    "parse_logical_table_reference",
+    "redact_secret_fields",
+    "render_table_reference_placeholders",
+    "rest_api_descriptor",
+    "normalize_file_format",
+    "object_storage_provider",
+    "rds_iam_review_options",
+    "source_capabilities",
+    "source_connector_details",
+    "source_metadata_from_contract",
+    "source_metadata_from_mapping",
+    "source_provider",
+    "source_logical_table_reference",
+    "validate_jdbc_source",
+]

contractforge_core/connectors/api/__init__.py

@@ -0,0 +1,17 @@
+"""Facade for the API connector family."""
+
+from contractforge_core.connectors.api.rest import (
+    REST_API_CONNECTORS,
+    is_rest_api_connector,
+    read_rest_api_records,
+    rest_api_descriptor,
+    rest_request_headers,
+)
+
+__all__ = [
+    "REST_API_CONNECTORS",
+    "is_rest_api_connector",
+    "read_rest_api_records",
+    "rest_api_descriptor",
+    "rest_request_headers",
+]

contractforge_core/connectors/api/rest/__init__.py

@@ -0,0 +1,51 @@
+"""Facade for the bounded REST API connector."""
+
+from contractforge_core.connectors.api.rest.auth import rest_request_headers
+from contractforge_core.connectors.api.rest.pagination import (
+    json_path,
+    link_header_next,
+    max_pages_for_source,
+    next_url,
+    page_urls,
+    pagination_type,
+    url_with_params,
+)
+from contractforge_core.connectors.api.rest.reader import read_rest_api_records
+from contractforge_core.connectors.api.rest.retry import (
+    RETRYABLE_HTTP_STATUS,
+    is_retryable_http_error,
+    is_retryable_network_error,
+    sleep_retry_backoff,
+)
+from contractforge_core.connectors.api.rest.safety import (
+    ALLOW_PRIVATE_FLAG,
+    ALLOWED_SCHEMES,
+    validate_http_target,
+)
+from contractforge_core.connectors.api.rest.source import (
+    REST_API_CONNECTORS,
+    is_rest_api_connector,
+    rest_api_descriptor,
+)
+
+__all__ = [
+    "ALLOWED_SCHEMES",
+    "ALLOW_PRIVATE_FLAG",
+    "REST_API_CONNECTORS",
+    "RETRYABLE_HTTP_STATUS",
+    "is_retryable_http_error",
+    "is_retryable_network_error",
+    "is_rest_api_connector",
+    "json_path",
+    "link_header_next",
+    "max_pages_for_source",
+    "next_url",
+    "page_urls",
+    "pagination_type",
+    "read_rest_api_records",
+    "rest_api_descriptor",
+    "rest_request_headers",
+    "sleep_retry_backoff",
+    "url_with_params",
+    "validate_http_target",
+]

contractforge_core/connectors/api/rest/auth.py

@@ -0,0 +1,104 @@
+"""Auth header construction for the platform-neutral bounded REST client.
+
+This module operates on already-resolved values. Adapters are responsible for
+resolving any secret placeholders (e.g. via their platform secret store) before
+calling these helpers, so the core never depends on a platform secret backend.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import urllib.parse
+import urllib.request
+from typing import Any
+
+from contractforge_core.connectors.api.rest.safety import validate_http_target
+from contractforge_core.connectors.api.rest.transport import open_request as _open_request
+
+
+def rest_request_headers(
+    source: dict[str, Any],
+    incremental: dict[str, Any] | None = None,
+    watermark: str | None = None,
+) -> dict[str, str]:
+    request = _dict(source.get("request"))
+    headers = _string_dict(request.get("headers"))
+    auth = _dict(source.get("auth"))
+    auth_type = str(auth.get("type") or "").strip().lower()
+    if auth_type == "bearer_token":
+        token = auth.get("token")
+        if not token:
+            raise ValueError("REST API bearer_token auth requires auth.token")
+        headers["Authorization"] = f"Bearer {token}"
+    if auth_type == "api_key":
+        api_key = auth.get("value")
+        if not api_key:
+            raise ValueError("REST API api_key auth requires auth.value")
+        headers[str(auth.get("header") or "X-Api-Key")] = str(api_key)
+    if auth_type == "basic":
+        username = auth.get("username")
+        password = auth.get("password")
+        if not username or not password:
+            raise ValueError("REST API basic auth requires auth.username and auth.password")
+        raw = f"{username}:{password}".encode("utf-8")
+        headers["Authorization"] = "Basic " + base64.b64encode(raw).decode("ascii")
+    if auth_type == "oauth_client_credentials":
+        headers["Authorization"] = f"Bearer {_oauth_client_credentials_token(auth, source)}"
+    if auth and auth_type not in {"bearer_token", "api_key", "basic", "oauth_client_credentials"}:
+        raise ValueError(f"REST API auth.type={auth_type!r} is not supported")
+    incremental = incremental or {}
+    if watermark and incremental.get("watermark_header"):
+        headers[str(incremental["watermark_header"])] = watermark
+    return headers
+
+
+def _oauth_client_credentials_token(auth: dict[str, Any], source: dict[str, Any]) -> str:
+    token_url = str(auth.get("token_url") or "").strip()
+    tenant_id = str(auth.get("tenant_id") or "").strip()
+    if not token_url and tenant_id:
+        token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token"
+    client_id = auth.get("client_id")
+    client_secret = auth.get("client_secret")
+    if not token_url or not client_id or not client_secret:
+        raise ValueError(
+            "REST API OAuth client credentials auth requires auth.token_url, auth.client_id and auth.client_secret; "
+            "auth.tenant_id can replace auth.token_url for Microsoft Entra ID."
+        )
+    fields = {
+        "grant_type": "client_credentials",
+        "client_id": str(client_id),
+        "client_secret": str(client_secret),
+    }
+    scope = auth.get("scope")
+    scopes = auth.get("scopes")
+    if scope:
+        fields["scope"] = str(scope)
+    elif isinstance(scopes, (list, tuple, set)):
+        fields["scope"] = " ".join(str(item) for item in scopes if str(item).strip())
+    elif scopes:
+        fields["scope"] = str(scopes)
+    timeout = int(_dict(source.get("limits")).get("timeout_seconds", 60))
+    validate_http_target(token_url, context="REST API OAuth token URL")
+    request = urllib.request.Request(
+        token_url,
+        method="POST",
+        data=urllib.parse.urlencode(fields).encode("utf-8"),
+        headers={"Content-Type": "application/x-www-form-urlencoded"},
+    )
+    with _open_request(request, timeout=timeout) as response:
+        raw = response.read()
+        encoding = response.headers.get_content_charset() if hasattr(response.headers, "get_content_charset") else None
+        payload = json.loads(raw.decode(encoding or "utf-8")) if raw else {}
+    token = payload.get("access_token")
+    if not token:
+        raise ValueError("OAuth response did not return access_token")
+    return str(token)
+
+
+def _dict(value: object) -> dict[str, Any]:
+    return dict(value) if isinstance(value, dict) else {}
+
+
+def _string_dict(value: object) -> dict[str, str]:
+    return {str(key): str(item) for key, item in _dict(value).items()}