kiri-cli 1.0.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of kiri-cli might be problematic. Click here for more details.
- kiri_adapters/__init__.py +193 -0
- kiri_adapters/backends/__init__.py +0 -0
- kiri_adapters/backends/databricks.manifest.yml +55 -0
- kiri_adapters/backends/databricks.py +2980 -0
- kiri_adapters/backends/fabric_dw.manifest.yml +17 -0
- kiri_adapters/backends/fabric_dw.py +307 -0
- kiri_adapters/backends/fabric_lakehouse.manifest.yml +19 -0
- kiri_adapters/backends/fabric_lakehouse.py +365 -0
- kiri_adapters/backends/iceberg.manifest.yml +17 -0
- kiri_adapters/backends/iceberg.py +451 -0
- kiri_adapters/backends/localduckdb.manifest.yml +43 -0
- kiri_adapters/backends/localduckdb.py +1295 -0
- kiri_adapters/backends/mssql.manifest.yml +17 -0
- kiri_adapters/backends/mssql.py +366 -0
- kiri_adapters/backends/postgres.manifest.yml +17 -0
- kiri_adapters/backends/postgres.py +434 -0
- kiri_adapters/backends/trino.manifest.yml +17 -0
- kiri_adapters/backends/trino.py +542 -0
- kiri_adapters/base.py +892 -0
- kiri_adapters/doctor_checks.py +81 -0
- kiri_adapters/dv_planner.py +1205 -0
- kiri_adapters/dv_planner_bridge.py +265 -0
- kiri_adapters/dv_planner_pit.py +265 -0
- kiri_adapters/dv_planner_ref.py +403 -0
- kiri_adapters/dv_planner_sat_variants.py +668 -0
- kiri_adapters/framework.py +112 -0
- kiri_adapters/incident/__init__.py +30 -0
- kiri_adapters/incident/_http.py +121 -0
- kiri_adapters/incident/base.py +89 -0
- kiri_adapters/incident/jira.py +365 -0
- kiri_adapters/incident/servicenow.py +314 -0
- kiri_adapters/incident/zendesk.py +285 -0
- kiri_adapters/manifest.py +466 -0
- kiri_adapters/orchestration/__init__.py +30 -0
- kiri_adapters/orchestration/airflow.py +117 -0
- kiri_adapters/orchestration/artifact_emitter.py +99 -0
- kiri_adapters/orchestration/autoloader_emitter.py +141 -0
- kiri_adapters/orchestration/azure_data_factory.py +137 -0
- kiri_adapters/orchestration/base.py +626 -0
- kiri_adapters/orchestration/databricks_workflows.py +1768 -0
- kiri_adapters/orchestration/k8s_cron.py +555 -0
- kiri_adapters/orchestration/lifecycle.py +222 -0
- kiri_adapters/orchestration/pipeline_compat.py +145 -0
- kiri_adapters/orchestration/runtime_python.py +188 -0
- kiri_adapters/paths.py +38 -0
- kiri_adapters/protocols/__init__.py +64 -0
- kiri_adapters/protocols/ai_service/__init__.py +7 -0
- kiri_adapters/protocols/ai_service/conformance.py +120 -0
- kiri_adapters/protocols/base.py +220 -0
- kiri_adapters/protocols/catalog/__init__.py +7 -0
- kiri_adapters/protocols/catalog/conformance.py +138 -0
- kiri_adapters/protocols/framework/__init__.py +7 -0
- kiri_adapters/protocols/framework/conformance.py +98 -0
- kiri_adapters/protocols/incident/__init__.py +7 -0
- kiri_adapters/protocols/incident/conformance.py +64 -0
- kiri_adapters/protocols/orchestration/__init__.py +7 -0
- kiri_adapters/protocols/orchestration/conformance.py +84 -0
- kiri_adapters/protocols/platform/__init__.py +7 -0
- kiri_adapters/protocols/platform/conformance.py +120 -0
- kiri_adapters/protocols/sql_dialect/__init__.py +7 -0
- kiri_adapters/protocols/sql_dialect/conformance.py +91 -0
- kiri_adapters/protocols/vault/__init__.py +7 -0
- kiri_adapters/protocols/vault/conformance.py +65 -0
- kiri_adapters/py.typed +0 -0
- kiri_adapters/registry.py +104 -0
- kiri_adapters/semantic/__init__.py +60 -0
- kiri_adapters/semantic/apply.py +274 -0
- kiri_adapters/semantic/base.py +530 -0
- kiri_adapters/semantic/cube.py +280 -0
- kiri_adapters/semantic/dbt_semantic.py +92 -0
- kiri_adapters/semantic/metric_views.py +249 -0
- kiri_adapters/semantic/metricflow.py +345 -0
- kiri_adapters/semantic/snowflake_views.py +75 -0
- kiri_adapters/sql_dialect.py +1592 -0
- kiri_adapters/tsql_to_spark.py +653 -0
- kiri_ai_gateway/__init__.py +147 -0
- kiri_ai_gateway/audit.py +334 -0
- kiri_ai_gateway/batch.py +293 -0
- kiri_ai_gateway/config.py +165 -0
- kiri_ai_gateway/gateway.py +642 -0
- kiri_ai_gateway/placeholder_guard.py +145 -0
- kiri_ai_gateway/prices/__init__.py +77 -0
- kiri_ai_gateway/prices/prices.yaml +64 -0
- kiri_ai_gateway/prompts/__init__.py +54 -0
- kiri_ai_gateway/prompts/airbyte_source_setup.yaml +212 -0
- kiri_ai_gateway/prompts/bdv_translation.yaml +90 -0
- kiri_ai_gateway/prompts/contract_draft_mvp.yaml +117 -0
- kiri_ai_gateway/prompts/describe_table.yaml +19 -0
- kiri_ai_gateway/prompts/design_gold_from_question.yaml +130 -0
- kiri_ai_gateway/prompts/detect_pii.yaml +59 -0
- kiri_ai_gateway/prompts/discover_classify_column.yaml +69 -0
- kiri_ai_gateway/prompts/discover_describe_table.yaml +61 -0
- kiri_ai_gateway/prompts/dv_quality_summary.yaml +36 -0
- kiri_ai_gateway/prompts/incident_assist.yaml +58 -0
- kiri_ai_gateway/prompts/infer_source_schema.yaml +31 -0
- kiri_ai_gateway/prompts/ingest_backend_chooser.yaml +116 -0
- kiri_ai_gateway/prompts/interactive_source_edit.yaml +94 -0
- kiri_ai_gateway/prompts/map_needed_entity_to_source.yaml +72 -0
- kiri_ai_gateway/prompts/meet_kiri.yaml +127 -0
- kiri_ai_gateway/prompts/native_source_setup.yaml +211 -0
- kiri_ai_gateway/prompts/source_grouping_reason.yaml +29 -0
- kiri_ai_gateway/prompts/suggest_attribute_classification.yaml +42 -0
- kiri_ai_gateway/prompts/suggest_data_tests.yaml +71 -0
- kiri_ai_gateway/prompts/suggest_gold_dimension.yaml +108 -0
- kiri_ai_gateway/prompts/suggest_gold_fact.yaml +107 -0
- kiri_ai_gateway/prompts/suggest_gold_model.yaml +70 -0
- kiri_ai_gateway/prompts/suggest_semantic_intent.yaml +91 -0
- kiri_ai_gateway/prompts/suggest_silver_model.yaml +140 -0
- kiri_ai_gateway/providers/__init__.py +7 -0
- kiri_ai_gateway/providers/anthropic_provider.py +26 -0
- kiri_ai_gateway/providers/gateway_provider.py +188 -0
- kiri_ai_gateway/providers/local_provider.py +177 -0
- kiri_ai_gateway/py.typed +0 -0
- kiri_ai_gateway/service.py +283 -0
- kiri_ai_gateway/transport.py +103 -0
- kiri_ai_gateway/transports/__init__.py +12 -0
- kiri_ai_gateway/transports/anthropic_transport.py +73 -0
- kiri_ai_gateway/transports/ollama_transport.py +82 -0
- kiri_ai_gateway/transports/openai_compatible_transport.py +125 -0
- kiri_ai_gateway/workspace_policy.py +52 -0
- kiri_catalog/__init__.py +199 -0
- kiri_catalog/annotation_plan.py +183 -0
- kiri_catalog/appearances.py +155 -0
- kiri_catalog/backends/__init__.py +79 -0
- kiri_catalog/backends/base.py +167 -0
- kiri_catalog/backends/memory.py +241 -0
- kiri_catalog/backends/postgres.py +497 -0
- kiri_catalog/backends/redacting.py +195 -0
- kiri_catalog/backends/sqlite.py +480 -0
- kiri_catalog/context.py +94 -0
- kiri_catalog/federation/__init__.py +114 -0
- kiri_catalog/federation/api.py +200 -0
- kiri_catalog/federation/cache.py +149 -0
- kiri_catalog/federation/emit.py +209 -0
- kiri_catalog/federation/fs_resolver.py +295 -0
- kiri_catalog/federation/http_resolver.py +486 -0
- kiri_catalog/federation/metrics.py +123 -0
- kiri_catalog/federation/release.py +342 -0
- kiri_catalog/federation/resolver.py +332 -0
- kiri_catalog/federation/schema.py +460 -0
- kiri_catalog/importer.py +668 -0
- kiri_catalog/lineage_drift.py +280 -0
- kiri_catalog/lineage_graph.py +176 -0
- kiri_catalog/lineage_graph_explain.py +96 -0
- kiri_catalog/lineage_graph_mermaid.py +230 -0
- kiri_catalog/observed_uc_lineage.py +216 -0
- kiri_catalog/pii_detect.py +176 -0
- kiri_catalog/push.py +122 -0
- kiri_catalog/py.typed +0 -0
- kiri_catalog/redact.py +101 -0
- kiri_catalog/review_actions.py +119 -0
- kiri_catalog/schema/__init__.py +70 -0
- kiri_catalog/schema/asset.py +142 -0
- kiri_catalog/schema/column.py +247 -0
- kiri_catalog/schema/enums.py +195 -0
- kiri_catalog/schema/job.py +51 -0
- kiri_catalog/schema/lineage.py +33 -0
- kiri_catalog/schema/policy.py +69 -0
- kiri_catalog/schema/primitives.py +90 -0
- kiri_catalog/schema/review_state.py +111 -0
- kiri_catalog/schema/urn.py +45 -0
- kiri_catalog/schema_discovery.py +160 -0
- kiri_catalog/sinks/__init__.py +64 -0
- kiri_catalog/sinks/base.py +178 -0
- kiri_catalog/sinks/horizon.py +195 -0
- kiri_catalog/sinks/purview.py +216 -0
- kiri_catalog/sinks/unity.py +352 -0
- kiri_catalog/sinks/unity_executor.py +543 -0
- kiri_catalog/sources/__init__.py +42 -0
- kiri_catalog/sources/base.py +104 -0
- kiri_catalog/sources/unity.py +465 -0
- kiri_catalog/steward_writeback.py +158 -0
- kiri_catalog/tag_sql.py +99 -0
- kiri_cli/LICENSE +105 -0
- kiri_cli/__init__.py +1 -0
- kiri_cli/_apply_cli.py +63 -0
- kiri_cli/_apply_reporter.py +38 -0
- kiri_cli/_closed_menu.py +36 -0
- kiri_cli/_fixture_option.py +18 -0
- kiri_cli/_layer_detect.py +158 -0
- kiri_cli/_project_option.py +79 -0
- kiri_cli/adapter_cmd.py +192 -0
- kiri_cli/ai_batch_cmd.py +258 -0
- kiri_cli/ai_cmd.py +352 -0
- kiri_cli/approve_cmd.py +659 -0
- kiri_cli/attr_cmd.py +575 -0
- kiri_cli/audit_cmd.py +348 -0
- kiri_cli/audit_correlate_cmd.py +648 -0
- kiri_cli/audit_join.py +437 -0
- kiri_cli/audit_join_cmd.py +160 -0
- kiri_cli/auto_cmd.py +97 -0
- kiri_cli/azure_cmd.py +243 -0
- kiri_cli/catalog.py +1293 -0
- kiri_cli/cli_reference_live.py +147 -0
- kiri_cli/compile_cmd.py +493 -0
- kiri_cli/compliance_cmd.py +221 -0
- kiri_cli/contract_approval_cmd.py +94 -0
- kiri_cli/contract_cmd.py +64 -0
- kiri_cli/contract_codeowners_cmd.py +162 -0
- kiri_cli/contract_diff.py +111 -0
- kiri_cli/contract_emit.py +90 -0
- kiri_cli/contract_export.py +192 -0
- kiri_cli/contract_lint.py +1505 -0
- kiri_cli/contract_merge_cmd.py +264 -0
- kiri_cli/contract_new.py +326 -0
- kiri_cli/contract_promote.py +216 -0
- kiri_cli/contract_scaffold_cmd.py +134 -0
- kiri_cli/contract_schema_export.py +69 -0
- kiri_cli/contract_show.py +150 -0
- kiri_cli/contract_stubs.py +60 -0
- kiri_cli/contract_sync_from_live.py +327 -0
- kiri_cli/contract_validate.py +154 -0
- kiri_cli/contract_verify_live.py +242 -0
- kiri_cli/cost_cmd.py +243 -0
- kiri_cli/databricks_health.py +1254 -0
- kiri_cli/databricks_setup_cmd.py +712 -0
- kiri_cli/discover_cmd.py +600 -0
- kiri_cli/docs_cmd.py +35 -0
- kiri_cli/doctor_cmd.py +145 -0
- kiri_cli/dv_cmd.py +814 -0
- kiri_cli/dv_conformance_cmd.py +154 -0
- kiri_cli/dv_install.py +493 -0
- kiri_cli/dv_migrate_cmd.py +183 -0
- kiri_cli/dv_test_cmd.py +175 -0
- kiri_cli/federation_cmd.py +315 -0
- kiri_cli/fk_cmd.py +160 -0
- kiri_cli/flows_cmd.py +910 -0
- kiri_cli/governance_cmd.py +358 -0
- kiri_cli/incident_api_cmd.py +180 -0
- kiri_cli/incident_cmd.py +410 -0
- kiri_cli/incident_setup_cmd.py +239 -0
- kiri_cli/ingest.py +427 -0
- kiri_cli/ingest_discover.py +480 -0
- kiri_cli/ingest_new_source.py +386 -0
- kiri_cli/intent_cmd.py +122 -0
- kiri_cli/invariants_cmd.py +118 -0
- kiri_cli/kiri_banner.py +161 -0
- kiri_cli/kpi_cmd.py +222 -0
- kiri_cli/layer_cmd.py +464 -0
- kiri_cli/lineage.py +601 -0
- kiri_cli/main.py +2472 -0
- kiri_cli/maturity_cmd.py +222 -0
- kiri_cli/migrate_cmd.py +1347 -0
- kiri_cli/onboard_cmd.py +223 -0
- kiri_cli/pack_cmd.py +496 -0
- kiri_cli/pipeline_cmd.py +127 -0
- kiri_cli/pre_classify_cmd.py +274 -0
- kiri_cli/project_cmd.py +592 -0
- kiri_cli/py.typed +0 -0
- kiri_cli/quality_cmd.py +295 -0
- kiri_cli/rbac.py +139 -0
- kiri_cli/rbac_cmd.py +291 -0
- kiri_cli/release.py +850 -0
- kiri_cli/release_semver.py +265 -0
- kiri_cli/repo_cmd.py +155 -0
- kiri_cli/retention_cmd.py +333 -0
- kiri_cli/sample_data/__init__.py +12 -0
- kiri_cli/sample_data/customers.contract.yml +47 -0
- kiri_cli/sample_data/customers.csv +25 -0
- kiri_cli/scaffold_cmd.py +721 -0
- kiri_cli/schema_cmd.py +185 -0
- kiri_cli/semantic_cmd.py +605 -0
- kiri_cli/silver_cmd.py +348 -0
- kiri_cli/skills_cmd.py +243 -0
- kiri_cli/skills_data/README.md +51 -0
- kiri_cli/skills_data/add-source/SKILL.md +116 -0
- kiri_cli/skills_data/azure-deploy/SKILL.md +261 -0
- kiri_cli/skills_data/azure-status/SKILL.md +111 -0
- kiri_cli/skills_data/compliance-check/SKILL.md +99 -0
- kiri_cli/skills_data/databricks-build-pipeline/SKILL.md +112 -0
- kiri_cli/skills_data/databricks-project-setup/SKILL.md +157 -0
- kiri_cli/skills_data/debug-apply/SKILL.md +186 -0
- kiri_cli/skills_data/draft-gold-model/SKILL.md +203 -0
- kiri_cli/skills_data/explain-kpi/SKILL.md +158 -0
- kiri_cli/skills_data/incident-coach/SKILL.md +135 -0
- kiri_cli/skills_data/incident-setup/SKILL.md +112 -0
- kiri_cli/skills_data/maturity-coach/SKILL.md +120 -0
- kiri_cli/skills_data/promote-contract/SKILL.md +91 -0
- kiri_cli/skills_data/review-attributes/SKILL.md +94 -0
- kiri_cli/sla_cmd.py +769 -0
- kiri_cli/source_cmd.py +909 -0
- kiri_cli/templates/__init__.py +6 -0
- kiri_cli/templates/dv4dbt_packages.yml.j2 +11 -0
- kiri_cli/upgrade_cmd.py +896 -0
- kiri_cli/use_cmd.py +120 -0
- kiri_cli/vault_cmd.py +241 -0
- kiri_cli/verify_generated.py +241 -0
- kiri_cli-1.0.0b3.dist-info/METADATA +212 -0
- kiri_cli-1.0.0b3.dist-info/RECORD +743 -0
- kiri_cli-1.0.0b3.dist-info/WHEEL +4 -0
- kiri_cli-1.0.0b3.dist-info/entry_points.txt +4 -0
- kiri_compliance/__init__.py +33 -0
- kiri_compliance/audit_coverage.py +354 -0
- kiri_compliance/controls.py +763 -0
- kiri_compliance/evidence.py +155 -0
- kiri_compliance/py.typed +0 -0
- kiri_compliance/quality_engine.py +246 -0
- kiri_compliance/renderer.py +195 -0
- kiri_compliance/report.py +44 -0
- kiri_compliance/schema.py +129 -0
- kiri_core/__init__.py +185 -0
- kiri_core/adapter_surfaces.py +149 -0
- kiri_core/ai_query_gate/__init__.py +92 -0
- kiri_core/ai_query_gate/audit.py +153 -0
- kiri_core/ai_query_gate/evaluator.py +484 -0
- kiri_core/ai_query_gate/models.py +162 -0
- kiri_core/ai_query_gate/parser.py +436 -0
- kiri_core/ai_query_gate/resolver.py +274 -0
- kiri_core/ai_query_gate/store.py +234 -0
- kiri_core/ai_query_gate/store_postgres.py +409 -0
- kiri_core/api_keys.py +243 -0
- kiri_core/apply/__init__.py +27 -0
- kiri_core/apply/conformance.py +99 -0
- kiri_core/apply/incident_router.py +176 -0
- kiri_core/apply/models.py +140 -0
- kiri_core/apply/reporter.py +46 -0
- kiri_core/apply/runner.py +3083 -0
- kiri_core/apply_log_store.py +155 -0
- kiri_core/approval_flow/__init__.py +133 -0
- kiri_core/approval_flow/integration.py +156 -0
- kiri_core/approval_flow/models.py +443 -0
- kiri_core/approval_flow/notifier.py +263 -0
- kiri_core/approval_flow/orchestrator.py +660 -0
- kiri_core/approval_flow/reconcile_signoff.py +53 -0
- kiri_core/approval_flow/store.py +220 -0
- kiri_core/approval_flow/store_jsonl.py +217 -0
- kiri_core/approval_flow/store_postgres.py +580 -0
- kiri_core/approval_policy.py +116 -0
- kiri_core/asgi_trace.py +160 -0
- kiri_core/compliance.py +802 -0
- kiri_core/cutover_approval.py +187 -0
- kiri_core/discovery/__init__.py +109 -0
- kiri_core/discovery/ai/__init__.py +37 -0
- kiri_core/discovery/ai/_payload.py +115 -0
- kiri_core/discovery/ai/budget.py +83 -0
- kiri_core/discovery/ai/classify.py +223 -0
- kiri_core/discovery/ai/describe.py +200 -0
- kiri_core/discovery/ai/errors.py +49 -0
- kiri_core/discovery/discover.py +648 -0
- kiri_core/discovery/heuristics/__init__.py +42 -0
- kiri_core/discovery/heuristics/classification.py +174 -0
- kiri_core/discovery/heuristics/medallion.py +157 -0
- kiri_core/discovery/heuristics/pii.py +92 -0
- kiri_core/discovery/introspector.py +636 -0
- kiri_core/discovery/introspector_databricks.py +396 -0
- kiri_core/discovery/introspector_postgres.py +394 -0
- kiri_core/discovery/introspector_snowflake.py +383 -0
- kiri_core/discovery/reporting/__init__.py +17 -0
- kiri_core/discovery/reporting/discovery_report.py +190 -0
- kiri_core/discovery/scaffold/__init__.py +41 -0
- kiri_core/discovery/scaffold/contract_writer.py +632 -0
- kiri_core/discovery/scaffold/provenance_overlay.py +143 -0
- kiri_core/discovery/scaffold/source_writer.py +211 -0
- kiri_core/discovery/scaffold/todo_marker.py +98 -0
- kiri_core/domain_policy.py +122 -0
- kiri_core/goal_coverage.py +82 -0
- kiri_core/gold_registry.py +211 -0
- kiri_core/health.py +192 -0
- kiri_core/incident.py +204 -0
- kiri_core/incident_builders.py +256 -0
- kiri_core/incident_log.py +418 -0
- kiri_core/intent/__init__.py +46 -0
- kiri_core/intent/agent_runner.py +82 -0
- kiri_core/intent/project_dv.py +261 -0
- kiri_core/intent/propose.py +132 -0
- kiri_core/intent/semantic_intent.py +446 -0
- kiri_core/invariants.py +301 -0
- kiri_core/kpi.py +373 -0
- kiri_core/latency.py +150 -0
- kiri_core/lifecycle/__init__.py +555 -0
- kiri_core/lifecycle/ai_index.py +453 -0
- kiri_core/lifecycle/auto_mode.py +172 -0
- kiri_core/lifecycle/bootstrap.py +316 -0
- kiri_core/lifecycle/cli_reference.py +1064 -0
- kiri_core/lifecycle/commit.py +258 -0
- kiri_core/lifecycle/compile_dispatch.py +296 -0
- kiri_core/lifecycle/compile_range.py +665 -0
- kiri_core/lifecycle/contract_inventory.py +354 -0
- kiri_core/lifecycle/contracts.py +250 -0
- kiri_core/lifecycle/doctor.py +412 -0
- kiri_core/lifecycle/git_remote.py +948 -0
- kiri_core/lifecycle/identity.py +147 -0
- kiri_core/lifecycle/ingestion_modes.py +215 -0
- kiri_core/lifecycle/layer_contract.py +452 -0
- kiri_core/lifecycle/layer_contract_migrate.py +969 -0
- kiri_core/lifecycle/layer_contract_parser.py +436 -0
- kiri_core/lifecycle/layer_contract_validator.py +2291 -0
- kiri_core/lifecycle/layout.py +298 -0
- kiri_core/lifecycle/onboarding.py +306 -0
- kiri_core/lifecycle/openlineage_emit.py +153 -0
- kiri_core/lifecycle/patch.py +231 -0
- kiri_core/lifecycle/plan.py +241 -0
- kiri_core/lifecycle/probe.py +204 -0
- kiri_core/lifecycle/project_context.py +334 -0
- kiri_core/lifecycle/project_status.py +532 -0
- kiri_core/lifecycle/scaffold.py +455 -0
- kiri_core/lifecycle/source_grouping.py +725 -0
- kiri_core/lifecycle/source_introspection.py +488 -0
- kiri_core/lifecycle/source_inventory.py +561 -0
- kiri_core/lifecycle/source_patch.py +353 -0
- kiri_core/lifecycle/sources.py +470 -0
- kiri_core/lifecycle/sync_from_live.py +727 -0
- kiri_core/lifecycle/validate.py +116 -0
- kiri_core/lifecycle/verify_annotations.py +391 -0
- kiri_core/lifecycle/verify_live.py +389 -0
- kiri_core/lifecycle/verify_metric_views.py +321 -0
- kiri_core/lifecycle/watermark.py +371 -0
- kiri_core/lifecycle/watermark_apply.py +656 -0
- kiri_core/lineage.py +418 -0
- kiri_core/loader.py +377 -0
- kiri_core/logging.py +374 -0
- kiri_core/maturity.py +592 -0
- kiri_core/migration_layer_policy.py +285 -0
- kiri_core/names.py +163 -0
- kiri_core/observed_lineage.py +62 -0
- kiri_core/odcs/__init__.py +15 -0
- kiri_core/otel_bootstrap.py +326 -0
- kiri_core/pipeline_ir.py +332 -0
- kiri_core/pipeline_loader.py +82 -0
- kiri_core/planning.py +110 -0
- kiri_core/project.py +1576 -0
- kiri_core/project_docs.py +199 -0
- kiri_core/py.typed +0 -0
- kiri_core/rate_limit.py +456 -0
- kiri_core/release_state/__init__.py +37 -0
- kiri_core/release_state/models.py +108 -0
- kiri_core/release_state/store.py +634 -0
- kiri_core/resilience/__init__.py +288 -0
- kiri_core/resilience/retry.py +173 -0
- kiri_core/runtime.py +40 -0
- kiri_core/schema/__init__.py +62 -0
- kiri_core/schema/column.py +140 -0
- kiri_core/schema/ingestion.py +665 -0
- kiri_core/schema/model.py +44 -0
- kiri_core/schema/reporting_goal.py +160 -0
- kiri_core/schema/source.py +188 -0
- kiri_core/silver/__init__.py +148 -0
- kiri_core/silver/apply_runner.py +830 -0
- kiri_core/silver/backfill.py +280 -0
- kiri_core/silver/dv_backend.py +426 -0
- kiri_core/silver/dv_graph.py +932 -0
- kiri_core/silver/dv_graph_mermaid.py +443 -0
- kiri_core/silver/dv_graph_payload.py +79 -0
- kiri_core/silver/dv_graph_views.py +416 -0
- kiri_core/silver/graph_projector.py +566 -0
- kiri_core/silver/hash_canonicaliser.py +172 -0
- kiri_core/silver/hash_policy.py +195 -0
- kiri_core/silver/load_ts_policy.py +202 -0
- kiri_core/silver/schema_evolution.py +410 -0
- kiri_core/silver/zone_rules.py +81 -0
- kiri_core/sizing.py +134 -0
- kiri_core/storage_paths.py +130 -0
- kiri_core/technique_validator.py +700 -0
- kiri_core/trace.py +409 -0
- kiri_core/types.py +41 -0
- kiri_core/urn.py +161 -0
- kiri_dbt_bridge/__init__.py +22 -0
- kiri_dbt_bridge/_schema_resolve.py +38 -0
- kiri_dbt_bridge/bridge.py +94 -0
- kiri_dbt_bridge/dbt.manifest.yml +26 -0
- kiri_dbt_bridge/dv4dbt/__init__.py +143 -0
- kiri_dbt_bridge/dv4dbt/backend.py +204 -0
- kiri_dbt_bridge/dv4dbt/bridge_generator.py +289 -0
- kiri_dbt_bridge/dv4dbt/compat.py +232 -0
- kiri_dbt_bridge/dv4dbt/hub_generator.py +282 -0
- kiri_dbt_bridge/dv4dbt/link_generator.py +306 -0
- kiri_dbt_bridge/dv4dbt/manifest.py +203 -0
- kiri_dbt_bridge/dv4dbt/pit_generator.py +318 -0
- kiri_dbt_bridge/dv4dbt/policy_translation.py +249 -0
- kiri_dbt_bridge/dv4dbt/project_writer.py +588 -0
- kiri_dbt_bridge/dv4dbt/ref_generator.py +404 -0
- kiri_dbt_bridge/dv4dbt/render.py +180 -0
- kiri_dbt_bridge/dv4dbt/sat_generator.py +480 -0
- kiri_dbt_bridge/dv4dbt/sat_variant_generator.py +529 -0
- kiri_dbt_bridge/dv4dbt/templates/__init__.py +16 -0
- kiri_dbt_bridge/dv4dbt/templates/bridge.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/dbt_project.yml.j2 +37 -0
- kiri_dbt_bridge/dv4dbt/templates/hub.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/link.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/packages.yml.j2 +10 -0
- kiri_dbt_bridge/dv4dbt/templates/pit.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/ref_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_effectivity.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_multi_active.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_non_historised.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_record_tracking.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sat_virtual_end_dated.sql.j2 +9 -0
- kiri_dbt_bridge/dv4dbt/templates/sources.yml.j2 +39 -0
- kiri_dbt_bridge/framework_adapter.py +310 -0
- kiri_dbt_bridge/generate.py +372 -0
- kiri_dbt_bridge/gold/__init__.py +21 -0
- kiri_dbt_bridge/gold/_sql_emit.py +128 -0
- kiri_dbt_bridge/gold/assemble.py +90 -0
- kiri_dbt_bridge/gold/canonical_columns.py +168 -0
- kiri_dbt_bridge/gold/computed_expressions.py +179 -0
- kiri_dbt_bridge/gold/dialects/__init__.py +69 -0
- kiri_dbt_bridge/gold/dialects/databricks.py +150 -0
- kiri_dbt_bridge/gold/dialects/duckdb.py +120 -0
- kiri_dbt_bridge/gold/dialects/trino.py +136 -0
- kiri_dbt_bridge/gold/dim_generator.py +688 -0
- kiri_dbt_bridge/gold/dimension_refs.py +255 -0
- kiri_dbt_bridge/gold/dispatch.py +69 -0
- kiri_dbt_bridge/gold/fact_generator.py +534 -0
- kiri_dbt_bridge/gold/flat_generator.py +144 -0
- kiri_dbt_bridge/gold/lineage.py +484 -0
- kiri_dbt_bridge/gold/localized_twins.py +218 -0
- kiri_dbt_bridge/gold/measures.py +49 -0
- kiri_dbt_bridge/gold/role_playing.py +75 -0
- kiri_dbt_bridge/gold/surrogate_key.py +114 -0
- kiri_dbt_bridge/macros/kiri_scd1_dim.sql +57 -0
- kiri_dbt_bridge/macros/kiri_scd2_dim.sql +188 -0
- kiri_dbt_bridge/macros/kiri_sha256.sql +32 -0
- kiri_dbt_bridge/macros/kiri_string_cast.sql +37 -0
- kiri_dbt_bridge/macros/kiri_surrogate_key.sql +43 -0
- kiri_dbt_bridge/profiles.py +162 -0
- kiri_dbt_bridge/py.typed +0 -0
- kiri_dbt_bridge/safe_ident.py +44 -0
- kiri_dbt_bridge/silver_conformance.py +188 -0
- kiri_dbt_bridge/silver_kimball/__init__.py +27 -0
- kiri_dbt_bridge/silver_kimball/assemble.py +174 -0
- kiri_dbt_bridge/silver_kimball/dim_generator.py +602 -0
- kiri_dbt_bridge/silver_kimball/dispatch.py +86 -0
- kiri_dbt_bridge/silver_kimball/fact_generator.py +320 -0
- kiri_dbt_bridge/silver_kimball/flat_generator.py +156 -0
- kiri_dbt_bridge/silver_kimball/lineage.py +464 -0
- kiri_dbt_bridge/silver_kimball/watermark_window.py +199 -0
- kiri_dbt_bridge/silver_kimball/zone.py +68 -0
- kiri_dbt_bridge/silver_scd2_curated/__init__.py +46 -0
- kiri_dbt_bridge/silver_scd2_curated/assemble.py +213 -0
- kiri_dbt_bridge/silver_scd2_curated/curated_generator.py +351 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_generator.py +261 -0
- kiri_dbt_bridge/silver_scd2_curated/standardized_plan.py +254 -0
- kiri_iceberg/__init__.py +11 -0
- kiri_iceberg/catalog.py +128 -0
- kiri_iceberg/profile.py +131 -0
- kiri_iceberg/py.typed +0 -0
- kiri_ingestion/__init__.py +63 -0
- kiri_ingestion/airbyte_client.py +428 -0
- kiri_ingestion/database_fetcher.py +224 -0
- kiri_ingestion/layout.py +71 -0
- kiri_ingestion/mssql_introspect.py +565 -0
- kiri_ingestion/py.typed +0 -0
- kiri_ingestion/rest_fetcher.py +472 -0
- kiri_ingestion/runner.py +341 -0
- kiri_ingestion/schema_drift.py +222 -0
- kiri_ingestion/storage.py +621 -0
- kiri_mcp/__init__.py +5 -0
- kiri_mcp/audit.py +99 -0
- kiri_mcp/audit_decorators.py +252 -0
- kiri_mcp/main.py +22 -0
- kiri_mcp/py.typed +0 -0
- kiri_mcp/rbac.py +110 -0
- kiri_mcp/server.py +766 -0
- kiri_mcp/skills.py +297 -0
- kiri_mcp/tools/__init__.py +70 -0
- kiri_mcp/tools/_shared.py +58 -0
- kiri_mcp/tools/adapter.py +150 -0
- kiri_mcp/tools/cli_reference.py +59 -0
- kiri_mcp/tools/compliance.py +119 -0
- kiri_mcp/tools/contract.py +374 -0
- kiri_mcp/tools/dv.py +62 -0
- kiri_mcp/tools/federation.py +152 -0
- kiri_mcp/tools/intent.py +71 -0
- kiri_mcp/tools/lineage.py +204 -0
- kiri_mcp/tools/lineage_drift.py +75 -0
- kiri_mcp/tools/plan.py +66 -0
- kiri_mcp/tools/project.py +311 -0
- kiri_migration/__init__.py +56 -0
- kiri_migration/name_preservation.py +271 -0
- kiri_migration/predeploy.py +311 -0
- kiri_migration/py.typed +0 -0
- kiri_migration/reconcile.py +1891 -0
- kiri_migration/scaffold/__init__.py +90 -0
- kiri_migration/scaffold/_audit_columns.py +74 -0
- kiri_migration/scaffold/_classify.py +100 -0
- kiri_migration/scaffold/_typemap.py +71 -0
- kiri_migration/scaffold/_yaml.py +33 -0
- kiri_migration/scaffold/bronze.py +279 -0
- kiri_migration/scaffold/compile_targets/__init__.py +271 -0
- kiri_migration/scaffold/compile_targets/_common.py +193 -0
- kiri_migration/scaffold/compile_targets/copy_into.py +33 -0
- kiri_migration/scaffold/compile_targets/dlt.py +125 -0
- kiri_migration/scaffold/compile_targets/dlt_with_append_flows.py +163 -0
- kiri_migration/scaffold/compile_targets/dlt_with_apply_changes.py +192 -0
- kiri_migration/scaffold/compile_targets/dlt_with_expectations.py +276 -0
- kiri_migration/scaffold/compile_targets/layer_dispatch.py +258 -0
- kiri_migration/scaffold/compile_targets/sql_full_snapshot.py +385 -0
- kiri_migration/scaffold/compile_targets/streaming_table.py +37 -0
- kiri_migration/scaffold/mart.py +234 -0
- kiri_migration/scaffold/plans/__init__.py +49 -0
- kiri_migration/scaffold/plans/_registry.py +308 -0
- kiri_migration/scaffold/plans/minimal.py +550 -0
- kiri_migration/scaffold/plans/plan_b_layer.py +640 -0
- kiri_migration/scaffold/silver_from_marts.py +256 -0
- kiri_migration/scaffold/silver_pattern.py +451 -0
- kiri_migration/sp_translator.py +398 -0
- kiri_odcs/LICENSE +204 -0
- kiri_odcs/__init__.py +234 -0
- kiri_odcs/ai_trust_tier.py +475 -0
- kiri_odcs/canonical.py +263 -0
- kiri_odcs/diff.py +63 -0
- kiri_odcs/errors.py +51 -0
- kiri_odcs/invariant_sql.py +136 -0
- kiri_odcs/kiri_bridge.py +71 -0
- kiri_odcs/kiri_ext.py +3166 -0
- kiri_odcs/kiri_fk.py +447 -0
- kiri_odcs/kiri_pit.py +121 -0
- kiri_odcs/kiri_ref.py +57 -0
- kiri_odcs/openlineage_adapter.py +105 -0
- kiri_odcs/parser.py +1213 -0
- kiri_odcs/pii_guardrails.py +159 -0
- kiri_odcs/py.typed +0 -0
- kiri_odcs/schema_export.py +223 -0
- kiri_openlineage/__init__.py +58 -0
- kiri_openlineage/builder.py +146 -0
- kiri_openlineage/cost.py +190 -0
- kiri_openlineage/emitter.py +165 -0
- kiri_openlineage/py.typed +0 -0
- kiri_openlineage/schema.py +121 -0
- kiri_packs/__init__.py +67 -0
- kiri_packs/installer.py +500 -0
- kiri_packs/manifest.py +134 -0
- kiri_packs/publisher.py +605 -0
- kiri_packs/py.typed +0 -0
- kiri_packs/validator.py +295 -0
- kiri_quality/__init__.py +108 -0
- kiri_quality/ai_audit_correlator.py +293 -0
- kiri_quality/ai_audit_correlator_apply.py +526 -0
- kiri_quality/conformance_evidence.py +272 -0
- kiri_quality/dbt_test_collector.py +214 -0
- kiri_quality/dq_codegen.py +300 -0
- kiri_quality/dv_checks.py +393 -0
- kiri_quality/dv_column_lineage.py +279 -0
- kiri_quality/dv_column_lineage_apply.py +856 -0
- kiri_quality/dv_conformance.py +841 -0
- kiri_quality/dv_fixture_runner.py +478 -0
- kiri_quality/dv_invariants.py +986 -0
- kiri_quality/dv_kiri.py +97 -0
- kiri_quality/dv_markdown.py +169 -0
- kiri_quality/dv_observed_uc_reader.py +187 -0
- kiri_quality/dv_runner.py +389 -0
- kiri_quality/evidence_status.py +337 -0
- kiri_quality/fk_checks.py +800 -0
- kiri_quality/fk_ddl.py +402 -0
- kiri_quality/fk_pit_contract.py +168 -0
- kiri_quality/fk_runner.py +950 -0
- kiri_quality/kernel.py +376 -0
- kiri_quality/pii.py +60 -0
- kiri_quality/pre_classify/__init__.py +55 -0
- kiri_quality/pre_classify/engine.py +346 -0
- kiri_quality/pre_classify/locale_packs/en.yaml +117 -0
- kiri_quality/pre_classify/locale_packs/sv.yaml +73 -0
- kiri_quality/pre_classify/merge.py +297 -0
- kiri_quality/pre_classify/model.py +163 -0
- kiri_quality/py.typed +0 -0
- kiri_quality/reject_lane.py +261 -0
- kiri_quality/rule_runner.py +565 -0
- kiri_quality/schema_evolution_check.py +88 -0
- kiri_quality/silver_quality.py +494 -0
- kiri_quality/silver_quality_markdown.py +187 -0
- kiri_quality/uc_security_compile.py +951 -0
- kiri_quality/uc_security_compile_apply.py +1186 -0
- kiri_rbac/__init__.py +87 -0
- kiri_rbac/access.py +371 -0
- kiri_rbac/loader.py +104 -0
- kiri_rbac/pr_gate.py +213 -0
- kiri_rbac/providers/__init__.py +29 -0
- kiri_rbac/providers/_jwt.py +121 -0
- kiri_rbac/providers/azure_ad.py +93 -0
- kiri_rbac/providers/base.py +107 -0
- kiri_rbac/providers/factory.py +48 -0
- kiri_rbac/providers/github.py +134 -0
- kiri_rbac/providers/oidc_generic.py +99 -0
- kiri_rbac/py.typed +0 -0
- kiri_rbac/resolver.py +118 -0
- kiri_rbac/schema.py +219 -0
- kiri_schema_registry/__init__.py +22 -0
- kiri_schema_registry/avro.py +127 -0
- kiri_schema_registry/base.py +71 -0
- kiri_schema_registry/confluent.py +184 -0
- kiri_schema_registry/glue.py +178 -0
- kiri_schema_registry/py.typed +0 -0
- kiri_slack_bot/__init__.py +24 -0
- kiri_slack_bot/handler.py +248 -0
- kiri_slack_bot/py.typed +0 -0
- kiri_slack_bot/queries.py +90 -0
- kiri_slack_bot/signature.py +94 -0
- kiri_vault/__init__.py +59 -0
- kiri_vault/audit.py +64 -0
- kiri_vault/aws_vault.py +123 -0
- kiri_vault/azure_vault.py +236 -0
- kiri_vault/boundaries.py +219 -0
- kiri_vault/client.py +54 -0
- kiri_vault/databricks_vault.py +292 -0
- kiri_vault/env_vault.py +77 -0
- kiri_vault/py.typed +0 -0
- kiri_vault/ref.py +84 -0
- kirimana_automation/__init__.py +0 -0
- kirimana_automation/business_calendar.py +179 -0
- kirimana_automation/dag_snapshot.py +252 -0
- kirimana_automation/db/__init__.py +5 -0
- kirimana_automation/db/migrate.py +571 -0
- kirimana_automation/db/migrations/0001_release_state.down.duckdb.sql +29 -0
- kirimana_automation/db/migrations/0001_release_state.down.postgres.sql +37 -0
- kirimana_automation/db/migrations/0001_release_state.duckdb.sql +170 -0
- kirimana_automation/db/migrations/0001_release_state.postgres.sql +176 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.down.postgres.sql +9 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0002_rate_limit_buckets.postgres.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.duckdb.sql +18 -0
- kirimana_automation/db/migrations/0003_flow_runs.down.postgres.sql +19 -0
- kirimana_automation/db/migrations/0003_flow_runs.duckdb.sql +85 -0
- kirimana_automation/db/migrations/0003_flow_runs.postgres.sql +102 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.duckdb.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.duckdb.sql +15 -0
- kirimana_automation/db/migrations/0004_flow_dag_snapshots.postgres.sql +31 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.duckdb.sql +109 -0
- kirimana_automation/db/migrations/0005_calendar_skip.down.postgres.sql +42 -0
- kirimana_automation/db/migrations/0005_calendar_skip.duckdb.sql +113 -0
- kirimana_automation/db/migrations/0005_calendar_skip.postgres.sql +42 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.duckdb.sql +9 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.down.postgres.sql +11 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.duckdb.sql +45 -0
- kirimana_automation/db/migrations/0006_flow_apply_log.postgres.sql +49 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.duckdb.sql +14 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.down.postgres.sql +2 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.duckdb.sql +7 -0
- kirimana_automation/db/migrations/0007_flow_task_error_redacted.postgres.sql +11 -0
- kirimana_automation/db/migrations/README.md +84 -0
- kirimana_automation/flow_loader.py +220 -0
- kirimana_automation/freeze.py +133 -0
- kirimana_automation/kiri_recommender.py +313 -0
- kirimana_automation/main.py +195 -0
- kirimana_automation/py.typed +0 -0
- kirimana_automation/run_history.py +593 -0
- kirimana_automation/run_history_analysis.py +607 -0
- kirimana_automation/schedule.py +343 -0
- kirimana_automation/schedule_validator.py +572 -0
- kirimana_automation/workflow_compiler.py +1288 -0
|
@@ -0,0 +1,472 @@
|
|
|
1
|
+
"""REST API runtime fetcher — pulls one source/table into the landing zone.
|
|
2
|
+
|
|
3
|
+
Resolves ${vault:...} refs for auth, fetches one or more HTTP pages via
|
|
4
|
+
stdlib ``urllib``, and writes the landing payload under the table's
|
|
5
|
+
``_landing/`` directory. When ADR 0090 ``record_path`` is unset, the legacy
|
|
6
|
+
single-request raw-body write is preserved byte-for-byte. When ``record_path``
|
|
7
|
+
is set, JSON records are unwrapped before landing so downstream JSON ingestion
|
|
8
|
+
receives a clean row array. See ADR 0011 for source schema, ADR 0009 for this
|
|
9
|
+
runtime.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import re
|
|
17
|
+
import ssl
|
|
18
|
+
import time
|
|
19
|
+
import urllib.error
|
|
20
|
+
import urllib.parse
|
|
21
|
+
import urllib.request
|
|
22
|
+
import uuid
|
|
23
|
+
from datetime import UTC, datetime
|
|
24
|
+
from decimal import Decimal, InvalidOperation
|
|
25
|
+
from functools import lru_cache
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import TYPE_CHECKING, Any
|
|
28
|
+
|
|
29
|
+
import certifi
|
|
30
|
+
from kiri_core.lifecycle.source_introspection import resolve_dotted_path
|
|
31
|
+
from kiri_core.schema.ingestion import RestApiPagination
|
|
32
|
+
from pydantic import BaseModel, ConfigDict
|
|
33
|
+
|
|
34
|
+
from kiri_ingestion.storage import LandingStorage, LocalLandingStorage
|
|
35
|
+
|
|
36
|
+
if TYPE_CHECKING: # pragma: no cover — type-only import
|
|
37
|
+
from kiri_core.schema import IngestionFormat, RestApiSpec
|
|
38
|
+
from kiri_vault import VaultAuditLog, VaultClient
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@lru_cache(maxsize=1)
|
|
42
|
+
def _tls_context() -> ssl.SSLContext:
|
|
43
|
+
"""TLS context backed by certifi's CA bundle.
|
|
44
|
+
|
|
45
|
+
Python's stdlib ``urllib`` has no CA store of its own; on macOS and
|
|
46
|
+
minimal containers the default context can't verify public certs
|
|
47
|
+
(CERTIFICATE_VERIFY_FAILED). certifi is already in the dependency
|
|
48
|
+
tree, so pin verification to it for deterministic REST ingest.
|
|
49
|
+
"""
|
|
50
|
+
return ssl.create_default_context(cafile=certifi.where())
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class FetchError(RuntimeError):
|
|
54
|
+
"""Raised for any unrecoverable fetch failure."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class FetchResult(BaseModel):
|
|
58
|
+
"""Outcome of one REST fetch."""
|
|
59
|
+
|
|
60
|
+
model_config = ConfigDict(extra="forbid")
|
|
61
|
+
|
|
62
|
+
source: str
|
|
63
|
+
table: str
|
|
64
|
+
url: str
|
|
65
|
+
http_status: int
|
|
66
|
+
bytes_written: int
|
|
67
|
+
landing_path: str
|
|
68
|
+
trace_id: str
|
|
69
|
+
high_watermark: str | None = None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
_ABSOLUTE_MAX_PAGES = 1000
|
|
73
|
+
_INTEGER_TEXT_RE = re.compile(r"^[+-]?\d+$")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _resolve_vault_ref(
|
|
77
|
+
ref: str,
|
|
78
|
+
vault: VaultClient,
|
|
79
|
+
audit: VaultAuditLog | None,
|
|
80
|
+
caller: str,
|
|
81
|
+
) -> str:
|
|
82
|
+
"""Turn a ${vault:id:key} string into its resolved value, with audit."""
|
|
83
|
+
from kiri_vault import VaultRef
|
|
84
|
+
from kiri_vault.audit import VaultAuditEntry
|
|
85
|
+
|
|
86
|
+
parsed = VaultRef.parse(ref)
|
|
87
|
+
if parsed is None:
|
|
88
|
+
raise FetchError(f"expected vault ref, got {ref!r} — check source YAML.")
|
|
89
|
+
try:
|
|
90
|
+
value = vault.get(parsed.secret_id, parsed.key)
|
|
91
|
+
except Exception as e:
|
|
92
|
+
if audit is not None:
|
|
93
|
+
audit.write(
|
|
94
|
+
VaultAuditEntry.make(
|
|
95
|
+
caller=caller,
|
|
96
|
+
secret_id=parsed.secret_id,
|
|
97
|
+
key=parsed.key,
|
|
98
|
+
provider=getattr(vault, "kind", "unknown"),
|
|
99
|
+
status="error",
|
|
100
|
+
error=str(e),
|
|
101
|
+
)
|
|
102
|
+
)
|
|
103
|
+
raise FetchError(f"vault miss: {ref}: {e}") from e
|
|
104
|
+
if audit is not None:
|
|
105
|
+
audit.write(
|
|
106
|
+
VaultAuditEntry.make(
|
|
107
|
+
caller=caller,
|
|
108
|
+
secret_id=parsed.secret_id,
|
|
109
|
+
key=parsed.key,
|
|
110
|
+
provider=getattr(vault, "kind", "unknown"),
|
|
111
|
+
status="ok",
|
|
112
|
+
)
|
|
113
|
+
)
|
|
114
|
+
return value
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _auth_header(
|
|
118
|
+
spec: RestApiSpec, vault: VaultClient, audit: VaultAuditLog | None, caller: str
|
|
119
|
+
) -> tuple[str, str] | None:
|
|
120
|
+
"""Build the (name, value) header tuple for the configured auth, or None."""
|
|
121
|
+
auth = spec.auth
|
|
122
|
+
if auth.kind == "none":
|
|
123
|
+
return None
|
|
124
|
+
if auth.kind == "bearer":
|
|
125
|
+
token = _resolve_vault_ref(auth.value_vault_ref or "", vault, audit, caller)
|
|
126
|
+
return ("Authorization", f"Bearer {token}")
|
|
127
|
+
if auth.kind == "header":
|
|
128
|
+
value = _resolve_vault_ref(auth.value_vault_ref or "", vault, audit, caller)
|
|
129
|
+
return (auth.header_name or "", value)
|
|
130
|
+
if auth.kind == "basic":
|
|
131
|
+
import base64
|
|
132
|
+
|
|
133
|
+
u = _resolve_vault_ref(auth.username_vault_ref or "", vault, audit, caller)
|
|
134
|
+
p = _resolve_vault_ref(auth.password_vault_ref or "", vault, audit, caller)
|
|
135
|
+
token = base64.b64encode(f"{u}:{p}".encode()).decode()
|
|
136
|
+
return ("Authorization", f"Basic {token}")
|
|
137
|
+
raise FetchError(f"unknown auth kind: {auth.kind!r}")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _file_extension_for(fmt: IngestionFormat) -> str:
|
|
141
|
+
return {
|
|
142
|
+
"json": ".json",
|
|
143
|
+
"csv": ".csv",
|
|
144
|
+
"xml": ".xml",
|
|
145
|
+
"parquet": ".parquet",
|
|
146
|
+
}.get(fmt.value, ".bin")
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def fetch_rest_to_landing(
|
|
150
|
+
*,
|
|
151
|
+
source_name: str,
|
|
152
|
+
table_name: str,
|
|
153
|
+
spec: RestApiSpec,
|
|
154
|
+
fmt: IngestionFormat,
|
|
155
|
+
vault: VaultClient,
|
|
156
|
+
bronze_root: Path | None = None,
|
|
157
|
+
storage: LandingStorage | None = None,
|
|
158
|
+
audit: VaultAuditLog | None = None,
|
|
159
|
+
caller: str = "cli.fetch.rest",
|
|
160
|
+
last_watermark: str | None = None,
|
|
161
|
+
) -> FetchResult:
|
|
162
|
+
"""Fetch one endpoint and drop the response body into the landing zone.
|
|
163
|
+
|
|
164
|
+
Caller is responsible for the subsequent ``kiri ingest`` that converts
|
|
165
|
+
landing files to Delta. This function is the *producer*; the ingestor is
|
|
166
|
+
the *consumer*.
|
|
167
|
+
|
|
168
|
+
Pass either ``bronze_root`` (back-compat — constructs a
|
|
169
|
+
:class:`LocalLandingStorage` internally) or ``storage`` (ADR 0046
|
|
170
|
+
phase 2 — adapter-provided handle for ADLS / S3 / etc.). At least
|
|
171
|
+
one must be set.
|
|
172
|
+
"""
|
|
173
|
+
landing = _resolve_storage(storage=storage, bronze_root=bronze_root)
|
|
174
|
+
|
|
175
|
+
headers: dict[str, str] = {"User-Agent": "kiri-fetch/0.4"}
|
|
176
|
+
headers.update(spec.headers or {})
|
|
177
|
+
auth_header = _auth_header(spec, vault, audit, caller)
|
|
178
|
+
if auth_header is not None:
|
|
179
|
+
headers[auth_header[0]] = auth_header[1]
|
|
180
|
+
|
|
181
|
+
trace_id = uuid.uuid4().hex
|
|
182
|
+
landing_name = f"{int(time.time() * 1000)}_{trace_id[:12]}{_file_extension_for(fmt)}"
|
|
183
|
+
|
|
184
|
+
pagination = _normalise_pagination(spec.pagination)
|
|
185
|
+
incremental = spec.incremental
|
|
186
|
+
base_params = dict(spec.query_params)
|
|
187
|
+
if incremental is not None and incremental.request_param and last_watermark is not None:
|
|
188
|
+
base_params[incremental.request_param] = last_watermark
|
|
189
|
+
|
|
190
|
+
url = _url_with_params(spec.url, base_params)
|
|
191
|
+
|
|
192
|
+
if spec.record_path is None and pagination.kind == "none" and incremental is None:
|
|
193
|
+
status, body = _fetch_once(
|
|
194
|
+
url=url,
|
|
195
|
+
method=spec.method,
|
|
196
|
+
headers=headers,
|
|
197
|
+
timeout_seconds=spec.timeout_seconds,
|
|
198
|
+
)
|
|
199
|
+
if status >= 400:
|
|
200
|
+
raise FetchError(f"HTTP {status} from {url}")
|
|
201
|
+
|
|
202
|
+
ref = landing.write(source_name, table_name, landing_name, body)
|
|
203
|
+
return FetchResult(
|
|
204
|
+
source=source_name,
|
|
205
|
+
table=table_name,
|
|
206
|
+
url=url,
|
|
207
|
+
http_status=status,
|
|
208
|
+
bytes_written=len(body),
|
|
209
|
+
landing_path=ref.path,
|
|
210
|
+
trace_id=trace_id,
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
rows: list[dict[str, Any]] = []
|
|
214
|
+
status = 0
|
|
215
|
+
page_count = 0
|
|
216
|
+
cursor: str | None = None
|
|
217
|
+
while True:
|
|
218
|
+
page_count += 1
|
|
219
|
+
if page_count > _max_pages(pagination):
|
|
220
|
+
raise FetchError(
|
|
221
|
+
"pagination exceeded safety bound "
|
|
222
|
+
f"({_max_pages(pagination)} pages); set provider_page_cap/page_size "
|
|
223
|
+
"to a tighter provider limit."
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
page_params = dict(base_params)
|
|
227
|
+
if pagination.kind == "page":
|
|
228
|
+
page_params[pagination.page_param] = str(page_count)
|
|
229
|
+
if pagination.page_size is not None:
|
|
230
|
+
page_params[pagination.limit_param] = str(pagination.page_size)
|
|
231
|
+
elif pagination.kind == "cursor" and cursor:
|
|
232
|
+
page_params[pagination.cursor_param] = cursor
|
|
233
|
+
|
|
234
|
+
page_url = _url_with_params(spec.url, page_params)
|
|
235
|
+
status, body = _fetch_once(
|
|
236
|
+
url=page_url,
|
|
237
|
+
method=spec.method,
|
|
238
|
+
headers=headers,
|
|
239
|
+
timeout_seconds=spec.timeout_seconds,
|
|
240
|
+
)
|
|
241
|
+
if status >= 400:
|
|
242
|
+
raise FetchError(f"HTTP {status} from {page_url}")
|
|
243
|
+
|
|
244
|
+
payload = _decode_json_body(body, page_url)
|
|
245
|
+
page_rows = _records_at_path(payload, spec.record_path)
|
|
246
|
+
rows.extend(page_rows)
|
|
247
|
+
|
|
248
|
+
if pagination.provider_page_cap is not None and len(rows) >= pagination.provider_page_cap:
|
|
249
|
+
rows = rows[: pagination.provider_page_cap]
|
|
250
|
+
break
|
|
251
|
+
if pagination.kind == "none":
|
|
252
|
+
break
|
|
253
|
+
if pagination.kind == "page":
|
|
254
|
+
if not page_rows:
|
|
255
|
+
break
|
|
256
|
+
if pagination.page_size is not None and len(page_rows) < pagination.page_size:
|
|
257
|
+
break
|
|
258
|
+
continue
|
|
259
|
+
cursor = _next_cursor(payload)
|
|
260
|
+
if not cursor or not page_rows:
|
|
261
|
+
break
|
|
262
|
+
|
|
263
|
+
rows = _apply_post_fetch_incremental(rows, spec, last_watermark)
|
|
264
|
+
high_watermark = _high_watermark(rows, spec)
|
|
265
|
+
body = json.dumps(rows, ensure_ascii=False, separators=(",", ":")).encode()
|
|
266
|
+
ref = landing.write(source_name, table_name, landing_name, body)
|
|
267
|
+
return FetchResult(
|
|
268
|
+
source=source_name,
|
|
269
|
+
table=table_name,
|
|
270
|
+
url=url,
|
|
271
|
+
http_status=status,
|
|
272
|
+
bytes_written=len(body),
|
|
273
|
+
landing_path=ref.path,
|
|
274
|
+
trace_id=trace_id,
|
|
275
|
+
high_watermark=high_watermark,
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _fetch_once(
|
|
280
|
+
*,
|
|
281
|
+
url: str,
|
|
282
|
+
method: str,
|
|
283
|
+
headers: dict[str, str],
|
|
284
|
+
timeout_seconds: float,
|
|
285
|
+
) -> tuple[int, bytes]:
|
|
286
|
+
req = urllib.request.Request( # noqa: S310 — URL + method + headers are metadata-validated
|
|
287
|
+
url=url,
|
|
288
|
+
method=method,
|
|
289
|
+
headers=headers,
|
|
290
|
+
)
|
|
291
|
+
try:
|
|
292
|
+
with urllib.request.urlopen( # noqa: S310
|
|
293
|
+
req, timeout=timeout_seconds, context=_tls_context()
|
|
294
|
+
) as resp:
|
|
295
|
+
status = resp.status
|
|
296
|
+
body = resp.read()
|
|
297
|
+
except urllib.error.HTTPError as e:
|
|
298
|
+
raise FetchError(f"HTTP {e.code} from {url}: {e.reason}") from e
|
|
299
|
+
except Exception as e:
|
|
300
|
+
raise FetchError(f"fetch failed for {url}: {e}") from e
|
|
301
|
+
return status, body
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _url_with_params(url: str, params: dict[str, str]) -> str:
|
|
305
|
+
if not params:
|
|
306
|
+
return url
|
|
307
|
+
joiner = "&" if "?" in url else "?"
|
|
308
|
+
return url + joiner + urllib.parse.urlencode(params)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _normalise_pagination(pagination: Any) -> RestApiPagination:
|
|
312
|
+
if isinstance(pagination, RestApiPagination):
|
|
313
|
+
return pagination
|
|
314
|
+
if pagination == "page":
|
|
315
|
+
return RestApiPagination(kind="page")
|
|
316
|
+
if pagination == "cursor":
|
|
317
|
+
return RestApiPagination(kind="cursor")
|
|
318
|
+
return RestApiPagination(kind="none")
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _max_pages(pagination: RestApiPagination) -> int:
|
|
322
|
+
if pagination.kind == "none":
|
|
323
|
+
return 1
|
|
324
|
+
if pagination.provider_page_cap is not None and pagination.page_size:
|
|
325
|
+
pages_for_cap = (
|
|
326
|
+
pagination.provider_page_cap + pagination.page_size - 1
|
|
327
|
+
) // pagination.page_size
|
|
328
|
+
return max(1, min(_ABSOLUTE_MAX_PAGES, pages_for_cap))
|
|
329
|
+
return _ABSOLUTE_MAX_PAGES
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _decode_json_body(body: bytes, url: str) -> Any:
|
|
333
|
+
try:
|
|
334
|
+
return json.loads(body.decode("utf-8"))
|
|
335
|
+
except UnicodeDecodeError as exc:
|
|
336
|
+
raise FetchError(f"response from {url} was not UTF-8 JSON") from exc
|
|
337
|
+
except json.JSONDecodeError as exc:
|
|
338
|
+
raise FetchError(f"response from {url} was not valid JSON") from exc
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _records_at_path(payload: Any, record_path: str | None) -> list[dict[str, Any]]:
|
|
342
|
+
value = resolve_dotted_path(payload, record_path)
|
|
343
|
+
label = "response root" if record_path is None else f"record_path {record_path!r}"
|
|
344
|
+
if not isinstance(value, list):
|
|
345
|
+
raise FetchError(f"{label} did not resolve to an array of records.")
|
|
346
|
+
if not all(isinstance(item, dict) for item in value):
|
|
347
|
+
raise FetchError(f"{label} resolved to an array, but not every item is an object record.")
|
|
348
|
+
return list(value)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _next_cursor(payload: Any) -> str | None:
|
|
352
|
+
if not isinstance(payload, dict):
|
|
353
|
+
return None
|
|
354
|
+
for key in ("next_cursor", "nextCursor", "cursor", "next"):
|
|
355
|
+
value = payload.get(key)
|
|
356
|
+
if isinstance(value, str) and value:
|
|
357
|
+
return value
|
|
358
|
+
pagination = payload.get("pagination")
|
|
359
|
+
if isinstance(pagination, dict):
|
|
360
|
+
for key in ("next_cursor", "nextCursor", "cursor", "next"):
|
|
361
|
+
value = pagination.get(key)
|
|
362
|
+
if isinstance(value, str) and value:
|
|
363
|
+
return value
|
|
364
|
+
return None
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _apply_post_fetch_incremental(
|
|
368
|
+
rows: list[dict[str, Any]],
|
|
369
|
+
spec: RestApiSpec,
|
|
370
|
+
last_watermark: str | None,
|
|
371
|
+
) -> list[dict[str, Any]]:
|
|
372
|
+
incremental = spec.incremental
|
|
373
|
+
if incremental is None or incremental.strategy != "post_fetch_filter" or last_watermark is None:
|
|
374
|
+
return rows
|
|
375
|
+
column = incremental.incremental_column
|
|
376
|
+
return [row for row in rows if _watermark_gt(row.get(column), last_watermark)]
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _high_watermark(rows: list[dict[str, Any]], spec: RestApiSpec) -> str | None:
|
|
380
|
+
incremental = spec.incremental
|
|
381
|
+
if incremental is None:
|
|
382
|
+
return None
|
|
383
|
+
chosen: Any | None = None
|
|
384
|
+
for row in rows:
|
|
385
|
+
value = row.get(incremental.incremental_column)
|
|
386
|
+
if value is None:
|
|
387
|
+
continue
|
|
388
|
+
if chosen is None or _watermark_compare(value, chosen) > 0:
|
|
389
|
+
chosen = value
|
|
390
|
+
return str(chosen) if chosen is not None else None
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _watermark_gt(value: Any, last_watermark: str) -> bool:
|
|
394
|
+
if value is None:
|
|
395
|
+
return False
|
|
396
|
+
return _watermark_compare(value, last_watermark) > 0
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _watermark_compare(left: Any, right: Any) -> int:
|
|
400
|
+
left_number = _coerce_finite_number(left)
|
|
401
|
+
right_number = _coerce_finite_number(right)
|
|
402
|
+
if left_number is not None and right_number is not None:
|
|
403
|
+
return _cmp(left_number, right_number)
|
|
404
|
+
|
|
405
|
+
left_datetime = _coerce_iso_datetime(left)
|
|
406
|
+
right_datetime = _coerce_iso_datetime(right)
|
|
407
|
+
if left_datetime is not None and right_datetime is not None:
|
|
408
|
+
return _cmp(left_datetime, right_datetime)
|
|
409
|
+
|
|
410
|
+
return _cmp(str(left), str(right))
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def _cmp(left: Any, right: Any) -> int:
|
|
414
|
+
if left > right:
|
|
415
|
+
return 1
|
|
416
|
+
if left < right:
|
|
417
|
+
return -1
|
|
418
|
+
return 0
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _coerce_finite_number(value: Any) -> int | Decimal | None:
|
|
422
|
+
if isinstance(value, bool):
|
|
423
|
+
return None
|
|
424
|
+
if isinstance(value, int):
|
|
425
|
+
return value
|
|
426
|
+
if isinstance(value, Decimal):
|
|
427
|
+
return value if value.is_finite() else None
|
|
428
|
+
if isinstance(value, float):
|
|
429
|
+
if not math.isfinite(value):
|
|
430
|
+
return None
|
|
431
|
+
return Decimal(str(value))
|
|
432
|
+
if isinstance(value, str):
|
|
433
|
+
text = value.strip()
|
|
434
|
+
if not text:
|
|
435
|
+
return None
|
|
436
|
+
if _INTEGER_TEXT_RE.fullmatch(text):
|
|
437
|
+
# Watermark ids are ordered by numeric value; preserve the original
|
|
438
|
+
# row value for the stored high-watermark string.
|
|
439
|
+
return int(text)
|
|
440
|
+
try:
|
|
441
|
+
number = Decimal(text)
|
|
442
|
+
except InvalidOperation:
|
|
443
|
+
return None
|
|
444
|
+
return number if number.is_finite() else None
|
|
445
|
+
return None
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _coerce_iso_datetime(value: Any) -> datetime | None:
|
|
449
|
+
if not isinstance(value, str):
|
|
450
|
+
return None
|
|
451
|
+
text = value.strip()
|
|
452
|
+
if not text:
|
|
453
|
+
return None
|
|
454
|
+
try:
|
|
455
|
+
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
456
|
+
except ValueError:
|
|
457
|
+
return None
|
|
458
|
+
if parsed.tzinfo is not None:
|
|
459
|
+
parsed = parsed.astimezone(UTC).replace(tzinfo=None)
|
|
460
|
+
return parsed
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _resolve_storage(*, storage: LandingStorage | None, bronze_root: Path | None) -> LandingStorage:
|
|
464
|
+
"""Pick the storage handle: explicit ``storage`` wins, otherwise
|
|
465
|
+
construct a :class:`LocalLandingStorage` from ``bronze_root`` for
|
|
466
|
+
back-compat with pre-ADR-0046 callers.
|
|
467
|
+
"""
|
|
468
|
+
if storage is not None:
|
|
469
|
+
return storage
|
|
470
|
+
if bronze_root is None:
|
|
471
|
+
raise ValueError("fetcher requires either ``storage`` or ``bronze_root``")
|
|
472
|
+
return LocalLandingStorage(bronze_root.resolve())
|