contractforge-aws 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- contractforge_aws-0.1.0/.gitignore +18 -0
- contractforge_aws-0.1.0/CHANGELOG.md +15 -0
- contractforge_aws-0.1.0/PKG-INFO +241 -0
- contractforge_aws-0.1.0/README.md +210 -0
- contractforge_aws-0.1.0/pyproject.toml +46 -0
- contractforge_aws-0.1.0/src/contractforge_aws/__init__.py +126 -0
- contractforge_aws-0.1.0/src/contractforge_aws/adapter.py +62 -0
- contractforge_aws-0.1.0/src/contractforge_aws/annotations/__init__.py +13 -0
- contractforge_aws-0.1.0/src/contractforge_aws/annotations/api.py +54 -0
- contractforge_aws-0.1.0/src/contractforge_aws/annotations/rendering.py +209 -0
- contractforge_aws-0.1.0/src/contractforge_aws/annotations/runtime.py +168 -0
- contractforge_aws-0.1.0/src/contractforge_aws/api.py +214 -0
- contractforge_aws-0.1.0/src/contractforge_aws/capabilities/__init__.py +5 -0
- contractforge_aws-0.1.0/src/contractforge_aws/capabilities/glue_iceberg.py +45 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/__init__.py +62 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/apply.py +91 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/deploy.py +53 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/glue.py +167 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/performance.py +65 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/plan.py +85 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/project.py +194 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/project_cleanup.py +176 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/project_cost.py +139 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/project_orchestration.py +208 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/project_orchestration_cost.py +75 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/project_run.py +108 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/project_step.py +62 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/project_support.py +135 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/project_validation.py +49 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/runtime.py +120 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/smoke.py +46 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/stabilization.py +129 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cli/support.py +124 -0
- contractforge_aws-0.1.0/src/contractforge_aws/contract_extensions.py +100 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cost/__init__.py +6 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cost/model.py +19 -0
- contractforge_aws-0.1.0/src/contractforge_aws/cost/sql.py +132 -0
- contractforge_aws-0.1.0/src/contractforge_aws/deployment_api.py +84 -0
- contractforge_aws-0.1.0/src/contractforge_aws/diagnostics/__init__.py +5 -0
- contractforge_aws-0.1.0/src/contractforge_aws/diagnostics/portability.py +153 -0
- contractforge_aws-0.1.0/src/contractforge_aws/environment.py +53 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/__init__.py +70 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/athena_ddl.py +127 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/database.py +10 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/ddl.py +139 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/error_runtime.py +102 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/failure_runtime.py +82 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/glue.py +128 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/metadata_runtime.py +88 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/run_context_runtime.py +51 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/run_helper_runtime.py +47 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/run_metadata.py +42 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/run_success_runtime.py +85 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/runtime.py +31 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/source.py +41 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/sql.py +99 -0
- contractforge_aws-0.1.0/src/contractforge_aws/evidence/stream_runtime.py +136 -0
- contractforge_aws-0.1.0/src/contractforge_aws/glue_job_definition.py +117 -0
- contractforge_aws-0.1.0/src/contractforge_aws/governance/__init__.py +15 -0
- contractforge_aws-0.1.0/src/contractforge_aws/governance/evidence.py +190 -0
- contractforge_aws-0.1.0/src/contractforge_aws/governance/lakeformation.py +199 -0
- contractforge_aws-0.1.0/src/contractforge_aws/governance/runtime.py +114 -0
- contractforge_aws-0.1.0/src/contractforge_aws/lineage/__init__.py +16 -0
- contractforge_aws-0.1.0/src/contractforge_aws/lineage/openlineage.py +178 -0
- contractforge_aws-0.1.0/src/contractforge_aws/lineage/runtime.py +110 -0
- contractforge_aws-0.1.0/src/contractforge_aws/operations/__init__.py +9 -0
- contractforge_aws-0.1.0/src/contractforge_aws/operations/sql.py +128 -0
- contractforge_aws-0.1.0/src/contractforge_aws/orchestration/__init__.py +13 -0
- contractforge_aws-0.1.0/src/contractforge_aws/orchestration/execution_name.py +15 -0
- contractforge_aws-0.1.0/src/contractforge_aws/orchestration/project_graph.py +55 -0
- contractforge_aws-0.1.0/src/contractforge_aws/orchestration/scheduler.py +99 -0
- contractforge_aws-0.1.0/src/contractforge_aws/orchestration/stepfunctions.py +145 -0
- contractforge_aws-0.1.0/src/contractforge_aws/performance/__init__.py +6 -0
- contractforge_aws-0.1.0/src/contractforge_aws/performance/profile.py +97 -0
- contractforge_aws-0.1.0/src/contractforge_aws/performance/sql.py +95 -0
- contractforge_aws-0.1.0/src/contractforge_aws/preparation/__init__.py +19 -0
- contractforge_aws-0.1.0/src/contractforge_aws/preparation/arrays.py +67 -0
- contractforge_aws-0.1.0/src/contractforge_aws/preparation/flatten.py +95 -0
- contractforge_aws-0.1.0/src/contractforge_aws/preparation/metadata.py +77 -0
- contractforge_aws-0.1.0/src/contractforge_aws/preparation/rendering.py +81 -0
- contractforge_aws-0.1.0/src/contractforge_aws/preparation/shape.py +175 -0
- contractforge_aws-0.1.0/src/contractforge_aws/preparation/transform.py +198 -0
- contractforge_aws-0.1.0/src/contractforge_aws/preparation/utils.py +17 -0
- contractforge_aws-0.1.0/src/contractforge_aws/quality/__init__.py +27 -0
- contractforge_aws-0.1.0/src/contractforge_aws/quality/dqdl.py +178 -0
- contractforge_aws-0.1.0/src/contractforge_aws/quality/enforcement.py +46 -0
- contractforge_aws-0.1.0/src/contractforge_aws/quality/expression.py +114 -0
- contractforge_aws-0.1.0/src/contractforge_aws/quality/quarantine.py +56 -0
- contractforge_aws-0.1.0/src/contractforge_aws/quality/runtime.py +173 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/__init__.py +51 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/artifact_registry.py +167 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/cloudformation.py +112 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/deployment.py +209 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/error_handler.py +38 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/glue_job.py +186 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/glue_job_common.py +158 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/glue_job_outline.py +111 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/iam.py +172 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/iam_s3.py +86 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/iceberg_config.py +64 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/library_runner.py +26 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/manifest.py +214 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/manifest_boundaries.py +84 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/manifest_size.py +28 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/names.py +34 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/review.py +161 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/streaming_job.py +210 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/terraform.py +82 -0
- contractforge_aws-0.1.0/src/contractforge_aws/rendering/write_mode_review.py +84 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/__init__.py +122 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/api.py +194 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/athena.py +136 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/audit.py +92 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/dependencies.py +16 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/deploy.py +94 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/evidence.py +96 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/evidence_api.py +25 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/glue_job_payload.py +58 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/glue_jobs.py +168 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/glue_wait.py +34 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/lakeformation_api.py +53 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/library_runner.py +208 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/operations.py +33 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/operations_api.py +33 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/orchestration.py +189 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/publishable.py +60 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime/s3_artifacts.py +151 -0
- contractforge_aws-0.1.0/src/contractforge_aws/runtime_args.py +7 -0
- contractforge_aws-0.1.0/src/contractforge_aws/schema/__init__.py +9 -0
- contractforge_aws-0.1.0/src/contractforge_aws/schema/runtime.py +105 -0
- contractforge_aws-0.1.0/src/contractforge_aws/schema_columns.py +7 -0
- contractforge_aws-0.1.0/src/contractforge_aws/security/__init__.py +32 -0
- contractforge_aws-0.1.0/src/contractforge_aws/security/http_safety.py +9 -0
- contractforge_aws-0.1.0/src/contractforge_aws/security/secrets.py +141 -0
- contractforge_aws-0.1.0/src/contractforge_aws/security/source_policy.py +105 -0
- contractforge_aws-0.1.0/src/contractforge_aws/smoke/__init__.py +6 -0
- contractforge_aws-0.1.0/src/contractforge_aws/smoke/environment.py +139 -0
- contractforge_aws-0.1.0/src/contractforge_aws/smoke/minimal.py +45 -0
- contractforge_aws-0.1.0/src/contractforge_aws/smoke/models.py +64 -0
- contractforge_aws-0.1.0/src/contractforge_aws/smoke/runner.py +84 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/__init__.py +40 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/classification.py +215 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/http_file.py +184 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/interpret.py +105 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/jdbc.py +122 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/native_passthrough.py +165 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/native_passthrough_candidates.py +125 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/rds_iam.py +74 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/rest_api.py +60 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/spark.py +197 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/streams.py +61 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/support.py +33 -0
- contractforge_aws-0.1.0/src/contractforge_aws/sources/table_refs.py +77 -0
- contractforge_aws-0.1.0/src/contractforge_aws/state/__init__.py +5 -0
- contractforge_aws-0.1.0/src/contractforge_aws/state/runtime.py +112 -0
- contractforge_aws-0.1.0/src/contractforge_aws/subtargets.py +27 -0
- contractforge_aws-0.1.0/src/contractforge_aws/validation.py +10 -0
- contractforge_aws-0.1.0/src/contractforge_aws/write_modes/__init__.py +5 -0
- contractforge_aws-0.1.0/src/contractforge_aws/write_modes/hash_diff.py +134 -0
- contractforge_aws-0.1.0/src/contractforge_aws/write_modes/iceberg.py +151 -0
- contractforge_aws-0.1.0/src/contractforge_aws/write_modes/writer.py +53 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `contractforge-aws` are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format follows Keep a Changelog, and this package follows semantic
|
|
6
|
+
versioning as described in `../../docs/specs/api-stability.md`.
|
|
7
|
+
|
|
8
|
+
## [0.1.0] - 2026-06-08
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Initial public alpha release of the AWS adapter.
|
|
13
|
+
- AWS Glue/Iceberg planning, artifact rendering, quality, evidence, lineage,
|
|
14
|
+
RDS IAM, REST/HTTP source handling, orchestration and Athena-facing runtime
|
|
15
|
+
helpers.
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: contractforge-aws
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: AWS adapter for ContractForge Core.
|
|
5
|
+
Project-URL: Homepage, https://github.com/marquesantero/contractforge-core/tree/main/adapters/aws
|
|
6
|
+
Project-URL: Documentation, https://marquesantero.github.io/contractforge-core/docs/adapters/aws
|
|
7
|
+
Project-URL: Repository, https://github.com/marquesantero/contractforge-core
|
|
8
|
+
Project-URL: Issues, https://github.com/marquesantero/contractforge-core/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/marquesantero/contractforge-core/blob/main/adapters/aws/CHANGELOG.md
|
|
10
|
+
Author: ContractForge contributors
|
|
11
|
+
License: MIT
|
|
12
|
+
Keywords: aws,contractforge,data-contracts,glue,iceberg,ingestion
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Database
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: contractforge-core<0.2,>=0.1
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
27
|
+
Provides-Extra: runtime
|
|
28
|
+
Requires-Dist: boto3>=1.34; extra == 'runtime'
|
|
29
|
+
Requires-Dist: botocore[crt]>=1.34; extra == 'runtime'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# ContractForge AWS Adapter
|
|
33
|
+
|
|
34
|
+
`contractforge-aws` is the AWS adapter package for ContractForge.
|
|
35
|
+
|
|
36
|
+
The initial target is `aws_glue_iceberg`: AWS Glue Spark jobs writing Apache Iceberg tables in Amazon S3, cataloged through AWS Glue Data Catalog and governed by AWS Lake Formation.
|
|
37
|
+
|
|
38
|
+
The base package declares capabilities, calls the core planner and renders review artifacts. It also renders first-pass AWS Glue Spark scripts for `scd0_append`, `scd0_overwrite`, `scd1_upsert` and `scd1_hash_diff`.
|
|
39
|
+
|
|
40
|
+
JDBC credentials are never baked into generated scripts: `{{ secret:scope/key }}` placeholders render an AWS Secrets Manager lookup that resolves when the Glue job runs, and inline passwords are refused. JDBC `auth.type: rds_iam` renders a runtime RDS IAM token (`rds.generate_db_auth_token`) instead of a static password.
|
|
41
|
+
|
|
42
|
+
The default install intentionally does not depend on `boto3`. Runtime/apply commands that call AWS APIs will live behind the optional `runtime` extra.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install contractforge-core contractforge-aws
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
For AWS API helpers such as S3 artifact publishing:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install "contractforge-aws[runtime]"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Minimal usage
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from contractforge_aws import plan_aws_contract, render_aws_contract
|
|
60
|
+
|
|
61
|
+
contract = {
|
|
62
|
+
"source": {"type": "s3", "path": "s3://landing/orders", "format": "parquet"},
|
|
63
|
+
"target": {"catalog": "glue", "schema": "bronze", "table": "orders"},
|
|
64
|
+
"mode": "scd0_append",
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
result = plan_aws_contract(contract)
|
|
68
|
+
print(result.status)
|
|
69
|
+
|
|
70
|
+
artifacts = render_aws_contract(contract)
|
|
71
|
+
print(artifacts.artifacts["glue_bronze_orders.glue_job.py"])
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
The SCD1 upsert script uses Iceberg `MERGE INTO` and validates missing, null and duplicate merge keys before executing the merge. Rendered Glue job definitions include the adapter-owned Iceberg Spark extension startup configuration through `--conf`; user-provided Glue `default_arguments` cannot override this managed argument.
|
|
75
|
+
|
|
76
|
+
The runtime renderer preserves top-level `select_columns`, `column_mapping`, `filter_expression`, portable transforms for `cast`, `standardize`, `derive`, `composite_keys` and `deduplicate`, plus supported `shape` sections (`parse_json`, arrays, columns and flattening). If a section cannot be preserved faithfully, the adapter emits a review artifact instead of a runnable Glue job.
|
|
77
|
+
|
|
78
|
+
Quality rules are evaluated in-job. Rules with faithful AWS Glue Data Quality equivalents (`required_columns`, `not_null`, `unique_key`, `row_count_minimum`, `accepted_values`, `max_null_ratio`) use `EvaluateDataQuality` against a DQDL ruleset. `expression` rules use Spark SQL DataFrame filters because they have no faithful DQDL mapping. Enforcement stays consistent: `abort` rules raise and fail the run; `warn` rules are recorded and continue; row-level `quarantine` rules write offending rows to `ctrl_ingestion_quarantine`, remove them before the target write and record quality evidence. Every evaluated rule appends one immutable row to `ctrl_ingestion_quality`.
|
|
79
|
+
|
|
80
|
+
Portable quality rules are also rendered as an AWS Glue Data Quality DQDL ruleset (`*.quality.dqdl`) for native evaluation:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from contractforge_aws import render_aws_quality_dqdl
|
|
84
|
+
|
|
85
|
+
dqdl = render_aws_quality_dqdl(contract)
|
|
86
|
+
print(dqdl) # Rules = [ ColumnExists "order_id", IsUnique "order_id", ... ]
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`required_columns`, `not_null`, `unique_key`, `accepted_values`, `row_count_minimum` and `max_null_ratio` map to DQDL rules; `expression` rules are reported as unmapped rather than approximated.
|
|
90
|
+
|
|
91
|
+
The `access` section renders Lake Formation review/apply artifacts (`*.lakeformation.json`):
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from contractforge_aws import render_aws_lake_formation_plan
|
|
95
|
+
|
|
96
|
+
plan = render_aws_lake_formation_plan(contract)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`access.grants` become applyable `GrantPermissions` requests. `access.row_filters` and `access.column_masks` render `CreateDataCellsFilter` scaffolds: row filters are fail-closed (`false`) because Lake Formation uses a SQL `FilterExpression`, not the contract's row-filter function; column masks exclude the column (LF has no value-masking function). These two stay `REVIEW_REQUIRED` in planning.
|
|
100
|
+
|
|
101
|
+
When Lake Formation artifacts are rendered, the adapter also renders `*.lakeformation_evidence.sql` for `ctrl_ingestion_access`: grants are recorded as `PLANNED`, while row-filter and column-mask scaffolds are recorded as `REVIEW_REQUIRED` until a reviewer completes the Lake Formation expression/design.
|
|
102
|
+
|
|
103
|
+
## Publish artifacts to S3
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from contractforge_aws import publish_aws_contract_artifacts_to_s3
|
|
107
|
+
|
|
108
|
+
published = publish_aws_contract_artifacts_to_s3(
|
|
109
|
+
contract,
|
|
110
|
+
bucket="contractforge-artifacts",
|
|
111
|
+
prefix="dev/orders",
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
print([item.uri for item in published])
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
CLI:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
contractforge-aws publish-s3 contract.yaml --bucket contractforge-artifacts --prefix dev/orders
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Register a Glue job
|
|
124
|
+
|
|
125
|
+
After publishing the generated `.glue_job.py` artifact to S3, register or update an AWS Glue job definition:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
from contractforge_aws import register_aws_glue_job
|
|
129
|
+
|
|
130
|
+
registered = register_aws_glue_job(
|
|
131
|
+
job_name="cf-orders",
|
|
132
|
+
role_arn="arn:aws:iam::123456789012:role/ContractForgeGlueRole",
|
|
133
|
+
script_s3_uri="s3://contractforge-artifacts/dev/orders/glue_bronze_orders.glue_job.py",
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
print(registered.action)
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
CLI:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
contractforge-aws register-glue-job --job-name cf-orders --role-arn arn:aws:iam::123456789012:role/ContractForgeGlueRole --script-s3-uri s3://contractforge-artifacts/dev/orders/glue_bronze_orders.glue_job.py
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## Start and inspect a Glue job run
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
from contractforge_aws import get_aws_glue_job_run_status, start_aws_glue_job_run
|
|
149
|
+
|
|
150
|
+
run = start_aws_glue_job_run(
|
|
151
|
+
job_name="cf-orders",
|
|
152
|
+
arguments={"--contractforge-run-id": "run-123"},
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
status = get_aws_glue_job_run_status(job_name="cf-orders", run_id=run.run_id)
|
|
156
|
+
print(status.state)
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
Starting a job is intentionally separate from the post-hoc reconciliation API. The rendered Glue job itself writes evidence in-job: after the Iceberg write it reads the target snapshot (table version + summary), records AWS state as append-only observations in `ctrl_ingestion_state`, appends source metadata to `ctrl_ingestion_metadata`, appends an OpenLineage-compatible event to `ctrl_ingestion_lineage`, and only then appends the final successful row to `ctrl_ingestion_runs`. This prevents failed post-write evidence steps from leaving false successful run rows. Run evidence fills platform-neutral columns (`source_*`, `rows_read`, `rows_written` from `added-records`, `table_version_after` from the snapshot id, `operation_metrics_json`, `runtime_type`, `runtime_entrypoint`, engine/Python versions, etc.) per the [evidence mapping matrix](../../docs/specs/evidence-mapping-matrix.md). Available-now streaming jobs write per-micro-batch rows to `ctrl_ingestion_streams` and roll those totals into final run evidence. If the Glue script fails, it writes one row to `ctrl_ingestion_errors` and re-raises. The state, run, metadata, lineage, stream and error control tables are created (`CREATE TABLE IF NOT EXISTS`) by the job if missing.
|
|
160
|
+
|
|
161
|
+
## Reconcile Glue run evidence
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
from contractforge_aws import reconcile_aws_glue_job_run_evidence
|
|
165
|
+
|
|
166
|
+
evidence = reconcile_aws_glue_job_run_evidence(
|
|
167
|
+
job_name="cf-orders",
|
|
168
|
+
run_id=run.run_id,
|
|
169
|
+
target_table="glue.bronze.orders",
|
|
170
|
+
mode="scd0_append",
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
print(evidence.run.status)
|
|
174
|
+
print(evidence.cost)
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
This maps Glue `JobRun` metadata into core evidence record objects. It does not persist control-table rows yet.
|
|
178
|
+
|
|
179
|
+
To render Iceberg `INSERT` statements for review or an explicit apply step:
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
from contractforge_aws import render_aws_glue_job_run_evidence_sql
|
|
183
|
+
|
|
184
|
+
sql = render_aws_glue_job_run_evidence_sql(
|
|
185
|
+
job_name="cf-orders",
|
|
186
|
+
run_id=run.run_id,
|
|
187
|
+
target_table="glue.bronze.orders",
|
|
188
|
+
mode="scd0_append",
|
|
189
|
+
database="contractforge_ops",
|
|
190
|
+
)
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
For query-only operational cost reporting over ContractForge evidence tables:
|
|
194
|
+
|
|
195
|
+
```python
|
|
196
|
+
from contractforge_aws import CostModel, render_aws_operational_cost_query
|
|
197
|
+
|
|
198
|
+
query = render_aws_operational_cost_query(
|
|
199
|
+
database="lake_bronze_ops",
|
|
200
|
+
cost_model=CostModel(dpu_hour_usd=0.44),
|
|
201
|
+
)
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
The query estimates cost only from `ctrl_ingestion_cost.signal_name = 'glue_dpu_seconds'`. If no explicit DPU-hour rate is supplied, cost fields stay `NULL`.
|
|
205
|
+
|
|
206
|
+
## Minimal AWS smoke test
|
|
207
|
+
|
|
208
|
+
The adapter includes a cost-gated smoke runner for the smallest real AWS validation path:
|
|
209
|
+
|
|
210
|
+
- creates/uses a tagged S3 bucket;
|
|
211
|
+
- uploads a tiny JSON input file;
|
|
212
|
+
- creates/uses a tagged Glue IAM role;
|
|
213
|
+
- renders and publishes ContractForge AWS artifacts through the adapter;
|
|
214
|
+
- registers a Glue Spark/Iceberg job;
|
|
215
|
+
- optionally starts one `scd0_overwrite` run.
|
|
216
|
+
|
|
217
|
+
Dry-run is the default and does not call AWS:
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
contractforge-aws smoke-minimal \
|
|
221
|
+
--account-id 123456789012 \
|
|
222
|
+
--bucket contractforge-aws-smoke-123456789012-us-east-1 \
|
|
223
|
+
--max-estimated-cost-usd 1.00
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
Real execution requires both `--execute` and a cost ceiling that covers the configured timeout:
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
contractforge-aws smoke-minimal \
|
|
230
|
+
--account-id 123456789012 \
|
|
231
|
+
--bucket contractforge-aws-smoke-123456789012-us-east-1 \
|
|
232
|
+
--max-estimated-cost-usd 1.00 \
|
|
233
|
+
--execute \
|
|
234
|
+
--wait
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
The default ceiling estimate is based on 2 `G.1X` workers, 10 minutes, and `$0.44` per DPU-hour. The actual successful smoke run is normally much cheaper, but the guardrail uses timeout ceiling rather than optimistic runtime.
|
|
238
|
+
|
|
239
|
+
## Scope
|
|
240
|
+
|
|
241
|
+
See `docs/specs/aws-adapter.md` and `docs/adapters/aws.md` in the repository root.
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# ContractForge AWS Adapter
|
|
2
|
+
|
|
3
|
+
`contractforge-aws` is the AWS adapter package for ContractForge.
|
|
4
|
+
|
|
5
|
+
The initial target is `aws_glue_iceberg`: AWS Glue Spark jobs writing Apache Iceberg tables in Amazon S3, cataloged through AWS Glue Data Catalog and governed by AWS Lake Formation.
|
|
6
|
+
|
|
7
|
+
The base package declares capabilities, calls the core planner and renders review artifacts. It also renders first-pass AWS Glue Spark scripts for `scd0_append`, `scd0_overwrite`, `scd1_upsert` and `scd1_hash_diff`.
|
|
8
|
+
|
|
9
|
+
JDBC credentials are never baked into generated scripts: `{{ secret:scope/key }}` placeholders render an AWS Secrets Manager lookup that resolves when the Glue job runs, and inline passwords are refused. JDBC `auth.type: rds_iam` renders a runtime RDS IAM token (`rds.generate_db_auth_token`) instead of a static password.
|
|
10
|
+
|
|
11
|
+
The default install intentionally does not depend on `boto3`. Runtime/apply commands that call AWS APIs will live behind the optional `runtime` extra.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install contractforge-core contractforge-aws
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
For AWS API helpers such as S3 artifact publishing:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install "contractforge-aws[runtime]"
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Minimal usage
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
from contractforge_aws import plan_aws_contract, render_aws_contract
|
|
29
|
+
|
|
30
|
+
contract = {
|
|
31
|
+
"source": {"type": "s3", "path": "s3://landing/orders", "format": "parquet"},
|
|
32
|
+
"target": {"catalog": "glue", "schema": "bronze", "table": "orders"},
|
|
33
|
+
"mode": "scd0_append",
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
result = plan_aws_contract(contract)
|
|
37
|
+
print(result.status)
|
|
38
|
+
|
|
39
|
+
artifacts = render_aws_contract(contract)
|
|
40
|
+
print(artifacts.artifacts["glue_bronze_orders.glue_job.py"])
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The SCD1 upsert script uses Iceberg `MERGE INTO` and validates missing, null and duplicate merge keys before executing the merge. Rendered Glue job definitions include the adapter-owned Iceberg Spark extension startup configuration through `--conf`; user-provided Glue `default_arguments` cannot override this managed argument.
|
|
44
|
+
|
|
45
|
+
The runtime renderer preserves top-level `select_columns`, `column_mapping`, `filter_expression`, portable transforms for `cast`, `standardize`, `derive`, `composite_keys` and `deduplicate`, plus supported `shape` sections (`parse_json`, arrays, columns and flattening). If a section cannot be preserved faithfully, the adapter emits a review artifact instead of a runnable Glue job.
|
|
46
|
+
|
|
47
|
+
Quality rules are evaluated in-job. Rules with faithful AWS Glue Data Quality equivalents (`required_columns`, `not_null`, `unique_key`, `row_count_minimum`, `accepted_values`, `max_null_ratio`) use `EvaluateDataQuality` against a DQDL ruleset. `expression` rules use Spark SQL DataFrame filters because they have no faithful DQDL mapping. Enforcement stays consistent: `abort` rules raise and fail the run; `warn` rules are recorded and continue; row-level `quarantine` rules write offending rows to `ctrl_ingestion_quarantine`, remove them before the target write and record quality evidence. Every evaluated rule appends one immutable row to `ctrl_ingestion_quality`.
|
|
48
|
+
|
|
49
|
+
Portable quality rules are also rendered as an AWS Glue Data Quality DQDL ruleset (`*.quality.dqdl`) for native evaluation:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from contractforge_aws import render_aws_quality_dqdl
|
|
53
|
+
|
|
54
|
+
dqdl = render_aws_quality_dqdl(contract)
|
|
55
|
+
print(dqdl) # Rules = [ ColumnExists "order_id", IsUnique "order_id", ... ]
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
`required_columns`, `not_null`, `unique_key`, `accepted_values`, `row_count_minimum` and `max_null_ratio` map to DQDL rules; `expression` rules are reported as unmapped rather than approximated.
|
|
59
|
+
|
|
60
|
+
The `access` section renders Lake Formation review/apply artifacts (`*.lakeformation.json`):
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from contractforge_aws import render_aws_lake_formation_plan
|
|
64
|
+
|
|
65
|
+
plan = render_aws_lake_formation_plan(contract)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`access.grants` become applyable `GrantPermissions` requests. `access.row_filters` and `access.column_masks` render `CreateDataCellsFilter` scaffolds: row filters are fail-closed (`false`) because Lake Formation uses a SQL `FilterExpression`, not the contract's row-filter function; column masks exclude the column (LF has no value-masking function). These two stay `REVIEW_REQUIRED` in planning.
|
|
69
|
+
|
|
70
|
+
When Lake Formation artifacts are rendered, the adapter also renders `*.lakeformation_evidence.sql` for `ctrl_ingestion_access`: grants are recorded as `PLANNED`, while row-filter and column-mask scaffolds are recorded as `REVIEW_REQUIRED` until a reviewer completes the Lake Formation expression/design.
|
|
71
|
+
|
|
72
|
+
## Publish artifacts to S3
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from contractforge_aws import publish_aws_contract_artifacts_to_s3
|
|
76
|
+
|
|
77
|
+
published = publish_aws_contract_artifacts_to_s3(
|
|
78
|
+
contract,
|
|
79
|
+
bucket="contractforge-artifacts",
|
|
80
|
+
prefix="dev/orders",
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
print([item.uri for item in published])
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
CLI:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
contractforge-aws publish-s3 contract.yaml --bucket contractforge-artifacts --prefix dev/orders
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Register a Glue job
|
|
93
|
+
|
|
94
|
+
After publishing the generated `.glue_job.py` artifact to S3, register or update an AWS Glue job definition:
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from contractforge_aws import register_aws_glue_job
|
|
98
|
+
|
|
99
|
+
registered = register_aws_glue_job(
|
|
100
|
+
job_name="cf-orders",
|
|
101
|
+
role_arn="arn:aws:iam::123456789012:role/ContractForgeGlueRole",
|
|
102
|
+
script_s3_uri="s3://contractforge-artifacts/dev/orders/glue_bronze_orders.glue_job.py",
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
print(registered.action)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
CLI:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
contractforge-aws register-glue-job --job-name cf-orders --role-arn arn:aws:iam::123456789012:role/ContractForgeGlueRole --script-s3-uri s3://contractforge-artifacts/dev/orders/glue_bronze_orders.glue_job.py
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## Start and inspect a Glue job run
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from contractforge_aws import get_aws_glue_job_run_status, start_aws_glue_job_run
|
|
118
|
+
|
|
119
|
+
run = start_aws_glue_job_run(
|
|
120
|
+
job_name="cf-orders",
|
|
121
|
+
arguments={"--contractforge-run-id": "run-123"},
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
status = get_aws_glue_job_run_status(job_name="cf-orders", run_id=run.run_id)
|
|
125
|
+
print(status.state)
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Starting a job is intentionally separate from the post-hoc reconciliation API. The rendered Glue job itself writes evidence in-job: after the Iceberg write it reads the target snapshot (table version + summary), records AWS state as append-only observations in `ctrl_ingestion_state`, appends source metadata to `ctrl_ingestion_metadata`, appends an OpenLineage-compatible event to `ctrl_ingestion_lineage`, and only then appends the final successful row to `ctrl_ingestion_runs`. This prevents failed post-write evidence steps from leaving false successful run rows. Run evidence fills platform-neutral columns (`source_*`, `rows_read`, `rows_written` from `added-records`, `table_version_after` from the snapshot id, `operation_metrics_json`, `runtime_type`, `runtime_entrypoint`, engine/Python versions, etc.) per the [evidence mapping matrix](../../docs/specs/evidence-mapping-matrix.md). Available-now streaming jobs write per-micro-batch rows to `ctrl_ingestion_streams` and roll those totals into final run evidence. If the Glue script fails, it writes one row to `ctrl_ingestion_errors` and re-raises. The state, run, metadata, lineage, stream and error control tables are created (`CREATE TABLE IF NOT EXISTS`) by the job if missing.
|
|
129
|
+
|
|
130
|
+
## Reconcile Glue run evidence
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from contractforge_aws import reconcile_aws_glue_job_run_evidence
|
|
134
|
+
|
|
135
|
+
evidence = reconcile_aws_glue_job_run_evidence(
|
|
136
|
+
job_name="cf-orders",
|
|
137
|
+
run_id=run.run_id,
|
|
138
|
+
target_table="glue.bronze.orders",
|
|
139
|
+
mode="scd0_append",
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
print(evidence.run.status)
|
|
143
|
+
print(evidence.cost)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
This maps Glue `JobRun` metadata into core evidence record objects. It does not persist control-table rows yet.
|
|
147
|
+
|
|
148
|
+
To render Iceberg `INSERT` statements for review or an explicit apply step:
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
from contractforge_aws import render_aws_glue_job_run_evidence_sql
|
|
152
|
+
|
|
153
|
+
sql = render_aws_glue_job_run_evidence_sql(
|
|
154
|
+
job_name="cf-orders",
|
|
155
|
+
run_id=run.run_id,
|
|
156
|
+
target_table="glue.bronze.orders",
|
|
157
|
+
mode="scd0_append",
|
|
158
|
+
database="contractforge_ops",
|
|
159
|
+
)
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
For query-only operational cost reporting over ContractForge evidence tables:
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
from contractforge_aws import CostModel, render_aws_operational_cost_query
|
|
166
|
+
|
|
167
|
+
query = render_aws_operational_cost_query(
|
|
168
|
+
database="lake_bronze_ops",
|
|
169
|
+
cost_model=CostModel(dpu_hour_usd=0.44),
|
|
170
|
+
)
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
The query estimates cost only from `ctrl_ingestion_cost.signal_name = 'glue_dpu_seconds'`. If no explicit DPU-hour rate is supplied, cost fields stay `NULL`.
|
|
174
|
+
|
|
175
|
+
## Minimal AWS smoke test
|
|
176
|
+
|
|
177
|
+
The adapter includes a cost-gated smoke runner for the smallest real AWS validation path:
|
|
178
|
+
|
|
179
|
+
- creates/uses a tagged S3 bucket;
|
|
180
|
+
- uploads a tiny JSON input file;
|
|
181
|
+
- creates/uses a tagged Glue IAM role;
|
|
182
|
+
- renders and publishes ContractForge AWS artifacts through the adapter;
|
|
183
|
+
- registers a Glue Spark/Iceberg job;
|
|
184
|
+
- optionally starts one `scd0_overwrite` run.
|
|
185
|
+
|
|
186
|
+
Dry-run is the default and does not call AWS:
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
contractforge-aws smoke-minimal \
|
|
190
|
+
--account-id 123456789012 \
|
|
191
|
+
--bucket contractforge-aws-smoke-123456789012-us-east-1 \
|
|
192
|
+
--max-estimated-cost-usd 1.00
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Real execution requires both `--execute` and a cost ceiling that covers the configured timeout:
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
contractforge-aws smoke-minimal \
|
|
199
|
+
--account-id 123456789012 \
|
|
200
|
+
--bucket contractforge-aws-smoke-123456789012-us-east-1 \
|
|
201
|
+
--max-estimated-cost-usd 1.00 \
|
|
202
|
+
--execute \
|
|
203
|
+
--wait
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
The default ceiling estimate is based on 2 `G.1X` workers, 10 minutes, and `$0.44` per DPU-hour. The actual successful smoke run is normally much cheaper, but the guardrail uses timeout ceiling rather than optimistic runtime.
|
|
207
|
+
|
|
208
|
+
## Scope
|
|
209
|
+
|
|
210
|
+
See `docs/specs/aws-adapter.md` and `docs/adapters/aws.md` in the repository root.
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "contractforge-aws"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "AWS adapter for ContractForge Core."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "ContractForge contributors" }]
|
|
13
|
+
keywords = ["contractforge", "aws", "glue", "iceberg", "data-contracts", "ingestion"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Developers",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Topic :: Database",
|
|
24
|
+
"Topic :: Software Development :: Libraries",
|
|
25
|
+
]
|
|
26
|
+
dependencies = ["contractforge-core>=0.1,<0.2"]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/marquesantero/contractforge-core/tree/main/adapters/aws"
|
|
30
|
+
Documentation = "https://marquesantero.github.io/contractforge-core/docs/adapters/aws"
|
|
31
|
+
Repository = "https://github.com/marquesantero/contractforge-core"
|
|
32
|
+
Issues = "https://github.com/marquesantero/contractforge-core/issues"
|
|
33
|
+
Changelog = "https://github.com/marquesantero/contractforge-core/blob/main/adapters/aws/CHANGELOG.md"
|
|
34
|
+
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
dev = ["pytest>=8"]
|
|
37
|
+
runtime = ["boto3>=1.34", "botocore[crt]>=1.34"]
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
contractforge-aws = "contractforge_aws.cli:main"
|
|
41
|
+
|
|
42
|
+
[tool.hatch.build.targets.wheel]
|
|
43
|
+
packages = ["src/contractforge_aws"]
|
|
44
|
+
|
|
45
|
+
[tool.hatch.build.targets.sdist]
|
|
46
|
+
include = ["/src/contractforge_aws", "/README.md", "/CHANGELOG.md", "/pyproject.toml"]
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Public API for the ContractForge AWS adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import PackageNotFoundError
|
|
6
|
+
from importlib.metadata import version as _version
|
|
7
|
+
|
|
8
|
+
from contractforge_aws.adapter import AWSAdapter
|
|
9
|
+
from contractforge_aws.api import (
|
|
10
|
+
plan_aws_contract,
|
|
11
|
+
render_aws_annotations_evidence_sql,
|
|
12
|
+
render_aws_annotations_plan,
|
|
13
|
+
render_aws_contract,
|
|
14
|
+
render_aws_deployment_manifest,
|
|
15
|
+
render_aws_glue_job_cloudformation,
|
|
16
|
+
render_aws_glue_job_definition,
|
|
17
|
+
render_aws_glue_job_iam_policy,
|
|
18
|
+
render_aws_glue_job_terraform,
|
|
19
|
+
render_aws_lake_formation_evidence_sql,
|
|
20
|
+
render_aws_lake_formation_plan,
|
|
21
|
+
render_aws_native_passthrough_plan,
|
|
22
|
+
render_aws_operations_evidence_sql,
|
|
23
|
+
render_aws_operations_json,
|
|
24
|
+
render_aws_operational_cost_query,
|
|
25
|
+
render_aws_quality_dqdl,
|
|
26
|
+
)
|
|
27
|
+
from contractforge_aws.runtime import (
|
|
28
|
+
AthenaQueryResult,
|
|
29
|
+
AthenaSqlRunner,
|
|
30
|
+
audit_evidence_tables,
|
|
31
|
+
apply_aws_annotations_contract,
|
|
32
|
+
apply_aws_annotations_plan,
|
|
33
|
+
apply_aws_lake_formation_contract,
|
|
34
|
+
apply_aws_lake_formation_plan,
|
|
35
|
+
create_or_update_schedule_payload,
|
|
36
|
+
create_or_update_state_machine_payload,
|
|
37
|
+
ensure_aws_evidence_tables,
|
|
38
|
+
deploy_aws_contract_to_glue,
|
|
39
|
+
get_aws_glue_job_run_status,
|
|
40
|
+
get_state_machine_execution_status,
|
|
41
|
+
publish_aws_contract_artifacts_to_s3,
|
|
42
|
+
record_aws_operations_contract,
|
|
43
|
+
reconcile_aws_glue_job_run_evidence,
|
|
44
|
+
register_aws_glue_job,
|
|
45
|
+
register_aws_glue_job_definition_payload,
|
|
46
|
+
render_aws_glue_job_run_evidence_sql,
|
|
47
|
+
start_aws_glue_job_run,
|
|
48
|
+
start_state_machine_execution,
|
|
49
|
+
wait_aws_glue_job_run,
|
|
50
|
+
wait_state_machine_execution,
|
|
51
|
+
)
|
|
52
|
+
from contractforge_aws.orchestration import (
|
|
53
|
+
render_eventbridge_scheduler_payload,
|
|
54
|
+
render_stepfunctions_state_machine_definition,
|
|
55
|
+
render_stepfunctions_state_machine_payload,
|
|
56
|
+
)
|
|
57
|
+
from contractforge_aws.capabilities import AWS_SUBTARGET_GLUE_ICEBERG, glue_iceberg_capabilities
|
|
58
|
+
from contractforge_aws.cost import CostModel, render_operational_cost_query
|
|
59
|
+
from contractforge_aws.environment import AWSEnvironment
|
|
60
|
+
from contractforge_aws.lineage import (
|
|
61
|
+
build_openlineage_event,
|
|
62
|
+
openlineage_namespace,
|
|
63
|
+
render_openlineage_insert_sql,
|
|
64
|
+
)
|
|
65
|
+
from contractforge_aws.subtargets import list_aws_subtargets
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
__version__ = _version("contractforge-aws")
|
|
69
|
+
except PackageNotFoundError: # pragma: no cover - editable/source tree without installed metadata
|
|
70
|
+
__version__ = "0.1.0"
|
|
71
|
+
|
|
72
|
+
__all__ = [
|
|
73
|
+
"AWSAdapter",
|
|
74
|
+
"AWSEnvironment",
|
|
75
|
+
"AWS_SUBTARGET_GLUE_ICEBERG",
|
|
76
|
+
"AthenaQueryResult",
|
|
77
|
+
"AthenaSqlRunner",
|
|
78
|
+
"CostModel",
|
|
79
|
+
"__version__",
|
|
80
|
+
"apply_aws_annotations_contract",
|
|
81
|
+
"apply_aws_annotations_plan",
|
|
82
|
+
"audit_evidence_tables",
|
|
83
|
+
"create_or_update_schedule_payload",
|
|
84
|
+
"create_or_update_state_machine_payload",
|
|
85
|
+
"ensure_aws_evidence_tables",
|
|
86
|
+
"deploy_aws_contract_to_glue",
|
|
87
|
+
"get_aws_glue_job_run_status",
|
|
88
|
+
"get_state_machine_execution_status",
|
|
89
|
+
"glue_iceberg_capabilities",
|
|
90
|
+
"build_openlineage_event",
|
|
91
|
+
"apply_aws_lake_formation_contract",
|
|
92
|
+
"apply_aws_lake_formation_plan",
|
|
93
|
+
"list_aws_subtargets",
|
|
94
|
+
"openlineage_namespace",
|
|
95
|
+
"plan_aws_contract",
|
|
96
|
+
"publish_aws_contract_artifacts_to_s3",
|
|
97
|
+
"record_aws_operations_contract",
|
|
98
|
+
"reconcile_aws_glue_job_run_evidence",
|
|
99
|
+
"register_aws_glue_job",
|
|
100
|
+
"register_aws_glue_job_definition_payload",
|
|
101
|
+
"render_aws_annotations_evidence_sql",
|
|
102
|
+
"render_aws_annotations_plan",
|
|
103
|
+
"render_aws_contract",
|
|
104
|
+
"render_aws_deployment_manifest",
|
|
105
|
+
"render_aws_glue_job_cloudformation",
|
|
106
|
+
"render_aws_glue_job_definition",
|
|
107
|
+
"render_aws_glue_job_iam_policy",
|
|
108
|
+
"render_aws_glue_job_terraform",
|
|
109
|
+
"render_aws_glue_job_run_evidence_sql",
|
|
110
|
+
"render_aws_lake_formation_evidence_sql",
|
|
111
|
+
"render_aws_lake_formation_plan",
|
|
112
|
+
"render_aws_native_passthrough_plan",
|
|
113
|
+
"render_aws_operations_evidence_sql",
|
|
114
|
+
"render_aws_operations_json",
|
|
115
|
+
"render_aws_operational_cost_query",
|
|
116
|
+
"render_aws_quality_dqdl",
|
|
117
|
+
"render_operational_cost_query",
|
|
118
|
+
"render_openlineage_insert_sql",
|
|
119
|
+
"render_eventbridge_scheduler_payload",
|
|
120
|
+
"render_stepfunctions_state_machine_definition",
|
|
121
|
+
"render_stepfunctions_state_machine_payload",
|
|
122
|
+
"start_aws_glue_job_run",
|
|
123
|
+
"start_state_machine_execution",
|
|
124
|
+
"wait_aws_glue_job_run",
|
|
125
|
+
"wait_state_machine_execution",
|
|
126
|
+
]
|