pyingestkit 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. pyingestkit-1.0.0/LICENSE +21 -0
  2. pyingestkit-1.0.0/PKG-INFO +325 -0
  3. pyingestkit-1.0.0/README.md +270 -0
  4. pyingestkit-1.0.0/pyproject.toml +95 -0
  5. pyingestkit-1.0.0/setup.cfg +4 -0
  6. pyingestkit-1.0.0/src/pyingestkit/__init__.py +111 -0
  7. pyingestkit-1.0.0/src/pyingestkit/_version.py +1 -0
  8. pyingestkit-1.0.0/src/pyingestkit/artifacts/__init__.py +17 -0
  9. pyingestkit-1.0.0/src/pyingestkit/artifacts/base.py +145 -0
  10. pyingestkit-1.0.0/src/pyingestkit/artifacts/factory.py +30 -0
  11. pyingestkit-1.0.0/src/pyingestkit/artifacts/filesystem.py +97 -0
  12. pyingestkit-1.0.0/src/pyingestkit/artifacts/naming.py +32 -0
  13. pyingestkit-1.0.0/src/pyingestkit/artifacts/raw.py +43 -0
  14. pyingestkit-1.0.0/src/pyingestkit/artifacts/s3.py +271 -0
  15. pyingestkit-1.0.0/src/pyingestkit/artifacts/stored.py +26 -0
  16. pyingestkit-1.0.0/src/pyingestkit/artifacts/uri.py +89 -0
  17. pyingestkit-1.0.0/src/pyingestkit/cli/__init__.py +5 -0
  18. pyingestkit-1.0.0/src/pyingestkit/cli/app.py +73 -0
  19. pyingestkit-1.0.0/src/pyingestkit/cli/commands/__init__.py +23 -0
  20. pyingestkit-1.0.0/src/pyingestkit/cli/commands/config.py +232 -0
  21. pyingestkit-1.0.0/src/pyingestkit/cli/commands/inspect.py +68 -0
  22. pyingestkit-1.0.0/src/pyingestkit/cli/commands/jobs.py +72 -0
  23. pyingestkit-1.0.0/src/pyingestkit/cli/commands/published.py +40 -0
  24. pyingestkit-1.0.0/src/pyingestkit/cli/commands/replay.py +92 -0
  25. pyingestkit-1.0.0/src/pyingestkit/cli/commands/run.py +162 -0
  26. pyingestkit-1.0.0/src/pyingestkit/cli/commands/runs.py +89 -0
  27. pyingestkit-1.0.0/src/pyingestkit/cli/commands/status.py +264 -0
  28. pyingestkit-1.0.0/src/pyingestkit/cli/commands/versions.py +56 -0
  29. pyingestkit-1.0.0/src/pyingestkit/cli/common.py +149 -0
  30. pyingestkit-1.0.0/src/pyingestkit/cli/console.py +6 -0
  31. pyingestkit-1.0.0/src/pyingestkit/cli/main.py +60 -0
  32. pyingestkit-1.0.0/src/pyingestkit/config/__init__.py +33 -0
  33. pyingestkit-1.0.0/src/pyingestkit/config/loader.py +122 -0
  34. pyingestkit-1.0.0/src/pyingestkit/config/models.py +166 -0
  35. pyingestkit-1.0.0/src/pyingestkit/contracts/__init__.py +3 -0
  36. pyingestkit-1.0.0/src/pyingestkit/contracts/dataset.py +613 -0
  37. pyingestkit-1.0.0/src/pyingestkit/core/__init__.py +22 -0
  38. pyingestkit-1.0.0/src/pyingestkit/core/context.py +27 -0
  39. pyingestkit-1.0.0/src/pyingestkit/core/events.py +80 -0
  40. pyingestkit-1.0.0/src/pyingestkit/core/exceptions.py +70 -0
  41. pyingestkit-1.0.0/src/pyingestkit/core/job.py +28 -0
  42. pyingestkit-1.0.0/src/pyingestkit/core/pipeline.py +20 -0
  43. pyingestkit-1.0.0/src/pyingestkit/core/registry.py +34 -0
  44. pyingestkit-1.0.0/src/pyingestkit/core/result.py +49 -0
  45. pyingestkit-1.0.0/src/pyingestkit/core/step.py +21 -0
  46. pyingestkit-1.0.0/src/pyingestkit/core/types.py +7 -0
  47. pyingestkit-1.0.0/src/pyingestkit/dataset.py +86 -0
  48. pyingestkit-1.0.0/src/pyingestkit/declarative/__init__.py +6 -0
  49. pyingestkit-1.0.0/src/pyingestkit/declarative/builder.py +44 -0
  50. pyingestkit-1.0.0/src/pyingestkit/declarative/decorators.py +62 -0
  51. pyingestkit-1.0.0/src/pyingestkit/declarative/invocation.py +20 -0
  52. pyingestkit-1.0.0/src/pyingestkit/declarative/job_definition.py +68 -0
  53. pyingestkit-1.0.0/src/pyingestkit/declarative/step_definition.py +70 -0
  54. pyingestkit-1.0.0/src/pyingestkit/deprecations.py +35 -0
  55. pyingestkit-1.0.0/src/pyingestkit/diff/__init__.py +4 -0
  56. pyingestkit-1.0.0/src/pyingestkit/diff/engine.py +243 -0
  57. pyingestkit-1.0.0/src/pyingestkit/diff/models.py +149 -0
  58. pyingestkit-1.0.0/src/pyingestkit/diff/report.py +119 -0
  59. pyingestkit-1.0.0/src/pyingestkit/errors.py +74 -0
  60. pyingestkit-1.0.0/src/pyingestkit/logging/__init__.py +13 -0
  61. pyingestkit-1.0.0/src/pyingestkit/logging/context.py +38 -0
  62. pyingestkit-1.0.0/src/pyingestkit/logging/filters.py +79 -0
  63. pyingestkit-1.0.0/src/pyingestkit/logging/formatters.py +75 -0
  64. pyingestkit-1.0.0/src/pyingestkit/logging/setup.py +113 -0
  65. pyingestkit-1.0.0/src/pyingestkit/metadata/__init__.py +51 -0
  66. pyingestkit-1.0.0/src/pyingestkit/metadata/_artifact_locations.py +49 -0
  67. pyingestkit-1.0.0/src/pyingestkit/metadata/_schema.py +266 -0
  68. pyingestkit-1.0.0/src/pyingestkit/metadata/_sqlalchemy.py +703 -0
  69. pyingestkit-1.0.0/src/pyingestkit/metadata/_target_loads.py +178 -0
  70. pyingestkit-1.0.0/src/pyingestkit/metadata/_types.py +35 -0
  71. pyingestkit-1.0.0/src/pyingestkit/metadata/base.py +112 -0
  72. pyingestkit-1.0.0/src/pyingestkit/metadata/capabilities.py +108 -0
  73. pyingestkit-1.0.0/src/pyingestkit/metadata/factory.py +26 -0
  74. pyingestkit-1.0.0/src/pyingestkit/metadata/memory.py +292 -0
  75. pyingestkit-1.0.0/src/pyingestkit/metadata/models.py +216 -0
  76. pyingestkit-1.0.0/src/pyingestkit/metadata/postgres.py +79 -0
  77. pyingestkit-1.0.0/src/pyingestkit/metadata/sqlite.py +67 -0
  78. pyingestkit-1.0.0/src/pyingestkit/parsers/__init__.py +16 -0
  79. pyingestkit-1.0.0/src/pyingestkit/parsers/base.py +14 -0
  80. pyingestkit-1.0.0/src/pyingestkit/parsers/csv.py +69 -0
  81. pyingestkit-1.0.0/src/pyingestkit/parsers/excel.py +121 -0
  82. pyingestkit-1.0.0/src/pyingestkit/parsers/json.py +76 -0
  83. pyingestkit-1.0.0/src/pyingestkit/parsers/ndjson.py +51 -0
  84. pyingestkit-1.0.0/src/pyingestkit/parsers/parquet.py +89 -0
  85. pyingestkit-1.0.0/src/pyingestkit/plugins/__init__.py +19 -0
  86. pyingestkit-1.0.0/src/pyingestkit/plugins/discovery.py +138 -0
  87. pyingestkit-1.0.0/src/pyingestkit/profiling/__init__.py +4 -0
  88. pyingestkit-1.0.0/src/pyingestkit/profiling/models.py +63 -0
  89. pyingestkit-1.0.0/src/pyingestkit/profiling/profiler.py +120 -0
  90. pyingestkit-1.0.0/src/pyingestkit/provenance/__init__.py +4 -0
  91. pyingestkit-1.0.0/src/pyingestkit/provenance/hashing.py +16 -0
  92. pyingestkit-1.0.0/src/pyingestkit/provenance/manifest.py +82 -0
  93. pyingestkit-1.0.0/src/pyingestkit/publication/__init__.py +3 -0
  94. pyingestkit-1.0.0/src/pyingestkit/publication/atomic.py +27 -0
  95. pyingestkit-1.0.0/src/pyingestkit/py.typed +0 -0
  96. pyingestkit-1.0.0/src/pyingestkit/quality/__init__.py +3 -0
  97. pyingestkit-1.0.0/src/pyingestkit/quality/report.py +31 -0
  98. pyingestkit-1.0.0/src/pyingestkit/replay/__init__.py +11 -0
  99. pyingestkit-1.0.0/src/pyingestkit/replay/models.py +117 -0
  100. pyingestkit-1.0.0/src/pyingestkit/replay/resolver.py +63 -0
  101. pyingestkit-1.0.0/src/pyingestkit/replay/service.py +227 -0
  102. pyingestkit-1.0.0/src/pyingestkit/retry/__init__.py +15 -0
  103. pyingestkit-1.0.0/src/pyingestkit/retry/policy.py +175 -0
  104. pyingestkit-1.0.0/src/pyingestkit/runtime/__init__.py +3 -0
  105. pyingestkit-1.0.0/src/pyingestkit/runtime/runner.py +699 -0
  106. pyingestkit-1.0.0/src/pyingestkit/sources/__init__.py +4 -0
  107. pyingestkit-1.0.0/src/pyingestkit/sources/base.py +12 -0
  108. pyingestkit-1.0.0/src/pyingestkit/sources/http/__init__.py +23 -0
  109. pyingestkit-1.0.0/src/pyingestkit/sources/http/client.py +75 -0
  110. pyingestkit-1.0.0/src/pyingestkit/sources/http/exceptions.py +47 -0
  111. pyingestkit-1.0.0/src/pyingestkit/sources/http/request.py +71 -0
  112. pyingestkit-1.0.0/src/pyingestkit/sources/http/response.py +65 -0
  113. pyingestkit-1.0.0/src/pyingestkit/sources/http/security.py +57 -0
  114. pyingestkit-1.0.0/src/pyingestkit/sources/http/source.py +185 -0
  115. pyingestkit-1.0.0/src/pyingestkit/sources/local.py +39 -0
  116. pyingestkit-1.0.0/src/pyingestkit/targets/__init__.py +45 -0
  117. pyingestkit-1.0.0/src/pyingestkit/targets/base.py +50 -0
  118. pyingestkit-1.0.0/src/pyingestkit/targets/capabilities.py +18 -0
  119. pyingestkit-1.0.0/src/pyingestkit/targets/errors.py +35 -0
  120. pyingestkit-1.0.0/src/pyingestkit/targets/idempotency.py +258 -0
  121. pyingestkit-1.0.0/src/pyingestkit/targets/models.py +141 -0
  122. pyingestkit-1.0.0/src/pyingestkit/targets/postgres.py +333 -0
  123. pyingestkit-1.0.0/src/pyingestkit/targets/schema.py +160 -0
  124. pyingestkit-1.0.0/src/pyingestkit/validation/__init__.py +15 -0
  125. pyingestkit-1.0.0/src/pyingestkit/validation/report.py +81 -0
  126. pyingestkit-1.0.0/src/pyingestkit/validation/result.py +42 -0
  127. pyingestkit-1.0.0/src/pyingestkit/validation/rules.py +92 -0
  128. pyingestkit-1.0.0/src/pyingestkit/versioning/__init__.py +17 -0
  129. pyingestkit-1.0.0/src/pyingestkit/versioning/_canonical.py +66 -0
  130. pyingestkit-1.0.0/src/pyingestkit/versioning/_metadata.py +61 -0
  131. pyingestkit-1.0.0/src/pyingestkit/versioning/_s3_objects.py +196 -0
  132. pyingestkit-1.0.0/src/pyingestkit/versioning/fingerprint.py +83 -0
  133. pyingestkit-1.0.0/src/pyingestkit/versioning/models.py +31 -0
  134. pyingestkit-1.0.0/src/pyingestkit/versioning/s3.py +279 -0
  135. pyingestkit-1.0.0/src/pyingestkit/versioning/snapshot.py +147 -0
  136. pyingestkit-1.0.0/src/pyingestkit/versioning/store.py +341 -0
  137. pyingestkit-1.0.0/src/pyingestkit.egg-info/PKG-INFO +325 -0
  138. pyingestkit-1.0.0/src/pyingestkit.egg-info/SOURCES.txt +140 -0
  139. pyingestkit-1.0.0/src/pyingestkit.egg-info/dependency_links.txt +1 -0
  140. pyingestkit-1.0.0/src/pyingestkit.egg-info/entry_points.txt +2 -0
  141. pyingestkit-1.0.0/src/pyingestkit.egg-info/requires.txt +37 -0
  142. pyingestkit-1.0.0/src/pyingestkit.egg-info/top_level.txt +1 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Thomas Awounfouet
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,325 @@
1
+ Metadata-Version: 2.4
2
+ Name: pyingestkit
3
+ Version: 1.0.0
4
+ Summary: Composable ingestion tooling for Python.
5
+ Author: Thomas Awounfouet
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/tawounfouet/pyingestkit
8
+ Project-URL: Documentation, https://github.com/tawounfouet/pyingestkit#readme
9
+ Project-URL: Repository, https://github.com/tawounfouet/pyingestkit
10
+ Project-URL: Issues, https://github.com/tawounfouet/pyingestkit/issues
11
+ Project-URL: Changelog, https://github.com/tawounfouet/pyingestkit/releases
12
+ Classifier: Development Status :: 5 - Production/Stable
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
19
+ Requires-Python: >=3.11
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Requires-Dist: typer<0.28,>=0.27
23
+ Requires-Dist: rich<16,>=15
24
+ Requires-Dist: pydantic<3,>=2.11
25
+ Requires-Dist: PyYAML<7,>=6
26
+ Requires-Dist: SQLAlchemy<3,>=2.0
27
+ Requires-Dist: httpx<1,>=0.28.1
28
+ Requires-Dist: tenacity<10,>=9.1.4
29
+ Requires-Dist: python-dotenv<2,>=1.0
30
+ Provides-Extra: dev
31
+ Requires-Dist: pytest>=8; extra == "dev"
32
+ Requires-Dist: pytest-cov>=6; extra == "dev"
33
+ Requires-Dist: pytest-randomly>=3.15; extra == "dev"
34
+ Requires-Dist: ruff>=0.6; extra == "dev"
35
+ Requires-Dist: mypy>=1.11; extra == "dev"
36
+ Requires-Dist: types-PyYAML; extra == "dev"
37
+ Requires-Dist: bandit>=1.8; extra == "dev"
38
+ Requires-Dist: pip-audit>=2.9; extra == "dev"
39
+ Requires-Dist: pre-commit>=4; extra == "dev"
40
+ Requires-Dist: build>=1.2; extra == "dev"
41
+ Requires-Dist: twine>=6; extra == "dev"
42
+ Requires-Dist: openpyxl<4,>=3.1; extra == "dev"
43
+ Requires-Dist: pyarrow<24,>=16; extra == "dev"
44
+ Requires-Dist: psycopg[binary]<4,>=3.2; extra == "dev"
45
+ Requires-Dist: boto3<2,>=1.35; extra == "dev"
46
+ Provides-Extra: postgres
47
+ Requires-Dist: psycopg[binary]<4,>=3.2; extra == "postgres"
48
+ Provides-Extra: s3
49
+ Requires-Dist: boto3<2,>=1.35; extra == "s3"
50
+ Provides-Extra: excel
51
+ Requires-Dist: openpyxl<4,>=3.1; extra == "excel"
52
+ Provides-Extra: parquet
53
+ Requires-Dist: pyarrow<24,>=16; extra == "parquet"
54
+ Dynamic: license-file
55
+
56
+ # PyIngestKit
57
+
58
+ [![CI](https://github.com/tawounfouet/pyingestkit/actions/workflows/ci.yml/badge.svg)](https://github.com/tawounfouet/pyingestkit/actions/workflows/ci.yml)
59
+ [![Security](https://github.com/tawounfouet/pyingestkit/actions/workflows/security.yml/badge.svg)](https://github.com/tawounfouet/pyingestkit/actions/workflows/security.yml)
60
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/)
61
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
62
+ [![Stable: v1.0.0](https://img.shields.io/badge/stable-v1.0.0-brightgreen.svg)](docs/releases/v1.0.0.md)
63
+
64
+ **PyIngestKit** is a focused Python framework for reliable, traceable batch ingestion.
65
+
66
+ > Transform an external source into a reliable, validated, reproducible and publishable dataset without rewriting ingestion plumbing for every job.
67
+
68
+ **V1.0.0 Stable** is the first protected 1.x framework contract. It promotes the qualified RC1
69
+ baseline without adding product scope and retains the immutable `v0.6.0` release as executable
70
+ historical upgrade evidence. The annotated `v1.0.0` release tag is created only after the exact stable
71
+ merge SHA passes post-merge CI and Security.
72
+
73
+ ## Product boundary
74
+
75
+ PyIngestKit owns **HOW TO INGEST**. External orchestrators own **WHEN TO RUN**.
76
+
77
+ It is not Airflow, Dagster, Prefect, Celery, a distributed scheduler, a Data Platform, a Data Catalog, an IAM platform, or a cloud-provisioning framework.
78
+
79
+ ```text
80
+ ArtifactStore != Target
81
+ ArtifactStore != MetadataStore
82
+ DatasetVersionStore != ArtifactStore
83
+ S3-compatible != AWS-only
84
+ DatasetVersion != S3 object version
85
+ Replay != new source acquisition
86
+ PyIngestKit != orchestrator
87
+ ```
88
+
89
+ ## V1.0.0 stable capabilities
90
+
91
+ - immutable RAW with SHA-256 provenance;
92
+ - CSV, JSON, NDJSON, Excel and Parquet parsing behind a dependency-neutral `Dataset`;
93
+ - contracts, validation, profiling and portable quality reports;
94
+ - deterministic Dataset fingerprints and diff reports;
95
+ - immutable content-addressed DatasetVersion snapshots and PublishedDataset pointers;
96
+ - strict replay from historical RAW;
97
+ - transactional PostgreSQL target loads with COPY and idempotency;
98
+ - `ArtifactURI` and `StoredArtifact` portable durable references;
99
+ - optional `S3ArtifactStore` for RAW/reports/manifests;
100
+ - optional `S3DatasetVersionStore` for remote snapshots/publication;
101
+ - MinIO-tested S3-compatible behavior;
102
+ - full replay from a fresh host/workspace using shared PostgreSQL metadata + object storage;
103
+ - deterministic plugin/config/error/CLI/logging behavior governed for 1.x;
104
+ - explicit stable Python/public/persisted compatibility contracts;
105
+ - five representative pilots covering nine executable reference jobs;
106
+ - clean-wheel packaging plus an executable V0.6.0 -> 1.0.0 upgrade smoke.
107
+
108
+ ## Installation
109
+
110
+ ```bash
111
+ python3.12 -m venv .venv
112
+ source .venv/bin/activate
113
+ python -m pip install -e ".[dev,excel,parquet,postgres,s3]"
114
+ python -m pip install -e examples/plugin_package
115
+ ```
116
+
117
+ Production consumers can select only the required extras:
118
+
119
+ ```bash
120
+ pip install "pyingestkit[s3]>=1,<2"
121
+ pip install "pyingestkit[postgres]>=1,<2"
122
+ ```
123
+
124
+ Stable qualification builds and installs the generated `1.0.0` wheels in clean environments before
125
+ the immutable release tag is published.
126
+
127
+ ## Minimal S3-compatible configuration
128
+
129
+ ```yaml
130
+ runtime:
131
+ workspace: .pyingest
132
+
133
+ artifacts:
134
+ backend: s3
135
+ s3:
136
+ bucket: my-pyingest-artifacts
137
+ prefix: pyingest
138
+ region_name: eu-west-3
139
+ endpoint_url_env: PYINGEST_S3_ENDPOINT_URL
140
+ cache_path: .pyingest
141
+ ```
142
+
143
+ Credentials use the boto3/AWS provider chain and are not stored in YAML.
144
+
145
+ For MinIO:
146
+
147
+ ```bash
148
+ export PYINGEST_S3_ENDPOINT_URL='https://minio.example.internal'
149
+ ```
150
+
151
+ ## API
152
+
153
+ Decorator style:
154
+
155
+ ```python
156
+ from pyingestkit import RunContext, job, step
157
+
158
+ @step(name="Fetch")
159
+ def fetch(context: RunContext):
160
+ ...
161
+
162
+ @step
163
+ def normalize(data):
164
+ ...
165
+
166
+ @job(id="public.postal_codes", version="1.0.0")
167
+ def postal_codes() -> None:
168
+ fetch()
169
+ normalize()
170
+ ```
171
+
172
+ Imperative style:
173
+
174
+ ```python
175
+ from pyingestkit import Job, Pipeline, RunContext, Step
176
+
177
+ class Fetch(Step):
178
+ def execute(self, context: RunContext, data):
179
+ ...
180
+
181
+ class MyJob(Job):
182
+ id = "public.example"
183
+
184
+ def pipeline(self) -> Pipeline:
185
+ return Pipeline([Fetch()])
186
+ ```
187
+
188
+ ## Configuration Management & Backend Requirements
189
+
190
+ PyIngestKit resolves configuration in the following order:
191
+ 1. Explicit `--config <path.yml>` CLI flag.
192
+ 2. `PYINGEST_CONFIG` environment variable path.
193
+ 3. `PYINGEST_ENV=<env>` selecting `pyingest.yml.<env>`.
194
+ 4. Automatic discovery of `pyingest.yml`, `pyingestkit.yml`, or `.pyingest.yml` in the working directory.
195
+ 5. Local in-memory defaults (`filesystem` artifacts & `sqlite` metadata).
196
+
197
+ Explicit environment/profile selectors are fail closed: a missing selected config is an error.
198
+ Workspace precedence is `--workspace` → `PYINGEST_WORKSPACE` → `runtime.workspace` → `.pyingest`.
199
+
200
+ Jobs can explicitly declare backend requirements (e.g. `requires_artifacts="s3"`, `requires_metadata="postgres"`). If a job's requirements are not met by the active configuration, `pyingest run` halts before executing any step with a clear error message.
201
+
202
+ ### Configuration Profiles & Environment Files
203
+
204
+ Three ready-to-use YAML profiles and corresponding environment templates in `envs/` are provided.
205
+ The `*.example` dotenv files are templates only and are never auto-loaded.
206
+
207
+ - `pyingest.yml.dev` & `envs/.env.dev.example`: local filesystem artifacts + SQLite metadata.
208
+ - `pyingest.yml.stg` & `envs/.env.stg.example`: S3-compatible artifacts via MinIO + PostgreSQL metadata.
209
+ - `pyingest.yml.prod` & `envs/.env.prod.example`: S3-compatible artifacts via AWS S3 / Cloudflare R2 + PostgreSQL metadata.
210
+
211
+ ```bash
212
+ cp envs/.env.dev.example .env
213
+ cp pyingest.yml.dev pyingest.yml
214
+ ```
215
+
216
+ ### Project auto-discovery
217
+
218
+ ```bash
219
+ pyingest --version
220
+ pyingest config
221
+ pyingest jobs
222
+ pyingest inspect demo.versioned_s3
223
+
224
+ pyingest run demo.local_file --param path=examples/plugin_package/data/sample.txt
225
+ pyingest run demo.http_csv
226
+ pyingest run demo.http_json
227
+ pyingest run demo.ndjson_quality
228
+ pyingest run demo.excel_quality
229
+ pyingest run demo.parquet_quality
230
+
231
+ pyingest run demo.versioned_postgres --param revision=1
232
+ pyingest run demo.versioned_s3 --param revision=1
233
+ pyingest run demo.versioned_s3 --param revision=2
234
+
235
+ pyingest versions demo.versioned_s3
236
+ pyingest published demo.versioned_s3
237
+ pyingest runs
238
+ pyingest status
239
+ pyingest replay
240
+ ```
241
+
242
+ ### Explicit demo configuration
243
+
244
+ ```bash
245
+ pyingest run demo.local_file --config examples/plugin_package/demo.yml
246
+ pyingest run demo.http_csv --config examples/plugin_package/demo-http.yml
247
+ pyingest run demo.http_json --config examples/plugin_package/demo-http.yml
248
+ pyingest run demo.ndjson_quality --config examples/plugin_package/demo-quality.yml
249
+ pyingest run demo.excel_quality --config examples/plugin_package/demo-quality.yml
250
+ pyingest run demo.parquet_quality --config examples/plugin_package/demo-quality.yml
251
+ pyingest run demo.versioned_ndjson --config examples/plugin_package/demo-versioned.yml --param revision=1
252
+ pyingest run demo.versioned_postgres --config examples/plugin_package/demo-versioned-postgres.yml --param revision=1
253
+ pyingest run demo.versioned_s3 --config examples/plugin_package/demo-versioned-s3.yml --param revision=1
254
+ pyingest run demo.versioned_s3 --config examples/plugin_package/demo-versioned-s3.yml --param revision=2
255
+ ```
256
+
257
+ ## V1 stable reference jobs
258
+
259
+ ```text
260
+ demo.local_file
261
+ demo.http_csv
262
+ demo.http_json
263
+ demo.ndjson_quality
264
+ demo.excel_quality
265
+ demo.parquet_quality
266
+ demo.versioned_ndjson
267
+ demo.versioned_postgres
268
+ demo.versioned_s3
269
+ ```
270
+
271
+ `demo.versioned_s3` remains the full cross-host vertical slice: V1 → V2 → remote RAW/reports/snapshots → PostgreSQL → publish V2 → destroy workspace A → strict replay from workspace B → fingerprint match → idempotent target SKIP.
272
+
273
+ ## Durable storage model
274
+
275
+ ```text
276
+ PostgreSQL metadata
277
+ └── runs / artifact locations / lineage / target loads
278
+
279
+ S3-compatible object storage
280
+ └── RAW / reports / manifests / DatasetVersion snapshots / PublishedDataset pointer
281
+
282
+ PostgreSQL Target
283
+ └── consumable dataset
284
+ ```
285
+
286
+ ## Quality and release gates
287
+
288
+ ```bash
289
+ make test
290
+ make quality
291
+ make security
292
+ make build
293
+ make check
294
+ make release-check
295
+ ```
296
+
297
+ GitHub CI qualifies Python 3.11/3.12/3.13, PostgreSQL 16, MinIO/S3 integration, full cross-host
298
+ object-storage replay, A1/A2/B1/B2 governance, historical RC1 evidence, the stable release contract,
299
+ clean-wheel installation and the real `v0.6.0` → `1.0.0` upgrade path.
300
+
301
+ See:
302
+ - `docs/guides/v1-quickstart.md`
303
+ - `docs/guides/v1-production-pilot.md`
304
+ - `docs/guides/migrate-v0.6-to-v1.md`
305
+ - `docs/guides/release-validation-v1.0.0.md`
306
+ - `docs/reference/stable-contract-v1.md`
307
+ - `docs/reference/public-api.md`
308
+ - `docs/reference/compatibility-v1.md`
309
+ - `docs/reference/stability-v1.md`
310
+ - `docs/reference/pilots-v1.md`
311
+ - `SECURITY.md`
312
+
313
+ ## V1.0.0 stable build artifacts
314
+
315
+ ```text
316
+ pyingestkit-1.0.0-py3-none-any.whl
317
+ pyingestkit-1.0.0.tar.gz
318
+ pyingestkit_demo_jobs-1.0.0-py3-none-any.whl
319
+ pyingestkit_demo_jobs-1.0.0.tar.gz
320
+ SHA256SUMS
321
+ ```
322
+
323
+ CI groups these as `pyingestkit-v1.0.0-source` and `pyingestkit-v1.0.0-dist`. The historical `v0.6.0`
324
+ and RC1 evidence remain immutable and separate. The annotated `v1.0.0` tag is created only after the
325
+ exact stable merge SHA passes post-merge CI and Security.
@@ -0,0 +1,270 @@
1
+ # PyIngestKit
2
+
3
+ [![CI](https://github.com/tawounfouet/pyingestkit/actions/workflows/ci.yml/badge.svg)](https://github.com/tawounfouet/pyingestkit/actions/workflows/ci.yml)
4
+ [![Security](https://github.com/tawounfouet/pyingestkit/actions/workflows/security.yml/badge.svg)](https://github.com/tawounfouet/pyingestkit/actions/workflows/security.yml)
5
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/)
6
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
7
+ [![Stable: v1.0.0](https://img.shields.io/badge/stable-v1.0.0-brightgreen.svg)](docs/releases/v1.0.0.md)
8
+
9
+ **PyIngestKit** is a focused Python framework for reliable, traceable batch ingestion.
10
+
11
+ > Transform an external source into a reliable, validated, reproducible and publishable dataset without rewriting ingestion plumbing for every job.
12
+
13
+ **V1.0.0 Stable** is the first protected 1.x framework contract. It promotes the qualified RC1
14
+ baseline without adding product scope and retains the immutable `v0.6.0` release as executable
15
+ historical upgrade evidence. The annotated `v1.0.0` release tag is created only after the exact stable
16
+ merge SHA passes post-merge CI and Security.
17
+
18
+ ## Product boundary
19
+
20
+ PyIngestKit owns **HOW TO INGEST**. External orchestrators own **WHEN TO RUN**.
21
+
22
+ It is not Airflow, Dagster, Prefect, Celery, a distributed scheduler, a Data Platform, a Data Catalog, an IAM platform, or a cloud-provisioning framework.
23
+
24
+ ```text
25
+ ArtifactStore != Target
26
+ ArtifactStore != MetadataStore
27
+ DatasetVersionStore != ArtifactStore
28
+ S3-compatible != AWS-only
29
+ DatasetVersion != S3 object version
30
+ Replay != new source acquisition
31
+ PyIngestKit != orchestrator
32
+ ```
33
+
34
+ ## V1.0.0 stable capabilities
35
+
36
+ - immutable RAW with SHA-256 provenance;
37
+ - CSV, JSON, NDJSON, Excel and Parquet parsing behind a dependency-neutral `Dataset`;
38
+ - contracts, validation, profiling and portable quality reports;
39
+ - deterministic Dataset fingerprints and diff reports;
40
+ - immutable content-addressed DatasetVersion snapshots and PublishedDataset pointers;
41
+ - strict replay from historical RAW;
42
+ - transactional PostgreSQL target loads with COPY and idempotency;
43
+ - `ArtifactURI` and `StoredArtifact` portable durable references;
44
+ - optional `S3ArtifactStore` for RAW/reports/manifests;
45
+ - optional `S3DatasetVersionStore` for remote snapshots/publication;
46
+ - MinIO-tested S3-compatible behavior;
47
+ - full replay from a fresh host/workspace using shared PostgreSQL metadata + object storage;
48
+ - deterministic plugin/config/error/CLI/logging behavior governed for 1.x;
49
+ - explicit stable Python/public/persisted compatibility contracts;
50
+ - five representative pilots covering nine executable reference jobs;
51
+ - clean-wheel packaging plus an executable V0.6.0 -> 1.0.0 upgrade smoke.
52
+
53
+ ## Installation
54
+
55
+ ```bash
56
+ python3.12 -m venv .venv
57
+ source .venv/bin/activate
58
+ python -m pip install -e ".[dev,excel,parquet,postgres,s3]"
59
+ python -m pip install -e examples/plugin_package
60
+ ```
61
+
62
+ Production consumers can select only the required extras:
63
+
64
+ ```bash
65
+ pip install "pyingestkit[s3]>=1,<2"
66
+ pip install "pyingestkit[postgres]>=1,<2"
67
+ ```
68
+
69
+ Stable qualification builds and installs the generated `1.0.0` wheels in clean environments before
70
+ the immutable release tag is published.
71
+
72
+ ## Minimal S3-compatible configuration
73
+
74
+ ```yaml
75
+ runtime:
76
+ workspace: .pyingest
77
+
78
+ artifacts:
79
+ backend: s3
80
+ s3:
81
+ bucket: my-pyingest-artifacts
82
+ prefix: pyingest
83
+ region_name: eu-west-3
84
+ endpoint_url_env: PYINGEST_S3_ENDPOINT_URL
85
+ cache_path: .pyingest
86
+ ```
87
+
88
+ Credentials use the boto3/AWS provider chain and are not stored in YAML.
89
+
90
+ For MinIO:
91
+
92
+ ```bash
93
+ export PYINGEST_S3_ENDPOINT_URL='https://minio.example.internal'
94
+ ```
95
+
96
+ ## API
97
+
98
+ Decorator style:
99
+
100
+ ```python
101
+ from pyingestkit import RunContext, job, step
102
+
103
+ @step(name="Fetch")
104
+ def fetch(context: RunContext):
105
+ ...
106
+
107
+ @step
108
+ def normalize(data):
109
+ ...
110
+
111
+ @job(id="public.postal_codes", version="1.0.0")
112
+ def postal_codes() -> None:
113
+ fetch()
114
+ normalize()
115
+ ```
116
+
117
+ Imperative style:
118
+
119
+ ```python
120
+ from pyingestkit import Job, Pipeline, RunContext, Step
121
+
122
+ class Fetch(Step):
123
+ def execute(self, context: RunContext, data):
124
+ ...
125
+
126
+ class MyJob(Job):
127
+ id = "public.example"
128
+
129
+ def pipeline(self) -> Pipeline:
130
+ return Pipeline([Fetch()])
131
+ ```
132
+
133
+ ## Configuration Management & Backend Requirements
134
+
135
+ PyIngestKit resolves configuration in the following order:
136
+ 1. Explicit `--config <path.yml>` CLI flag.
137
+ 2. `PYINGEST_CONFIG` environment variable path.
138
+ 3. `PYINGEST_ENV=<env>` selecting `pyingest.yml.<env>`.
139
+ 4. Automatic discovery of `pyingest.yml`, `pyingestkit.yml`, or `.pyingest.yml` in the working directory.
140
+ 5. Local in-memory defaults (`filesystem` artifacts & `sqlite` metadata).
141
+
142
+ Explicit environment/profile selectors are fail closed: a missing selected config is an error.
143
+ Workspace precedence is `--workspace` → `PYINGEST_WORKSPACE` → `runtime.workspace` → `.pyingest`.
144
+
145
+ Jobs can explicitly declare backend requirements (e.g. `requires_artifacts="s3"`, `requires_metadata="postgres"`). If a job's requirements are not met by the active configuration, `pyingest run` halts before executing any step with a clear error message.
146
+
147
+ ### Configuration Profiles & Environment Files
148
+
149
+ Three ready-to-use YAML profiles and corresponding environment templates in `envs/` are provided.
150
+ The `*.example` dotenv files are templates only and are never auto-loaded.
151
+
152
+ - `pyingest.yml.dev` & `envs/.env.dev.example`: local filesystem artifacts + SQLite metadata.
153
+ - `pyingest.yml.stg` & `envs/.env.stg.example`: S3-compatible artifacts via MinIO + PostgreSQL metadata.
154
+ - `pyingest.yml.prod` & `envs/.env.prod.example`: S3-compatible artifacts via AWS S3 / Cloudflare R2 + PostgreSQL metadata.
155
+
156
+ ```bash
157
+ cp envs/.env.dev.example .env
158
+ cp pyingest.yml.dev pyingest.yml
159
+ ```
160
+
161
+ ### Project auto-discovery
162
+
163
+ ```bash
164
+ pyingest --version
165
+ pyingest config
166
+ pyingest jobs
167
+ pyingest inspect demo.versioned_s3
168
+
169
+ pyingest run demo.local_file --param path=examples/plugin_package/data/sample.txt
170
+ pyingest run demo.http_csv
171
+ pyingest run demo.http_json
172
+ pyingest run demo.ndjson_quality
173
+ pyingest run demo.excel_quality
174
+ pyingest run demo.parquet_quality
175
+
176
+ pyingest run demo.versioned_postgres --param revision=1
177
+ pyingest run demo.versioned_s3 --param revision=1
178
+ pyingest run demo.versioned_s3 --param revision=2
179
+
180
+ pyingest versions demo.versioned_s3
181
+ pyingest published demo.versioned_s3
182
+ pyingest runs
183
+ pyingest status
184
+ pyingest replay
185
+ ```
186
+
187
+ ### Explicit demo configuration
188
+
189
+ ```bash
190
+ pyingest run demo.local_file --config examples/plugin_package/demo.yml
191
+ pyingest run demo.http_csv --config examples/plugin_package/demo-http.yml
192
+ pyingest run demo.http_json --config examples/plugin_package/demo-http.yml
193
+ pyingest run demo.ndjson_quality --config examples/plugin_package/demo-quality.yml
194
+ pyingest run demo.excel_quality --config examples/plugin_package/demo-quality.yml
195
+ pyingest run demo.parquet_quality --config examples/plugin_package/demo-quality.yml
196
+ pyingest run demo.versioned_ndjson --config examples/plugin_package/demo-versioned.yml --param revision=1
197
+ pyingest run demo.versioned_postgres --config examples/plugin_package/demo-versioned-postgres.yml --param revision=1
198
+ pyingest run demo.versioned_s3 --config examples/plugin_package/demo-versioned-s3.yml --param revision=1
199
+ pyingest run demo.versioned_s3 --config examples/plugin_package/demo-versioned-s3.yml --param revision=2
200
+ ```
201
+
202
+ ## V1 stable reference jobs
203
+
204
+ ```text
205
+ demo.local_file
206
+ demo.http_csv
207
+ demo.http_json
208
+ demo.ndjson_quality
209
+ demo.excel_quality
210
+ demo.parquet_quality
211
+ demo.versioned_ndjson
212
+ demo.versioned_postgres
213
+ demo.versioned_s3
214
+ ```
215
+
216
+ `demo.versioned_s3` remains the full cross-host vertical slice: V1 → V2 → remote RAW/reports/snapshots → PostgreSQL → publish V2 → destroy workspace A → strict replay from workspace B → fingerprint match → idempotent target SKIP.
217
+
218
+ ## Durable storage model
219
+
220
+ ```text
221
+ PostgreSQL metadata
222
+ └── runs / artifact locations / lineage / target loads
223
+
224
+ S3-compatible object storage
225
+ └── RAW / reports / manifests / DatasetVersion snapshots / PublishedDataset pointer
226
+
227
+ PostgreSQL Target
228
+ └── consumable dataset
229
+ ```
230
+
231
+ ## Quality and release gates
232
+
233
+ ```bash
234
+ make test
235
+ make quality
236
+ make security
237
+ make build
238
+ make check
239
+ make release-check
240
+ ```
241
+
242
+ GitHub CI qualifies Python 3.11/3.12/3.13, PostgreSQL 16, MinIO/S3 integration, full cross-host
243
+ object-storage replay, A1/A2/B1/B2 governance, historical RC1 evidence, the stable release contract,
244
+ clean-wheel installation and the real `v0.6.0` → `1.0.0` upgrade path.
245
+
246
+ See:
247
+ - `docs/guides/v1-quickstart.md`
248
+ - `docs/guides/v1-production-pilot.md`
249
+ - `docs/guides/migrate-v0.6-to-v1.md`
250
+ - `docs/guides/release-validation-v1.0.0.md`
251
+ - `docs/reference/stable-contract-v1.md`
252
+ - `docs/reference/public-api.md`
253
+ - `docs/reference/compatibility-v1.md`
254
+ - `docs/reference/stability-v1.md`
255
+ - `docs/reference/pilots-v1.md`
256
+ - `SECURITY.md`
257
+
258
+ ## V1.0.0 stable build artifacts
259
+
260
+ ```text
261
+ pyingestkit-1.0.0-py3-none-any.whl
262
+ pyingestkit-1.0.0.tar.gz
263
+ pyingestkit_demo_jobs-1.0.0-py3-none-any.whl
264
+ pyingestkit_demo_jobs-1.0.0.tar.gz
265
+ SHA256SUMS
266
+ ```
267
+
268
+ CI groups these as `pyingestkit-v1.0.0-source` and `pyingestkit-v1.0.0-dist`. The historical `v0.6.0`
269
+ and RC1 evidence remain immutable and separate. The annotated `v1.0.0` tag is created only after the
270
+ exact stable merge SHA passes post-merge CI and Security.