consist 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. consist-0.1.0/PKG-INFO +270 -0
  2. consist-0.1.0/README.md +169 -0
  3. consist-0.1.0/pyproject.toml +152 -0
  4. consist-0.1.0/src/consist/.DS_Store +0 -0
  5. consist-0.1.0/src/consist/__init__.py +187 -0
  6. consist-0.1.0/src/consist/api.py +2672 -0
  7. consist-0.1.0/src/consist/cli.py +3882 -0
  8. consist-0.1.0/src/consist/core/__init__.py +4 -0
  9. consist-0.1.0/src/consist/core/_coupler_shared.py +266 -0
  10. consist-0.1.0/src/consist/core/artifact_facets.py +145 -0
  11. consist-0.1.0/src/consist/core/artifact_schemas.py +658 -0
  12. consist-0.1.0/src/consist/core/artifacts.py +741 -0
  13. consist-0.1.0/src/consist/core/cache.py +501 -0
  14. consist-0.1.0/src/consist/core/cache_output_logging.py +155 -0
  15. consist-0.1.0/src/consist/core/config_canonicalization.py +543 -0
  16. consist-0.1.0/src/consist/core/config_facets.py +238 -0
  17. consist-0.1.0/src/consist/core/context.py +125 -0
  18. consist-0.1.0/src/consist/core/coupler.py +282 -0
  19. consist-0.1.0/src/consist/core/decorators.py +163 -0
  20. consist-0.1.0/src/consist/core/drivers.py +475 -0
  21. consist-0.1.0/src/consist/core/error_messages.py +6 -0
  22. consist-0.1.0/src/consist/core/events.py +117 -0
  23. consist-0.1.0/src/consist/core/facet_common.py +123 -0
  24. consist-0.1.0/src/consist/core/fs.py +200 -0
  25. consist-0.1.0/src/consist/core/identity.py +856 -0
  26. consist-0.1.0/src/consist/core/indexing.py +57 -0
  27. consist-0.1.0/src/consist/core/ingestion.py +163 -0
  28. consist-0.1.0/src/consist/core/input_utils.py +21 -0
  29. consist-0.1.0/src/consist/core/lifecycle.py +44 -0
  30. consist-0.1.0/src/consist/core/lineage.py +267 -0
  31. consist-0.1.0/src/consist/core/maintenance.py +3461 -0
  32. consist-0.1.0/src/consist/core/materialize.py +369 -0
  33. consist-0.1.0/src/consist/core/matrix.py +182 -0
  34. consist-0.1.0/src/consist/core/metadata_resolver.py +224 -0
  35. consist-0.1.0/src/consist/core/netcdf_utils.py +10 -0
  36. consist-0.1.0/src/consist/core/netcdf_views.py +447 -0
  37. consist-0.1.0/src/consist/core/noop.py +573 -0
  38. consist-0.1.0/src/consist/core/openlineage.py +296 -0
  39. consist-0.1.0/src/consist/core/openmatrix_views.py +454 -0
  40. consist-0.1.0/src/consist/core/persistence.py +2741 -0
  41. consist-0.1.0/src/consist/core/queries.py +331 -0
  42. consist-0.1.0/src/consist/core/run_invocation.py +623 -0
  43. consist-0.1.0/src/consist/core/run_options.py +156 -0
  44. consist-0.1.0/src/consist/core/run_resolution.py +200 -0
  45. consist-0.1.0/src/consist/core/schema_export.py +397 -0
  46. consist-0.1.0/src/consist/core/settings.py +68 -0
  47. consist-0.1.0/src/consist/core/spatial_views.py +205 -0
  48. consist-0.1.0/src/consist/core/step_context.py +166 -0
  49. consist-0.1.0/src/consist/core/tracker.py +5178 -0
  50. consist-0.1.0/src/consist/core/tracker_artifact_logging.py +269 -0
  51. consist-0.1.0/src/consist/core/tracker_config.py +52 -0
  52. consist-0.1.0/src/consist/core/tracker_lifecycle.py +630 -0
  53. consist-0.1.0/src/consist/core/tracker_orchestration.py +1706 -0
  54. consist-0.1.0/src/consist/core/validation.py +140 -0
  55. consist-0.1.0/src/consist/core/views.py +1026 -0
  56. consist-0.1.0/src/consist/core/workflow.py +1300 -0
  57. consist-0.1.0/src/consist/integrations/.DS_Store +0 -0
  58. consist-0.1.0/src/consist/integrations/__init__.py +5 -0
  59. consist-0.1.0/src/consist/integrations/_config_adapter_shared.py +375 -0
  60. consist-0.1.0/src/consist/integrations/activitysim/__init__.py +10 -0
  61. consist-0.1.0/src/consist/integrations/activitysim/config_adapter.py +2839 -0
  62. consist-0.1.0/src/consist/integrations/beam/__init__.py +9 -0
  63. consist-0.1.0/src/consist/integrations/beam/config_adapter.py +1048 -0
  64. consist-0.1.0/src/consist/integrations/containers/__init__.py +4 -0
  65. consist-0.1.0/src/consist/integrations/containers/api.py +735 -0
  66. consist-0.1.0/src/consist/integrations/containers/backends.py +642 -0
  67. consist-0.1.0/src/consist/integrations/containers/models.py +81 -0
  68. consist-0.1.0/src/consist/integrations/dlt_loader.py +899 -0
  69. consist-0.1.0/src/consist/models/__init__.py +51 -0
  70. consist-0.1.0/src/consist/models/activitysim.py +143 -0
  71. consist-0.1.0/src/consist/models/artifact.py +367 -0
  72. consist-0.1.0/src/consist/models/artifact_facet.py +60 -0
  73. consist-0.1.0/src/consist/models/artifact_kv.py +70 -0
  74. consist-0.1.0/src/consist/models/artifact_schema.py +151 -0
  75. consist-0.1.0/src/consist/models/beam.py +49 -0
  76. consist-0.1.0/src/consist/models/config_facet.py +56 -0
  77. consist-0.1.0/src/consist/models/run.py +389 -0
  78. consist-0.1.0/src/consist/models/run_config_kv.py +66 -0
  79. consist-0.1.0/src/consist/protocols.py +58 -0
  80. consist-0.1.0/src/consist/py.typed +0 -0
  81. consist-0.1.0/src/consist/runset.py +644 -0
  82. consist-0.1.0/src/consist/runtime.py +44 -0
  83. consist-0.1.0/src/consist/tools/__init__.py +4 -0
  84. consist-0.1.0/src/consist/tools/file_batches.py +62 -0
  85. consist-0.1.0/src/consist/tools/generator.py +34 -0
  86. consist-0.1.0/src/consist/tools/mount_diagnostics.py +153 -0
  87. consist-0.1.0/src/consist/tools/queries.py +162 -0
  88. consist-0.1.0/src/consist/tools/schema_profile.py +456 -0
  89. consist-0.1.0/src/consist/types.py +212 -0
  90. consist-0.1.0/src/consist/utils/__init__.py +8 -0
  91. consist-0.1.0/src/consist/utils/introspection.py +108 -0
  92. consist-0.1.0/src/consist/utils/keys.py +51 -0
consist-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,270 @@
1
+ Metadata-Version: 2.3
2
+ Name: consist
3
+ Version: 0.1.0
4
+ Summary: Provenance tracking, intelligent caching, and data virtualization for scientific simulation workflows.
5
+ Keywords: provenance,caching,lineage,reproducibility,duckdb,scientific-computing
6
+ Author: Zach Needell
7
+ Author-email: Zach Needell <zaneedell@lbl.gov>
8
+ Classifier: Development Status :: 5 - Production/Stable
9
+ Classifier: Intended Audience :: Science/Research
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Programming Language :: Python :: 3 :: Only
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Typing :: Typed
15
+ Requires-Dist: duckdb>=1.4.2
16
+ Requires-Dist: duckdb-engine>=0.17.0
17
+ Requires-Dist: pandas>=2.1.0
18
+ Requires-Dist: pydantic>=2.12.4
19
+ Requires-Dist: rich>=13.7.1
20
+ Requires-Dist: sqlalchemy>=2.0.44,<2.0.45
21
+ Requires-Dist: sqlmodel>=0.0.31
22
+ Requires-Dist: tqdm>=4.67.1
23
+ Requires-Dist: ty>=0.0.15
24
+ Requires-Dist: typer>=0.12.3
25
+ Requires-Dist: zensical>=0.0.21
26
+ Requires-Dist: pyyaml>=6.0 ; extra == 'activitysim'
27
+ Requires-Dist: cftime>=1.6.0 ; extra == 'all'
28
+ Requires-Dist: docker>=7.1.0 ; extra == 'all'
29
+ Requires-Dist: dlt>=1.21.0 ; extra == 'all'
30
+ Requires-Dist: geopandas>=0.14.0 ; extra == 'all'
31
+ Requires-Dist: h5netcdf>=1.3.0 ; extra == 'all'
32
+ Requires-Dist: h5py>=3.15.1 ; extra == 'all'
33
+ Requires-Dist: netcdf4>=1.6.0 ; extra == 'all'
34
+ Requires-Dist: openmatrix>=0.3.5.0 ; extra == 'all'
35
+ Requires-Dist: pyarrow>=22.0.0 ; extra == 'all'
36
+ Requires-Dist: pyhocon>=0.3.60 ; extra == 'all'
37
+ Requires-Dist: pyyaml>=6.0 ; extra == 'all'
38
+ Requires-Dist: tables>=3.10.2 ; extra == 'all'
39
+ Requires-Dist: xarray>=2024.9.0 ; extra == 'all'
40
+ Requires-Dist: zarr>=2.18,<3 ; extra == 'all'
41
+ Requires-Dist: pyhocon>=0.3.60 ; extra == 'beam'
42
+ Requires-Dist: polars>=1.35.2 ; extra == 'bench'
43
+ Requires-Dist: psutil>6.0.0 ; extra == 'bench'
44
+ Requires-Dist: dlt>=1.21.0 ; extra == 'dev'
45
+ Requires-Dist: h5py>=3.15.1 ; extra == 'dev'
46
+ Requires-Dist: pyarrow>=22.0.0 ; extra == 'dev'
47
+ Requires-Dist: pytest>=9.0.1 ; extra == 'dev'
48
+ Requires-Dist: pytest-cov>=7.0.0 ; extra == 'dev'
49
+ Requires-Dist: pytest-mock>=3.15.1 ; extra == 'dev'
50
+ Requires-Dist: ruff>=0.14.6 ; extra == 'dev'
51
+ Requires-Dist: tables>=3.10.2 ; extra == 'dev'
52
+ Requires-Dist: docker>=7.1.0 ; extra == 'docker'
53
+ Requires-Dist: mkdocs-material>=9.7.0 ; extra == 'docs'
54
+ Requires-Dist: mkdocstrings[python]>=1.0.0 ; extra == 'docs'
55
+ Requires-Dist: ipykernel>=6.29.0 ; extra == 'examples'
56
+ Requires-Dist: jupyterlab>=4.0.0 ; extra == 'examples'
57
+ Requires-Dist: matplotlib>=3.8.0 ; extra == 'examples'
58
+ Requires-Dist: seaborn>=0.13.0 ; extra == 'examples'
59
+ Requires-Dist: ipywidgets>=8.1.8 ; extra == 'examples'
60
+ Requires-Dist: h5py>=3.15.1 ; extra == 'hdf5'
61
+ Requires-Dist: tables>=3.10.2 ; extra == 'hdf5'
62
+ Requires-Dist: dlt>=1.21.0 ; extra == 'ingest'
63
+ Requires-Dist: pyarrow>=22.0.0 ; extra == 'ingest'
64
+ Requires-Dist: cftime>=1.6.0 ; extra == 'netcdf'
65
+ Requires-Dist: h5netcdf>=1.3.0 ; extra == 'netcdf'
66
+ Requires-Dist: xarray>=2024.9.0 ; extra == 'netcdf'
67
+ Requires-Dist: netcdf4>=1.6.0 ; extra == 'netcdf4'
68
+ Requires-Dist: xarray>=2024.9.0 ; extra == 'netcdf4'
69
+ Requires-Dist: openmatrix>=0.3.5.0 ; extra == 'omx'
70
+ Requires-Dist: pyarrow>=22.0.0 ; extra == 'parquet'
71
+ Requires-Dist: geopandas>=0.14.0 ; extra == 'spatial'
72
+ Requires-Dist: pytest>=9.0.1 ; extra == 'test'
73
+ Requires-Dist: pytest-cov>=7.0.0 ; extra == 'test'
74
+ Requires-Dist: pytest-mock>=3.15.1 ; extra == 'test'
75
+ Requires-Dist: pyhocon>=0.3.60 ; extra == 'test'
76
+ Requires-Dist: ruff>=0.14.6 ; extra == 'test'
77
+ Requires-Dist: pandas-stubs~=2.3.3 ; extra == 'typecheck'
78
+ Requires-Dist: ty>=0.0.7 ; extra == 'typecheck'
79
+ Requires-Dist: xarray>=2024.9.0 ; extra == 'zarr'
80
+ Requires-Dist: zarr>=2.18,<3 ; extra == 'zarr'
81
+ Requires-Python: >=3.11
82
+ Provides-Extra: activitysim
83
+ Provides-Extra: all
84
+ Provides-Extra: beam
85
+ Provides-Extra: bench
86
+ Provides-Extra: dev
87
+ Provides-Extra: docker
88
+ Provides-Extra: docs
89
+ Provides-Extra: examples
90
+ Provides-Extra: hdf5
91
+ Provides-Extra: ingest
92
+ Provides-Extra: netcdf
93
+ Provides-Extra: netcdf4
94
+ Provides-Extra: omx
95
+ Provides-Extra: parquet
96
+ Provides-Extra: spatial
97
+ Provides-Extra: test
98
+ Provides-Extra: typecheck
99
+ Provides-Extra: zarr
100
+ Description-Content-Type: text/markdown
101
+
102
+ <p align="center">
103
+ <img src="docs/assets/logo.png" alt="Consist" width="320">
104
+ </p>
105
+
106
+ <p align="center">
107
+ <a href="https://github.com/LBNL-UCB-STI/consist/actions/workflows/ci.yml">
108
+ <img src="https://github.com/LBNL-UCB-STI/consist/actions/workflows/ci.yml/badge.svg" alt="CI">
109
+ </a>
110
+ <img src="https://img.shields.io/badge/python-3.11+-blue.svg" alt="Python 3.11+">
111
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-BSD--3--Clause-blue.svg" alt="License BSD 3-Clause"></a>
112
+ </p>
113
+
114
+ **Consist** is a caching layer for scientific simulation workflows that makes provenance queryable. It automatically
115
+ records what code, configuration, and input data produced each output in your pipeline—eliminating redundant computation
116
+ and enabling post-hoc inspection of results via SQL.
117
+
118
+ ### Why Consist?
119
+
120
+ Multi-run simulation workflows typically accumulate friction:
121
+
122
+ - **Provenance ambiguity**: "Which configuration produced those results in Figure 3?"
123
+ - **Redundant computation**: Re-running a 4-hour pipeline because you changed one unrelated parameter.
124
+ - **Scattered outputs**: Finding and comparing results across scenario variants manually.
125
+ - **Hidden wiring**: Tools with implicit dependencies (name-based injection, global state) are hard to debug and modify
126
+ when something breaks.
127
+
128
+ Consist tracks lineage explicitly. Tasks are ordinary Python functions; dependencies flow through concrete values, not
129
+ framework magic. Your pipeline remains inspectable and testable.
130
+
131
+ ---
132
+
133
+ ## Installation
134
+
135
+ ```bash
136
+ pip install consist
137
+ ```
138
+
139
+ Optional extras:
140
+
141
+ ```bash
142
+ pip install "consist[parquet]"
143
+ pip install "consist[ingest]"
144
+ ```
145
+
146
+ > [!NOTE]
147
+ > Consist is pre-`1.0`. The library is ready for real workflows, but minor
148
+ > releases may still include breaking changes while the API continues to settle.
149
+
150
+ ---
151
+
152
+ ## Quick Example
153
+
154
+ ```python
155
+ import consist
156
+ from pathlib import Path
157
+ from consist import ExecutionOptions, Tracker
158
+ import pandas as pd
159
+
160
+ tracker = Tracker(run_dir="./runs", db_path="./provenance.duckdb")
161
+
162
+
163
+ def clean_data(raw: Path, threshold: float = 0.5) -> dict[str, Path]:
164
+ df = pd.read_parquet(raw)
165
+ out = Path("./cleaned.parquet")
166
+ df[df["value"] > threshold].to_parquet(out)
167
+ return {"cleaned": out}
168
+
169
+
170
+ # Executes function and records inputs, config, and output artifact
171
+ result = tracker.run(
172
+ fn=clean_data,
173
+ inputs={"raw": Path("raw.parquet")}, # hashed for cache identity
174
+ config={"threshold": 0.5}, # hashed for cache identity
175
+ outputs=["cleaned"],
176
+ execution_options=ExecutionOptions(input_binding="paths"),
177
+ )
178
+
179
+ # Second call with identical inputs: instant cache hit, no execution
180
+ result = tracker.run(
181
+ fn=clean_data,
182
+ inputs={"raw": Path("raw.parquet")},
183
+ config={"threshold": 0.5},
184
+ outputs=["cleaned"],
185
+ execution_options=ExecutionOptions(input_binding="paths"),
186
+ )
187
+
188
+ # Artifact: the output file with provenance metadata attached
189
+ artifact = result.outputs["cleaned"]
190
+ print(artifact.path) # -> PosixPath('./cleaned.parquet')
191
+
192
+ # Load as a DataFrame when needed
193
+ cleaned_df = consist.load_df(artifact)
194
+ ```
195
+
196
+ `input_binding="paths"` keeps the file boundary explicit: the function receives
197
+ the local `Path` values named in `inputs`, while those same inputs still define
198
+ cache identity and lineage.
199
+
200
+ **Summary**: Consist computes a deterministic fingerprint from your code version, config, and input files. If you change
201
+ anything upstream, only affected downstream steps will re-execute.
202
+
203
+ ### Multi-Step Pipeline
204
+
205
+ Dependencies are explicit: the output of one step becomes the input of the next via a concrete reference, not name
206
+ matching or injection.
207
+
208
+ ```python
209
+ def analyze_data(cleaned: Path) -> dict[str, Path]:
210
+ df = pd.read_parquet(cleaned)
211
+ out = Path("./analysis.parquet")
212
+ summary = df.groupby("category")["value"].mean()
213
+ summary.to_parquet(out)
214
+ return {"analysis": out}
215
+
216
+
217
+ preprocess = tracker.run(
218
+ fn=clean_data,
219
+ inputs={"raw": Path("raw.parquet")},
220
+ config={"threshold": 0.5},
221
+ outputs=["cleaned"],
222
+ execution_options=ExecutionOptions(input_binding="paths"),
223
+ )
224
+ analyze = tracker.run(
225
+ fn=analyze_data,
226
+ inputs={"cleaned": consist.ref(preprocess, key="cleaned")}, # explicit artifact reference
227
+ outputs=["analysis"],
228
+ execution_options=ExecutionOptions(input_binding="paths"),
229
+ )
230
+ ```
231
+
232
+ Use `output_paths` when a function returns `None` but writes files, or when you need explicit destination control.
233
+
234
+ ---
235
+
236
+ ## Key Features
237
+
238
+ - **Deterministic Caching**: Cache identity is based on an inspectable fingerprint of code, config, and inputs. Only
239
+ affected downstream steps re-execute when any upstream piece changes.
240
+ - **Plain Python**: Tasks are ordinary Python functions—callable and testable without the tracker. The tracker is
241
+ additive and does not restructure your code.
242
+ - **Complete Lineage**: Every result is tagged with the exact code and config that created it. Trace lineage from any
243
+ output back to its sources.
244
+ - **SQL-Native Analysis**: All metadata is indexed in DuckDB. Query across runs, join tables, and compare variants using
245
+ standard SQL.
246
+ - **HPC and Container Support**: Track Docker and Singularity containers as pure functions—image digests and mounted
247
+ volumes become part of the cache signature. Ideal for long-running jobs on shared compute.
248
+ - **Queryable CLI**: Inspect history, trace lineage, and compare results from the command line after a job completes. No
249
+ code required.
250
+
251
+ ---
252
+
253
+ ## Documentation Index
254
+
255
+ | Section | Description |
256
+ |:----------------------------------------------------------|:-------------------------------------------------------------|
257
+ | **[Getting Started](docs/getting-started/quickstart.md)** | 5-minute guide to your first tracked run. |
258
+ | **[Usage Guide](docs/usage-guide.md)** | Detailed patterns for scenarios and complex workflows. |
259
+ | **[Architecture](docs/architecture.md)** | Deep dive into hashing, lineage, and the DuckDB core. |
260
+ | **[CLI Reference](docs/cli-reference.md)** | Guide to the `consist` command-line tools. |
261
+ | **[DB Maintenance](docs/db-maintenance.md)** | Operational runbooks for inspect/doctor/purge/merge/rebuild. |
262
+ | **[Example Gallery](docs/examples.md)** | Interactive notebooks for Monte Carlo, Demand Modeling, etc. |
263
+
264
+ ---
265
+
266
+ ## Etymology
267
+
268
+ In railroad terminology, a **consist** (noun, pronounced *CON-sist*) is the specific lineup of locomotives and cars that
269
+ make up a train. In this library, a **consist** is the immutable record of exactly which components—code, config, and
270
+ inputs—were coupled together to produce a result.
@@ -0,0 +1,169 @@
1
+ <p align="center">
2
+ <img src="docs/assets/logo.png" alt="Consist" width="320">
3
+ </p>
4
+
5
+ <p align="center">
6
+ <a href="https://github.com/LBNL-UCB-STI/consist/actions/workflows/ci.yml">
7
+ <img src="https://github.com/LBNL-UCB-STI/consist/actions/workflows/ci.yml/badge.svg" alt="CI">
8
+ </a>
9
+ <img src="https://img.shields.io/badge/python-3.11+-blue.svg" alt="Python 3.11+">
10
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-BSD--3--Clause-blue.svg" alt="License BSD 3-Clause"></a>
11
+ </p>
12
+
13
+ **Consist** is a caching layer for scientific simulation workflows that makes provenance queryable. It automatically
14
+ records what code, configuration, and input data produced each output in your pipeline—eliminating redundant computation
15
+ and enabling post-hoc inspection of results via SQL.
16
+
17
+ ### Why Consist?
18
+
19
+ Multi-run simulation workflows typically accumulate friction:
20
+
21
+ - **Provenance ambiguity**: "Which configuration produced those results in Figure 3?"
22
+ - **Redundant computation**: Re-running a 4-hour pipeline because you changed one unrelated parameter.
23
+ - **Scattered outputs**: Finding and comparing results across scenario variants manually.
24
+ - **Hidden wiring**: Tools with implicit dependencies (name-based injection, global state) are hard to debug and modify
25
+ when something breaks.
26
+
27
+ Consist tracks lineage explicitly. Tasks are ordinary Python functions; dependencies flow through concrete values, not
28
+ framework magic. Your pipeline remains inspectable and testable.
29
+
30
+ ---
31
+
32
+ ## Installation
33
+
34
+ ```bash
35
+ pip install consist
36
+ ```
37
+
38
+ Optional extras:
39
+
40
+ ```bash
41
+ pip install "consist[parquet]"
42
+ pip install "consist[ingest]"
43
+ ```
44
+
45
+ > [!NOTE]
46
+ > Consist is pre-`1.0`. The library is ready for real workflows, but minor
47
+ > releases may still include breaking changes while the API continues to settle.
48
+
49
+ ---
50
+
51
+ ## Quick Example
52
+
53
+ ```python
54
+ import consist
55
+ from pathlib import Path
56
+ from consist import ExecutionOptions, Tracker
57
+ import pandas as pd
58
+
59
+ tracker = Tracker(run_dir="./runs", db_path="./provenance.duckdb")
60
+
61
+
62
+ def clean_data(raw: Path, threshold: float = 0.5) -> dict[str, Path]:
63
+ df = pd.read_parquet(raw)
64
+ out = Path("./cleaned.parquet")
65
+ df[df["value"] > threshold].to_parquet(out)
66
+ return {"cleaned": out}
67
+
68
+
69
+ # Executes function and records inputs, config, and output artifact
70
+ result = tracker.run(
71
+ fn=clean_data,
72
+ inputs={"raw": Path("raw.parquet")}, # hashed for cache identity
73
+ config={"threshold": 0.5}, # hashed for cache identity
74
+ outputs=["cleaned"],
75
+ execution_options=ExecutionOptions(input_binding="paths"),
76
+ )
77
+
78
+ # Second call with identical inputs: instant cache hit, no execution
79
+ result = tracker.run(
80
+ fn=clean_data,
81
+ inputs={"raw": Path("raw.parquet")},
82
+ config={"threshold": 0.5},
83
+ outputs=["cleaned"],
84
+ execution_options=ExecutionOptions(input_binding="paths"),
85
+ )
86
+
87
+ # Artifact: the output file with provenance metadata attached
88
+ artifact = result.outputs["cleaned"]
89
+ print(artifact.path) # -> PosixPath('./cleaned.parquet')
90
+
91
+ # Load as a DataFrame when needed
92
+ cleaned_df = consist.load_df(artifact)
93
+ ```
94
+
95
+ `input_binding="paths"` keeps the file boundary explicit: the function receives
96
+ the local `Path` values named in `inputs`, while those same inputs still define
97
+ cache identity and lineage.
98
+
99
+ **Summary**: Consist computes a deterministic fingerprint from your code version, config, and input files. If you change
100
+ anything upstream, only affected downstream steps will re-execute.
101
+
102
+ ### Multi-Step Pipeline
103
+
104
+ Dependencies are explicit: the output of one step becomes the input of the next via a concrete reference, not name
105
+ matching or injection.
106
+
107
+ ```python
108
+ def analyze_data(cleaned: Path) -> dict[str, Path]:
109
+ df = pd.read_parquet(cleaned)
110
+ out = Path("./analysis.parquet")
111
+ summary = df.groupby("category")["value"].mean()
112
+ summary.to_parquet(out)
113
+ return {"analysis": out}
114
+
115
+
116
+ preprocess = tracker.run(
117
+ fn=clean_data,
118
+ inputs={"raw": Path("raw.parquet")},
119
+ config={"threshold": 0.5},
120
+ outputs=["cleaned"],
121
+ execution_options=ExecutionOptions(input_binding="paths"),
122
+ )
123
+ analyze = tracker.run(
124
+ fn=analyze_data,
125
+ inputs={"cleaned": consist.ref(preprocess, key="cleaned")}, # explicit artifact reference
126
+ outputs=["analysis"],
127
+ execution_options=ExecutionOptions(input_binding="paths"),
128
+ )
129
+ ```
130
+
131
+ Use `output_paths` when a function returns `None` but writes files, or when you need explicit destination control.
132
+
133
+ ---
134
+
135
+ ## Key Features
136
+
137
+ - **Deterministic Caching**: Cache identity is based on an inspectable fingerprint of code, config, and inputs. Only
138
+ affected downstream steps re-execute when any upstream piece changes.
139
+ - **Plain Python**: Tasks are ordinary Python functions—callable and testable without the tracker. The tracker is
140
+ additive and does not restructure your code.
141
+ - **Complete Lineage**: Every result is tagged with the exact code and config that created it. Trace lineage from any
142
+ output back to its sources.
143
+ - **SQL-Native Analysis**: All metadata is indexed in DuckDB. Query across runs, join tables, and compare variants using
144
+ standard SQL.
145
+ - **HPC and Container Support**: Track Docker and Singularity containers as pure functions—image digests and mounted
146
+ volumes become part of the cache signature. Ideal for long-running jobs on shared compute.
147
+ - **Queryable CLI**: Inspect history, trace lineage, and compare results from the command line after a job completes. No
148
+ code required.
149
+
150
+ ---
151
+
152
+ ## Documentation Index
153
+
154
+ | Section | Description |
155
+ |:----------------------------------------------------------|:-------------------------------------------------------------|
156
+ | **[Getting Started](docs/getting-started/quickstart.md)** | 5-minute guide to your first tracked run. |
157
+ | **[Usage Guide](docs/usage-guide.md)** | Detailed patterns for scenarios and complex workflows. |
158
+ | **[Architecture](docs/architecture.md)** | Deep dive into hashing, lineage, and the DuckDB core. |
159
+ | **[CLI Reference](docs/cli-reference.md)** | Guide to the `consist` command-line tools. |
160
+ | **[DB Maintenance](docs/db-maintenance.md)** | Operational runbooks for inspect/doctor/purge/merge/rebuild. |
161
+ | **[Example Gallery](docs/examples.md)** | Interactive notebooks for Monte Carlo, Demand Modeling, etc. |
162
+
163
+ ---
164
+
165
+ ## Etymology
166
+
167
+ In railroad terminology, a **consist** (noun, pronounced *CON-sist*) is the specific lineup of locomotives and cars that
168
+ make up a train. In this library, a **consist** is the immutable record of exactly which components—code, config, and
169
+ inputs—were coupled together to produce a result.
@@ -0,0 +1,152 @@
1
+ [project]
2
+ name = "consist"
3
+ version = "0.1.0"
4
+ description = "Provenance tracking, intelligent caching, and data virtualization for scientific simulation workflows."
5
+ readme = "README.md"
6
+ authors = [
7
+ { name = "Zach Needell", email = "zaneedell@lbl.gov" }
8
+ ]
9
+ requires-python = ">=3.11"
10
+ dependencies = [
11
+ "duckdb>=1.4.2",
12
+ "duckdb-engine>=0.17.0",
13
+ "pandas>=2.1.0",
14
+ "pydantic>=2.12.4",
15
+ "rich>=13.7.1",
16
+ "sqlalchemy>=2.0.44,<2.0.45",
17
+ "sqlmodel>=0.0.31",
18
+ "tqdm>=4.67.1",
19
+ "ty>=0.0.15",
20
+ "typer>=0.12.3",
21
+ "zensical>=0.0.21",
22
+ ]
23
+ keywords = ["provenance", "caching", "lineage", "reproducibility", "duckdb", "scientific-computing"]
24
+ classifiers = [
25
+ "Development Status :: 5 - Production/Stable",
26
+ "Intended Audience :: Science/Research",
27
+ "Operating System :: OS Independent",
28
+ "Programming Language :: Python :: 3 :: Only",
29
+ "Programming Language :: Python :: 3.11",
30
+ "Programming Language :: Python :: 3.12",
31
+ "Typing :: Typed",
32
+ ]
33
+
34
+ [project.scripts]
35
+ consist = "consist.cli:app"
36
+
37
+ [project.optional-dependencies]
38
+ # Optional runtime integrations (install as needed).
39
+ docker = ["docker>=7.1.0"]
40
+ hdf5 = ["h5py>=3.15.1", "tables>=3.10.2"]
41
+ ingest = ["dlt>=1.21.0", "pyarrow>=22.0.0"]
42
+ activitysim = ["PyYAML>=6.0"]
43
+ beam = ["pyhocon>=0.3.60"]
44
+ netcdf = ["cftime>=1.6.0", "h5netcdf>=1.3.0", "xarray>=2024.9.0"]
45
+ netcdf4 = ["netCDF4>=1.6.0", "xarray>=2024.9.0"]
46
+ parquet = ["pyarrow>=22.0.0"]
47
+ spatial = ["geopandas>=0.14.0"]
48
+ zarr = ["xarray>=2024.9.0", "zarr>=2.18,<3"]
49
+ omx = ["openmatrix>=0.3.5.0"]
50
+ examples = [
51
+ "ipykernel>=6.29.0",
52
+ "jupyterlab>=4.0.0",
53
+ "matplotlib>=3.8.0",
54
+ "seaborn>=0.13.0",
55
+ "ipywidgets>=8.1.8"
56
+ ]
57
+ typecheck = [
58
+ "pandas-stubs~=2.3.3",
59
+ "ty>=0.0.7",
60
+ ]
61
+
62
+ # Convenience extras.
63
+ all = [
64
+ "cftime>=1.6.0",
65
+ "docker>=7.1.0",
66
+ "dlt>=1.21.0",
67
+ "geopandas>=0.14.0",
68
+ "h5netcdf>=1.3.0",
69
+ "h5py>=3.15.1",
70
+ "netCDF4>=1.6.0",
71
+ "openmatrix>=0.3.5.0",
72
+ "pyarrow>=22.0.0",
73
+ "pyhocon>=0.3.60",
74
+ "PyYAML>=6.0",
75
+ "tables>=3.10.2",
76
+ "xarray>=2024.9.0",
77
+ "zarr>=2.18,<3",
78
+ ]
79
+ bench = ["polars>=1.35.2", "psutil>6.0.0"]
80
+ docs = ["mkdocs-material>=9.7.0", "mkdocstrings[python]>=1.0.0"]
81
+ # Test + lint tools used by CI and local development.
82
+ test = [
83
+ "pytest>=9.0.1",
84
+ "pytest-cov>=7.0.0",
85
+ "pytest-mock>=3.15.1",
86
+ "pyhocon>=0.3.60",
87
+ "ruff>=0.14.6",
88
+ ]
89
+ dev = [
90
+ "dlt>=1.21.0",
91
+ "h5py>=3.15.1",
92
+ "pyarrow>=22.0.0",
93
+ "pytest>=9.0.1",
94
+ "pytest-cov>=7.0.0",
95
+ "pytest-mock>=3.15.1",
96
+ "ruff>=0.14.6",
97
+ "tables>=3.10.2",
98
+ ]
99
+
100
+ [build-system]
101
+ requires = ["uv_build>=0.8.15,<0.9.0"]
102
+ build-backend = "uv_build"
103
+
104
+ [dependency-groups]
105
+ bench = [
106
+ "psutil>=7.1.3",
107
+ ]
108
+ examples = [
109
+ "ipykernel>=6.29.0",
110
+ "jupyter>=1.1.1",
111
+ "jupyterlab>=4.0.0",
112
+ "matplotlib>=3.8.0",
113
+ "seaborn>=0.13.0",
114
+ ]
115
+ dev = [
116
+ "dlt>=1.21.0",
117
+ "h5py>=3.15.1",
118
+ "pyarrow>=22.0.0",
119
+ "pytest>=9.0.1",
120
+ "pytest-cov>=7.0.0",
121
+ "pytest-mock>=3.15.1",
122
+ "ruff>=0.14.6",
123
+ "tables>=3.10.2",
124
+ ]
125
+ docs = [
126
+ "mkdocs-material>=9.7.0",
127
+ "mkdocstrings[python]>=1.0.0",
128
+ "mkdocstrings-python>=2.0.0",
129
+ "zensical>=0.0.24",
130
+ ]
131
+
132
+ [tool.pytest.ini_options]
133
+ markers = [
134
+ "heavy: marks tests as heavyweight/long-running",
135
+ "flaky: marks tests as flaky/retryable",
136
+ "timeout: marks tests with explicit timeout expectations",
137
+ ]
138
+
139
+ [tool.ruff]
140
+ target-version = "py311"
141
+ line-length = 88
142
+
143
+ [tool.ty.src]
144
+ include = ["src", "tests"]
145
+
146
+ [[tool.ty.overrides]]
147
+ include = ["src/consist/integrations/**"]
148
+ rules = { "invalid-assignment" = "ignore" }
149
+
150
+ [[tool.ty.overrides]]
151
+ include = ["tests/**"]
152
+ rules = { "invalid-argument-type" = "ignore", "invalid-assignment" = "ignore", "not-subscriptable" = "ignore", "possibly-missing-attribute" = "ignore", "unresolved-attribute" = "ignore" }
Binary file