consist 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- consist-0.1.0/PKG-INFO +270 -0
- consist-0.1.0/README.md +169 -0
- consist-0.1.0/pyproject.toml +152 -0
- consist-0.1.0/src/consist/.DS_Store +0 -0
- consist-0.1.0/src/consist/__init__.py +187 -0
- consist-0.1.0/src/consist/api.py +2672 -0
- consist-0.1.0/src/consist/cli.py +3882 -0
- consist-0.1.0/src/consist/core/__init__.py +4 -0
- consist-0.1.0/src/consist/core/_coupler_shared.py +266 -0
- consist-0.1.0/src/consist/core/artifact_facets.py +145 -0
- consist-0.1.0/src/consist/core/artifact_schemas.py +658 -0
- consist-0.1.0/src/consist/core/artifacts.py +741 -0
- consist-0.1.0/src/consist/core/cache.py +501 -0
- consist-0.1.0/src/consist/core/cache_output_logging.py +155 -0
- consist-0.1.0/src/consist/core/config_canonicalization.py +543 -0
- consist-0.1.0/src/consist/core/config_facets.py +238 -0
- consist-0.1.0/src/consist/core/context.py +125 -0
- consist-0.1.0/src/consist/core/coupler.py +282 -0
- consist-0.1.0/src/consist/core/decorators.py +163 -0
- consist-0.1.0/src/consist/core/drivers.py +475 -0
- consist-0.1.0/src/consist/core/error_messages.py +6 -0
- consist-0.1.0/src/consist/core/events.py +117 -0
- consist-0.1.0/src/consist/core/facet_common.py +123 -0
- consist-0.1.0/src/consist/core/fs.py +200 -0
- consist-0.1.0/src/consist/core/identity.py +856 -0
- consist-0.1.0/src/consist/core/indexing.py +57 -0
- consist-0.1.0/src/consist/core/ingestion.py +163 -0
- consist-0.1.0/src/consist/core/input_utils.py +21 -0
- consist-0.1.0/src/consist/core/lifecycle.py +44 -0
- consist-0.1.0/src/consist/core/lineage.py +267 -0
- consist-0.1.0/src/consist/core/maintenance.py +3461 -0
- consist-0.1.0/src/consist/core/materialize.py +369 -0
- consist-0.1.0/src/consist/core/matrix.py +182 -0
- consist-0.1.0/src/consist/core/metadata_resolver.py +224 -0
- consist-0.1.0/src/consist/core/netcdf_utils.py +10 -0
- consist-0.1.0/src/consist/core/netcdf_views.py +447 -0
- consist-0.1.0/src/consist/core/noop.py +573 -0
- consist-0.1.0/src/consist/core/openlineage.py +296 -0
- consist-0.1.0/src/consist/core/openmatrix_views.py +454 -0
- consist-0.1.0/src/consist/core/persistence.py +2741 -0
- consist-0.1.0/src/consist/core/queries.py +331 -0
- consist-0.1.0/src/consist/core/run_invocation.py +623 -0
- consist-0.1.0/src/consist/core/run_options.py +156 -0
- consist-0.1.0/src/consist/core/run_resolution.py +200 -0
- consist-0.1.0/src/consist/core/schema_export.py +397 -0
- consist-0.1.0/src/consist/core/settings.py +68 -0
- consist-0.1.0/src/consist/core/spatial_views.py +205 -0
- consist-0.1.0/src/consist/core/step_context.py +166 -0
- consist-0.1.0/src/consist/core/tracker.py +5178 -0
- consist-0.1.0/src/consist/core/tracker_artifact_logging.py +269 -0
- consist-0.1.0/src/consist/core/tracker_config.py +52 -0
- consist-0.1.0/src/consist/core/tracker_lifecycle.py +630 -0
- consist-0.1.0/src/consist/core/tracker_orchestration.py +1706 -0
- consist-0.1.0/src/consist/core/validation.py +140 -0
- consist-0.1.0/src/consist/core/views.py +1026 -0
- consist-0.1.0/src/consist/core/workflow.py +1300 -0
- consist-0.1.0/src/consist/integrations/.DS_Store +0 -0
- consist-0.1.0/src/consist/integrations/__init__.py +5 -0
- consist-0.1.0/src/consist/integrations/_config_adapter_shared.py +375 -0
- consist-0.1.0/src/consist/integrations/activitysim/__init__.py +10 -0
- consist-0.1.0/src/consist/integrations/activitysim/config_adapter.py +2839 -0
- consist-0.1.0/src/consist/integrations/beam/__init__.py +9 -0
- consist-0.1.0/src/consist/integrations/beam/config_adapter.py +1048 -0
- consist-0.1.0/src/consist/integrations/containers/__init__.py +4 -0
- consist-0.1.0/src/consist/integrations/containers/api.py +735 -0
- consist-0.1.0/src/consist/integrations/containers/backends.py +642 -0
- consist-0.1.0/src/consist/integrations/containers/models.py +81 -0
- consist-0.1.0/src/consist/integrations/dlt_loader.py +899 -0
- consist-0.1.0/src/consist/models/__init__.py +51 -0
- consist-0.1.0/src/consist/models/activitysim.py +143 -0
- consist-0.1.0/src/consist/models/artifact.py +367 -0
- consist-0.1.0/src/consist/models/artifact_facet.py +60 -0
- consist-0.1.0/src/consist/models/artifact_kv.py +70 -0
- consist-0.1.0/src/consist/models/artifact_schema.py +151 -0
- consist-0.1.0/src/consist/models/beam.py +49 -0
- consist-0.1.0/src/consist/models/config_facet.py +56 -0
- consist-0.1.0/src/consist/models/run.py +389 -0
- consist-0.1.0/src/consist/models/run_config_kv.py +66 -0
- consist-0.1.0/src/consist/protocols.py +58 -0
- consist-0.1.0/src/consist/py.typed +0 -0
- consist-0.1.0/src/consist/runset.py +644 -0
- consist-0.1.0/src/consist/runtime.py +44 -0
- consist-0.1.0/src/consist/tools/__init__.py +4 -0
- consist-0.1.0/src/consist/tools/file_batches.py +62 -0
- consist-0.1.0/src/consist/tools/generator.py +34 -0
- consist-0.1.0/src/consist/tools/mount_diagnostics.py +153 -0
- consist-0.1.0/src/consist/tools/queries.py +162 -0
- consist-0.1.0/src/consist/tools/schema_profile.py +456 -0
- consist-0.1.0/src/consist/types.py +212 -0
- consist-0.1.0/src/consist/utils/__init__.py +8 -0
- consist-0.1.0/src/consist/utils/introspection.py +108 -0
- consist-0.1.0/src/consist/utils/keys.py +51 -0
consist-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: consist
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Provenance tracking, intelligent caching, and data virtualization for scientific simulation workflows.
|
|
5
|
+
Keywords: provenance,caching,lineage,reproducibility,duckdb,scientific-computing
|
|
6
|
+
Author: Zach Needell
|
|
7
|
+
Author-email: Zach Needell <zaneedell@lbl.gov>
|
|
8
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Dist: duckdb>=1.4.2
|
|
16
|
+
Requires-Dist: duckdb-engine>=0.17.0
|
|
17
|
+
Requires-Dist: pandas>=2.1.0
|
|
18
|
+
Requires-Dist: pydantic>=2.12.4
|
|
19
|
+
Requires-Dist: rich>=13.7.1
|
|
20
|
+
Requires-Dist: sqlalchemy>=2.0.44,<2.0.45
|
|
21
|
+
Requires-Dist: sqlmodel>=0.0.31
|
|
22
|
+
Requires-Dist: tqdm>=4.67.1
|
|
23
|
+
Requires-Dist: ty>=0.0.15
|
|
24
|
+
Requires-Dist: typer>=0.12.3
|
|
25
|
+
Requires-Dist: zensical>=0.0.21
|
|
26
|
+
Requires-Dist: pyyaml>=6.0 ; extra == 'activitysim'
|
|
27
|
+
Requires-Dist: cftime>=1.6.0 ; extra == 'all'
|
|
28
|
+
Requires-Dist: docker>=7.1.0 ; extra == 'all'
|
|
29
|
+
Requires-Dist: dlt>=1.21.0 ; extra == 'all'
|
|
30
|
+
Requires-Dist: geopandas>=0.14.0 ; extra == 'all'
|
|
31
|
+
Requires-Dist: h5netcdf>=1.3.0 ; extra == 'all'
|
|
32
|
+
Requires-Dist: h5py>=3.15.1 ; extra == 'all'
|
|
33
|
+
Requires-Dist: netcdf4>=1.6.0 ; extra == 'all'
|
|
34
|
+
Requires-Dist: openmatrix>=0.3.5.0 ; extra == 'all'
|
|
35
|
+
Requires-Dist: pyarrow>=22.0.0 ; extra == 'all'
|
|
36
|
+
Requires-Dist: pyhocon>=0.3.60 ; extra == 'all'
|
|
37
|
+
Requires-Dist: pyyaml>=6.0 ; extra == 'all'
|
|
38
|
+
Requires-Dist: tables>=3.10.2 ; extra == 'all'
|
|
39
|
+
Requires-Dist: xarray>=2024.9.0 ; extra == 'all'
|
|
40
|
+
Requires-Dist: zarr>=2.18,<3 ; extra == 'all'
|
|
41
|
+
Requires-Dist: pyhocon>=0.3.60 ; extra == 'beam'
|
|
42
|
+
Requires-Dist: polars>=1.35.2 ; extra == 'bench'
|
|
43
|
+
Requires-Dist: psutil>6.0.0 ; extra == 'bench'
|
|
44
|
+
Requires-Dist: dlt>=1.21.0 ; extra == 'dev'
|
|
45
|
+
Requires-Dist: h5py>=3.15.1 ; extra == 'dev'
|
|
46
|
+
Requires-Dist: pyarrow>=22.0.0 ; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest>=9.0.1 ; extra == 'dev'
|
|
48
|
+
Requires-Dist: pytest-cov>=7.0.0 ; extra == 'dev'
|
|
49
|
+
Requires-Dist: pytest-mock>=3.15.1 ; extra == 'dev'
|
|
50
|
+
Requires-Dist: ruff>=0.14.6 ; extra == 'dev'
|
|
51
|
+
Requires-Dist: tables>=3.10.2 ; extra == 'dev'
|
|
52
|
+
Requires-Dist: docker>=7.1.0 ; extra == 'docker'
|
|
53
|
+
Requires-Dist: mkdocs-material>=9.7.0 ; extra == 'docs'
|
|
54
|
+
Requires-Dist: mkdocstrings[python]>=1.0.0 ; extra == 'docs'
|
|
55
|
+
Requires-Dist: ipykernel>=6.29.0 ; extra == 'examples'
|
|
56
|
+
Requires-Dist: jupyterlab>=4.0.0 ; extra == 'examples'
|
|
57
|
+
Requires-Dist: matplotlib>=3.8.0 ; extra == 'examples'
|
|
58
|
+
Requires-Dist: seaborn>=0.13.0 ; extra == 'examples'
|
|
59
|
+
Requires-Dist: ipywidgets>=8.1.8 ; extra == 'examples'
|
|
60
|
+
Requires-Dist: h5py>=3.15.1 ; extra == 'hdf5'
|
|
61
|
+
Requires-Dist: tables>=3.10.2 ; extra == 'hdf5'
|
|
62
|
+
Requires-Dist: dlt>=1.21.0 ; extra == 'ingest'
|
|
63
|
+
Requires-Dist: pyarrow>=22.0.0 ; extra == 'ingest'
|
|
64
|
+
Requires-Dist: cftime>=1.6.0 ; extra == 'netcdf'
|
|
65
|
+
Requires-Dist: h5netcdf>=1.3.0 ; extra == 'netcdf'
|
|
66
|
+
Requires-Dist: xarray>=2024.9.0 ; extra == 'netcdf'
|
|
67
|
+
Requires-Dist: netcdf4>=1.6.0 ; extra == 'netcdf4'
|
|
68
|
+
Requires-Dist: xarray>=2024.9.0 ; extra == 'netcdf4'
|
|
69
|
+
Requires-Dist: openmatrix>=0.3.5.0 ; extra == 'omx'
|
|
70
|
+
Requires-Dist: pyarrow>=22.0.0 ; extra == 'parquet'
|
|
71
|
+
Requires-Dist: geopandas>=0.14.0 ; extra == 'spatial'
|
|
72
|
+
Requires-Dist: pytest>=9.0.1 ; extra == 'test'
|
|
73
|
+
Requires-Dist: pytest-cov>=7.0.0 ; extra == 'test'
|
|
74
|
+
Requires-Dist: pytest-mock>=3.15.1 ; extra == 'test'
|
|
75
|
+
Requires-Dist: pyhocon>=0.3.60 ; extra == 'test'
|
|
76
|
+
Requires-Dist: ruff>=0.14.6 ; extra == 'test'
|
|
77
|
+
Requires-Dist: pandas-stubs~=2.3.3 ; extra == 'typecheck'
|
|
78
|
+
Requires-Dist: ty>=0.0.7 ; extra == 'typecheck'
|
|
79
|
+
Requires-Dist: xarray>=2024.9.0 ; extra == 'zarr'
|
|
80
|
+
Requires-Dist: zarr>=2.18,<3 ; extra == 'zarr'
|
|
81
|
+
Requires-Python: >=3.11
|
|
82
|
+
Provides-Extra: activitysim
|
|
83
|
+
Provides-Extra: all
|
|
84
|
+
Provides-Extra: beam
|
|
85
|
+
Provides-Extra: bench
|
|
86
|
+
Provides-Extra: dev
|
|
87
|
+
Provides-Extra: docker
|
|
88
|
+
Provides-Extra: docs
|
|
89
|
+
Provides-Extra: examples
|
|
90
|
+
Provides-Extra: hdf5
|
|
91
|
+
Provides-Extra: ingest
|
|
92
|
+
Provides-Extra: netcdf
|
|
93
|
+
Provides-Extra: netcdf4
|
|
94
|
+
Provides-Extra: omx
|
|
95
|
+
Provides-Extra: parquet
|
|
96
|
+
Provides-Extra: spatial
|
|
97
|
+
Provides-Extra: test
|
|
98
|
+
Provides-Extra: typecheck
|
|
99
|
+
Provides-Extra: zarr
|
|
100
|
+
Description-Content-Type: text/markdown
|
|
101
|
+
|
|
102
|
+
<p align="center">
|
|
103
|
+
<img src="docs/assets/logo.png" alt="Consist" width="320">
|
|
104
|
+
</p>
|
|
105
|
+
|
|
106
|
+
<p align="center">
|
|
107
|
+
<a href="https://github.com/LBNL-UCB-STI/consist/actions/workflows/ci.yml">
|
|
108
|
+
<img src="https://github.com/LBNL-UCB-STI/consist/actions/workflows/ci.yml/badge.svg" alt="CI">
|
|
109
|
+
</a>
|
|
110
|
+
<img src="https://img.shields.io/badge/python-3.11+-blue.svg" alt="Python 3.11+">
|
|
111
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-BSD--3--Clause-blue.svg" alt="License BSD 3-Clause"></a>
|
|
112
|
+
</p>
|
|
113
|
+
|
|
114
|
+
**Consist** is a caching layer for scientific simulation workflows that makes provenance queryable. It automatically
|
|
115
|
+
records what code, configuration, and input data produced each output in your pipeline—eliminating redundant computation
|
|
116
|
+
and enabling post-hoc inspection of results via SQL.
|
|
117
|
+
|
|
118
|
+
### Why Consist?
|
|
119
|
+
|
|
120
|
+
Multi-run simulation workflows typically accumulate friction:
|
|
121
|
+
|
|
122
|
+
- **Provenance ambiguity**: "Which configuration produced those results in Figure 3?"
|
|
123
|
+
- **Redundant computation**: Re-running a 4-hour pipeline because you changed one unrelated parameter.
|
|
124
|
+
- **Scattered outputs**: Finding and comparing results across scenario variants manually.
|
|
125
|
+
- **Hidden wiring**: Tools with implicit dependencies (name-based injection, global state) are hard to debug and modify
|
|
126
|
+
when something breaks.
|
|
127
|
+
|
|
128
|
+
Consist tracks lineage explicitly. Tasks are ordinary Python functions; dependencies flow through concrete values, not
|
|
129
|
+
framework magic. Your pipeline remains inspectable and testable.
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
## Installation
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
pip install consist
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Optional extras:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
pip install "consist[parquet]"
|
|
143
|
+
pip install "consist[ingest]"
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
> [!NOTE]
|
|
147
|
+
> Consist is pre-`1.0`. The library is ready for real workflows, but minor
|
|
148
|
+
> releases may still include breaking changes while the API continues to settle.
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
## Quick Example
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
import consist
|
|
156
|
+
from pathlib import Path
|
|
157
|
+
from consist import ExecutionOptions, Tracker
|
|
158
|
+
import pandas as pd
|
|
159
|
+
|
|
160
|
+
tracker = Tracker(run_dir="./runs", db_path="./provenance.duckdb")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def clean_data(raw: Path, threshold: float = 0.5) -> dict[str, Path]:
|
|
164
|
+
df = pd.read_parquet(raw)
|
|
165
|
+
out = Path("./cleaned.parquet")
|
|
166
|
+
df[df["value"] > threshold].to_parquet(out)
|
|
167
|
+
return {"cleaned": out}
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# Executes function and records inputs, config, and output artifact
|
|
171
|
+
result = tracker.run(
|
|
172
|
+
fn=clean_data,
|
|
173
|
+
inputs={"raw": Path("raw.parquet")}, # hashed for cache identity
|
|
174
|
+
config={"threshold": 0.5}, # hashed for cache identity
|
|
175
|
+
outputs=["cleaned"],
|
|
176
|
+
execution_options=ExecutionOptions(input_binding="paths"),
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
# Second call with identical inputs: instant cache hit, no execution
|
|
180
|
+
result = tracker.run(
|
|
181
|
+
fn=clean_data,
|
|
182
|
+
inputs={"raw": Path("raw.parquet")},
|
|
183
|
+
config={"threshold": 0.5},
|
|
184
|
+
outputs=["cleaned"],
|
|
185
|
+
execution_options=ExecutionOptions(input_binding="paths"),
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# Artifact: the output file with provenance metadata attached
|
|
189
|
+
artifact = result.outputs["cleaned"]
|
|
190
|
+
print(artifact.path) # -> PosixPath('./cleaned.parquet')
|
|
191
|
+
|
|
192
|
+
# Load as a DataFrame when needed
|
|
193
|
+
cleaned_df = consist.load_df(artifact)
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
`input_binding="paths"` keeps the file boundary explicit: the function receives
|
|
197
|
+
the local `Path` values named in `inputs`, while those same inputs still define
|
|
198
|
+
cache identity and lineage.
|
|
199
|
+
|
|
200
|
+
**Summary**: Consist computes a deterministic fingerprint from your code version, config, and input files. If you change
|
|
201
|
+
anything upstream, only affected downstream steps will re-execute.
|
|
202
|
+
|
|
203
|
+
### Multi-Step Pipeline
|
|
204
|
+
|
|
205
|
+
Dependencies are explicit: the output of one step becomes the input of the next via a concrete reference, not name
|
|
206
|
+
matching or injection.
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
def analyze_data(cleaned: Path) -> dict[str, Path]:
|
|
210
|
+
df = pd.read_parquet(cleaned)
|
|
211
|
+
out = Path("./analysis.parquet")
|
|
212
|
+
summary = df.groupby("category")["value"].mean()
|
|
213
|
+
summary.to_parquet(out)
|
|
214
|
+
return {"analysis": out}
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
preprocess = tracker.run(
|
|
218
|
+
fn=clean_data,
|
|
219
|
+
inputs={"raw": Path("raw.parquet")},
|
|
220
|
+
config={"threshold": 0.5},
|
|
221
|
+
outputs=["cleaned"],
|
|
222
|
+
execution_options=ExecutionOptions(input_binding="paths"),
|
|
223
|
+
)
|
|
224
|
+
analyze = tracker.run(
|
|
225
|
+
fn=analyze_data,
|
|
226
|
+
inputs={"cleaned": consist.ref(preprocess, key="cleaned")}, # explicit artifact reference
|
|
227
|
+
outputs=["analysis"],
|
|
228
|
+
execution_options=ExecutionOptions(input_binding="paths"),
|
|
229
|
+
)
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Use `output_paths` when a function returns `None` but writes files, or when you need explicit destination control.
|
|
233
|
+
|
|
234
|
+
---
|
|
235
|
+
|
|
236
|
+
## Key Features
|
|
237
|
+
|
|
238
|
+
- **Deterministic Caching**: Cache identity is based on an inspectable fingerprint of code, config, and inputs. Only
|
|
239
|
+
affected downstream steps re-execute when any upstream piece changes.
|
|
240
|
+
- **Plain Python**: Tasks are ordinary Python functions—callable and testable without the tracker. The tracker is
|
|
241
|
+
additive and does not restructure your code.
|
|
242
|
+
- **Complete Lineage**: Every result is tagged with the exact code and config that created it. Trace lineage from any
|
|
243
|
+
output back to its sources.
|
|
244
|
+
- **SQL-Native Analysis**: All metadata is indexed in DuckDB. Query across runs, join tables, and compare variants using
|
|
245
|
+
standard SQL.
|
|
246
|
+
- **HPC and Container Support**: Track Docker and Singularity containers as pure functions—image digests and mounted
|
|
247
|
+
volumes become part of the cache signature. Ideal for long-running jobs on shared compute.
|
|
248
|
+
- **Queryable CLI**: Inspect history, trace lineage, and compare results from the command line after a job completes. No
|
|
249
|
+
code required.
|
|
250
|
+
|
|
251
|
+
---
|
|
252
|
+
|
|
253
|
+
## Documentation Index
|
|
254
|
+
|
|
255
|
+
| Section | Description |
|
|
256
|
+
|:----------------------------------------------------------|:-------------------------------------------------------------|
|
|
257
|
+
| **[Getting Started](docs/getting-started/quickstart.md)** | 5-minute guide to your first tracked run. |
|
|
258
|
+
| **[Usage Guide](docs/usage-guide.md)** | Detailed patterns for scenarios and complex workflows. |
|
|
259
|
+
| **[Architecture](docs/architecture.md)** | Deep dive into hashing, lineage, and the DuckDB core. |
|
|
260
|
+
| **[CLI Reference](docs/cli-reference.md)** | Guide to the `consist` command-line tools. |
|
|
261
|
+
| **[DB Maintenance](docs/db-maintenance.md)** | Operational runbooks for inspect/doctor/purge/merge/rebuild. |
|
|
262
|
+
| **[Example Gallery](docs/examples.md)** | Interactive notebooks for Monte Carlo, Demand Modeling, etc. |
|
|
263
|
+
|
|
264
|
+
---
|
|
265
|
+
|
|
266
|
+
## Etymology
|
|
267
|
+
|
|
268
|
+
In railroad terminology, a **consist** (noun, pronounced *CON-sist*) is the specific lineup of locomotives and cars that
|
|
269
|
+
make up a train. In this library, a **consist** is the immutable record of exactly which components—code, config, and
|
|
270
|
+
inputs—were coupled together to produce a result.
|
consist-0.1.0/README.md
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="docs/assets/logo.png" alt="Consist" width="320">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
<a href="https://github.com/LBNL-UCB-STI/consist/actions/workflows/ci.yml">
|
|
7
|
+
<img src="https://github.com/LBNL-UCB-STI/consist/actions/workflows/ci.yml/badge.svg" alt="CI">
|
|
8
|
+
</a>
|
|
9
|
+
<img src="https://img.shields.io/badge/python-3.11+-blue.svg" alt="Python 3.11+">
|
|
10
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-BSD--3--Clause-blue.svg" alt="License BSD 3-Clause"></a>
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
**Consist** is a caching layer for scientific simulation workflows that makes provenance queryable. It automatically
|
|
14
|
+
records what code, configuration, and input data produced each output in your pipeline—eliminating redundant computation
|
|
15
|
+
and enabling post-hoc inspection of results via SQL.
|
|
16
|
+
|
|
17
|
+
### Why Consist?
|
|
18
|
+
|
|
19
|
+
Multi-run simulation workflows typically accumulate friction:
|
|
20
|
+
|
|
21
|
+
- **Provenance ambiguity**: "Which configuration produced those results in Figure 3?"
|
|
22
|
+
- **Redundant computation**: Re-running a 4-hour pipeline because you changed one unrelated parameter.
|
|
23
|
+
- **Scattered outputs**: Finding and comparing results across scenario variants manually.
|
|
24
|
+
- **Hidden wiring**: Tools with implicit dependencies (name-based injection, global state) are hard to debug and modify
|
|
25
|
+
when something breaks.
|
|
26
|
+
|
|
27
|
+
Consist tracks lineage explicitly. Tasks are ordinary Python functions; dependencies flow through concrete values, not
|
|
28
|
+
framework magic. Your pipeline remains inspectable and testable.
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install consist
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Optional extras:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install "consist[parquet]"
|
|
42
|
+
pip install "consist[ingest]"
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
> [!NOTE]
|
|
46
|
+
> Consist is pre-`1.0`. The library is ready for real workflows, but minor
|
|
47
|
+
> releases may still include breaking changes while the API continues to settle.
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## Quick Example
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
import consist
|
|
55
|
+
from pathlib import Path
|
|
56
|
+
from consist import ExecutionOptions, Tracker
|
|
57
|
+
import pandas as pd
|
|
58
|
+
|
|
59
|
+
tracker = Tracker(run_dir="./runs", db_path="./provenance.duckdb")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def clean_data(raw: Path, threshold: float = 0.5) -> dict[str, Path]:
|
|
63
|
+
df = pd.read_parquet(raw)
|
|
64
|
+
out = Path("./cleaned.parquet")
|
|
65
|
+
df[df["value"] > threshold].to_parquet(out)
|
|
66
|
+
return {"cleaned": out}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
# Executes function and records inputs, config, and output artifact
|
|
70
|
+
result = tracker.run(
|
|
71
|
+
fn=clean_data,
|
|
72
|
+
inputs={"raw": Path("raw.parquet")}, # hashed for cache identity
|
|
73
|
+
config={"threshold": 0.5}, # hashed for cache identity
|
|
74
|
+
outputs=["cleaned"],
|
|
75
|
+
execution_options=ExecutionOptions(input_binding="paths"),
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
# Second call with identical inputs: instant cache hit, no execution
|
|
79
|
+
result = tracker.run(
|
|
80
|
+
fn=clean_data,
|
|
81
|
+
inputs={"raw": Path("raw.parquet")},
|
|
82
|
+
config={"threshold": 0.5},
|
|
83
|
+
outputs=["cleaned"],
|
|
84
|
+
execution_options=ExecutionOptions(input_binding="paths"),
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Artifact: the output file with provenance metadata attached
|
|
88
|
+
artifact = result.outputs["cleaned"]
|
|
89
|
+
print(artifact.path) # -> PosixPath('./cleaned.parquet')
|
|
90
|
+
|
|
91
|
+
# Load as a DataFrame when needed
|
|
92
|
+
cleaned_df = consist.load_df(artifact)
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
`input_binding="paths"` keeps the file boundary explicit: the function receives
|
|
96
|
+
the local `Path` values named in `inputs`, while those same inputs still define
|
|
97
|
+
cache identity and lineage.
|
|
98
|
+
|
|
99
|
+
**Summary**: Consist computes a deterministic fingerprint from your code version, config, and input files. If you change
|
|
100
|
+
anything upstream, only affected downstream steps will re-execute.
|
|
101
|
+
|
|
102
|
+
### Multi-Step Pipeline
|
|
103
|
+
|
|
104
|
+
Dependencies are explicit: the output of one step becomes the input of the next via a concrete reference, not name
|
|
105
|
+
matching or injection.
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
def analyze_data(cleaned: Path) -> dict[str, Path]:
|
|
109
|
+
df = pd.read_parquet(cleaned)
|
|
110
|
+
out = Path("./analysis.parquet")
|
|
111
|
+
summary = df.groupby("category")["value"].mean()
|
|
112
|
+
summary.to_parquet(out)
|
|
113
|
+
return {"analysis": out}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
preprocess = tracker.run(
|
|
117
|
+
fn=clean_data,
|
|
118
|
+
inputs={"raw": Path("raw.parquet")},
|
|
119
|
+
config={"threshold": 0.5},
|
|
120
|
+
outputs=["cleaned"],
|
|
121
|
+
execution_options=ExecutionOptions(input_binding="paths"),
|
|
122
|
+
)
|
|
123
|
+
analyze = tracker.run(
|
|
124
|
+
fn=analyze_data,
|
|
125
|
+
inputs={"cleaned": consist.ref(preprocess, key="cleaned")}, # explicit artifact reference
|
|
126
|
+
outputs=["analysis"],
|
|
127
|
+
execution_options=ExecutionOptions(input_binding="paths"),
|
|
128
|
+
)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Use `output_paths` when a function returns `None` but writes files, or when you need explicit destination control.
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## Key Features
|
|
136
|
+
|
|
137
|
+
- **Deterministic Caching**: Cache identity is based on an inspectable fingerprint of code, config, and inputs. Only
|
|
138
|
+
affected downstream steps re-execute when any upstream piece changes.
|
|
139
|
+
- **Plain Python**: Tasks are ordinary Python functions—callable and testable without the tracker. The tracker is
|
|
140
|
+
additive and does not restructure your code.
|
|
141
|
+
- **Complete Lineage**: Every result is tagged with the exact code and config that created it. Trace lineage from any
|
|
142
|
+
output back to its sources.
|
|
143
|
+
- **SQL-Native Analysis**: All metadata is indexed in DuckDB. Query across runs, join tables, and compare variants using
|
|
144
|
+
standard SQL.
|
|
145
|
+
- **HPC and Container Support**: Track Docker and Singularity containers as pure functions—image digests and mounted
|
|
146
|
+
volumes become part of the cache signature. Ideal for long-running jobs on shared compute.
|
|
147
|
+
- **Queryable CLI**: Inspect history, trace lineage, and compare results from the command line after a job completes. No
|
|
148
|
+
code required.
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
## Documentation Index
|
|
153
|
+
|
|
154
|
+
| Section | Description |
|
|
155
|
+
|:----------------------------------------------------------|:-------------------------------------------------------------|
|
|
156
|
+
| **[Getting Started](docs/getting-started/quickstart.md)** | 5-minute guide to your first tracked run. |
|
|
157
|
+
| **[Usage Guide](docs/usage-guide.md)** | Detailed patterns for scenarios and complex workflows. |
|
|
158
|
+
| **[Architecture](docs/architecture.md)** | Deep dive into hashing, lineage, and the DuckDB core. |
|
|
159
|
+
| **[CLI Reference](docs/cli-reference.md)** | Guide to the `consist` command-line tools. |
|
|
160
|
+
| **[DB Maintenance](docs/db-maintenance.md)** | Operational runbooks for inspect/doctor/purge/merge/rebuild. |
|
|
161
|
+
| **[Example Gallery](docs/examples.md)** | Interactive notebooks for Monte Carlo, Demand Modeling, etc. |
|
|
162
|
+
|
|
163
|
+
---
|
|
164
|
+
|
|
165
|
+
## Etymology
|
|
166
|
+
|
|
167
|
+
In railroad terminology, a **consist** (noun, pronounced *CON-sist*) is the specific lineup of locomotives and cars that
|
|
168
|
+
make up a train. In this library, a **consist** is the immutable record of exactly which components—code, config, and
|
|
169
|
+
inputs—were coupled together to produce a result.
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "consist"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Provenance tracking, intelligent caching, and data virtualization for scientific simulation workflows."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "Zach Needell", email = "zaneedell@lbl.gov" }
|
|
8
|
+
]
|
|
9
|
+
requires-python = ">=3.11"
|
|
10
|
+
dependencies = [
|
|
11
|
+
"duckdb>=1.4.2",
|
|
12
|
+
"duckdb-engine>=0.17.0",
|
|
13
|
+
"pandas>=2.1.0",
|
|
14
|
+
"pydantic>=2.12.4",
|
|
15
|
+
"rich>=13.7.1",
|
|
16
|
+
"sqlalchemy>=2.0.44,<2.0.45",
|
|
17
|
+
"sqlmodel>=0.0.31",
|
|
18
|
+
"tqdm>=4.67.1",
|
|
19
|
+
"ty>=0.0.15",
|
|
20
|
+
"typer>=0.12.3",
|
|
21
|
+
"zensical>=0.0.21",
|
|
22
|
+
]
|
|
23
|
+
keywords = ["provenance", "caching", "lineage", "reproducibility", "duckdb", "scientific-computing"]
|
|
24
|
+
classifiers = [
|
|
25
|
+
"Development Status :: 5 - Production/Stable",
|
|
26
|
+
"Intended Audience :: Science/Research",
|
|
27
|
+
"Operating System :: OS Independent",
|
|
28
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
29
|
+
"Programming Language :: Python :: 3.11",
|
|
30
|
+
"Programming Language :: Python :: 3.12",
|
|
31
|
+
"Typing :: Typed",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.scripts]
|
|
35
|
+
consist = "consist.cli:app"
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
# Optional runtime integrations (install as needed).
|
|
39
|
+
docker = ["docker>=7.1.0"]
|
|
40
|
+
hdf5 = ["h5py>=3.15.1", "tables>=3.10.2"]
|
|
41
|
+
ingest = ["dlt>=1.21.0", "pyarrow>=22.0.0"]
|
|
42
|
+
activitysim = ["PyYAML>=6.0"]
|
|
43
|
+
beam = ["pyhocon>=0.3.60"]
|
|
44
|
+
netcdf = ["cftime>=1.6.0", "h5netcdf>=1.3.0", "xarray>=2024.9.0"]
|
|
45
|
+
netcdf4 = ["netCDF4>=1.6.0", "xarray>=2024.9.0"]
|
|
46
|
+
parquet = ["pyarrow>=22.0.0"]
|
|
47
|
+
spatial = ["geopandas>=0.14.0"]
|
|
48
|
+
zarr = ["xarray>=2024.9.0", "zarr>=2.18,<3"]
|
|
49
|
+
omx = ["openmatrix>=0.3.5.0"]
|
|
50
|
+
examples = [
|
|
51
|
+
"ipykernel>=6.29.0",
|
|
52
|
+
"jupyterlab>=4.0.0",
|
|
53
|
+
"matplotlib>=3.8.0",
|
|
54
|
+
"seaborn>=0.13.0",
|
|
55
|
+
"ipywidgets>=8.1.8"
|
|
56
|
+
]
|
|
57
|
+
typecheck = [
|
|
58
|
+
"pandas-stubs~=2.3.3",
|
|
59
|
+
"ty>=0.0.7",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
# Convenience extras.
|
|
63
|
+
all = [
|
|
64
|
+
"cftime>=1.6.0",
|
|
65
|
+
"docker>=7.1.0",
|
|
66
|
+
"dlt>=1.21.0",
|
|
67
|
+
"geopandas>=0.14.0",
|
|
68
|
+
"h5netcdf>=1.3.0",
|
|
69
|
+
"h5py>=3.15.1",
|
|
70
|
+
"netCDF4>=1.6.0",
|
|
71
|
+
"openmatrix>=0.3.5.0",
|
|
72
|
+
"pyarrow>=22.0.0",
|
|
73
|
+
"pyhocon>=0.3.60",
|
|
74
|
+
"PyYAML>=6.0",
|
|
75
|
+
"tables>=3.10.2",
|
|
76
|
+
"xarray>=2024.9.0",
|
|
77
|
+
"zarr>=2.18,<3",
|
|
78
|
+
]
|
|
79
|
+
bench = ["polars>=1.35.2", "psutil>6.0.0"]
|
|
80
|
+
docs = ["mkdocs-material>=9.7.0", "mkdocstrings[python]>=1.0.0"]
|
|
81
|
+
# Test + lint tools used by CI and local development.
|
|
82
|
+
test = [
|
|
83
|
+
"pytest>=9.0.1",
|
|
84
|
+
"pytest-cov>=7.0.0",
|
|
85
|
+
"pytest-mock>=3.15.1",
|
|
86
|
+
"pyhocon>=0.3.60",
|
|
87
|
+
"ruff>=0.14.6",
|
|
88
|
+
]
|
|
89
|
+
dev = [
|
|
90
|
+
"dlt>=1.21.0",
|
|
91
|
+
"h5py>=3.15.1",
|
|
92
|
+
"pyarrow>=22.0.0",
|
|
93
|
+
"pytest>=9.0.1",
|
|
94
|
+
"pytest-cov>=7.0.0",
|
|
95
|
+
"pytest-mock>=3.15.1",
|
|
96
|
+
"ruff>=0.14.6",
|
|
97
|
+
"tables>=3.10.2",
|
|
98
|
+
]
|
|
99
|
+
|
|
100
|
+
[build-system]
|
|
101
|
+
requires = ["uv_build>=0.8.15,<0.9.0"]
|
|
102
|
+
build-backend = "uv_build"
|
|
103
|
+
|
|
104
|
+
[dependency-groups]
|
|
105
|
+
bench = [
|
|
106
|
+
"psutil>=7.1.3",
|
|
107
|
+
]
|
|
108
|
+
examples = [
|
|
109
|
+
"ipykernel>=6.29.0",
|
|
110
|
+
"jupyter>=1.1.1",
|
|
111
|
+
"jupyterlab>=4.0.0",
|
|
112
|
+
"matplotlib>=3.8.0",
|
|
113
|
+
"seaborn>=0.13.0",
|
|
114
|
+
]
|
|
115
|
+
dev = [
|
|
116
|
+
"dlt>=1.21.0",
|
|
117
|
+
"h5py>=3.15.1",
|
|
118
|
+
"pyarrow>=22.0.0",
|
|
119
|
+
"pytest>=9.0.1",
|
|
120
|
+
"pytest-cov>=7.0.0",
|
|
121
|
+
"pytest-mock>=3.15.1",
|
|
122
|
+
"ruff>=0.14.6",
|
|
123
|
+
"tables>=3.10.2",
|
|
124
|
+
]
|
|
125
|
+
docs = [
|
|
126
|
+
"mkdocs-material>=9.7.0",
|
|
127
|
+
"mkdocstrings[python]>=1.0.0",
|
|
128
|
+
"mkdocstrings-python>=2.0.0",
|
|
129
|
+
"zensical>=0.0.24",
|
|
130
|
+
]
|
|
131
|
+
|
|
132
|
+
[tool.pytest.ini_options]
|
|
133
|
+
markers = [
|
|
134
|
+
"heavy: marks tests as heavyweight/long-running",
|
|
135
|
+
"flaky: marks tests as flaky/retryable",
|
|
136
|
+
"timeout: marks tests with explicit timeout expectations",
|
|
137
|
+
]
|
|
138
|
+
|
|
139
|
+
[tool.ruff]
|
|
140
|
+
target-version = "py311"
|
|
141
|
+
line-length = 88
|
|
142
|
+
|
|
143
|
+
[tool.ty.src]
|
|
144
|
+
include = ["src", "tests"]
|
|
145
|
+
|
|
146
|
+
[[tool.ty.overrides]]
|
|
147
|
+
include = ["src/consist/integrations/**"]
|
|
148
|
+
rules = { "invalid-assignment" = "ignore" }
|
|
149
|
+
|
|
150
|
+
[[tool.ty.overrides]]
|
|
151
|
+
include = ["tests/**"]
|
|
152
|
+
rules = { "invalid-argument-type" = "ignore", "invalid-assignment" = "ignore", "not-subscriptable" = "ignore", "possibly-missing-attribute" = "ignore", "unresolved-attribute" = "ignore" }
|
|
Binary file
|