marsh-lib 0.3.7__tar.gz → 0.3.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/PKG-INFO +41 -1
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/README.md +40 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/api-migration.md +23 -2
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/architecture.md +64 -0
- marsh_lib-0.3.8/docs/artifacts-and-reproducibility.md +130 -0
- marsh_lib-0.3.8/docs/release-notes-v0.3.8.md +36 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/pyproject.toml +1 -1
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/__init__.py +4 -0
- marsh_lib-0.3.8/src/marsh/core/artifact_store.py +156 -0
- marsh_lib-0.3.8/src/marsh/core/artifacts.py +110 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/cache.py +7 -11
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/domain.py +9 -0
- marsh_lib-0.3.8/src/marsh/core/identity.py +63 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/runtime.py +172 -17
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.circleci/config.yml +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/ISSUE_TEMPLATE/bug-report.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/ISSUE_TEMPLATE/other.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/pull_request_template.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/docker-integration.yml +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/release-rehearsal.yml +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/release.yml +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/ssh-integration.yml +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/test.yml +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.gitignore +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/LICENSE +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/Taskfile.yml +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/P0_REPOSITORY_AUDIT.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/P6_UX_COMPOSITION_EVALUATION.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/releases/v0.3.3.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/releases/v0.3.6.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/releases/v0.3.7.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/v0.3.2-test-portability.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/v0.3.6-workflow-ir.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/workflow-guide.md +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/run_act.sh +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/run_pytest.sh +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/run_tests_with_sysbox.sh +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/bash_factory.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/bash_grammar.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/bash_runner_decorators.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/bash_script.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/constants.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/authenticator.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/cmd_run_decorator.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/cmd_runner_spec.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/command_grammar.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/configuration.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/connector.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/conveyor.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/dag_bridge.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/diagnostics.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/executor.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/expression.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/codecs.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/errors.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/identifiers.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/models.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/operations.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/observability.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/policies.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/providers.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/scheduler.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/script.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/serialization.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/validation.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/dag/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/dag/dag.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/dag/node.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/dag/startable.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/docker/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/docker/docker_command_grammar.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/docker/docker_executor.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/exceptions.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/logger.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/modifier_functions/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/modifier_functions/case_conversion.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/modifier_functions/readers.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/powershell/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/processor_functions/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/processor_functions/printers.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/processor_functions/raisers.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/processor_functions/redirections.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/providers/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/providers/docker_provider.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/runtime/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/signals.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/ssh/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/ssh/ssh_command_grammar.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/ssh/ssh_connector.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/ssh/ssh_factory.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/utils/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/utils/output_streams.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/workflow/__init__.py +0 -0
- {marsh_lib-0.3.7 → marsh_lib-0.3.8}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: marsh-lib
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.8
|
|
4
4
|
Summary: Lightweight, extensible Python library for building, managing, and executing command workflows.
|
|
5
5
|
Project-URL: Repository, https://github.com/CedricAnover/marsh
|
|
6
6
|
Author-email: Cedric Anover <cedric.anover@hotmail.com>
|
|
@@ -584,6 +584,46 @@ The current release line intentionally has several boundaries:
|
|
|
584
584
|
|
|
585
585
|
These limitations describe the current implementation; they should not be interpreted as permanent exclusions.
|
|
586
586
|
|
|
587
|
+
|
|
588
|
+
## Artifacts and reproducibility (v0.3.8)
|
|
589
|
+
|
|
590
|
+
v0.3.8 adds provider-independent identity and artifact primitives around the canonical Workflow runtime.
|
|
591
|
+
|
|
592
|
+
A successful result can expose:
|
|
593
|
+
|
|
594
|
+
result.execution_id # stable semantic execution identity when portable
|
|
595
|
+
result.attempt_id # v0.3.7 per-attempt identity
|
|
596
|
+
result.artifact_refs # immutable content-addressed artifact references
|
|
597
|
+
result.provenance # allow-listed, serializable provenance
|
|
598
|
+
|
|
599
|
+
Execution identity is derived from canonical workflow semantics. Attempt identity is intentionally separate, so retries do not create a new logical execution identity. Artifact identity is content-derived with SHA-256 and does not depend on where the bytes are stored.
|
|
600
|
+
|
|
601
|
+
### Opt-in artifact persistence
|
|
602
|
+
|
|
603
|
+
Use the local content-addressed store when artifact persistence is required:
|
|
604
|
+
|
|
605
|
+
from pathlib import Path
|
|
606
|
+
from marsh import LocalArtifactStore, execute_workflow
|
|
607
|
+
|
|
608
|
+
store = LocalArtifactStore(Path(".marsh-artifacts"))
|
|
609
|
+
results = execute_workflow(workflow, artifact_store=store)
|
|
610
|
+
result = results["greet"]
|
|
611
|
+
|
|
612
|
+
for artifact in result.artifact_refs:
|
|
613
|
+
assert store.verify(artifact)
|
|
614
|
+
|
|
615
|
+
The store verifies artifact size and digest on reads and uses temporary files plus atomic finalization for writes. Artifact storage is opt-in; the normal runtime does not require a local artifact directory.
|
|
616
|
+
|
|
617
|
+
### Cache identity
|
|
618
|
+
|
|
619
|
+
Caching remains opt-in through the existing policy boundary. Cache keys now use the shared canonical identity primitives rather than a second JSON/hash implementation. A cache hit reuses the cached successful result and does not create another execution attempt.
|
|
620
|
+
|
|
621
|
+
### Reproducibility boundary
|
|
622
|
+
|
|
623
|
+
Portable identity data must be deterministic and serializable. Unsupported runtime objects, non-finite numeric values, and non-portable callable definitions are rejected by the identity boundary. Legacy workflows that use non-portable callables remain executable, but they do not receive a portable execution identity.
|
|
624
|
+
|
|
625
|
+
See docs/artifacts-and-reproducibility.md for the contract, storage layout, provenance rules, and reproducibility guidance.
|
|
626
|
+
|
|
587
627
|
## Development
|
|
588
628
|
|
|
589
629
|
Marsh uses `uv` for dependency and environment management.
|
|
@@ -555,6 +555,46 @@ The current release line intentionally has several boundaries:
|
|
|
555
555
|
|
|
556
556
|
These limitations describe the current implementation; they should not be interpreted as permanent exclusions.
|
|
557
557
|
|
|
558
|
+
|
|
559
|
+
## Artifacts and reproducibility (v0.3.8)
|
|
560
|
+
|
|
561
|
+
v0.3.8 adds provider-independent identity and artifact primitives around the canonical Workflow runtime.
|
|
562
|
+
|
|
563
|
+
A successful result can expose:
|
|
564
|
+
|
|
565
|
+
result.execution_id # stable semantic execution identity when portable
|
|
566
|
+
result.attempt_id # v0.3.7 per-attempt identity
|
|
567
|
+
result.artifact_refs # immutable content-addressed artifact references
|
|
568
|
+
result.provenance # allow-listed, serializable provenance
|
|
569
|
+
|
|
570
|
+
Execution identity is derived from canonical workflow semantics. Attempt identity is intentionally separate, so retries do not create a new logical execution identity. Artifact identity is content-derived with SHA-256 and does not depend on where the bytes are stored.
|
|
571
|
+
|
|
572
|
+
### Opt-in artifact persistence
|
|
573
|
+
|
|
574
|
+
Use the local content-addressed store when artifact persistence is required:
|
|
575
|
+
|
|
576
|
+
from pathlib import Path
|
|
577
|
+
from marsh import LocalArtifactStore, execute_workflow
|
|
578
|
+
|
|
579
|
+
store = LocalArtifactStore(Path(".marsh-artifacts"))
|
|
580
|
+
results = execute_workflow(workflow, artifact_store=store)
|
|
581
|
+
result = results["greet"]
|
|
582
|
+
|
|
583
|
+
for artifact in result.artifact_refs:
|
|
584
|
+
assert store.verify(artifact)
|
|
585
|
+
|
|
586
|
+
The store verifies artifact size and digest on reads and uses temporary files plus atomic finalization for writes. Artifact storage is opt-in; the normal runtime does not require a local artifact directory.
|
|
587
|
+
|
|
588
|
+
### Cache identity
|
|
589
|
+
|
|
590
|
+
Caching remains opt-in through the existing policy boundary. Cache keys now use the shared canonical identity primitives rather than a second JSON/hash implementation. A cache hit reuses the cached successful result and does not create another execution attempt.
|
|
591
|
+
|
|
592
|
+
### Reproducibility boundary
|
|
593
|
+
|
|
594
|
+
Portable identity data must be deterministic and serializable. Unsupported runtime objects, non-finite numeric values, and non-portable callable definitions are rejected by the identity boundary. Legacy workflows that use non-portable callables remain executable, but they do not receive a portable execution identity.
|
|
595
|
+
|
|
596
|
+
See docs/artifacts-and-reproducibility.md for the contract, storage layout, provenance rules, and reproducibility guidance.
|
|
597
|
+
|
|
558
598
|
## Development
|
|
559
599
|
|
|
560
600
|
Marsh uses `uv` for dependency and environment management.
|
|
@@ -198,7 +198,28 @@ round_trip = workflow_from_json(payload)
|
|
|
198
198
|
|
|
199
199
|
Portable JSON definitions should contain serializable values. Arbitrary Python callables are Python objects and are not general portable JSON definitions.
|
|
200
200
|
|
|
201
|
-
|
|
201
|
+
|
|
202
|
+
## 10. Artifacts, identity, and provenance
|
|
203
|
+
|
|
204
|
+
v0.3.8 adds identity and artifact fields additively to canonical Result values:
|
|
205
|
+
|
|
206
|
+
result.execution_id
|
|
207
|
+
result.attempt_id
|
|
208
|
+
result.artifact_refs
|
|
209
|
+
result.provenance
|
|
210
|
+
|
|
211
|
+
Existing code that only reads stdout, stderr, status, error, duration, or metadata does not need to change.
|
|
212
|
+
|
|
213
|
+
For portable workflows, execution_id is deterministic across fresh processes. Retry attempts keep the same execution identity while receiving a distinct attempt identity. Cache hits reuse the successful cached result rather than creating another attempt.
|
|
214
|
+
|
|
215
|
+
Artifact persistence is opt-in:
|
|
216
|
+
|
|
217
|
+
store = LocalArtifactStore(".marsh-artifacts")
|
|
218
|
+
results = execute_workflow(workflow, artifact_store=store)
|
|
219
|
+
|
|
220
|
+
Applications that require portability should avoid arbitrary local/lambda callables in workflow definitions. Legacy callable workflows remain supported, but the runtime does not invent a portable execution identity for definitions that cannot cross the canonical serialization boundary.
|
|
221
|
+
|
|
222
|
+
## 11. Deprecation discipline
|
|
202
223
|
|
|
203
224
|
The v0.3.x compatibility policy is deliberately conservative:
|
|
204
225
|
|
|
@@ -211,7 +232,7 @@ The v0.3.x compatibility policy is deliberately conservative:
|
|
|
211
232
|
|
|
212
233
|
This policy keeps application migrations incremental while the workflow kernel continues to converge.
|
|
213
234
|
|
|
214
|
-
##
|
|
235
|
+
## 12. Further reading
|
|
215
236
|
|
|
216
237
|
- [Workflow Guide](workflow-guide.md) — canonical examples and workflow semantics.
|
|
217
238
|
- [Architecture](architecture.md) — implemented architecture and boundaries.
|
|
@@ -150,3 +150,67 @@ The attempt lifecycle is: execute -> terminal outcome -> cleanup at most once ->
|
|
|
150
150
|
Cancellation is a request followed by mechanism-specific stop/terminate/kill and confirmation. A cancellation request is not itself a confirmed CANCELLED result. Timeout cleanup completes before retry eligibility is evaluated.
|
|
151
151
|
|
|
152
152
|
The policy evaluator is pure and inspectable. Backend differences must not change the policy decision for the same policy, task metadata, outcome, and control history. Runtime-only callbacks such as cleanup functions remain outside the portable IR serialization boundary.
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
## Artifact and identity boundary (v0.3.8)
|
|
156
|
+
|
|
157
|
+
v0.3.8 extends the existing canonical Workflow IR and runtime rather than introducing a second execution engine.
|
|
158
|
+
|
|
159
|
+
```mermaid
|
|
160
|
+
flowchart LR
|
|
161
|
+
W[Workflow / Task] --> C[Canonical IR]
|
|
162
|
+
C --> E[Execution identity]
|
|
163
|
+
E --> K[Opt-in cache]
|
|
164
|
+
W --> R[Existing runtime]
|
|
165
|
+
R --> A[Attempt identity]
|
|
166
|
+
R --> O[Result]
|
|
167
|
+
O --> S[ArtifactStore]
|
|
168
|
+
S --> AR[ArtifactRef]
|
|
169
|
+
O --> P[Provenance]
|
|
170
|
+
E --> P
|
|
171
|
+
A --> P
|
|
172
|
+
AR --> P
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Plaintext contract:
|
|
176
|
+
|
|
177
|
+
```text
|
|
178
|
+
Workflow semantics
|
|
179
|
+
-> canonical UTF-8 representation
|
|
180
|
+
-> execution identity
|
|
181
|
+
|
|
182
|
+
Existing scheduler / policy / provider path
|
|
183
|
+
-> attempt identity
|
|
184
|
+
-> Result
|
|
185
|
+
|
|
186
|
+
Result bytes
|
|
187
|
+
-> SHA-256 content identity
|
|
188
|
+
-> local ArtifactStore (optional)
|
|
189
|
+
-> ArtifactRef
|
|
190
|
+
|
|
191
|
+
Execution identity != attempt identity != artifact identity
|
|
192
|
+
|
|
193
|
+
Provenance is allow-listed and serializable.
|
|
194
|
+
Storage paths and runtime objects are not semantic identity.
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
### Identity layers
|
|
198
|
+
|
|
199
|
+
- Execution identity: semantic fingerprint of a portable workflow execution.
|
|
200
|
+
- Attempt identity: per-attempt runtime identity; retries receive distinct attempt IDs.
|
|
201
|
+
- Artifact identity: SHA-256 of artifact bytes; independent of storage location.
|
|
202
|
+
- Result: semantic outcome that may reference artifacts and provenance.
|
|
203
|
+
|
|
204
|
+
### ArtifactStore boundary
|
|
205
|
+
|
|
206
|
+
ArtifactStore is a narrow capability. LocalArtifactStore uses content-addressed paths, manifests, digest/size verification, secure temporary files, and atomic finalization. Remote object stores are not required by the canonical runtime.
|
|
207
|
+
|
|
208
|
+
### Compatibility boundary
|
|
209
|
+
|
|
210
|
+
Legacy callable workflows remain executable. When their definitions cannot be represented by the portable canonical IR, runtime identity is unavailable rather than fabricated from process-local object representations.
|
|
211
|
+
|
|
212
|
+
The runtime accepts an optional ArtifactStore. Without one, existing result streams remain in memory and no artifact directory is required.
|
|
213
|
+
|
|
214
|
+
### Security boundary
|
|
215
|
+
|
|
216
|
+
Portable provenance is allow-listed. It must not contain credentials, full environment dumps, arbitrary runtime objects, or storage implementation details. Artifact reads verify content identity and declared size before returning bytes.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# Artifacts and Reproducibility
|
|
2
|
+
|
|
3
|
+
## Scope
|
|
4
|
+
|
|
5
|
+
v0.3.8 adds a small provider-independent artifact and identity layer around the existing Workflow runtime.
|
|
6
|
+
|
|
7
|
+
It does not add a second scheduler, a distributed control plane, a mandatory remote object store, or a new CLI.
|
|
8
|
+
|
|
9
|
+
## Identity model
|
|
10
|
+
|
|
11
|
+
Marsh distinguishes four concepts:
|
|
12
|
+
|
|
13
|
+
- **Execution identity** — a SHA-256 fingerprint of canonical portable workflow semantics.
|
|
14
|
+
- **Attempt identity** — the runtime identity of one execution attempt. Retries create new attempt identities.
|
|
15
|
+
- **Artifact identity** — a SHA-256 digest of artifact bytes. It is independent of storage location.
|
|
16
|
+
- **Result** — the semantic execution outcome, which may reference artifacts and provenance.
|
|
17
|
+
|
|
18
|
+
The identities must not be conflated. In particular, a retry does not change the logical execution identity, and moving an artifact does not change its artifact identity.
|
|
19
|
+
|
|
20
|
+
## Canonical execution identity
|
|
21
|
+
|
|
22
|
+
Portable execution identity is derived from the existing canonical Workflow IR serializer.
|
|
23
|
+
|
|
24
|
+
The identity boundary uses:
|
|
25
|
+
|
|
26
|
+
- deterministic UTF-8 JSON;
|
|
27
|
+
- sorted mapping keys and compact separators for generic identity payloads;
|
|
28
|
+
- explicit schema/version markers;
|
|
29
|
+
- rejection of non-finite numeric values;
|
|
30
|
+
- explicit rejection of unsupported runtime objects.
|
|
31
|
+
|
|
32
|
+
Legacy workflows containing non-portable local callables remain executable. If a workflow cannot cross the canonical serialization boundary, Marsh does not fabricate a supposedly portable identity from process-local object representations.
|
|
33
|
+
|
|
34
|
+
## Result fields
|
|
35
|
+
|
|
36
|
+
Canonical Result values may include:
|
|
37
|
+
|
|
38
|
+
- execution_id;
|
|
39
|
+
- attempt_id;
|
|
40
|
+
- artifact_refs;
|
|
41
|
+
- provenance.
|
|
42
|
+
|
|
43
|
+
These fields are additive. Existing stdout, stderr, status, error, duration, metadata, and ok behavior remain the primary result semantics.
|
|
44
|
+
|
|
45
|
+
## ArtifactStore
|
|
46
|
+
|
|
47
|
+
LocalArtifactStore is content-addressed:
|
|
48
|
+
|
|
49
|
+
<root>/
|
|
50
|
+
objects/
|
|
51
|
+
<first-two-digest-characters>/
|
|
52
|
+
<remaining-digest>
|
|
53
|
+
manifests/
|
|
54
|
+
<full-digest>.json
|
|
55
|
+
tmp/
|
|
56
|
+
|
|
57
|
+
Writes use temporary files and atomic finalization. Existing objects are verified rather than silently accepted as valid. Reads verify both declared size and SHA-256 content identity, and manifest corruption is rejected.
|
|
58
|
+
|
|
59
|
+
Identical bytes deduplicate to the same content identity.
|
|
60
|
+
|
|
61
|
+
Artifact persistence is opt-in. The normal runtime does not create an artifact directory unless an ArtifactStore is supplied.
|
|
62
|
+
|
|
63
|
+
## Provenance
|
|
64
|
+
|
|
65
|
+
Provenance is an explicit allow-list rather than an environment dump.
|
|
66
|
+
|
|
67
|
+
The current contract can include:
|
|
68
|
+
|
|
69
|
+
- schema version;
|
|
70
|
+
- workflow and task identity;
|
|
71
|
+
- execution and attempt identity;
|
|
72
|
+
- parent execution references;
|
|
73
|
+
- operation reference;
|
|
74
|
+
- provider, machine, and policy fingerprints where relevant;
|
|
75
|
+
- input and output artifact references;
|
|
76
|
+
- creation time.
|
|
77
|
+
|
|
78
|
+
Credentials, full environment variables, arbitrary provider objects, process-local objects, and storage implementation details are not portable provenance.
|
|
79
|
+
|
|
80
|
+
## Cache semantics
|
|
81
|
+
|
|
82
|
+
The existing cache boundary remains authoritative.
|
|
83
|
+
|
|
84
|
+
Cache identity uses the shared canonical identity primitives rather than a separate serialization/hash implementation.
|
|
85
|
+
|
|
86
|
+
A successful cache hit reuses the cached Result and therefore does not create a new attempt. Corrupt or unverifiable artifact data must not be treated as a valid artifact.
|
|
87
|
+
|
|
88
|
+
## Reproducibility verification
|
|
89
|
+
|
|
90
|
+
v0.3.8 tests identity and artifact behavior across:
|
|
91
|
+
|
|
92
|
+
- fresh Python processes;
|
|
93
|
+
- repeated canonicalization;
|
|
94
|
+
- bounded concurrent artifact writes;
|
|
95
|
+
- artifact reopen/read from another process;
|
|
96
|
+
- manifest corruption;
|
|
97
|
+
- Python 3.10, 3.11, and 3.12 CI;
|
|
98
|
+
- Windows portability CI where the existing workflow permits it.
|
|
99
|
+
|
|
100
|
+
Reproducibility is a tested property, not an assumption based on implementation intent.
|
|
101
|
+
|
|
102
|
+
## Compatibility
|
|
103
|
+
|
|
104
|
+
The canonical runtime keeps the existing scheduler, policy, provider, and execution seams. Artifact storage is optional and is kept outside process-worker state; process workers return normal Results and the parent runtime may materialize artifacts.
|
|
105
|
+
|
|
106
|
+
This preserves the existing process-scheduler boundary while allowing the same artifact contract to be used by sequential, thread, async, and process execution modes.
|
|
107
|
+
|
|
108
|
+
## Security considerations
|
|
109
|
+
|
|
110
|
+
Before release, review:
|
|
111
|
+
|
|
112
|
+
1. canonicalization for accidental secret inclusion;
|
|
113
|
+
2. provenance allow-list completeness;
|
|
114
|
+
3. artifact path validation and traversal resistance;
|
|
115
|
+
4. digest and size verification;
|
|
116
|
+
5. atomic write behavior under concurrency;
|
|
117
|
+
6. temporary-file cleanup;
|
|
118
|
+
7. cache behavior on corrupt or stale data;
|
|
119
|
+
8. release artifacts and hashes.
|
|
120
|
+
|
|
121
|
+
## Non-goals
|
|
122
|
+
|
|
123
|
+
v0.3.8 does not attempt to provide:
|
|
124
|
+
|
|
125
|
+
- remote object storage;
|
|
126
|
+
- distributed cache invalidation;
|
|
127
|
+
- arbitrary callable serialization;
|
|
128
|
+
- cross-machine execution orchestration;
|
|
129
|
+
- automatic task-to-task dataflow;
|
|
130
|
+
- a general artifact retention/garbage-collection policy.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# v0.3.8 — Artifacts & Reproducibility
|
|
2
|
+
|
|
3
|
+
## Highlights
|
|
4
|
+
|
|
5
|
+
- Added deterministic canonical execution identity based on the existing Workflow IR.
|
|
6
|
+
- Added SHA-256 content identity for artifacts.
|
|
7
|
+
- Added immutable Artifact and ArtifactRef contracts.
|
|
8
|
+
- Added allow-listed serializable Provenance.
|
|
9
|
+
- Added a local content-addressed ArtifactStore with manifests and integrity verification.
|
|
10
|
+
- Added additive execution/artifact/provenance fields to Result.
|
|
11
|
+
- Added opt-in runtime artifact materialization.
|
|
12
|
+
- Routed cache identity through the shared canonical primitives.
|
|
13
|
+
- Added fresh-process, concurrent, corruption, and cross-process reproducibility coverage.
|
|
14
|
+
- Preserved legacy callable and process-scheduler compatibility.
|
|
15
|
+
|
|
16
|
+
## Compatibility
|
|
17
|
+
|
|
18
|
+
Portable execution identity is available when workflow semantics cross the canonical serialization boundary. Legacy non-portable callables remain executable but do not receive a fabricated portable identity.
|
|
19
|
+
|
|
20
|
+
Artifact persistence is opt-in. Existing workflow execution does not require an artifact directory.
|
|
21
|
+
|
|
22
|
+
## Verification
|
|
23
|
+
|
|
24
|
+
Release acceptance requires:
|
|
25
|
+
|
|
26
|
+
- CircleCI build;
|
|
27
|
+
- CircleCI lint;
|
|
28
|
+
- Python 3.10, 3.11, and 3.12 tests;
|
|
29
|
+
- CircleCI integration;
|
|
30
|
+
- Windows portability checks where configured;
|
|
31
|
+
- package build and artifact inspection;
|
|
32
|
+
- security review;
|
|
33
|
+
- exact artifact hashes;
|
|
34
|
+
- main/develop synchronization.
|
|
35
|
+
|
|
36
|
+
Upstash Box is intentionally reserved for the final tag/release step and is not used during normal development.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "marsh-lib"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.8"
|
|
8
8
|
description = "Lightweight, extensible Python library for building, managing, and executing command workflows."
|
|
9
9
|
authors = [{ name = "Cedric Anover", email = "cedric.anover@hotmail.com" }]
|
|
10
10
|
license = { text = "MIT" }
|
|
@@ -43,3 +43,7 @@ from marsh.core.providers import (
|
|
|
43
43
|
from marsh.core.cache import Cache, CachePolicy, MemoryCache, cache_key_for_task
|
|
44
44
|
from marsh.core.observability import EventType, Observer, RuntimeEvent, emit_event
|
|
45
45
|
from marsh.core.scheduler import AsyncScheduler, ConcurrentScheduler, ExecutorScheduler, ProcessScheduler, SchedulerState, TaskState, ThreadScheduler
|
|
46
|
+
|
|
47
|
+
from marsh.core.artifacts import Artifact, ArtifactRef, ArtifactStore, Provenance
|
|
48
|
+
from marsh.core.artifact_store import LocalArtifactStore
|
|
49
|
+
from marsh.core.identity import canonical_bytes, content_digest, execution_id
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import tempfile
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from marsh.core.artifacts import Artifact, ArtifactRef
|
|
9
|
+
from marsh.core.identity import content_digest
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class LocalArtifactStore:
|
|
13
|
+
def __init__(self, root: str | os.PathLike[str]):
|
|
14
|
+
self.root = Path(root)
|
|
15
|
+
self.objects = self.root / "objects"
|
|
16
|
+
self.manifests = self.root / "manifests"
|
|
17
|
+
self.tmp = self.root / "tmp"
|
|
18
|
+
self.objects.mkdir(parents=True, exist_ok=True)
|
|
19
|
+
self.manifests.mkdir(parents=True, exist_ok=True)
|
|
20
|
+
self.tmp.mkdir(parents=True, exist_ok=True)
|
|
21
|
+
|
|
22
|
+
def path_for(self, ref: ArtifactRef) -> Path:
|
|
23
|
+
digest = ref.digest.removeprefix("sha256:")
|
|
24
|
+
if len(digest) != 64 or any(
|
|
25
|
+
char not in "0123456789abcdef" for char in digest
|
|
26
|
+
):
|
|
27
|
+
raise ValueError("invalid artifact digest")
|
|
28
|
+
return self.objects / digest[:2] / digest[2:]
|
|
29
|
+
|
|
30
|
+
def manifest_path_for(self, ref: ArtifactRef) -> Path:
|
|
31
|
+
return self.manifests / f"{ref.digest.removeprefix('sha256:')}.json"
|
|
32
|
+
|
|
33
|
+
def put(
|
|
34
|
+
self,
|
|
35
|
+
data: bytes,
|
|
36
|
+
*,
|
|
37
|
+
media_type: str = "application/octet-stream",
|
|
38
|
+
) -> ArtifactRef:
|
|
39
|
+
if not isinstance(data, bytes):
|
|
40
|
+
raise TypeError("artifact data must be bytes")
|
|
41
|
+
|
|
42
|
+
digest = content_digest(data)
|
|
43
|
+
ref = Artifact(digest=digest, size=len(data), media_type=media_type)
|
|
44
|
+
destination = self.path_for(ref)
|
|
45
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
|
|
47
|
+
if destination.exists():
|
|
48
|
+
self.get(ref)
|
|
49
|
+
else:
|
|
50
|
+
fd, temp_name = tempfile.mkstemp(prefix="artifact-", dir=self.tmp)
|
|
51
|
+
try:
|
|
52
|
+
with os.fdopen(fd, "wb") as handle:
|
|
53
|
+
handle.write(data)
|
|
54
|
+
handle.flush()
|
|
55
|
+
os.fsync(handle.fileno())
|
|
56
|
+
try:
|
|
57
|
+
os.replace(temp_name, destination)
|
|
58
|
+
except PermissionError:
|
|
59
|
+
if not destination.exists():
|
|
60
|
+
raise
|
|
61
|
+
self.get(ref)
|
|
62
|
+
finally:
|
|
63
|
+
try:
|
|
64
|
+
os.unlink(temp_name)
|
|
65
|
+
except FileNotFoundError:
|
|
66
|
+
pass
|
|
67
|
+
|
|
68
|
+
manifest = {
|
|
69
|
+
"digest": digest,
|
|
70
|
+
"size": len(data),
|
|
71
|
+
"media_type": media_type,
|
|
72
|
+
}
|
|
73
|
+
manifest_path = self.manifest_path_for(ref)
|
|
74
|
+
if manifest_path.exists():
|
|
75
|
+
existing = json.loads(
|
|
76
|
+
manifest_path.read_text(encoding="utf-8")
|
|
77
|
+
)
|
|
78
|
+
if (
|
|
79
|
+
existing.get("digest") != digest
|
|
80
|
+
or existing.get("size") != len(data)
|
|
81
|
+
):
|
|
82
|
+
raise ValueError(
|
|
83
|
+
"artifact manifest integrity verification failed"
|
|
84
|
+
)
|
|
85
|
+
if existing.get("media_type") != media_type:
|
|
86
|
+
raise ValueError("artifact metadata conflict")
|
|
87
|
+
else:
|
|
88
|
+
fd, temp_name = tempfile.mkstemp(
|
|
89
|
+
prefix="manifest-", dir=self.tmp, text=True
|
|
90
|
+
)
|
|
91
|
+
try:
|
|
92
|
+
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
|
93
|
+
json.dump(
|
|
94
|
+
manifest,
|
|
95
|
+
handle,
|
|
96
|
+
sort_keys=True,
|
|
97
|
+
separators=(",", ":"),
|
|
98
|
+
)
|
|
99
|
+
handle.write("\n")
|
|
100
|
+
handle.flush()
|
|
101
|
+
os.fsync(handle.fileno())
|
|
102
|
+
try:
|
|
103
|
+
os.replace(temp_name, manifest_path)
|
|
104
|
+
except PermissionError:
|
|
105
|
+
if not manifest_path.exists():
|
|
106
|
+
raise
|
|
107
|
+
finally:
|
|
108
|
+
try:
|
|
109
|
+
os.unlink(temp_name)
|
|
110
|
+
except FileNotFoundError:
|
|
111
|
+
pass
|
|
112
|
+
|
|
113
|
+
return Artifact(
|
|
114
|
+
digest=digest,
|
|
115
|
+
size=len(data),
|
|
116
|
+
media_type=media_type,
|
|
117
|
+
storage_ref=str(destination),
|
|
118
|
+
manifest=manifest,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
def get(self, ref: ArtifactRef) -> bytes:
|
|
122
|
+
path = self.path_for(ref)
|
|
123
|
+
if not path.is_file():
|
|
124
|
+
raise FileNotFoundError(path)
|
|
125
|
+
data = path.read_bytes()
|
|
126
|
+
if len(data) != ref.size or content_digest(data) != ref.digest:
|
|
127
|
+
raise ValueError("artifact integrity verification failed")
|
|
128
|
+
manifest_path = self.manifest_path_for(ref)
|
|
129
|
+
if manifest_path.is_file():
|
|
130
|
+
manifest = json.loads(
|
|
131
|
+
manifest_path.read_text(encoding="utf-8")
|
|
132
|
+
)
|
|
133
|
+
if (
|
|
134
|
+
manifest.get("digest") != ref.digest
|
|
135
|
+
or manifest.get("size") != ref.size
|
|
136
|
+
):
|
|
137
|
+
raise ValueError(
|
|
138
|
+
"artifact manifest integrity verification failed"
|
|
139
|
+
)
|
|
140
|
+
return data
|
|
141
|
+
|
|
142
|
+
def exists(self, ref: ArtifactRef) -> bool:
|
|
143
|
+
return self.path_for(ref).is_file()
|
|
144
|
+
|
|
145
|
+
def verify(self, ref: ArtifactRef) -> bool:
|
|
146
|
+
self.get(ref)
|
|
147
|
+
return True
|
|
148
|
+
|
|
149
|
+
def delete(self, ref: ArtifactRef) -> None:
|
|
150
|
+
self.path_for(ref).unlink(missing_ok=True)
|
|
151
|
+
self.manifest_path_for(ref).unlink(missing_ok=True)
|
|
152
|
+
|
|
153
|
+
def object_count(self) -> int:
|
|
154
|
+
return sum(
|
|
155
|
+
1 for path in self.objects.glob("*/*") if path.is_file()
|
|
156
|
+
)
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Any, Mapping, Protocol
|
|
6
|
+
|
|
7
|
+
_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class Provenance:
|
|
12
|
+
schema_version: int
|
|
13
|
+
workflow_id: str
|
|
14
|
+
workflow_version: str | None = None
|
|
15
|
+
task_id: str | None = None
|
|
16
|
+
execution_id: str | None = None
|
|
17
|
+
attempt_id: str | None = None
|
|
18
|
+
parent_execution_ids: tuple[str, ...] = ()
|
|
19
|
+
operation_ref: Mapping[str, Any] | None = None
|
|
20
|
+
provider_fingerprint: str | None = None
|
|
21
|
+
machine_fingerprint: str | None = None
|
|
22
|
+
policy_fingerprint: str | None = None
|
|
23
|
+
input_artifact_refs: tuple[str, ...] = ()
|
|
24
|
+
output_artifact_refs: tuple[str, ...] = ()
|
|
25
|
+
created_at: str | None = None
|
|
26
|
+
|
|
27
|
+
def __post_init__(self) -> None:
|
|
28
|
+
if self.schema_version < 1:
|
|
29
|
+
raise ValueError("provenance schema_version must be positive")
|
|
30
|
+
if not self.workflow_id:
|
|
31
|
+
raise ValueError("provenance workflow_id must be non-empty")
|
|
32
|
+
object.__setattr__(
|
|
33
|
+
self, "parent_execution_ids", tuple(self.parent_execution_ids)
|
|
34
|
+
)
|
|
35
|
+
object.__setattr__(
|
|
36
|
+
self, "input_artifact_refs", tuple(self.input_artifact_refs)
|
|
37
|
+
)
|
|
38
|
+
object.__setattr__(
|
|
39
|
+
self, "output_artifact_refs", tuple(self.output_artifact_refs)
|
|
40
|
+
)
|
|
41
|
+
if self.operation_ref is not None:
|
|
42
|
+
object.__setattr__(self, "operation_ref", dict(self.operation_ref))
|
|
43
|
+
|
|
44
|
+
def to_dict(self) -> dict[str, Any]:
|
|
45
|
+
values = {
|
|
46
|
+
"schema_version": self.schema_version,
|
|
47
|
+
"workflow_id": self.workflow_id,
|
|
48
|
+
"workflow_version": self.workflow_version,
|
|
49
|
+
"task_id": self.task_id,
|
|
50
|
+
"execution_id": self.execution_id,
|
|
51
|
+
"attempt_id": self.attempt_id,
|
|
52
|
+
"parent_execution_ids": list(self.parent_execution_ids),
|
|
53
|
+
"operation_ref": (
|
|
54
|
+
dict(self.operation_ref) if self.operation_ref else None
|
|
55
|
+
),
|
|
56
|
+
"provider_fingerprint": self.provider_fingerprint,
|
|
57
|
+
"machine_fingerprint": self.machine_fingerprint,
|
|
58
|
+
"policy_fingerprint": self.policy_fingerprint,
|
|
59
|
+
"input_artifact_refs": list(self.input_artifact_refs),
|
|
60
|
+
"output_artifact_refs": list(self.output_artifact_refs),
|
|
61
|
+
"created_at": self.created_at,
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
key: value for key, value in values.items() if value is not None
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class Artifact:
|
|
70
|
+
digest: str
|
|
71
|
+
size: int
|
|
72
|
+
media_type: str = "application/octet-stream"
|
|
73
|
+
storage_ref: str | None = None
|
|
74
|
+
manifest: Mapping[str, Any] = field(default_factory=dict)
|
|
75
|
+
|
|
76
|
+
def __post_init__(self) -> None:
|
|
77
|
+
if not _DIGEST.fullmatch(self.digest):
|
|
78
|
+
raise ValueError(
|
|
79
|
+
"artifact digest must be sha256:<64 hex characters>"
|
|
80
|
+
)
|
|
81
|
+
if self.size < 0:
|
|
82
|
+
raise ValueError("artifact size cannot be negative")
|
|
83
|
+
if not self.media_type:
|
|
84
|
+
raise ValueError("artifact media_type must be non-empty")
|
|
85
|
+
object.__setattr__(self, "manifest", dict(self.manifest))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
ArtifactRef = Artifact
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class ArtifactStore(Protocol):
|
|
92
|
+
def put(
|
|
93
|
+
self,
|
|
94
|
+
data: bytes,
|
|
95
|
+
*,
|
|
96
|
+
media_type: str = "application/octet-stream",
|
|
97
|
+
) -> ArtifactRef:
|
|
98
|
+
...
|
|
99
|
+
|
|
100
|
+
def get(self, ref: ArtifactRef) -> bytes:
|
|
101
|
+
...
|
|
102
|
+
|
|
103
|
+
def exists(self, ref: ArtifactRef) -> bool:
|
|
104
|
+
...
|
|
105
|
+
|
|
106
|
+
def verify(self, ref: ArtifactRef) -> bool:
|
|
107
|
+
...
|
|
108
|
+
|
|
109
|
+
def delete(self, ref: ArtifactRef) -> None:
|
|
110
|
+
...
|