marsh-lib 0.3.7__tar.gz → 0.3.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/PKG-INFO +41 -1
  2. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/README.md +40 -0
  3. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/api-migration.md +23 -2
  4. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/architecture.md +64 -0
  5. marsh_lib-0.3.8/docs/artifacts-and-reproducibility.md +130 -0
  6. marsh_lib-0.3.8/docs/release-notes-v0.3.8.md +36 -0
  7. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/pyproject.toml +1 -1
  8. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/__init__.py +4 -0
  9. marsh_lib-0.3.8/src/marsh/core/artifact_store.py +156 -0
  10. marsh_lib-0.3.8/src/marsh/core/artifacts.py +110 -0
  11. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/cache.py +7 -11
  12. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/domain.py +9 -0
  13. marsh_lib-0.3.8/src/marsh/core/identity.py +63 -0
  14. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/runtime.py +172 -17
  15. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.circleci/config.yml +0 -0
  16. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/ISSUE_TEMPLATE/bug-report.md +0 -0
  17. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  18. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/ISSUE_TEMPLATE/other.md +0 -0
  19. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/pull_request_template.md +0 -0
  20. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/docker-integration.yml +0 -0
  21. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/release-rehearsal.yml +0 -0
  22. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/release.yml +0 -0
  23. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/ssh-integration.yml +0 -0
  24. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.github/workflows/test.yml +0 -0
  25. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/.gitignore +0 -0
  26. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/LICENSE +0 -0
  27. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/Taskfile.yml +0 -0
  28. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/P0_REPOSITORY_AUDIT.md +0 -0
  29. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/P6_UX_COMPOSITION_EVALUATION.md +0 -0
  30. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/releases/v0.3.3.md +0 -0
  31. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/releases/v0.3.6.md +0 -0
  32. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/releases/v0.3.7.md +0 -0
  33. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/v0.3.2-test-portability.md +0 -0
  34. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/v0.3.6-workflow-ir.md +0 -0
  35. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/docs/workflow-guide.md +0 -0
  36. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/run_act.sh +0 -0
  37. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/run_pytest.sh +0 -0
  38. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/run_tests_with_sysbox.sh +0 -0
  39. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/__init__.py +0 -0
  40. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/__init__.py +0 -0
  41. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/bash_factory.py +0 -0
  42. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/bash_grammar.py +0 -0
  43. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/bash_runner_decorators.py +0 -0
  44. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/bash/bash_script.py +0 -0
  45. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/constants.py +0 -0
  46. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/authenticator.py +0 -0
  47. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/cmd_run_decorator.py +0 -0
  48. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/cmd_runner_spec.py +0 -0
  49. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/command_grammar.py +0 -0
  50. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/configuration.py +0 -0
  51. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/connector.py +0 -0
  52. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/conveyor.py +0 -0
  53. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/dag_bridge.py +0 -0
  54. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/diagnostics.py +0 -0
  55. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/executor.py +0 -0
  56. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/expression.py +0 -0
  57. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/__init__.py +0 -0
  58. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/codecs.py +0 -0
  59. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/errors.py +0 -0
  60. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/identifiers.py +0 -0
  61. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/models.py +0 -0
  62. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/ir/operations.py +0 -0
  63. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/observability.py +0 -0
  64. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/policies.py +0 -0
  65. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/providers.py +0 -0
  66. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/scheduler.py +0 -0
  67. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/script.py +0 -0
  68. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/serialization.py +0 -0
  69. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/core/validation.py +0 -0
  70. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/dag/__init__.py +0 -0
  71. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/dag/dag.py +0 -0
  72. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/dag/node.py +0 -0
  73. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/dag/startable.py +0 -0
  74. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/docker/__init__.py +0 -0
  75. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/docker/docker_command_grammar.py +0 -0
  76. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/docker/docker_executor.py +0 -0
  77. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/exceptions.py +0 -0
  78. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/logger.py +0 -0
  79. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/modifier_functions/__init__.py +0 -0
  80. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/modifier_functions/case_conversion.py +0 -0
  81. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/modifier_functions/readers.py +0 -0
  82. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/powershell/__init__.py +0 -0
  83. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/processor_functions/__init__.py +0 -0
  84. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/processor_functions/printers.py +0 -0
  85. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/processor_functions/raisers.py +0 -0
  86. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/processor_functions/redirections.py +0 -0
  87. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/providers/__init__.py +0 -0
  88. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/providers/docker_provider.py +0 -0
  89. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/runtime/__init__.py +0 -0
  90. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/signals.py +0 -0
  91. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/ssh/__init__.py +0 -0
  92. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/ssh/ssh_command_grammar.py +0 -0
  93. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/ssh/ssh_connector.py +0 -0
  94. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/ssh/ssh_factory.py +0 -0
  95. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/utils/__init__.py +0 -0
  96. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/utils/output_streams.py +0 -0
  97. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/src/marsh/workflow/__init__.py +0 -0
  98. {marsh_lib-0.3.7 → marsh_lib-0.3.8}/uv.lock +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: marsh-lib
3
- Version: 0.3.7
3
+ Version: 0.3.8
4
4
  Summary: Lightweight, extensible Python library for building, managing, and executing command workflows.
5
5
  Project-URL: Repository, https://github.com/CedricAnover/marsh
6
6
  Author-email: Cedric Anover <cedric.anover@hotmail.com>
@@ -584,6 +584,46 @@ The current release line intentionally has several boundaries:
584
584
 
585
585
  These limitations describe the current implementation; they should not be interpreted as permanent exclusions.
586
586
 
587
+
588
+ ## Artifacts and reproducibility (v0.3.8)
589
+
590
+ v0.3.8 adds provider-independent identity and artifact primitives around the canonical Workflow runtime.
591
+
592
+ A successful result can expose:
593
+
594
+ result.execution_id # stable semantic execution identity when portable
595
+ result.attempt_id # v0.3.7 per-attempt identity
596
+ result.artifact_refs # immutable content-addressed artifact references
597
+ result.provenance # allow-listed, serializable provenance
598
+
599
+ Execution identity is derived from canonical workflow semantics. Attempt identity is intentionally separate, so retries do not create a new logical execution identity. Artifact identity is content-derived with SHA-256 and does not depend on where the bytes are stored.
600
+
601
+ ### Opt-in artifact persistence
602
+
603
+ Use the local content-addressed store when artifact persistence is required:
604
+
605
+ from pathlib import Path
606
+ from marsh import LocalArtifactStore, execute_workflow
607
+
608
+ store = LocalArtifactStore(Path(".marsh-artifacts"))
609
+ results = execute_workflow(workflow, artifact_store=store)
610
+ result = results["greet"]
611
+
612
+ for artifact in result.artifact_refs:
613
+ assert store.verify(artifact)
614
+
615
+ The store verifies artifact size and digest on reads and uses temporary files plus atomic finalization for writes. Artifact storage is opt-in; the normal runtime does not require a local artifact directory.
616
+
617
+ ### Cache identity
618
+
619
+ Caching remains opt-in through the existing policy boundary. Cache keys now use the shared canonical identity primitives rather than a second JSON/hash implementation. A cache hit reuses the cached successful result and does not create another execution attempt.
620
+
621
+ ### Reproducibility boundary
622
+
623
+ Portable identity data must be deterministic and serializable. Unsupported runtime objects, non-finite numeric values, and non-portable callable definitions are rejected by the identity boundary. Legacy workflows that use non-portable callables remain executable, but they do not receive a portable execution identity.
624
+
625
+ See docs/artifacts-and-reproducibility.md for the contract, storage layout, provenance rules, and reproducibility guidance.
626
+
587
627
  ## Development
588
628
 
589
629
  Marsh uses `uv` for dependency and environment management.
@@ -555,6 +555,46 @@ The current release line intentionally has several boundaries:
555
555
 
556
556
  These limitations describe the current implementation; they should not be interpreted as permanent exclusions.
557
557
 
558
+
559
+ ## Artifacts and reproducibility (v0.3.8)
560
+
561
+ v0.3.8 adds provider-independent identity and artifact primitives around the canonical Workflow runtime.
562
+
563
+ A successful result can expose:
564
+
565
+ result.execution_id # stable semantic execution identity when portable
566
+ result.attempt_id # v0.3.7 per-attempt identity
567
+ result.artifact_refs # immutable content-addressed artifact references
568
+ result.provenance # allow-listed, serializable provenance
569
+
570
+ Execution identity is derived from canonical workflow semantics. Attempt identity is intentionally separate, so retries do not create a new logical execution identity. Artifact identity is content-derived with SHA-256 and does not depend on where the bytes are stored.
571
+
572
+ ### Opt-in artifact persistence
573
+
574
+ Use the local content-addressed store when artifact persistence is required:
575
+
576
+ from pathlib import Path
577
+ from marsh import LocalArtifactStore, execute_workflow
578
+
579
+ store = LocalArtifactStore(Path(".marsh-artifacts"))
580
+ results = execute_workflow(workflow, artifact_store=store)
581
+ result = results["greet"]
582
+
583
+ for artifact in result.artifact_refs:
584
+ assert store.verify(artifact)
585
+
586
+ The store verifies artifact size and digest on reads and uses temporary files plus atomic finalization for writes. Artifact storage is opt-in; the normal runtime does not require a local artifact directory.
587
+
588
+ ### Cache identity
589
+
590
+ Caching remains opt-in through the existing policy boundary. Cache keys now use the shared canonical identity primitives rather than a second JSON/hash implementation. A cache hit reuses the cached successful result and does not create another execution attempt.
591
+
592
+ ### Reproducibility boundary
593
+
594
+ Portable identity data must be deterministic and serializable. Unsupported runtime objects, non-finite numeric values, and non-portable callable definitions are rejected by the identity boundary. Legacy workflows that use non-portable callables remain executable, but they do not receive a portable execution identity.
595
+
596
+ See docs/artifacts-and-reproducibility.md for the contract, storage layout, provenance rules, and reproducibility guidance.
597
+
558
598
  ## Development
559
599
 
560
600
  Marsh uses `uv` for dependency and environment management.
@@ -198,7 +198,28 @@ round_trip = workflow_from_json(payload)
198
198
 
199
199
  Portable JSON definitions should contain serializable values. Arbitrary Python callables are Python objects and are not general portable JSON definitions.
200
200
 
201
- ## 10. Deprecation discipline
201
+
202
+ ## 10. Artifacts, identity, and provenance
203
+
204
+ v0.3.8 adds identity and artifact fields additively to canonical Result values:
205
+
206
+ result.execution_id
207
+ result.attempt_id
208
+ result.artifact_refs
209
+ result.provenance
210
+
211
+ Existing code that only reads stdout, stderr, status, error, duration, or metadata does not need to change.
212
+
213
+ For portable workflows, execution_id is deterministic across fresh processes. Retry attempts keep the same execution identity while receiving a distinct attempt identity. Cache hits reuse the successful cached result rather than creating another attempt.
214
+
215
+ Artifact persistence is opt-in:
216
+
217
+ store = LocalArtifactStore(".marsh-artifacts")
218
+ results = execute_workflow(workflow, artifact_store=store)
219
+
220
+ Applications that require portability should avoid arbitrary local/lambda callables in workflow definitions. Legacy callable workflows remain supported, but the runtime does not invent a portable execution identity for definitions that cannot cross the canonical serialization boundary.
221
+
222
+ ## 11. Deprecation discipline
202
223
 
203
224
  The v0.3.x compatibility policy is deliberately conservative:
204
225
 
@@ -211,7 +232,7 @@ The v0.3.x compatibility policy is deliberately conservative:
211
232
 
212
233
  This policy keeps application migrations incremental while the workflow kernel continues to converge.
213
234
 
214
- ## 11. Further reading
235
+ ## 12. Further reading
215
236
 
216
237
  - [Workflow Guide](workflow-guide.md) — canonical examples and workflow semantics.
217
238
  - [Architecture](architecture.md) — implemented architecture and boundaries.
@@ -150,3 +150,67 @@ The attempt lifecycle is: execute -> terminal outcome -> cleanup at most once ->
150
150
  Cancellation is a request followed by mechanism-specific stop/terminate/kill and confirmation. A cancellation request is not itself a confirmed CANCELLED result. Timeout cleanup completes before retry eligibility is evaluated.
151
151
 
152
152
  The policy evaluator is pure and inspectable. Backend differences must not change the policy decision for the same policy, task metadata, outcome, and control history. Runtime-only callbacks such as cleanup functions remain outside the portable IR serialization boundary.
153
+
154
+
155
+ ## Artifact and identity boundary (v0.3.8)
156
+
157
+ v0.3.8 extends the existing canonical Workflow IR and runtime rather than introducing a second execution engine.
158
+
159
+ ```mermaid
160
+ flowchart LR
161
+ W[Workflow / Task] --> C[Canonical IR]
162
+ C --> E[Execution identity]
163
+ E --> K[Opt-in cache]
164
+ W --> R[Existing runtime]
165
+ R --> A[Attempt identity]
166
+ R --> O[Result]
167
+ O --> S[ArtifactStore]
168
+ S --> AR[ArtifactRef]
169
+ O --> P[Provenance]
170
+ E --> P
171
+ A --> P
172
+ AR --> P
173
+ ```
174
+
175
+ Plaintext contract:
176
+
177
+ ```text
178
+ Workflow semantics
179
+ -> canonical UTF-8 representation
180
+ -> execution identity
181
+
182
+ Existing scheduler / policy / provider path
183
+ -> attempt identity
184
+ -> Result
185
+
186
+ Result bytes
187
+ -> SHA-256 content identity
188
+ -> local ArtifactStore (optional)
189
+ -> ArtifactRef
190
+
191
+ Execution identity != attempt identity != artifact identity
192
+
193
+ Provenance is allow-listed and serializable.
194
+ Storage paths and runtime objects are not semantic identity.
195
+ ```
196
+
197
+ ### Identity layers
198
+
199
+ - Execution identity: semantic fingerprint of a portable workflow execution.
200
+ - Attempt identity: per-attempt runtime identity; retries receive distinct attempt IDs.
201
+ - Artifact identity: SHA-256 of artifact bytes; independent of storage location.
202
+ - Result: semantic outcome that may reference artifacts and provenance.
203
+
204
+ ### ArtifactStore boundary
205
+
206
+ ArtifactStore is a narrow capability. LocalArtifactStore uses content-addressed paths, manifests, digest/size verification, secure temporary files, and atomic finalization. Remote object stores are not required by the canonical runtime.
207
+
208
+ ### Compatibility boundary
209
+
210
+ Legacy callable workflows remain executable. When their definitions cannot be represented by the portable canonical IR, runtime identity is unavailable rather than fabricated from process-local object representations.
211
+
212
+ The runtime accepts an optional ArtifactStore. Without one, existing result streams remain in memory and no artifact directory is required.
213
+
214
+ ### Security boundary
215
+
216
+ Portable provenance is allow-listed. It must not contain credentials, full environment dumps, arbitrary runtime objects, or storage implementation details. Artifact reads verify content identity and declared size before returning bytes.
@@ -0,0 +1,130 @@
1
+ # Artifacts and Reproducibility
2
+
3
+ ## Scope
4
+
5
+ v0.3.8 adds a small provider-independent artifact and identity layer around the existing Workflow runtime.
6
+
7
+ It does not add a second scheduler, a distributed control plane, a mandatory remote object store, or a new CLI.
8
+
9
+ ## Identity model
10
+
11
+ Marsh distinguishes four concepts:
12
+
13
+ - **Execution identity** — a SHA-256 fingerprint of canonical portable workflow semantics.
14
+ - **Attempt identity** — the runtime identity of one execution attempt. Retries create new attempt identities.
15
+ - **Artifact identity** — a SHA-256 digest of artifact bytes. It is independent of storage location.
16
+ - **Result** — the semantic execution outcome, which may reference artifacts and provenance.
17
+
18
+ The identities must not be conflated. In particular, a retry does not change the logical execution identity, and moving an artifact does not change its artifact identity.
19
+
20
+ ## Canonical execution identity
21
+
22
+ Portable execution identity is derived from the existing canonical Workflow IR serializer.
23
+
24
+ The identity boundary uses:
25
+
26
+ - deterministic UTF-8 JSON;
27
+ - sorted mapping keys and compact separators for generic identity payloads;
28
+ - explicit schema/version markers;
29
+ - rejection of non-finite numeric values;
30
+ - explicit rejection of unsupported runtime objects.
31
+
32
+ Legacy workflows containing non-portable local callables remain executable. If a workflow cannot cross the canonical serialization boundary, Marsh does not fabricate a supposedly portable identity from process-local object representations.
33
+
34
+ ## Result fields
35
+
36
+ Canonical Result values may include:
37
+
38
+ - execution_id;
39
+ - attempt_id;
40
+ - artifact_refs;
41
+ - provenance.
42
+
43
+ These fields are additive. Existing stdout, stderr, status, error, duration, metadata, and ok behavior remain the primary result semantics.
44
+
45
+ ## ArtifactStore
46
+
47
+ LocalArtifactStore is content-addressed:
48
+
49
+ <root>/
50
+ objects/
51
+ <first-two-digest-characters>/
52
+ <remaining-digest>
53
+ manifests/
54
+ <full-digest>.json
55
+ tmp/
56
+
57
+ Writes use temporary files and atomic finalization. Existing objects are verified rather than silently accepted as valid. Reads verify both declared size and SHA-256 content identity, and manifest corruption is rejected.
58
+
59
+ Identical bytes deduplicate to the same content identity.
60
+
61
+ Artifact persistence is opt-in. The normal runtime does not create an artifact directory unless an ArtifactStore is supplied.
62
+
63
+ ## Provenance
64
+
65
+ Provenance is an explicit allow-list rather than an environment dump.
66
+
67
+ The current contract can include:
68
+
69
+ - schema version;
70
+ - workflow and task identity;
71
+ - execution and attempt identity;
72
+ - parent execution references;
73
+ - operation reference;
74
+ - provider, machine, and policy fingerprints where relevant;
75
+ - input and output artifact references;
76
+ - creation time.
77
+
78
+ Credentials, full environment variables, arbitrary provider objects, process-local objects, and storage implementation details are not portable provenance.
79
+
80
+ ## Cache semantics
81
+
82
+ The existing cache boundary remains authoritative.
83
+
84
+ Cache identity uses the shared canonical identity primitives rather than a separate serialization/hash implementation.
85
+
86
+ A successful cache hit reuses the cached Result and therefore does not create a new attempt. Corrupt or unverifiable artifact data must not be treated as a valid artifact.
87
+
88
+ ## Reproducibility verification
89
+
90
+ v0.3.8 tests identity and artifact behavior across:
91
+
92
+ - fresh Python processes;
93
+ - repeated canonicalization;
94
+ - bounded concurrent artifact writes;
95
+ - artifact reopen/read from another process;
96
+ - manifest corruption;
97
+ - Python 3.10, 3.11, and 3.12 CI;
98
+ - Windows portability CI where the existing workflow permits it.
99
+
100
+ Reproducibility is a tested property, not an assumption based on implementation intent.
101
+
102
+ ## Compatibility
103
+
104
+ The canonical runtime keeps the existing scheduler, policy, provider, and execution seams. Artifact storage is optional and is kept outside process-worker state; process workers return normal Results and the parent runtime may materialize artifacts.
105
+
106
+ This preserves the existing process-scheduler boundary while allowing the same artifact contract to be used by sequential, thread, async, and process execution modes.
107
+
108
+ ## Security considerations
109
+
110
+ Before release, review:
111
+
112
+ 1. canonicalization for accidental secret inclusion;
113
+ 2. provenance allow-list completeness;
114
+ 3. artifact path validation and traversal resistance;
115
+ 4. digest and size verification;
116
+ 5. atomic write behavior under concurrency;
117
+ 6. temporary-file cleanup;
118
+ 7. cache behavior on corrupt or stale data;
119
+ 8. release artifacts and hashes.
120
+
121
+ ## Non-goals
122
+
123
+ v0.3.8 does not attempt to provide:
124
+
125
+ - remote object storage;
126
+ - distributed cache invalidation;
127
+ - arbitrary callable serialization;
128
+ - cross-machine execution orchestration;
129
+ - automatic task-to-task dataflow;
130
+ - a general artifact retention/garbage-collection policy.
@@ -0,0 +1,36 @@
1
+ # v0.3.8 — Artifacts & Reproducibility
2
+
3
+ ## Highlights
4
+
5
+ - Added deterministic canonical execution identity based on the existing Workflow IR.
6
+ - Added SHA-256 content identity for artifacts.
7
+ - Added immutable Artifact and ArtifactRef contracts.
8
+ - Added allow-listed serializable Provenance.
9
+ - Added a local content-addressed ArtifactStore with manifests and integrity verification.
10
+ - Added additive execution/artifact/provenance fields to Result.
11
+ - Added opt-in runtime artifact materialization.
12
+ - Routed cache identity through the shared canonical primitives.
13
+ - Added fresh-process, concurrent, corruption, and cross-process reproducibility coverage.
14
+ - Preserved legacy callable and process-scheduler compatibility.
15
+
16
+ ## Compatibility
17
+
18
+ Portable execution identity is available when workflow semantics cross the canonical serialization boundary. Legacy non-portable callables remain executable but do not receive a fabricated portable identity.
19
+
20
+ Artifact persistence is opt-in. Existing workflow execution does not require an artifact directory.
21
+
22
+ ## Verification
23
+
24
+ Release acceptance requires:
25
+
26
+ - CircleCI build;
27
+ - CircleCI lint;
28
+ - Python 3.10, 3.11, and 3.12 tests;
29
+ - CircleCI integration;
30
+ - Windows portability checks where configured;
31
+ - package build and artifact inspection;
32
+ - security review;
33
+ - exact artifact hashes;
34
+ - main/develop synchronization.
35
+
36
+ Upstash Box is intentionally reserved for the final tag/release step and is not used during normal development.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "marsh-lib"
7
- version = "0.3.7"
7
+ version = "0.3.8"
8
8
  description = "Lightweight, extensible Python library for building, managing, and executing command workflows."
9
9
  authors = [{ name = "Cedric Anover", email = "cedric.anover@hotmail.com" }]
10
10
  license = { text = "MIT" }
@@ -43,3 +43,7 @@ from marsh.core.providers import (
43
43
  from marsh.core.cache import Cache, CachePolicy, MemoryCache, cache_key_for_task
44
44
  from marsh.core.observability import EventType, Observer, RuntimeEvent, emit_event
45
45
  from marsh.core.scheduler import AsyncScheduler, ConcurrentScheduler, ExecutorScheduler, ProcessScheduler, SchedulerState, TaskState, ThreadScheduler
46
+
47
+ from marsh.core.artifacts import Artifact, ArtifactRef, ArtifactStore, Provenance
48
+ from marsh.core.artifact_store import LocalArtifactStore
49
+ from marsh.core.identity import canonical_bytes, content_digest, execution_id
@@ -0,0 +1,156 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import os
5
+ import tempfile
6
+ from pathlib import Path
7
+
8
+ from marsh.core.artifacts import Artifact, ArtifactRef
9
+ from marsh.core.identity import content_digest
10
+
11
+
12
+ class LocalArtifactStore:
13
+ def __init__(self, root: str | os.PathLike[str]):
14
+ self.root = Path(root)
15
+ self.objects = self.root / "objects"
16
+ self.manifests = self.root / "manifests"
17
+ self.tmp = self.root / "tmp"
18
+ self.objects.mkdir(parents=True, exist_ok=True)
19
+ self.manifests.mkdir(parents=True, exist_ok=True)
20
+ self.tmp.mkdir(parents=True, exist_ok=True)
21
+
22
+ def path_for(self, ref: ArtifactRef) -> Path:
23
+ digest = ref.digest.removeprefix("sha256:")
24
+ if len(digest) != 64 or any(
25
+ char not in "0123456789abcdef" for char in digest
26
+ ):
27
+ raise ValueError("invalid artifact digest")
28
+ return self.objects / digest[:2] / digest[2:]
29
+
30
+ def manifest_path_for(self, ref: ArtifactRef) -> Path:
31
+ return self.manifests / f"{ref.digest.removeprefix('sha256:')}.json"
32
+
33
+ def put(
34
+ self,
35
+ data: bytes,
36
+ *,
37
+ media_type: str = "application/octet-stream",
38
+ ) -> ArtifactRef:
39
+ if not isinstance(data, bytes):
40
+ raise TypeError("artifact data must be bytes")
41
+
42
+ digest = content_digest(data)
43
+ ref = Artifact(digest=digest, size=len(data), media_type=media_type)
44
+ destination = self.path_for(ref)
45
+ destination.parent.mkdir(parents=True, exist_ok=True)
46
+
47
+ if destination.exists():
48
+ self.get(ref)
49
+ else:
50
+ fd, temp_name = tempfile.mkstemp(prefix="artifact-", dir=self.tmp)
51
+ try:
52
+ with os.fdopen(fd, "wb") as handle:
53
+ handle.write(data)
54
+ handle.flush()
55
+ os.fsync(handle.fileno())
56
+ try:
57
+ os.replace(temp_name, destination)
58
+ except PermissionError:
59
+ if not destination.exists():
60
+ raise
61
+ self.get(ref)
62
+ finally:
63
+ try:
64
+ os.unlink(temp_name)
65
+ except FileNotFoundError:
66
+ pass
67
+
68
+ manifest = {
69
+ "digest": digest,
70
+ "size": len(data),
71
+ "media_type": media_type,
72
+ }
73
+ manifest_path = self.manifest_path_for(ref)
74
+ if manifest_path.exists():
75
+ existing = json.loads(
76
+ manifest_path.read_text(encoding="utf-8")
77
+ )
78
+ if (
79
+ existing.get("digest") != digest
80
+ or existing.get("size") != len(data)
81
+ ):
82
+ raise ValueError(
83
+ "artifact manifest integrity verification failed"
84
+ )
85
+ if existing.get("media_type") != media_type:
86
+ raise ValueError("artifact metadata conflict")
87
+ else:
88
+ fd, temp_name = tempfile.mkstemp(
89
+ prefix="manifest-", dir=self.tmp, text=True
90
+ )
91
+ try:
92
+ with os.fdopen(fd, "w", encoding="utf-8") as handle:
93
+ json.dump(
94
+ manifest,
95
+ handle,
96
+ sort_keys=True,
97
+ separators=(",", ":"),
98
+ )
99
+ handle.write("\n")
100
+ handle.flush()
101
+ os.fsync(handle.fileno())
102
+ try:
103
+ os.replace(temp_name, manifest_path)
104
+ except PermissionError:
105
+ if not manifest_path.exists():
106
+ raise
107
+ finally:
108
+ try:
109
+ os.unlink(temp_name)
110
+ except FileNotFoundError:
111
+ pass
112
+
113
+ return Artifact(
114
+ digest=digest,
115
+ size=len(data),
116
+ media_type=media_type,
117
+ storage_ref=str(destination),
118
+ manifest=manifest,
119
+ )
120
+
121
+ def get(self, ref: ArtifactRef) -> bytes:
122
+ path = self.path_for(ref)
123
+ if not path.is_file():
124
+ raise FileNotFoundError(path)
125
+ data = path.read_bytes()
126
+ if len(data) != ref.size or content_digest(data) != ref.digest:
127
+ raise ValueError("artifact integrity verification failed")
128
+ manifest_path = self.manifest_path_for(ref)
129
+ if manifest_path.is_file():
130
+ manifest = json.loads(
131
+ manifest_path.read_text(encoding="utf-8")
132
+ )
133
+ if (
134
+ manifest.get("digest") != ref.digest
135
+ or manifest.get("size") != ref.size
136
+ ):
137
+ raise ValueError(
138
+ "artifact manifest integrity verification failed"
139
+ )
140
+ return data
141
+
142
+ def exists(self, ref: ArtifactRef) -> bool:
143
+ return self.path_for(ref).is_file()
144
+
145
+ def verify(self, ref: ArtifactRef) -> bool:
146
+ self.get(ref)
147
+ return True
148
+
149
+ def delete(self, ref: ArtifactRef) -> None:
150
+ self.path_for(ref).unlink(missing_ok=True)
151
+ self.manifest_path_for(ref).unlink(missing_ok=True)
152
+
153
+ def object_count(self) -> int:
154
+ return sum(
155
+ 1 for path in self.objects.glob("*/*") if path.is_file()
156
+ )
@@ -0,0 +1,110 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from dataclasses import dataclass, field
5
+ from typing import Any, Mapping, Protocol
6
+
7
+ _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class Provenance:
12
+ schema_version: int
13
+ workflow_id: str
14
+ workflow_version: str | None = None
15
+ task_id: str | None = None
16
+ execution_id: str | None = None
17
+ attempt_id: str | None = None
18
+ parent_execution_ids: tuple[str, ...] = ()
19
+ operation_ref: Mapping[str, Any] | None = None
20
+ provider_fingerprint: str | None = None
21
+ machine_fingerprint: str | None = None
22
+ policy_fingerprint: str | None = None
23
+ input_artifact_refs: tuple[str, ...] = ()
24
+ output_artifact_refs: tuple[str, ...] = ()
25
+ created_at: str | None = None
26
+
27
+ def __post_init__(self) -> None:
28
+ if self.schema_version < 1:
29
+ raise ValueError("provenance schema_version must be positive")
30
+ if not self.workflow_id:
31
+ raise ValueError("provenance workflow_id must be non-empty")
32
+ object.__setattr__(
33
+ self, "parent_execution_ids", tuple(self.parent_execution_ids)
34
+ )
35
+ object.__setattr__(
36
+ self, "input_artifact_refs", tuple(self.input_artifact_refs)
37
+ )
38
+ object.__setattr__(
39
+ self, "output_artifact_refs", tuple(self.output_artifact_refs)
40
+ )
41
+ if self.operation_ref is not None:
42
+ object.__setattr__(self, "operation_ref", dict(self.operation_ref))
43
+
44
+ def to_dict(self) -> dict[str, Any]:
45
+ values = {
46
+ "schema_version": self.schema_version,
47
+ "workflow_id": self.workflow_id,
48
+ "workflow_version": self.workflow_version,
49
+ "task_id": self.task_id,
50
+ "execution_id": self.execution_id,
51
+ "attempt_id": self.attempt_id,
52
+ "parent_execution_ids": list(self.parent_execution_ids),
53
+ "operation_ref": (
54
+ dict(self.operation_ref) if self.operation_ref else None
55
+ ),
56
+ "provider_fingerprint": self.provider_fingerprint,
57
+ "machine_fingerprint": self.machine_fingerprint,
58
+ "policy_fingerprint": self.policy_fingerprint,
59
+ "input_artifact_refs": list(self.input_artifact_refs),
60
+ "output_artifact_refs": list(self.output_artifact_refs),
61
+ "created_at": self.created_at,
62
+ }
63
+ return {
64
+ key: value for key, value in values.items() if value is not None
65
+ }
66
+
67
+
68
+ @dataclass(frozen=True)
69
+ class Artifact:
70
+ digest: str
71
+ size: int
72
+ media_type: str = "application/octet-stream"
73
+ storage_ref: str | None = None
74
+ manifest: Mapping[str, Any] = field(default_factory=dict)
75
+
76
+ def __post_init__(self) -> None:
77
+ if not _DIGEST.fullmatch(self.digest):
78
+ raise ValueError(
79
+ "artifact digest must be sha256:<64 hex characters>"
80
+ )
81
+ if self.size < 0:
82
+ raise ValueError("artifact size cannot be negative")
83
+ if not self.media_type:
84
+ raise ValueError("artifact media_type must be non-empty")
85
+ object.__setattr__(self, "manifest", dict(self.manifest))
86
+
87
+
88
+ ArtifactRef = Artifact
89
+
90
+
91
+ class ArtifactStore(Protocol):
92
+ def put(
93
+ self,
94
+ data: bytes,
95
+ *,
96
+ media_type: str = "application/octet-stream",
97
+ ) -> ArtifactRef:
98
+ ...
99
+
100
+ def get(self, ref: ArtifactRef) -> bytes:
101
+ ...
102
+
103
+ def exists(self, ref: ArtifactRef) -> bool:
104
+ ...
105
+
106
+ def verify(self, ref: ArtifactRef) -> bool:
107
+ ...
108
+
109
+ def delete(self, ref: ArtifactRef) -> None:
110
+ ...