marsh-lib 0.3.2__tar.gz → 0.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.circleci/config.yml +11 -4
  2. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.github/pull_request_template.md +3 -1
  3. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.github/workflows/release.yml +13 -1
  4. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/PKG-INFO +5 -3
  5. marsh_lib-0.3.4/docs/architecture.md +86 -0
  6. marsh_lib-0.3.4/docs/releases/v0.3.3.md +33 -0
  7. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/pyproject.toml +16 -3
  8. marsh_lib-0.3.4/src/marsh/__init__.py +9 -0
  9. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/__init__.py +11 -1
  10. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/providers.py +61 -16
  11. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/runtime.py +28 -2
  12. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/serialization.py +4 -26
  13. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/docker/docker_executor.py +5 -1
  14. marsh_lib-0.3.4/src/marsh/providers/docker_provider.py +253 -0
  15. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/uv.lock +8 -7
  16. marsh_lib-0.3.2/docs/architecture.md +0 -35
  17. marsh_lib-0.3.2/src/marsh/__init__.py +0 -2
  18. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.github/ISSUE_TEMPLATE/bug-report.md +0 -0
  19. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
  20. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.github/ISSUE_TEMPLATE/other.md +0 -0
  21. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.github/workflows/docker-integration.yml +0 -0
  22. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.github/workflows/release-rehearsal.yml +0 -0
  23. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.github/workflows/ssh-integration.yml +0 -0
  24. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.github/workflows/test.yml +0 -0
  25. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/.gitignore +0 -0
  26. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/LICENSE +0 -0
  27. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/README.md +0 -0
  28. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/Taskfile.yml +0 -0
  29. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/docs/P0_REPOSITORY_AUDIT.md +0 -0
  30. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/docs/P6_UX_COMPOSITION_EVALUATION.md +0 -0
  31. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/docs/api-migration.md +0 -0
  32. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/docs/v0.3.2-test-portability.md +0 -0
  33. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/docs/workflow-guide.md +0 -0
  34. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/run_act.sh +0 -0
  35. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/run_pytest.sh +0 -0
  36. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/run_tests_with_sysbox.sh +0 -0
  37. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/bash/__init__.py +0 -0
  38. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/bash/bash_factory.py +0 -0
  39. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/bash/bash_grammar.py +0 -0
  40. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/bash/bash_runner_decorators.py +0 -0
  41. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/bash/bash_script.py +0 -0
  42. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/constants.py +0 -0
  43. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/authenticator.py +0 -0
  44. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/cache.py +0 -0
  45. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/cmd_run_decorator.py +0 -0
  46. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/cmd_runner_spec.py +0 -0
  47. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/command_grammar.py +0 -0
  48. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/configuration.py +0 -0
  49. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/connector.py +0 -0
  50. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/conveyor.py +0 -0
  51. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/dag_bridge.py +0 -0
  52. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/diagnostics.py +0 -0
  53. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/domain.py +0 -0
  54. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/executor.py +0 -0
  55. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/expression.py +0 -0
  56. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/observability.py +0 -0
  57. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/policies.py +0 -0
  58. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/script.py +0 -0
  59. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/core/validation.py +0 -0
  60. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/dag/__init__.py +0 -0
  61. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/dag/dag.py +0 -0
  62. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/dag/node.py +0 -0
  63. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/dag/startable.py +0 -0
  64. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/docker/__init__.py +0 -0
  65. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/docker/docker_command_grammar.py +0 -0
  66. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/exceptions.py +0 -0
  67. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/logger.py +0 -0
  68. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/modifier_functions/__init__.py +0 -0
  69. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/modifier_functions/case_conversion.py +0 -0
  70. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/modifier_functions/readers.py +0 -0
  71. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/powershell/__init__.py +0 -0
  72. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/processor_functions/__init__.py +0 -0
  73. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/processor_functions/printers.py +0 -0
  74. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/processor_functions/raisers.py +0 -0
  75. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/processor_functions/redirections.py +0 -0
  76. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/providers/__init__.py +0 -0
  77. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/runtime/__init__.py +0 -0
  78. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/signals.py +0 -0
  79. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/ssh/__init__.py +0 -0
  80. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/ssh/ssh_command_grammar.py +0 -0
  81. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/ssh/ssh_connector.py +0 -0
  82. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/ssh/ssh_factory.py +0 -0
  83. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/utils/__init__.py +0 -0
  84. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/utils/output_streams.py +0 -0
  85. {marsh_lib-0.3.2 → marsh_lib-0.3.4}/src/marsh/workflow/__init__.py +0 -0
@@ -86,7 +86,7 @@ jobs:
86
86
  command -v uv
87
87
  - run:
88
88
  name: Install dependencies
89
- command: uv sync --all-groups
89
+ command: uv sync --all-groups --extra docker --extra ssh
90
90
  - run:
91
91
  name: Validate dependency and platform contract
92
92
  command: |
@@ -97,9 +97,14 @@ jobs:
97
97
 
98
98
  assert sys.version_info[:2] == (3, 12), sys.version
99
99
  project = tomllib.loads(Path("pyproject.toml").read_text())["project"]
100
- assert "optional-dependencies" not in project, "Unexpected optional dependency surface"
100
+ extras = project["optional-dependencies"]
101
+ dependencies = project["dependencies"]
102
+ assert not any(dep.startswith("docker") for dep in dependencies)
103
+ assert not any(dep.startswith("fabric") for dep in dependencies)
104
+ assert any(dep.startswith("docker") for dep in extras["docker"])
105
+ assert any(dep.startswith("fabric") for dep in extras["ssh"])
101
106
  assert project["requires-python"] == ">=3.10,<3.13"
102
- print("Python 3.12 integration environment validated; optional dependency surface is N/A.")
107
+ print("Python 3.12 integration environment and optional provider dependency contract validated.")
103
108
  PY
104
109
  - run:
105
110
  name: Pull Docker integration test images
@@ -108,7 +113,9 @@ jobs:
108
113
  docker pull ubuntu:24.04
109
114
  - run:
110
115
  name: Run Docker integration tests
111
- command: uv run pytest -vv --disable-warnings --tb=short tests/docker
116
+ command: |
117
+ uv run pytest -vv --disable-warnings --tb=short tests/docker
118
+ uv run pytest -vv --disable-warnings --tb=short tests/core/test_provider_architecture_v034.py::test_docker_provider_executes_real_container
112
119
  - run:
113
120
  name: Run SSH integration tests
114
121
  command: uv run pytest -vv --disable-warnings --tb=short tests/ssh
@@ -1,3 +1,6 @@
1
+ ## Summary
2
+ [1–3 concise, user-facing bullets describing what changed and why. This section is used in release notes.]
3
+
1
4
  ## Description
2
5
  [Brief and concise description of the changes]
3
6
 
@@ -17,7 +20,6 @@
17
20
  - [ ] Related API documentation, if any, has been updated.
18
21
  - [ ] My code is self-documented with clear comments for complex areas.
19
22
  - [ ] Public methods and classes include appropriate docstrings.
20
- - [ ] Function names, variables, and class names are descriptive and adhere to naming conventions.
21
23
  - [ ] I understand that pull requests will not be merged if they do not pass the automated tests.
22
24
  - [ ] New functionality is covered by automated tests.
23
25
  - [ ] Code changes have been tested across relevant environments or configurations.
@@ -127,8 +127,20 @@ jobs:
127
127
  ],
128
128
  "ignore_labels": ["wip", "do-not-merge"],
129
129
  "sort": {"order": "ASC", "on_property": "mergedAt"},
130
+ "custom_placeholders": [
131
+ {
132
+ "name": "SUMMARY",
133
+ "source": "BODY",
134
+ "transformer": {
135
+ "pattern": "^[\\s\\S]*?## Summary\\s*\\n([\\s\\S]*?)(?:\\n## |$)",
136
+ "target": "$1",
137
+ "flags": "g",
138
+ "on_empty": ""
139
+ }
140
+ }
141
+ ],
130
142
  "template": "# Release Notes for #{{TO_TAG}}\n\n#{{CHANGELOG}}\n\n<details>\n<summary>Uncategorized</summary>\n\n#{{UNCATEGORIZED}}\n</details>",
131
- "pr_template": "- #{{TITLE}} ([#{{NUMBER}}](#{{URL}})) by @#{{AUTHOR}}"
143
+ "pr_template": "- #{{TITLE}}\n #{{SUMMARY}}\n ([#{{NUMBER}}](#{{URL}})) by @#{{AUTHOR}}"
132
144
  }
133
145
  fetchReleaseInformation: true
134
146
  token: ${{ secrets.GITHUB_TOKEN }}
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: marsh-lib
3
- Version: 0.3.2
3
+ Version: 0.3.4
4
4
  Summary: Lightweight, extensible Python library for building, managing, and executing command workflows.
5
5
  Project-URL: Repository, https://github.com/CedricAnover/marsh
6
6
  Author-email: Cedric Anover <cedric.anover@hotmail.com>
@@ -21,8 +21,10 @@ Classifier: Topic :: System :: System Shells
21
21
  Classifier: Topic :: System :: Systems Administration
22
22
  Classifier: Topic :: Utilities
23
23
  Requires-Python: <3.13,>=3.10
24
- Requires-Dist: docker<8.0.0,>=7.1.0
25
- Requires-Dist: fabric<4.0.0,>=3.2.2
24
+ Provides-Extra: docker
25
+ Requires-Dist: docker<8.0.0,>=7.1.0; extra == 'docker'
26
+ Provides-Extra: ssh
27
+ Requires-Dist: fabric<4.0.0,>=3.2.2; extra == 'ssh'
26
28
  Description-Content-Type: text/markdown
27
29
 
28
30
  # Marsh
@@ -0,0 +1,86 @@
1
+ # Marsh Architecture
2
+
3
+ ## Architecture
4
+
5
+ Marsh is moving toward a small execution kernel with explicit boundaries:
6
+
7
+ ```mermaid
8
+ flowchart TD
9
+ A[UX / Python API] --> B[Workflow / Task]
10
+ B --> C[Validation]
11
+ C --> D[Planning]
12
+ D --> E[Scheduler]
13
+ E --> F[Machine / Process]
14
+ F --> G[Result]
15
+ ```
16
+
17
+ The architectural goal is to keep:
18
+
19
+ - workflow intent;
20
+ - execution mechanisms;
21
+ - machines;
22
+ - processes;
23
+ - scheduling;
24
+ - policies;
25
+ - providers;
26
+ - observability; and
27
+ - results
28
+
29
+ as distinct concepts.
30
+
31
+ The existing command and DAG APIs remain useful low-level building blocks and compatibility surfaces.
32
+
33
+ ## Implementation status
34
+
35
+ This document describes the architecture implemented by the current Alpha release. Future capabilities are described only as extension boundaries and are not presented as implemented features.
36
+
37
+
38
+ ## Provider boundary (v0.3.4)
39
+
40
+ Providers are execution-mechanism adapters. Workflow intent, planning, scheduling, and policy semantics remain provider-independent.
41
+
42
+ ```mermaid
43
+ classDiagram
44
+ Workflow --> ExecutionPlan
45
+ ExecutionPlan --> Scheduler
46
+ Scheduler --> Machine
47
+ Machine --> Process
48
+ Process --> Result
49
+ Workflow --> ProviderRegistry
50
+ ProviderRegistry --> Provider
51
+ ProviderConfig --> ProviderRegistry
52
+ Provider --> Machine
53
+ Provider --> Result
54
+
55
+ class Provider {
56
+ <<protocol>>
57
+ +capabilities
58
+ +create_machine()
59
+ }
60
+ class ProviderConfig {
61
+ +name
62
+ +options
63
+ }
64
+ class ProviderRegistry {
65
+ +register()
66
+ +resolve()
67
+ +find()
68
+ }
69
+ class LocalProvider
70
+ class DockerProvider
71
+ Provider <|.. LocalProvider
72
+ Provider <|.. DockerProvider
73
+ ```
74
+
75
+ Runtime resolution is deliberately outside the Workflow IR:
76
+
77
+ ```text
78
+ execute_workflow(workflow, provider=config)
79
+ -> resolve config.name in ProviderRegistry
80
+ -> verify required capabilities
81
+ -> materialize Provider Machine
82
+ -> run the existing Scheduler
83
+ -> return the existing Result model
84
+ ```
85
+
86
+ A new backend should therefore add a Provider/Adapter and conformance tests rather than introduce provider-specific branching into Workflow or planning.
@@ -0,0 +1,33 @@
1
+ # v0.3.3 — Kernel Contract Stabilization
2
+
3
+ ## Summary
4
+
5
+ v0.3.3 stabilizes the smallest Marsh execution kernel before further provider and concurrency work.
6
+
7
+ ## Contract changes
8
+
9
+ - `plan_workflow()` is the canonical dependency-planning semantic owner.
10
+ - `validate_workflow()` delegates to canonical planning instead of maintaining a second topological implementation.
11
+ - Deterministic task order and initial readiness are regression-tested.
12
+ - Scheduler and Machine replacement are covered by conformance tests.
13
+ - Lifecycle, failure, timeout, cancellation, dependency-skip, and fail-fast semantics are regression-tested.
14
+ - Modern namespace compatibility remains covered without removing legacy APIs.
15
+ - Observer failures remain non-authoritative and cannot change execution outcomes.
16
+
17
+ ## Reliability fix
18
+
19
+ - Hardened Docker container cleanup against auto-remove races that can surface as stale-container `404 Not Found` errors.
20
+
21
+ ## Verification
22
+
23
+ - CircleCI: Python 3.10, 3.11, 3.12
24
+ - CircleCI: lint and wheel build
25
+ - CircleCI: Docker integration
26
+ - CircleCI: SSH integration
27
+ - GitHub Actions: Linux test/coverage
28
+ - GitHub Actions: native Windows portability
29
+ - GitHub Actions: Docker integration smoke
30
+
31
+ ## Scope guardrails
32
+
33
+ No new runtime dependency, concurrency engine, remote execution model, workflow IR expansion, or provider architecture expansion was introduced in this release.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "marsh-lib"
7
- version = "0.3.2"
7
+ version = "0.3.4"
8
8
  description = "Lightweight, extensible Python library for building, managing, and executing command workflows."
9
9
  authors = [{ name = "Cedric Anover", email = "cedric.anover@hotmail.com" }]
10
10
  license = { text = "MIT" }
@@ -26,10 +26,15 @@ classifiers = [
26
26
  "Operating System :: Microsoft :: Windows",
27
27
  "Operating System :: POSIX :: Linux",
28
28
  ]
29
- dependencies = [
30
- "fabric>=3.2.2,<4.0.0",
29
+ dependencies = []
30
+
31
+ [project.optional-dependencies]
32
+ docker = [
31
33
  "docker>=7.1.0,<8.0.0",
32
34
  ]
35
+ ssh = [
36
+ "fabric>=3.2.2,<4.0.0",
37
+ ]
33
38
 
34
39
  [project.urls]
35
40
  Repository = "https://github.com/CedricAnover/marsh"
@@ -42,6 +47,8 @@ dev = [
42
47
  "fabric[pytest]>=3.2.2,<4.0.0",
43
48
  ]
44
49
  test = [
50
+ "docker>=7.1.0,<8.0.0",
51
+ "fabric[pytest]>=3.2.2,<4.0.0",
45
52
  "pytest>=8.3.2,<9.0.0",
46
53
  "pytest-mock>=3.14.0,<4.0.0",
47
54
  "testcontainers>=4.9.0,<5.0.0",
@@ -78,3 +85,9 @@ line-length = 88
78
85
  target-version = ['py310', 'py311', 'py312', 'py313']
79
86
  include = '\.pyi?$'
80
87
  #extend-exclude = '...'
88
+
89
+
90
+ [tool.pytest.ini_options]
91
+ markers = [
92
+ "integration: requires external runtime services such as Docker or SSH",
93
+ ]
@@ -0,0 +1,9 @@
1
+ from marsh.core import *
2
+
3
+
4
+ def __getattr__(name):
5
+ if name == "ssh":
6
+ import importlib
7
+
8
+ return importlib.import_module("marsh.ssh")
9
+ raise AttributeError(name)
@@ -19,6 +19,16 @@ from marsh.core.serialization import (
19
19
 
20
20
  from marsh.core.runtime import ExecutionPlan, LocalMachine, LocalProcess, SequentialScheduler, execute_workflow, plan_workflow
21
21
  from marsh.core.policies import ExecutionPolicy, FailurePolicy, ResourcePolicy, RetryPolicy, TimeoutPolicy
22
- from marsh.core.providers import LocalProvider, ProviderCapabilities, ProviderRegistry, UnsupportedCapabilityError
22
+ from marsh.core.providers import (
23
+ LocalProvider,
24
+ Provider,
25
+ ProviderCapabilities,
26
+ ProviderConfig,
27
+ ProviderConfigurationError,
28
+ ProviderError,
29
+ ProviderRegistry,
30
+ ProviderUnavailableError,
31
+ UnsupportedCapabilityError,
32
+ )
23
33
  from marsh.core.cache import Cache, CachePolicy, MemoryCache, cache_key_for_task
24
34
  from marsh.core.observability import EventType, Observer, RuntimeEvent, emit_event
@@ -1,15 +1,26 @@
1
- """Provider discovery and capability negotiation for Marsh."""
1
+ """Provider discovery, configuration, and capability negotiation for Marsh."""
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
5
  from dataclasses import dataclass
6
6
  from typing import Iterable, Mapping, Protocol, runtime_checkable
7
7
 
8
- from marsh.core.domain import Machine, ProcessSpec
9
- from marsh.core.runtime import LocalMachine
8
+ from marsh.core.domain import Machine
10
9
 
11
10
 
12
- class UnsupportedCapabilityError(ValueError):
11
+ class ProviderError(RuntimeError):
12
+ """Base error for provider-boundary failures."""
13
+
14
+
15
+ class ProviderConfigurationError(ProviderError, ValueError):
16
+ """Raised for invalid provider configuration."""
17
+
18
+
19
+ class ProviderUnavailableError(ProviderError):
20
+ """Raised when a configured provider cannot be reached."""
21
+
22
+
23
+ class UnsupportedCapabilityError(ProviderError, ValueError):
13
24
  """Raised when a provider cannot satisfy a required capability."""
14
25
 
15
26
 
@@ -35,6 +46,21 @@ class ProviderCapabilities:
35
46
  return set(required).issubset(self.values)
36
47
 
37
48
 
49
+ @dataclass(frozen=True)
50
+ class ProviderConfig:
51
+ """Provider selection/configuration without provider-specific semantics."""
52
+
53
+ name: str
54
+ options: Mapping[str, object]
55
+
56
+ def __init__(self, name: str, options: Mapping[str, object] | None = None):
57
+ normalized = name.strip()
58
+ if not normalized:
59
+ raise ProviderConfigurationError("provider name must be non-empty")
60
+ object.__setattr__(self, "name", normalized)
61
+ object.__setattr__(self, "options", dict(options or {}))
62
+
63
+
38
64
  @runtime_checkable
39
65
  class Provider(Protocol):
40
66
  """Mechanism boundary for materializing execution machines."""
@@ -68,14 +94,18 @@ class LocalProvider:
68
94
  }
69
95
  )
70
96
 
71
- def create_machine(self, **kwargs) -> LocalMachine:
97
+ def create_machine(self, **kwargs) -> Machine:
98
+ from marsh.core.runtime import LocalMachine
99
+
72
100
  if kwargs:
73
- raise TypeError(f"unsupported local provider options: {sorted(kwargs)}")
101
+ raise ProviderConfigurationError(
102
+ f"unsupported local provider options: {sorted(kwargs)}"
103
+ )
74
104
  return LocalMachine()
75
105
 
76
106
 
77
107
  class ProviderRegistry:
78
- """Named provider registry with explicit capability negotiation."""
108
+ """Deterministic named provider registry with capability negotiation."""
79
109
 
80
110
  def __init__(self, providers: Mapping[str, Provider] | None = None):
81
111
  self._providers: dict[str, Provider] = {}
@@ -83,15 +113,19 @@ class ProviderRegistry:
83
113
  self.register(name, provider)
84
114
 
85
115
  def register(self, name: str, provider: Provider) -> None:
86
- name = name.strip()
87
- if not name:
88
- raise ValueError("provider name must be non-empty")
89
- if name in self._providers:
90
- raise ValueError(f"provider {name!r} is already registered")
91
- self._providers[name] = provider
116
+ config_name = name.strip()
117
+ if not config_name:
118
+ raise ProviderConfigurationError("provider name must be non-empty")
119
+ if config_name in self._providers:
120
+ raise ProviderConfigurationError(
121
+ f"provider {config_name!r} is already registered"
122
+ )
123
+ if not isinstance(provider, Provider):
124
+ raise TypeError("provider does not satisfy the Provider contract")
125
+ self._providers[config_name] = provider
92
126
 
93
127
  def get(self, name: str) -> Provider | None:
94
- return self._providers.get(name)
128
+ return self._providers.get(name.strip())
95
129
 
96
130
  def find(self, capabilities: Iterable[str] = ()) -> tuple[tuple[str, Provider], ...]:
97
131
  required = frozenset(capabilities)
@@ -101,8 +135,12 @@ class ProviderRegistry:
101
135
  if self._providers[name].capabilities.satisfies(required)
102
136
  )
103
137
 
104
- def require(self, name: str, capabilities: Iterable[str] = ()) -> Provider:
105
- provider = self._providers.get(name)
138
+ def require(
139
+ self,
140
+ name: str,
141
+ capabilities: Iterable[str] = (),
142
+ ) -> Provider:
143
+ provider = self.get(name)
106
144
  if provider is None:
107
145
  raise KeyError(name)
108
146
  required = frozenset(capabilities)
@@ -112,3 +150,10 @@ class ProviderRegistry:
112
150
  f"provider {name!r} does not support capabilities: {', '.join(missing)}"
113
151
  )
114
152
  return provider
153
+
154
+ def resolve(
155
+ self,
156
+ config: ProviderConfig,
157
+ capabilities: Iterable[str] = (),
158
+ ) -> Provider:
159
+ return self.require(config.name, capabilities)
@@ -21,6 +21,21 @@ from marsh.core.observability import EventType, Observer, RuntimeEvent, emit_eve
21
21
  from marsh.core.policies import ExecutionPolicy
22
22
 
23
23
 
24
+ class _LocalProvider:
25
+ """Lazy local provider used only for runtime compatibility."""
26
+
27
+ @property
28
+ def capabilities(self):
29
+ from marsh.core.providers import LocalProvider
30
+
31
+ return LocalProvider().capabilities
32
+
33
+ def create_machine(self):
34
+ from marsh.core.providers import LocalProvider
35
+
36
+ return LocalProvider().create_machine()
37
+
38
+
24
39
  @dataclass(frozen=True)
25
40
  class ExecutionPlan:
26
41
  """Deterministic workflow order plus tasks initially ready to run."""
@@ -317,9 +332,20 @@ def execute_workflow(
317
332
  policy: ExecutionPolicy | None = None,
318
333
  observers: tuple[Observer, ...] = (),
319
334
  cache: Cache | None = None,
335
+ provider=None,
336
+ provider_registry=None,
320
337
  ) -> dict[str, Result]:
321
- """Execute a workflow sequentially with explicit, composable policies."""
322
- machine = machine or LocalMachine()
338
+ """Execute a workflow sequentially through a selected execution provider."""
339
+ if machine is not None and provider is not None:
340
+ raise ValueError("machine and provider cannot both be supplied")
341
+ if provider is not None:
342
+ from marsh.core.providers import ProviderConfig, ProviderRegistry
343
+
344
+ registry = provider_registry or ProviderRegistry({"local": _LocalProvider()})
345
+ config = provider if isinstance(provider, ProviderConfig) else ProviderConfig(str(provider))
346
+ machine = registry.resolve(config).create_machine(**dict(config.options))
347
+ else:
348
+ machine = machine or LocalMachine()
323
349
  scheduler = scheduler or SequentialScheduler()
324
350
  policy = policy or ExecutionPolicy()
325
351
  results: dict[str, Result] = {}
@@ -1,39 +1,17 @@
1
1
  """Workflow validation and deterministic serialization."""
2
2
 
3
3
  import json
4
- from heapq import heappop, heappush
5
4
  from typing import Any, Mapping
6
5
 
7
6
  from marsh.core.configuration import normalize_workflow
8
7
  from marsh.core.domain import Workflow
8
+ from marsh.core.runtime import plan_workflow
9
9
 
10
10
 
11
11
  def validate_workflow(workflow: Workflow | Mapping[str, Any]) -> tuple[str, ...]:
12
- workflow = normalize_workflow(workflow)
13
- tasks = {task.id: task for task in workflow.tasks}
14
- indegree = {task_id: 0 for task_id in tasks}
15
- dependents = {task_id: [] for task_id in tasks}
16
- for task in workflow.tasks:
17
- for dependency in task.dependencies:
18
- if dependency not in tasks:
19
- raise ValueError(f"task {task.id!r} has unknown dependencies: {dependency}")
20
- indegree[task.id] += 1
21
- dependents[dependency].append(task.id)
22
- ready = []
23
- for task_id, degree in indegree.items():
24
- if degree == 0:
25
- heappush(ready, task_id)
26
- order = []
27
- while ready:
28
- task_id = heappop(ready)
29
- order.append(task_id)
30
- for dependent in sorted(dependents[task_id]):
31
- indegree[dependent] -= 1
32
- if indegree[dependent] == 0:
33
- heappush(ready, dependent)
34
- if len(order) != len(tasks):
35
- raise ValueError("workflow contains a dependency cycle")
36
- return tuple(order)
12
+ """Validate dependencies using the canonical planning semantic owner."""
13
+
14
+ return plan_workflow(normalize_workflow(workflow)).order
37
15
 
38
16
 
39
17
  def _data(value: Any) -> Any:
@@ -103,7 +103,11 @@ class DockerContainer:
103
103
  self._timer.cancel()
104
104
 
105
105
  # Remove the Container
106
- all_containers: list[Container] = self._client.containers.list(all=True)
106
+ try:
107
+ all_containers: list[Container] = self._client.containers.list(all=True)
108
+ except NotFound:
109
+ all_containers = []
110
+
107
111
  for container in all_containers:
108
112
  if container.name == self._name:
109
113
  try:
@@ -0,0 +1,253 @@
1
+ """Docker-backed provider adapter for the canonical process contract."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+
7
+ from marsh.core.domain import Machine, Process, ProcessSpec, ProcessStatus, Result, can_transition
8
+ from marsh.core.providers import (
9
+ ProviderCapabilities,
10
+ ProviderConfigurationError,
11
+ ProviderError,
12
+ ProviderUnavailableError,
13
+ )
14
+
15
+
16
+ @dataclass(frozen=True)
17
+ class DockerProviderConfig:
18
+ """Configuration for Docker process materialization."""
19
+
20
+ image: str
21
+ client_kwargs: dict[str, object]
22
+
23
+ def __init__(self, image: str, client_kwargs: dict[str, object] | None = None):
24
+ image = image.strip()
25
+ if not image:
26
+ raise ProviderConfigurationError("docker image must be non-empty")
27
+ object.__setattr__(self, "image", image)
28
+ object.__setattr__(self, "client_kwargs", dict(client_kwargs or {}))
29
+
30
+
31
+ class DockerProcess:
32
+ """Process contract backed by one Docker container."""
33
+
34
+ def __init__(self, spec: ProcessSpec, config: DockerProviderConfig):
35
+ self.spec = spec
36
+ self.config = config
37
+ self._status = ProcessStatus.CREATED
38
+ self._result: Result | None = None
39
+ self._container = None
40
+ self._client = None
41
+
42
+ @property
43
+ def status(self) -> ProcessStatus:
44
+ return self._status
45
+
46
+ def _transition(self, target: ProcessStatus) -> None:
47
+ if target is self._status:
48
+ return
49
+ if not can_transition(self._status, target):
50
+ raise RuntimeError(
51
+ f"invalid process transition: {self._status.value} -> {target.value}"
52
+ )
53
+ self._status = target
54
+
55
+ def start(self) -> None:
56
+ if self._status is not ProcessStatus.CREATED:
57
+ raise RuntimeError(f"cannot start process in {self._status.value} state")
58
+ if self.spec.stdin is not None:
59
+ raise ProviderConfigurationError(
60
+ "DockerProcess does not support ProcessSpec.stdin yet"
61
+ )
62
+ self._transition(ProcessStatus.STARTING)
63
+ try:
64
+ import docker
65
+ from docker.errors import DockerException
66
+ except ImportError as exc:
67
+ self._transition(ProcessStatus.FAILED)
68
+ raise ProviderUnavailableError(
69
+ "Docker provider requires the 'docker' package"
70
+ ) from exc
71
+
72
+ try:
73
+ self._client = docker.DockerClient(**self.config.client_kwargs)
74
+ self._container = self._client.containers.create(
75
+ self.config.image,
76
+ command=[self.spec.executable, *self.spec.arguments],
77
+ working_dir=self.spec.working_directory,
78
+ environment=dict(self.spec.environment),
79
+ detach=True,
80
+ auto_remove=False,
81
+ )
82
+ self._container.start()
83
+ self._transition(ProcessStatus.RUNNING)
84
+ except DockerException as exc:
85
+ self._transition(ProcessStatus.FAILED)
86
+ self._result = Result(status=ProcessStatus.FAILED, error=str(exc))
87
+ self._cleanup()
88
+ raise ProviderUnavailableError(str(exc)) from exc
89
+ except Exception as exc:
90
+ self._transition(ProcessStatus.FAILED)
91
+ self._result = Result(status=ProcessStatus.FAILED, error=str(exc))
92
+ self._cleanup()
93
+ raise ProviderError(str(exc)) from exc
94
+
95
+ def wait(self) -> Result:
96
+ if self._result is not None:
97
+ return self._result
98
+ if self._container is None:
99
+ return Result(status=self._status, error="process was not started")
100
+ try:
101
+ status_code = self._container.wait(timeout=self.spec.timeout)["StatusCode"]
102
+ stdout = self._container.logs(stdout=True, stderr=False)
103
+ stderr = self._container.logs(stdout=False, stderr=True)
104
+ status = (
105
+ ProcessStatus.COMPLETED
106
+ if status_code == 0
107
+ else ProcessStatus.FAILED
108
+ )
109
+ error = (
110
+ None
111
+ if status is ProcessStatus.COMPLETED
112
+ else f"process exited with code {status_code}"
113
+ )
114
+ self._transition(status)
115
+ self._result = Result(
116
+ stdout=stdout,
117
+ stderr=stderr,
118
+ exit_code=status_code,
119
+ status=status,
120
+ error=error,
121
+ )
122
+ return self._result
123
+ except TimeoutError:
124
+ self._transition(ProcessStatus.TIMED_OUT)
125
+ self._result = Result(
126
+ status=ProcessStatus.TIMED_OUT,
127
+ error="process timed out",
128
+ )
129
+ return self._result
130
+ except Exception as exc:
131
+ if exc.__class__.__name__ == "ReadTimeout":
132
+ self._transition(ProcessStatus.TIMED_OUT)
133
+ self._result = Result(
134
+ status=ProcessStatus.TIMED_OUT,
135
+ error="process timed out",
136
+ )
137
+ return self._result
138
+ if self._status is ProcessStatus.RUNNING:
139
+ self._transition(ProcessStatus.FAILED)
140
+ self._result = Result(status=ProcessStatus.FAILED, error=str(exc))
141
+ raise ProviderError(str(exc)) from exc
142
+ finally:
143
+ self._cleanup()
144
+
145
+ def poll(self) -> ProcessStatus:
146
+ if self._container is None:
147
+ return self._status
148
+ self._container.reload()
149
+ if self._container.status == "exited" and self._status is ProcessStatus.RUNNING:
150
+ self._transition(
151
+ ProcessStatus.COMPLETED
152
+ if self._container.attrs["State"]["ExitCode"] == 0
153
+ else ProcessStatus.FAILED
154
+ )
155
+ return self._status
156
+
157
+ def stop(self) -> None:
158
+ if self._container is not None and self._status is ProcessStatus.RUNNING:
159
+ try:
160
+ self._transition(ProcessStatus.STOPPING)
161
+ self._container.stop(timeout=0)
162
+ self._transition(ProcessStatus.CANCELLED)
163
+ self._result = Result(status=ProcessStatus.CANCELLED)
164
+ self._cleanup()
165
+ except Exception as exc:
166
+ if self._status is ProcessStatus.STOPPING:
167
+ self._status = ProcessStatus.RUNNING
168
+ raise ProviderError(str(exc)) from exc
169
+
170
+ def terminate(self) -> None:
171
+ self.stop()
172
+
173
+ def kill(self) -> None:
174
+ if self._container is not None and self._status is ProcessStatus.RUNNING:
175
+ try:
176
+ self._transition(ProcessStatus.STOPPING)
177
+ self._container.kill()
178
+ self._transition(ProcessStatus.CANCELLED)
179
+ self._result = Result(status=ProcessStatus.CANCELLED)
180
+ self._cleanup()
181
+ except Exception as exc:
182
+ if self._status is ProcessStatus.STOPPING:
183
+ self._status = ProcessStatus.RUNNING
184
+ raise ProviderError(str(exc)) from exc
185
+
186
+ def cancel(self) -> None:
187
+ if self._status is ProcessStatus.CREATED:
188
+ self._transition(ProcessStatus.CANCELLED)
189
+ self._result = Result(status=ProcessStatus.CANCELLED)
190
+ return
191
+ self.stop()
192
+
193
+ def result(self) -> Result:
194
+ return self.wait()
195
+
196
+ def _cleanup(self) -> None:
197
+ if self._container is not None:
198
+ try:
199
+ self._container.remove(force=True)
200
+ except Exception:
201
+ pass
202
+ self._container = None
203
+ if self._client is not None:
204
+ self._client.close()
205
+ self._client = None
206
+
207
+
208
+ class DockerMachine:
209
+ """Machine that materializes Docker-backed processes."""
210
+
211
+ def __init__(self, config: DockerProviderConfig):
212
+ self.config = config
213
+
214
+ def create_process(self, spec: ProcessSpec) -> DockerProcess:
215
+ return DockerProcess(spec, self.config)
216
+
217
+
218
+ class DockerProvider:
219
+ """Reference non-local provider; no Workflow semantics live here."""
220
+
221
+ name = "docker"
222
+
223
+ def __init__(self, image: str = "python:3.12-slim", client_kwargs=None):
224
+ self.config = DockerProviderConfig(image, client_kwargs)
225
+
226
+ @property
227
+ def capabilities(self) -> ProviderCapabilities:
228
+ return ProviderCapabilities(
229
+ {
230
+ "machine.create",
231
+ "process.start",
232
+ "process.wait",
233
+ "process.poll",
234
+ "process.stop",
235
+ "process.terminate",
236
+ "process.kill",
237
+ "process.cancel",
238
+ "process.result",
239
+ }
240
+ )
241
+
242
+ def create_machine(self, **kwargs) -> Machine:
243
+ allowed = {"image", "client_kwargs"}
244
+ unsupported = sorted(set(kwargs) - allowed)
245
+ if unsupported:
246
+ raise ProviderConfigurationError(
247
+ f"unsupported docker provider options: {unsupported}"
248
+ )
249
+ config = DockerProviderConfig(
250
+ kwargs.get("image", self.config.image),
251
+ kwargs.get("client_kwargs", self.config.client_kwargs),
252
+ )
253
+ return DockerMachine(config)
@@ -421,12 +421,9 @@ wheels = [
421
421
 
422
422
  [[package]]
423
423
  name = "marsh-lib"
424
- version = "0.2.0"
424
+ version = "0.3.4"
425
425
  source = { editable = "." }
426
- dependencies = [
427
- { name = "docker" },
428
- { name = "fabric" },
429
- ]
426
+ dependencies = []
430
427
 
431
428
  [package.dev-dependencies]
432
429
  dev = [
@@ -436,6 +433,8 @@ dev = [
436
433
  { name = "pyproject-flake8" },
437
434
  ]
438
435
  test = [
436
+ { name = "docker" },
437
+ { name = "fabric", extra = ["pytest"] },
439
438
  { name = "pytest" },
440
439
  { name = "pytest-mock" },
441
440
  { name = "testcontainers" },
@@ -443,8 +442,8 @@ test = [
443
442
 
444
443
  [package.metadata]
445
444
  requires-dist = [
446
- { name = "docker", specifier = ">=7.1.0,<8.0.0" },
447
- { name = "fabric", specifier = ">=3.2.2,<4.0.0" },
445
+ { name = "docker", marker = "extra == 'docker'", specifier = ">=7.1.0,<8.0.0" },
446
+ { name = "fabric", marker = "extra == 'ssh'", specifier = ">=3.2.2,<4.0.0" },
448
447
  ]
449
448
 
450
449
  [package.metadata.requires-dev]
@@ -455,6 +454,8 @@ dev = [
455
454
  { name = "pyproject-flake8", specifier = ">=7.0.0,<8.0.0" },
456
455
  ]
457
456
  test = [
457
+ { name = "docker", specifier = ">=7.1.0,<8.0.0" },
458
+ { name = "fabric", extras = ["pytest"], specifier = ">=3.2.2,<4.0.0" },
458
459
  { name = "pytest", specifier = ">=8.3.2,<9.0.0" },
459
460
  { name = "pytest-mock", specifier = ">=3.14.0,<4.0.0" },
460
461
  { name = "testcontainers", specifier = ">=4.9.0,<5.0.0" },
@@ -1,35 +0,0 @@
1
- # Marsh Architecture
2
-
3
- ## Architecture
4
-
5
- Marsh is moving toward a small execution kernel with explicit boundaries:
6
-
7
- ```mermaid
8
- flowchart TD
9
- A[UX / Python API] --> B[Workflow / Task]
10
- B --> C[Validation]
11
- C --> D[Planning]
12
- D --> E[Scheduler]
13
- E --> F[Machine / Process]
14
- F --> G[Result]
15
- ```
16
-
17
- The architectural goal is to keep:
18
-
19
- - workflow intent;
20
- - execution mechanisms;
21
- - machines;
22
- - processes;
23
- - scheduling;
24
- - policies;
25
- - providers;
26
- - observability; and
27
- - results
28
-
29
- as distinct concepts.
30
-
31
- The existing command and DAG APIs remain useful low-level building blocks and compatibility surfaces.
32
-
33
- ## Implementation status
34
-
35
- This document describes the architecture implemented by the current Alpha release. Future capabilities are described only as extension boundaries and are not presented as implemented features.
@@ -1,2 +0,0 @@
1
- from marsh.core import *
2
- from marsh import ssh
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes