loom-pipeline 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loom_pipeline-0.0.1/LICENSE +21 -0
- loom_pipeline-0.0.1/PKG-INFO +302 -0
- loom_pipeline-0.0.1/README.md +257 -0
- loom_pipeline-0.0.1/pyproject.toml +94 -0
- loom_pipeline-0.0.1/setup.cfg +4 -0
- loom_pipeline-0.0.1/src/loom/__init__.py +3 -0
- loom_pipeline-0.0.1/src/loom/py.typed +0 -0
- loom_pipeline-0.0.1/src/loom/runner/__init__.py +37 -0
- loom_pipeline-0.0.1/src/loom/runner/clean.py +134 -0
- loom_pipeline-0.0.1/src/loom/runner/cli.py +259 -0
- loom_pipeline-0.0.1/src/loom/runner/config.py +287 -0
- loom_pipeline-0.0.1/src/loom/runner/executor.py +437 -0
- loom_pipeline-0.0.1/src/loom/runner/orchestrator.py +382 -0
- loom_pipeline-0.0.1/src/loom/runner/url.py +174 -0
- loom_pipeline-0.0.1/src/loom/ui/__init__.py +5 -0
- loom_pipeline-0.0.1/src/loom/ui/cli.py +117 -0
- loom_pipeline-0.0.1/src/loom/ui/execution.py +243 -0
- loom_pipeline-0.0.1/src/loom/ui/frontend/dist/assets/index-CqxcMPWV.css +32 -0
- loom_pipeline-0.0.1/src/loom/ui/frontend/dist/assets/index-rRhXrXo0.js +220 -0
- loom_pipeline-0.0.1/src/loom/ui/frontend/dist/favicon.svg +13 -0
- loom_pipeline-0.0.1/src/loom/ui/frontend/dist/index.html +14 -0
- loom_pipeline-0.0.1/src/loom/ui/server/__init__.py +140 -0
- loom_pipeline-0.0.1/src/loom/ui/server/endpoints.py +819 -0
- loom_pipeline-0.0.1/src/loom/ui/server/graph.py +491 -0
- loom_pipeline-0.0.1/src/loom/ui/server/models.py +101 -0
- loom_pipeline-0.0.1/src/loom/ui/server/state.py +54 -0
- loom_pipeline-0.0.1/src/loom/ui/server/terminal.py +953 -0
- loom_pipeline-0.0.1/src/loom/ui/server/thumbnails.py +237 -0
- loom_pipeline-0.0.1/src/loom/ui/server/validation.py +71 -0
- loom_pipeline-0.0.1/src/loom/ui/task_schema.py +295 -0
- loom_pipeline-0.0.1/src/loom_pipeline.egg-info/PKG-INFO +302 -0
- loom_pipeline-0.0.1/src/loom_pipeline.egg-info/SOURCES.txt +46 -0
- loom_pipeline-0.0.1/src/loom_pipeline.egg-info/dependency_links.txt +1 -0
- loom_pipeline-0.0.1/src/loom_pipeline.egg-info/entry_points.txt +3 -0
- loom_pipeline-0.0.1/src/loom_pipeline.egg-info/requires.txt +28 -0
- loom_pipeline-0.0.1/src/loom_pipeline.egg-info/top_level.txt +1 -0
- loom_pipeline-0.0.1/tests/test_clean.py +323 -0
- loom_pipeline-0.0.1/tests/test_editor_cli.py +258 -0
- loom_pipeline-0.0.1/tests/test_editor_execution.py +358 -0
- loom_pipeline-0.0.1/tests/test_editor_server.py +582 -0
- loom_pipeline-0.0.1/tests/test_editor_server_endpoints.py +1529 -0
- loom_pipeline-0.0.1/tests/test_editor_thumbnails.py +407 -0
- loom_pipeline-0.0.1/tests/test_orchestrator.py +409 -0
- loom_pipeline-0.0.1/tests/test_runner_cli.py +437 -0
- loom_pipeline-0.0.1/tests/test_runner_config.py +620 -0
- loom_pipeline-0.0.1/tests/test_runner_executor.py +796 -0
- loom_pipeline-0.0.1/tests/test_task_schema.py +401 -0
- loom_pipeline-0.0.1/tests/test_url.py +275 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Relja Ljubobratovic
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: loom-pipeline
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Visual pipeline editor and runner for task automation
|
|
5
|
+
Author-email: Relja Ljubobratovic <ljubobratovic.relja@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ljubobratovicrelja/loom
|
|
8
|
+
Project-URL: Repository, https://github.com/ljubobratovicrelja/loom
|
|
9
|
+
Project-URL: Documentation, https://ljubobratovicrelja.github.io/loom/
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: PyYAML>=6.0
|
|
20
|
+
Requires-Dist: requests>=2.28.0
|
|
21
|
+
Provides-Extra: ui
|
|
22
|
+
Requires-Dist: fastapi>=0.100.0; extra == "ui"
|
|
23
|
+
Requires-Dist: uvicorn>=0.23.0; extra == "ui"
|
|
24
|
+
Requires-Dist: websockets>=12.0; extra == "ui"
|
|
25
|
+
Requires-Dist: ruamel.yaml>=0.18.0; extra == "ui"
|
|
26
|
+
Requires-Dist: send2trash>=1.8.2; extra == "ui"
|
|
27
|
+
Requires-Dist: opencv-python-headless>=4.8.0; extra == "ui"
|
|
28
|
+
Provides-Extra: examples
|
|
29
|
+
Requires-Dist: numpy>=1.24.0; extra == "examples"
|
|
30
|
+
Requires-Dist: scipy>=1.10.0; extra == "examples"
|
|
31
|
+
Requires-Dist: matplotlib>=3.7.0; extra == "examples"
|
|
32
|
+
Requires-Dist: opencv-python-headless>=4.8.0; extra == "examples"
|
|
33
|
+
Provides-Extra: dev
|
|
34
|
+
Requires-Dist: loom-pipeline[examples,ui]; extra == "dev"
|
|
35
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
36
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
37
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
38
|
+
Requires-Dist: httpx>=0.24.0; extra == "dev"
|
|
39
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
40
|
+
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
41
|
+
Requires-Dist: types-PyYAML>=6.0.0; extra == "dev"
|
|
42
|
+
Requires-Dist: types-requests>=2.28.0; extra == "dev"
|
|
43
|
+
Requires-Dist: mkdocs-material>=9.0.0; extra == "dev"
|
|
44
|
+
Dynamic: license-file
|
|
45
|
+
|
|
46
|
+

|
|
47
|
+
|
|
48
|
+
[](https://loom-examples.onrender.com/)
|
|
49
|
+
[](https://github.com/ljubobratovicrelja/loom/actions/workflows/ci.yml)
|
|
50
|
+
[](https://ljubobratovicrelja.github.io/loom/)
|
|
51
|
+
[](https://pypi.org/project/loom-pipeline/)
|
|
52
|
+
|
|
53
|
+
A lightweight visual pipeline runner for research.
|
|
54
|
+
|
|
55
|
+
Connect your Python scripts into a graph, tweak parameters, run experiments, see results — without setting up Airflow or learning a workflow framework.
|
|
56
|
+
|
|
57
|
+
**[Try the live demo](https://loom-examples.onrender.com/)** — no installation required. Browse and run the [example pipelines](examples/) in your browser. (First load may take ~30s to wake up.)
|
|
58
|
+
|
|
59
|
+
**What it is:**
|
|
60
|
+
- A visual graph editor for your existing Python scripts
|
|
61
|
+
- A CLI runner with dependency tracking
|
|
62
|
+
- A way to organize experiments as YAML files you can version control
|
|
63
|
+
- Simple enough to install in any project's virtualenv
|
|
64
|
+
|
|
65
|
+
**What it isn't:**
|
|
66
|
+
- Not a replacement for Airflow/Kubeflow/Prefect (those are for production pipelines)
|
|
67
|
+
- Not an experiment tracker like W&B or MLflow (though it can be made to complement them)
|
|
68
|
+
|
|
69
|
+
## Installation
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
# Core runner only
|
|
73
|
+
pip install loom-pipeline
|
|
74
|
+
|
|
75
|
+
# With visual editor
|
|
76
|
+
pip install loom-pipeline[ui]
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
That's it. No configuration files to create, no external services to manage.
|
|
80
|
+
|
|
81
|
+
## Quick Start
|
|
82
|
+
|
|
83
|
+
Clone the repo and try an example:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
git clone https://github.com/ljubobratovicrelja/loom.git
|
|
87
|
+
cd loom
|
|
88
|
+
pip install -e .[ui,examples]
|
|
89
|
+
|
|
90
|
+
# Run a pipeline from the command line
|
|
91
|
+
loom examples/image-processing/pipeline.yml
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
```
|
|
95
|
+
Pipeline: 3 step(s) to run [parallel]
|
|
96
|
+
----------------------------------------
|
|
97
|
+
[RUNNING] grayscale
|
|
98
|
+
[grayscale] Converted to grayscale: .loom-url-cache/35bb4a6_Lenna.png -> data/grayscale.png
|
|
99
|
+
[SUCCESS] grayscale
|
|
100
|
+
[RUNNING] blur
|
|
101
|
+
[blur] Gaussian blur (radius=15): data/grayscale.png -> data/blurred.png
|
|
102
|
+
[SUCCESS] blur
|
|
103
|
+
[RUNNING] edge_detect
|
|
104
|
+
[edge_detect] Edge detection: data/grayscale.png -> data/edges.png
|
|
105
|
+
[SUCCESS] edge_detect
|
|
106
|
+
----------------------------------------
|
|
107
|
+
Completed: 3/3 steps succeeded
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Or open it in the visual editor:
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
# Edit a single pipeline
|
|
114
|
+
loom-ui examples/image-processing/pipeline.yml
|
|
115
|
+
|
|
116
|
+
# Browse all example pipelines
|
|
117
|
+
loom-ui examples/
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
The editor opens in your browser where you can see the pipeline graph, run steps, and view outputs.
|
|
121
|
+
|
|
122
|
+
## Building Your Own Pipeline
|
|
123
|
+
|
|
124
|
+
Add Loom to your project's environment:
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
pip install loom-pipeline[ui] # or just loom-pipeline for CLI only
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Now you can run pipelines from within your project. Here's how to set one up.
|
|
131
|
+
|
|
132
|
+
### 1. Point it at your scripts
|
|
133
|
+
|
|
134
|
+
Say you have some Python scripts that process data:
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
tasks/
|
|
138
|
+
extract_features.py # Takes video, outputs CSV
|
|
139
|
+
train_model.py # Takes CSV, outputs model
|
|
140
|
+
evaluate.py # Takes model + test data, outputs metrics
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### 2. Describe the pipeline in YAML
|
|
144
|
+
|
|
145
|
+
```yaml
|
|
146
|
+
# experiment.yml
|
|
147
|
+
variables:
|
|
148
|
+
video: data/raw/recording.mp4
|
|
149
|
+
features: data/processed/features.csv
|
|
150
|
+
model: models/classifier.pt
|
|
151
|
+
metrics: results/metrics.json
|
|
152
|
+
|
|
153
|
+
parameters:
|
|
154
|
+
learning_rate: 0.001
|
|
155
|
+
epochs: 100
|
|
156
|
+
|
|
157
|
+
pipeline:
|
|
158
|
+
- name: extract
|
|
159
|
+
task: tasks/extract_features.py
|
|
160
|
+
inputs:
|
|
161
|
+
video: $video
|
|
162
|
+
outputs:
|
|
163
|
+
-o: $features
|
|
164
|
+
|
|
165
|
+
- name: train
|
|
166
|
+
task: tasks/train_model.py
|
|
167
|
+
inputs:
|
|
168
|
+
data: $features
|
|
169
|
+
outputs:
|
|
170
|
+
-o: $model
|
|
171
|
+
args:
|
|
172
|
+
--lr: $learning_rate
|
|
173
|
+
--epochs: $epochs
|
|
174
|
+
|
|
175
|
+
- name: evaluate
|
|
176
|
+
task: tasks/evaluate.py
|
|
177
|
+
inputs:
|
|
178
|
+
model: $model
|
|
179
|
+
outputs:
|
|
180
|
+
-o: $metrics
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
### 3. Run it
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
# Run the full pipeline
|
|
187
|
+
loom experiment.yml
|
|
188
|
+
|
|
189
|
+
# Run just one step
|
|
190
|
+
loom experiment.yml --step train
|
|
191
|
+
|
|
192
|
+
# Run from a step onward
|
|
193
|
+
loom experiment.yml --from train
|
|
194
|
+
|
|
195
|
+
# Try different parameters
|
|
196
|
+
loom experiment.yml --set learning_rate=0.01 --set epochs=200
|
|
197
|
+
|
|
198
|
+
# Override file paths
|
|
199
|
+
loom experiment.yml --var video=other_recording.mp4
|
|
200
|
+
|
|
201
|
+
# Run steps in parallel
|
|
202
|
+
loom experiment.yml --parallel --max-workers 4
|
|
203
|
+
|
|
204
|
+
# Preview without executing
|
|
205
|
+
loom experiment.yml --dry-run
|
|
206
|
+
|
|
207
|
+
# Clean all data (move to trash) and re-run from scratch
|
|
208
|
+
loom experiment.yml --clean
|
|
209
|
+
loom experiment.yml
|
|
210
|
+
|
|
211
|
+
# Preview what would be cleaned
|
|
212
|
+
loom experiment.yml --clean-list
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### 4. Or use the visual editor
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
loom-ui experiment.yml
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
This opens a browser-based editor where you can:
|
|
222
|
+
- See your pipeline as a visual graph
|
|
223
|
+
- Drag and drop to reorganize
|
|
224
|
+
- Run individual steps and see output in real-time
|
|
225
|
+
- Quickly see which outputs exist (green) vs missing (grey)
|
|
226
|
+
|
|
227
|
+
You can also point it at a directory to browse multiple pipelines:
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
loom-ui experiments/ # Browse all pipelines in a folder
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
Each pipeline should be in its own subdirectory with a `pipeline.yml` file inside. See [examples/](examples/) for the expected structure.
|
|
234
|
+
|
|
235
|
+
## How Scripts Work
|
|
236
|
+
|
|
237
|
+
Your scripts stay normal Python with argparse. Just add a YAML block in the docstring so Loom knows the interface:
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
"""Extract features from video.
|
|
241
|
+
|
|
242
|
+
---
|
|
243
|
+
inputs:
|
|
244
|
+
video:
|
|
245
|
+
type: video
|
|
246
|
+
description: Input video file
|
|
247
|
+
outputs:
|
|
248
|
+
-o:
|
|
249
|
+
type: csv
|
|
250
|
+
description: Output features
|
|
251
|
+
args:
|
|
252
|
+
--sample-rate:
|
|
253
|
+
type: int
|
|
254
|
+
default: 30
|
|
255
|
+
description: Frames to sample per second
|
|
256
|
+
---
|
|
257
|
+
"""
|
|
258
|
+
|
|
259
|
+
import argparse
|
|
260
|
+
|
|
261
|
+
def main():
|
|
262
|
+
parser = argparse.ArgumentParser()
|
|
263
|
+
parser.add_argument("video")
|
|
264
|
+
parser.add_argument("-o", "--output", required=True)
|
|
265
|
+
parser.add_argument("--sample-rate", type=int, default=30)
|
|
266
|
+
args = parser.parse_args()
|
|
267
|
+
# ... your code ...
|
|
268
|
+
|
|
269
|
+
if __name__ == "__main__":
|
|
270
|
+
main()
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
The YAML frontmatter is optional but enables the editor to show input/output types and provide better validation.
|
|
274
|
+
|
|
275
|
+
## Use Cases
|
|
276
|
+
|
|
277
|
+
**Parameter exploration**: Create parallel branches in your pipeline to test different configurations side by side.
|
|
278
|
+
|
|
279
|
+
**Reproducible experiments**: The YAML file captures your entire experiment setup. Commit it to git alongside your code.
|
|
280
|
+
|
|
281
|
+
**Iterative development**: Run just the steps you're working on. Loom tracks dependencies so upstream steps run only when needed.
|
|
282
|
+
|
|
283
|
+
**Result organization**: Variables point to file paths, so your outputs are organized by experiment configuration.
|
|
284
|
+
|
|
285
|
+
## Philosophy
|
|
286
|
+
|
|
287
|
+
Loom is intentionally minimal:
|
|
288
|
+
|
|
289
|
+
- **No database** — Everything is files: your scripts, YAML configs, and outputs
|
|
290
|
+
- **No external services** — The visual editor runs a local server that stops when you close it
|
|
291
|
+
- **No lock-in** — Your scripts work with or without Loom
|
|
292
|
+
- **No magic** — Loom just builds shell commands and runs them
|
|
293
|
+
|
|
294
|
+
This makes it easy to adopt incrementally. Start with one experiment, see if it helps, expand from there.
|
|
295
|
+
|
|
296
|
+
## License
|
|
297
|
+
|
|
298
|
+
MIT License — see [LICENSE](LICENSE) for details.
|
|
299
|
+
|
|
300
|
+
---
|
|
301
|
+
|
|
302
|
+
*Built for researchers who want to see their experiments, not manage infrastructure.*
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
3
|
+
[](https://loom-examples.onrender.com/)
|
|
4
|
+
[](https://github.com/ljubobratovicrelja/loom/actions/workflows/ci.yml)
|
|
5
|
+
[](https://ljubobratovicrelja.github.io/loom/)
|
|
6
|
+
[](https://pypi.org/project/loom-pipeline/)
|
|
7
|
+
|
|
8
|
+
A lightweight visual pipeline runner for research.
|
|
9
|
+
|
|
10
|
+
Connect your Python scripts into a graph, tweak parameters, run experiments, see results — without setting up Airflow or learning a workflow framework.
|
|
11
|
+
|
|
12
|
+
**[Try the live demo](https://loom-examples.onrender.com/)** — no installation required. Browse and run the [example pipelines](examples/) in your browser. (First load may take ~30s to wake up.)
|
|
13
|
+
|
|
14
|
+
**What it is:**
|
|
15
|
+
- A visual graph editor for your existing Python scripts
|
|
16
|
+
- A CLI runner with dependency tracking
|
|
17
|
+
- A way to organize experiments as YAML files you can version control
|
|
18
|
+
- Simple enough to install in any project's virtualenv
|
|
19
|
+
|
|
20
|
+
**What it isn't:**
|
|
21
|
+
- Not a replacement for Airflow/Kubeflow/Prefect (those are for production pipelines)
|
|
22
|
+
- Not an experiment tracker like W&B or MLflow (though it can be made to complement them)
|
|
23
|
+
|
|
24
|
+
## Installation
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
# Core runner only
|
|
28
|
+
pip install loom-pipeline
|
|
29
|
+
|
|
30
|
+
# With visual editor
|
|
31
|
+
pip install loom-pipeline[ui]
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
That's it. No configuration files to create, no external services to manage.
|
|
35
|
+
|
|
36
|
+
## Quick Start
|
|
37
|
+
|
|
38
|
+
Clone the repo and try an example:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
git clone https://github.com/ljubobratovicrelja/loom.git
|
|
42
|
+
cd loom
|
|
43
|
+
pip install -e .[ui,examples]
|
|
44
|
+
|
|
45
|
+
# Run a pipeline from the command line
|
|
46
|
+
loom examples/image-processing/pipeline.yml
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
Pipeline: 3 step(s) to run [parallel]
|
|
51
|
+
----------------------------------------
|
|
52
|
+
[RUNNING] grayscale
|
|
53
|
+
[grayscale] Converted to grayscale: .loom-url-cache/35bb4a6_Lenna.png -> data/grayscale.png
|
|
54
|
+
[SUCCESS] grayscale
|
|
55
|
+
[RUNNING] blur
|
|
56
|
+
[blur] Gaussian blur (radius=15): data/grayscale.png -> data/blurred.png
|
|
57
|
+
[SUCCESS] blur
|
|
58
|
+
[RUNNING] edge_detect
|
|
59
|
+
[edge_detect] Edge detection: data/grayscale.png -> data/edges.png
|
|
60
|
+
[SUCCESS] edge_detect
|
|
61
|
+
----------------------------------------
|
|
62
|
+
Completed: 3/3 steps succeeded
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Or open it in the visual editor:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
# Edit a single pipeline
|
|
69
|
+
loom-ui examples/image-processing/pipeline.yml
|
|
70
|
+
|
|
71
|
+
# Browse all example pipelines
|
|
72
|
+
loom-ui examples/
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The editor opens in your browser where you can see the pipeline graph, run steps, and view outputs.
|
|
76
|
+
|
|
77
|
+
## Building Your Own Pipeline
|
|
78
|
+
|
|
79
|
+
Add Loom to your project's environment:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pip install loom-pipeline[ui] # or just loom-pipeline for CLI only
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Now you can run pipelines from within your project. Here's how to set one up.
|
|
86
|
+
|
|
87
|
+
### 1. Point it at your scripts
|
|
88
|
+
|
|
89
|
+
Say you have some Python scripts that process data:
|
|
90
|
+
|
|
91
|
+
```
|
|
92
|
+
tasks/
|
|
93
|
+
extract_features.py # Takes video, outputs CSV
|
|
94
|
+
train_model.py # Takes CSV, outputs model
|
|
95
|
+
evaluate.py # Takes model + test data, outputs metrics
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### 2. Describe the pipeline in YAML
|
|
99
|
+
|
|
100
|
+
```yaml
|
|
101
|
+
# experiment.yml
|
|
102
|
+
variables:
|
|
103
|
+
video: data/raw/recording.mp4
|
|
104
|
+
features: data/processed/features.csv
|
|
105
|
+
model: models/classifier.pt
|
|
106
|
+
metrics: results/metrics.json
|
|
107
|
+
|
|
108
|
+
parameters:
|
|
109
|
+
learning_rate: 0.001
|
|
110
|
+
epochs: 100
|
|
111
|
+
|
|
112
|
+
pipeline:
|
|
113
|
+
- name: extract
|
|
114
|
+
task: tasks/extract_features.py
|
|
115
|
+
inputs:
|
|
116
|
+
video: $video
|
|
117
|
+
outputs:
|
|
118
|
+
-o: $features
|
|
119
|
+
|
|
120
|
+
- name: train
|
|
121
|
+
task: tasks/train_model.py
|
|
122
|
+
inputs:
|
|
123
|
+
data: $features
|
|
124
|
+
outputs:
|
|
125
|
+
-o: $model
|
|
126
|
+
args:
|
|
127
|
+
--lr: $learning_rate
|
|
128
|
+
--epochs: $epochs
|
|
129
|
+
|
|
130
|
+
- name: evaluate
|
|
131
|
+
task: tasks/evaluate.py
|
|
132
|
+
inputs:
|
|
133
|
+
model: $model
|
|
134
|
+
outputs:
|
|
135
|
+
-o: $metrics
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### 3. Run it
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
# Run the full pipeline
|
|
142
|
+
loom experiment.yml
|
|
143
|
+
|
|
144
|
+
# Run just one step
|
|
145
|
+
loom experiment.yml --step train
|
|
146
|
+
|
|
147
|
+
# Run from a step onward
|
|
148
|
+
loom experiment.yml --from train
|
|
149
|
+
|
|
150
|
+
# Try different parameters
|
|
151
|
+
loom experiment.yml --set learning_rate=0.01 --set epochs=200
|
|
152
|
+
|
|
153
|
+
# Override file paths
|
|
154
|
+
loom experiment.yml --var video=other_recording.mp4
|
|
155
|
+
|
|
156
|
+
# Run steps in parallel
|
|
157
|
+
loom experiment.yml --parallel --max-workers 4
|
|
158
|
+
|
|
159
|
+
# Preview without executing
|
|
160
|
+
loom experiment.yml --dry-run
|
|
161
|
+
|
|
162
|
+
# Clean all data (move to trash) and re-run from scratch
|
|
163
|
+
loom experiment.yml --clean
|
|
164
|
+
loom experiment.yml
|
|
165
|
+
|
|
166
|
+
# Preview what would be cleaned
|
|
167
|
+
loom experiment.yml --clean-list
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### 4. Or use the visual editor
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
loom-ui experiment.yml
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
This opens a browser-based editor where you can:
|
|
177
|
+
- See your pipeline as a visual graph
|
|
178
|
+
- Drag and drop to reorganize
|
|
179
|
+
- Run individual steps and see output in real-time
|
|
180
|
+
- Quickly see which outputs exist (green) vs missing (grey)
|
|
181
|
+
|
|
182
|
+
You can also point it at a directory to browse multiple pipelines:
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
loom-ui experiments/ # Browse all pipelines in a folder
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
Each pipeline should be in its own subdirectory with a `pipeline.yml` file inside. See [examples/](examples/) for the expected structure.
|
|
189
|
+
|
|
190
|
+
## How Scripts Work
|
|
191
|
+
|
|
192
|
+
Your scripts stay normal Python with argparse. Just add a YAML block in the docstring so Loom knows the interface:
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
"""Extract features from video.
|
|
196
|
+
|
|
197
|
+
---
|
|
198
|
+
inputs:
|
|
199
|
+
video:
|
|
200
|
+
type: video
|
|
201
|
+
description: Input video file
|
|
202
|
+
outputs:
|
|
203
|
+
-o:
|
|
204
|
+
type: csv
|
|
205
|
+
description: Output features
|
|
206
|
+
args:
|
|
207
|
+
--sample-rate:
|
|
208
|
+
type: int
|
|
209
|
+
default: 30
|
|
210
|
+
description: Frames to sample per second
|
|
211
|
+
---
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
import argparse
|
|
215
|
+
|
|
216
|
+
def main():
|
|
217
|
+
parser = argparse.ArgumentParser()
|
|
218
|
+
parser.add_argument("video")
|
|
219
|
+
parser.add_argument("-o", "--output", required=True)
|
|
220
|
+
parser.add_argument("--sample-rate", type=int, default=30)
|
|
221
|
+
args = parser.parse_args()
|
|
222
|
+
# ... your code ...
|
|
223
|
+
|
|
224
|
+
if __name__ == "__main__":
|
|
225
|
+
main()
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
The YAML frontmatter is optional but enables the editor to show input/output types and provide better validation.
|
|
229
|
+
|
|
230
|
+
## Use Cases
|
|
231
|
+
|
|
232
|
+
**Parameter exploration**: Create parallel branches in your pipeline to test different configurations side by side.
|
|
233
|
+
|
|
234
|
+
**Reproducible experiments**: The YAML file captures your entire experiment setup. Commit it to git alongside your code.
|
|
235
|
+
|
|
236
|
+
**Iterative development**: Run just the steps you're working on. Loom tracks dependencies so upstream steps run only when needed.
|
|
237
|
+
|
|
238
|
+
**Result organization**: Variables point to file paths, so your outputs are organized by experiment configuration.
|
|
239
|
+
|
|
240
|
+
## Philosophy
|
|
241
|
+
|
|
242
|
+
Loom is intentionally minimal:
|
|
243
|
+
|
|
244
|
+
- **No database** — Everything is files: your scripts, YAML configs, and outputs
|
|
245
|
+
- **No external services** — The visual editor runs a local server that stops when you close it
|
|
246
|
+
- **No lock-in** — Your scripts work with or without Loom
|
|
247
|
+
- **No magic** — Loom just builds shell commands and runs them
|
|
248
|
+
|
|
249
|
+
This makes it easy to adopt incrementally. Start with one experiment, see if it helps, expand from there.
|
|
250
|
+
|
|
251
|
+
## License
|
|
252
|
+
|
|
253
|
+
MIT License — see [LICENSE](LICENSE) for details.
|
|
254
|
+
|
|
255
|
+
---
|
|
256
|
+
|
|
257
|
+
*Built for researchers who want to see their experiments, not manage infrastructure.*
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "loom-pipeline"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
description = "Visual pipeline editor and runner for task automation"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "Relja Ljubobratovic", email = "ljubobratovic.relja@gmail.com"}
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"PyYAML>=6.0",
|
|
25
|
+
"requests>=2.28.0",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/ljubobratovicrelja/loom"
|
|
30
|
+
Repository = "https://github.com/ljubobratovicrelja/loom"
|
|
31
|
+
Documentation = "https://ljubobratovicrelja.github.io/loom/"
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
ui = [
|
|
35
|
+
"fastapi>=0.100.0",
|
|
36
|
+
"uvicorn>=0.23.0",
|
|
37
|
+
"websockets>=12.0",
|
|
38
|
+
"ruamel.yaml>=0.18.0",
|
|
39
|
+
"send2trash>=1.8.2",
|
|
40
|
+
"opencv-python-headless>=4.8.0",
|
|
41
|
+
]
|
|
42
|
+
examples = [
|
|
43
|
+
"numpy>=1.24.0",
|
|
44
|
+
"scipy>=1.10.0",
|
|
45
|
+
"matplotlib>=3.7.0",
|
|
46
|
+
"opencv-python-headless>=4.8.0",
|
|
47
|
+
]
|
|
48
|
+
dev = [
|
|
49
|
+
"loom-pipeline[ui,examples]",
|
|
50
|
+
"pytest>=7.0.0",
|
|
51
|
+
"pytest-cov>=4.0.0",
|
|
52
|
+
"pytest-asyncio>=0.21.0",
|
|
53
|
+
"httpx>=0.24.0",
|
|
54
|
+
"ruff>=0.1.0",
|
|
55
|
+
"mypy>=1.0.0",
|
|
56
|
+
"types-PyYAML>=6.0.0",
|
|
57
|
+
"types-requests>=2.28.0",
|
|
58
|
+
"mkdocs-material>=9.0.0",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
[project.scripts]
|
|
62
|
+
loom = "loom.runner:main"
|
|
63
|
+
loom-ui = "loom.ui:main"
|
|
64
|
+
|
|
65
|
+
[tool.setuptools.packages.find]
|
|
66
|
+
where = ["src"]
|
|
67
|
+
exclude = ["loom.ui.frontend.node_modules*"]
|
|
68
|
+
|
|
69
|
+
[tool.setuptools.package-data]
|
|
70
|
+
"loom.ui" = ["frontend/dist/**/*"]
|
|
71
|
+
|
|
72
|
+
[tool.ruff]
|
|
73
|
+
line-length = 100
|
|
74
|
+
target-version = "py311"
|
|
75
|
+
|
|
76
|
+
[tool.ruff.lint]
|
|
77
|
+
select = ["E", "F", "I", "N", "W", "UP"]
|
|
78
|
+
ignore = ["E501"]
|
|
79
|
+
|
|
80
|
+
[tool.ruff.lint.isort]
|
|
81
|
+
known-first-party = ["loom"]
|
|
82
|
+
|
|
83
|
+
[tool.mypy]
|
|
84
|
+
python_version = "3.11"
|
|
85
|
+
warn_return_any = true
|
|
86
|
+
warn_unused_ignores = true
|
|
87
|
+
disallow_untyped_defs = true
|
|
88
|
+
# Exclude examples as they contain duplicate module names (multiple compute_stats.py, etc.)
|
|
89
|
+
exclude = ["examples/"]
|
|
90
|
+
|
|
91
|
+
[tool.pytest.ini_options]
|
|
92
|
+
testpaths = ["tests"]
|
|
93
|
+
python_files = ["test_*.py"]
|
|
94
|
+
asyncio_mode = "auto"
|
|
File without changes
|