pi-line 1.0__tar.gz

pi_line-1.0/.github/workflows/publish.yml

@@ -0,0 +1,51 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  test:
+    name: Run tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # needed for setuptools-scm to read tags
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest
+
+  publish:
+    name: Build and publish
+    needs: test
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for PyPI Trusted Publisher
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # needed for setuptools-scm to read tags
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

pi_line-1.0/.gitignore

@@ -0,0 +1,39 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+*.egg-info/
+dist/
+build/
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Testing / Linting caches
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+htmlcov/
+
+# piline runtime
+.piline/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Environment
+.env
+
+generate_docs.py

pi_line-1.0/PKG-INFO

@@ -0,0 +1,125 @@
+Metadata-Version: 2.4
+Name: pi-line
+Version: 1.0
+Summary: Zero-dependency parallel script runner.
+Author-email: Maximilian Todea <demon.and.max@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/importt-ant/pi-line
+Project-URL: Repository, https://github.com/importt-ant/pi-line
+Project-URL: Issues, https://github.com/importt-ant/pi-line/issues
+Keywords: python,automation,pipeline,devtools,parallel,task-runner,process-pool,zero-dependencies
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4; extra == "dev"
+Requires-Dist: pytest-cov>=4.1; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+
+# 🥧〽️❗: pi-line
+
+A zero-dependency parallel script runner for Python.
+
+Define units of work (**Pi**), execute them in parallel with a **Runner**, or feed them continuously through a **Line**.
+
+## Install
+
+```bash
+pip install pi-line
+```
+
+## Quick start
+
+### Run scripts in parallel
+
+```python
+from piline import Pi, Runner
+
+pis = [
+    Pi(name="train", script="train.py", args=["--epochs", "10"]),
+    Pi(name="eval", script="eval.py"),
+]
+
+runner = Runner(max_workers=4)
+results = runner.run(pis)
+
+for r in results:
+    print(f"{r.pi_name}: {'PASS' if r.succeeded else 'FAIL'}")
+```
+
+### Continuous feed with a Line
+
+```python
+from piline import Pi, Runner, Line
+
+runner = Runner(max_workers=4)
+
+with Line(runner, max_results=5000) as line:
+    line.put(Pi(name="job1", script="job.py"))
+    line.put(Pi(name="job2", script="job.py"))
+
+    # look up a result by id
+    result = line.get(pi_id)
+
+    # flush results to external storage
+    batch = line.drain_results()
+# line.stop() called automatically
+```
+
+### Argument templates
+
+Use `{artefact_dir}` and `{task_dir}` placeholders in args — they get resolved to the real paths before execution:
+
+```python
+Pi(
+    name="train",
+    script="train.py",
+    args=["--output", "{artefact_dir}/model.pt"],
+)
+```
+
+## Concepts
+
+- **Pi** — A unit of work. Wraps a script with args, env vars, and an optional timeout. Gets a unique ID on creation. `.py` files run with Python; anything else runs directly.
+- **Result** — Outcome of running a Pi. Exit code, timing, paths to stdout/stderr logs and artefacts.
+- **Runner** — Executes a batch of Pi's in parallel using `ProcessPoolExecutor`.
+- **Line** — Thread-safe queue that feeds Pi's to a Runner in batches. Results stored in a capped dict keyed by `pi_id`. Supports context manager, per-Pi and per-batch callbacks, and `drain_results()` for periodic flushing.
+
+## Output layout
+
+```
+.piline/runs/<pi_name>/<pi_id>/
+    stdout.log
+    stderr.log
+    artefact/
+```
+
+## Environment variables
+
+Scripts receive these env vars automatically:
+
+- `PILINE_PI_ID` — The Pi's unique ID
+- `PILINE_PI_NAME` — The Pi's name
+- `PILINE_TASK_DIR` — Path to the task directory
+- `PILINE_ARTEFACT_DIR` — Path to the artefact subdirectory
+
+## Callbacks
+
+```python
+Line(
+    runner,
+    on_pi_complete=lambda r: print(f"Done: {r.pi_name} ({'PASS' if r.succeeded else 'FAIL'})"),
+    on_batch_complete=lambda results: print(f"Batch of {len(results)} finished"),
+)
+```
+
+## API reference
+
+See [docs/index.md](docs/index.md).

pi_line-1.0/README.md

@@ -0,0 +1,100 @@
+# 🥧〽️❗: pi-line
+
+A zero-dependency parallel script runner for Python.
+
+Define units of work (**Pi**), execute them in parallel with a **Runner**, or feed them continuously through a **Line**.
+
+## Install
+
+```bash
+pip install pi-line
+```
+
+## Quick start
+
+### Run scripts in parallel
+
+```python
+from piline import Pi, Runner
+
+pis = [
+    Pi(name="train", script="train.py", args=["--epochs", "10"]),
+    Pi(name="eval", script="eval.py"),
+]
+
+runner = Runner(max_workers=4)
+results = runner.run(pis)
+
+for r in results:
+    print(f"{r.pi_name}: {'PASS' if r.succeeded else 'FAIL'}")
+```
+
+### Continuous feed with a Line
+
+```python
+from piline import Pi, Runner, Line
+
+runner = Runner(max_workers=4)
+
+with Line(runner, max_results=5000) as line:
+    line.put(Pi(name="job1", script="job.py"))
+    line.put(Pi(name="job2", script="job.py"))
+
+    # look up a result by id
+    result = line.get(pi_id)
+
+    # flush results to external storage
+    batch = line.drain_results()
+# line.stop() called automatically
+```
+
+### Argument templates
+
+Use `{artefact_dir}` and `{task_dir}` placeholders in args — they get resolved to the real paths before execution:
+
+```python
+Pi(
+    name="train",
+    script="train.py",
+    args=["--output", "{artefact_dir}/model.pt"],
+)
+```
+
+## Concepts
+
+- **Pi** — A unit of work. Wraps a script with args, env vars, and an optional timeout. Gets a unique ID on creation. `.py` files run with Python; anything else runs directly.
+- **Result** — Outcome of running a Pi. Exit code, timing, paths to stdout/stderr logs and artefacts.
+- **Runner** — Executes a batch of Pi's in parallel using `ProcessPoolExecutor`.
+- **Line** — Thread-safe queue that feeds Pi's to a Runner in batches. Results stored in a capped dict keyed by `pi_id`. Supports context manager, per-Pi and per-batch callbacks, and `drain_results()` for periodic flushing.
+
+## Output layout
+
+```
+.piline/runs/<pi_name>/<pi_id>/
+    stdout.log
+    stderr.log
+    artefact/
+```
+
+## Environment variables
+
+Scripts receive these env vars automatically:
+
+- `PILINE_PI_ID` — The Pi's unique ID
+- `PILINE_PI_NAME` — The Pi's name
+- `PILINE_TASK_DIR` — Path to the task directory
+- `PILINE_ARTEFACT_DIR` — Path to the artefact subdirectory
+
+## Callbacks
+
+```python
+Line(
+    runner,
+    on_pi_complete=lambda r: print(f"Done: {r.pi_name} ({'PASS' if r.succeeded else 'FAIL'})"),
+    on_batch_complete=lambda results: print(f"Batch of {len(results)} finished"),
+)
+```
+
+## API reference
+
+See [docs/index.md](docs/index.md).

pi_line-1.0/docs/index.md

@@ -0,0 +1,8 @@
+# API Reference
+
+| Module | Public API | Description |
+|---|---|---|
+| [pi.md](pi.md) | `Pi`, `Result` | Pi and Result data models. |
+| [runner.md](runner.md) | `Runner` | Parallel Pi executor using ProcessPoolExecutor. |
+| [line.md](line.md) | `Line` | Thread-safe feed queue that dispatches Pi's to a Runner. |
+| [worker.md](worker.md) | `execute_pi` | Single-Pi subprocess execution. |

pi_line-1.0/docs/line.md

@@ -0,0 +1,152 @@
+> Thread-safe feed queue that dispatches Pi's to a Runner.
+
+---
+
+## `Line`
+
+Thread-safe queue that feeds Pi's to a Runner from a background thread.
+
+Put Pi's onto the Line and they get batched and dispatched to the
+Runner in groups of up to `runner.max_workers`.  Results are stored
+in an ordered dict keyed by `pi_id`.  When the dict exceeds
+*max_results* entries, the oldest results are dropped (FIFO).
+
+Use as a context manager to handle start/stop:
+
+**Example**
+
+```python
+runner = Runner(max_workers=4)
+with Line(runner, max_results=5000) as line:
+    line.put(Pi(name="job1", script="job.py"))
+    line.put(Pi(name="job2", script="job.py"))
+
+    result = line.get(some_pi_id)
+    batch = line.drain_results()
+# consumer stops when the with-block exits
+```
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `runner` | The Runner that executes batches of Pi's. |
+| `maxsize` | Maximum queue depth.  `0` (default) means unlimited.  If the queue is full, `put` blocks until space is available. |
+| `max_results` | Maximum number of results to keep in memory.  Oldest results are evicted when this limit is exceeded.  Defaults to 2000. |
+| `on_batch_complete` | Called after each batch finishes, with the list of Results from that batch. |
+| `on_pi_complete` | Called once per finished Pi, with its Result. |
+
+### `size`
+
+Number of Pi's waiting in the queue (not yet dispatched).
+
+### `empty`
+
+`True` when nothing is queued.
+
+### `total_enqueued`
+
+Cumulative count of Pi's added since creation.  Keeps incrementing even after Pi's leave the queue, so it works as a progress counter.
+
+### `result_count`
+
+Number of results currently stored.  May be less than `total_enqueued` if results were drained or evicted.
+
+### `running`
+
+`True` while the background consumer thread is alive.
+
+---
+
+### `put(pi: Pi) → str`
+
+Add a Pi to the queue.
+
+If the consumer is running, the Pi will be picked up and
+executed in the next batch.
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `pi` | The Pi to enqueue. |
+
+**Returns**
+
+`str` — The Pi's unique ID, for later lookup via `get`.
+
+---
+
+### `put_many(pis: list[Pi]) → list[str]`
+
+Add several Pi's to the queue at once.
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `pis` | List of Pi's to enqueue. |
+
+**Returns**
+
+`list[str]` — IDs of the enqueued Pi's, in the same order as *pis*.
+
+---
+
+### `get(pi_id: str) → Result | None`
+
+Look up a result by its Pi's ID.
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `pi_id` | The ID returned by `put`. |
+
+**Returns**
+
+`Result | None` — The matching Result, or `None` if the Pi hasn't finished yet or has been evicted from the results dict.
+
+---
+
+### `drain_results() → dict[str, Result]`
+
+Remove and return all stored results.
+
+Useful for periodic flushing to a database or file without
+losing data.  After this call, `result_count` is 0.
+
+**Returns**
+
+`dict[str, Result]` — Mapping of `pi_id` to Result for every result that was stored.  Empty dict if nothing was stored.
+
+---
+
+### `start() → None`
+
+Start the background consumer thread.
+
+The consumer polls the queue, collects batches of up to
+`runner.max_workers` Pi's, and dispatches them to the Runner.
+Call `stop` (or use the context manager) to shut it down.
+
+**Raises**
+
+| Exception | When |
+|---|---|
+| `RuntimeError` | If the consumer is already running. |
+
+---
+
+### `stop(timeout: float | None = None) → None`
+
+Stop the background consumer and wait for it to finish.
+
+Blocks until the consumer thread exits.  Any batch currently in
+progress will complete before the thread ends.
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `timeout` | Maximum seconds to wait for the thread to join.  `None` (default) waits indefinitely. |

pi_line-1.0/docs/pi.md

@@ -0,0 +1,87 @@
+> Pi and Result data models.
+
+---
+
+## `Pi`
+
+A unit of work that wraps a script for parallel execution.
+
+Each Pi has a name, a script path, optional arguments and environment
+variables, and an optional timeout in seconds.  A unique 12-character
+hex ID is assigned on creation and cannot be overridden.
+
+`.py` files run under the current Python interpreter; everything
+else is executed directly (shell scripts, compiled binaries, etc.).
+
+**Example**
+
+```python
+pi = Pi(name="train", script="train.py", args=["--lr", "0.01"])
+print(pi.id)  # e.g. "a1b2c3d4e5f6"
+```
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `name` | Human-readable label, also used as the directory name in output. |
+| `script` | Path to the script to run. |
+| `args` | Command-line arguments passed to the script.  Supports `{task_dir}` and `{artefact_dir}` placeholders that are resolved to real paths before execution. |
+| `env` | Extra environment variables merged into the subprocess env. |
+| `timeout` | Maximum runtime in seconds.  The subprocess is killed if it exceeds this limit.  `None` means no limit. |
+
+---
+
+### `resolve_dirs(base_dir: Path | str) → tuple[Path, Path]`
+
+Build the task and artefact directory paths for this Pi.
+
+Directories are not created here; the worker handles that at
+execution time.
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `base_dir` | Root directory for all output.  Converted to `Path` if given as a string. |
+
+**Returns**
+
+`tuple[Path, Path]` — `(task_dir, artefact_dir)` where `task_dir` is `<base_dir>/<name>/<id>` and `artefact_dir` is `<task_dir>/artefact`.
+
+---
+
+## `Result`
+
+Outcome of running a single Pi.
+
+Created by the worker after a subprocess finishes (or fails to start).
+Contains the exit code, timing, log paths, and any error information.
+
+**Example**
+
+```python
+if result.succeeded:
+    print(f"{result.pi_name} passed in {result.duration_s}s")
+else:
+    print(result.error_message)
+```
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `pi_id` | Unique ID of the Pi that produced this result. |
+| `pi_name` | Name of the Pi that produced this result. |
+| `exit_code` | Subprocess return code.  `None` when the process could not start or the worker crashed. |
+| `started_at` | UTC timestamp when execution began. |
+| `finished_at` | UTC timestamp when execution ended. |
+| `duration_s` | Wall-clock duration in seconds, rounded to milliseconds. |
+| `task_dir` | Absolute path to the task output directory. |
+| `artefact_dir` | Absolute path to the artefact subdirectory. |
+| `error_message` | Human-readable error for timeouts, launch failures, or worker crashes.  `None` on success. |
+| `timed_out` | `True` if the subprocess was killed for exceeding its timeout. |
+
+### `succeeded`
+
+`True` when exit_code is 0 and the Pi did not time out.

pi_line-1.0/docs/runner.md

@@ -0,0 +1,52 @@
+> Parallel Pi executor using ProcessPoolExecutor.
+
+---
+
+## `Runner`
+
+Runs a batch of Pi's in parallel using a process pool.
+
+Each Pi gets its own output directory under *base_dir*:
+`<base_dir>/<pi_name>/<pi_id>/`, containing `stdout.log`,
+`stderr.log`, and an `artefact/` subdirectory.
+
+`results` is reset on every call to `run`.
+
+**Example**
+
+```python
+runner = Runner(base_dir="/tmp/runs", max_workers=4)
+results = runner.run([
+    Pi(name="train", script="train.py"),
+    Pi(name="eval", script="eval.py"),
+])
+for r in results:
+    print(r.pi_name, r.succeeded)
+```
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `base_dir` | Root directory for task output.  Defaults to `.piline/runs`. |
+| `max_workers` | Maximum number of parallel processes.  Defaults to `os.cpu_count()`, falling back to 4. |
+
+---
+
+### `run(pis: list[Pi]) → list[Result]`
+
+Execute *pis* in parallel and return one Result per Pi.
+
+Clears `self.results` before starting. The number of worker
+processes is capped at `min(max_workers, len(pis))` so short
+lists don't spawn idle processes.
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `pis` | Pi's to execute.  An empty list returns `[]` immediately. |
+
+**Returns**
+
+`list[Result]` — One Result per Pi, in completion order (not input order).

pi_line-1.0/docs/worker.md

@@ -0,0 +1,32 @@
+> Single-Pi subprocess execution.
+
+---
+
+## `execute_pi(pi: Pi, task_dir: Path, artefact_dir: str) → Result`
+
+Run a single Pi as a subprocess and return the outcome.
+
+Creates *task_dir* and *artefact_dir* if they don't exist, writes
+stdout and stderr to log files inside *task_dir*, and populates a
+`Result` with exit code, timing, and error details.
+
+The subprocess receives four extra environment variables:
+`PILINE_PI_ID`, `PILINE_PI_NAME`, `PILINE_TASK_DIR`, and
+`PILINE_ARTEFACT_DIR`, plus any vars from `pi.env`.
+`{task_dir}` and `{artefact_dir}` placeholders in `pi.args`
+are resolved to real paths before the command is built.
+
+Meant to run inside a `ProcessPoolExecutor` via `Runner`,
+but works fine standalone.
+
+**Parameters**
+
+| Name | Description |
+|---|---|
+| `pi` | The Pi to execute. |
+| `task_dir` | Directory for stdout/stderr logs.  Created if missing. |
+| `artefact_dir` | Directory the script can write output files to.  Passed to the subprocess via `PILINE_ARTEFACT_DIR`.  Created if missing. |
+
+**Returns**
+
+`Result` — Contains exit code, timing, log paths, and any error or timeout information.

pi_line-1.0/pyproject.toml

@@ -0,0 +1,55 @@
+[build-system]
+requires = ["setuptools>=68.0", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pi-line"
+dynamic = ["version"]
+description = "Zero-dependency parallel script runner."
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.10"
+authors = [
+    {name = "Maximilian Todea", email = "demon.and.max@gmail.com"},
+]
+keywords = ["python", "automation", "pipeline", "devtools", "parallel", "task-runner", "process-pool", "zero-dependencies"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+dependencies = []
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7.4",
+    "pytest-cov>=4.1",
+    "ruff>=0.4",
+]
+
+[project.urls]
+Homepage = "https://github.com/importt-ant/pi-line"
+Repository = "https://github.com/importt-ant/pi-line"
+Issues = "https://github.com/importt-ant/pi-line/issues"
+
+[tool.setuptools_scm]
+# version is read from git tags (e.g. v1.0 → 1.0)
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["piline*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "W", "UP"]