ms-flow 0.1.0__tar.gz

ms_flow-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,58 @@
+name: Documentation
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "v*"
+  workflow_dispatch:
+
+concurrency:
+  group: documentation
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+  pages: write
+  id-token: write
+
+jobs:
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - uses: actions/configure-pages@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.x
+      - run: pip install zensical "mkdocstrings[python]" "mike @ git+https://github.com/squidfunk/mike.git"
+      - name: Configure Git author
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+      - name: Deploy development documentation
+        if: github.ref == 'refs/heads/main'
+        run: |
+          mike deploy --push dev
+          if ! mike list latest >/dev/null 2>&1; then
+            mike set-default --push dev
+          fi
+      - name: Deploy release documentation
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          mike deploy --push --update-aliases --alias-type redirect "$GITHUB_REF_NAME" latest
+          mike set-default --push latest
+      - name: Export versioned site
+        run: |
+          mkdir -p site
+          git archive gh-pages | tar -x -C site
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: site
+      - uses: actions/deploy-pages@v5
+        id: deployment

ms_flow-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,31 @@
+# Publish to PyPI on a GitHub release.
+#
+# No secrets: the job authenticates to PyPI over OIDC (Trusted Publishing).
+# It has to be enabled on the PyPI side *before* the first upload -- for a
+# project that does not exist yet, that is a "pending publisher" under
+# Your projects > Publishing, and it must name this repo, `publish.yml`
+# and the `pypi` environment below.
+name: publish
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      # A permissions block sets every scope it does not name to none, so
+      # checkout needs contents spelled out or it cannot read a private repo.
+      contents: read
+      id-token: write  # mint the OIDC token PyPI checks; no secret involved
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0  # setuptools-scm reads the version off the tag
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - run: pipx run build
+      - uses: pypa/gh-action-pypi-publish@release/v1

ms_flow-0.1.0/.gitignore

@@ -0,0 +1,37 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.eggs/
+
+# Test / coverage
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Benchmark output (disposable, large)
+.bench_runs/
+
+# IDE / editor
+.idea/
+.vscode/
+
+# Envs
+.venv/
+venv/
+
+# Artefactos de corridas (ejemplos, proyectos de prueba)
+demo_project/
+chunk_payloads/
+*.db
+*.db-shm
+*.db-wal
+*.log
+
+# Agentes / herramientas locales
+.claude/
+.aiassistant/
+
+.idea/

ms_flow-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco, Ernesto Moreno, Pedro A. Valiente
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ms_flow-0.1.0/PKG-INFO

@@ -0,0 +1,77 @@
+Metadata-Version: 2.4
+Name: ms_flow
+Version: 0.1.0
+Summary: In-process, desktop-first, domain-agnostic job/task/executor orchestrator for massive data flows.
+Author: Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco, Ernesto Moreno, Pedro A. Valiente
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/MolSuite/ms_flow
+Project-URL: Documentation, https://github.com/MolSuite/ms_flow/tree/main/docs
+Project-URL: Issues, https://github.com/MolSuite/ms_flow/issues
+Keywords: orchestration,executor,jobs,pipeline,scientific-computing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: blinker>=1.6
+Requires-Dist: loky>=3.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: sqlmodel>=0.0.14
+Requires-Dist: toml>=0.10
+Requires-Dist: tqdm>=4.66
+Provides-Extra: ray
+Requires-Dist: ray>=2.6; extra == "ray"
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Dynamic: license-file
+
+# ms_flow
+
+In-process, desktop-first, domain-agnostic orchestrator for massive data flows:
+jobs, executors and persistence in the same process as your app — no broker, no
+daemon, no server. Works from a Qt app, a notebook or a plain script.
+
+```python
+from ms_flow.api import MolSuite, table_sink
+from workers import double_value  # el handler vive en un modulo importable
+
+ms = MolSuite(app_id="demo")
+ms.create_or_open_project(name="demo", folder="./demo_project", activate=True)
+
+job_id = ms.run(
+    name="double_values",
+    input=[{"value": v} for v in range(1000)],
+    process=double_value,
+    output=table_sink("results", columns=("value", "double_value")),
+    executor="thread",
+)
+print(ms.wait_for_job(job_id).status)
+```
+
+## Install
+
+```bash
+pip install ms_flow            # core: control plane + SQLite + loky
+pip install "ms_flow[ray]"     # Ray executor
+```
+
+## Where to look
+
+- [5-minute quickstart](docs/guides/quickstart_5min.md) — first working job.
+- [Runnable examples](examples/README.md) — scripts y notebook.
+- [Public API](docs/guides/public_api.md) — API.
+- [Docs index](docs/README.md) — guides, contracts, operation
+- [Benchmarks](benchmarks/README.md) — performance.
+
+## Status
+
+Beta. The public API (`ms_flow.api`) is the stable surface; `ms_flow.core.*` may change without notice.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

ms_flow-0.1.0/README.md

@@ -0,0 +1,45 @@
+# ms_flow
+
+In-process, desktop-first, domain-agnostic orchestrator for massive data flows:
+jobs, executors and persistence in the same process as your app — no broker, no
+daemon, no server. Works from a Qt app, a notebook or a plain script.
+
+```python
+from ms_flow.api import MolSuite, table_sink
+from workers import double_value  # el handler vive en un modulo importable
+
+ms = MolSuite(app_id="demo")
+ms.create_or_open_project(name="demo", folder="./demo_project", activate=True)
+
+job_id = ms.run(
+    name="double_values",
+    input=[{"value": v} for v in range(1000)],
+    process=double_value,
+    output=table_sink("results", columns=("value", "double_value")),
+    executor="thread",
+)
+print(ms.wait_for_job(job_id).status)
+```
+
+## Install
+
+```bash
+pip install ms_flow            # core: control plane + SQLite + loky
+pip install "ms_flow[ray]"     # Ray executor
+```
+
+## Where to look
+
+- [5-minute quickstart](docs/guides/quickstart_5min.md) — first working job.
+- [Runnable examples](examples/README.md) — scripts y notebook.
+- [Public API](docs/guides/public_api.md) — API.
+- [Docs index](docs/README.md) — guides, contracts, operation
+- [Benchmarks](benchmarks/README.md) — performance.
+
+## Status
+
+Beta. The public API (`ms_flow.api`) is the stable surface; `ms_flow.core.*` may change without notice.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

ms_flow-0.1.0/docs/api_guide.md

@@ -0,0 +1,191 @@
+# API guide
+
+Use `MolSuite` as the public facade. A `MolSuite` instance owns the active
+project and its executors, so close it with `shutdown()` when the work is done.
+
+```python
+from ms_flow.api import MolSuite
+```
+
+## Projects
+
+Projects are isolated per `app_id`. Opening a project activates it; jobs run
+against the active project.
+
+```python
+ms = MolSuite(app_id="demo")
+try:
+    context = ms.create_or_open_project(name="demo", folder="./demo-project")
+    print(context.id)
+
+    ms.close_project()          # the executors stay alive
+    ms.open_project(context.id)
+finally:
+    ms.shutdown()               # stops the whole runtime
+```
+
+`close_project()` and `open_project()` fail explicitly if the active project
+cannot drain its running jobs.
+
+## Run a job
+
+`run()` is the shortest path: `input → process → output`. It returns a job
+identifier immediately; `wait_for_job()` blocks until the job ends and returns
+a `JobSnapshot`.
+
+```python
+from workers import double_value
+
+job_id = ms.run(
+    name="double-values",
+    input=[{"value": 1}, {"value": 2}, {"value": 3}],
+    process=double_value,
+    executor="thread",
+    store_results=True,
+)
+final = ms.wait_for_job(job_id)
+print(final.status, final.chunks_done)
+print(ms.get_job_outputs(job_id))
+```
+
+!!! note "Workers must be importable"
+    The `process` callable has to live in an importable module. Functions
+    defined in the script you launch (`__main__`), lambdas, nested functions,
+    and notebook cells are rejected, because process and remote executors must
+    re-import the worker by name.
+
+Each worker receives one payload dictionary and returns a dictionary.
+
+## Persist results with sinks
+
+By default outputs are not kept. Pass a sink to write results incrementally,
+and keep `store_results=False` so the operational database does not grow with
+every result.
+
+```python
+from ms_flow.api import file_sink, table_sink
+
+job_id = ms.run(
+    name="double-values",
+    input=[{"value": value} for value in range(1000)],
+    process=double_value,
+    output=table_sink("results", columns=("value", "double_value")),
+    flush_every=100,
+    executor="thread",
+)
+```
+
+| Sink | Destination |
+| --- | --- |
+| `table_sink(table, columns=...)` | A table in the project database; the table must exist |
+| `file_sink(path, fmt="json")` | A file under the project folder |
+| `graph_sink(nodes=..., relations=...)` | Several related tables written in one transaction |
+
+## Read project data
+
+`db_input_for()` describes a query instead of materialising rows. The chunk
+carries the specification and the rows are read inside the worker.
+
+```python
+from ms_flow.query import db_input_for
+from workers import summarize_molecules
+
+job_id = ms.run(
+    name="summarize-ready",
+    input=[
+        {
+            "molecules": db_input_for(
+                "molecules",
+                fields=("id", "name"),
+                filters={"status": "ready"},
+                order=("id",),
+            )
+        }
+    ],
+    process=summarize_molecules,
+    executor="thread",
+    store_results=True,
+)
+```
+
+## Reusable definitions
+
+Separate definition from execution with `workflow()`:
+
+```python
+from ms_flow.api import workflow
+
+spec = workflow(
+    name="double-values",
+    input=[{"value": 1}, {"value": 2}],
+    process=double_value,
+    executor="thread",
+    store_results=True,
+)
+job_id = ms.run(spec=spec)
+```
+
+When a job is shared between scripts, a UI, and an application runtime, declare
+it once with `streaming_job()` (one payload per chunk) or `batch_job()` (one
+batch per chunk) and submit it with parameters:
+
+```python
+from ms_flow.api import streaming_job
+from workers import double_value, value_chunks
+
+double_job = streaming_job(
+    name="double-values",
+    run_chunk=double_value,
+    chunker=value_chunks,
+    executor="thread",
+    store_results=True,
+)
+job_id = ms.submit_job(double_job, params={"count": 10})
+```
+
+## Follow progress
+
+`watch_job()` renders a tqdm progress bar in a terminal or notebook and returns
+the final snapshot. Pass `follow=False` for a non-blocking snapshot.
+
+```python
+from ms_flow import watch_job
+
+snapshot = watch_job(ms, job_id)
+```
+
+Cancel a running job with `ms.cancel_job(job_id)`.
+
+## Application runtimes
+
+Subclass `BaseRuntime` to build a domain facade on top of `MolSuite`. The
+runtime fixes the `app_id` once and exposes the engine as `self.molsuite`.
+
+```python
+from ms_flow.runtime import BaseRuntime
+
+
+class DemoRuntime(BaseRuntime):
+    def __init__(self):
+        super().__init__("demoapp")
+
+    def double(self, values):
+        return self.run(
+            name="double-values",
+            input=[{"value": value} for value in values],
+            process=double_value,
+            executor="thread",
+            store_results=True,
+        )
+```
+
+## Choose the API level
+
+| Need | Recommended API |
+| --- | --- |
+| Run `input → process → output` | `MolSuite.run()` |
+| Reuse a common definition | `workflow()`, `streaming_job()`, or `batch_job()` |
+| Describe project queries | `ms_flow.query` |
+| Control tasks, staging, and finalize | `ms_flow.tasking` |
+| Build a domain facade | `BaseRuntime` / `AppRuntime` |
+| Extend the engine | `ms_flow.advanced` and, exceptionally, `ms_flow.core` |

ms_flow-0.1.0/docs/api_reference.md

@@ -0,0 +1,46 @@
+# API reference
+
+The reference below is generated directly from the current Python sources.
+Names beginning with an underscore are implementation details.
+
+## MolSuite
+
+::: ms_flow.main.MolSuite
+    options:
+      inherited_members: true
+      # The mixin methods have no docstrings yet; hidden by default.
+      show_if_no_docstring: true
+
+## Application runtimes
+
+::: ms_flow.runtime.BaseRuntime
+
+::: ms_flow.runtime.AppRuntime
+
+## Workflow specs
+
+::: ms_flow.specs
+
+## Job templates
+
+::: ms_flow.job_templates
+
+## Sinks
+
+::: ms_flow.sinks
+
+## Queries
+
+::: ms_flow.query
+
+## Selections
+
+::: ms_flow.selection
+
+## Progress
+
+::: ms_flow.progress
+
+## Tasking
+
+::: ms_flow.tasking

ms_flow-0.1.0/docs/examples.md

@@ -0,0 +1,191 @@
+# Examples
+
+Every example imports its worker functions from `workers.py`, because workers
+must be importable. Save this file next to the scripts:
+
+```python
+# workers.py
+def double_value(payload: dict) -> dict:
+    value = int(payload["value"])
+    return {"value": value, "double_value": value * 2}
+
+
+def summarize_molecules(payload: dict) -> dict:
+    molecules = list(payload["molecules"])
+    return {
+        "molecule_count": len(molecules),
+        "molecule_names": [row["name"] for row in molecules],
+    }
+
+
+def value_chunks(params: dict):
+    for value in range(int(params["count"])):
+        yield {"value": value}
+```
+
+Each example creates its project in the current directory, so run them in a
+scratch folder.
+
+## Persist results to a table
+
+```python
+import sqlite3
+from pathlib import Path
+
+from ms_flow.api import MolSuite, table_sink
+from workers import double_value
+
+ms = MolSuite(app_id="demo")
+try:
+    ms.create_or_open_project(
+        name="table-demo",
+        folder=Path("./table-demo").resolve(),
+        activate=True,
+    )
+
+    # A table sink writes to the project database: the table has to exist.
+    with sqlite3.connect(str(ms.project_db.db_path)) as conn:
+        conn.execute("DROP TABLE IF EXISTS results")
+        conn.execute("CREATE TABLE results (value INTEGER, double_value INTEGER)")
+
+    job_id = ms.run(
+        name="double-values",
+        input=[{"value": value} for value in (1, 2, 3, 4)],
+        process=double_value,
+        output=table_sink("results", columns=("value", "double_value")),
+        flush_every=2,
+        executor="thread",
+    )
+    final = ms.wait_for_job(job_id, poll_s=0.05)
+    print(final.status, final.chunks_done)
+
+    with sqlite3.connect(str(ms.project_db.db_path)) as conn:
+        rows = conn.execute("SELECT value, double_value FROM results ORDER BY value")
+        print(rows.fetchall())
+finally:
+    ms.shutdown()
+```
+
+## Feed a job from project rows
+
+The chunk carries the query, not the rows: they are read inside the worker.
+
+```python
+import sqlite3
+from pathlib import Path
+
+from ms_flow.api import MolSuite
+from ms_flow.query import db_input_for
+from workers import summarize_molecules
+
+ms = MolSuite(app_id="demo")
+try:
+    ms.create_or_open_project(
+        name="query-demo",
+        folder=Path("./query-demo").resolve(),
+        activate=True,
+    )
+
+    with sqlite3.connect(str(ms.project_db.db_path)) as conn:
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS molecules "
+            "(id INTEGER PRIMARY KEY, name TEXT, status TEXT)"
+        )
+        conn.execute("DELETE FROM molecules")
+        conn.executemany(
+            "INSERT INTO molecules (id, name, status) VALUES (?, ?, ?)",
+            [(1, "MolA", "ready"), (2, "MolB", "ready"), (3, "MolC", "archived")],
+        )
+
+    job_id = ms.run(
+        name="summarize-ready",
+        input=[
+            {
+                "molecules": db_input_for(
+                    "molecules",
+                    fields=("id", "name"),
+                    filters={"status": "ready"},
+                    order=("id",),
+                )
+            }
+        ],
+        process=summarize_molecules,
+        executor="thread",
+        store_results=True,
+    )
+    final = ms.wait_for_job(job_id, poll_s=0.05)
+    print(final.status, ms.get_job_outputs(job_id))
+finally:
+    ms.shutdown()
+```
+
+## Reusable streaming job
+
+Declare the job once and submit it with different parameters.
+
+```python
+from pathlib import Path
+
+from ms_flow import watch_job
+from ms_flow.api import MolSuite, streaming_job
+from workers import double_value, value_chunks
+
+double_job = streaming_job(
+    name="double-values",
+    run_chunk=double_value,
+    chunker=value_chunks,
+    executor="thread",
+    store_results=True,
+)
+
+ms = MolSuite(app_id="demo")
+try:
+    ms.create_or_open_project(
+        name="streaming-demo",
+        folder=Path("./streaming-demo").resolve(),
+        activate=True,
+    )
+    job_id = ms.submit_job(double_job, params={"count": 5})
+    final = watch_job(ms, job_id)
+    print(final.status, len(ms.get_job_outputs(job_id)))
+finally:
+    ms.shutdown()
+```
+
+## Application runtime
+
+A domain facade over `MolSuite` for an application.
+
+```python
+from pathlib import Path
+
+from ms_flow.runtime import BaseRuntime
+from workers import double_value
+
+
+class DemoRuntime(BaseRuntime):
+    def __init__(self):
+        super().__init__("demoapp")
+
+    def double(self, values):
+        return self.run(
+            name="double-values",
+            input=[{"value": value} for value in values],
+            process=double_value,
+            executor="thread",
+            store_results=True,
+        )
+
+
+runtime = DemoRuntime()
+try:
+    runtime.create_or_open_project(
+        name="runtime-demo",
+        folder=Path("./runtime-demo").resolve(),
+    )
+    job_id = runtime.double([1, 2, 3])
+    final = runtime.molsuite.wait_for_job(job_id, poll_s=0.05)
+    print(final.status, runtime.molsuite.get_job_outputs(job_id))
+finally:
+    runtime.shutdown()
+```