ms-flow 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ms_flow-0.2.0 → ms_flow-0.3.0}/PKG-INFO +30 -17
- ms_flow-0.3.0/README.md +64 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/README.md +3 -1
- ms_flow-0.3.0/benchmarks/refactor_optimization/bench_volumes.py +987 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/docs/03-impacto.html +79 -1
- ms_flow-0.3.0/benchmarks/refactor_optimization/docs/08-plan-volumenes.md +495 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/report.py +246 -1
- ms_flow-0.3.0/benchmarks/refactor_optimization/results/volumes_2026-10-15.json +114 -0
- ms_flow-0.3.0/benchmarks/refactor_optimization/results/volumes_V1-9.json +67 -0
- ms_flow-0.3.0/benchmarks/refactor_optimization/results/volumes_V2-12.json +48 -0
- ms_flow-0.3.0/benchmarks/refactor_optimization/results/volumes_V3.json +52 -0
- ms_flow-0.3.0/benchmarks/refactor_optimization/results/volumes_V4-10.json +44 -0
- ms_flow-0.3.0/benchmarks/refactor_optimization/results/volumes_V5-8.json +69 -0
- ms_flow-0.3.0/benchmarks/refactor_optimization/results/volumes_baseline.json +173 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/workers.py +34 -0
- ms_flow-0.3.0/docs/api_reference.md +50 -0
- ms_flow-0.3.0/docs/guide.md +181 -0
- ms_flow-0.3.0/docs/index.md +74 -0
- ms_flow-0.3.0/docs/payloads.md +170 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/pyproject.toml +2 -1
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/_api_impl/project_runtime.py +1 -1
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/database/base.py +1 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/database/project.py +23 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/database/project_models.py +42 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/database/project_store_ops.py +14 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/project/catalog.py +0 -5
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/project/manager.py +2 -1
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/project/resources.py +0 -7
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/engine/flow.py +2 -0
- ms_flow-0.3.0/src/ms_flow/payloads.py +936 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow.egg-info/PKG-INFO +30 -17
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow.egg-info/SOURCES.txt +18 -2
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow.egg-info/requires.txt +1 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow.egg-info/scm_file_list.json +18 -2
- ms_flow-0.3.0/src/ms_flow.egg-info/scm_version.json +8 -0
- ms_flow-0.3.0/tests/test_bench_volumes.py +98 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_engine_flow.py +1 -1
- ms_flow-0.3.0/tests/test_payloads.py +387 -0
- ms_flow-0.3.0/tests/test_payloads_codecs.py +119 -0
- ms_flow-0.3.0/tests/test_payloads_jobs.py +327 -0
- ms_flow-0.3.0/tests/test_payloads_maintenance.py +391 -0
- ms_flow-0.3.0/tests/test_payloads_read.py +172 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_project_catalog.py +3 -2
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_project_manager.py +2 -6
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_workspace_runtime.py +1 -1
- {ms_flow-0.2.0 → ms_flow-0.3.0}/zensical.toml +2 -2
- ms_flow-0.2.0/README.md +0 -52
- ms_flow-0.2.0/docs/api_guide.md +0 -191
- ms_flow-0.2.0/docs/api_reference.md +0 -46
- ms_flow-0.2.0/docs/examples.md +0 -191
- ms_flow-0.2.0/docs/index.md +0 -83
- ms_flow-0.2.0/src/ms_flow.egg-info/scm_version.json +0 -8
- {ms_flow-0.2.0 → ms_flow-0.3.0}/.github/workflows/docs.yml +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/.github/workflows/publish.yml +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/.gitignore +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/LICENSE +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/bench.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/docs/01-auditoria.md +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/docs/02-diseno-motor.md +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/docs/04-auditoria-tasks-jobs.md +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/docs/05-diseno-tasks-jobs.md +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/docs/06-diseno-ejecutor-remoto.md +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/docs/07-diseno-ms-env.md +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/baseline.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/batching.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/engine_core.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/feeders.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/forced_batch.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/grouped_delivery.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/journal.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/lazy_import.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/memory.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/new_engine_only.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/pack_target.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/remote_executor.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/resume.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/resume_count.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/retries.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/table_sources.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/benchmarks/refactor_optimization/results/task_api.json +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/setup.cfg +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/_api_impl/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/_api_impl/advanced.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/_api_impl/job_api.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/advanced.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/api.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/artifacts.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/app_settings.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/apps.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/configuration.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/data/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/data/backends.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/data/bridge.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/data/contracts.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/data/planner.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/data/project_output.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/data/runtime.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/data/shard.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/database/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/database/master.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/database/master_models.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/database/project_records.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/database/sqlite_utils.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/events.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/project/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/project/context.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/project/repository.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/project/state.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/settings/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/settings/defaults.toml +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/settings/manager.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/core/settings/models.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/engine/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/engine/engine.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/engine/executors.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/engine/files.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/engine/journal.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/engine/tokens.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/logger.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/main.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/progress.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/query.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/ray_hosts.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/remote.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/runner.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/runtime.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/selection.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/sinks.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/specs/__init__.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/specs/input.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow/specs/output.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow.egg-info/dependency_links.txt +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/src/ms_flow.egg-info/top_level.txt +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/conftest.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_app_registry.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_artifact_registry.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_data_api.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_data_backend.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_database_manager.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_database_migrations.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_db_pages.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_engine_core.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_engine_journal.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_file_sink_append.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_files.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_graph_deferred_relation.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_hosts_executors.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_logging_manager.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_outlives.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_output_helpers.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_project_storage.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_project_store.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_prune_missing_projects.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_public_api.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_pydantic_configuration.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_remote.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_runner.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_selection.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_settings_manager.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_shard_container.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_sink_column_defaults.py +0 -0
- {ms_flow-0.2.0 → ms_flow-0.3.0}/tests/test_sql_compiler.py +0 -0
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ms_flow
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: In-process, desktop-first, domain-agnostic job/task/executor orchestrator for massive data flows.
|
|
5
5
|
Author: Mario S. Valdés-Tresanco, Mario E. Valdés-Tresanco, Ernesto Moreno, Pedro A. Valiente
|
|
6
6
|
License-Expression: MIT
|
|
7
7
|
Project-URL: Homepage, https://github.com/MolSuite/ms_flow
|
|
8
|
-
Project-URL: Documentation, https://github.
|
|
8
|
+
Project-URL: Documentation, https://molsuite.github.io/ms_flow/
|
|
9
9
|
Project-URL: Issues, https://github.com/MolSuite/ms_flow/issues
|
|
10
10
|
Keywords: orchestration,executor,jobs,pipeline,scientific-computing
|
|
11
11
|
Classifier: Development Status :: 4 - Beta
|
|
@@ -24,6 +24,7 @@ Requires-Dist: pydantic>=2.0
|
|
|
24
24
|
Requires-Dist: sqlalchemy>=2.0
|
|
25
25
|
Requires-Dist: sqlmodel>=0.0.14
|
|
26
26
|
Requires-Dist: toml>=0.10
|
|
27
|
+
Requires-Dist: zstandard>=0.22
|
|
27
28
|
Provides-Extra: dev
|
|
28
29
|
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
29
30
|
Provides-Extra: ray
|
|
@@ -36,10 +37,19 @@ In-process, desktop-first, domain-agnostic orchestrator for massive data flows:
|
|
|
36
37
|
jobs, executors and persistence in the same process as your app — no broker, no
|
|
37
38
|
daemon, no server. Works from a Qt app, a notebook or a plain script.
|
|
38
39
|
|
|
40
|
+
`workers.py` (tasks and jobs live in a module the workers can import):
|
|
41
|
+
|
|
39
42
|
```python
|
|
40
|
-
import
|
|
43
|
+
from sqlmodel import Field, SQLModel
|
|
44
|
+
|
|
45
|
+
from ms_flow import job, table_sink, task
|
|
41
46
|
|
|
42
|
-
|
|
47
|
+
|
|
48
|
+
class Result(SQLModel, table=True): # the project creates this table when it opens
|
|
49
|
+
__tablename__ = "results"
|
|
50
|
+
id: int | None = Field(default=None, primary_key=True)
|
|
51
|
+
value: int
|
|
52
|
+
double_value: int
|
|
43
53
|
|
|
44
54
|
|
|
45
55
|
@task()
|
|
@@ -50,15 +60,21 @@ def double_value(value: int) -> dict:
|
|
|
50
60
|
@job(task=double_value, output=table_sink("results", columns=("value", "double_value")))
|
|
51
61
|
def double_values(params, project):
|
|
52
62
|
yield from range(1000)
|
|
63
|
+
```
|
|
53
64
|
|
|
65
|
+
`run.py`:
|
|
54
66
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
db.execute("CREATE TABLE IF NOT EXISTS results (value INTEGER, double_value INTEGER)")
|
|
67
|
+
```python
|
|
68
|
+
from ms_flow import Workspace
|
|
69
|
+
from workers import Result, double_values
|
|
59
70
|
|
|
60
|
-
|
|
61
|
-
|
|
71
|
+
ms = Workspace(app_id="demo")
|
|
72
|
+
try:
|
|
73
|
+
ms.create_or_open_project(name="demo", folder="./demo_project", activate=True)
|
|
74
|
+
print(ms.engine.wait(ms.submit(double_values)).status) # completed
|
|
75
|
+
print(ms.project_db.count(Result)) # 1000
|
|
76
|
+
finally:
|
|
77
|
+
ms.shutdown()
|
|
62
78
|
```
|
|
63
79
|
|
|
64
80
|
## Install
|
|
@@ -67,17 +83,14 @@ ms.shutdown()
|
|
|
67
83
|
pip install ms_flow # engine + projects: SQLite and loky
|
|
68
84
|
```
|
|
69
85
|
|
|
70
|
-
##
|
|
86
|
+
## Documentation
|
|
71
87
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
- [Public API](docs/guides/public_api.md) — API.
|
|
75
|
-
- [Docs index](docs/README.md) — guides, contracts, operation
|
|
76
|
-
- [Benchmarks](benchmarks/README.md) — performance.
|
|
88
|
+
<https://molsuite.github.io/ms_flow/>: a guide to tasks, jobs, sinks, following and
|
|
89
|
+
continuing jobs and where they run, and the API reference.
|
|
77
90
|
|
|
78
91
|
## Status
|
|
79
92
|
|
|
80
|
-
Beta. The public API (`ms_flow.api`) is the stable surface; `ms_flow.core.*` may change without notice.
|
|
93
|
+
Beta. The public API (`ms_flow`, `ms_flow.api`, `ms_flow.engine`) is the stable surface; `ms_flow.core.*` may change without notice.
|
|
81
94
|
|
|
82
95
|
## License
|
|
83
96
|
|
ms_flow-0.3.0/README.md
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# ms_flow
|
|
2
|
+
|
|
3
|
+
In-process, desktop-first, domain-agnostic orchestrator for massive data flows:
|
|
4
|
+
jobs, executors and persistence in the same process as your app — no broker, no
|
|
5
|
+
daemon, no server. Works from a Qt app, a notebook or a plain script.
|
|
6
|
+
|
|
7
|
+
`workers.py` (tasks and jobs live in a module the workers can import):
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
from sqlmodel import Field, SQLModel
|
|
11
|
+
|
|
12
|
+
from ms_flow import job, table_sink, task
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Result(SQLModel, table=True): # the project creates this table when it opens
|
|
16
|
+
__tablename__ = "results"
|
|
17
|
+
id: int | None = Field(default=None, primary_key=True)
|
|
18
|
+
value: int
|
|
19
|
+
double_value: int
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@task()
|
|
23
|
+
def double_value(value: int) -> dict:
|
|
24
|
+
return {"value": value, "double_value": 2 * value}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@job(task=double_value, output=table_sink("results", columns=("value", "double_value")))
|
|
28
|
+
def double_values(params, project):
|
|
29
|
+
yield from range(1000)
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
`run.py`:
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from ms_flow import Workspace
|
|
36
|
+
from workers import Result, double_values
|
|
37
|
+
|
|
38
|
+
ms = Workspace(app_id="demo")
|
|
39
|
+
try:
|
|
40
|
+
ms.create_or_open_project(name="demo", folder="./demo_project", activate=True)
|
|
41
|
+
print(ms.engine.wait(ms.submit(double_values)).status) # completed
|
|
42
|
+
print(ms.project_db.count(Result)) # 1000
|
|
43
|
+
finally:
|
|
44
|
+
ms.shutdown()
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Install
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pip install ms_flow # engine + projects: SQLite and loky
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Documentation
|
|
54
|
+
|
|
55
|
+
<https://molsuite.github.io/ms_flow/>: a guide to tasks, jobs, sinks, following and
|
|
56
|
+
continuing jobs and where they run, and the API reference.
|
|
57
|
+
|
|
58
|
+
## Status
|
|
59
|
+
|
|
60
|
+
Beta. The public API (`ms_flow`, `ms_flow.api`, `ms_flow.engine`) is the stable surface; `ms_flow.core.*` may change without notice.
|
|
61
|
+
|
|
62
|
+
## License
|
|
63
|
+
|
|
64
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -10,8 +10,10 @@ Refactor del motor de ms_flow: el banco de pruebas, lo que midió cada paso y lo
|
|
|
10
10
|
| [docs/04-auditoria-tasks-jobs.md](docs/04-auditoria-tasks-jobs.md) | Auditoría del diseño de tasks y jobs contra el uso real en AMDockVS |
|
|
11
11
|
| [docs/05-diseno-tasks-jobs.md](docs/05-diseno-tasks-jobs.md) | Diseño de tasks y jobs sobre el motor nuevo, estado de cada pieza y restos del motor viejo |
|
|
12
12
|
| [docs/06-diseno-ejecutor-remoto.md](docs/06-diseno-ejecutor-remoto.md) | Propuesta del ejecutor remoto: Ray dentro del job del HPC, runner de `ms_flow` y decisiones abiertas |
|
|
13
|
+
| [docs/08-plan-volumenes.md](docs/08-plan-volumenes.md) | Plan de los volúmenes para payloads por entidad: invariantes, fallos y recuperación, hitos V0-V7 (`ms_flow`) y A0-A8 (AMDockVS) con sus tareas |
|
|
13
14
|
| `bench.py` | Mide el coste de orquestación y las comprobaciones de robustez; escribe `results/<paso>.json` |
|
|
14
|
-
| `
|
|
15
|
+
| `bench_volumes.py` | Banco de los volúmenes (plan 08): compara SQLite con blobs y volúmenes en disco (modo serie y paralelo, lecturas en caliente y en frío); escribe `results/volumes_<etiqueta>.json` |
|
|
16
|
+
| `report.py` | Lee `results/*.json` y escribe el informe; los `volumes_*.json` van a su sección «Volumes» |
|
|
15
17
|
| `workers.py` | Funciones de chunk del banco de pruebas |
|
|
16
18
|
|
|
17
19
|
## Un cambio, una rama, un PR
|