industream-flowmaker-sdk 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- industream_flowmaker_sdk-1.0.2/.gitignore +168 -0
- industream_flowmaker_sdk-1.0.2/PKG-INFO +20 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/__init__.py +39 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/flowbox.py +24 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/flowbox_core.py +59 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/flowbox_decorator.py +19 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/flowbox_init_params.py +32 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/flowbox_registration_info.py +79 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/flowmaker_event.py +9 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/flowmaker_worker.py +147 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/flowmaker_worker_builder.py +23 -0
- industream_flowmaker_sdk-1.0.2/industream/flowmaker/sdk/flowmaker_worker_options.py +38 -0
- industream_flowmaker_sdk-1.0.2/pyproject.toml +39 -0
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py,cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
#Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
#uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
#poetry.lock
|
|
109
|
+
|
|
110
|
+
# pdm
|
|
111
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
112
|
+
#pdm.lock
|
|
113
|
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
|
114
|
+
# in version control.
|
|
115
|
+
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
|
|
116
|
+
.pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
121
|
+
__pypackages__/
|
|
122
|
+
|
|
123
|
+
# Celery stuff
|
|
124
|
+
celerybeat-schedule
|
|
125
|
+
celerybeat.pid
|
|
126
|
+
|
|
127
|
+
# SageMath parsed files
|
|
128
|
+
*.sage.py
|
|
129
|
+
|
|
130
|
+
# Environments
|
|
131
|
+
.env
|
|
132
|
+
.venv
|
|
133
|
+
env/
|
|
134
|
+
venv/
|
|
135
|
+
ENV/
|
|
136
|
+
env.bak/
|
|
137
|
+
venv.bak/
|
|
138
|
+
|
|
139
|
+
# Spyder project settings
|
|
140
|
+
.spyderproject
|
|
141
|
+
.spyproject
|
|
142
|
+
|
|
143
|
+
# Rope project settings
|
|
144
|
+
.ropeproject
|
|
145
|
+
|
|
146
|
+
# mkdocs documentation
|
|
147
|
+
/site
|
|
148
|
+
|
|
149
|
+
# mypy
|
|
150
|
+
.mypy_cache/
|
|
151
|
+
.dmypy.json
|
|
152
|
+
dmypy.json
|
|
153
|
+
|
|
154
|
+
# Pyre type checker
|
|
155
|
+
.pyre/
|
|
156
|
+
|
|
157
|
+
# pytype static type analyzer
|
|
158
|
+
.pytype/
|
|
159
|
+
|
|
160
|
+
# Cython debug symbols
|
|
161
|
+
cython_debug/
|
|
162
|
+
|
|
163
|
+
# PyCharm
|
|
164
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
165
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
166
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
167
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
168
|
+
#.idea/
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: industream-flowmaker-sdk
|
|
3
|
+
Version: 1.0.2
|
|
4
|
+
Summary: Industream FlowMaker SDK for Python
|
|
5
|
+
Project-URL: Homepage, https://github.com/industream/industream-flowmaker
|
|
6
|
+
Project-URL: Repository, https://github.com/industream/industream-flowmaker
|
|
7
|
+
Project-URL: Issues, https://github.com/industream/industream-flowmaker/issues
|
|
8
|
+
Author-email: Sylvain Bruyère <sylvain.bruyere@industream.com>
|
|
9
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Python: >=3.12
|
|
16
|
+
Requires-Dist: httpx>=0.28.1
|
|
17
|
+
Requires-Dist: msgpack-types>=0.5.0
|
|
18
|
+
Requires-Dist: msgpack>=1.1.0
|
|
19
|
+
Requires-Dist: pyzmq>=26.2.0
|
|
20
|
+
Requires-Dist: reactivex>=4.0.4
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from .flowbox_init_params import *
|
|
2
|
+
from .flowbox import *
|
|
3
|
+
from .flowbox_core import *
|
|
4
|
+
from .flowbox_registration_info import *
|
|
5
|
+
from .flowbox_decorator import *
|
|
6
|
+
from .flowmaker_event import *
|
|
7
|
+
from .flowmaker_worker_options import *
|
|
8
|
+
from .flowmaker_worker import *
|
|
9
|
+
from .flowmaker_worker_builder import *
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"FlowBoxRaw",
|
|
13
|
+
"FlowBoxSource",
|
|
14
|
+
"FlowBoxSink",
|
|
15
|
+
"FlowBoxDestroy",
|
|
16
|
+
"FlowBoxCore",
|
|
17
|
+
|
|
18
|
+
"flowbox",
|
|
19
|
+
"FlowBoxRegistrationInfo",
|
|
20
|
+
"FlowBoxType",
|
|
21
|
+
"FlowBoxIOInterfaces",
|
|
22
|
+
"FlowBoxIO",
|
|
23
|
+
"SerializationFormat",
|
|
24
|
+
"FlowBoxUIConfig",
|
|
25
|
+
|
|
26
|
+
"FlowBoxInitParams",
|
|
27
|
+
"FlowRuntimeContext",
|
|
28
|
+
|
|
29
|
+
"FlowMakerWorkerBuilder",
|
|
30
|
+
"FlowMakerWorkerOptions",
|
|
31
|
+
"FlowMakerWorker"
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
import asyncio
|
|
36
|
+
import sys
|
|
37
|
+
|
|
38
|
+
if sys.platform == "win32":
|
|
39
|
+
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from collections.abc import Callable
|
|
3
|
+
from . import FlowBoxInitParams
|
|
4
|
+
|
|
5
|
+
class FlowBoxRaw(ABC):
|
|
6
|
+
@abstractmethod
|
|
7
|
+
def __init__(self, init_params: FlowBoxInitParams) -> None: ...
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class FlowBoxSource(ABC):
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def on_output_ready(self, output_name: str, header: bytes,
|
|
13
|
+
fn_ready_for_next_item: Callable[[bytes, bytes], None]) -> None: ...
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class FlowBoxSink(ABC):
|
|
17
|
+
@abstractmethod
|
|
18
|
+
def on_input_received(self, input_name: str, header: bytes, data: bytes,
|
|
19
|
+
fn_ready_for_next_item: Callable[[], None]) -> None: ...
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class FlowBoxDestroy(ABC):
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def on_destroy(self) -> None: ...
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from abc import abstractmethod
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from reactivex import zip
|
|
5
|
+
from reactivex.subject import Subject
|
|
6
|
+
from . import FlowBoxRaw, FlowBoxSink, FlowBoxSource, FlowBoxInitParams
|
|
7
|
+
|
|
8
|
+
class FlowBoxCore(FlowBoxRaw, FlowBoxSink, FlowBoxSource):
|
|
9
|
+
def __init__(self, init_params: FlowBoxInitParams) -> None:
|
|
10
|
+
self._output_synchronizers: dict[str, OutputSynchronizer] = {}
|
|
11
|
+
|
|
12
|
+
@abstractmethod
|
|
13
|
+
async def on_input(self, input_name: str, header: bytes, data: bytes) -> None: ...
|
|
14
|
+
|
|
15
|
+
def on_input_received(self, input_name: str, header: bytes, data: bytes,
|
|
16
|
+
fn_ready_for_next_item: Callable[[], None]) -> None:
|
|
17
|
+
asyncio.create_task(self.on_input(input_name, header, data)) \
|
|
18
|
+
.add_done_callback(lambda _: fn_ready_for_next_item())
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def on_output_ready(self, output_name: str, header: bytes,
|
|
22
|
+
fn_ready_for_next_item: Callable[[bytes, bytes], None]) -> None:
|
|
23
|
+
output_synchronizer = (self._output_synchronizers.get(output_name) or OutputSynchronizer())
|
|
24
|
+
self._output_synchronizers[output_name] = output_synchronizer
|
|
25
|
+
|
|
26
|
+
output_synchronizer.output_ready_subject.on_next(fn_ready_for_next_item)
|
|
27
|
+
|
|
28
|
+
async def push(self, output_name: str, header: bytes, data: bytes, output_buffer_size: int = 0):
|
|
29
|
+
future = asyncio.get_running_loop().create_future()
|
|
30
|
+
|
|
31
|
+
output_synchronizer = (self._output_synchronizers.get(output_name) or OutputSynchronizer())
|
|
32
|
+
self._output_synchronizers[output_name] = output_synchronizer
|
|
33
|
+
|
|
34
|
+
if output_synchronizer.msg_count_in_buffer < output_buffer_size:
|
|
35
|
+
future.set_result(None)
|
|
36
|
+
|
|
37
|
+
output_synchronizer.msg_count_in_buffer += 1
|
|
38
|
+
output_synchronizer.push_subject.on_next((header, data, lambda: future.set_result(None) if not future.done() else None))
|
|
39
|
+
|
|
40
|
+
await future
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class OutputSynchronizer:
|
|
44
|
+
"""Synchronize pushed data with the "on output ready" event."""
|
|
45
|
+
|
|
46
|
+
def __init__(self) -> None:
|
|
47
|
+
self.msg_count_in_buffer = 0
|
|
48
|
+
self.push_subject: Subject[tuple[bytes, bytes, Callable[[], None]]] = Subject()
|
|
49
|
+
self.output_ready_subject: Subject[Callable[[bytes, bytes], None]] = Subject()
|
|
50
|
+
|
|
51
|
+
zip(self.push_subject, self.output_ready_subject) \
|
|
52
|
+
.subscribe(lambda pair: self._on_next(*pair))
|
|
53
|
+
|
|
54
|
+
def _on_next(self, push_info: tuple, fn_ready_for_next_item: Callable):
|
|
55
|
+
data_to_push, header_to_push, complete_push = push_info
|
|
56
|
+
|
|
57
|
+
self.msg_count_in_buffer -= 1
|
|
58
|
+
fn_ready_for_next_item(data_to_push, header_to_push)
|
|
59
|
+
complete_push()
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from . import FlowBoxRegistrationInfo, FlowBoxRaw
|
|
3
|
+
|
|
4
|
+
def flowbox(registration_info: FlowBoxRegistrationInfo):
|
|
5
|
+
def decorator(cls):
|
|
6
|
+
cls.registration_info = registration_info
|
|
7
|
+
return cls
|
|
8
|
+
|
|
9
|
+
return decorator
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def get_registration_info(flowbox_type: type[FlowBoxRaw]) -> FlowBoxRegistrationInfo:
|
|
13
|
+
if not hasattr(flowbox_type, "registration_info"):
|
|
14
|
+
raise AttributeError(f"Missing registration info for the {flowbox_type.__name__} box. Please add a @flowbox() decorator to the {flowbox_type.__name__} class.")
|
|
15
|
+
|
|
16
|
+
if not isinstance(flowbox_type.registration_info, FlowBoxRegistrationInfo):
|
|
17
|
+
raise TypeError(f"Invalid registration info type for the {flowbox_type.__name__} box.")
|
|
18
|
+
|
|
19
|
+
return flowbox_type.registration_info
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
@dataclass
|
|
6
|
+
class FlowBoxInitParams:
|
|
7
|
+
runtime_context: FlowRuntimeContext
|
|
8
|
+
options: dict[str, Any]
|
|
9
|
+
|
|
10
|
+
@staticmethod
|
|
11
|
+
def from_dict(data: dict[str, Any]) -> FlowBoxInitParams:
|
|
12
|
+
return FlowBoxInitParams(
|
|
13
|
+
runtime_context=FlowRuntimeContext.from_dict(data["runtimeContext"]),
|
|
14
|
+
options=data["options"]
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class FlowRuntimeContext:
|
|
20
|
+
flowbox_id: str
|
|
21
|
+
job_id: str
|
|
22
|
+
node_id: str
|
|
23
|
+
used_by: str | None
|
|
24
|
+
|
|
25
|
+
@staticmethod
|
|
26
|
+
def from_dict(data: dict[str, Any]) -> FlowRuntimeContext:
|
|
27
|
+
return FlowRuntimeContext(
|
|
28
|
+
flowbox_id=data["flowBoxId"],
|
|
29
|
+
job_id=data["jobId"],
|
|
30
|
+
node_id=data["nodeId"],
|
|
31
|
+
used_by=data.get("usedBy")
|
|
32
|
+
)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
from enum import IntEnum
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
class FlowBoxType(IntEnum):
|
|
7
|
+
SOURCE = 1
|
|
8
|
+
SINK = 2
|
|
9
|
+
PIPE = 3
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SerializationFormat(IntEnum):
|
|
13
|
+
MSGPACK = 1
|
|
14
|
+
JSON = 2
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class FlowBoxIOInterfaces:
|
|
19
|
+
inputs: list[FlowBoxIO] = field(default_factory=list)
|
|
20
|
+
outputs: list[FlowBoxIO] = field(default_factory=list)
|
|
21
|
+
|
|
22
|
+
def to_dict(self) -> dict[str, Any]:
|
|
23
|
+
return {
|
|
24
|
+
"inputs": [i.to_dict() for i in self.inputs],
|
|
25
|
+
"outputs": [o.to_dict() for o in self.outputs],
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class FlowBoxIO:
|
|
31
|
+
name: str
|
|
32
|
+
display_name: str
|
|
33
|
+
supported_formats: list[SerializationFormat] = field(default_factory=lambda: [SerializationFormat.MSGPACK])
|
|
34
|
+
|
|
35
|
+
def to_dict(self) -> dict[str, Any]:
|
|
36
|
+
return {
|
|
37
|
+
"name": self.name,
|
|
38
|
+
"displayName": self.display_name,
|
|
39
|
+
"supportedFormats": [f.name.lower() for f in self.supported_formats]
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass(frozen=True)
|
|
44
|
+
class FlowBoxUIConfig:
|
|
45
|
+
default_options: dict[str, Any] | None = None
|
|
46
|
+
implementation: str | None = None
|
|
47
|
+
|
|
48
|
+
def to_dict(self) -> dict[str, Any]:
|
|
49
|
+
return {
|
|
50
|
+
"defaultOptions": self.default_options,
|
|
51
|
+
"implementation": self.implementation
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class FlowBoxRegistrationInfo:
|
|
57
|
+
id: str
|
|
58
|
+
display_name: str
|
|
59
|
+
current_version: str
|
|
60
|
+
type: FlowBoxType
|
|
61
|
+
icon: str
|
|
62
|
+
stateless: bool = False
|
|
63
|
+
io_interfaces: FlowBoxIOInterfaces = FlowBoxIOInterfaces()
|
|
64
|
+
ui_config: FlowBoxUIConfig = FlowBoxUIConfig()
|
|
65
|
+
|
|
66
|
+
def to_dict(self, agent: str, worker_id: str, worker_data_connection_string: str) -> dict[str, Any]:
|
|
67
|
+
return {
|
|
68
|
+
"id": self.id,
|
|
69
|
+
"displayName": self.display_name,
|
|
70
|
+
"currentVersion": self.current_version,
|
|
71
|
+
"type": self.type.name.lower(),
|
|
72
|
+
"icon": self.icon,
|
|
73
|
+
"stateless": self.stateless,
|
|
74
|
+
"agent": agent,
|
|
75
|
+
"workerId": worker_id,
|
|
76
|
+
"workerDataConnectionString": worker_data_connection_string,
|
|
77
|
+
"ioInterfaces": self.io_interfaces.to_dict(),
|
|
78
|
+
"uiConfig": self.ui_config.to_dict()
|
|
79
|
+
}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import httpx
|
|
3
|
+
import msgpack
|
|
4
|
+
import zmq
|
|
5
|
+
import zmq.asyncio
|
|
6
|
+
from typing import Any
|
|
7
|
+
from .flowbox_decorator import get_registration_info
|
|
8
|
+
from . import (
|
|
9
|
+
FlowBoxRaw,
|
|
10
|
+
FlowBoxSource,
|
|
11
|
+
FlowBoxSink,
|
|
12
|
+
FlowBoxDestroy,
|
|
13
|
+
FlowBoxInitParams,
|
|
14
|
+
FlowMakerEvent,
|
|
15
|
+
FlowMakerImmutableWorkerOptions
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
class FlowMakerWorker:
|
|
19
|
+
ACK_HEADER = (0x9000).to_bytes(8, byteorder="little")
|
|
20
|
+
ERROR_HEADER = (0xFFFF).to_bytes(8, byteorder="little")
|
|
21
|
+
|
|
22
|
+
def __init__(self, options: FlowMakerImmutableWorkerOptions, implementations: dict[str, type[FlowBoxRaw]]) -> None:
|
|
23
|
+
self._options = options
|
|
24
|
+
self._implementations = implementations
|
|
25
|
+
self._flowbox_instances: dict[str, FlowBoxRaw] = {}
|
|
26
|
+
self._socket = zmq.asyncio.Context().socket(zmq.SocketType.ROUTER)
|
|
27
|
+
|
|
28
|
+
async def run(self):
|
|
29
|
+
http_client = httpx.AsyncClient(base_url=self._options.runtime_http_address)
|
|
30
|
+
|
|
31
|
+
# Register all declared Flow boxes.
|
|
32
|
+
await self._register(http_client)
|
|
33
|
+
|
|
34
|
+
self._socket.bind(self._options.router_transport_address)
|
|
35
|
+
await self._listen_incoming_messages()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
async def _register(self, http_client: httpx.AsyncClient) -> None:
|
|
39
|
+
registrations = [get_registration_info(impl) for impl in self._implementations.values()]
|
|
40
|
+
register_request = [
|
|
41
|
+
registration.to_dict(
|
|
42
|
+
"flowmaker-python-sdk/1.0.1;os=linux;tag=docker", # TODO: build this agent string using environment info
|
|
43
|
+
self._options.worker_id, self._options.worker_transport_adv_address)
|
|
44
|
+
for registration in registrations
|
|
45
|
+
]
|
|
46
|
+
result = await http_client.post("/workers/register", json=register_request)
|
|
47
|
+
result.raise_for_status()
|
|
48
|
+
|
|
49
|
+
flowbow_ids = [f'"{r.id}"' for r in registrations]
|
|
50
|
+
print(f"Worker started with {', '.join(flowbow_ids)} registered.")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
async def _listen_incoming_messages(self) -> None:
|
|
54
|
+
while True:
|
|
55
|
+
msg_parts = await self._socket.recv_multipart()
|
|
56
|
+
|
|
57
|
+
# Caution: keep the processing of each message in a separate function, to maintain
|
|
58
|
+
# a local scope of variables (since Python doesn't enforce local scope in while/for loops).
|
|
59
|
+
await self._process_incoming_message(msg_parts)
|
|
60
|
+
|
|
61
|
+
async def _process_incoming_message(self, msg_parts: list[bytes]) -> None:
|
|
62
|
+
if len(msg_parts) < 6:
|
|
63
|
+
print("Invalid message received: it must contain at least 6 parts.")
|
|
64
|
+
return
|
|
65
|
+
|
|
66
|
+
routing_id, token, encoded_node_ref, encoded_io_name, header, data, *_ = msg_parts
|
|
67
|
+
|
|
68
|
+
node_ref = encoded_node_ref.decode()
|
|
69
|
+
io_name = encoded_io_name.decode()
|
|
70
|
+
event_id = int.from_bytes(header[:4], byteorder="little")
|
|
71
|
+
|
|
72
|
+
if event_id == FlowMakerEvent.HEARTBEAT_EVENT:
|
|
73
|
+
# print("HEARTBEAT_EVENT received")
|
|
74
|
+
# Acknowledge to indicate that the worker is still alive.
|
|
75
|
+
await self._socket.send_multipart([routing_id, token, self.ACK_HEADER])
|
|
76
|
+
|
|
77
|
+
elif event_id == FlowMakerEvent.INIT_FLOW_BOX:
|
|
78
|
+
# print("INIT_FLOW_BOX received")
|
|
79
|
+
msg = msgpack.unpackb(data)
|
|
80
|
+
init_params = FlowBoxInitParams.from_dict(msg)
|
|
81
|
+
|
|
82
|
+
implementation = self._implementations[init_params.runtime_context.flowbox_id]
|
|
83
|
+
self._flowbox_instances[node_ref] = implementation(init_params)
|
|
84
|
+
|
|
85
|
+
print(f"Flow box {node_ref} initialized.")
|
|
86
|
+
await self._socket.send_multipart([routing_id, token, self.ACK_HEADER])
|
|
87
|
+
|
|
88
|
+
elif event_id == FlowMakerEvent.CURRENT_EVENT:
|
|
89
|
+
# print("CURRENT_EVENT received")
|
|
90
|
+
flowbox_instance = await self._get_flowbox_instance(routing_id, token, node_ref)
|
|
91
|
+
if flowbox_instance is None:
|
|
92
|
+
return
|
|
93
|
+
|
|
94
|
+
if isinstance(flowbox_instance, FlowBoxSink):
|
|
95
|
+
flowbox_instance.on_input_received(
|
|
96
|
+
io_name, header, data,
|
|
97
|
+
lambda: self._send_multipart_and_forget([routing_id, token, self.ACK_HEADER]))
|
|
98
|
+
else:
|
|
99
|
+
print(f"Flow box {node_ref} does not implement FlowBoxSink.")
|
|
100
|
+
|
|
101
|
+
elif event_id == FlowMakerEvent.CAN_SEND_NEXT:
|
|
102
|
+
# print("CAN_SEND_NEXT received")
|
|
103
|
+
flowbox_instance = await self._get_flowbox_instance(routing_id, token, node_ref)
|
|
104
|
+
if flowbox_instance is None:
|
|
105
|
+
return
|
|
106
|
+
|
|
107
|
+
if isinstance(flowbox_instance, FlowBoxSource):
|
|
108
|
+
flowbox_instance.on_output_ready(
|
|
109
|
+
io_name, header,
|
|
110
|
+
lambda header, data: self._send_multipart_and_forget([routing_id, token, header, data]))
|
|
111
|
+
else:
|
|
112
|
+
print(f"Flow box {node_ref} does not implement FlowBoxSource.")
|
|
113
|
+
|
|
114
|
+
elif event_id == FlowMakerEvent.DESTROY_EVENT:
|
|
115
|
+
# print("DESTROY_EVENT received")
|
|
116
|
+
flowbox_instance = await self._get_flowbox_instance(routing_id, token, node_ref)
|
|
117
|
+
if flowbox_instance is None:
|
|
118
|
+
return
|
|
119
|
+
|
|
120
|
+
if isinstance(flowbox_instance, FlowBoxDestroy):
|
|
121
|
+
flowbox_instance.on_destroy()
|
|
122
|
+
self._flowbox_instances.pop(node_ref)
|
|
123
|
+
|
|
124
|
+
print(f"Flow box {node_ref} destroyed.")
|
|
125
|
+
await self._socket.send_multipart([routing_id, token, self.ACK_HEADER])
|
|
126
|
+
|
|
127
|
+
elif event_id == FlowMakerEvent.SCHEDULER_RESTART:
|
|
128
|
+
# print("SCHEDULER_RESTART received")
|
|
129
|
+
for flowbox_instance in self._flowbox_instances.values():
|
|
130
|
+
if isinstance(flowbox_instance, FlowBoxDestroy):
|
|
131
|
+
flowbox_instance.on_destroy()
|
|
132
|
+
self._flowbox_instances.clear()
|
|
133
|
+
|
|
134
|
+
print("Scheduler restarted! All Flow box instances have just been destroyed.")
|
|
135
|
+
await self._socket.send_multipart([routing_id, token, self.ACK_HEADER])
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
async def _get_flowbox_instance(self, routing_id: bytes, token: bytes, node_ref: str) -> FlowBoxRaw | None:
|
|
139
|
+
flowbox_instance = self._flowbox_instances.get(node_ref)
|
|
140
|
+
if flowbox_instance is None:
|
|
141
|
+
await self._socket.send_multipart([routing_id, token, self.ERROR_HEADER])
|
|
142
|
+
print(f"Flow box {node_ref} not found or deleted.")
|
|
143
|
+
|
|
144
|
+
return flowbox_instance
|
|
145
|
+
|
|
146
|
+
def _send_multipart_and_forget(self, msg_parts: Any) -> None:
|
|
147
|
+
asyncio.ensure_future(self._socket.send_multipart(msg_parts)) # Fire and forget
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from collections.abc import Callable
|
|
3
|
+
from . import FlowMakerWorkerOptions, FlowMakerWorker, FlowBoxRaw
|
|
4
|
+
from .flowbox_decorator import get_registration_info
|
|
5
|
+
|
|
6
|
+
class FlowMakerWorkerBuilder:
|
|
7
|
+
def __init__(self) -> None:
|
|
8
|
+
self._options = FlowMakerWorkerOptions()
|
|
9
|
+
self._implementations: dict[str, type[FlowBoxRaw]] = {}
|
|
10
|
+
|
|
11
|
+
def configure(self, configureOptions: Callable[[FlowMakerWorkerOptions], None]) -> FlowMakerWorkerBuilder:
|
|
12
|
+
configureOptions(self._options)
|
|
13
|
+
return self
|
|
14
|
+
|
|
15
|
+
def declare_flowbox(self, flowbox_type: type[FlowBoxRaw]) -> FlowMakerWorkerBuilder:
|
|
16
|
+
registration_info = get_registration_info(flowbox_type)
|
|
17
|
+
id = f"{registration_info.id}/{registration_info.current_version}"
|
|
18
|
+
self._implementations[id] = flowbox_type
|
|
19
|
+
return self
|
|
20
|
+
|
|
21
|
+
def build(self) -> FlowMakerWorker:
|
|
22
|
+
immutable_options = self._options.to_immutable()
|
|
23
|
+
return FlowMakerWorker(immutable_options, self._implementations)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
class FlowMakerWorkerOptions:
|
|
6
|
+
def __init__(self) -> None:
|
|
7
|
+
self.worker_id = os.getenv("FM_WORKER_ID")
|
|
8
|
+
self.worker_transport_adv_address = os.getenv("FM_WORKER_TRANSPORT_ADV_ADDRESS")
|
|
9
|
+
self.router_transport_address = os.getenv("FM_ROUTER_TRANSPORT_ADDRESS")
|
|
10
|
+
self.runtime_http_address = os.getenv("FM_RUNTIME_HTTP_ADDRESS")
|
|
11
|
+
|
|
12
|
+
def to_immutable(self) -> FlowMakerImmutableWorkerOptions:
|
|
13
|
+
if not self.worker_id or self.worker_id.isspace():
|
|
14
|
+
raise ValueError("The 'FM_WORKER_ID' environment variable (or 'worker_id' option) is required.")
|
|
15
|
+
|
|
16
|
+
if not self.worker_transport_adv_address or self.worker_transport_adv_address.isspace():
|
|
17
|
+
raise ValueError("The 'FM_WORKER_TRANSPORT_ADV_ADDRESS' environment variable (or 'worker_transport_adv_address' option) is required.")
|
|
18
|
+
|
|
19
|
+
if not self.router_transport_address or self.router_transport_address.isspace():
|
|
20
|
+
raise ValueError("The 'FM_ROUTER_TRANSPORT_ADDRESS' environment variable (or 'router_transport_address' option) is required.")
|
|
21
|
+
|
|
22
|
+
if not self.runtime_http_address or self.runtime_http_address.isspace():
|
|
23
|
+
raise ValueError("The 'FM_RUNTIME_HTTP_ADDRESS' environment variable (or 'runtime_http_address' option) is required.")
|
|
24
|
+
|
|
25
|
+
return FlowMakerImmutableWorkerOptions(
|
|
26
|
+
self.worker_id,
|
|
27
|
+
self.worker_transport_adv_address,
|
|
28
|
+
self.router_transport_address,
|
|
29
|
+
self.runtime_http_address
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass(frozen=True)
|
|
34
|
+
class FlowMakerImmutableWorkerOptions:
|
|
35
|
+
worker_id: str
|
|
36
|
+
worker_transport_adv_address: str
|
|
37
|
+
router_transport_address: str
|
|
38
|
+
runtime_http_address: str
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "industream-flowmaker-sdk"
|
|
7
|
+
version = "1.0.2"
|
|
8
|
+
description = "Industream FlowMaker SDK for Python"
|
|
9
|
+
authors = [
|
|
10
|
+
{ name = "Sylvain Bruyère", email = "sylvain.bruyere@industream.com" },
|
|
11
|
+
]
|
|
12
|
+
# license = { text = "Apache-2.0" }
|
|
13
|
+
# readme = "README.md"
|
|
14
|
+
requires-python = ">=3.12"
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
18
|
+
"License :: OSI Approved :: Apache Software License",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"Typing :: Typed",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"httpx >= 0.28.1",
|
|
25
|
+
"pyzmq >= 26.2.0",
|
|
26
|
+
"msgpack >= 1.1.0",
|
|
27
|
+
"msgpack-types >= 0.5.0",
|
|
28
|
+
"reactivex >= 4.0.4"
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/industream/industream-flowmaker"
|
|
33
|
+
# Documentation = ""
|
|
34
|
+
Repository = "https://github.com/industream/industream-flowmaker"
|
|
35
|
+
Issues = "https://github.com/industream/industream-flowmaker/issues"
|
|
36
|
+
# Changelog = ""
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build]
|
|
39
|
+
include = ["industream"]
|