industream-flowmaker-sdk 1.0.2__py3-none-any.whl

industream/flowmaker/sdk/__init__.py

@@ -0,0 +1,39 @@
+from .flowbox_init_params import *
+from .flowbox import *
+from .flowbox_core import *
+from .flowbox_registration_info import *
+from .flowbox_decorator import *
+from .flowmaker_event import *
+from .flowmaker_worker_options import *
+from .flowmaker_worker import *
+from .flowmaker_worker_builder import *
+
+__all__ = [
+    "FlowBoxRaw",
+    "FlowBoxSource",
+    "FlowBoxSink",
+    "FlowBoxDestroy",
+    "FlowBoxCore",
+
+    "flowbox",
+    "FlowBoxRegistrationInfo",
+    "FlowBoxType",
+    "FlowBoxIOInterfaces",
+    "FlowBoxIO",
+    "SerializationFormat",
+    "FlowBoxUIConfig",
+
+    "FlowBoxInitParams",
+    "FlowRuntimeContext",
+
+    "FlowMakerWorkerBuilder",
+    "FlowMakerWorkerOptions",
+    "FlowMakerWorker"
+]
+
+
+import asyncio
+import sys
+
+if sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())

industream/flowmaker/sdk/flowbox.py

@@ -0,0 +1,24 @@
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from . import FlowBoxInitParams
+
+class FlowBoxRaw(ABC):
+    @abstractmethod
+    def __init__(self, init_params: FlowBoxInitParams) -> None: ...
+
+
+class FlowBoxSource(ABC):
+    @abstractmethod
+    def on_output_ready(self, output_name: str, header: bytes,
+                        fn_ready_for_next_item: Callable[[bytes, bytes], None]) -> None: ...
+
+
+class FlowBoxSink(ABC):
+    @abstractmethod
+    def on_input_received(self, input_name: str, header: bytes, data: bytes,
+                          fn_ready_for_next_item: Callable[[], None]) -> None: ...
+
+
+class FlowBoxDestroy(ABC):
+    @abstractmethod
+    def on_destroy(self) -> None: ...

industream/flowmaker/sdk/flowbox_core.py

@@ -0,0 +1,59 @@
+import asyncio
+from abc import abstractmethod
+from collections.abc import Callable
+from reactivex import zip
+from reactivex.subject import Subject
+from . import FlowBoxRaw, FlowBoxSink, FlowBoxSource, FlowBoxInitParams
+
+class FlowBoxCore(FlowBoxRaw, FlowBoxSink, FlowBoxSource):
+    def __init__(self, init_params: FlowBoxInitParams) -> None:
+        self._output_synchronizers: dict[str, OutputSynchronizer] = {}
+
+    @abstractmethod
+    async def on_input(self, input_name: str, header: bytes, data: bytes) -> None: ...
+
+    def on_input_received(self, input_name: str, header: bytes, data: bytes,
+                          fn_ready_for_next_item: Callable[[], None]) -> None:
+        asyncio.create_task(self.on_input(input_name, header, data)) \
+            .add_done_callback(lambda _: fn_ready_for_next_item())
+
+
+    def on_output_ready(self, output_name: str, header: bytes,
+                        fn_ready_for_next_item: Callable[[bytes, bytes], None]) -> None:
+        output_synchronizer = (self._output_synchronizers.get(output_name) or OutputSynchronizer())
+        self._output_synchronizers[output_name] = output_synchronizer
+
+        output_synchronizer.output_ready_subject.on_next(fn_ready_for_next_item)
+
+    async def push(self, output_name: str, header: bytes, data: bytes, output_buffer_size: int = 0):
+        future = asyncio.get_running_loop().create_future()
+
+        output_synchronizer = (self._output_synchronizers.get(output_name) or OutputSynchronizer())
+        self._output_synchronizers[output_name] = output_synchronizer
+
+        if output_synchronizer.msg_count_in_buffer < output_buffer_size:
+            future.set_result(None)
+
+        output_synchronizer.msg_count_in_buffer += 1
+        output_synchronizer.push_subject.on_next((header, data, lambda: future.set_result(None) if not future.done() else None))
+
+        await future
+
+
+class OutputSynchronizer:
+    """Synchronize pushed data with the "on output ready" event."""
+
+    def __init__(self) -> None:
+        self.msg_count_in_buffer = 0
+        self.push_subject: Subject[tuple[bytes, bytes, Callable[[], None]]] = Subject()
+        self.output_ready_subject: Subject[Callable[[bytes, bytes], None]] = Subject()
+
+        zip(self.push_subject, self.output_ready_subject) \
+            .subscribe(lambda pair: self._on_next(*pair))
+
+    def _on_next(self, push_info: tuple, fn_ready_for_next_item: Callable):
+        data_to_push, header_to_push, complete_push = push_info
+
+        self.msg_count_in_buffer -= 1
+        fn_ready_for_next_item(data_to_push, header_to_push)
+        complete_push()

industream/flowmaker/sdk/flowbox_decorator.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+from . import FlowBoxRegistrationInfo, FlowBoxRaw
+
+def flowbox(registration_info: FlowBoxRegistrationInfo):
+    def decorator(cls):
+        cls.registration_info = registration_info
+        return cls
+
+    return decorator
+
+
+def get_registration_info(flowbox_type: type[FlowBoxRaw]) -> FlowBoxRegistrationInfo:
+    if not hasattr(flowbox_type, "registration_info"):
+        raise AttributeError(f"Missing registration info for the {flowbox_type.__name__} box. Please add a @flowbox() decorator to the {flowbox_type.__name__} class.")
+
+    if not isinstance(flowbox_type.registration_info, FlowBoxRegistrationInfo):
+        raise TypeError(f"Invalid registration info type for the {flowbox_type.__name__} box.")
+
+    return flowbox_type.registration_info

industream/flowmaker/sdk/flowbox_init_params.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+from dataclasses import dataclass
+from typing import Any
+
+@dataclass
+class FlowBoxInitParams:
+    runtime_context: FlowRuntimeContext
+    options: dict[str, Any]
+
+    @staticmethod
+    def from_dict(data: dict[str, Any]) -> FlowBoxInitParams:
+        return FlowBoxInitParams(
+            runtime_context=FlowRuntimeContext.from_dict(data["runtimeContext"]),
+            options=data["options"]
+        )
+
+
+@dataclass
+class FlowRuntimeContext:
+    flowbox_id: str
+    job_id: str
+    node_id: str
+    used_by: str | None
+
+    @staticmethod
+    def from_dict(data: dict[str, Any]) -> FlowRuntimeContext:
+        return FlowRuntimeContext(
+            flowbox_id=data["flowBoxId"],
+            job_id=data["jobId"],
+            node_id=data["nodeId"],
+            used_by=data.get("usedBy")
+        )

industream/flowmaker/sdk/flowbox_registration_info.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+from enum import IntEnum
+from typing import Any
+
+class FlowBoxType(IntEnum):
+    SOURCE = 1
+    SINK = 2
+    PIPE = 3
+
+
+class SerializationFormat(IntEnum):
+    MSGPACK = 1
+    JSON = 2
+
+
+@dataclass(frozen=True)
+class FlowBoxIOInterfaces:
+    inputs: list[FlowBoxIO] = field(default_factory=list)
+    outputs: list[FlowBoxIO] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "inputs": [i.to_dict() for i in self.inputs],
+            "outputs": [o.to_dict() for o in self.outputs],
+        }
+
+
+@dataclass(frozen=True)
+class FlowBoxIO:
+    name: str
+    display_name: str
+    supported_formats: list[SerializationFormat] = field(default_factory=lambda: [SerializationFormat.MSGPACK])
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "displayName": self.display_name,
+            "supportedFormats": [f.name.lower() for f in self.supported_formats]
+        }
+
+
+@dataclass(frozen=True)
+class FlowBoxUIConfig:
+    default_options: dict[str, Any] | None = None
+    implementation: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "defaultOptions": self.default_options,
+            "implementation": self.implementation
+        }
+
+
+@dataclass(frozen=True)
+class FlowBoxRegistrationInfo:
+    id: str
+    display_name: str
+    current_version: str
+    type: FlowBoxType
+    icon: str
+    stateless: bool = False
+    io_interfaces: FlowBoxIOInterfaces = FlowBoxIOInterfaces()
+    ui_config: FlowBoxUIConfig = FlowBoxUIConfig()
+
+    def to_dict(self, agent: str, worker_id: str, worker_data_connection_string: str) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "displayName": self.display_name,
+            "currentVersion": self.current_version,
+            "type": self.type.name.lower(),
+            "icon": self.icon,
+            "stateless": self.stateless,
+            "agent": agent,
+            "workerId": worker_id,
+            "workerDataConnectionString": worker_data_connection_string,
+            "ioInterfaces": self.io_interfaces.to_dict(),
+            "uiConfig": self.ui_config.to_dict()
+        }

industream/flowmaker/sdk/flowmaker_event.py

@@ -0,0 +1,9 @@
+from enum import IntEnum
+
+class FlowMakerEvent(IntEnum):
+    INIT_FLOW_BOX = 0x4026
+    CURRENT_EVENT = 0x8200
+    CAN_SEND_NEXT = 0x8001
+    DESTROY_EVENT = 0x82FF
+    HEARTBEAT_EVENT = 0x8210
+    SCHEDULER_RESTART = 0x8220

industream/flowmaker/sdk/flowmaker_worker.py

@@ -0,0 +1,147 @@
+import asyncio
+import httpx
+import msgpack
+import zmq
+import zmq.asyncio
+from typing import Any
+from .flowbox_decorator import get_registration_info
+from . import (
+    FlowBoxRaw,
+    FlowBoxSource,
+    FlowBoxSink,
+    FlowBoxDestroy,
+    FlowBoxInitParams,
+    FlowMakerEvent,
+    FlowMakerImmutableWorkerOptions
+)
+
+class FlowMakerWorker:
+    ACK_HEADER = (0x9000).to_bytes(8, byteorder="little")
+    ERROR_HEADER = (0xFFFF).to_bytes(8, byteorder="little")
+
+    def __init__(self, options: FlowMakerImmutableWorkerOptions, implementations: dict[str, type[FlowBoxRaw]]) -> None:
+        self._options = options
+        self._implementations = implementations
+        self._flowbox_instances: dict[str, FlowBoxRaw] = {}
+        self._socket = zmq.asyncio.Context().socket(zmq.SocketType.ROUTER)
+
+    async def run(self):
+        http_client = httpx.AsyncClient(base_url=self._options.runtime_http_address)
+
+        # Register all declared Flow boxes.
+        await self._register(http_client)
+
+        self._socket.bind(self._options.router_transport_address)
+        await self._listen_incoming_messages()
+
+
+    async def _register(self, http_client: httpx.AsyncClient) -> None:
+        registrations = [get_registration_info(impl) for impl in self._implementations.values()]
+        register_request = [
+            registration.to_dict(
+                "flowmaker-python-sdk/1.0.1;os=linux;tag=docker", # TODO: build this agent string using environment info
+                self._options.worker_id, self._options.worker_transport_adv_address)
+            for registration in registrations
+        ]
+        result = await http_client.post("/workers/register", json=register_request)
+        result.raise_for_status()
+
+        flowbow_ids = [f'"{r.id}"' for r in registrations]
+        print(f"Worker started with {', '.join(flowbow_ids)} registered.")
+
+
+    async def _listen_incoming_messages(self) -> None:
+        while True:
+            msg_parts = await self._socket.recv_multipart()
+
+            # Caution: keep the processing of each message in a separate function, to maintain
+            # a local scope of variables (since Python doesn't enforce local scope in while/for loops).
+            await self._process_incoming_message(msg_parts)
+
+    async def _process_incoming_message(self, msg_parts: list[bytes]) -> None:
+        if len(msg_parts) < 6:
+            print("Invalid message received: it must contain at least 6 parts.")
+            return
+
+        routing_id, token, encoded_node_ref, encoded_io_name, header, data, *_ = msg_parts
+
+        node_ref = encoded_node_ref.decode()
+        io_name = encoded_io_name.decode()
+        event_id = int.from_bytes(header[:4], byteorder="little")
+
+        if event_id == FlowMakerEvent.HEARTBEAT_EVENT:
+            # print("HEARTBEAT_EVENT received")
+            # Acknowledge to indicate that the worker is still alive.
+            await self._socket.send_multipart([routing_id, token, self.ACK_HEADER])
+
+        elif event_id == FlowMakerEvent.INIT_FLOW_BOX:
+            # print("INIT_FLOW_BOX received")
+            msg = msgpack.unpackb(data)
+            init_params = FlowBoxInitParams.from_dict(msg)
+
+            implementation = self._implementations[init_params.runtime_context.flowbox_id]
+            self._flowbox_instances[node_ref] = implementation(init_params)
+
+            print(f"Flow box {node_ref} initialized.")
+            await self._socket.send_multipart([routing_id, token, self.ACK_HEADER])
+
+        elif event_id == FlowMakerEvent.CURRENT_EVENT:
+            # print("CURRENT_EVENT received")
+            flowbox_instance = await self._get_flowbox_instance(routing_id, token, node_ref)
+            if flowbox_instance is None:
+                return
+
+            if isinstance(flowbox_instance, FlowBoxSink):
+                flowbox_instance.on_input_received(
+                    io_name, header, data,
+                    lambda: self._send_multipart_and_forget([routing_id, token, self.ACK_HEADER]))
+            else:
+                print(f"Flow box {node_ref} does not implement FlowBoxSink.")
+
+        elif event_id == FlowMakerEvent.CAN_SEND_NEXT:
+            # print("CAN_SEND_NEXT received")
+            flowbox_instance = await self._get_flowbox_instance(routing_id, token, node_ref)
+            if flowbox_instance is None:
+                return
+
+            if isinstance(flowbox_instance, FlowBoxSource):
+                flowbox_instance.on_output_ready(
+                    io_name, header,
+                    lambda header, data: self._send_multipart_and_forget([routing_id, token, header, data]))
+            else:
+                print(f"Flow box {node_ref} does not implement FlowBoxSource.")
+
+        elif event_id == FlowMakerEvent.DESTROY_EVENT:
+            # print("DESTROY_EVENT received")
+            flowbox_instance = await self._get_flowbox_instance(routing_id, token, node_ref)
+            if flowbox_instance is None:
+                return
+
+            if isinstance(flowbox_instance, FlowBoxDestroy):
+                flowbox_instance.on_destroy()
+            self._flowbox_instances.pop(node_ref)
+
+            print(f"Flow box {node_ref} destroyed.")
+            await self._socket.send_multipart([routing_id, token, self.ACK_HEADER])
+
+        elif event_id == FlowMakerEvent.SCHEDULER_RESTART:
+            # print("SCHEDULER_RESTART received")
+            for flowbox_instance in self._flowbox_instances.values():
+                if isinstance(flowbox_instance, FlowBoxDestroy):
+                    flowbox_instance.on_destroy()
+            self._flowbox_instances.clear()
+
+            print("Scheduler restarted! All Flow box instances have just been destroyed.")
+            await self._socket.send_multipart([routing_id, token, self.ACK_HEADER])
+
+
+    async def _get_flowbox_instance(self, routing_id: bytes, token: bytes, node_ref: str) -> FlowBoxRaw | None:
+        flowbox_instance = self._flowbox_instances.get(node_ref)
+        if flowbox_instance is None:
+            await self._socket.send_multipart([routing_id, token, self.ERROR_HEADER])
+            print(f"Flow box {node_ref} not found or deleted.")
+
+        return flowbox_instance
+
+    def _send_multipart_and_forget(self, msg_parts: Any) -> None:
+        asyncio.ensure_future(self._socket.send_multipart(msg_parts)) # Fire and forget

industream/flowmaker/sdk/flowmaker_worker_builder.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+from collections.abc import Callable
+from . import FlowMakerWorkerOptions, FlowMakerWorker, FlowBoxRaw
+from .flowbox_decorator import get_registration_info
+
+class FlowMakerWorkerBuilder:
+    def __init__(self) -> None:
+        self._options = FlowMakerWorkerOptions()
+        self._implementations: dict[str, type[FlowBoxRaw]] = {}
+
+    def configure(self, configureOptions: Callable[[FlowMakerWorkerOptions], None]) -> FlowMakerWorkerBuilder:
+        configureOptions(self._options)
+        return self
+
+    def declare_flowbox(self, flowbox_type: type[FlowBoxRaw]) -> FlowMakerWorkerBuilder:
+        registration_info = get_registration_info(flowbox_type)
+        id = f"{registration_info.id}/{registration_info.current_version}"
+        self._implementations[id] = flowbox_type
+        return self
+
+    def build(self) -> FlowMakerWorker:
+        immutable_options = self._options.to_immutable()
+        return FlowMakerWorker(immutable_options, self._implementations)

industream/flowmaker/sdk/flowmaker_worker_options.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+from dataclasses import dataclass
+import os
+
+class FlowMakerWorkerOptions:
+    def __init__(self) -> None:
+        self.worker_id = os.getenv("FM_WORKER_ID")
+        self.worker_transport_adv_address = os.getenv("FM_WORKER_TRANSPORT_ADV_ADDRESS")
+        self.router_transport_address = os.getenv("FM_ROUTER_TRANSPORT_ADDRESS")
+        self.runtime_http_address = os.getenv("FM_RUNTIME_HTTP_ADDRESS")
+
+    def to_immutable(self) -> FlowMakerImmutableWorkerOptions:
+        if not self.worker_id or self.worker_id.isspace():
+            raise ValueError("The 'FM_WORKER_ID' environment variable (or 'worker_id' option) is required.")
+        
+        if not self.worker_transport_adv_address or self.worker_transport_adv_address.isspace():
+            raise ValueError("The 'FM_WORKER_TRANSPORT_ADV_ADDRESS' environment variable (or 'worker_transport_adv_address' option) is required.")
+        
+        if not self.router_transport_address or self.router_transport_address.isspace():
+            raise ValueError("The 'FM_ROUTER_TRANSPORT_ADDRESS' environment variable (or 'router_transport_address' option) is required.")
+        
+        if not self.runtime_http_address or self.runtime_http_address.isspace():
+            raise ValueError("The 'FM_RUNTIME_HTTP_ADDRESS' environment variable (or 'runtime_http_address' option) is required.")
+
+        return FlowMakerImmutableWorkerOptions(
+            self.worker_id,
+            self.worker_transport_adv_address,
+            self.router_transport_address,
+            self.runtime_http_address
+        )
+
+
+@dataclass(frozen=True)
+class FlowMakerImmutableWorkerOptions:
+    worker_id: str
+    worker_transport_adv_address: str
+    router_transport_address: str
+    runtime_http_address: str

industream_flowmaker_sdk-1.0.2.dist-info/METADATA

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: industream-flowmaker-sdk
+Version: 1.0.2
+Summary: Industream FlowMaker SDK for Python
+Project-URL: Homepage, https://github.com/industream/industream-flowmaker
+Project-URL: Repository, https://github.com/industream/industream-flowmaker
+Project-URL: Issues, https://github.com/industream/industream-flowmaker/issues
+Author-email: Sylvain Bruyère <sylvain.bruyere@industream.com>
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: msgpack-types>=0.5.0
+Requires-Dist: msgpack>=1.1.0
+Requires-Dist: pyzmq>=26.2.0
+Requires-Dist: reactivex>=4.0.4

industream_flowmaker_sdk-1.0.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+industream/flowmaker/sdk/__init__.py,sha256=aVhFHBW1cpN9VDPfLf0ZibzTkUA7JOkZMHvkPTP9JQA,884
+industream/flowmaker/sdk/flowbox.py,sha256=1sJUpLn3zBZu6Fc9mZ1eb7GcPavtoNKQZITOJbLPycA,738
+industream/flowmaker/sdk/flowbox_core.py,sha256=jw7JmkPfjzlaUxhCGndE0eAYo2uguVBf93RnQPvCqYI,2655
+industream/flowmaker/sdk/flowbox_decorator.py,sha256=4AC-jI-0HZfT6t8LMmMqsJEj9SDfdb2BA0ReUAX2xU4,808
+industream/flowmaker/sdk/flowbox_init_params.py,sha256=Um7__12cc-wnkebZje223pB_9tbE3ygvuQvEVSRVOUU,866
+industream/flowmaker/sdk/flowbox_registration_info.py,sha256=u_LR6gS7NsWTB9SuK2wU0Vd-g01GR91ocHAiOOZHOvs,2334
+industream/flowmaker/sdk/flowmaker_event.py,sha256=gajEZ3IKzh8AIYRKUVWtfudxv7xjUEpW__SjWSQzbIk,234
+industream/flowmaker/sdk/flowmaker_worker.py,sha256=5coORY-nIAGYUq-hUC8Wgo-TcuZ8xnGclyzqqBUNe2Q,6471
+industream/flowmaker/sdk/flowmaker_worker_builder.py,sha256=kiS04H7G3t8C04jF0eappq6zm7igNbXMRrO-VW4w5BI,1031
+industream/flowmaker/sdk/flowmaker_worker_options.py,sha256=b_kwk8WW8eaeCfdQAE7pxFu6bkjw53ZY4mC9SCvC3No,1818
+industream_flowmaker_sdk-1.0.2.dist-info/METADATA,sha256=cbi0iLvpnS82iYCABrHoA0EXlqsRJQ1xIhDLsjRLyPg,864
+industream_flowmaker_sdk-1.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+industream_flowmaker_sdk-1.0.2.dist-info/RECORD,,

industream_flowmaker_sdk-1.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any