syrtis-python-dataflow 1.0.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

syrtis_python_dataflow/__init__.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from syrtis_python_dataflow.common.abstract_operator import AbstractOperator
+from syrtis_python_dataflow.common.abstract_projection_storage import AbstractProjectionStorage
+from syrtis_python_dataflow.common.element import Element
+from syrtis_python_dataflow.common.executor import Executor
+from syrtis_python_dataflow.common.file_projection_storage import FileProjectionStorage
+from syrtis_python_dataflow.common.memory_projection_storage import MemoryProjectionStorage
+from syrtis_python_dataflow.common.operator_context import OperatorContext
+from syrtis_python_dataflow.common.pipeline import Pipeline
+from syrtis_python_dataflow.common.port import Port
+from syrtis_python_dataflow.common.run_report import RunReport
+from syrtis_python_dataflow.common.stage import Stage
+from syrtis_python_dataflow.common.stage_input import StageInput
+from syrtis_python_dataflow.common.stage_report import StageReport
+
+__all__ = [
+    "AbstractOperator",
+    "AbstractProjectionStorage",
+    "Element",
+    "Executor",
+    "FileProjectionStorage",
+    "MemoryProjectionStorage",
+    "OperatorContext",
+    "Pipeline",
+    "Port",
+    "RunReport",
+    "Stage",
+    "StageInput",
+    "StageReport",
+]

syrtis_python_dataflow/common/abstract_operator.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable
+
+from wexample_helpers.classes.abstract_method import abstract_method
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.decorator.base_class import base_class
+
+from syrtis_python_dataflow.const.config import CONFIG_TRANSMIT_CONTENT_TYPE
+from syrtis_python_dataflow.const.port import PORT_NAME_DEFAULT
+
+if TYPE_CHECKING:
+    from syrtis_python_dataflow.common.element import Element
+    from syrtis_python_dataflow.common.operator_context import OperatorContext
+    from syrtis_python_dataflow.common.port import Port
+
+
+@base_class
+class AbstractOperator(BaseClass):
+    """What a stage runs: named, with its ports. Given, for each input port,
+    the whole of what reaches it — the elements of every stage linked to the
+    port, in the links' order —, it gives back the elements of its
+    projection. Whether it gives one per element read, ten, or one for the
+    lot is its own business. A stage whose operator has no input port is
+    where a pipe starts: what it reads lives outside the pipe.
+
+    An element keeps its key through what derives from it (`derive`): what
+    two branches made of the same thing is told apart by it, and put back
+    together by it.
+
+    An operator that passes through (`passes_through`, or a stage's config
+    saying so: `transmit_content_type`) — a text in capitals is still the
+    text it was — gives each element the type it came with,
+    whatever its output port declares; a stage reading it may then name a
+    type in place of a port, and read the elements of that type alone."""
+
+    name: str = ""
+
+    @classmethod
+    def passes_through(cls) -> bool:
+        """Whether what it gives keeps the type of what it read."""
+        return False
+
+    @classmethod
+    def transmits(cls, config: dict[str, Any] | None) -> bool:
+        """Whether a stage set with this config keeps the type of what it
+        reads: as set (`transmit_content_type`), else as the operator does
+        by default."""
+        said = (config or {}).get(CONFIG_TRANSMIT_CONTENT_TYPE)
+        return cls.passes_through() if said is None else bool(said)
+
+    @classmethod
+    def get_ports_input(cls) -> list[Port]:
+        from syrtis_python_dataflow.common.port import Port
+
+        return [Port(name=PORT_NAME_DEFAULT)]
+
+    @classmethod
+    def get_ports_output(cls) -> list[Port]:
+        from syrtis_python_dataflow.common.port import Port
+
+        return [Port(name=PORT_NAME_DEFAULT)]
+
+    @classmethod
+    def get_port_input(cls, name: str) -> Port | None:
+        return next((port for port in cls.get_ports_input() if port.name == name), None)
+
+    @classmethod
+    def get_port_output(cls, name: str) -> Port | None:
+        return next((port for port in cls.get_ports_output() if port.name == name), None)
+
+    @abstract_method
+    def run(self, inputs: dict[str, list[Element]], config: dict[str, Any], context: OperatorContext) -> list[Element]:
+        """The elements of the projection, from what each input port reads."""
+
+    def each(self, elements: list[Element], make: Callable[[Element], Element | list[Element] | None], context: OperatorContext) -> list[Element]:
+        """What `make` gives of each element — one, several, none —, an
+        element it fails on said in the stage's report (`context.fail`) and
+        left out: one page out of reach costs that page, not the stage."""
+        produced: list[Element] = []
+
+        for element in elements:
+            try:
+                made = make(element)
+            except Exception as exception:  # noqa: BLE001 — whatever it failed with is the element's report.
+                context.fail(element, exception)
+
+                continue
+
+            if isinstance(made, list):
+                produced.extend(made)
+            elif made is not None:
+                produced.append(made)
+
+        return produced
+
+    def get_output_content_type(self, config: dict[str, Any], port: str = PORT_NAME_DEFAULT) -> str | None:
+        """The content type the elements of an output port carry; an
+        operator whose type depends on its config says it here. None when
+        it depends on what is read: the element keeps the type it came with."""
+        found = self.get_port_output(port)
+        return found.get_content_type() if found else None
+
+    def element(
+        self,
+        key: str,
+        data: Any,
+        content_type: str | None = None,
+        parents: list[str] | None = None,
+        metadata: dict[str, Any] | None = None,
+        config: dict[str, Any] | None = None,
+    ) -> Element:
+        """A new element, its type the default output port's unless given."""
+        from syrtis_python_dataflow.common.element import Element
+
+        return Element(
+            content_type=content_type or self.get_output_content_type(config or {}) or "content/text",
+            data=data,
+            key=key,
+            metadata=metadata or {},
+            parents=parents or [],
+        )
+
+    def derive(
+        self,
+        source: Element,
+        data: Any,
+        content_type: str | None = None,
+        metadata: dict[str, Any] | None = None,
+        key: str | None = None,
+        config: dict[str, Any] | None = None,
+    ) -> Element:
+        """An element made from another: the same key unless given — a
+        paragraph of `moby-dick` is `moby-dick/3` —, descending from it, its
+        metadata carried over, its type the source's own when the operator
+        passes through, else the output port's, else the source's."""
+        kept = source.content_type if self.transmits(config) else None
+
+        return self.element(
+            key=key if key is not None else source.key,
+            data=data,
+            content_type=content_type or kept or self.get_output_content_type(config or {}) or source.content_type,
+            parents=[source.key],
+            metadata={**source.metadata, **(metadata or {})},
+            config=config,
+        )

syrtis_python_dataflow/common/abstract_projection_storage.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from wexample_helpers.classes.abstract_method import abstract_method
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.decorator.base_class import base_class
+
+if TYPE_CHECKING:
+    from syrtis_python_dataflow.common.element import Element
+    from syrtis_python_dataflow.common.projection_index import ProjectionIndex
+    from syrtis_python_dataflow.common.stored_element import StoredElement
+
+
+@base_class
+class AbstractProjectionStorage(BaseClass):
+    """Where a pipeline's projections are kept between runs: an index per
+    projection, and each element's content by key. What a run computes is
+    written here; what it finds unchanged is read back from here."""
+
+    @abstract_method
+    def read_index(self, pipeline: str, projection: str) -> ProjectionIndex:
+        """The projection as it stands; empty before any run."""
+
+    @abstract_method
+    def write_index(self, pipeline: str, projection: str, index: ProjectionIndex) -> None:
+        pass
+
+    @abstract_method
+    def read_content(self, pipeline: str, projection: str, key: str) -> bytes:
+        pass
+
+    @abstract_method
+    def write_content(self, pipeline: str, projection: str, key: str, content: bytes) -> None:
+        pass
+
+    @abstract_method
+    def delete_content(self, pipeline: str, projection: str, key: str) -> None:
+        pass
+
+    def read_element(self, pipeline: str, projection: str, stored: StoredElement) -> Element:
+        """An element read back whole from what the index keeps of it."""
+        from syrtis_python_dataflow.common.element import Element
+        from syrtis_python_dataflow.helper.content import content_from_bytes
+
+        return Element(
+            content_type=stored.content_type,
+            data=content_from_bytes(self.read_content(pipeline, projection, stored.key), stored.content_type),
+            key=stored.key,
+            metadata=dict(stored.metadata),
+            parents=list(stored.parents),
+        )
+
+    def read_elements(self, pipeline: str, projection: str) -> list[Element]:
+        """Every element of a projection, in its order."""
+        index = self.read_index(pipeline, projection)
+        return [self.read_element(pipeline, projection, stored) for stored in index.elements.values()]

syrtis_python_dataflow/common/element.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import Any
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+from syrtis_python_dataflow.helper.content import ElementData, content_to_bytes
+
+
+@base_class
+class Element(BaseClass):
+    """One thing of a projection: a book, a paragraph, a summary. Its key is
+    its identity, posed by the operator that first makes it and kept by what
+    derives from it — a paragraph of `moby-dick` is `moby-dick/12`, its
+    summary keeps that key —; its parents say which elements it was made
+    from."""
+
+    content_type: str = public_field(description="The Syrtis content type its data follows, e.g. `content/text`.")
+    data: ElementData = public_field(description="What it holds: text, bytes, or a JSON structure.")
+    key: str = public_field(description="Its name in the projection, unique there.")
+    metadata: dict[str, Any] = public_field(
+        factory=dict, description="What goes with it without being it: a chapter number, a source url."
+    )
+    parents: list[str] = public_field(
+        factory=list, description="The keys of the elements it was made from, in the projections upstream."
+    )
+
+    def to_bytes(self) -> bytes:
+        return content_to_bytes(self.data)
+
+    def __repr__(self) -> str:
+        return f"Element(key={self.key!r}, content_type={self.content_type!r})"

syrtis_python_dataflow/common/executor.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+if TYPE_CHECKING:
+    from syrtis_python_dataflow.common.abstract_projection_storage import AbstractProjectionStorage
+    from syrtis_python_dataflow.common.element import Element
+    from syrtis_python_dataflow.common.pipeline import Pipeline
+    from syrtis_python_dataflow.common.run_report import RunReport
+    from syrtis_python_dataflow.common.stage import Stage
+    from syrtis_python_dataflow.common.stage_input import StageInput
+    from syrtis_python_dataflow.common.stage_report import StageReport
+
+
+@base_class
+class Executor(BaseClass):
+    """Runs a pipeline, stage after stage, from the start down: each
+    operator given the whole of what its ports read, its elements written
+    in the stage's place of the run before. An element the operator fails
+    on (`each`, `context.fail`) is reported and left out, the others kept;
+    a stage whose operator fails whole is reported and left empty, the
+    stages reading it read nothing. The run goes on."""
+
+    host: Any = public_field(default=None, description="What every operator is handed in its context: a kernel, a wallet.")
+    storage: AbstractProjectionStorage = public_field(description="Where the projections are kept between runs.")
+    listener: Callable[[str, Stage, StageReport | None], None] | None = public_field(
+        default=None,
+        description="Told of each stage as it starts (`stage.start`, no report yet) and ends (`stage.done`, its report): whoever follows the run.",
+    )
+
+    def run(self, pipeline: Pipeline) -> RunReport:
+        from syrtis_python_dataflow.common.run_report import RunReport
+
+        report = RunReport(pipeline=pipeline.name)
+        for stage in pipeline.get_stages_ordered():
+            self._tell("stage.start", stage, None)
+            stage_report = self._run_stage(pipeline, stage)
+            report.stages.append(stage_report)
+            self._tell("stage.done", stage, stage_report)
+
+        return report
+
+    def _tell(self, event: str, stage: Stage, report: StageReport | None) -> None:
+        if self.listener is not None:
+            self.listener(event, stage, report)
+
+    def _run_stage(self, pipeline: Pipeline, stage: Stage) -> StageReport:
+        from syrtis_python_dataflow.common.operator_context import OperatorContext
+        from syrtis_python_dataflow.common.stage_report import StageReport
+
+        report = StageReport(name=stage.name)
+        # Each port the union of what its links bring, in the links' order; a
+        # link naming a type in place of a port brings the elements of that
+        # type alone.
+        inputs = {
+            port.name: [
+                element
+                for read in stage.inputs.get(port.name, [])
+                for element in self._read(pipeline, read)
+            ]
+            for port in stage.operator.get_ports_input()
+        }
+
+        context = OperatorContext(host=self.host, stage_name=stage.name)
+
+        try:
+            produced = list(stage.operator.run(inputs, stage.config, context))
+            # What it made, and the elements it failed on.
+            report.errors.extend(context.errors)
+            report.failed = len(context.errors)
+        except Exception as exception:  # noqa: BLE001 — whatever the operator failed with is the stage's report.
+            report.failed = 1
+            report.errors.append({"key": None, "reason": f"{type(exception).__name__}: {exception}"})
+            produced = []
+
+        for position, element in enumerate(produced):
+            if not element.key:
+                element.key = str(position)
+
+        self._write(pipeline, stage, produced)
+        report.computed = len(produced)
+        report.total = len({element.key for element in produced})
+
+        return report
+
+    def _read(self, pipeline: Pipeline, read: StageInput) -> list[Element]:
+        elements = self.storage.read_elements(pipeline.name, read.stage)
+        upstream = pipeline.get_stage(read.stage)
+
+        if upstream and upstream.operator.get_port_output(read.port) is None:
+            return [element for element in elements if element.content_type == read.port]
+
+        return elements
+
+    def _write(self, pipeline: Pipeline, stage: Stage, elements: list[Element]) -> None:
+        """The stage's elements written in place of the run before's: two
+        under one key, the last one stands."""
+        from syrtis_python_dataflow.common.projection_index import ProjectionIndex
+        from syrtis_python_dataflow.common.stored_element import StoredElement
+
+        before = self.storage.read_index(pipeline.name, stage.name)
+        index = ProjectionIndex()
+
+        for element in elements:
+            self.storage.write_content(pipeline.name, stage.name, element.key, element.to_bytes())
+            index.elements[element.key] = StoredElement(
+                content_type=element.content_type,
+                key=element.key,
+                metadata=dict(element.metadata),
+                parents=list(element.parents),
+            )
+
+        for key in before.elements:
+            if key not in index.elements:
+                self.storage.delete_content(pipeline.name, stage.name, key)
+
+        self.storage.write_index(pipeline.name, stage.name, index)

syrtis_python_dataflow/common/file_projection_storage.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+from syrtis_python_dataflow.common.abstract_projection_storage import AbstractProjectionStorage
+
+if TYPE_CHECKING:
+    from syrtis_python_storage.common.abstract_file_storage import AbstractFileStorage
+
+    from syrtis_python_dataflow.common.projection_index import ProjectionIndex
+
+# Where a projection's index stands among its contents.
+FILE_PROJECTION_STORAGE_INDEX_NAME: str = "index.json"
+
+
+@base_class
+class FileProjectionStorage(AbstractProjectionStorage):
+    """Projections kept in a syrtis-python-storage file storage — a directory,
+    a bucket —: `<pipeline>/<projection>/index.json` and one file per
+    element, named by a digest of its key so that `a` and `a/b` never
+    collide as a file and a directory would."""
+
+    storage: AbstractFileStorage = public_field(description="The file storage the projections are written to.")
+
+    def read_index(self, pipeline: str, projection: str) -> ProjectionIndex:
+        from syrtis_python_dataflow.common.projection_index import ProjectionIndex
+
+        path = self._index_path(pipeline, projection)
+        if not self.storage.exists(path):
+            return ProjectionIndex()
+
+        return ProjectionIndex.from_dict(json.loads(self.storage.read(path).decode("utf-8")))
+
+    def write_index(self, pipeline: str, projection: str, index: ProjectionIndex) -> None:
+        self.storage.write(
+            self._index_path(pipeline, projection),
+            json.dumps(index.to_dict(), ensure_ascii=False, indent=2).encode("utf-8"),
+        )
+
+    def read_content(self, pipeline: str, projection: str, key: str) -> bytes:
+        return self.storage.read(self._content_path(pipeline, projection, key))
+
+    def write_content(self, pipeline: str, projection: str, key: str, content: bytes) -> None:
+        self.storage.write(self._content_path(pipeline, projection, key), content)
+
+    def delete_content(self, pipeline: str, projection: str, key: str) -> None:
+        self.storage.delete(self._content_path(pipeline, projection, key))
+
+    def _index_path(self, pipeline: str, projection: str) -> str:
+        return f"{pipeline}/{projection}/{FILE_PROJECTION_STORAGE_INDEX_NAME}"
+
+    def _content_path(self, pipeline: str, projection: str, key: str) -> str:
+        from syrtis_python_dataflow.helper.fingerprint import fingerprint_bytes
+
+        return f"{pipeline}/{projection}/{fingerprint_bytes(key.encode('utf-8'))[:32]}"

syrtis_python_dataflow/common/memory_projection_storage.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+from syrtis_python_dataflow.common.abstract_projection_storage import AbstractProjectionStorage
+
+if TYPE_CHECKING:
+    from syrtis_python_dataflow.common.projection_index import ProjectionIndex
+
+
+@base_class
+class MemoryProjectionStorage(AbstractProjectionStorage):
+    """Projections kept in memory: for tests, and for a run whose result
+    is read at once and never again."""
+
+    contents: dict[str, bytes] = public_field(factory=dict, description="Contents by pipeline, projection and key.")
+    indexes: dict[str, ProjectionIndex] = public_field(factory=dict, description="Indexes by pipeline and projection.")
+
+    def read_index(self, pipeline: str, projection: str) -> ProjectionIndex:
+        from syrtis_python_dataflow.common.projection_index import ProjectionIndex
+
+        index = self.indexes.get(f"{pipeline}/{projection}")
+        # Read back as written: a run's own edits never reach the storage before it writes.
+        return ProjectionIndex.from_dict(index.to_dict()) if index else ProjectionIndex()
+
+    def write_index(self, pipeline: str, projection: str, index: ProjectionIndex) -> None:
+        from syrtis_python_dataflow.common.projection_index import ProjectionIndex
+
+        self.indexes[f"{pipeline}/{projection}"] = ProjectionIndex.from_dict(index.to_dict())
+
+    def read_content(self, pipeline: str, projection: str, key: str) -> bytes:
+        return self.contents[f"{pipeline}/{projection}/{key}"]
+
+    def write_content(self, pipeline: str, projection: str, key: str, content: bytes) -> None:
+        self.contents[f"{pipeline}/{projection}/{key}"] = content
+
+    def delete_content(self, pipeline: str, projection: str, key: str) -> None:
+        self.contents.pop(f"{pipeline}/{projection}/{key}", None)

syrtis_python_dataflow/common/operator_context.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+if TYPE_CHECKING:
+    from syrtis_python_dataflow.common.element import Element
+
+
+@base_class
+class OperatorContext(BaseClass):
+    """What an operator is handed besides its inputs and its config: the
+    stage it runs for, whatever the host gives every operator — a kernel, a
+    wallet —, and where it says which elements it failed on."""
+
+    errors: list[dict[str, str | None]] = public_field(factory=list, description="The elements the operator could not make anything of, each its key and why: the stage's report.")
+    host: Any = public_field(default=None, description="What the host lends its operators; nothing in a bare run.")
+    stage_name: str = public_field(description="The stage the operator runs for.")
+
+    def fail(self, element: Element, exception: BaseException) -> None:
+        """An element the operator could not make anything of: said in the
+        stage's report, the others going on."""
+        self.errors.append({"key": element.key, "reason": f"{type(exception).__name__}: {exception}"})

syrtis_python_dataflow/common/pipeline.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+if TYPE_CHECKING:
+    from syrtis_python_dataflow.common.stage import Stage
+
+
+@base_class
+class Pipeline(BaseClass):
+    """Stages reading one another: one pipe, branches allowed, no loop.
+    Validated once, so that a run never meets a stage reading a port that
+    does not exist or a type its operator refuses. A stage reading nothing
+    is where the pipe starts — or a leak the caller chose to leave in."""
+
+    name: str = public_field(description="Its name, under which its projections are kept.")
+    stages: list[Stage] = public_field(factory=list, description="Its stages, in any order.")
+
+    def get_stage(self, name: str) -> Stage | None:
+        return next((stage for stage in self.stages if stage.name == name), None)
+
+    def get_stages_ordered(self) -> list[Stage]:
+        """The stages from the start down, each after every stage it reads;
+        validated first."""
+        self.validate()
+
+        ordered: list[Stage] = []
+        remaining = {stage.name: stage for stage in self.stages}
+        while remaining:
+            ready = [
+                stage for stage in remaining.values() if all(read.stage not in remaining for read in stage.get_reads())
+            ]
+            # Validation refused cycles: something is always ready.
+            ready.sort(key=lambda stage: stage.name)
+            for stage in ready:
+                ordered.append(stage)
+                del remaining[stage.name]
+
+        return ordered
+
+    def get_readers(self, name: str) -> list[Stage]:
+        """The stages reading the one named."""
+        return [stage for stage in self.stages if any(read.stage == name for read in stage.get_reads())]
+
+    def validate(self) -> None:
+        from syrtis_python_dataflow.exception.pipeline_definition_exception import (
+            PipelineDefinitionException,
+        )
+
+        names = [stage.name for stage in self.stages]
+        if len(names) != len(set(names)):
+            raise PipelineDefinitionException(pipeline_name=self.name, problem="two stages share a name")
+
+        for stage in self.stages:
+            operator = stage.operator
+            ports_input = {port.name: port for port in operator.get_ports_input()}
+
+            for port_name, reads in stage.inputs.items():
+                port = ports_input.get(port_name)
+                if port is None:
+                    self._refuse(stage, f"its operator '{operator.name}' has no input port '{port_name}'")
+
+                for read in reads:
+                    upstream = self.get_stage(read.stage)
+                    if upstream is None:
+                        self._refuse(stage, f"port '{port_name}' reads stage '{read.stage}', which does not exist")
+                    if upstream is stage:
+                        self._refuse(stage, "it reads itself")
+                    # A type named in place of a port, on an operator passing
+                    # through: the elements of that type alone.
+                    if upstream.operator.get_port_output(read.port) is None and not upstream.operator.transmits(upstream.config):
+                        self._refuse(stage, f"port '{port_name}' reads port '{read.port}' of stage '{read.stage}', which its operator does not have")
+                    given = read.port if upstream.operator.get_port_output(read.port) is None else upstream.operator.get_output_content_type(upstream.config, read.port)
+                    if given is not None and not port.accepts(given):
+                        self._refuse(
+                            stage,
+                            f"port '{port_name}' takes {', '.join(port.content_types)} but stage '{read.stage}' gives {given}",
+                        )
+
+        self._refuse_cycles()
+
+    def _refuse(self, stage: Stage, problem: str) -> None:
+        from syrtis_python_dataflow.exception.pipeline_definition_exception import (
+            PipelineDefinitionException,
+        )
+
+        raise PipelineDefinitionException(pipeline_name=self.name, problem=problem, stage_name=stage.name)
+
+    def _refuse_cycles(self) -> None:
+        from syrtis_python_dataflow.exception.pipeline_definition_exception import (
+            PipelineDefinitionException,
+        )
+
+        # Kahn: what cannot be ordered is in a cycle.
+        remaining = {stage.name: {read.stage for read in stage.get_reads()} for stage in self.stages}
+        while remaining:
+            ready = [name for name, reads in remaining.items() if not reads & set(remaining)]
+            if not ready:
+                raise PipelineDefinitionException(
+                    pipeline_name=self.name,
+                    problem=f"stages {', '.join(sorted(remaining))} read one another in a cycle",
+                )
+            for name in ready:
+                del remaining[name]

syrtis_python_dataflow/common/port.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+
+@base_class
+class Port(BaseClass):
+    """Where elements enter or leave an operator. An input port says the
+    content types it takes — none, any —; an output port the one it gives."""
+
+    content_types: list[str] = public_field(
+        factory=list, description="The content types an input takes, or the single one an output gives."
+    )
+    name: str = public_field(description="Its name, unique among the operator's ports of its direction.")
+    title: str | None = public_field(default=None, description="Its name as a person reads it.")
+
+    def accepts(self, content_type: str) -> bool:
+        return not self.content_types or content_type in self.content_types
+
+    def get_content_type(self) -> str | None:
+        """What an output port gives; None when it does not say."""
+        return self.content_types[0] if self.content_types else None

syrtis_python_dataflow/common/projection_index.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+if TYPE_CHECKING:
+    from syrtis_python_dataflow.common.stored_element import StoredElement
+
+
+@base_class
+class ProjectionIndex(BaseClass):
+    """What a projection holds, as its storage keeps it: its elements in order."""
+
+    elements: dict[str, StoredElement] = public_field(factory=dict, description="Its elements by key, in order.")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"elements": [element.to_dict() for element in self.elements.values()]}
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ProjectionIndex:
+        from syrtis_python_dataflow.common.stored_element import StoredElement
+
+        elements = [StoredElement.from_dict(element) for element in data.get("elements") or []]
+        return cls(elements={element.key: element for element in elements})

syrtis_python_dataflow/common/run_report.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+if TYPE_CHECKING:
+    from syrtis_python_dataflow.common.stage_report import StageReport
+
+
+@base_class
+class RunReport(BaseClass):
+    """What a run of a pipeline did, stage by stage."""
+
+    pipeline: str = public_field(description="The pipeline run.")
+    stages: list[StageReport] = public_field(factory=list, description="One report per stage, in the order run.")
+
+    def get_stage(self, name: str) -> StageReport | None:
+        return next((stage for stage in self.stages if stage.name == name), None)
+
+    def has_failures(self) -> bool:
+        return any(stage.failed for stage in self.stages)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"pipeline": self.pipeline, "stages": [stage.to_dict() for stage in self.stages]}

syrtis_python_dataflow/common/stage.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+if TYPE_CHECKING:
+    from syrtis_python_dataflow.common.abstract_operator import AbstractOperator
+    from syrtis_python_dataflow.common.stage_input import StageInput
+
+
+@base_class
+class Stage(BaseClass):
+    """One projection of a pipeline: the elements an operator computes, with
+    its config, from the stages it reads — any number per input port of the
+    operator, their elements read together. Keyed by its name, under which
+    its elements are kept."""
+
+    config: dict[str, Any] = public_field(factory=dict, description="What the operator is configured with here.")
+    inputs: dict[str, list[StageInput]] = public_field(
+        factory=dict, description="What each input port of the operator reads, by port name: the links reaching it, in order."
+    )
+    name: str = public_field(description="Its name, unique in the pipeline.")
+    operator: AbstractOperator = public_field(description="What computes its elements.")
+    title: str | None = public_field(default=None, description="Its name as a person reads it.")
+
+    def get_reads(self) -> list[StageInput]:
+        """Every link reaching it, whatever the port."""
+        return [read for reads in self.inputs.values() for read in reads]

syrtis_python_dataflow/common/stage_input.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+from syrtis_python_dataflow.const.port import PORT_NAME_DEFAULT
+
+
+@base_class
+class StageInput(BaseClass):
+    """Where a stage reads from: a port of a stage upstream."""
+
+    port: str = public_field(default=PORT_NAME_DEFAULT, description="The output port of the stage read.")
+    stage: str = public_field(description="The stage read.")

syrtis_python_dataflow/common/stage_report.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import Any
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+
+@base_class
+class StageReport(BaseClass):
+    """What a run did on one stage: how many elements it made, and what it
+    failed on — elements, each its key and why, the others kept; or the
+    stage whole, its key none, the stage then holding nothing."""
+
+    computed: int = public_field(default=0, description="Elements made in this run.")
+    errors: list[dict[str, str | None]] = public_field(factory=list, description="What it failed on: an element each — its key, why —, or the stage whole, its key none.")
+    failed: int = public_field(default=0, description="The elements it failed on; 1, its key none, when the operator failed whole and the stage was left empty.")
+    name: str = public_field(description="The stage.")
+    total: int = public_field(default=0, description="Elements the projection holds after the run.")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "computed": self.computed,
+            "failed": self.failed,
+            "total": self.total,
+            "errors": list(self.errors),
+        }

syrtis_python_dataflow/common/stored_element.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Any
+
+from wexample_helpers.classes.base_class import BaseClass
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+
+
+@base_class
+class StoredElement(BaseClass):
+    """What a projection's index keeps of an element: enough to read it back
+    without the run that made it."""
+
+    content_type: str = public_field(description="The content type of its data.")
+    key: str = public_field(description="Its key in the projection.")
+    metadata: dict[str, Any] = public_field(factory=dict, description="What goes with it.")
+    parents: list[str] = public_field(factory=list, description="The keys it was made from, upstream.")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "content_type": self.content_type,
+            "key": self.key,
+            "metadata": self.metadata,
+            "parents": self.parents,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> StoredElement:
+        return cls(
+            content_type=data["content_type"],
+            key=data["key"],
+            metadata=data.get("metadata") or {},
+            parents=list(data.get("parents") or []),
+        )

syrtis_python_dataflow/const/config.py

@@ -0,0 +1,4 @@
+from __future__ import annotations
+
+# A stage's config saying whether what it gives keeps the type of what it read.
+CONFIG_TRANSMIT_CONTENT_TYPE: str = "transmit_content_type"

syrtis_python_dataflow/const/port.py

@@ -0,0 +1,4 @@
+from __future__ import annotations
+
+# The port an element goes through when the operator names none.
+PORT_NAME_DEFAULT: str = "default"

syrtis_python_dataflow/exception/pipeline_definition_exception.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from typing import ClassVar
+
+from wexample_helpers.classes.field import public_field
+from wexample_helpers.decorator.base_class import base_class
+from wexample_helpers.exception.undefined_exception import UndefinedException
+
+
+@base_class
+class PipelineDefinitionException(UndefinedException):
+    """The pipeline cannot run as defined: a stage names an input that does
+    not exist, a port its operator lacks, a content type the next operator
+    refuses, or the stages form a cycle."""
+
+    error_code: ClassVar[str] = "DATAFLOW_PIPELINE_DEFINITION"
+    pipeline_name: str = public_field(description="The pipeline at fault.")
+    problem: str = public_field(description="What is wrong with it.")
+    stage_name: str | None = public_field(default=None, description="The stage at fault, when one is.")
+
+    def _build_message(self) -> str:
+        where = f"stage '{self.stage_name}' of pipeline '{self.pipeline_name}'" if self.stage_name else f"pipeline '{self.pipeline_name}'"
+        return f"In {where}: {self.problem}"

syrtis_python_dataflow/helper/content.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+ElementData = bytes | str | dict[str, Any] | list[Any]
+
+
+def content_to_bytes(data: ElementData) -> bytes:
+    """What an element holds, as it is kept: text as utf-8, bytes as they
+    are, a structure as JSON."""
+    if isinstance(data, bytes):
+        return data
+    if isinstance(data, str):
+        return data.encode("utf-8")
+    return json.dumps(data, ensure_ascii=False, sort_keys=True).encode("utf-8")
+
+
+def content_from_bytes(content: bytes, content_type: str) -> ElementData:
+    """What was kept, read back as the content type says: JSON for a
+    structured type, text for a textual one, bytes otherwise."""
+    if content_type_is_structured(content_type):
+        return json.loads(content.decode("utf-8"))
+    if content_type_is_text(content_type):
+        return content.decode("utf-8")
+    return content
+
+
+def content_type_is_structured(content_type: str) -> bool:
+    return content_type.endswith(("/json", "/object", "/list"))
+
+
+def content_type_is_text(content_type: str) -> bool:
+    """A type whose content is read as text: Syrtis' shapes of text and
+    whatever names itself so."""
+    return not content_type.endswith(("/binary", "/image", "/file", "/bytes"))

syrtis_python_dataflow/helper/fingerprint.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+
+def fingerprint_bytes(content: bytes) -> str:
+    return hashlib.sha256(content).hexdigest()
+
+
+def fingerprint_json(value: Any) -> str:
+    """The same value written the same way gives the same fingerprint,
+    whatever the order its keys were given in."""
+    return fingerprint_bytes(
+        json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
+    )
+
+
+def fingerprint_parts(*parts: str) -> str:
+    """One fingerprint of several, each kept apart so that `ab` + `c` and
+    `a` + `bc` do not meet."""
+    return fingerprint_bytes("\x1f".join(parts).encode("utf-8"))

{syrtis_python_dataflow-1.0.1.dist-info → syrtis_python_dataflow-2.0.0.dist-info}/METADATA

@@ -1,12 +1,14 @@
 Metadata-Version: 2.1
 Name: syrtis-python-dataflow
-Version: 1.0.1
+Version: 2.0.0
 Author-Email: weeger <contact@syrtis.ai>
 License: MIT
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
+Requires-Dist: syrtis-python-storage>=2.0.0
+Requires-Dist: wexample-helpers>=20.6.0
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -14,16 +16,23 @@ Description-Content-Type: text/markdown
 
 # python-dataflow
 
-Version: 1.0.1
+Version: 2.0.0
 
 ## Table of Contents
 
+- [Introduction](#introduction)
 - [Install](#install)
 - [Tests](#tests)
 - [Roadmap](#roadmap)
 - [Status Compatibility](#status-compatibility)
 - [Useful Links](#useful-links)
 
+`syrtis-python-dataflow` runs a pipeline: stages reading one another from the sources down, each a projection of elements an operator computes — a book, its paragraphs, their summaries, the chapters', the book's. It is the engine of the data side of Syrtis, where the scenarios' engine holds a conversation: here nothing reasons, and no session is kept; collections are read, transformed and kept, in a storage.
+
+What it does that the scenarios' engine does not: it works on collections, element by element; it remembers what every element was computed from, so that a run computes only what changed — a stage's config, a source refreshed — and keeps the rest; it runs a stage's elements on several threads; an element an operator fails on is reported and left out, the stage going on.
+
+It knows nothing of Syrtis' database: the host — the core — builds the pipeline from what it keeps, hands the operators what they need, and reads the report back.
+
 ## Installation
 
 ```bash

syrtis_python_dataflow-2.0.0.dist-info/RECORD

@@ -0,0 +1,32 @@
+syrtis_python_dataflow-2.0.0.dist-info/METADATA,sha256=hyHAbYp-PFuNQRy1sUiAsgWTZvc2gRZUprRNEAuvq7I,3885
+syrtis_python_dataflow-2.0.0.dist-info/WHEEL,sha256=IuIk3wbaEnFH1cx75iM2R1CmliRI3dFLXTbRakgvgFQ,90
+syrtis_python_dataflow-2.0.0.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
+syrtis_python_dataflow/__init__.py,sha256=JMYsyGKsdT609sZ36Os7-HXsedKCtTMpoPo1gUJsDTw,1224
+syrtis_python_dataflow/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+syrtis_python_dataflow/common/abstract_operator.py,sha256=axeia69wb0KbJhx6kE--ocoAEVAkMOvhoUlHAAmhMy8,6028
+syrtis_python_dataflow/common/abstract_projection_storage.py,sha256=GXUsG2S8iMcMVgBDMrWgV43qs4hUtgUq4PhNexlqqNs,2330
+syrtis_python_dataflow/common/element.py,sha256=ht5E39dfqRGMGRXtPwDZBAJaR7RXUbNmkygkiVzOEuI,1471
+syrtis_python_dataflow/common/executor.py,sha256=Nf4wdKxbuTxzamz3UPAYEMeFizGeeeM9wYiaqFl6t4s,5474
+syrtis_python_dataflow/common/file_projection_storage.py,sha256=DR8YVQCuyJ2T1y-zf7nmO-ho4021Vd1q3ut2G8n5uRY,2596
+syrtis_python_dataflow/common/memory_projection_storage.py,sha256=zuaZbMyLJNuJf1Jexu8s_wX9tezOfdAIuSHCuCysFOQ,1932
+syrtis_python_dataflow/common/operator_context.py,sha256=ODVem5xrpCkiwwwkiHJq7rAm4jXdlGe_j6WfKaa9pnY,1275
+syrtis_python_dataflow/common/pipeline.py,sha256=LgCQ1-L3Yr5sJraL6oKCuq-Fi8P0Tw8cTJ44oUzTSvU,5017
+syrtis_python_dataflow/common/port.py,sha256=75oxG_6CQcTZWM4vcOAgujfHt8AM9Jk2YZYq50g1_Uc,1074
+syrtis_python_dataflow/common/projection_index.py,sha256=ojCaS9av6uhMiD-UQgJwX2oTuypks9W3dJjidg4AheU,1054
+syrtis_python_dataflow/common/run_report.py,sha256=ONDVDwJHpMejzj5IW0m7e2mso11ZhbZzUMFk42UUaX0,994
+syrtis_python_dataflow/common/stage.py,sha256=MPT0EWjrzCBikWFKLHyBe8-E-kCkHNVbLI4MLwRxRfA,1462
+syrtis_python_dataflow/common/stage_input.py,sha256=xWPPZDoGjGfkZiTQ5kyzpYAvPNkKsLR5Y-cdlLJjA2E,551
+syrtis_python_dataflow/common/stage_report.py,sha256=ipsZ06MHFBfGDRzc1QpncFfDG0u_STMjGzx3girajTw,1322
+syrtis_python_dataflow/common/stored_element.py,sha256=0YgcmQ5XijVE18CFS8XZ7RVDwQb0DbW-6u88chFZOk0,1254
+syrtis_python_dataflow/const/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+syrtis_python_dataflow/const/config.py,sha256=3QV_JJWJtTilAOU5oN0jh4vUPfdRNY42idBn8eM3bRU,176
+syrtis_python_dataflow/const/port.py,sha256=4U_sfjIda6XfBMFaQQDbUl3cV8uYMhbNFDRmv_v2m5Y,136
+syrtis_python_dataflow/exception/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+syrtis_python_dataflow/exception/pipeline_definition_exception.py,sha256=ZXdn7bTk4a6ltAw50fmBfUE7nSs35kAf-XL69oCUbtc,1060
+syrtis_python_dataflow/helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+syrtis_python_dataflow/helper/content.py,sha256=JBDeEgA1C-YC3uULlQwOwd7joIgp3YqD7vXpUbqRkzs,1259
+syrtis_python_dataflow/helper/fingerprint.py,sha256=5giH4XC8g0c9Psr0joXpBUpDF8rd7C6yejcKOLJ_X-A,690
+syrtis_python_dataflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+syrtis_syrtis_python_dataflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+syrtis_syrtis_python_dataflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+syrtis_python_dataflow-2.0.0.dist-info/RECORD,,

syrtis_python_dataflow-1.0.1.dist-info/RECORD

@@ -1,6 +0,0 @@
-syrtis_python_dataflow-1.0.1.dist-info/METADATA,sha256=m1XeStBE0eRvCKhVEk9uo-W70CtnYZAiytl8q7EEnYg,2808
-syrtis_python_dataflow-1.0.1.dist-info/WHEEL,sha256=IuIk3wbaEnFH1cx75iM2R1CmliRI3dFLXTbRakgvgFQ,90
-syrtis_python_dataflow-1.0.1.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
-syrtis_syrtis_python_dataflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syrtis_syrtis_python_dataflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-syrtis_python_dataflow-1.0.1.dist-info/RECORD,,