wexample-wex-addon-process 1.0.4__py3-none-any.whl

wexample_wex_addon_process/commands/run/execute.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from wexample_cli.const.tags import ScopeTag
+from wexample_cli.decorator.command import command
+from wexample_cli.decorator.middleware import middleware
+from wexample_cli.decorator.option import option
+from wexample_wex_addon_app.middleware.app_middleware import AppMiddleware
+from wexample_wex_core.const.globals import COMMAND_TYPE_ADDON
+
+if TYPE_CHECKING:
+    from wexample_app.response.abstract_response import AbstractResponse
+    from wexample_cli.context.execution_context import ExecutionContext
+    from wexample_wex_addon_app.workdir.managed_workdir import ManagedWorkdir
+
+
+@option(
+    name="run",
+    type=str,
+    required=True,
+    description="Identity of the run to execute, which is the name of its record under `.wex/data/process_run/`.",
+)
+@middleware(middleware=AppMiddleware)
+@command(
+    type=COMMAND_TYPE_ADDON,
+    description="Run one process run, here and now, without a queue. What the worker does for a message, done by hand — which is what makes a process runnable on a repository that has no board.",
+    tags=[
+        ScopeTag.APP,
+        ScopeTag.LOCAL,
+    ],
+)
+def process__run__execute(
+    context: ExecutionContext,
+    run: str,
+    app_workdir: ManagedWorkdir | None = None,
+) -> AbstractResponse | None:
+    from wexample_app.response.success_response import SuccessResponse
+
+    from wexample_wex_addon_process.process_type.process_type_registry import (
+        ProcessTypeRegistry,
+    )
+    from wexample_wex_addon_process.runner.process_runner import ProcessRunner
+
+    runner = ProcessRunner(io=context.io, registry=ProcessTypeRegistry.default())
+
+    return SuccessResponse(
+        kernel=context.kernel,
+        data=runner.execute(
+            workdir=app_workdir.get_path(),
+            run_id=run,
+        ),
+    )

wexample_wex_addon_process/commands/worker/start.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from wexample_cli.const.tags import EffectTag, ScopeTag
+from wexample_cli.decorator.command import command
+from wexample_cli.decorator.option import option
+from wexample_wex_core.const.globals import COMMAND_TYPE_ADDON
+
+if TYPE_CHECKING:
+    from wexample_app.response.abstract_response import AbstractResponse
+    from wexample_cli.context.execution_context import ExecutionContext
+
+
+@option(
+    name="queue",
+    type=str,
+    required=False,
+    description="Queue to consume. The default is the one a board rings on to ask for a run; another one is how a treatment with its own appetite gets its own workers.",
+)
+@command(
+    type=COMMAND_TYPE_ADDON,
+    description="Consume runs from a queue until stopped. Blocking — this is the process a worker container runs, and there may be as many as the parallelism asked for.",
+    tags=[
+        EffectTag.LONG_RUNNING,
+        ScopeTag.LOCAL,
+    ],
+)
+def process__worker__start(
+    context: ExecutionContext,
+    queue: str | None = None,
+) -> AbstractResponse | None:
+    from wexample_wex_addon_process.service.process_run_service import (
+        QUEUE_PROCESS_RUN,
+        ProcessRunService,
+    )
+
+    service = ProcessRunService(
+        kernel=context.kernel,
+        queue_name=queue or QUEUE_PROCESS_RUN,
+    )
+
+    # Blocks until the process is signalled: the container's whole life is this
+    # call, and a worker that returns is a worker that has been asked to stop.
+    service.start()
+
+    return None

wexample_wex_addon_process/const/process.py

@@ -0,0 +1,34 @@
+"""What a process, a selection and a run are called on disk.
+
+Spelled once here because wex is not the only reader: the board writes these very
+keys into the same files, so a name changed on one side has to be changed on the
+other or the two stop understanding each other.
+"""
+
+from __future__ import annotations
+
+# filestate: python-constant-sort
+KIND_PROCESS: str = "process"
+KIND_PROCESS_RUN: str = "process_run"
+KIND_SELECTION: str = "selection"
+
+PROCESS_KEY_OPTIONS: str = "options"
+PROCESS_KEY_SELECTION_ID: str = "selection_id"
+PROCESS_KEY_TITLE: str = "title"
+PROCESS_KEY_TYPE: str = "type"
+
+RUN_KEY_DATA: str = "data"
+RUN_KEY_DATE_ENDED: str = "date_ended"
+RUN_KEY_DATE_STARTED: str = "date_started"
+RUN_KEY_ITEMS_DONE: str = "items_done"
+RUN_KEY_ITEMS_TOTAL: str = "items_total"
+RUN_KEY_PROCESS_ID: str = "process_id"
+RUN_KEY_STATE: str = "state"
+
+SELECTION_KEY_PATTERNS: str = "patterns"
+SELECTION_KEY_TITLE: str = "title"
+
+STATE_COMPLETE: str = "complete"
+STATE_FAILED: str = "failed"
+STATE_PENDING: str = "pending"
+STATE_RUNNING: str = "running"

wexample_wex_addon_process/process_addon_manager.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from typing import Any
+
+from wexample_helpers.decorator.base_class import base_class
+from wexample_wex_core.common.abstract_addon_manager import AbstractAddonManager
+
+
+@base_class
+class ProcessAddonManager(AbstractAddonManager):
+    @classmethod
+    def get_package_module(cls) -> Any:
+        import wexample_wex_addon_process
+
+        return wexample_wex_addon_process

wexample_wex_addon_process/process_type/abstract_process_type.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from wexample_helpers.classes.abstract_method import abstract_method
+
+if TYPE_CHECKING:
+    from wexample_wex_addon_process.process_type.process_context import ProcessContext
+
+
+class AbstractProcessType:
+    """One treatment a process can name.
+
+    A type is a name and a `run()`. It is deliberately an ordinary class with no
+    kernel and no state: what it needs comes in the context, and a type
+    contributed from a bundle should be readable by whoever wrote the bundle
+    rather than by whoever wrote wex.
+
+    The name is prefixed by the family the type belongs to — `filestate:mode`,
+    `agent:reviewer` — so that what runs a treatment is read off the name before
+    anything is loaded. A type is free to set it on the class when there is one
+    of it, or on the instance when a family makes many.
+    """
+    label: str
+    name: str
+
+    @abstract_method
+    def run(self, context: ProcessContext) -> dict[str, Any]:
+        """Does the work, and answers what is worth keeping about it.
+
+        What is returned goes into the run's `data`, as it is. Raising is how a
+        type says it could not go on: the runner writes the failure down, so
+        nothing here has to know what a failed run looks like.
+        """

wexample_wex_addon_process/process_type/family/abstract_process_type_family.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from wexample_helpers.classes.abstract_method import abstract_method
+
+if TYPE_CHECKING:
+    from wexample_wex_addon_process.process_type.abstract_process_type import (
+        AbstractProcessType,
+    )
+
+
+class AbstractProcessTypeFamily:
+    """What a whole way of treating things brings, under one prefix.
+
+    A family is not a branch taken at run time: the name written in a process
+    record already says which family answers for it. A family only declares
+    names, which is why it can declare a great many of them without anything
+    being loaded.
+    """
+
+    prefix: ClassVar[str]
+
+    @abstract_method
+    def types(self) -> list[AbstractProcessType]:
+        """Every type this family answers for, named with its prefix."""

wexample_wex_addon_process/process_type/family/file_state_process_type_family.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from wexample_wex_addon_process.process_type.family.abstract_process_type_family import (
+    AbstractProcessTypeFamily,
+)
+
+if TYPE_CHECKING:
+    from wexample_config.config_option.abstract_config_option import (
+        AbstractConfigOption,
+    )
+
+    from wexample_wex_addon_process.process_type.abstract_process_type import (
+        AbstractProcessType,
+    )
+
+
+class FileStateProcessTypeFamily(AbstractProcessTypeFamily):
+    """Everything filestate can do to a file, as treatments a process may name.
+
+    The list is not written: it is read off filestate's own options, so an option
+    added there is a treatment available here the same day, under the very name
+    filestate gives it.
+    """
+
+    prefix: ClassVar[str] = "filestate"
+
+    def types(self) -> list[AbstractProcessType]:
+        from wexample_filestate.options_provider.default_options_provider import (
+            DefaultOptionsProvider,
+        )
+
+        from wexample_wex_addon_process.process_type.filestate.file_state_option_process_type import (
+            FileStateOptionProcessType,
+        )
+        from wexample_wex_addon_process.process_type.filestate.file_state_report_process_type import (
+            FileStateReportProcessType,
+        )
+
+        types: list[AbstractProcessType] = [FileStateReportProcessType()]
+
+        for option_class in DefaultOptionsProvider.get_options():
+            if self._treats(option_class):
+                types.append(FileStateOptionProcessType(option_class))
+
+        return types
+
+    def _treats(self, option_class: type[AbstractConfigOption]) -> bool:
+        """Whether that option is a treatment rather than a piece of structure.
+
+        Asked of the option itself: one that can produce an operation is
+        something that can be done to a file, one that cannot is how a tree is
+        described. `children` and `class` are not treatments; `mode` is.
+        """
+        from wexample_filestate.option.mixin.option_mixin import OptionMixin
+
+        return (
+            option_class.create_required_operation
+            is not OptionMixin.create_required_operation
+        )

wexample_wex_addon_process/process_type/filestate/file_state_option_process_type.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from wexample_wex_addon_process.process_type.abstract_process_type import (
+    AbstractProcessType,
+)
+
+if TYPE_CHECKING:
+    from wexample_config.config_option.abstract_config_option import (
+        AbstractConfigOption,
+    )
+
+    from wexample_wex_addon_process.process_type.process_context import ProcessContext
+
+OPTION_KEY_DRY_RUN: str = "dry_run"
+"""Written in a process's options to see what a run would do, and do none of it."""
+
+
+class FileStateOptionProcessType(AbstractProcessType):
+    """One filestate option, asked of every file a selection takes.
+
+    There is no code here per treatment. A filestate option already says what a
+    file should be — `mode: '644'`, `should_exist: false` — and already knows how
+    to turn the difference with the disk into an operation that can be described,
+    applied and undone. So a type of this family is that option under a name, and
+    the process's own options are the configuration handed to each file.
+    """
+
+    def __init__(self, option_class: type[AbstractConfigOption]) -> None:
+        self.option_class = option_class
+        self.name = f"filestate:{option_class.get_name()}"
+        self.label = option_class.get_name().replace("_", " ").capitalize()
+
+    def run(self, context: ProcessContext) -> dict[str, Any]:
+        from wexample_filestate.enum.scopes import Scope
+
+        pattern = dict(context.options)
+        dry_run = bool(pattern.pop(OPTION_KEY_DRY_RUN, False))
+
+        if self.option_class.get_name() not in pattern:
+            raise ValueError(
+                f"« {self.name} » wants « {self.option_class.get_name()} » in its "
+                f"options, which is what it would ask of each file."
+            )
+
+        target = context.selection.build_target(pattern)
+
+        # Counted before anything is done to them: a run says how far it has got
+        # against what it set out to do, and filestate reports only what it had
+        # to change.
+        context.advance(len(context.files))
+
+        result = (
+            target.dry_run(scopes=Scope.default())
+            if dry_run
+            else target.apply()
+        )
+
+        return {
+            "dry_run": dry_run,
+            "operations": [
+                {
+                    "name": operation.get_name(),
+                    "path": str(operation.target.get_path()),
+                    "description": str(operation.description),
+                }
+                for operation in result.operations
+            ],
+        }

wexample_wex_addon_process/process_type/filestate/file_state_report_process_type.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from wexample_wex_addon_process.process_type.abstract_process_type import (
+    AbstractProcessType,
+)
+
+if TYPE_CHECKING:
+    from wexample_wex_addon_process.process_type.process_context import ProcessContext
+
+
+class FileStateReportProcessType(AbstractProcessType):
+    """Says what a selection takes, and changes nothing.
+
+    The first type on purpose: it exercises the whole chain — a run asked for, a
+    selection read, files walked, progress reported, a result written back — and
+    the worst it can do to a repository is read it.
+
+    It belongs to the filestate family because it walks the same tree as its
+    siblings, but it emits no operation: filestate's own rule is that a scope
+    names something observable on disk, and a report changes nothing observable.
+    An operation with no scope would be an operation nothing could ever run.
+    """
+    label = "Report"
+    name = "filestate:report"
+
+    def run(self, context: ProcessContext) -> dict[str, Any]:
+        by_extension: dict[str, int] = {}
+        total_bytes = 0
+
+        for path in context.files:
+            extension = path.suffix or "(none)"
+            by_extension[extension] = by_extension.get(extension, 0) + 1
+            total_bytes += path.stat().st_size
+
+            context.advance()
+
+        return {
+            "files": len(context.files),
+            "bytes": total_bytes,
+            "by_extension": dict(sorted(by_extension.items())),
+        }

wexample_wex_addon_process/process_type/process_context.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from pathlib import Path
+
+    from wexample_wex_addon_process.process_type.selection import Selection
+
+
+class ProcessContext:
+    """What a type is given to do its work: the selection, the options, and a way
+    to say how far it has got.
+
+    It holds no run and no record. A type that could reach the record could write
+    anywhere in it, and what a run is in is decided by the runner alone — a type
+    says what it did, not that it is finished.
+    """
+
+    def __init__(
+        self,
+        selection: Selection,
+        options: dict[str, Any],
+        advance: Callable[[int], None],
+    ) -> None:
+        self.selection = selection
+        self.options = options
+        self._advance = advance
+        self._files: list[Path] | None = None
+
+    @property
+    def files(self) -> list[Path]:
+        """The files the selection takes, read once for the length of the run."""
+        if self._files is None:
+            self._files = self.selection.files()
+
+        return self._files
+
+    @property
+    def workdir(self) -> Path:
+        return self.selection.workdir
+
+    def advance(self, count: int = 1) -> None:
+        """Says that many more files have been dealt with."""
+        self._advance(count)

wexample_wex_addon_process/process_type/process_type_registry.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from wexample_wex_addon_process.process_type.abstract_process_type import (
+        AbstractProcessType,
+    )
+    from wexample_wex_addon_process.process_type.family.abstract_process_type_family import (
+        AbstractProcessTypeFamily,
+    )
+
+
+class ProcessTypeRegistry:
+    """The types a run may name, by the name it names them with.
+
+    A collection of families rather than a list of types: a family declares its
+    own names — filestate declares one per option it can enforce — so adding a
+    way of treating things is adding a family and not a hundred entries. A
+    process naming a type nothing declares is an error the runner reports rather
+    than something to guess at.
+    """
+
+    def __init__(self, families: list[AbstractProcessTypeFamily]) -> None:
+        self._types: dict[str, AbstractProcessType] = {}
+
+        for family in families:
+            for process_type in family.types():
+                self._types[process_type.name] = process_type
+
+    @classmethod
+    def default(cls) -> ProcessTypeRegistry:
+        from wexample_wex_addon_process.process_type.family.file_state_process_type_family import (
+            FileStateProcessTypeFamily,
+        )
+
+        return cls([FileStateProcessTypeFamily()])
+
+    def get(self, name: str) -> AbstractProcessType | None:
+        return self._types.get(name)
+
+    def names(self) -> list[str]:
+        return sorted(self._types)

wexample_wex_addon_process/process_type/selection.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from wexample_filestate.utils.file_state_manager import FileStateManager
+    from wexample_prompt.mixin.with_io_methods import WithIoMethods
+
+
+class Selection:
+    """The files a process runs on, as the patterns that name them.
+
+    Nothing here holds a list: a selection is a rule, and the files it takes are
+    read from disk each time they are asked for. What turns the rule into a tree
+    is filestate, so a type that only wants to read them and a type that wants to
+    change them walk the very same thing.
+    """
+
+    def __init__(
+        self,
+        io: WithIoMethods,
+        workdir: Path,
+        patterns: list[str],
+    ) -> None:
+        from wexample_file.common.path_matcher import PathMatcher
+
+        self.io = io
+        self.workdir = workdir
+        self.matcher = PathMatcher(patterns)
+
+    def build_target(self, pattern: dict[str, Any]) -> FileStateManager:
+        """A filestate tree of the app, holding the selected files and that config.
+
+        The config is what every taken file is asked to be — `mode: '644'`,
+        `should_exist: false` — and it is filestate that turns the difference with
+        the disk into operations.
+        """
+        from wexample_filestate.const.disk import DiskItemType
+        from wexample_filestate.option.children_filter_option import (
+            ChildrenFilterOption,
+        )
+        from wexample_filestate.utils.file_state_manager import FileStateManager
+
+        manager = FileStateManager.create_from_path(
+            io=self.io,
+            path=self.workdir,
+            configure=False,
+        )
+
+        manager.configure(
+            {
+                "children": [
+                    ChildrenFilterOption(
+                        pattern={"type": DiskItemType.FILE, **pattern},
+                        filter=self._takes,
+                        recursive=True,
+                    )
+                ]
+            }
+        )
+
+        return manager
+
+    def files(self) -> list[Path]:
+        """The files taken, asking for nothing to be done to them."""
+        return sorted(
+            child.get_path()
+            for child in self._leaves(self.build_target({}))
+            if child.get_path().is_file()
+        )
+
+    def _leaves(self, item: Any) -> list[Any]:
+        leaves = []
+
+        for child in item.get_children_list():
+            if child.is_directory():
+                leaves.extend(self._leaves(child))
+            else:
+                leaves.append(child)
+
+        return leaves
+
+    def _takes(self, path: Path) -> bool:
+        return self.matcher.matches(str(path.relative_to(self.workdir)))

wexample_wex_addon_process/record/record_file.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def read_record(path: Path) -> dict[str, Any]:
+    """The values one record holds."""
+    import yaml
+
+    return yaml.safe_load(path.read_text()) or {}
+
+
+def record_path(app_path: Path, kind: str, identity: str) -> Path:
+    """Where the record of that kind bearing that identity sits."""
+    from wexample_wex_core.const.data import RECORD_EXTENSION
+    from wexample_wex_core.helper.data_record import data_kind_dir
+
+    return data_kind_dir(app_path, kind) / f"{identity}{RECORD_EXTENSION}"
+
+
+def write_record(path: Path, values: dict[str, Any]) -> None:
+    """Merges those values into the record and leaves the rest alone.
+
+    A run record is written from both ends — the board opens it, a worker fills
+    it in — so it is read, merged and dumped rather than rebuilt. Two spaces, no
+    flow style and unicode left alone, like every `.wex` file the board writes,
+    so a rewrite does not read as a change nobody made.
+    """
+    import yaml
+
+    current = read_record(path) if path.is_file() else {}
+    current.update(values)
+
+    path.write_text(
+        yaml.safe_dump(
+            current,
+            default_flow_style=False,
+            indent=2,
+            sort_keys=False,
+            allow_unicode=True,
+        )
+    )