wexample-wex-addon-process 3.0.1__tar.gz → 4.0.0__tar.gz

{wexample_wex_addon_process-3.0.1 → wexample_wex_addon_process-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wexample-wex-addon-process
-Version: 3.0.1
+Version: 4.0.0
 Summary: Executes process runs defined on disk—either directly or by consuming them from a queue as a worker—against filestate selections
 Author-Email: weeger <contact@wexample.com>
 License: MIT
@@ -10,7 +10,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Requires-Dist: wexample-file>=9.2.0
 Requires-Dist: wexample-queue>=2.0.0
-Requires-Dist: wexample-wex-addon-app>=32.1.0
+Requires-Dist: wexample-wex-addon-app>=32.2.0
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
@@ -18,7 +18,7 @@ Description-Content-Type: text/markdown
 
 # wex-addon-process
 
-Version: 3.0.1
+Version: 4.0.0
 
 `wexample-wex-addon-process` runs the treatments a board declares. A *process* says what to run and on which selection of files; a *run* is one execution of it, asked for by writing a record and picked up by a worker.
 
@@ -249,10 +249,18 @@ moves on: the run still completes, and lists what it could not do under its data
 "data": {"errors": [{"path": "docs/a.pdf", "reason": "docling timed out"}]}
 ```
 
+An item left as it was on purpose — its output already up to date — is said with
+`context.skip(path, reason)`, listed under `data.skipped` the same way. It counts as
+dealt with and is never committed; that is what lets a long batch be started again
+without redoing what is done.
+
+A run whose type raises ends `failed`, its `error` beside the `artifacts`, `errors` and
+`skipped` collected until then.
+
 To stop a run, write `"cancel": true` into its record. The runner checks it before every
 item (`context.each()`, `context.advance()`, or `context.check_cancel()` in a type's own
-loop) and ends the run `cancelled`, with `items_done` where it got to and the errors and
-artifacts collected so far. A run cancelled before it was taken ends `cancelled` without
+loop) and ends the run `cancelled`, with `items_done` where it got to and the errors,
+artifacts and skipped items collected so far. A run cancelled before it was taken ends `cancelled` without
 starting.
 
 ## Tests
@@ -358,7 +366,7 @@ Visit the [Wexample Suite documentation](https://docs.wexample.com) for the comp
 
 - wexample-file: >=9.2.0
 - wexample-queue: >=2.0.0
-- wexample-wex-addon-app: >=32.1.0
+- wexample-wex-addon-app: >=32.2.0
 
 ## Versioning & Compatibility Policy
 

{wexample_wex_addon_process-3.0.1 → wexample_wex_addon_process-4.0.0}/README.md

@@ -1,6 +1,6 @@
 # wex-addon-process
 
-Version: 3.0.1
+Version: 4.0.0
 
 `wexample-wex-addon-process` runs the treatments a board declares. A *process* says what to run and on which selection of files; a *run* is one execution of it, asked for by writing a record and picked up by a worker.
 
@@ -231,10 +231,18 @@ moves on: the run still completes, and lists what it could not do under its data
 "data": {"errors": [{"path": "docs/a.pdf", "reason": "docling timed out"}]}
 ```
 
+An item left as it was on purpose — its output already up to date — is said with
+`context.skip(path, reason)`, listed under `data.skipped` the same way. It counts as
+dealt with and is never committed; that is what lets a long batch be started again
+without redoing what is done.
+
+A run whose type raises ends `failed`, its `error` beside the `artifacts`, `errors` and
+`skipped` collected until then.
+
 To stop a run, write `"cancel": true` into its record. The runner checks it before every
 item (`context.each()`, `context.advance()`, or `context.check_cancel()` in a type's own
-loop) and ends the run `cancelled`, with `items_done` where it got to and the errors and
-artifacts collected so far. A run cancelled before it was taken ends `cancelled` without
+loop) and ends the run `cancelled`, with `items_done` where it got to and the errors,
+artifacts and skipped items collected so far. A run cancelled before it was taken ends `cancelled` without
 starting.
 
 ## Tests
@@ -340,7 +348,7 @@ Visit the [Wexample Suite documentation](https://docs.wexample.com) for the comp
 
 - wexample-file: >=9.2.0
 - wexample-queue: >=2.0.0
-- wexample-wex-addon-app: >=32.1.0
+- wexample-wex-addon-app: >=32.2.0
 
 ## Versioning & Compatibility Policy
 

{wexample_wex_addon_process-3.0.1 → wexample_wex_addon_process-4.0.0}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "wexample-wex-addon-process"
-version = "3.0.1"
+version = "4.0.0"
 description = "Executes process runs defined on disk—either directly or by consuming them from a queue as a worker—against filestate selections"
 authors = [
     { name = "weeger", email = "contact@wexample.com" },
@@ -20,7 +20,7 @@ classifiers = [
 dependencies = [
     "wexample-file>=9.2.0",
     "wexample-queue>=2.0.0",
-    "wexample-wex-addon-app>=32.1.0",
+    "wexample-wex-addon-app>=32.2.0",
 ]
 
 [project.readme]

{wexample_wex_addon_process-3.0.1 → wexample_wex_addon_process-4.0.0}/src/wexample_wex_addon_process/const/process.py

@@ -18,6 +18,10 @@ DATA_KEY_ARTIFACTS: str = "artifacts"
 # The run still completes — one bad file does not sink the batch.
 DATA_KEY_ERRORS: str = "errors"
 
+# Under a run's data: the items a type left as they were on purpose, each
+# {path, reason} — an output already up to date. They count as dealt with.
+DATA_KEY_SKIPPED: str = "skipped"
+
 DEFAULT_COMMIT_AUTHOR: str = "wex <wex@localhost>"
 """Who commits a run's work when neither the run nor git names anyone."""
 

{wexample_wex_addon_process-3.0.1 → wexample_wex_addon_process-4.0.0}/src/wexample_wex_addon_process/process_type/process_context.py

@@ -34,6 +34,7 @@ class ProcessContext:
         touch: Callable[[Path], None] | None = None,
         artifact: Callable[[Path, str | None], None] | None = None,
         fail: Callable[[Path, str], None] | None = None,
+        skip: Callable[[Path, str], None] | None = None,
         check_cancel: Callable[[], None] | None = None,
     ) -> None:
         self.selection = selection
@@ -42,6 +43,7 @@ class ProcessContext:
         self._touch = touch
         self._artifact = artifact
         self._fail = fail
+        self._skip = skip
         self._check_cancel = check_cancel
         self.options = options
         self._advance = advance
@@ -104,6 +106,17 @@ class ProcessContext:
         if self._fail is not None:
             self._fail(path, reason)
 
+    def skip(self, path: Path, reason: str) -> None:
+        """Says that item was left as it was on purpose, and why — « output up to
+        date ».
+
+        Listed under `data.skipped` whatever the run ends in, so a run started
+        again over a long batch says what it did not need to redo. The item still
+        counts as dealt with, and is not touched: nothing of it is committed.
+        """
+        if self._skip is not None:
+            self._skip(path, reason)
+
     def touch(self, path: Path) -> None:
         """Says the run wrote that path — changed, created, or removed it.
 

{wexample_wex_addon_process-3.0.1 → wexample_wex_addon_process-4.0.0}/src/wexample_wex_addon_process/runner/process_runner.py

@@ -162,6 +162,7 @@ class ProcessRunner:
         touched: set[Path] = set()
         artifacts: list[dict[str, Any]] = []
         errors: list[dict[str, str]] = []
+        skipped: list[dict[str, str]] = []
 
         def advance(count: int) -> None:
             progress[RUN_KEY_ITEMS_DONE] += count
@@ -204,13 +205,14 @@ class ProcessRunner:
                         kernel=self.kernel,
                         touch=touched.add,
                         artifact=self._artifact_collector(workdir, artifacts),
-                        fail=self._error_collector(workdir, errors),
+                        fail=self._reason_collector(workdir, errors),
+                        skip=self._reason_collector(workdir, skipped),
                         check_cancel=self._cancel_checker(run_path),
                     )
                 )
         except RunCancelledException:
             data: dict[str, Any] = {}
-            self._add_collected(data, artifacts, errors)
+            self._add_collected(data, artifacts, errors, skipped)
             return self._write(
                 run_path,
                 run_id,
@@ -225,15 +227,20 @@ class ProcessRunner:
         except Exception as exception:
             self.io.debug(traceback.format_exc())
 
-            # The file it was on stays named: a failed run says where it failed.
+            # The file it was on stays named: a failed run says where it failed,
+            # and what it did before keeps being said — a run started again over a
+            # long batch is told by it what is already there.
+            data = {}
+            self._add_collected(data, artifacts, errors, skipped)
             return self._fail(
                 run_path,
                 run_id,
                 f"{type(exception).__name__}: {exception}",
                 progress,
+                data,
             )
 
-        self._add_collected(data, artifacts, errors)
+        self._add_collected(data, artifacts, errors, skipped)
 
         values: dict[str, Any] = {
             RUN_KEY_STATE: STATE_COMPLETE,
@@ -348,6 +355,7 @@ class ProcessRunner:
         touched: set[Path] = set()
         artifacts: list[dict[str, Any]] = []
         errors: list[dict[str, str]] = []
+        skipped: list[dict[str, str]] = []
         data = process_type.run(
             ProcessContext(
                 selection=selection,
@@ -358,10 +366,11 @@ class ProcessRunner:
                 kernel=self.kernel,
                 touch=touched.add,
                 artifact=self._artifact_collector(workdir, artifacts),
-                fail=self._error_collector(workdir, errors),
+                fail=self._reason_collector(workdir, errors),
+                skip=self._reason_collector(workdir, skipped),
             )
         )
-        self._add_collected(data, artifacts, errors)
+        self._add_collected(data, artifacts, errors, skipped)
 
         if commit:
             from wexample_wex_addon_process.const.filestate import OPTION_KEY_ACTION
@@ -381,18 +390,23 @@ class ProcessRunner:
         data: dict[str, Any],
         artifacts: list[dict[str, Any]],
         errors: list[dict[str, str]],
+        skipped: list[dict[str, str]],
     ) -> None:
         """Writes what every type may report the same way under its data — the files
-        it produced, the items it could not deal with — left out when empty."""
+        it produced, the items it could not deal with, those it left as they were —
+        left out when empty."""
         from wexample_wex_addon_process.const.process import (
             DATA_KEY_ARTIFACTS,
             DATA_KEY_ERRORS,
+            DATA_KEY_SKIPPED,
         )
 
         if artifacts:
             data[DATA_KEY_ARTIFACTS] = artifacts
         if errors:
             data[DATA_KEY_ERRORS] = errors
+        if skipped:
+            data[DATA_KEY_SKIPPED] = skipped
 
     def _alive(self, run_path: Path) -> Any:
         """Keeps the run's record saying it is alive while the block runs.
@@ -496,23 +510,16 @@ class ProcessRunner:
         )
         return git_commit_paths(workdir, paths, message, author=author)
 
-    def _error_collector(
-        self, workdir: Path, errors: list[dict[str, str]]
-    ) -> Any:
-        """What a context calls to say an item could not be dealt with."""
-
-        def collect(path: Path, reason: str) -> None:
-            errors.append({"path": str(path.relative_to(workdir)), "reason": reason})
-
-        return collect
-
     def _fail(
         self,
         run_path: Path,
         run_id: str,
         message: str,
         progress: dict[str, Any] | None = None,
+        data: dict[str, Any] | None = None,
     ) -> dict[str, Any]:
+        """Says the run failed, with what it had collected until then beside the
+        error: its artifacts are on disk all the same."""
         from wexample_wex_addon_process.const.process import (
             RUN_KEY_DATA,
             RUN_KEY_DATE_ENDED,
@@ -529,7 +536,7 @@ class ProcessRunner:
                 **(progress or {}),
                 RUN_KEY_STATE: STATE_FAILED,
                 RUN_KEY_DATE_ENDED: self._now(),
-                RUN_KEY_DATA: {"error": message},
+                RUN_KEY_DATA: {"error": message, **(data or {})},
             },
         )
 
@@ -587,6 +594,19 @@ class ProcessRunner:
             write_record(run_path, dict(progress))
         self._ring_now(run_id)
 
+    def _reason_collector(
+        self, workdir: Path, collected: list[dict[str, str]]
+    ) -> Any:
+        """What a context calls to say something of an item, and why — that it
+        could not be dealt with, or that it was left as it was."""
+
+        def collect(path: Path, reason: str) -> None:
+            collected.append(
+                {"path": str(path.relative_to(workdir)), "reason": reason}
+            )
+
+        return collect
+
     def _ring_now(self, run_id: str) -> None:
         import time
 

wexample_wex_addon_process-4.0.0/tests/unit/test_skip_and_failure.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+def _registry() -> Any:
+    """A type that converts each item, skips those already converted, and stops
+    dead on an item named `crash`."""
+    from wexample_wex_addon_process.process_type.abstract_process_type import (
+        AbstractProcessType,
+    )
+    from wexample_wex_addon_process.process_type.family.abstract_process_type_family import (
+        AbstractProcessTypeFamily,
+    )
+    from wexample_wex_addon_process.process_type.process_type_registry import (
+        ProcessTypeRegistry,
+    )
+
+    class ConvertType(AbstractProcessType):
+        label = "Convert"
+        name = "test:convert"
+
+        def run(self, context: Any) -> dict[str, Any]:
+            for path in context.each():
+                target = path.with_suffix(".txt")
+                if target.exists():
+                    context.skip(path, "output up to date")
+                    continue
+                if path.stem == "crash":
+                    raise ConnectionError("docling unreachable")
+                if path.stem == "bad":
+                    context.fail(path, "malformed")
+                    continue
+                target.write_text(path.read_text())
+                context.artifact(target)
+
+            return {}
+
+    class ConvertFamily(AbstractProcessTypeFamily):
+        prefix = "test"
+
+        def types(self) -> list[AbstractProcessType]:
+            return [ConvertType()]
+
+    return ProcessTypeRegistry([ConvertFamily()])
+
+
+def _execute(app: Path) -> dict[str, Any]:
+    from wexample_prompt.common.io_manager import IoManager
+
+    from wexample_wex_addon_process.runner.process_runner import ProcessRunner
+
+    data = app / ".wex" / "data"
+    for kind, identity, values in (
+        ("selection", "s", {"patterns": "docs/*.md"}),
+        ("process", "p", {"title": "Convert", "type": "test:convert", "selection_id": "s"}),
+        ("process_run", "r", {"process_id": "p", "state": "pending"}),
+    ):
+        (data / kind).mkdir(parents=True, exist_ok=True)
+        (data / kind / f"{identity}.json").write_text(json.dumps(values))
+
+    return ProcessRunner(io=IoManager(), registry=_registry()).execute(
+        workdir=app, run_id="r"
+    )
+
+
+def _app(tmp_path: Path, *stems: str) -> Path:
+    app = tmp_path / "app"
+    (app / "docs").mkdir(parents=True)
+    for stem in stems:
+        (app / "docs" / f"{stem}.md").write_text(f"{stem}\n")
+    return app
+
+
+def test_a_skipped_item_is_listed_and_counted(tmp_path: Path) -> None:
+    app = _app(tmp_path, "a", "b")
+    (app / "docs" / "a.txt").write_text("a\n")
+
+    run = _execute(app)
+
+    assert run["state"] == "complete", run.get("data")
+    assert run["data"]["skipped"] == [
+        {"path": "docs/a.md", "reason": "output up to date"}
+    ]
+    assert run["items_done"] == 2, "A skipped item counts as dealt with"
+
+
+def test_nothing_skipped_leaves_no_key(tmp_path: Path) -> None:
+    run = _execute(_app(tmp_path, "a"))
+
+    assert run["state"] == "complete"
+    assert "skipped" not in run["data"]
+
+
+def test_a_failed_run_keeps_what_it_collected(tmp_path: Path) -> None:
+    # Sorted: a.md is skipped, b.md converted, bad.md in error, crash.md raises.
+    app = _app(tmp_path, "a", "b", "bad", "crash")
+    (app / "docs" / "a.txt").write_text("a\n")
+
+    run = _execute(app)
+
+    assert run["state"] == "failed"
+    assert run["data"]["error"] == "ConnectionError: docling unreachable"
+    assert run["data"]["skipped"] == [
+        {"path": "docs/a.md", "reason": "output up to date"}
+    ]
+    assert [a["path"] for a in run["data"]["artifacts"]] == ["docs/b.txt"]
+    assert run["data"]["errors"] == [{"path": "docs/bad.md", "reason": "malformed"}]
+    assert run["current"] == "docs/crash.md", "A failed run still says where"