starbash 0.2.6__tar.gz → 0.2.7__tar.gz

{starbash-0.2.6 → starbash-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: starbash
-Version: 0.2.6
+Version: 0.2.7
 Summary: A tool for automating/standardizing/sharing astrophotography workflows.
 License-File: LICENSE
 Author: Kevin Hester
@@ -20,7 +20,7 @@ Requires-Dist: restrictedpython (>=8.1,<9.0)
 Requires-Dist: rich (>=14.2.0,<15.0.0)
 Requires-Dist: sentry-sdk (>=2.42.1,<3.0.0)
 Requires-Dist: textual (>=6.8.0,<7.0.0)
-Requires-Dist: toml-repo (>=0.1.2,<0.2.0)
+Requires-Dist: toml-repo (>=0.1.6,<0.2.0)
 Requires-Dist: typer (>=0.20.0,<0.21.0)
 Requires-Dist: update-checker (>=0.18.0,<0.19.0)
 Project-URL: Documentation, https://github.com/geeksville/starbash/blob/main/README.md

{starbash-0.2.6 → starbash-0.2.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "starbash"
-version = "0.2.6"
+version = "0.2.7"
 description = "A tool for automating/standardizing/sharing astrophotography workflows."
 authors = ["Kevin Hester <kevinh@geeksville.com>"]
 readme = "README.md"
@@ -34,7 +34,7 @@ graxpert = {version = "^3.2.0a2", extras = ["cpuonly"]}
 # experimenting with siril-scripts/processing/VeraLux_HyperMetric_Stretch.py
 numpy = "2.2.6"
 pyqt6 = "^6.10.1"
-toml-repo = "^0.1.2"
+toml-repo = "^0.1.6"
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=8.4.2,<10.0.0"

{starbash-0.2.6 → starbash-0.2.7}/src/starbash/filtering.py

@@ -1,4 +1,6 @@
 import logging
+import os
+import re
 
 from starbash import InputDef, Metadata, RequireDef
 from starbash.aliases import get_aliases
@@ -62,6 +64,16 @@ def _apply_filter(requires: RequireDef, candidates: list[ImageRow]) -> list[Imag
         repo_kind = metadata["repo"].kind()  # I think repo is guaranteed to be present
         return repo_kind != "processed" and repo_kind != "master"
 
+    def _filter_filename(metadata: Metadata) -> bool:
+        """Keep images whose basename matches (or, with mode='exclude', does not match) a regex."""
+        pattern = get_safe(requires, "value")
+        mode = requires.get("mode", "include")
+        if mode not in ("include", "exclude"):
+            raise ValueError(f"Unknown filename filter mode: {mode}")
+        basename = os.path.basename(str(metadata.get("path") or metadata.get("abspath") or ""))
+        matched = re.search(pattern, basename) is not None
+        return matched if mode == "include" else not matched
+
     def _filter_min_count(metadata: Metadata) -> bool:
         """Min_count is handled in stage 2, so always return True here."""
         return True
@@ -71,6 +83,7 @@ def _apply_filter(requires: RequireDef, candidates: list[ImageRow]) -> list[Imag
         "metadata": _filter_metadata,
         "camera": _filter_camera,
         "unprocessed": _filter_unprocessed,
+        "filename": _filter_filename,
         "min_count": _filter_min_count,
     }
 

{starbash-0.2.6 → starbash-0.2.7}/src/starbash/processing.py

@@ -14,7 +14,7 @@ from toml_repo import Repo
 from tomlkit.items import AoT
 
 import starbash
-from starbash import InputDef, OutputDef, StageDict
+from starbash import InputDef, OutputDef, RequireDef, StageDict
 from starbash.aliases import get_aliases, normalize_target_name
 from starbash.app import Starbash
 from starbash.database import (
@@ -900,6 +900,18 @@ class Processing(ProcessingLike):
         # Collect all image rows from prior stage outputs
         child_exception: Exception | None = None
 
+        # `filename` requires match the actual files we consume (the prior stage's
+        # OUTPUT names), which differ from its inputs; every other requires kind
+        # (metadata/camera/unprocessed/min_count) gates on the prior stage's rich
+        # INPUT rows. So split them and apply each to the correct row set.
+        all_requires: list[RequireDef] = input.get("requires", [])
+        filename_requires = [r for r in all_requires if r.get("kind") == "filename"]
+        gating_input: InputDef = (
+            {**input, "requires": [r for r in all_requires if r.get("kind") != "filename"]}
+            if filename_requires
+            else input
+        )
+
         base: str | None = None  # We will set base from the first matching prior task output
         for task in prior_tasks:
             task_context: dict[str, Any] = task["meta"]["context"]  # type: ignore
@@ -911,7 +923,7 @@ class Processing(ProcessingLike):
                     images = file_info.image_rows
                     images = [self._with_defaults(img) for img in images]
                     try:
-                        task_filtered_input = filter_by_requires(input, images)
+                        task_filtered_input = filter_by_requires(gating_input, images)
                         if (
                             task_filtered_input
                         ):  # This task had matching inputs for us, so therefore we want its outputs
@@ -922,7 +934,13 @@ class Processing(ProcessingLike):
                                 and task_output.image_rows
                             ):
                                 base = task_output.base
-                                for img in task_output.image_rows:
+                                # filename filters select which produced files we want.
+                                output_rows = task_output.image_rows
+                                if filename_requires:
+                                    output_rows = filter_by_requires(
+                                        {"requires": filename_requires}, output_rows
+                                    )
+                                for img in output_rows:
                                     image_rows[img["abspath"]] = img
                     except NotEnoughFilesError as e:
                         child_exception = e  # In case we need to raise later
@@ -933,6 +951,12 @@ class Processing(ProcessingLike):
             # we failed on every child, give up
             raise child_exception
 
+        if filename_requires and len(image_rows) == 0:
+            # Prior outputs existed but none matched the filename filter; skip this stage.
+            raise NotEnoughFilesError(
+                "No prior stage outputs matched the input filename filter", []
+            )
+
         return FileInfo(base=base, image_rows=list(image_rows.values()), definition=input)
 
     def preflight_tasks(self, pt: ProcessedTarget, tasks: list[TaskDict]) -> list[TaskDict]:

{starbash-0.2.6 → starbash-0.2.7}/src/starbash/tool/base.py

@@ -8,12 +8,16 @@ import subprocess
 import sys
 import tempfile
 import textwrap
+from collections import deque
 from collections.abc import Callable
 from contextlib import nullcontext
 from typing import Any
 
+from rich.console import Group, RenderableType
 from rich.live import Live
+from rich.padding import Padding
 from rich.spinner import Spinner
+from rich.text import Text
 
 from starbash.commands import SPINNER_STYLE
 from starbash.exception import UserHandledError
@@ -95,6 +99,56 @@ def color_lines(lines: list[str]) -> str:
     return "\n".join(color_line(line) for line in lines)
 
 
+class ToolLiveDisplay:
+    """Live renderable: a running spinner plus the most recent tool output lines.
+
+    Fed incrementally by ``tool_run_streaming`` via :meth:`add_line`.  stderr lines
+    render red, stdout lines yellow.  Each line is truncated to a single row so the
+    display stays a fixed height while the tool runs.
+    """
+
+    MAX_LINES = 3
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.spinner = Spinner(
+            "arc",
+            text=f"Tool running: [bold]{name}[/bold]...",
+            speed=2.0,
+            style=SPINNER_STYLE,
+        )
+        self._lines: deque[Text] = deque(maxlen=self.MAX_LINES)
+        self._done = False
+
+    def add_line(self, line: str, is_stderr: bool) -> None:
+        """Append a tool output line, keeping only the most recent MAX_LINES."""
+        text = line.rstrip("\n")
+        if not text.strip():
+            return
+        self._lines.append(
+            Text(
+                text,
+                style="red" if is_stderr else "yellow",
+                no_wrap=True,
+                overflow="ellipsis",
+            )
+        )
+
+    def finish(self) -> None:
+        """Swap the spinner for a static header so the final frame persists."""
+        self._done = True
+
+    def __rich__(self) -> RenderableType:
+        header: RenderableType = (
+            Text(f"Tool completed: {self.name}", style="dim")
+            if self._done
+            else self.spinner
+        )
+        # Indent each output line by 4 spaces beneath the header.
+        lines = [Padding(line, (0, 0, 0, 4)) for line in self._lines]
+        return Group(header, *lines)
+
+
 def tool_emit_logs(lines: str, log_level: int = logging.INFO) -> None:
     """Emit log lines from a tool to the logger at the specified log level.
 
@@ -172,10 +226,10 @@ def tool_run_streaming(
     """Execute an external tool, invoking on_line for each stdout line as it arrives.
 
     Unlike a blocking communicate()-based runner, this streams stdout line-by-line so
-    callers can react to incremental output (e.g. JSON progress events).  A reader
-    thread feeds a queue so the timeout fires correctly even for silent processes.
-    stderr is collected and handled after the process exits.  A non-zero exit code
-    raises ToolError.
+    callers can react to incremental output (e.g. JSON progress events).  Separate
+    reader threads for stdout and stderr feed a shared queue so the timeout fires
+    correctly even for silent processes and both streams are written to log_out in
+    approximate arrival order.  A non-zero exit code raises ToolError.
     """
     import queue
     import threading
@@ -196,6 +250,7 @@ def tool_run_streaming(
         env=env,
     )
     assert process.stdout is not None
+    assert process.stderr is not None
 
     if commands:
         assert process.stdin is not None
@@ -209,57 +264,70 @@ def tool_run_streaming(
 
         threading.Thread(target=_write_stdin, daemon=True).start()
 
-    line_queue: queue.Queue[str | None] = queue.Queue()
+    # Both reader threads push (stream_name, line) tuples so we can interleave the
+    # two streams into log_out in approximate arrival order.  A None line marks EOF
+    # for that reader; the main loop stops once both readers have finished.
+    line_queue: queue.Queue[tuple[str, str | None]] = queue.Queue()
 
-    def _reader() -> None:
+    def _reader(name: str, stream: io.TextIOBase) -> None:
         try:
-            for line in process.stdout:  # type: ignore[union-attr]
-                line_queue.put(line)
+            for line in stream:
+                line_queue.put((name, line))
         except Exception:
             pass
         finally:
-            line_queue.put(None)  # EOF sentinel
+            line_queue.put((name, None))  # EOF sentinel
 
-    threading.Thread(target=_reader, daemon=True).start()
+    threading.Thread(
+        target=_reader, args=("stdout", process.stdout), daemon=True
+    ).start()
+    threading.Thread(
+        target=_reader, args=("stderr", process.stderr), daemon=True
+    ).start()
 
     deadline = (time.monotonic() + timeout) if timeout else None
     stdout_captured: list[str] = []
+    stderr_captured: list[str] = []
+    readers_done = 0
 
     try:
-        while True:
+        while readers_done < 2:
             remaining = max(0.0, deadline - time.monotonic()) if deadline else None
             try:
-                line = line_queue.get(timeout=remaining)
+                stream_name, line = line_queue.get(timeout=remaining)
             except queue.Empty:
                 process.kill()
                 process.wait()
-                if process.stderr:
-                    process.stderr.close()
                 raise RuntimeError(f"Tool timed out after {timeout} seconds")
-            if line is None:  # EOF
-                break
-            stdout_captured.append(line)
+            if line is None:  # EOF for one of the streams
+                readers_done += 1
+                continue
             if log_out:
                 log_out.write(line)
                 log_out.flush()  # Just in case the user is 'tailing' the file
-            if on_line:
-                try:
-                    on_line(line)
-                except Exception:
-                    logger.exception("Error in tool output line handler")
+            active_display = Tool._active_display
+            if active_display is not None:
+                active_display.add_line(line, is_stderr=stream_name == "stderr")
+            if stream_name == "stdout":
+                stdout_captured.append(line)
+                if on_line:
+                    try:
+                        on_line(line)
+                    except Exception:
+                        logger.exception("Error in tool output line handler")
+            else:
+                stderr_captured.append(line)
     finally:
         process.stdout.close()
-
-    stderr_text = process.stderr.read() if process.stderr else ""
-    if process.stderr:
         process.stderr.close()
+
     returncode = process.wait()
 
     stdout_str = "".join(stdout_captured)
     if returncode != 0:
         tool_emit_logs(stdout_str, log_level=logging.ERROR)
 
-    stderr_list = stderr_text.splitlines() if stderr_text else []
+    stderr_list = [line.rstrip("\n") for line in stderr_captured]
     if stderr_fixup:
         stderr_list = stderr_fixup(stdout_captured, stderr_list)
     if stderr_list:
@@ -285,6 +353,10 @@ class Tool:
     # of our call tree, that variables get passed down to all sub-tools.
     _default_log_out: io.TextIOWrapper | None = None
 
+    # The live output display owned by the outermost tool in a call chain.  Nested
+    # tools (and tool_run_streaming) feed their output lines into this same display.
+    _active_display: "ToolLiveDisplay | None" = None
+
     # If True, this tool renders its own progress display in _run() (e.g. a Rich
     # Progress bar), so Tool.run() suppresses the default spinner to avoid nested
     # Live displays.
@@ -320,22 +392,19 @@ class Tool:
         from starbash import console  # Lazy import to avoid circular dependency
 
         temp_dir = None
-        spinner = (
-            None
-            if self.manages_own_progress
-            else Spinner(
-                "arc",
-                text=f"Tool running: [bold]{self.name}[/bold]...",
-                speed=2.0,
-                style=SPINNER_STYLE,
-            )
-        )
+        # Only the outermost tool in a call chain owns the live display; nested tools
+        # (and tool_run_streaming) feed their output into the already-active display.
+        owns_display = not self.manages_own_progress and Tool._active_display is None
+        display_obj = ToolLiveDisplay(self.name) if owns_display else None
         display = (
-            nullcontext()
-            if spinner is None
-            else Live(spinner, console=console, refresh_per_second=5, transient=True)
+            Live(display_obj, console=console, refresh_per_second=8, transient=False)
+            if display_obj is not None
+            else nullcontext()
         )
         with display:
+            if display_obj is not None:
+                Tool._active_display = display_obj
+
             did_set_default_log = (
                 False  # Assume we are not the top entry into the chain of tool calls
             )
@@ -359,8 +428,9 @@ class Tool:
 
                 self._run(cwd, commands, context=context, log_out=my_log, **kwargs)
             finally:
-                if spinner is not None:
-                    spinner.update(text=f"Tool completed: [bold]{self.name}[/bold].")
+                if display_obj is not None:
+                    display_obj.finish()
+                    Tool._active_display = None
                 if temp_dir:
                     shutil.rmtree(temp_dir)
                     context.pop("temp_dir", None)