starbash 0.1.0__py3-none-any.whl

starbash/repo/manager.py

@@ -0,0 +1,248 @@
+"""
+Manages the repository of processing recipes and configurations.
+"""
+
+from __future__ import annotations
+import logging
+from pathlib import Path
+
+import tomlkit
+from tomlkit.items import AoT
+from multidict import MultiDict
+
+
+repo_suffix = "starbash.toml"
+
+
+class Repo:
+    """
+    Represents a single starbash repository."""
+
+    def __init__(self, manager: RepoManager, url: str, config: str | None = None):
+        """
+        Initializes a Repo instance.
+
+        Args:
+            url: The URL to the repository (file or general http/https urls are acceptable).
+        """
+        self.manager = manager
+        self.url = url
+        self.config = tomlkit.parse(config) if config else self._load_config()
+        self.manager.add_all_repos(self.config, self.get_path())
+
+    def __str__(self) -> str:
+        """Return a concise one-line description of this repo.
+
+        Example: "Repo(kind=recipe, local=True, url=file:///path/to/repo)"
+        """
+        return f"Repo(kind={self.kind}, local={self.is_local}, url={self.url})"
+
+    __repr__ = __str__
+
+    @property
+    def kind(self) -> str:
+        """
+        Read-only attribute for the repository kind (e.g., "recipe", "data", etc.).
+
+        Returns:
+            The kind of the repository as a string.
+        """
+        return str(self.get("repo.kind", "unknown"))
+
+    @property
+    def is_local(self) -> bool:
+        """
+        Read-only attribute indicating whether the repository URL points to a
+        local file system path (file:// scheme).
+
+        Returns:
+            bool: True if the URL is a local file path, False otherwise.
+        """
+        return self.url.startswith("file://")
+
+    def get_path(self) -> Path | None:
+        """
+        Resolves the URL to a local file system path if it's a file URI.
+
+        Args:
+            url: The repository URL.
+
+        Returns:
+            A Path object if the URL is a local file, otherwise None.
+        """
+        if self.is_local:
+            return Path(self.url[len("file://") :])
+
+        return None
+
+    def read(self, filepath: str) -> str:
+        """
+        Read a filepath relative to the base of this repo. Return the contents in a string.
+
+        Args:
+            filepath: The path to the file, relative to the repository root.
+
+        Returns:
+            The content of the file as a string.
+        """
+        base_path = self.get_path()
+        if base_path is None:
+            raise ValueError("Cannot read files from non-local repositories")
+        target_path = (base_path / filepath).resolve()
+
+        # Security check to prevent reading files outside the repo directory
+        if base_path not in target_path.parents and target_path != base_path:
+            raise PermissionError("Attempted to read file outside of repository")
+
+        return target_path.read_text()
+
+    def _load_config(self) -> dict:
+        """
+        Loads the repository's configuration file (e.g., repo.sb.toml).
+
+        If the config file does not exist, it logs a warning and returns an empty dict.
+
+        Returns:
+            A dictionary containing the parsed configuration.
+        """
+        try:
+            config_content = self.read(repo_suffix)
+            logging.debug(f"Loading repo config from {repo_suffix}")
+            return tomlkit.parse(config_content)
+        except FileNotFoundError:
+            logging.warning(f"No {repo_suffix} found")
+            return {}
+
+    def get(self, key: str, default=None):
+        """
+        Gets a value from this repo's config for a given key.
+        The key can be a dot-separated string for nested values.
+
+        Args:
+            key: The dot-separated key to search for (e.g., "repo.kind").
+            default: The value to return if the key is not found.
+
+        Returns:
+            The found value or the default.
+        """
+        value = self.config
+        for k in key.split("."):
+            if not isinstance(value, dict):
+                return default
+            value = value.get(k)
+        return value if value is not None else default
+
+
+class RepoManager:
+    """
+    Manages the collection of starbash repositories.
+
+    This class is responsible for finding, loading, and providing an API
+    for searching through known repositories defined in TOML configuration
+    files (like appdefaults.sb.toml).
+    """
+
+    def __init__(self, app_defaults: str):
+        """
+        Initializes the RepoManager by loading the application default repos.
+        """
+        self.repos = []
+
+        # We expose the app default preferences as a special root repo with a private URL
+        root_repo = Repo(self, "pkg://starbash-defaults", config=app_defaults)
+        self.repos.append(root_repo)
+
+        # Most users will just want to read from merged
+        self.merged = self._union()
+
+    def add_all_repos(self, toml: dict, base_path: Path | None = None) -> None:
+        # From appdefaults.sb.toml, repo.ref is a list of tables
+        repo_refs = toml.get("repo", {}).get("ref", [])
+
+        for ref in repo_refs:
+            if "url" in ref:
+                url = ref["url"]
+            elif "dir" in ref:
+                path = Path(ref["dir"])
+                if base_path and not path.is_absolute():
+                    # Resolve relative to the current TOML file's directory
+                    path = (base_path / path).resolve()
+                else:
+                    # Expand ~ and resolve from CWD
+                    path = path.expanduser().resolve()
+                url = f"file://{path}"
+            else:
+                raise ValueError(f"Invalid repo reference: {ref}")
+            self.add_repo(url)
+
+    def add_repo(self, url: str) -> None:
+        logging.debug(f"Adding repo: {url}")
+        self.repos.append(Repo(self, url))
+
+    def get(self, key: str, default=None):
+        """
+        Searches for a key across all repositories and returns the first value found.
+        The search is performed in reverse order of repository loading, so the
+        most recently added repositories have precedence.
+
+        Args:
+            key: The dot-separated key to search for (e.g., "repo.kind").
+            default: The value to return if the key is not found in any repo.
+
+        Returns:
+            The found value or the default.
+        """
+        # Iterate in reverse to give precedence to later-loaded repos
+        for repo in reversed(self.repos):
+            value = repo.get(key)
+            if value is not None:
+                return value
+
+        return default
+
+    def dump(self):
+        """
+        Prints a detailed, multi-line description of the combined top-level keys
+        and values from all repositories, using a MultiDict for aggregation.
+        This is useful for debugging and inspecting the consolidated configuration.
+        """
+
+        combined_config = self.merged
+        logging.info("RepoManager Dump")
+        for key, value in combined_config.items():
+            # tomlkit.items() can return complex types (e.g., ArrayOfTables, Table)
+            # For a debug dump, a simple string representation is usually sufficient.
+            logging.info(f"  %s: %s", key, value)
+
+    def _union(self) -> MultiDict:
+        """
+        Merges the top-level keys from all repository configurations into a MultiDict.
+
+        This method iterates through all loaded repositories in their original order
+        and combines their top-level configuration keys. If a key exists in multiple
+        repositories, all of its values will be present in the returned MultiDict.
+
+        Returns:
+            A MultiDict containing the union of all top-level keys.
+        """
+        merged_dict = MultiDict()
+        for repo in self.repos:
+            for key, value in repo.config.items():
+                # if the toml object is an AoT type, monkey patch each element in the array instead
+                if isinstance(value, AoT):
+                    for v in value:
+                        setattr(v, "source", repo)
+                else:
+                    # We monkey patch source into any object that came from a repo, so that users can
+                    # find the source repo (for attribution, URL relative resolution, whatever...)
+                    setattr(value, "source", repo)
+
+                merged_dict.add(key, value)
+
+        return merged_dict
+
+    def __str__(self):
+        lines = [f"RepoManager with {len(self.repos)} repositories:"]
+        for i, repo in enumerate(self.repos):
+            lines.append(f"  [{i}] {repo.url}")
+        return "\n".join(lines)

starbash/tool.py

@@ -0,0 +1,260 @@
+import os
+import shutil
+import textwrap
+import tempfile
+import subprocess
+import re
+
+import logging
+
+import RestrictedPython
+
+logger = logging.getLogger(__name__)
+
+
+class _SafeFormatter(dict):
+    """A dictionary for safe string formatting that ignores missing keys during expansion."""
+
+    def __missing__(self, key):
+        return "{" + key + "}"
+
+
+def expand_context(s: str, context: dict) -> str:
+    """Expand any named variables in the provided string
+
+    Will expand strings of the form MyStr{somevar}a{someothervar} using vars listed in context.
+    Guaranteed safe, doesn't run any python scripts.
+    """
+    # Iteratively expand the command string to handle nested placeholders.
+    # The loop continues until the string no longer changes.
+    expanded = s
+    previous = None
+    max_iterations = 10  # Safety break for infinite recursion
+    for i in range(max_iterations):
+        if expanded == previous:
+            break  # Expansion is complete
+        previous = expanded
+        expanded = expanded.format_map(_SafeFormatter(context))
+    else:
+        logger.warning(
+            f"Template expansion reached max iterations ({max_iterations}). Possible recursive definition in '{s}'."
+        )
+
+    logger.debug(f"Expanded '{s}' into '{expanded}'")
+
+    # throw an error if any remaining unexpanded variables remain unexpanded
+    unexpanded_vars = re.findall(r"\{([^{}]+)\}", expanded)
+    if unexpanded_vars:
+        raise KeyError("Missing context variable(s): " + ", ".join(unexpanded_vars))
+
+    return expanded
+
+
+def make_safe_globals(context: dict = {}) -> dict:
+    """Generate a set of RestrictedPython globals for AstoGlue exec/eval usage"""
+    # Define the global and local namespaces for the restricted execution.
+    # FIXME - this is still unsafe, policies need to be added to limit import/getattr etc...
+    # see https://restrictedpython.readthedocs.io/en/latest/usage/policy.html#implementing-a-policy
+
+    builtins = RestrictedPython.safe_builtins.copy()
+
+    def write_test(obj):
+        """``_write_`` is a guard function taking a single argument.  If the
+        object passed to it may be written to, it should be returned,
+        otherwise the guard function should raise an exception.  ``_write_``
+        is typically called on an object before a ``setattr`` operation."""
+        return obj
+
+    def getitem_glue(baseobj, index):
+        return baseobj[index]
+
+    extras = {
+        "__import__": __import__,  # FIXME very unsafe
+        "_getitem_": getitem_glue,  # why isn't the default guarded getitem found?
+        "_getiter_": iter,  # Allows for loops and other iterations.
+        "_write_": write_test,
+        # Add common built-in types
+        "list": list,
+        "dict": dict,
+        "str": str,
+        "int": int,
+        "all": all,
+    }
+    builtins.update(extras)
+
+    execution_globals = {
+        # Required for RestrictedPython
+        "__builtins__": builtins,
+        "__name__": "__starbash_script__",
+        "__metaclass__": type,
+        # Extra globals auto imported into the scripts context
+        "context": context,
+        "logger": logging.getLogger("script"),  # Allow logging within the script
+    }
+    return execution_globals
+
+
+def strip_comments(text: str) -> str:
+    """Removes comments from a string.
+
+    This function removes both full-line comments (lines starting with '#')
+    and inline comments (text after '#' on a line).
+    """
+    lines = []
+    for line in text.splitlines():
+        lines.append(line.split("#", 1)[0].rstrip())
+    return "\n".join(lines)
+
+
+def tool_run(cmd: str, cwd: str, commands: str | None = None) -> None:
+    """Executes an external tool with an optional script of commands in a given working directory."""
+
+    logger.debug(f"Running {cmd} in {cwd}: stdin={commands}")
+    result = subprocess.run(
+        cmd, input=commands, shell=True, capture_output=True, text=True, cwd=cwd
+    )
+
+    if result.stderr:
+        logger.warning(f"Tool error message:\n{result.stderr.strip()}")
+
+    if result.returncode != 0:
+        # If we got an error, print the entire tool stdout as a warning
+        logger.warning(f"Tool output:\n{result.stdout.strip()}")
+        raise RuntimeError(f"Tool failed with exit code {result.returncode}")
+    else:
+        logger.debug("Tool command successful.")
+
+    if result.stdout:
+        logger.debug(f"Tool output:\n{result.stdout.strip()}")
+
+
+# siril_path = "/home/kevinh/packages/Siril-1.4.0~beta3-x86_64.AppImage"
+siril_path = "org.siril.Siril"  # flatpak
+
+
+def siril_run(temp_dir: str, commands: str, input_files: list[str] = []) -> None:
+    """Executes Siril with a script of commands in a given working directory."""
+
+    # Create symbolic links for all input files in the temp directory
+    for f in input_files:
+        os.symlink(
+            os.path.abspath(str(f)), os.path.join(temp_dir, os.path.basename(str(f)))
+        )
+
+    # We dedent here because the commands are often indented multiline strings
+    script_content = textwrap.dedent(
+        f"""
+        requires 1.4.0-beta3
+        {textwrap.dedent(strip_comments(commands))}
+        """
+    )
+
+    logger.info(
+        f"Running Siril in {temp_dir}, ({len(input_files)} input files) cmds:\n{script_content}"
+    )
+
+    # The `-s -` arguments tell Siril to run in script mode and read commands from stdin.
+    # It seems like the -d command may also be required when siril is in a flatpak
+    cmd = f"{siril_path} -d {temp_dir} -s -"
+
+    tool_run(cmd, temp_dir, script_content)
+
+
+def graxpert_run(cwd: str, arguments: str) -> None:
+    """Executes Graxpert with the specified command line arguments"""
+
+    # Arguments look similar to: graxpert -cmd background-extraction -output /tmp/testout tests/test_images/real_crummy.fits
+    cmd = f"graxpert {arguments}"
+
+    tool_run(cmd, cwd)
+
+
+class Tool:
+    """A tool for stage execution"""
+
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+        # default script file name
+        self.default_script_file = None
+
+    def run_in_temp_dir(self, commands: str, context: dict = {}) -> None:
+        """Run commands inside this tool (with cwd pointing to a temp directory)"""
+        # Create a temporary directory for processing
+        temp_dir = tempfile.mkdtemp(prefix=self.name)
+
+        context["temp_dir"] = (
+            temp_dir  # pass our directory path in for the tool's usage
+        )
+
+        try:
+            self.run(temp_dir, commands, context=context)
+        finally:
+            shutil.rmtree(temp_dir)
+
+    def run(self, cwd: str, commands: str, context: dict = {}) -> None:
+        """Run commands inside this tool (with cwd pointing to the specified directory)"""
+        raise NotImplementedError()
+
+
+class SirilTool(Tool):
+    """Expose Siril as a tool"""
+
+    def __init__(self) -> None:
+        super().__init__("siril")
+
+    def run(self, cwd: str, commands: str, context: dict = {}) -> None:
+
+        # Iteratively expand the command string to handle nested placeholders.
+        # The loop continues until the string no longer changes.
+        expanded = expand_context(commands, context)
+
+        input_files = context.get("input_files", [])
+
+        siril_run(cwd, expanded, input_files)
+
+
+class GraxpertTool(Tool):
+    """Expose Graxpert as a tool"""
+
+    def __init__(self) -> None:
+        super().__init__("graxpert")
+
+    def run(self, cwd: str, commands: str, context: dict = {}) -> None:
+        graxpert_run(cwd, commands)
+
+
+class PythonTool(Tool):
+    """Expose Python as a tool"""
+
+    def __init__(self) -> None:
+        super().__init__("python")
+
+        # default script file override
+        self.default_script_file = "starbash.py"
+
+    def run(self, cwd: str, commands: str, context: dict = {}) -> None:
+        original_cwd = os.getcwd()
+        try:
+            os.chdir(cwd)  # cd to where this script expects to run
+
+            logger.info(f"Executing python script in {cwd} using RestrictedPython")
+            try:
+                byte_code = RestrictedPython.compile_restricted(
+                    commands, filename="<python script>", mode="exec"
+                )
+                # No locals yet
+                execution_locals = None
+                exec(byte_code, make_safe_globals(context), execution_locals)
+            except SyntaxError as e:
+                logger.error(f"Syntax error in python script: {e}")
+                raise
+            except Exception as e:
+                logger.error(f"Error during python script execution: {e}")
+                raise
+        finally:
+            os.chdir(original_cwd)
+
+
+# A dictionary mapping tool names to their respective tool instances.
+tools = {tool.name: tool for tool in [SirilTool(), GraxpertTool(), PythonTool()]}