nf-meta 0.2.3__tar.gz → 0.3.0.dev0__tar.gz

{nf_meta-0.2.3/src/nf_meta.egg-info → nf_meta-0.3.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nf-meta
-Version: 0.2.3
+Version: 0.3.0.dev0
 Summary: nf-meta is a cli tool for building, validating and running meta-pipelines based on the Nextflow workflow language
 Author-email: Julian Flesch <julian.flesch@qbic.uni-tuebingen.de>
 License-Expression: Apache-2.0
@@ -85,7 +85,7 @@ This project features a small editor which intends to ease the creation and upda
 of these config files, by visualizing the config as a graph and 
 offering user-firendly form for entering and validating values.
 
-![editor](assets/nf-meta-screenshot.png)
+![editor](https://raw.githubusercontent.com/bmds-tue/nf-meta/main/assets/nf-meta-screenshot.png)
 
 
 ## Metapipeline Runners

{nf_meta-0.2.3 → nf_meta-0.3.0.dev0}/README.md

@@ -61,7 +61,7 @@ This project features a small editor which intends to ease the creation and upda
 of these config files, by visualizing the config as a graph and 
 offering user-firendly form for entering and validating values.
 
-![editor](assets/nf-meta-screenshot.png)
+![editor](https://raw.githubusercontent.com/bmds-tue/nf-meta/main/assets/nf-meta-screenshot.png)
 
 
 ## Metapipeline Runners

{nf_meta-0.2.3 → nf_meta-0.3.0.dev0}/pyproject.toml

@@ -13,7 +13,7 @@ namespaces = false
 
 [project]
 name = "nf-meta"
-version = "0.2.3"
+version = "0.3.0-dev"
 description = "nf-meta is a cli tool for building, validating and running meta-pipelines based on the Nextflow workflow language"
 readme = "README.md"
 authors = [
@@ -45,4 +45,21 @@ nf-meta = "nf_meta.__main__:cli"
 [dependency-groups]
 dev = [
     "twine>=6.2.0",
+    "pytest>=9.0.3",
+    "pytest-mock>=3.15.1",
+    "mypy>=2.1.0",
+    "types-networkx>=3.6.1.20260408",
+    "types-pyyaml>=6.0.12.20260408",
+    "types-requests>=2.33.0.20260503",
 ]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.mypy]
+python_version = "3.12"
+mypy_path = "src"
+
+[[tool.mypy.overrides]]
+module = ["uvicorn", "uvicorn.*"]
+ignore_missing_imports = true

nf_meta-0.3.0.dev0/src/nf_meta/__main__.py

@@ -0,0 +1,135 @@
+import click
+from functools import wraps
+from nf_meta.core.errors import (
+    GraphValidationError,
+    ValidationError,
+    format_errors_for_cli,
+)
+from nf_meta.runner import (
+    run_metapipeline,
+    get_registered_runners,
+    RunOptions,
+    NfMetaRunnerError,
+)
+from nf_meta.core.graph import MetaworkflowGraph
+from nf_meta.core.session import start_session
+from nf_meta.editor import get_registered_editors, EditorOptions, run_editor
+
+
+@click.group()
+@click.version_option()
+def cli() -> None:
+    return
+
+
+@click.command("editor")
+@click.option("--verbose", "-v", is_flag=True, help="Enables verbose mode")
+@click.option(
+    "--editor",
+    "-e",
+    type=click.Choice(get_registered_editors()),
+    default="browser",
+    help="Editor backend to use",
+)
+@click.option("--host", help="Host to bind the editor server to")
+@click.option(
+    "--port",
+    type=int,
+    help="Port for the editor server (auto-assigned if omitted)",
+)
+@click.argument("config", required=False, type=click.Path())
+def edit_browser(config, verbose, editor, host, port):
+    try:
+        start_session(config)
+        opts = EditorOptions(editor_name=editor)
+        if host is not None:
+            opts.host = host
+        if port is not None:
+            opts.port = port
+        run_editor(opts)
+    except (GraphValidationError, ValidationError) as e:
+        click.echo(format_errors_for_cli(e))
+        raise SystemExit(1)
+    except FileNotFoundError as e:
+        click.echo(click.style(e, fg="red"))
+        raise SystemExit(1)
+
+
+@click.command("validate")
+@click.argument("config", type=click.Path())
+@click.option("--verbose", "-v", is_flag=True, help="Enables verbose mode")
+def validate_config(config, verbose):
+    try:
+        g = MetaworkflowGraph.from_file(config)
+        click.echo(click.style("✓ Config is valid", fg="green"))
+    except (GraphValidationError, ValidationError) as e:
+        click.echo(format_errors_for_cli(e))
+        raise SystemExit(1)
+    except FileNotFoundError as e:
+        click.echo(click.style(e, fg="red"))
+        raise SystemExit(1)
+
+
+@click.command("run")
+@click.argument("config", type=click.Path())
+@click.option("--verbose", "-v", is_flag=True, help="Enables verbose mode")
+@click.option(
+    "--runner",
+    "-r",
+    prompt=True,
+    type=click.Choice(get_registered_runners()),
+    default="python",
+)
+@click.option("--resume", is_flag=True, help="Resume a previous run")
+@click.option(
+    "--output-lines",
+    "-l",
+    default="25",
+    type=int,
+    help="Number of lines of workflow output to stream to output window (Only for Python Runner!)",
+)
+@click.option("--start", "-s", type=str, help="ID of workflow to start from")
+@click.option("--target", "-t", type=str, help="ID of workflow to run until")
+@click.option(
+    "--profile",
+    "-p",
+    type=str,
+    help="Nextflow profile to apply globally, taking precedent over config values.",
+)
+@click.option(
+    "--stub",
+    is_flag=True,
+    help="Run all workflows as stub runs (passes -stub to Nextflow)",
+)
+def run(config, verbose, runner, resume, output_lines, start, target, profile, stub):
+    try:
+        run_options = RunOptions(
+            runner_name=runner,
+            verbose=verbose,
+            output_lines=output_lines,
+            nf_profile=profile,
+            stub=stub,
+            resume=resume,
+            start=start,
+            target=target,
+        )
+        g = MetaworkflowGraph.from_file(config)
+        run_metapipeline(g, run_options)
+    except (GraphValidationError, ValidationError) as e:
+        click.echo(format_errors_for_cli(e))
+        raise SystemExit(1)
+    except NfMetaRunnerError as e:
+        click.echo(click.style(e.message, fg="red"))
+        raise SystemExit(1)
+    except FileNotFoundError as e:
+        click.echo(click.style(e, fg="red"))
+        raise SystemExit(1)
+
+
+cli.add_command(edit_browser)
+cli.add_command(validate_config)
+cli.add_command(run)
+
+
+if __name__ == "__main__":
+    cli()

nf_meta-0.3.0.dev0/src/nf_meta/core/cache.py

@@ -0,0 +1,39 @@
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_ENV_VAR = "NF_META_CACHE_DIR"
+GLOBAL_CACHE_DIR: Path = Path(
+    os.environ.get(_ENV_VAR) or Path.home() / ".cache" / "nf-meta"
+)
+
+
+def schema_cache_path(url: str, version: str) -> Path:
+    key = hashlib.sha256(f"{url}@{version}".encode()).hexdigest()[:16]
+    return GLOBAL_CACHE_DIR / "schemas" / f"{key}.json"
+
+
+def read_schema_cache(url: str, version: str) -> dict | None:
+    path = schema_cache_path(url, version)
+    if not path.exists():
+        return None
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except Exception as e:
+        logger.warning("Failed to read schema cache at %s: %s", path, e)
+        return None
+
+
+def write_schema_cache(url: str, version: str, schema: dict) -> None:
+    path = schema_cache_path(url, version)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, "w") as f:
+            json.dump(schema, f)
+    except Exception as e:
+        logger.warning("Failed to write schema cache to %s: %s", path, e)

{nf_meta-0.2.3 → nf_meta-0.3.0.dev0}/src/nf_meta/core/errors.py

@@ -25,7 +25,7 @@ class WorkflowReferenceErrors(GraphValidationError):
 class SessionCommandError(Exception):
     @dataclass
     class FieldError:
-        workflow_id: str
+        workflow_id: Optional[str]
         field: str
         message: str
 
@@ -33,7 +33,13 @@ class SessionCommandError(Exception):
     graph_errors: list[str]
 
     @classmethod
-    def from_exception(cls, e: WorkflowReferenceError | WorkflowReferenceErrors | GraphValidationError):
+    def from_exception(
+        cls,
+        e: WorkflowReferenceError
+        | WorkflowReferenceErrors
+        | GraphValidationError
+        | ValueError,
+    ):
         if isinstance(e, WorkflowReferenceError):
             e = WorkflowReferenceErrors([e])
         if isinstance(e, WorkflowReferenceErrors):
@@ -42,23 +48,30 @@ class SessionCommandError(Exception):
                     cls.FieldError(
                         workflow_id=ref_error.reference.source_wf_id,
                         field="params",  # TODO: For now ok. Think about more finegrained errors for Workflow output/input references
-                        message=ref_error.message
+                        message=ref_error.message,
                     )
                     for ref_error in e.errors
                 ],
-                graph_errors=[]
+                graph_errors=[],
             )
         return cls(field_errors=[], graph_errors=[str(e)])
-    
+
     def to_dict(self):
         return {
-            "field_errors": [{"workflow_id": e.workflow_id, "field": e.field, "message": e.message} 
-                             for e in self.field_errors],
-            "graph_errors": self.graph_errors
+            "field_errors": [
+                {"workflow_id": e.workflow_id, "field": e.field, "message": e.message}
+                for e in self.field_errors
+            ],
+            "graph_errors": self.graph_errors,
         }
-    
 
-def format_errors_for_cli(e: WorkflowReferenceError | WorkflowReferenceErrors | GraphValidationError | ValidationError) -> str:
+
+def format_errors_for_cli(
+    e: WorkflowReferenceError
+    | WorkflowReferenceErrors
+    | GraphValidationError
+    | ValidationError,
+) -> str:
     lines = []
     if isinstance(e, WorkflowReferenceError):
         e = WorkflowReferenceErrors([e])
@@ -74,5 +87,5 @@ def format_errors_for_cli(e: WorkflowReferenceError | WorkflowReferenceErrors |
         lines.append(click.style("Validation failed:", fg="red", bold=True))
         for err in e.errors():
             field = ".".join(str(l) for l in err["loc"])
-            lines.append(f" {click.style(field, fg="yellow")}: {err["msg"]}")
+            lines.append(f" {click.style(field, fg='yellow')}: {err['msg']}")
     return "\n".join(lines)

{nf_meta-0.2.3 → nf_meta-0.3.0.dev0}/src/nf_meta/core/events.py

@@ -7,7 +7,6 @@ from .models import Workflow, Transition, GlobalOptions
 # Event Handler
 # TODO: Test that Metaworkflow implements GraphEventHandler: isinstance(graph, GraphEventHandler)
 class GraphEventHandler(Protocol):
-
     def pop_events(self) -> tuple["Event"]: ...
 
     def add_workflow(self, w: Workflow) -> None: ...
@@ -22,13 +21,14 @@ class GraphEventHandler(Protocol):
 
     def update_global_options(self, g: GlobalOptions) -> None: ...
 
-    def deferred_validation(self) -> None: ...
+    def deferred_validation(self): ...
 
 
 # ------------------------------------------
 # ----- Events: State Changes --------------
 # ------------------------------------------
 
+
 class Event(Protocol):
     def get_undo_cmd(self) -> "Command": ...
 
@@ -87,14 +87,14 @@ class GlobalOptionsUpdated:
 # ----- COMMANDS: State Change Intents -----
 # ------------------------------------------
 
-class Command(Protocol):
 
+class Command(Protocol):
     def apply(self, graph: GraphEventHandler) -> None: ...
 
 
 @dataclass(frozen=True)
 class Transaction:
-    commands: tuple[Command]
+    commands: tuple[Command, ...]
 
     def apply(self, graph: GraphEventHandler):
         # TODO: Handle errors?

{nf_meta-0.2.3 → nf_meta-0.3.0.dev0}/src/nf_meta/core/graph.py

@@ -1,13 +1,33 @@
 from contextlib import contextmanager
-from typing import Dict, Any, Optional
+from typing import Optional
 from pathlib import Path
 import logging
 
 import networkx as nx
 
-from .models import MetaworkflowConfig, Workflow, GlobalOptions, Transition, load_config, dump_config, CONFIG_VERSION_MAX
-from .events import Event, WorkflowAdded, WorkflowRemoved, WorkflowUpdated, TransitionAdded, TransitionRemoved, GlobalOptionsUpdated
-from .errors import GraphValidationError, WorkflowReferenceError, WorkflowReferenceErrors
+from .models import (
+    MetaworkflowConfig,
+    GlobalOptions,
+    Transition,
+    Workflow,
+    load_config,
+    dump_config,
+    CONFIG_VERSION_MAX,
+)
+from .events import (
+    Event,
+    WorkflowAdded,
+    WorkflowRemoved,
+    WorkflowUpdated,
+    TransitionAdded,
+    TransitionRemoved,
+    GlobalOptionsUpdated,
+)
+from .errors import (
+    GraphValidationError,
+    WorkflowReferenceError,
+    WorkflowReferenceErrors,
+)
 
 logger = logging.getLogger()
 
@@ -22,7 +42,7 @@ class MetaworkflowGraph:
 
     def __init__(self):
         self.G: nx.DiGraph = nx.DiGraph()
-        self.global_options: Optional[GlobalOptions] = None
+        self.global_options = GlobalOptions()
         self.config_version: str = CONFIG_VERSION_MAX
         self._events: list[Event] = []
         self._validation_suspended = False
@@ -41,7 +61,7 @@ class MetaworkflowGraph:
 
         if not cfg:
             raise ValueError(f"No config data loaded from {cfg_file}")
-    
+
         return cls.from_config(cfg)
 
     @classmethod
@@ -82,17 +102,17 @@ class MetaworkflowGraph:
         if wf.params and not self._validation_suspended:
             self.validate_param_references(wf)
 
-        if not wf.id in self.G.nodes:
+        if wf.id not in self.G.nodes:
             raise ValueError("Workflow has invalid id. Update unsuccessful!")
 
         old_wf = self.get_workflow_by_id(wf.id)
         self.G.nodes[wf.id]["workflow"] = wf.model_copy()
-        self._emit(WorkflowUpdated(old_workflow=old_wf, new_workflow=wf))
+        self._emit(WorkflowUpdated(old_workflow=old_wf, new_workflow=wf))  # type: ignore
 
     def remove_workflow(self, wf_id: str, recursive=False):
         if wf_id not in self.G.nodes:
             raise ValueError("Invalid wf_id")
-        
+
         for edge in list(self.G.in_edges(wf_id)):
             self.remove_transition(*edge)
 
@@ -101,14 +121,18 @@ class MetaworkflowGraph:
 
         removed_wf = self.get_workflow_by_id(wf_id)
         self.G.remove_node(wf_id)
-        self._emit(WorkflowRemoved(removed_wf))
+        self._emit(WorkflowRemoved(removed_wf))  # type: ignore
 
     def add_transition(self, tr: Transition):
         if tr.source not in self.G.nodes:
-            raise ValueError(f"Unknown node {tr.source} found in transition {tr.source}->{tr.target}")
+            raise ValueError(
+                f"Unknown node {tr.source} found in transition {tr.source}->{tr.target}"
+            )
 
         if tr.target not in self.G.nodes:
-            raise ValueError(f"Unknown node {tr.source} found in transition {tr.source}->{tr.target}")
+            raise ValueError(
+                f"Unknown node {tr.source} found in transition {tr.source}->{tr.target}"
+            )
 
         if self.G.has_edge(tr.source, tr.target):
             print(f"[warning] Edge already exists: {tr.source}->{tr.target}")
@@ -130,7 +154,7 @@ class MetaworkflowGraph:
 
         if not self._validation_suspended:
             source_wf = self.get_workflow_by_id(source)
-            self.validate_param_references(source_wf)
+            self.validate_param_references(source_wf)  # type: ignore
 
     def update_global_options(self, glob: GlobalOptions):
         old_globals = self.global_options
@@ -143,22 +167,41 @@ class MetaworkflowGraph:
     def validate_param_references(self, wf: Workflow):
         errors = []
         for ref in wf.field_refs:
-            if ref.target_wf_id not in self.G.nodes:
-                errors.append(WorkflowReferenceError(ref, f"Reference to unknown workflow: {ref.target_wf_id}"))
+            if ref.namespace != "params":
+                continue  # output references not yet validated
+            referenced_wf = self.get_workflow_by_id(ref.target_wf_id)
+            if not referenced_wf:
+                errors.append(
+                    WorkflowReferenceError(
+                        ref, f"Reference to unknown workflow: {ref.target_wf_id}"
+                    )
+                )
                 continue
-            
+
             if ref.target_wf_id not in list(self.G.predecessors(wf.id)):
-                errors.append(WorkflowReferenceError(ref, f"Reference to workflow {ref.target_wf_id} that is not a predecessor of {wf.id}"))
+                errors.append(
+                    WorkflowReferenceError(
+                        ref,
+                        f"Reference to workflow {ref.target_wf_id} that is not a predecessor of {wf.id}",
+                    )
+                )
                 continue
 
-            referenced_wf = self.get_workflow_by_id(ref.target_wf_id)
             if not referenced_wf.params or not ref.target_key:
-                errors.append(WorkflowReferenceError(ref, f"No referencable params in workflow {referenced_wf.id}"))
+                errors.append(
+                    WorkflowReferenceError(
+                        ref, f"No referencable params in workflow {referenced_wf.id}"
+                    )
+                )
                 continue
-       
+
             if not referenced_wf.params.get(ref.target_key):
-                errors.append(WorkflowReferenceError(ref, f"Reference to unresolvable param: {ref.target_key}"))
-        
+                errors.append(
+                    WorkflowReferenceError(
+                        ref, f"Reference to unresolvable param: {ref.target_key}"
+                    )
+                )
+
         if errors:
             raise WorkflowReferenceErrors(errors)
 
@@ -191,51 +234,55 @@ class MetaworkflowGraph:
     # ===========================
     #   EXPORT BACK TO CONFIG
     # ===========================
-    def to_config(self) -> Dict[str, Any]:
+    def to_config(self) -> MetaworkflowConfig:
         workflows = self.get_workflows()
         transitions = self.get_transitions()
 
-        return MetaworkflowConfig.model_validate({
-            "config_version": self.config_version,
-            "globals": self.global_options,
-            "workflows": workflows,
-            "transitions": transitions,
-        })
+        return MetaworkflowConfig.model_validate(
+            {
+                "config_version": self.config_version,
+                "globals": self.global_options,
+                "workflows": workflows,
+                "transitions": transitions,
+            }
+        )
 
-    def to_file(self, file: Path|str) -> None:
+    def to_file(self, file: Path | str) -> None:
         dump_config(self.to_config(), Path(file))
 
     # ===========================
     #        UTILITIES
     # ===========================
-    def get_workflow_by_id(self, id: str) -> Workflow:
+    def get_workflow_by_id(self, id: str) -> Optional[Workflow]:
         try:
             return self.G.nodes[id].get("workflow")
         except KeyError:
             return None
 
-    def get_transition(self, source: str, target: str) -> Transition:
+    def get_transition(self, source: str, target: str) -> Optional[Transition]:
         try:
             return self.G.edges[(source, target)].get("transition")
         except KeyError:
             return None
 
     def get_transitions(self) -> list[Transition]:
-        return [self.get_transition(*e) for e in self.G.edges]
+        return [self.get_transition(*e) for e in self.G.edges]  # type: ignore
 
     def get_workflows(self) -> list[Workflow]:
-        return [self.get_workflow_by_id(n) for n in self.G.nodes]
+        return [self.get_workflow_by_id(n) for n in self.G.nodes]  # type: ignore
 
     def get_workflows_sorted(self) -> list[Workflow]:
         """Returns workflows in valid execution order."""
         nodes_sorted = list(nx.topological_sort(self.G))
         workflows = [self.get_workflow_by_id(n) for n in nodes_sorted]
-        return workflows
+        return workflows  # type: ignore
 
-    def subset_workflows(self,
-                         start: Optional[str] = None,
-                         target: Optional[str] = None,
-                         workflows: Optional[list[Workflow]] = None):
+    def subset_workflows(
+        self,
+        start: Optional[str] = None,
+        target: Optional[str] = None,
+        workflows: Optional[list[Workflow]] = None,
+    ):
 
         if workflows is None:
             workflows = self.get_workflows_sorted()
@@ -264,10 +311,10 @@ class MetaworkflowGraph:
             return workflows_sorted[0]
 
     def successors(self, wf: Workflow) -> list[Workflow]:
-        return [self.get_workflow_by_id(n) for n in self.G.successors(wf.id)]
+        return [self.get_workflow_by_id(n) for n in self.G.successors(wf.id)]  # type: ignore
 
     def predecessors(self, wf: Workflow) -> list[Workflow]:
-        return [self.get_workflow_by_id(n) for n in self.G.predecessors(wf.id)]
+        return [self.get_workflow_by_id(n) for n in self.G.predecessors(wf.id)]  # type: ignore
 
     @contextmanager
     def deferred_validation(self):