LbAPLocal 0.10.2__tar.gz → 0.11.0__tar.gz

{lbaplocal-0.10.2 → lbaplocal-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: LbAPLocal
-Version: 0.10.2
+Version: 0.11.0
 Summary: Tool to locally run tests for AnalysisProductions
 Author: LHCb
 Project-URL: Homepage, https://gitlab.cern.ch/lhcb-dpa/analysis-productions/lbaplocal
@@ -13,9 +13,10 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: LbAPCommon>=0.15.10
+Requires-Dist: LbAPCommon>=0.19.0
 Requires-Dist: LbProdRun>1.12.2
-Requires-Dist: dirac-cwl>=1.1.3
+Requires-Dist: cwltool>=3.1.20260108
+Requires-Dist: cwl-utils>=0.41
 Requires-Dist: LbDiracWrappers
 Requires-Dist: LbEnv
 Requires-Dist: apd>=0.6.0

{lbaplocal-0.10.2 → lbaplocal-0.11.0}/pyproject.toml

@@ -19,9 +19,11 @@ classifiers = [
 ]
 requires-python = ">=3.12"
 dependencies = [
-    "LbAPCommon>=0.15.10",
+    "LbAPCommon>=0.19.0",
     "LbProdRun>1.12.2",
-    "dirac-cwl>=1.1.3",
+    # floor = oldest verified build that routes JS eval through cwl-utils (set_js_engine)
+    "cwltool>=3.1.20260108",
+    "cwl-utils>=0.41",
     "LbDiracWrappers",
     "LbEnv",
     "apd>=0.6.0",
@@ -57,6 +59,8 @@ testing = [
 [project.scripts]
 lb-ap = "LbAPLocal.cli.main:app"
 lb-ap-legacy = "LbAPLocal.cli.legacy:main"
+lb-ap-run-app = "LbAPLocal.cwl.app_runner:main"
+lb-ap-cwl-run = "LbAPLocal.cwl.runner:app"
 
 [tool.black]
 exclude = '''

{lbaplocal-0.10.2 → lbaplocal-0.11.0}/src/LbAPLocal/cli/main.py

@@ -29,6 +29,7 @@ from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
 from typing import Annotated, Optional
+from urllib.parse import unquote, urlparse
 
 import click
 import typer
@@ -599,7 +600,7 @@ def version_callback(value: bool):
             "LbAPCommon",
             "LbEnv",
             "LbDiracWrappers",
-            "dirac-cwl",
+            "cwltool",
         ]:
             try:
                 version = importlib.metadata.version(package)
@@ -3699,9 +3700,7 @@ def cwl_test(
         lb-ap cwl test MyAnalysis MyJob --pick-smallest-lfn  # Use smallest LFN
         lb-ap cwl test MyAnalysis MyJob --pretty           # Split-view real-time logs
         lb-ap cwl test MyAnalysis MyJob -o ./output
-        lb-ap cwl test MyAnalysis MyJob --executor toil-cwl-runner
     """
-    executor = "dirac-cwl-run"  # Default executor
     handler = get_handler(state.repo_type)
     # Get available productions for wizard
     productions_list = None
@@ -3741,6 +3740,12 @@ def cwl_test(
             )
             raise typer.Exit(code=1)
 
+    # Fail fast if the CWL runner is missing — before any slow work
+    # (YAML rendering, BK queries, replica resolution over the network).
+    if not shutil.which("lb-ap-cwl-run"):
+        err_console.print("[bold red]✗ lb-ap-cwl-run not found on PATH![/bold red]\n")
+        raise typer.Exit(code=1)
+
     # Set up output directory with timestamp for easy identification
     import datetime
 
@@ -3757,7 +3762,6 @@ def cwl_test(
             f"[cyan]Job:[/cyan] {job_name}\n"
             f"[cyan]Events:[/cyan] {n_events or 'default'}\n"
             f"[cyan]Input:[/cyan] {dependent_input or 'default'}\n"
-            f"[cyan]Executor:[/cyan] {executor}\n"
             f"[cyan]Output:[/cyan] {output_dir}",
             title="🧪 CWL Test Configuration",
             border_style="cyan",
@@ -3932,79 +3936,165 @@ def cwl_test(
         # Replace dependent_input with the sliced local file
         dependent_input = str(sliced_path)
 
-    # Step 2b: Rewrite CWL for custom input (if provided)
+    # Step 2b: Resolve/rewrite input data for execution.
+    #
+    # Case A (no -i): resolve inputs directly from the workflow's
+    #   dirac:MetadataQuery via a BK query -- writes a plain inputs.yml,
+    #   no CWL rewriting needed.
+    # Case B (-i <local path>): local file, no resolution needed -- rewrite
+    #   the CWL directly with the given path (skips the producing steps).
+    # Case C (-i LFN:<lfn>): resolve the LFN to a PFN first (BK query for a
+    #   specific LFN), then rewrite the CWL with the resolved PFN string --
+    #   never pass an unresolved "LFN:..." string to
+    #   rewrite_cwl_for_custom_input, which raises on that.
     cwl_inputs_path = None
+    # NOTE: replica_map_path is a user-facing debug artifact (and, in the
+    # LFN case, the resolve_lfn_to_pfn input); unlike the retired DIRAC-based
+    # CWL runner's --replica-map flag, the runner itself never consumes it —
+    # resolved PFNs flow through the inputs YAML instead.
     replica_map_path = None
-    if dependent_input:
+
+    def rewrite_for_custom_input(input_value):
+        """Rewrite the CWL for a resolved custom input, exiting on failure."""
         from LbAPLocal.cwl.rewrite import rewrite_cwl_for_custom_input
 
         console.print(
-            f"\n[bold cyan]Rewriting CWL for custom input:[/bold cyan] {dependent_input}"
+            f"\n[bold cyan]Rewriting CWL for custom input:[/bold cyan] {input_value}"
         )
         try:
-            cwl_workflow_path, cwl_inputs_path = rewrite_cwl_for_custom_input(
-                cwl_workflow_path, [dependent_input], job_name
+            workflow_path, inputs_path = rewrite_cwl_for_custom_input(
+                cwl_workflow_path, [input_value], job_name
             )
-            console.print(
-                f"[green]✓ Modified CWL workflow:[/green] {cwl_workflow_path}"
-            )
-            console.print(f"[green]✓ Custom inputs file:[/green] {cwl_inputs_path}")
         except Exception as e:
             err_console.print(f"[bold red]✗ CWL rewriting failed:[/bold red] {e}")
             import traceback
 
             err_console.print(traceback.format_exc())
             raise typer.Exit(code=1)
+        console.print(f"[green]✓ Modified CWL workflow:[/green] {workflow_path}")
+        console.print(f"[green]✓ Custom inputs file:[/green] {inputs_path}")
+        return workflow_path, inputs_path
+
+    if dependent_input and dependent_input.startswith("LFN:"):
+        console.print(
+            "\n[bold cyan]Generating replica map for custom input LFN[/bold cyan]"
+        )
+        import importlib.util
+
+        spec = importlib.util.find_spec("LbAPLocal.cwl.generate_replica_map")
+        if spec is None or spec.origin is None:
+            err_console.print(
+                "[bold red]✗ Could not find LbAPLocal.cwl.generate_replica_map module[/bold red]"
+            )
+            raise typer.Exit(code=1)
+
+        map_path = output_dir / f"{cwl_workflow_path.stem}-replica-map.json"
+        # Strip the LFN: prefix — DIRAC APIs expect bare paths
+        lfn_path = dependent_input.removeprefix("LFN:")
+        cmd_map = [
+            "lb-dirac",
+            "python",
+            spec.origin,
+            "--lfns",
+            lfn_path,
+            "--output-map",
+            str(map_path),
+        ]
+        if download:
+            cmd_map.append("--download")
+        result = subprocess.run(cmd_map, capture_output=True, text=True)
+        if result.returncode != 0:
+            # generate_replica_map prints errors to stdout via rich Console
+            error_output = result.stderr or result.stdout
+            err_console.print(
+                f"[bold red]✗ Failed to generate replica map:[/bold red]\n{error_output}"
+            )
+            raise typer.Exit(code=1)
+        replica_map_path = map_path
+        console.print(f"[green]✓ Generated replica map:[/green] {replica_map_path}")
+
+        from LbAPLocal.cwl.generate_replica_map import resolve_lfn_to_pfn
+
+        lfn_pfn_map = resolve_lfn_to_pfn(map_path)
+        if not lfn_pfn_map:
+            err_console.print(
+                f"[bold red]✗ Replica map contains no entries:[/bold red] {map_path}"
+            )
+            raise typer.Exit(code=1)
+        resolved_input = next(iter(lfn_pfn_map.values()))
+        console.print(
+            f"[green]✓ Resolved custom input to PFN:[/green] {resolved_input}"
+        )
 
-        # Generate replica map for LFN custom inputs so the executor can resolve them
-        if dependent_input.startswith("LFN:"):
+        cwl_workflow_path, cwl_inputs_path = rewrite_for_custom_input(resolved_input)
+
+    elif dependent_input:
+        cwl_workflow_path, cwl_inputs_path = rewrite_for_custom_input(dependent_input)
+
+    else:
+        # Resolve input data for the default (no -i) case: BK query from the
+        # dirac:MetadataQuery on the workflow's input-data parameter.
+        from LbAPLocal.cwl.generate_replica_map import has_metadata_query
+
+        if not has_metadata_query(cwl_workflow_path):
+            # Simulation workflows whose first step is Gauss have no
+            # input-data parameter at all -- there is nothing to resolve
+            # from Bookkeeping. Leave cwl_inputs_path as None so the
+            # runner is invoked without an inputs file; workflow defaults
+            # (event-type/run-number/number-of-events) apply.
             console.print(
-                "\n[bold cyan]Generating replica map for custom input LFN[/bold cyan]"
+                "[dim]No dirac:MetadataQuery on this workflow (e.g. Gauss-first "
+                "simulation) - skipping input resolution[/dim]"
             )
+        else:
             import importlib.util
 
             spec = importlib.util.find_spec("LbAPLocal.cwl.generate_replica_map")
             if spec is None or spec.origin is None:
                 err_console.print(
-                    "[bold red]✗ Could not find LbAPLocal.cwl.generate_replica_map module[/bold red]"
+                    "[bold red]✗ Could not find generate_replica_map module[/bold red]"
                 )
                 raise typer.Exit(code=1)
 
-            map_path = output_dir / f"{cwl_workflow_path.stem}-replica-map.json"
-            # Strip the LFN: prefix — DIRAC APIs expect bare paths
-            lfn_path = dependent_input.removeprefix("LFN:")
+            cwl_inputs_path = output_dir / f"{cwl_workflow_path.stem}-inputs.yml"
+            replica_map_path = output_dir / f"{cwl_workflow_path.stem}-replica-map.json"
             cmd_map = [
                 "lb-dirac",
                 "python",
                 spec.origin,
-                "--lfns",
-                lfn_path,
+                str(cwl_workflow_path),
+                "--output-yaml",
+                str(cwl_inputs_path),
                 "--output-map",
-                str(map_path),
+                str(replica_map_path),
             ]
-            result = subprocess.run(cmd_map, capture_output=True, text=True)
+            if n_lfns is not None:
+                cmd_map.extend(["--n-lfns", str(n_lfns)])
+            if pick_smallest:
+                cmd_map.append("--pick-smallest-lfn")
+            if download:
+                cmd_map.append("--download")
+
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                console=console,
+            ) as progress:
+                progress.add_task(
+                    "Resolving input data from Bookkeeping...", total=None
+                )
+                result = subprocess.run(cmd_map, capture_output=True, text=True)
             if result.returncode != 0:
-                # generate_replica_map prints errors to stdout via rich Console
                 error_output = result.stderr or result.stdout
                 err_console.print(
-                    f"[bold red]✗ Failed to generate replica map:[/bold red]\n{error_output}"
+                    f"[bold red]✗ Input resolution failed:[/bold red]\n{error_output}"
                 )
                 raise typer.Exit(code=1)
-            replica_map_path = map_path
-            console.print(f"[green]✓ Generated replica map:[/green] {replica_map_path}")
+            console.print(f"[green]✓ Resolved inputs:[/green] {cwl_inputs_path}")
 
     # Step 3: Execute CWL workflow
     console.print("\n[bold cyan]Step 3/3: Executing CWL workflow[/bold cyan]")
 
-    # Check if executor is available
-    executor_check = shutil.which(executor)
-    if not executor_check:
-        err_console.print(
-            f"[bold red]✗ CWL executor '{executor}' not found![/bold red]\n"
-            f"Install with: pip install cwltool (or your preferred CWL runner)"
-        )
-        raise typer.Exit(code=1)
-
     # Set up lb-run environment for local AnalysisProductions testing
     # This creates a fake siteroot so lb-run can find AnalysisProductions.v999999999999 locally
     import asyncio
@@ -4084,49 +4174,22 @@ def cwl_test(
     cwl_output_dir = output_dir / "cwl_output"
     cwl_output_dir.mkdir(parents=True, exist_ok=True)
 
-    # Use absolute paths to avoid path resolution issues
+    # Use absolute paths to avoid path resolution issues. The whole curated
+    # custom_env is passed straight through to the subprocess below; the
+    # runner itself preserves the entire environment (see cwl/runner.py), so
+    # no --preserve-environment allowlisting is needed here.
+    summary_path = output_dir / "summary.json"
     cmd = [
-        executor,
+        "lb-ap-cwl-run",
         "--outdir",
         str(cwl_output_dir.resolve()),
+        "--leave-tmpdir",
+        "--tmpdir-prefix",
+        f"{output_dir}/tmp_step_",
+        "--write-summary",
+        str(summary_path),
     ]
-
-    # Add dirac-cwl-run specific options for input generation
-    if executor == "dirac-cwl-run":
-        if not cwl_inputs_path:
-            # Only add BK query options when not using custom input
-            cmd.extend(["--n-lfns", str(n_lfns)])
-            if pick_smallest:
-                cmd.append("--pick-smallest-lfn")
-            if download:
-                cmd.append("--download")
-            # Force regeneration to avoid interactive prompts
-            cmd.append("--force-regenerate")
-        # Keep temp directories for debugging
-        cmd.append("--leave-tmpdir")
-        # set a specific prefix for temp directories to easily identify them
-        cmd.extend(["--tmpdir-prefix", f"{output_dir}/tmp_step_"])
-        # Enable verbose logging to see what's happening
-        cmd.append("--verbose")
-        cmd.extend(["--preserve-environment", "CMAKE_PREFIX_PATH"])
-        cmd.extend(["--preserve-environment", "ANALYSIS_PRODUCTIONS_DYNAMIC"])
-        # Preserve X509/XRootD authentication variables so that grid
-        # authentication works
-        from LbDiracWrappers import (
-            ENV_VAR_WHITELIST,
-        )  # compiled RE to decide env vars to keep
-
-        for var in sorted(os.environ):
-            if ENV_VAR_WHITELIST.match(var):
-                cmd.extend(["--preserve-environment", var])
-
-        if replica_map_path:
-            cmd.extend(["--replica-map", str(replica_map_path)])
-
-    # Add workflow file
     cmd.append(str(cwl_workflow_path.resolve()))
-
-    # Add inputs file if custom input was provided
     if cwl_inputs_path:
         cmd.append(str(cwl_inputs_path.resolve()))
 
@@ -4159,8 +4222,17 @@ def cwl_test(
     current_run_tmpdirs = []  # Track temp directories used in this run
 
     def find_log_files(include_temp_dirs=False):
-        """Find and return log files, optionally including temp directories"""
-        log_files = list(cwl_output_dir.glob("**/*.log"))
+        """Find and return log files, optionally including temp directories.
+
+        Merges the .log File entries recorded in the runner's
+        --write-summary output object with filesystem globbing (the two
+        sources are deduplicated). On a workflow failure the summary only
+        covers steps that completed, so the failing step's log may exist
+        only in a --leave-tmpdir directory — never let a partial summary
+        short-circuit the temp-dir search.
+        """
+        log_files = _logs_from_summary(summary_path)
+        log_files.extend(cwl_output_dir.glob("**/*.log"))
 
         if include_temp_dirs and current_run_tmpdirs:
             # Only search in temp directories from this specific run
@@ -4293,8 +4365,9 @@ def cwl_test(
 
         raise typer.Exit(code=1)
 
-    # Success! Find and list log files in cwl_output directory only
-    log_files = list(cwl_output_dir.glob("**/*.log"))
+    # Success! Find and list log files (from the summary, falling back to
+    # a directory glob).
+    log_files = find_log_files()
 
     log_files_str = ""
     if log_files:
@@ -4363,6 +4436,50 @@ def cwl_test(
     _show_log_analysis_for_files(log_files, console)
 
 
+def _logs_from_summary(summary_file: Path) -> list[Path]:
+    """Extract .log File paths from the CWL output object.
+
+    ``summary_file`` is the raw Factory output object written by
+    ``lb-ap-cwl-run --write-summary``: File/Directory entries are keyed by
+    ``location`` file-URIs only (no ``path`` key), and the object may
+    contain cwltool-namespaced keys such as
+    ``"http://commonwl.org/cwltool#generation"`` alongside ordinary CWL
+    output names — treat every dict as a plain, URI-keyed bag to walk.
+
+    Success/failure of the run is intentionally NOT considered here: the
+    runner writes this file both on a clean success and on a workflow
+    failure with partial output (see cwl/runner.py); it is only absent for
+    a load error that happened before any job ran. Callers should key off
+    this file's existence, not the runner's exit code, to decide whether
+    there is anything to parse.
+    """
+    if not summary_file.is_file():
+        return []
+    try:
+        output_object = json.loads(summary_file.read_text())
+    except json.JSONDecodeError:
+        return []
+    logs: list[Path] = []
+
+    def walk(node):
+        if isinstance(node, dict):
+            location = str(node.get("location", ""))
+            if (
+                node.get("class") == "File"
+                and location.startswith("file://")
+                and location.endswith(".log")
+            ):
+                logs.append(Path(unquote(urlparse(location).path)))
+            for v in node.values():
+                walk(v)
+        elif isinstance(node, list):
+            for v in node:
+                walk(v)
+
+    walk(output_object)
+    return sorted(set(logs), key=lambda p: p.name)
+
+
 def _show_log_analysis_for_files(log_files: list[Path], console) -> None:
     """Parse Gaudi log files and render analysis summaries."""
     if not log_files: