LbAPLocal 0.12.2__tar.gz → 0.14.0__tar.gz

{lbaplocal-0.12.2 → lbaplocal-0.14.0}/.gitlab-ci.yml

@@ -16,7 +16,7 @@ pixi:
   parallel:
     matrix:
       - task: pre-commit run --all-files
-      - task: pytest --runslow
+      - task: pytest --runslow --cov-report=xml
   rules:
     - when: always
   image: gitlab-registry.cern.ch/lhcb-docker/os-base/alma9-devel:latest
@@ -28,6 +28,12 @@ pixi:
     - source ~/.bashrc || true
   script:
     - pixi run $task
+  coverage: '/TOTAL.*\s+(\d+%)$/'
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
 
 test-build:
   stage: test

{lbaplocal-0.12.2 → lbaplocal-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: LbAPLocal
-Version: 0.12.2
+Version: 0.14.0
 Summary: Tool to locally run tests for AnalysisProductions
 Author: LHCb
 Project-URL: Homepage, https://gitlab.cern.ch/lhcb-dpa/analysis-productions/lbaplocal

{lbaplocal-0.12.2 → lbaplocal-0.14.0}/src/LbAPLocal/cli/legacy.py

@@ -20,7 +20,7 @@ from typing import Iterable
 
 import click
 from apd.authentication import get_auth_headers
-from LbAPCommon import parse_yaml, render_yaml, validate_yaml
+from LbAPCommon import render_yaml, validate_yaml
 
 from ..utils import (
     LBAPI_HOST,
@@ -28,6 +28,7 @@ from ..utils import (
     check_production,
     inside_ap_datapkg,
     logging_subprocess_run,
+    parse_yaml_with_catalog,
     production_name_type,
     production_to_versions,
     validate_environment,
@@ -63,7 +64,14 @@ def main():
 
     NOTE: This is the legacy CLI. Consider using the new 'lb-ap' command instead.
     """
-    pass
+    click.secho(
+        "WARNING: lb-ap-legacy will be retired soon; please switch to 'lb-ap'.\n"
+        "If anything stops you from using 'lb-ap', please open an issue so it "
+        "can be resolved ASAP:\n"
+        "https://gitlab.cern.ch/lhcb-dpa/analysis-productions/lbaplocal/-/work_items",
+        fg="yellow",
+        err=True,
+    )
 
 
 @main.command()
@@ -274,7 +282,7 @@ def list(production_name):
         # Get rendered yaml and find all the production names
         with open(os.path.join(production_name, "info.yaml"), "rt") as fp:
             raw_yaml = fp.read()
-        job_data = parse_yaml(render_yaml(raw_yaml))
+        job_data = parse_yaml_with_catalog(render_yaml(raw_yaml))
         for job_name in job_data:
             click.echo(f"* {job_name}")
     else:
@@ -299,7 +307,7 @@ def render(production_name):
     render = render_yaml(raw_yaml)
     click.echo(render)
     try:
-        job_data = parse_yaml(render)
+        job_data = parse_yaml_with_catalog(render)
         warnings = validate_yaml(job_data, ".", production_name)
     except Exception:
         click.secho("Rendered YAML has errors!", fg="red")
@@ -327,7 +335,7 @@ def validate(production_name):
     render = render_yaml(raw_yaml)
 
     try:
-        job_data = parse_yaml(render)
+        job_data = parse_yaml_with_catalog(render)
     except Exception as e:
         click.secho("Error parsing YAML!", fg="red")
         raise click.ClickException(str(e))
@@ -401,7 +409,7 @@ def test(production_name, job_name, dependent_input, n_events, validate):
     with open(os.path.join(production_name, "info.yaml"), "rt") as fp:
         raw_yaml = fp.read()
     # TODO do we really need to redo parsing and validating when prepare test already does it?
-    prod_data = parse_yaml(render_yaml(raw_yaml))
+    prod_data = parse_yaml_with_catalog(render_yaml(raw_yaml))
     yaml_warnings = validate_yaml(prod_data, ".", production_name)
     try:
         job_data = prod_data[job_name]

{lbaplocal-0.12.2 → lbaplocal-0.14.0}/src/LbAPLocal/cli/main.py

@@ -1830,6 +1830,13 @@ def test(
             help="Validate and render configuration without executing the test",
         ),
     ] = False,
+    no_reasons: Annotated[
+        bool,
+        typer.Option(
+            "--no-reasons",
+            help="Skip LHCbDIRAC reason analysis of failed steps",
+        ),
+    ] = False,
 ):
     """
     🧪 Test a job using CWL workflow
@@ -1896,7 +1903,9 @@ def test(
             )
             raise typer.Exit(code=1)
 
-        from LbAPCommon import parse_yaml, render_yaml, validate_yaml
+        from LbAPCommon import render_yaml, validate_yaml
+
+        from ..utils import parse_yaml_with_catalog
 
         raw_yaml = info_yaml_path.read_text()
         rendered = render_yaml(raw_yaml)
@@ -1908,7 +1917,7 @@ def test(
         syntax = Syntax(rendered, "yaml", theme="monokai", line_numbers=True)
         console.print(syntax)
 
-        jobs_data = parse_yaml(rendered, production_name, None)
+        jobs_data = parse_yaml_with_catalog(rendered, production_name, None)
 
         if job_name not in jobs_data:
             available_jobs = list(jobs_data.keys())
@@ -1948,6 +1957,7 @@ def test(
         pick_smallest=pick_smallest,
         pretty=pretty,
         download=download,
+        no_reasons=no_reasons,
     )
 
 
@@ -2296,6 +2306,13 @@ def parse_log_cmd(
             help="Minimum number of instances required to show WARNING messages",
         ),
     ] = 5,
+    reasons: Annotated[
+        bool,
+        typer.Option(
+            "--reasons",
+            help="Also classify failures with LHCbDIRAC's reason engine (via lb-dirac)",
+        ),
+    ] = False,
 ):
     """Analyse a Gaudi log file and show a summary with advice.
 
@@ -2309,10 +2326,61 @@ def parse_log_cmd(
     analysis = analyse_log(log_text, suppress=suppress)
     render_log_analysis(analysis, console)
 
+    if reasons:
+        from ..reason_analysis import (
+            ReasonAnalysisUnavailable,
+            render_reason_analysis,
+            run_reason_analysis,
+            steps_from_log_files,
+        )
+
+        try:
+            reason_result = run_reason_analysis(steps_from_log_files([log_file]))
+        except ReasonAnalysisUnavailable as e:
+            console.print(f"[dim]Reason analysis unavailable: {e}[/dim]")
+        else:
+            render_reason_analysis(reason_result, console)
+
     if analysis.has_fatal_issues:
         raise typer.Exit(code=1)
 
 
+@app.command(name="analyse-job")
+def analyse_job_cmd(
+    path: Annotated[
+        Path,
+        typer.Argument(
+            help="Run/output directory or a single Gaudi log file to analyse",
+            exists=True,
+        ),
+    ],
+):
+    """Classify failures using LHCbDIRAC's reason engine (runs via lb-dirac).
+
+    Scans application logs (pairing them with prodConf JSONs when present)
+    against the same curated failure patterns used by the production
+    TransformationDebugAgent.
+    """
+    from ..reason_analysis import (
+        ReasonAnalysisUnavailable,
+        render_reason_analysis,
+        run_reason_analysis,
+        steps_from_path,
+    )
+
+    steps = steps_from_path(path)
+    if not steps:
+        err_console.print(f"[bold red]✗ No .log files found under:[/bold red] {path}")
+        raise typer.Exit(code=1)
+    console.print(f"[dim]Analysing {len(steps)} step log(s) via lb-dirac…[/dim]")
+    try:
+        analysis = run_reason_analysis(steps)
+    except ReasonAnalysisUnavailable as e:
+        err_console.print(f"[bold red]✗ Reason analysis unavailable:[/bold red] {e}")
+        raise typer.Exit(code=1)
+    render_reason_analysis(analysis, console)
+
+
 # ============================================================================
 # Reproducer Commands
 # ============================================================================
@@ -3877,6 +3945,13 @@ def cwl_test(
             help="Download input files locally instead of using remote protocol where available",
         ),
     ] = False,
+    no_reasons: Annotated[
+        bool,
+        typer.Option(
+            "--no-reasons",
+            help="Skip LHCbDIRAC reason analysis of failed steps",
+        ),
+    ] = False,
 ):
     """
     🧪 Test a job using CWL workflow
@@ -4557,6 +4632,7 @@ def cwl_test(
 
         # Analyse log files to show what went wrong
         _show_log_analysis_for_files(log_files, err_console)
+        _maybe_show_reason_analysis(log_files, err_console, no_reasons)
 
         raise typer.Exit(code=1)
 
@@ -4707,6 +4783,29 @@ def _show_log_analysis_for_files(log_files: list[Path], console) -> None:
                 )
 
 
+def _maybe_show_reason_analysis(
+    log_files: list[Path], console, no_reasons: bool
+) -> None:
+    """Run the LHCbDIRAC reason engine over failed-run logs. Never raises."""
+    from ..reason_analysis import (
+        ReasonAnalysisUnavailable,
+        reason_analysis_disabled,
+        render_reason_analysis,
+        run_reason_analysis,
+        steps_from_log_files,
+    )
+
+    if no_reasons or reason_analysis_disabled() or not log_files:
+        return
+    console.print("\n[dim]Running LHCbDIRAC reason analysis (lb-dirac)…[/dim]")
+    try:
+        analysis = run_reason_analysis(steps_from_log_files(log_files))
+    except ReasonAnalysisUnavailable as e:
+        console.print(f"[dim]Reason analysis unavailable: {e}[/dim]")
+        return
+    render_reason_analysis(analysis, console)
+
+
 # ============================================================================
 # Helper Functions
 # ============================================================================

{lbaplocal-0.12.2 → lbaplocal-0.14.0}/src/LbAPLocal/cli/wizards.py

@@ -247,7 +247,9 @@ def test_wizard(productions: Optional[List[str]] = None) -> dict:
     Returns:
         Dictionary with test parameters or None if cancelled
     """
-    from LbAPCommon import parse_yaml, render_yaml
+    from LbAPCommon import render_yaml
+
+    from ..utils import parse_yaml_with_catalog
 
     welcome_wizard()
 
@@ -271,7 +273,7 @@ def test_wizard(productions: Optional[List[str]] = None) -> dict:
         if info_yaml_path.exists():
             with open(info_yaml_path, "rt") as fp:
                 raw_yaml = fp.read()
-            job_data = parse_yaml(render_yaml(raw_yaml))
+            job_data = parse_yaml_with_catalog(render_yaml(raw_yaml))
             jobs = list(job_data.keys())
 
             if jobs:
@@ -338,7 +340,9 @@ def cwl_generate_wizard(productions: Optional[List[str]] = None) -> dict:
     Returns:
         Dictionary with generation parameters or None if cancelled
     """
-    from LbAPCommon import parse_yaml, render_yaml
+    from LbAPCommon import render_yaml
+
+    from ..utils import parse_yaml_with_catalog
 
     welcome_wizard()
 
@@ -372,7 +376,7 @@ def cwl_generate_wizard(productions: Optional[List[str]] = None) -> dict:
         if info_yaml_path.exists():
             with open(info_yaml_path, "rt") as fp:
                 raw_yaml = fp.read()
-            job_data = parse_yaml(render_yaml(raw_yaml))
+            job_data = parse_yaml_with_catalog(render_yaml(raw_yaml))
             jobs = list(job_data.keys())
 
             if jobs:

{lbaplocal-0.12.2 → lbaplocal-0.14.0}/src/LbAPLocal/handlers/anaprod.py

@@ -43,12 +43,14 @@ class APHandler:
         return [dirname(f) for f in folders]
 
     def list_jobs(self, production: str) -> list[str]:
-        from LbAPCommon import parse_yaml, render_yaml
+        from LbAPCommon import render_yaml
+
+        from LbAPLocal.utils import parse_yaml_with_catalog
 
         info_yaml = Path(production) / "info.yaml"
         raw = info_yaml.read_text()
         rendered = render_yaml(raw)
-        jobs = parse_yaml(rendered, production, None)
+        jobs = parse_yaml_with_catalog(rendered, production, None)
         return list(jobs.keys())
 
     def generate_request_yaml(

lbaplocal-0.14.0/src/LbAPLocal/reason_analysis.py

@@ -0,0 +1,281 @@
+###############################################################################
+# (c) Copyright 2026 CERN for the benefit of the LHCb Collaboration           #
+#                                                                             #
+# This software is distributed under the terms of the GNU General Public      #
+# Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING".   #
+#                                                                             #
+# In applying this licence, CERN does not waive the privileges and immunities #
+# granted to it by virtue of its status as an Intergovernmental Organization  #
+# or submit itself to any jurisdiction.                                       #
+###############################################################################
+"""Host side of the LHCbDIRAC reason-engine integration.
+
+Discovers local run artifacts, runs ``reason_driver.py`` inside the
+``lb-dirac`` (CVMFS LHCbDIRAC) environment, parses its JSON output and
+renders the result with Rich.  See the accepted design
+"LbAPLocal Log Analysis via lb-dirac".
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import subprocess
+import tempfile
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+SCHEMA_VERSION = 1
+
+
+class ReasonAnalysisUnavailable(RuntimeError):
+    """Reason analysis could not run; callers degrade gracefully."""
+
+
+class ReasonAnalysisError(ReasonAnalysisUnavailable):
+    """The driver produced output we cannot interpret."""
+
+
+@dataclass(frozen=True)
+class ReasonHit:
+    type: str
+    category: str
+    match: str
+    explanation: str | None
+    issue_url: str | None
+    scope: str
+    priority: int
+    count: int
+
+
+@dataclass(frozen=True)
+class StepReasons:
+    name: str
+    mode: str
+    reasons: list[ReasonHit]
+
+
+@dataclass(frozen=True)
+class ReasonAnalysis:
+    lhcbdirac_version: str | None
+    steps: list[StepReasons]
+    errors: list[str]
+
+
+def parse_driver_output(text: str) -> ReasonAnalysis:
+    """Parse and validate the driver's JSON output.
+
+    Raises:
+        ReasonAnalysisError: on invalid JSON, schema mismatch, or missing fields.
+    """
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError as e:
+        raise ReasonAnalysisError(f"driver returned invalid JSON: {e}") from e
+    if not isinstance(data, dict):
+        raise ReasonAnalysisError(
+            f"driver output is not a JSON object: {type(data).__name__}"
+        )
+    if data.get("schema_version") != SCHEMA_VERSION:
+        raise ReasonAnalysisError(
+            "unexpected schema_version: "
+            f"{data.get('schema_version')!r} (expected {SCHEMA_VERSION})"
+        )
+    try:
+        steps = []
+        for raw_step in data.get("steps", []):
+            reasons = [
+                ReasonHit(
+                    type=r["type"],
+                    category=r["category"],
+                    match=r["match"],
+                    explanation=r.get("explanation"),
+                    issue_url=r.get("issue_url"),
+                    scope=r.get("scope", "unattributed"),
+                    priority=int(r.get("priority", 0)),
+                    count=int(r.get("count", 1)),
+                )
+                for r in raw_step.get("reasons", [])
+            ]
+            steps.append(
+                StepReasons(
+                    name=raw_step["name"],
+                    mode=raw_step.get("mode", "flat"),
+                    reasons=reasons,
+                )
+            )
+        engine = data.get("engine") or {}
+        return ReasonAnalysis(
+            lhcbdirac_version=engine.get("lhcbdirac_version"),
+            steps=steps,
+            errors=[str(e) for e in data.get("errors", [])],
+        )
+    except (KeyError, TypeError, ValueError, AttributeError) as e:
+        raise ReasonAnalysisError(f"malformed driver output: {e!r}") from e
+
+
+@dataclass(frozen=True)
+class StepInput:
+    name: str
+    log: Path
+    prodconf: Path | None
+
+
+def _find_prodconf(log: Path) -> Path | None:
+    """Pair a step log with its prodConf JSON, mirroring job_slice.find_failed_step."""
+    exact = log.parent / f"prodConf_{log.stem}.json"
+    if exact.is_file():
+        return exact
+    candidates = sorted(log.parent.glob(f"prodConf*{log.stem}*.json"))
+    return candidates[0] if candidates else None
+
+
+def steps_from_log_files(log_files: Iterable[Path]) -> list[StepInput]:
+    """Build driver step inputs from application log paths."""
+    steps = []
+    for log in set(log_files):
+        steps.append(StepInput(name=log.stem, log=log, prodconf=_find_prodconf(log)))
+    steps.sort(key=lambda s: s.name)
+    return steps
+
+
+def steps_from_path(path: Path) -> list[StepInput]:
+    """Discover step inputs from a run directory or a single log file.
+
+    A nonexistent path yields an empty list; callers decide how to report that.
+    """
+    if path.is_file():
+        return steps_from_log_files([path])
+    return steps_from_log_files(path.rglob("*.log"))
+
+
+DRIVER_TIMEOUT = 120.0
+
+
+def reason_analysis_disabled() -> bool:
+    """True when the user opted out via LBAP_NO_REASON_ANALYSIS."""
+    return bool(os.environ.get("LBAP_NO_REASON_ANALYSIS"))
+
+
+def run_reason_analysis(
+    steps: list[StepInput], *, timeout: float = DRIVER_TIMEOUT
+) -> ReasonAnalysis:
+    """Run reason_driver.py under lb-dirac and return the parsed result.
+
+    Raises:
+        ReasonAnalysisUnavailable: for every infrastructure failure --
+            missing lb-dirac, timeout, driver crash, or malformed output.
+    """
+    if not steps:
+        raise ReasonAnalysisUnavailable("no log files to analyse")
+    spec = importlib.util.find_spec("LbAPLocal.reason_driver")
+    if spec is None or spec.origin is None:
+        raise ReasonAnalysisUnavailable("could not locate LbAPLocal.reason_driver")
+
+    request = {
+        "steps": [
+            {
+                "name": s.name,
+                "log": str(s.log),
+                "prodconf": str(s.prodconf) if s.prodconf else None,
+            }
+            for s in steps
+        ]
+    }
+    with tempfile.NamedTemporaryFile(
+        "w", suffix=".json", prefix="lbap-reasons-", delete=False
+    ) as f:
+        json.dump(request, f)
+        request_path = f.name
+    try:
+        try:
+            result = subprocess.run(
+                ["lb-dirac", "python", spec.origin, request_path],
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+        except FileNotFoundError as e:
+            raise ReasonAnalysisUnavailable(
+                "lb-dirac not found (is CVMFS available?)"
+            ) from e
+        except subprocess.TimeoutExpired as e:
+            raise ReasonAnalysisUnavailable(
+                f"reason analysis timed out after {timeout:.0f}s"
+            ) from e
+        if result.returncode != 0:
+            detail = (result.stderr or result.stdout or "").strip()
+            raise ReasonAnalysisUnavailable(
+                f"driver failed (exit {result.returncode}): {detail[:500]}"
+            )
+        return parse_driver_output(result.stdout)
+    finally:
+        os.unlink(request_path)
+
+
+_CATEGORY_STYLES = {
+    "KnownUnrecoverableFailure": "red",
+    "UnknownFailure": "red",
+    "EventTimeout": "dark_orange",
+    "RecoverableFailureWithManualIntervention": "yellow",
+    "WorkerNodeIssue": "yellow",
+    "RecoverableFailure": "yellow",
+}
+_MAX_MATCH_DISPLAY_LINES = 6
+
+
+def render_reason_analysis(analysis: ReasonAnalysis, console) -> None:
+    """Render a ReasonAnalysis with Rich panels (one per step with findings)."""
+    from rich.console import Group
+    from rich.panel import Panel
+    from rich.text import Text
+
+    for step in analysis.steps:
+        if not step.reasons:
+            console.print(
+                f"[dim]Reason analysis ({step.name}): "
+                "no known failure patterns matched[/dim]"
+            )
+            continue
+        parts = []
+        border = "yellow"
+        for hit in step.reasons:
+            style = _CATEGORY_STYLES.get(hit.category, "yellow")
+            if style == "red":
+                border = "red"
+            header = Text()
+            header.append(hit.type, style=f"bold {style}")
+            header.append(f"  ({hit.category}, {hit.scope}", style="dim")
+            if hit.count > 1:
+                header.append(f", {hit.count}×", style="dim")
+            header.append(")", style="dim")
+            parts.append(header)
+            if hit.explanation:
+                parts.append(Text(hit.explanation))
+            if hit.issue_url:
+                parts.append(Text(hit.issue_url, style="cyan underline"))
+            match_lines = hit.match.strip().splitlines()
+            snippet = "\n".join(match_lines[:_MAX_MATCH_DISPLAY_LINES])
+            if len(match_lines) > _MAX_MATCH_DISPLAY_LINES:
+                snippet += (
+                    f"\n… ({len(match_lines) - _MAX_MATCH_DISPLAY_LINES} more lines)"
+                )
+            parts.append(Text(snippet, style="dim"))
+            parts.append(Text())
+        console.print(
+            Panel(
+                Group(*parts),
+                title=f"[bold]Reason analysis: {step.name}[/bold]"
+                + (" [dim](unsegmented)[/dim]" if step.mode == "flat" else ""),
+                title_align="left",
+                border_style=border,
+            )
+        )
+    for err in analysis.errors:
+        console.print(f"[dim]Reason analysis: {err}[/dim]")
+    if analysis.lhcbdirac_version:
+        console.print(
+            f"[dim]Reason engine: LHCbDIRAC {analysis.lhcbdirac_version}[/dim]"
+        )