nthlayer-workers 1.7.1__tar.gz → 2.0.0__tar.gz

{nthlayer_workers-1.7.1 → nthlayer_workers-2.0.0}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: nthlayer-workers
-Version: 1.7.1
+Version: 2.0.0
 Summary: NthLayer workers — Tier 2 background computation (observe, measure, correlate, respond, learn)
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: nthlayer-common<2.0.0,>=1.5.0
+Requires-Dist: nthlayer-common<3.0.0,>=2.1.2
 Requires-Dist: httpx>=0.27
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: structlog>=24.1.0

{nthlayer_workers-1.7.1 → nthlayer_workers-2.0.0}/pyproject.toml

@@ -4,12 +4,24 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nthlayer-workers"
-version = "1.7.1"
+version = "2.0.0"
 description = "NthLayer workers — Tier 2 background computation (observe, measure, correlate, respond, learn)"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
-    "nthlayer-common>=1.5.0,<2.0.0",
+    # >=2.1.2, NOT >=2.1.0 or 2.1.1. 2.1.0 introduced foreign_yaml_reason with an
+    # INERT body-recovery step: it tested spec.slos as a list, but v1 writes
+    # a mapping and v2 has no spec.slos at all, so a header-stripped manifest
+    # was dropped with parse_failures == 0 — the exact silent drop this
+    # package uses the gate to prevent. Fixed across 2.1.1 and 2.1.2;
+    # 2.1.1 still counted every Backstage catalog-info.yaml as a broken
+    # manifest, which 2.1.2 corrected.
+    #
+    # CI resolves nthlayer-common from a sibling checkout, so it would not
+    # notice a bad resolve. tests/observe/test_slo_spec_loader.py's
+    # TestSharedGateResolvesCorrectly exists for that: it uses the shapes the
+    # PARSERS produce and fails against 2.1.0.
+    "nthlayer-common>=2.1.2,<3.0.0",
     "httpx>=0.27",
     "pyyaml>=6.0",
     "structlog>=24.1.0",

nthlayer_workers-2.0.0/src/nthlayer_workers/cli_output.py

@@ -0,0 +1,26 @@
+"""Operator-facing CLI output shared across worker modules."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+
+def warn_parse_failures(parse_failures: int, specs_dir: str | Path) -> None:
+    """Tell the operator when the SLOs that follow cover only part of a directory.
+
+    Surfaced on stderr rather than only logged: whoever reads SLO output is
+    the one who needs to know the view is partial, and a warning in a log
+    they are not tailing does not reach them (opensrm-3470).
+
+    Takes the count, not a ``LoadedSpecs`` — observe and measure each have
+    their own, carrying differently-shaped SLOs under differently-named
+    fields (``service_slos`` vs ``slos``). The only thing they share is this
+    number, so that is the whole parameter (opensrm-fxln).
+    """
+    if parse_failures:
+        print(
+            f"Warning: {parse_failures} manifest(s) in {specs_dir} "
+            f"failed to parse and were not evaluated",
+            file=sys.stderr,
+        )

{nthlayer_workers-1.7.1 → nthlayer_workers-2.0.0}/src/nthlayer_workers/learn/cli.py

@@ -139,6 +139,15 @@ def _cmd_retrospective(args: argparse.Namespace) -> None:
         impact = custom.get("financial_impact")
         if impact:
             print(f"  Financial impact: ${impact.get('estimated', 0):.2f} ({impact.get('failure_mode')})")
+        # opensrm-oh27: an unparseable manifest drops its service from the
+        # figures above. Print the count so the reader knows the numbers
+        # were computed over a subset rather than over everything.
+        parse_failures = custom.get("manifest_parse_failures", 0)
+        if parse_failures:
+            print(
+                f"  Manifest parse failures: {parse_failures} "
+                "(figures computed over a subset; see manifest_parse_failed logs)"
+            )
     except KeyError as e:
         print(f"Error: {e}", file=__import__("sys").stderr)
         __import__("sys").exit(1)

{nthlayer_workers-1.7.1 → nthlayer_workers-2.0.0}/src/nthlayer_workers/learn/retrospective.py

@@ -1,6 +1,7 @@
 """Retrospective builder — walks verdict lineage to produce a post-incident analysis."""
 from __future__ import annotations
 
+from dataclasses import dataclass
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import Any
@@ -9,6 +10,8 @@ import structlog
 from nthlayer_common.manifest import (
     ManifestLoadError,
     extract_declared_dependencies,
+    foreign_yaml_reason,
+    iter_manifest_files,
     load_manifest,
 )
 from nthlayer_common.outcomes import (
@@ -116,8 +119,8 @@ def build_retrospective(
     # ground-truth view of operator-declared deps that downstream
     # add_dependency recommendation logic compares against the
     # incident's observed call graph.
-    loaded_manifests = _load_manifests_from_specs(specs_dir)
-    declared_dependencies_by_service = _extract_declared_dependencies(loaded_manifests)
+    loaded_specs = _load_manifests_from_specs(specs_dir)
+    declared_dependencies_by_service = _extract_declared_dependencies(loaded_specs.manifests)
 
     # Financial impact (if specs available)
     financial_impact = _compute_financial_impact(
@@ -125,7 +128,7 @@ def build_retrospective(
         duration_minutes,
         breach_counts_by_service,
         specs_dir,
-        loaded_manifests=loaded_manifests,
+        manifests_by_service=loaded_specs.manifests,
     )
 
     # Generate recommendations
@@ -155,6 +158,10 @@ def build_retrospective(
         "financial_impact": financial_impact,
         "recommendations": recommendations,
         "declared_dependencies_by_service": declared_dependencies_by_service,
+        # opensrm-oh27: always present, 0 on a clean load. An absent key
+        # would read the same as "no failures", which is the ambiguity
+        # the silent skip created in the first place.
+        "manifest_parse_failures": loaded_specs.parse_failures,
     }
     if trigger_service is not None:
         retro_custom["trigger_service"] = trigger_service
@@ -225,32 +232,74 @@ def _parse_ts(ts: Any) -> datetime:
     return dt
 
 
-def _load_manifests_from_specs(specs_dir: str | None) -> dict[str, Any]:
+@dataclass(frozen=True)
+class LoadedManifests:
+    """Outcome of scanning a specs directory (opensrm-oh27).
+
+    ``manifests`` is the parsed view keyed by service name; ``parse_failures``
+    counts *files* that raised ``ManifestLoadError``, not services missing
+    from ``manifests`` — a failed environment overlay for a service whose
+    base manifest parsed still counts one. The count is deliberately a
+    coverage-doubt signal rather than a precise miss-count, and it travels
+    with the manifests because the downstream financial figure is computed
+    over ``manifests`` alone: without it, a partial view is
+    indistinguishable from a complete one. The per-file
+    ``manifest_parse_failed`` warning carries which file and why.
+    """
+
+    manifests: dict[str, Any]
+    parse_failures: int = 0
+
+
+def _load_manifests_from_specs(specs_dir: str | None) -> LoadedManifests:
     """Load all ``*.yaml`` manifests under ``specs_dir`` once, keyed by
     ``manifest.name`` (the canonical service identity).
 
-    Returns an empty dict when ``specs_dir`` is None or not a directory,
+    Returns empty manifests when ``specs_dir`` is None or not a directory,
     so callers can treat the result uniformly without re-checking the
     filesystem. Duplicate manifests for the same service (template forks,
     environment overlays) are deduplicated by keeping the first one
     encountered and emitting a warning — this matches the pre-jmy.21
     behaviour of ``_compute_financial_impact``.
 
-    Parse failures from ``load_manifest`` are silently skipped (consistent
-    with the prior inline ``ManifestLoadError`` swallow); the financial-
-    impact and dependency-extraction paths both tolerate a partial view.
+    Parse failures from ``load_manifest`` are logged and counted, then
+    skipped (opensrm-oh27). ``UnicodeDecodeError`` is caught alongside
+    ``ManifestLoadError`` because ``load_manifest`` opens with the default
+    encoding and only wraps ``YAMLError`` — non-UTF-8 bytes would otherwise
+    abort the whole retrospective, a hard stop where the design says log and
+    continue. The financial-impact and dependency-extraction
+    paths both tolerate a partial view, but the caller is told how partial
+    it is via ``LoadedManifests.parse_failures``.
     """
     if not specs_dir:
-        return {}
+        return LoadedManifests(manifests={})
     specs_path = Path(specs_dir)
     if not specs_path.is_dir():
-        return {}
+        return LoadedManifests(manifests={})
 
     manifests: dict[str, Any] = {}
-    for spec_file in specs_path.glob("*.yaml"):
+    parse_failures = 0
+    for spec_file in iter_manifest_files(specs_path):
         try:
             manifest = load_manifest(str(spec_file))
-        except ManifestLoadError:
+        except (ManifestLoadError, UnicodeDecodeError) as exc:
+            foreign_reason = foreign_yaml_reason(spec_file)
+            if foreign_reason is not None:
+                # Foreign YAML sharing the directory, not a broken manifest.
+                # Recorded rather than dropped without trace — skipping
+                # without saying so is the shape this bead exists to remove.
+                log.debug(
+                    "manifest_file_ignored",
+                    spec_file=str(spec_file),
+                    reason=foreign_reason,
+                )
+                continue
+            parse_failures += 1
+            log.warning(
+                "manifest_parse_failed",
+                spec_file=str(spec_file),
+                error=str(exc),
+            )
             continue
         if manifest.name in manifests:
             log.warning(
@@ -260,16 +309,16 @@ def _load_manifests_from_specs(specs_dir: str | None) -> dict[str, Any]:
             )
             continue
         manifests[manifest.name] = manifest
-    return manifests
+    return LoadedManifests(manifests=manifests, parse_failures=parse_failures)
 
 
 def _extract_declared_dependencies(
-    loaded_manifests: dict[str, Any],
+    manifests_by_service: dict[str, Any],
 ) -> dict[str, list[str]]:
     """Wrapper around the shared helper (opensrm-dpws). The CLI path
     works with Manifest dataclass instances loaded via load_manifest().
     """
-    return extract_declared_dependencies(from_manifests=loaded_manifests)
+    return extract_declared_dependencies(from_manifests=manifests_by_service)
 
 
 def _compute_financial_impact(
@@ -278,7 +327,7 @@ def _compute_financial_impact(
     breach_counts_by_service: dict[str, int],
     specs_dir: str | None,
     *,
-    loaded_manifests: dict[str, Any] | None = None,
+    manifests_by_service: dict[str, Any] | None = None,
 ) -> dict | None:
     """Compute spec § 1 financial impact across blast-radius services.
 
@@ -289,10 +338,14 @@ def _compute_financial_impact(
     services that share a currency; mixed-currency manifests are
     skipped with a warning.
 
-    ``loaded_manifests`` is the shared cache produced by
-    ``_load_manifests_from_specs`` (opensrm-jmy.21). When omitted the
-    function loads manifests itself — preserves the pre-jmy.21 calling
-    convention used by ``tests/learn/test_retrospective_financial.py``.
+    ``manifests_by_service`` is the shared cache produced by
+    ``_load_manifests_from_specs(...).manifests`` (opensrm-jmy.21). When
+    omitted the function loads manifests itself — preserves the pre-jmy.21
+    calling convention used by ``tests/learn/test_retrospective_financial.py``.
+    That self-loading path discards the parse-failure count; only
+    ``build_retrospective`` surfaces it (as
+    ``metadata.custom["manifest_parse_failures"]``), and the per-file
+    warning log fires either way.
     """
     if not specs_dir or not blast_radius:
         return None
@@ -307,9 +360,9 @@ def _compute_financial_impact(
         )
         return None
 
-    if loaded_manifests is None:
-        loaded_manifests = _load_manifests_from_specs(specs_dir)
-    if not loaded_manifests:
+    if manifests_by_service is None:
+        manifests_by_service = _load_manifests_from_specs(specs_dir).manifests
+    if not manifests_by_service:
         return None
 
     affected_services = {
@@ -322,7 +375,7 @@ def _compute_financial_impact(
     currency: str | None = None
     aggregate_source: str | None = None
 
-    for service_name, manifest in loaded_manifests.items():
+    for service_name, manifest in manifests_by_service.items():
         if service_name not in affected_services:
             continue
         if manifest.outcomes is None:

{nthlayer_workers-1.7.1 → nthlayer_workers-2.0.0}/src/nthlayer_workers/learn/worker.py

@@ -323,8 +323,8 @@ class LearnRetrospectiveModule:
                     ]
                     if len(trigger_matches) > 1:
                         # First-wins (matches the existing
-                        # _load_manifests_from_specs dedup behaviour
-                        # on the CLI side, retrospective.py line 238).
+                        # _load_manifests_from_specs dedup behaviour on the
+                        # CLI side, which logs manifest_duplicate_skipped).
                         logger.warning(
                             "learn_manifest_duplicate_skipped",
                             service=trigger_service,