nthlayer-workers 1.7.1__tar.gz → 2.0.1__tar.gz

{nthlayer_workers-1.7.1 → nthlayer_workers-2.0.1}/PKG-INFO

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: nthlayer-workers
-Version: 1.7.1
+Version: 2.0.1
 Summary: NthLayer workers — Tier 2 background computation (observe, measure, correlate, respond, learn)
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
-Requires-Dist: nthlayer-common<2.0.0,>=1.5.0
+Requires-Dist: nthlayer-common<4.0.0,>=3.0.0
 Requires-Dist: httpx>=0.27
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: structlog>=24.1.0

nthlayer_workers-2.0.1/pyproject.toml

@@ -0,0 +1,110 @@
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "nthlayer-workers"
+version = "2.0.1"
+description = "NthLayer workers — Tier 2 background computation (observe, measure, correlate, respond, learn)"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    # >=3.0.0, NOT >=2.1.2. Two of the six breaks documented in
+    # nthlayer-common/docs/upgrading-3.0.md reach this package. Labelled rather
+    # than numbered, because that doc numbers its items 1-6 globally and its
+    # item 2 is a different break from either of these.
+    #
+    # BREAK A (that doc's item 1) — target conversion moved to the INBOUND
+    #   manifest boundary (opensrm-ocvu), so a v2 objective target of 0.999 now
+    #   parses to 99.9 where it used to parse to 0.999. The measure adapter
+    #   compares SLODefinition.target against a measured value directly, so the
+    #   two conventions disagree on the v2 path — exactly 100x for classical
+    #   targets, while judgment maxima both complement AND scale (0.05 -> 95.0).
+    #
+    # BREAK B (that doc's item 6) — iter_manifest_files returns one manifest per
+    #   stem (opensrm-xvwt), which retires this package's own same-stem
+    #   duplicate handling. The remaining service-name duplicate branch in
+    #   learn/retrospective.py is still live and is covered by
+    #   TestDuplicateServiceAcrossDifferentStems.
+    #
+    # So the floor must admit only the convention this code is written against.
+    #
+    # The historical 2.1.2 floor is inherited and still satisfied: 2.1.0's
+    # foreign_yaml_reason had an INERT body-recovery step (it tested spec.slos
+    # as a list, but v1 writes a mapping and v2 has no such key), fixed across
+    # 2.1.1 and 2.1.2 — and what made it 2.1.2 specifically rather than 2.1.1
+    # is that 2.1.1 still counted every Backstage catalog-info.yaml as a broken
+    # manifest.
+    #
+    # CI resolves nthlayer-common from a sibling checkout, so it would not
+    # notice a bad resolve. tests/observe/test_slo_spec_loader.py's
+    # TestSharedGateResolvesCorrectly exists for that: it uses the shapes the
+    # PARSERS produce and fails against 2.1.0.
+    #
+    # tests/test_dependency_declarations.py guards THIS declaration as of
+    # [opensrm-ir5m] — ported from nthlayer-core once a mutation showed the
+    # range could be reverted to >=2.1.2,<3.0.0 with all 1958 tests still
+    # green. Still missing, and tracked as opensrm-p62o: the EXPECTED_MAJORS
+    # half, which reads the BUILT wheel's own metadata in the release
+    # container, where deps come from PyPI instead of the sibling checkout.
+    "nthlayer-common>=3.0.0,<4.0.0",
+    "httpx>=0.27",
+    "pyyaml>=6.0",
+    "structlog>=24.1.0",
+    "scipy>=1.11",
+    "numpy>=1.24",
+    "starlette>=0.40",
+    "uvicorn>=0.30",
+    # Required by ApprovalServer's Slack interaction endpoint at
+    # `respond/server.py:197` — `await request.form()` hard-imports
+    # python-multipart via Starlette. Operators running with
+    # `slack_signing_secret` set need this at runtime, not just for
+    # tests; declared as a top-level production dep.
+    "python-multipart>=0.0.18",
+    "ruamel.yaml>=0.18",
+    "textual>=1.0",
+]
+
+[project.scripts]
+nthlayer-workers = "nthlayer_workers.cli:main"
+nthlayer-learn = "nthlayer_workers.learn.cli:main"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.2",
+    "pytest-asyncio>=0.23",
+    "respx>=0.21",
+    "ruff>=0.8",
+]
+
+[tool.uv.sources]
+nthlayer-common = { path = "../nthlayer-common", editable = true }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+
+# Ruff configuration (opensrm-d96q floor). Explicit defaults so future
+# violations show up as new findings rather than blending in. Tightening
+# beyond E/F (isort/pyupgrade/simplify/bugbear) is tracked in
+# opensrm-d96q.1 .. d96q.4 — each rule family is its own bead because
+# the cumulative diff across all four is ~640 findings.
+[tool.ruff]
+target-version = "py311"
+line-length = 100
+
+[tool.ruff.lint]
+# Mirrors ruff's literal default select (E4/E7/E9 + F) plus I (isort,
+# opensrm-ncy0), UP (pyupgrade, opensrm-6nn1), SIM (simplify,
+# opensrm-y7dd), and B (bugbear, opensrm-po23). E501 / W are separate
+# hygiene calls (line-too-long has 393 hits today).
+select = ["E4", "E7", "E9", "F", "I", "UP", "SIM", "B"]
+
+[tool.ruff.lint.per-file-ignores]
+# Correlate test files defer some imports until after pytest.importorskip
+# / pytestmark blocks (the modules under test optional-import heavy deps).
+# Pin E402 here instead of scattering noqa: E402 across the call sites.
+"tests/correlate/test_*.py" = ["E402"]

nthlayer_workers-2.0.1/src/nthlayer_workers/cli_output.py

@@ -0,0 +1,26 @@
+"""Operator-facing CLI output shared across worker modules."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+
+def warn_parse_failures(parse_failures: int, specs_dir: str | Path) -> None:
+    """Tell the operator when the SLOs that follow cover only part of a directory.
+
+    Surfaced on stderr rather than only logged: whoever reads SLO output is
+    the one who needs to know the view is partial, and a warning in a log
+    they are not tailing does not reach them (opensrm-3470).
+
+    Takes the count, not a ``LoadedSpecs`` — observe and measure each have
+    their own, carrying differently-shaped SLOs under differently-named
+    fields (``service_slos`` vs ``slos``). The only thing they share is this
+    number, so that is the whole parameter (opensrm-fxln).
+    """
+    if parse_failures:
+        print(
+            f"Warning: {parse_failures} manifest(s) in {specs_dir} "
+            f"failed to parse and were not evaluated",
+            file=sys.stderr,
+        )

{nthlayer_workers-1.7.1 → nthlayer_workers-2.0.1}/src/nthlayer_workers/learn/cli.py

@@ -139,6 +139,15 @@ def _cmd_retrospective(args: argparse.Namespace) -> None:
         impact = custom.get("financial_impact")
         if impact:
             print(f"  Financial impact: ${impact.get('estimated', 0):.2f} ({impact.get('failure_mode')})")
+        # opensrm-oh27: an unparseable manifest drops its service from the
+        # figures above. Print the count so the reader knows the numbers
+        # were computed over a subset rather than over everything.
+        parse_failures = custom.get("manifest_parse_failures", 0)
+        if parse_failures:
+            print(
+                f"  Manifest parse failures: {parse_failures} "
+                "(figures computed over a subset; see manifest_parse_failed logs)"
+            )
     except KeyError as e:
         print(f"Error: {e}", file=__import__("sys").stderr)
         __import__("sys").exit(1)

{nthlayer_workers-1.7.1 → nthlayer_workers-2.0.1}/src/nthlayer_workers/learn/retrospective.py

@@ -1,6 +1,7 @@
 """Retrospective builder — walks verdict lineage to produce a post-incident analysis."""
 from __future__ import annotations
 
+from dataclasses import dataclass
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import Any
@@ -9,6 +10,8 @@ import structlog
 from nthlayer_common.manifest import (
     ManifestLoadError,
     extract_declared_dependencies,
+    foreign_yaml_reason,
+    iter_manifest_files,
     load_manifest,
 )
 from nthlayer_common.outcomes import (
@@ -116,8 +119,8 @@ def build_retrospective(
     # ground-truth view of operator-declared deps that downstream
     # add_dependency recommendation logic compares against the
     # incident's observed call graph.
-    loaded_manifests = _load_manifests_from_specs(specs_dir)
-    declared_dependencies_by_service = _extract_declared_dependencies(loaded_manifests)
+    loaded_specs = _load_manifests_from_specs(specs_dir)
+    declared_dependencies_by_service = _extract_declared_dependencies(loaded_specs.manifests)
 
     # Financial impact (if specs available)
     financial_impact = _compute_financial_impact(
@@ -125,7 +128,7 @@ def build_retrospective(
         duration_minutes,
         breach_counts_by_service,
         specs_dir,
-        loaded_manifests=loaded_manifests,
+        manifests_by_service=loaded_specs.manifests,
     )
 
     # Generate recommendations
@@ -155,6 +158,10 @@ def build_retrospective(
         "financial_impact": financial_impact,
         "recommendations": recommendations,
         "declared_dependencies_by_service": declared_dependencies_by_service,
+        # opensrm-oh27: always present, 0 on a clean load. An absent key
+        # would read the same as "no failures", which is the ambiguity
+        # the silent skip created in the first place.
+        "manifest_parse_failures": loaded_specs.parse_failures,
     }
     if trigger_service is not None:
         retro_custom["trigger_service"] = trigger_service
@@ -225,32 +232,74 @@ def _parse_ts(ts: Any) -> datetime:
     return dt
 
 
-def _load_manifests_from_specs(specs_dir: str | None) -> dict[str, Any]:
+@dataclass(frozen=True)
+class LoadedManifests:
+    """Outcome of scanning a specs directory (opensrm-oh27).
+
+    ``manifests`` is the parsed view keyed by service name; ``parse_failures``
+    counts *files* that raised ``ManifestLoadError``, not services missing
+    from ``manifests`` — a failed environment overlay for a service whose
+    base manifest parsed still counts one. The count is deliberately a
+    coverage-doubt signal rather than a precise miss-count, and it travels
+    with the manifests because the downstream financial figure is computed
+    over ``manifests`` alone: without it, a partial view is
+    indistinguishable from a complete one. The per-file
+    ``manifest_parse_failed`` warning carries which file and why.
+    """
+
+    manifests: dict[str, Any]
+    parse_failures: int = 0
+
+
+def _load_manifests_from_specs(specs_dir: str | None) -> LoadedManifests:
     """Load all ``*.yaml`` manifests under ``specs_dir`` once, keyed by
     ``manifest.name`` (the canonical service identity).
 
-    Returns an empty dict when ``specs_dir`` is None or not a directory,
+    Returns empty manifests when ``specs_dir`` is None or not a directory,
     so callers can treat the result uniformly without re-checking the
     filesystem. Duplicate manifests for the same service (template forks,
     environment overlays) are deduplicated by keeping the first one
     encountered and emitting a warning — this matches the pre-jmy.21
     behaviour of ``_compute_financial_impact``.
 
-    Parse failures from ``load_manifest`` are silently skipped (consistent
-    with the prior inline ``ManifestLoadError`` swallow); the financial-
-    impact and dependency-extraction paths both tolerate a partial view.
+    Parse failures from ``load_manifest`` are logged and counted, then
+    skipped (opensrm-oh27). ``UnicodeDecodeError`` is caught alongside
+    ``ManifestLoadError`` because ``load_manifest`` opens with the default
+    encoding and only wraps ``YAMLError`` — non-UTF-8 bytes would otherwise
+    abort the whole retrospective, a hard stop where the design says log and
+    continue. The financial-impact and dependency-extraction
+    paths both tolerate a partial view, but the caller is told how partial
+    it is via ``LoadedManifests.parse_failures``.
     """
     if not specs_dir:
-        return {}
+        return LoadedManifests(manifests={})
     specs_path = Path(specs_dir)
     if not specs_path.is_dir():
-        return {}
+        return LoadedManifests(manifests={})
 
     manifests: dict[str, Any] = {}
-    for spec_file in specs_path.glob("*.yaml"):
+    parse_failures = 0
+    for spec_file in iter_manifest_files(specs_path):
         try:
             manifest = load_manifest(str(spec_file))
-        except ManifestLoadError:
+        except (ManifestLoadError, UnicodeDecodeError) as exc:
+            foreign_reason = foreign_yaml_reason(spec_file)
+            if foreign_reason is not None:
+                # Foreign YAML sharing the directory, not a broken manifest.
+                # Recorded rather than dropped without trace — skipping
+                # without saying so is the shape this bead exists to remove.
+                log.debug(
+                    "manifest_file_ignored",
+                    spec_file=str(spec_file),
+                    reason=foreign_reason,
+                )
+                continue
+            parse_failures += 1
+            log.warning(
+                "manifest_parse_failed",
+                spec_file=str(spec_file),
+                error=str(exc),
+            )
             continue
         if manifest.name in manifests:
             log.warning(
@@ -260,16 +309,16 @@ def _load_manifests_from_specs(specs_dir: str | None) -> dict[str, Any]:
             )
             continue
         manifests[manifest.name] = manifest
-    return manifests
+    return LoadedManifests(manifests=manifests, parse_failures=parse_failures)
 
 
 def _extract_declared_dependencies(
-    loaded_manifests: dict[str, Any],
+    manifests_by_service: dict[str, Any],
 ) -> dict[str, list[str]]:
     """Wrapper around the shared helper (opensrm-dpws). The CLI path
     works with Manifest dataclass instances loaded via load_manifest().
     """
-    return extract_declared_dependencies(from_manifests=loaded_manifests)
+    return extract_declared_dependencies(from_manifests=manifests_by_service)
 
 
 def _compute_financial_impact(
@@ -278,7 +327,7 @@ def _compute_financial_impact(
     breach_counts_by_service: dict[str, int],
     specs_dir: str | None,
     *,
-    loaded_manifests: dict[str, Any] | None = None,
+    manifests_by_service: dict[str, Any] | None = None,
 ) -> dict | None:
     """Compute spec § 1 financial impact across blast-radius services.
 
@@ -289,10 +338,14 @@ def _compute_financial_impact(
     services that share a currency; mixed-currency manifests are
     skipped with a warning.
 
-    ``loaded_manifests`` is the shared cache produced by
-    ``_load_manifests_from_specs`` (opensrm-jmy.21). When omitted the
-    function loads manifests itself — preserves the pre-jmy.21 calling
-    convention used by ``tests/learn/test_retrospective_financial.py``.
+    ``manifests_by_service`` is the shared cache produced by
+    ``_load_manifests_from_specs(...).manifests`` (opensrm-jmy.21). When
+    omitted the function loads manifests itself — preserves the pre-jmy.21
+    calling convention used by ``tests/learn/test_retrospective_financial.py``.
+    That self-loading path discards the parse-failure count; only
+    ``build_retrospective`` surfaces it (as
+    ``metadata.custom["manifest_parse_failures"]``), and the per-file
+    warning log fires either way.
     """
     if not specs_dir or not blast_radius:
         return None
@@ -307,9 +360,9 @@ def _compute_financial_impact(
         )
         return None
 
-    if loaded_manifests is None:
-        loaded_manifests = _load_manifests_from_specs(specs_dir)
-    if not loaded_manifests:
+    if manifests_by_service is None:
+        manifests_by_service = _load_manifests_from_specs(specs_dir).manifests
+    if not manifests_by_service:
         return None
 
     affected_services = {
@@ -322,7 +375,7 @@ def _compute_financial_impact(
     currency: str | None = None
     aggregate_source: str | None = None
 
-    for service_name, manifest in loaded_manifests.items():
+    for service_name, manifest in manifests_by_service.items():
         if service_name not in affected_services:
             continue
         if manifest.outcomes is None:

{nthlayer_workers-1.7.1 → nthlayer_workers-2.0.1}/src/nthlayer_workers/learn/worker.py

@@ -323,8 +323,8 @@ class LearnRetrospectiveModule:
                     ]
                     if len(trigger_matches) > 1:
                         # First-wins (matches the existing
-                        # _load_manifests_from_specs dedup behaviour
-                        # on the CLI side, retrospective.py line 238).
+                        # _load_manifests_from_specs dedup behaviour on the
+                        # CLI side, which logs manifest_duplicate_skipped).
                         logger.warning(
                             "learn_manifest_duplicate_skipped",
                             service=trigger_service,