nthlayer-workers 4.1.0__tar.gz → 4.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/PKG-INFO +2 -2
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/pyproject.toml +2 -2
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/prometheus.py +19 -5
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/prometheus.py +83 -88
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/cli.py +28 -1
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/drift/analyzer.py +38 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/worker.py +20 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers.egg-info/PKG-INFO +2 -2
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers.egg-info/requires.txt +1 -1
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/LICENSE +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/README.md +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/setup.cfg +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/cli.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/cli_output.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/cli.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/config.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/correlation/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/correlation/changes.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/correlation/dedup.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/correlation/engine.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/correlation/temporal.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/correlation/topology.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/ingestion/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/ingestion/protocol.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/ingestion/severity.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/ingestion/webhook.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/notifications.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/reasoning.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/session.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/snapshot/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/snapshot/generator.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/snapshot/model.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/snapshot/token.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/state.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/store/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/store/protocol.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/store/sqlite.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/summary.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/traces/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/traces/protocol.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/traces/tempo.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/traces/topology.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/types.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/verdict_chain.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/worker.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/__main__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/_apply.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/_gh.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/_interactive.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/_interactive_app.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/_preview.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/_trigger.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/_yaml.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/cli.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/recommendations.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/retrospective.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/trends.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/worker.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/llm_provider.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/__main__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/_parsing.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/_util.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/devin.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/gastown.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/protocol.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/webhook.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/calibration/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/calibration/loop.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/calibration/slos.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/calibration/verdict_calibration.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/config.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/detection/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/detection/detector.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/detection/protocol.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/governance/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/governance/engine.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/manifest.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/notifications.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/pipeline/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/pipeline/evaluator.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/pipeline/router.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/store/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/store/protocol.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/store/sqlite.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/telemetry.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/thresholds/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/thresholds/phase.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/thresholds/sampler.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/thresholds/state.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/tiering/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/tiering/classifier.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/tiering/promotion.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/trends/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/trends/tracker.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/types.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/worker.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/__main__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/api/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/assessment.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/cli.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/config.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/db/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/decision_records.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/discovery.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/providers/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/providers/backstage.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/providers/base.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/providers/consul.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/providers/etcd.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/providers/kubernetes.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/providers/prometheus.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/dependencies/providers/zookeeper.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/deployments/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/discovery/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/discovery/classifier.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/discovery/client.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/discovery/models.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/drift/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/drift/models.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/drift/patterns.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/explanation.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/gate/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/gate/conditions.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/gate/correlator.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/gate/evaluator.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/gate/policies.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/gate_adapter.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/incident.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/portfolio/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/portfolio/aggregator.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/portfolio/scorer.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/slo/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/slo/collector.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/slo/spec_loader.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/sqlite_store.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/store.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/verification/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/verification/exporter_guidance.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/verification/extractor.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/verification/models.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/verification/verifier.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/__main__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/agents/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/agents/base.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/agents/communication.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/agents/investigation.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/agents/remediation.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/agents/response_models.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/agents/triage.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/cli.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/config.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/context_store.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/coordinator.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/metrics.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/notification_backends/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/notification_backends/ntfy_backend.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/notification_backends/protocol.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/notification_backends/slack_backend.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/notification_backends/stdout_backend.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/notifications.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/oncall/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/oncall/escalation.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/oncall/runner.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/oncall/schedule.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/safe_actions/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/safe_actions/actions.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/safe_actions/registry.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/safe_actions/webhook.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/server.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/sre/__init__.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/sre/brief.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/sre/delegation.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/sre/post_incident.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/sre/shift_report.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/sre/suppression.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/types.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/verdict_submission.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/worker.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/respond/worker_helpers.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/runner.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers.egg-info/SOURCES.txt +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers.egg-info/dependency_links.txt +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers.egg-info/entry_points.txt +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers.egg-info/top_level.txt +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/tests/test_cli_gate.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/tests/test_cli_telemetry.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/tests/test_dependency_declarations.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/tests/test_llm_provider.py +0 -0
- {nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/tests/test_runner.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nthlayer-workers
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.2.0
|
|
4
4
|
Summary: NthLayer workers — Tier 2 background computation (observe, measure, correlate, respond, learn)
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Requires-Python: >=3.11
|
|
7
7
|
Description-Content-Type: text/markdown
|
|
8
8
|
License-File: LICENSE
|
|
9
|
-
Requires-Dist: nthlayer-common[otel]<6.0.0,>=5.
|
|
9
|
+
Requires-Dist: nthlayer-common[otel]<6.0.0,>=5.12.0
|
|
10
10
|
Requires-Dist: httpx>=0.27
|
|
11
11
|
Requires-Dist: pyyaml>=6.0
|
|
12
12
|
Requires-Dist: structlog>=24.1.0
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "nthlayer-workers"
|
|
7
|
-
version = "4.
|
|
7
|
+
version = "4.2.0"
|
|
8
8
|
description = "NthLayer workers — Tier 2 background computation (observe, measure, correlate, respond, learn)"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "Apache-2.0"}
|
|
@@ -59,7 +59,7 @@ dependencies = [
|
|
|
59
59
|
# [otel] (opensrm-quqo): the OTel SDK and OTLP log exporter, so the events
|
|
60
60
|
# workers emits are exported once init_telemetry finds an endpoint. Without
|
|
61
61
|
# them every event is dropped, which is how they all went missing.
|
|
62
|
-
"nthlayer-common[otel]>=5.
|
|
62
|
+
"nthlayer-common[otel]>=5.12.0,<6.0.0",
|
|
63
63
|
"httpx>=0.27",
|
|
64
64
|
"pyyaml>=6.0",
|
|
65
65
|
"structlog>=24.1.0",
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/prometheus.py
RENAMED
|
@@ -62,11 +62,14 @@ async def fetch_metric_breaches(
|
|
|
62
62
|
|
|
63
63
|
for service in services:
|
|
64
64
|
# Check error budget: the worst across the service's SLOs, since
|
|
65
|
-
# generate writes one series per
|
|
66
|
-
|
|
65
|
+
# generate writes one series per classical SLO (opensrm-h0at,
|
|
66
|
+
# opensrm-zo0o). bottomk, not min: min aggregates the labels away,
|
|
67
|
+
# and the event must say WHICH SLO is overspent.
|
|
68
|
+
worst = await _query_instant_sample(
|
|
67
69
|
client, prometheus_url,
|
|
68
|
-
f"
|
|
70
|
+
f"bottomk(1, {ERROR_BUDGET_REMAINING.selector(service=service)})",
|
|
69
71
|
)
|
|
72
|
+
budget, budget_labels = worst if worst is not None else (None, {})
|
|
70
73
|
if budget is not None and budget < 0.0:
|
|
71
74
|
events.append(SitRepEvent(
|
|
72
75
|
id=f"prom-metric-{uuid.uuid4().hex[:8]}",
|
|
@@ -80,6 +83,7 @@ async def fetch_metric_breaches(
|
|
|
80
83
|
"metric": ERROR_BUDGET_REMAINING.metric,
|
|
81
84
|
"value": budget,
|
|
82
85
|
"breach": "error_budget_exhausted",
|
|
86
|
+
"slo": budget_labels.get("slo"),
|
|
83
87
|
},
|
|
84
88
|
))
|
|
85
89
|
|
|
@@ -214,6 +218,16 @@ async def _query_instant(
|
|
|
214
218
|
query: str,
|
|
215
219
|
) -> float | None:
|
|
216
220
|
"""Execute a PromQL instant query and return the scalar value."""
|
|
221
|
+
sample = await _query_instant_sample(client, prometheus_url, query)
|
|
222
|
+
return None if sample is None else sample[0]
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
async def _query_instant_sample(
|
|
226
|
+
client: httpx.AsyncClient,
|
|
227
|
+
prometheus_url: str,
|
|
228
|
+
query: str,
|
|
229
|
+
) -> tuple[float, dict[str, str]] | None:
|
|
230
|
+
"""Execute a PromQL instant query; return the first sample's value and labels."""
|
|
217
231
|
try:
|
|
218
232
|
resp = await client.get(
|
|
219
233
|
f"{prometheus_url}/api/v1/query",
|
|
@@ -227,8 +241,8 @@ async def _query_instant(
|
|
|
227
241
|
val = float(results[0].get("value", [None, None])[1])
|
|
228
242
|
if val != val: # NaN
|
|
229
243
|
return None
|
|
230
|
-
return val
|
|
231
|
-
except (httpx.HTTPError, ValueError, KeyError, IndexError, TypeError):
|
|
244
|
+
return val, dict(results[0].get("metric") or {})
|
|
245
|
+
except (httpx.HTTPError, ValueError, KeyError, IndexError, TypeError, AttributeError):
|
|
232
246
|
return None
|
|
233
247
|
|
|
234
248
|
|
|
@@ -36,7 +36,8 @@ class SLODefinition:
|
|
|
36
36
|
# How to read `query`'s value, chosen by _query_for when it built the
|
|
37
37
|
# query. The breach rule and the query are one decision: an
|
|
38
38
|
# overrides/decisions RATE must be inverted, a `slo:*:ratio` recording
|
|
39
|
-
# rule must not,
|
|
39
|
+
# rule must not, an error budget is signed against zero, and
|
|
40
|
+
# `not_evaluable` has an empty query that is never sent. Keeping them apart
|
|
40
41
|
# meant the four judgment types with no builder got an SLI query and the
|
|
41
42
|
# inverting rule, reading a healthy 0.99 as 1.0% (opensrm-fxln).
|
|
42
43
|
#
|
|
@@ -45,12 +46,6 @@ class SLODefinition:
|
|
|
45
46
|
# answer — the failure shape this whole bead is about.
|
|
46
47
|
query_kind: str
|
|
47
48
|
judgment_type: str | None = None
|
|
48
|
-
# The manifest's declared unit for a duration target. The parser
|
|
49
|
-
# preserves it verbatim, so `latency: {target: 2, unit: s}` and
|
|
50
|
-
# `{target: 2000, unit: ms}` both arrive as target=2000.0/2.0 with the
|
|
51
|
-
# unit that makes sense of them. Hard-coding milliseconds here compared
|
|
52
|
-
# a seconds SLO against a threshold 1000x too small (opensrm-fxln).
|
|
53
|
-
unit: str | None = None
|
|
54
49
|
|
|
55
50
|
|
|
56
51
|
@dataclass
|
|
@@ -135,24 +130,59 @@ def load_specs(specs_dir: Path) -> LoadedSpecs:
|
|
|
135
130
|
window=slo.window or "7d",
|
|
136
131
|
query=query,
|
|
137
132
|
query_kind=query_kind,
|
|
138
|
-
unit=slo.unit,
|
|
139
133
|
))
|
|
140
134
|
|
|
141
135
|
return LoadedSpecs(slos=slos, parse_failures=parse_failures)
|
|
142
136
|
|
|
143
137
|
|
|
138
|
+
# The classical kinds generate writes an error budget for (ruling D5,
|
|
139
|
+
# opensrm-zo0o). Throughput, freshness, lag and custom have none until
|
|
140
|
+
# time-slice budgets exist (opensrm-vcxm).
|
|
141
|
+
BUDGETED_KINDS = frozenset({"availability", "latency", "error_rate"})
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _classical_kind(slo: ManifestSLO) -> str | None:
|
|
145
|
+
"""The SLO's kind as generate classifies it, or None for one it writes
|
|
146
|
+
no rules for (freshness, lag, custom).
|
|
147
|
+
|
|
148
|
+
A copy of generate's ``_slo_kind`` (recording_rules/manifest_builder.py),
|
|
149
|
+
kept identical so measure asks for exactly the budgets generate writes:
|
|
150
|
+
substring on the lowered type, falling back to the name when no type is
|
|
151
|
+
set, because srm/v1 and legacy pass an explicit ``type:`` through
|
|
152
|
+
unnormalised (opensrm-hiw2). TODO(opensrm-ltlw): move it to
|
|
153
|
+
nthlayer-common so the two cannot drift.
|
|
154
|
+
"""
|
|
155
|
+
# str(): srm/v1 loads `name: 404` as an int.
|
|
156
|
+
slo_type = str(slo.slo_type or slo.name).lower()
|
|
157
|
+
for needle, kind in (
|
|
158
|
+
("availability", "availability"),
|
|
159
|
+
("latency", "latency"),
|
|
160
|
+
("error", "error_rate"),
|
|
161
|
+
("throughput", "throughput"),
|
|
162
|
+
):
|
|
163
|
+
if needle in slo_type:
|
|
164
|
+
return kind
|
|
165
|
+
return None
|
|
166
|
+
|
|
167
|
+
|
|
144
168
|
def _query_for(service: str, slo: ManifestSLO) -> tuple[str, str]:
|
|
145
169
|
"""PromQL for an SLO, always synthesised, never taken from the manifest.
|
|
146
170
|
|
|
147
171
|
Returns ``(query, query_kind)``. The kind is not decoration: it is how
|
|
148
172
|
evaluate_slos knows whether the number coming back is a rate to invert,
|
|
149
|
-
a good-ratio SLI,
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
Judgment SLOs get the interim raw-metric queries below
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
173
|
+
a good-ratio SLI, or a signed error budget. Deriving it later from the
|
|
174
|
+
SLO's name or type cannot work — that is precisely what broke.
|
|
175
|
+
|
|
176
|
+
Judgment SLOs get the interim raw-metric queries below, or the
|
|
177
|
+
recording-rule convention when their type has no builder. A classical
|
|
178
|
+
SLO of a request-ratio kind reads the error budget generate writes for
|
|
179
|
+
it: by ruling Z1 (opensrm-zo0o) each one is good/total against its
|
|
180
|
+
objective, so one series and one breach rule cover them all. Every
|
|
181
|
+
other classical kind is ``not_evaluable`` with an empty query (ruling
|
|
182
|
+
D5): generate writes no budget for it, so there is nothing to ask
|
|
183
|
+
Prometheus, and evaluate_slos reports it rather than querying a series
|
|
184
|
+
nobody writes. Judgment SLOs always get a query: one reaching
|
|
185
|
+
evaluation with none would be dropped silently.
|
|
156
186
|
"""
|
|
157
187
|
name = slo.name
|
|
158
188
|
if slo.judgment_type:
|
|
@@ -165,26 +195,15 @@ def _query_for(service: str, slo: ManifestSLO) -> tuple[str, str]:
|
|
|
165
195
|
if builder is not None:
|
|
166
196
|
kind = _JUDGMENT_QUERY_KINDS[slo.judgment_type]
|
|
167
197
|
return builder(service, slo.window or "7d"), kind
|
|
168
|
-
if not slo.judgment_type
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
#
|
|
172
|
-
#
|
|
173
|
-
# left-hand `not judgment_type` stands in for generate only ever
|
|
174
|
-
# passing classical SLOs there; judgment SLOs are availability-shaped
|
|
175
|
-
# too, and one with no builder above keeps the fallback below. The
|
|
176
|
-
# selector filters to this SLO, as each series carries slo=<name>
|
|
177
|
-
# (opensrm-h0at).
|
|
198
|
+
if not slo.judgment_type:
|
|
199
|
+
if _classical_kind(slo) not in BUDGETED_KINDS:
|
|
200
|
+
return "", "not_evaluable"
|
|
201
|
+
# The selector filters to this SLO, as each series carries
|
|
202
|
+
# slo=<name> (opensrm-h0at).
|
|
178
203
|
return (
|
|
179
204
|
ERROR_BUDGET_REMAINING.selector(service=service, slo=name),
|
|
180
205
|
"error_budget",
|
|
181
206
|
)
|
|
182
|
-
if name == "latency":
|
|
183
|
-
percentile = slo.percentile or "p99"
|
|
184
|
-
return (
|
|
185
|
-
f'slo:http_request_duration_seconds:{percentile}{{service="{service}"}}',
|
|
186
|
-
"latency_seconds",
|
|
187
|
-
)
|
|
188
207
|
return f'slo:{name}:ratio{{service="{service}"}}', "ratio"
|
|
189
208
|
|
|
190
209
|
|
|
@@ -234,54 +253,6 @@ def evaluation_custom_metadata(result: EvaluationResult) -> dict:
|
|
|
234
253
|
}
|
|
235
254
|
|
|
236
255
|
|
|
237
|
-
# Seconds per unit of a duration target. Prometheus latency queries return
|
|
238
|
-
# seconds, so the target is converted TO seconds rather than the reverse.
|
|
239
|
-
# Keys are matched case-insensitively after stripping; the spelled-out and
|
|
240
|
-
# abbreviated forms are aliases because manifests are hand-written and
|
|
241
|
-
# `unit: seconds` reaching the unknown-unit path would assume milliseconds —
|
|
242
|
-
# a threshold 1000x too small that breaches every window.
|
|
243
|
-
_DURATION_UNIT_SECONDS = {
|
|
244
|
-
"us": 0.000_001, "µs": 0.000_001, "microsecond": 0.000_001,
|
|
245
|
-
"microseconds": 0.000_001,
|
|
246
|
-
"ms": 0.001, "msec": 0.001, "millisecond": 0.001, "milliseconds": 0.001,
|
|
247
|
-
"s": 1.0, "sec": 1.0, "secs": 1.0, "second": 1.0, "seconds": 1.0,
|
|
248
|
-
"m": 60.0, "min": 60.0, "minute": 60.0, "minutes": 60.0,
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
# What a duration target means when the manifest declared no unit. The
|
|
252
|
-
# schema's own latency example uses `ms`, and so does every manifest in the
|
|
253
|
-
# ecosystem, so that is the assumption — but it IS an assumption, so it is
|
|
254
|
-
# logged rather than applied silently. Refusing to evaluate instead would
|
|
255
|
-
# drop the SLO, which is the failure this bead exists to remove.
|
|
256
|
-
_DEFAULT_DURATION_UNIT = "ms"
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
def _target_seconds(target: float, unit: str | None) -> float:
|
|
260
|
-
"""Convert a duration target to seconds using its declared unit."""
|
|
261
|
-
if unit is None:
|
|
262
|
-
logger.warning(
|
|
263
|
-
"Latency target %s has no unit; assuming %s",
|
|
264
|
-
target,
|
|
265
|
-
_DEFAULT_DURATION_UNIT,
|
|
266
|
-
)
|
|
267
|
-
unit = _DEFAULT_DURATION_UNIT
|
|
268
|
-
# str() before strip(): parser/v1.py passes config.get("unit") through
|
|
269
|
-
# unvalidated, so `unit: 100` arrives as an int. Calling .strip() on it
|
|
270
|
-
# would raise out of evaluate_slos' loop and abort the cycle with
|
|
271
|
-
# verdicts already written for earlier SLOs — a partial cycle rather
|
|
272
|
-
# than a clean failure. Coercing routes it to the warning below.
|
|
273
|
-
scale = _DURATION_UNIT_SECONDS.get(str(unit).strip().lower())
|
|
274
|
-
if scale is None:
|
|
275
|
-
logger.warning(
|
|
276
|
-
"Unknown duration unit %r for target %s; assuming %s",
|
|
277
|
-
unit,
|
|
278
|
-
target,
|
|
279
|
-
_DEFAULT_DURATION_UNIT,
|
|
280
|
-
)
|
|
281
|
-
scale = _DURATION_UNIT_SECONDS[_DEFAULT_DURATION_UNIT]
|
|
282
|
-
return target * scale
|
|
283
|
-
|
|
284
|
-
|
|
285
256
|
async def query_prometheus(
|
|
286
257
|
client: httpx.AsyncClient,
|
|
287
258
|
prometheus_url: str,
|
|
@@ -408,9 +379,36 @@ async def evaluate_slos(
|
|
|
408
379
|
|
|
409
380
|
async with httpx.AsyncClient() as client:
|
|
410
381
|
for slo in slos:
|
|
382
|
+
if slo.query_kind == "not_evaluable":
|
|
383
|
+
# E3's no_data_source: no series is defined for this kind.
|
|
384
|
+
# The CLI reports it; there is nothing to query or to count
|
|
385
|
+
# toward hysteresis.
|
|
386
|
+
logger.info(
|
|
387
|
+
"%s/%s is not evaluable (no_data_source): generate writes "
|
|
388
|
+
"no error budget for its kind",
|
|
389
|
+
slo.service, slo.slo_name,
|
|
390
|
+
)
|
|
391
|
+
continue
|
|
411
392
|
current_value = await query_prometheus(client, prometheus_url, slo.query)
|
|
412
393
|
if current_value is None:
|
|
413
|
-
|
|
394
|
+
if slo.query_kind == "error_budget":
|
|
395
|
+
# Not debug: for a classical SLO this is usually not a
|
|
396
|
+
# scrape gap but a budget generate chose not to write (an
|
|
397
|
+
# objective or latency bound it cannot trust), or rules
|
|
398
|
+
# older than generate 4.1.0, and the SLO is then never
|
|
399
|
+
# evaluated. WARNING because workers configures no
|
|
400
|
+
# logging, so stdlib shows nothing below it; the CLI also
|
|
401
|
+
# lists the SLO as NO DATA.
|
|
402
|
+
logger.warning(
|
|
403
|
+
"No error budget value for %s/%s; not evaluated "
|
|
404
|
+
"(Prometheus did not answer, or the series is missing: "
|
|
405
|
+
"generate skips a budget whose objective or latency "
|
|
406
|
+
"bound it cannot trust, and before 4.1.0 wrote only "
|
|
407
|
+
"availability budgets)",
|
|
408
|
+
slo.service, slo.slo_name,
|
|
409
|
+
)
|
|
410
|
+
else:
|
|
411
|
+
logger.debug("No data for %s/%s, skipping", slo.service, slo.slo_name)
|
|
414
412
|
continue
|
|
415
413
|
|
|
416
414
|
# Dispatch on the kind of number the query returns, decided by
|
|
@@ -433,15 +431,12 @@ async def evaluate_slos(
|
|
|
433
431
|
# The error budget is signed — negative is overspent.
|
|
434
432
|
# The target is not consulted; it is baked into the rule.
|
|
435
433
|
raw_breach = current_value < 0.0
|
|
436
|
-
elif slo.query_kind == "latency_seconds":
|
|
437
|
-
# Query is seconds; the target is in whatever unit the
|
|
438
|
-
# manifest declared.
|
|
439
|
-
raw_breach = current_value > _target_seconds(slo.target, slo.unit)
|
|
440
434
|
else:
|
|
441
|
-
# "ratio"
|
|
442
|
-
#
|
|
443
|
-
#
|
|
444
|
-
#
|
|
435
|
+
# "ratio", only for a judgment type with no builder: the
|
|
436
|
+
# recording-rule convention's 0-1 good-ratio SLI against a
|
|
437
|
+
# 0-100 target, so it scales but does not invert — the same
|
|
438
|
+
# arithmetic as worker.py:242. Comparing the two conventions
|
|
439
|
+
# directly, as this branch used to, breaches every window.
|
|
445
440
|
raw_breach = current_value * 100 < slo.target
|
|
446
441
|
|
|
447
442
|
# Get consecutive breach count from verdict store
|
|
@@ -113,6 +113,24 @@ def _build_adapter(config: MeasureConfig):
|
|
|
113
113
|
# status, calibrate, overrides, tiering — remain.
|
|
114
114
|
|
|
115
115
|
|
|
116
|
+
def unevaluated_slos(slos, results):
|
|
117
|
+
"""The SLOs a run produced no verdict for, as ``(not_evaluable, no_data)``.
|
|
118
|
+
|
|
119
|
+
``not_evaluable``: kinds with no error budget, never queried (ruling D5,
|
|
120
|
+
opensrm-zo0o; E3 no_data_source). ``no_data``: budgeted SLOs whose query
|
|
121
|
+
returned nothing, because the series is missing (a budget generate
|
|
122
|
+
skipped, or pre-4.1.0 rules) or Prometheus did not answer. Both are
|
|
123
|
+
listed by evaluate-once so neither can vanish from a run unseen.
|
|
124
|
+
"""
|
|
125
|
+
evaluated = {(r.service, r.slo_name) for r in results}
|
|
126
|
+
not_evaluable = [s for s in slos if s.query_kind == "not_evaluable"]
|
|
127
|
+
no_data = [
|
|
128
|
+
s for s in slos
|
|
129
|
+
if s.query_kind == "error_budget" and (s.service, s.slo_name) not in evaluated
|
|
130
|
+
]
|
|
131
|
+
return not_evaluable, no_data
|
|
132
|
+
|
|
133
|
+
|
|
116
134
|
def cmd_evaluate_once(args: argparse.Namespace) -> None:
|
|
117
135
|
"""One-shot Prometheus SLO evaluation — evaluate all SLOs, write verdicts, exit."""
|
|
118
136
|
from nthlayer_common.verdicts import SQLiteVerdictStore
|
|
@@ -212,7 +230,16 @@ def cmd_evaluate_once(args: argparse.Namespace) -> None:
|
|
|
212
230
|
v.metadata.custom["slack_thread_ts"] = thread_ts
|
|
213
231
|
verdict_store.put(v)
|
|
214
232
|
|
|
215
|
-
|
|
233
|
+
not_evaluable, no_data = unevaluated_slos(slos, results)
|
|
234
|
+
for s in not_evaluable:
|
|
235
|
+
print(f" {s.service}/{s.slo_name}: NOT EVALUABLE (no_data_source)")
|
|
236
|
+
for s in no_data:
|
|
237
|
+
print(f" {s.service}/{s.slo_name}: NO DATA (no error budget value)")
|
|
238
|
+
|
|
239
|
+
print(
|
|
240
|
+
f"\nEvaluated {len(results)} SLOs, {breach_count} in breach"
|
|
241
|
+
f", {len(not_evaluable)} not evaluable, {len(no_data)} with no data."
|
|
242
|
+
)
|
|
216
243
|
return results
|
|
217
244
|
|
|
218
245
|
try:
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/observe/drift/analyzer.py
RENAMED
|
@@ -150,6 +150,44 @@ class DriftAnalyzer:
|
|
|
150
150
|
exit_code=exit_code,
|
|
151
151
|
)
|
|
152
152
|
|
|
153
|
+
async def budgeted_slos(self, service: str) -> set[str]:
|
|
154
|
+
"""Names of the service's SLOs that have an error budget series.
|
|
155
|
+
|
|
156
|
+
Generate decides which SLOs get a budget (opensrm-zo0o: availability,
|
|
157
|
+
latency and error_rate, minus objectives it cannot trust), so drift
|
|
158
|
+
asks for the series it wrote rather than re-deriving that rule from
|
|
159
|
+
the manifest, which in worker mode lacks the fields it needs.
|
|
160
|
+
Raises DriftAnalysisError when Prometheus cannot answer, so a caller
|
|
161
|
+
can tell "none" from "unknown".
|
|
162
|
+
"""
|
|
163
|
+
import httpx
|
|
164
|
+
|
|
165
|
+
query = f"group by (slo) ({ERROR_BUDGET_REMAINING.selector(service=service)})"
|
|
166
|
+
auth = (self.username, self.password) if self.username and self.password else None
|
|
167
|
+
try:
|
|
168
|
+
async with httpx.AsyncClient(auth=auth, timeout=30.0) as client:
|
|
169
|
+
response = await client.get(
|
|
170
|
+
f"{self.prometheus_url}/api/v1/query", params={"query": query}
|
|
171
|
+
)
|
|
172
|
+
response.raise_for_status()
|
|
173
|
+
result = response.json()
|
|
174
|
+
except (httpx.HTTPError, httpx.InvalidURL, ValueError) as e:
|
|
175
|
+
raise DriftAnalysisError(f"Failed to query Prometheus: {e}") from e
|
|
176
|
+
if not isinstance(result, dict):
|
|
177
|
+
raise DriftAnalysisError(f"Malformed Prometheus response: {type(result).__name__}")
|
|
178
|
+
if result.get("status") != "success":
|
|
179
|
+
raise DriftAnalysisError(f"Prometheus query failed: {result.get('error', 'Unknown')}")
|
|
180
|
+
try:
|
|
181
|
+
return {
|
|
182
|
+
series["metric"]["slo"]
|
|
183
|
+
for series in result["data"]["result"]
|
|
184
|
+
if "slo" in series.get("metric", {})
|
|
185
|
+
}
|
|
186
|
+
except (KeyError, TypeError, AttributeError) as e:
|
|
187
|
+
# A success status with a malformed body must still read as
|
|
188
|
+
# "could not ask", not escape process_cycle.
|
|
189
|
+
raise DriftAnalysisError(f"Malformed Prometheus response: {e}") from e
|
|
190
|
+
|
|
153
191
|
async def _query_budget_history(
|
|
154
192
|
self,
|
|
155
193
|
service: str,
|
|
@@ -168,7 +168,27 @@ class ObserveDriftModule:
|
|
|
168
168
|
|
|
169
169
|
analyzer = DriftAnalyzer(self.prometheus_url)
|
|
170
170
|
|
|
171
|
+
# Only SLOs with an error budget series can drift (opensrm-zo0o):
|
|
172
|
+
# judgment SLOs and the classical kinds generate writes no budget for
|
|
173
|
+
# would otherwise fail every cycle with a warning each.
|
|
174
|
+
# None = the lookup failed this cycle: skip the service. An empty set
|
|
175
|
+
# = it answered, and the service has no budget.
|
|
176
|
+
budgeted: dict[str, set[str] | None] = {}
|
|
171
177
|
for service, tier, slo_name, window in _drift_targets(manifests):
|
|
178
|
+
if service not in budgeted:
|
|
179
|
+
try:
|
|
180
|
+
budgeted[service] = await analyzer.budgeted_slos(service)
|
|
181
|
+
except DriftAnalysisError:
|
|
182
|
+
logger.warning("drift_budget_discovery_failed", service=service)
|
|
183
|
+
budgeted[service] = None
|
|
184
|
+
except Exception:
|
|
185
|
+
# As analyze() below: one service's surprise must not end
|
|
186
|
+
# the cycle for every other service.
|
|
187
|
+
logger.exception("drift_budget_discovery_unexpected_error", service=service)
|
|
188
|
+
budgeted[service] = None
|
|
189
|
+
names = budgeted[service]
|
|
190
|
+
if names is None or str(slo_name) not in names:
|
|
191
|
+
continue
|
|
172
192
|
try:
|
|
173
193
|
result = await analyzer.analyze(service, tier, slo_name, window)
|
|
174
194
|
except DriftAnalysisError:
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nthlayer-workers
|
|
3
|
-
Version: 4.
|
|
3
|
+
Version: 4.2.0
|
|
4
4
|
Summary: NthLayer workers — Tier 2 background computation (observe, measure, correlate, respond, learn)
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Requires-Python: >=3.11
|
|
7
7
|
Description-Content-Type: text/markdown
|
|
8
8
|
License-File: LICENSE
|
|
9
|
-
Requires-Dist: nthlayer-common[otel]<6.0.0,>=5.
|
|
9
|
+
Requires-Dist: nthlayer-common[otel]<6.0.0,>=5.12.0
|
|
10
10
|
Requires-Dist: httpx>=0.27
|
|
11
11
|
Requires-Dist: pyyaml>=6.0
|
|
12
12
|
Requires-Dist: structlog>=24.1.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/notifications.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/reasoning.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/snapshot/model.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/snapshot/token.py
RENAMED
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/store/__init__.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/store/protocol.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/store/sqlite.py
RENAMED
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/traces/__init__.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/traces/protocol.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/traces/tempo.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/traces/topology.py
RENAMED
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/correlate/verdict_chain.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/_interactive.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/_interactive_app.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/recommendations.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/learn/retrospective.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/__init__.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/_util.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/devin.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/gastown.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/protocol.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/adapters/webhook.py
RENAMED
|
File without changes
|
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/calibration/loop.py
RENAMED
|
File without changes
|
{nthlayer_workers-4.1.0 → nthlayer_workers-4.2.0}/src/nthlayer_workers/measure/calibration/slos.py
RENAMED
|
File without changes
|