failroute 0.3.0__tar.gz

failroute-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 feiiiiii5
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

failroute-0.3.0/PKG-INFO

@@ -0,0 +1,205 @@
+Metadata-Version: 2.4
+Name: failroute
+Version: 0.3.0
+Summary: Static detection of failure-routing anti-patterns in Python
+Author: fc
+License: MIT
+Keywords: static-analysis,exception-handling,code-quality,llm,reliability
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Dynamic: license-file
+
+# failroute
+
+Static detection of **failure-routing** anti-patterns in Python: the practice of
+converting an underlying failure into a success-like outcome at the wrong layer.
+
+Failure-routing is the root cause behind some of the most insidious correctness
+bugs in real LLM/eval/agent codebases:
+
+```python
+# Before — a judge API outage becomes a perfect "0.0 score" with no way to tell
+try:
+    score = await llm_judge(prompt)
+except Exception:
+    return 0.0  # ← silent fallback: failure looks like a legitimate low score
+
+# After — the failure propagates; callers can route it to the right outcome
+return await llm_judge(prompt)
+```
+
+## What it detects
+
+| Mode | Pattern |
+| --- | --- |
+| `no-action` | `except ...: pass` — the exception is discarded, callers never learn |
+| `silent-fallback` | handler returns/assigns a constant (`None`, `0`, `0.0`, `False`, `[]`, …) without re-raising |
+| `masked-exception` | **catch-all** handler re-raises conditionally yet also falls through to a success-looking return |
+| `name-shadowing` | `except E as e:` whose body rebinds `e` — Python deletes the binding at handler exit, so later uses raise `NameError` |
+
+Findings are emitted as `file:line: mode: message`, or as JSON for CI.
+
+### Logging exemption (two tiers)
+
+A handler that *records* the failure is informational, not silent — but what
+counts as a record depends on how wide the handler is:
+
+- **Catch-all handlers** (`except:` / `except Exception:`) must log at a
+  severity worth reading (`warning`+). A `debug` line or a bare `print(...)`
+  does not survive production triage, so it does not exempt.
+- **Typed handlers** name an anticipated failure mode; recording it at *any*
+  level (even `logger.info`) is enough.
+
+## Usage
+
+```console
+$ failroute path/to/file.py
+$ failroute path/to/dir      # recursive
+$ failroute --repo .         # skip .git/.venv/build/...
+$ failroute --repo . --exclude tests/corpus   # repeatable path exclusions
+$ failroute --json --repo .  | jq 'select(.mode=="silent-fallback")'
+$ failroute --format sarif --output results.sarif --repo .   # code scanning
+$ failroute --threshold 5    # exit 1 when more than 5 findings
+$ python -m failroute .      # module form (no console script needed)
+```
+
+Exit codes: `0` clean, `1` findings above threshold, `2` usage error.
+
+### Output formats
+
+```console
+$ failroute --format text   path/       # default: file:line: mode: message
+$ failroute --format json   path/       # one JSON object per finding
+$ failroute --format sarif  --output scan.sarif path/   # SARIF 2.1.0
+```
+
+SARIF output plugs straight into [GitHub code scanning](
+https://docs.github.com/en/code-security/code-scanning) via the
+`upload-sarif` action, so findings appear inline on pull requests:
+
+```yaml
+- run: failroute --format sarif --output results.sarif --repo .
+- uses: github/codeql-action/upload-sarif@v3
+  with:
+    sarif_file: results.sarif
+```
+
+Or use the bundled composite action, which installs failroute, scans, and
+uploads SARIF in one step:
+
+```yaml
+- uses: feiiiiii5/failroute/action@main
+  with:
+    path: src
+    exclude: tests/corpus fixtures
+    threshold: "0"
+```
+
+Severity mapping: `silent-fallback` → `error`, `no-action` and
+`masked-exception` → `warning`.
+
+### Suppressing findings
+
+Reviewed-and-accepted handlers can be opted out with a line marker (the
+scanner honors both):
+
+```python
+try:
+    return best_effort()
+except Exception:  # failroute: ignore - documented fallback semantics
+    return None
+```
+
+`# pragma: no cover` markers are honored as well (explicitly defensive code).
+
+## Examples that trip it
+
+```python
+def classify(text):                       # no-action
+    try:
+        return model.predict(text)
+    except Exception:
+        pass                                # 💥 swallowed
+
+def score(prompt):                          # silent-fallback
+    try:
+        return judge(prompt)
+    except Exception:
+        return 0.0                          # 💥 outage == "0.0 score"
+
+def fetch(url):                             # silent-fallback (assign)
+    data = None
+    try:
+        data = download(url)
+    except Exception:
+        data = {"items": []}                # 💥 error looks like an empty result
+    return data
+```
+
+## What it does *not* flag (by design)
+
+* `except KeyboardInterrupt` / `except SystemExit` — normally intentional.
+* Handlers that re-raise unconditionally without a fallback.
+* `except` bodies that log at `warning`/`error` **and** re-raise — the failure
+  still propagates; we only flag the success-looking path.
+
+Run `failroute` on its own checkout as a smoke test:
+
+```console
+$ pip install -e .
+$ failroute --repo .     # expected: zero findings (self-hosting)
+```
+
+## Benchmarks & validation
+
+All numbers below are reproducible from this checkout; nothing here is
+copy-pasted from a run that cannot be re-executed.
+
+### Labelled corpus (precision / recall)
+
+`tests/corpus/` holds 19 hand-labelled exception handlers (10 positives across
+all three modes, 9 negatives covering re-raise, log-and-raise, derived values,
+dead code, opt-out markers, and non-fallback constants). Ground truth lives in
+`tests/corpus/manifest.json` and was written from the *semantics* of each
+fixture, independently of tool output.
+
+```
+corpus v1   TP=10  FP=0  FN=0  TN=9
+precision=1.0  recall=1.0
+```
+
+Re-run: `python tools/benchmark.py` (also enforced by `pytest`).
+
+### What syntactic linters miss
+
+Against the source packages of 8 real AI/eval repositories (garak,
+inspect_ai, pydantic-ai, uqlm, trl, smolagents, deepteam, fickling),
+failroute reported **647 findings**; ruff's exception-handling rules
+(`S110` try-except-pass, `S112` try-except-continue) reported **80**, of which
+70 overlap failroute's `no-action` mode. The remaining **390 findings are
+silent-fallback / masked-exception handlers** -- failures converted into
+success-looking values -- a class syntactic rules cannot express by
+construction.
+
+Re-run: `python tools/compare_ruff.py <repo> [<repo> ...]`.
+Results are checked into `bench/`.
+
+## Development
+
+```console
+$ pip install -e ".[test]"
+$ pytest
+$ ruff check .
+```

failroute-0.3.0/README.md

@@ -0,0 +1,182 @@
+# failroute
+
+Static detection of **failure-routing** anti-patterns in Python: the practice of
+converting an underlying failure into a success-like outcome at the wrong layer.
+
+Failure-routing is the root cause behind some of the most insidious correctness
+bugs in real LLM/eval/agent codebases:
+
+```python
+# Before — a judge API outage becomes a perfect "0.0 score" with no way to tell
+try:
+    score = await llm_judge(prompt)
+except Exception:
+    return 0.0  # ← silent fallback: failure looks like a legitimate low score
+
+# After — the failure propagates; callers can route it to the right outcome
+return await llm_judge(prompt)
+```
+
+## What it detects
+
+| Mode | Pattern |
+| --- | --- |
+| `no-action` | `except ...: pass` — the exception is discarded, callers never learn |
+| `silent-fallback` | handler returns/assigns a constant (`None`, `0`, `0.0`, `False`, `[]`, …) without re-raising |
+| `masked-exception` | **catch-all** handler re-raises conditionally yet also falls through to a success-looking return |
+| `name-shadowing` | `except E as e:` whose body rebinds `e` — Python deletes the binding at handler exit, so later uses raise `NameError` |
+
+Findings are emitted as `file:line: mode: message`, or as JSON for CI.
+
+### Logging exemption (two tiers)
+
+A handler that *records* the failure is informational, not silent — but what
+counts as a record depends on how wide the handler is:
+
+- **Catch-all handlers** (`except:` / `except Exception:`) must log at a
+  severity worth reading (`warning`+). A `debug` line or a bare `print(...)`
+  does not survive production triage, so it does not exempt.
+- **Typed handlers** name an anticipated failure mode; recording it at *any*
+  level (even `logger.info`) is enough.
+
+## Usage
+
+```console
+$ failroute path/to/file.py
+$ failroute path/to/dir      # recursive
+$ failroute --repo .         # skip .git/.venv/build/...
+$ failroute --repo . --exclude tests/corpus   # repeatable path exclusions
+$ failroute --json --repo .  | jq 'select(.mode=="silent-fallback")'
+$ failroute --format sarif --output results.sarif --repo .   # code scanning
+$ failroute --threshold 5    # exit 1 when more than 5 findings
+$ python -m failroute .      # module form (no console script needed)
+```
+
+Exit codes: `0` clean, `1` findings above threshold, `2` usage error.
+
+### Output formats
+
+```console
+$ failroute --format text   path/       # default: file:line: mode: message
+$ failroute --format json   path/       # one JSON object per finding
+$ failroute --format sarif  --output scan.sarif path/   # SARIF 2.1.0
+```
+
+SARIF output plugs straight into [GitHub code scanning](
+https://docs.github.com/en/code-security/code-scanning) via the
+`upload-sarif` action, so findings appear inline on pull requests:
+
+```yaml
+- run: failroute --format sarif --output results.sarif --repo .
+- uses: github/codeql-action/upload-sarif@v3
+  with:
+    sarif_file: results.sarif
+```
+
+Or use the bundled composite action, which installs failroute, scans, and
+uploads SARIF in one step:
+
+```yaml
+- uses: feiiiiii5/failroute/action@main
+  with:
+    path: src
+    exclude: tests/corpus fixtures
+    threshold: "0"
+```
+
+Severity mapping: `silent-fallback` → `error`, `no-action` and
+`masked-exception` → `warning`.
+
+### Suppressing findings
+
+Reviewed-and-accepted handlers can be opted out with a line marker (the
+scanner honors both):
+
+```python
+try:
+    return best_effort()
+except Exception:  # failroute: ignore - documented fallback semantics
+    return None
+```
+
+`# pragma: no cover` markers are honored as well (explicitly defensive code).
+
+## Examples that trip it
+
+```python
+def classify(text):                       # no-action
+    try:
+        return model.predict(text)
+    except Exception:
+        pass                                # 💥 swallowed
+
+def score(prompt):                          # silent-fallback
+    try:
+        return judge(prompt)
+    except Exception:
+        return 0.0                          # 💥 outage == "0.0 score"
+
+def fetch(url):                             # silent-fallback (assign)
+    data = None
+    try:
+        data = download(url)
+    except Exception:
+        data = {"items": []}                # 💥 error looks like an empty result
+    return data
+```
+
+## What it does *not* flag (by design)
+
+* `except KeyboardInterrupt` / `except SystemExit` — normally intentional.
+* Handlers that re-raise unconditionally without a fallback.
+* `except` bodies that log at `warning`/`error` **and** re-raise — the failure
+  still propagates; we only flag the success-looking path.
+
+Run `failroute` on its own checkout as a smoke test:
+
+```console
+$ pip install -e .
+$ failroute --repo .     # expected: zero findings (self-hosting)
+```
+
+## Benchmarks & validation
+
+All numbers below are reproducible from this checkout; nothing here is
+copy-pasted from a run that cannot be re-executed.
+
+### Labelled corpus (precision / recall)
+
+`tests/corpus/` holds 19 hand-labelled exception handlers (10 positives across
+all three modes, 9 negatives covering re-raise, log-and-raise, derived values,
+dead code, opt-out markers, and non-fallback constants). Ground truth lives in
+`tests/corpus/manifest.json` and was written from the *semantics* of each
+fixture, independently of tool output.
+
+```
+corpus v1   TP=10  FP=0  FN=0  TN=9
+precision=1.0  recall=1.0
+```
+
+Re-run: `python tools/benchmark.py` (also enforced by `pytest`).
+
+### What syntactic linters miss
+
+Against the source packages of 8 real AI/eval repositories (garak,
+inspect_ai, pydantic-ai, uqlm, trl, smolagents, deepteam, fickling),
+failroute reported **647 findings**; ruff's exception-handling rules
+(`S110` try-except-pass, `S112` try-except-continue) reported **80**, of which
+70 overlap failroute's `no-action` mode. The remaining **390 findings are
+silent-fallback / masked-exception handlers** -- failures converted into
+success-looking values -- a class syntactic rules cannot express by
+construction.
+
+Re-run: `python tools/compare_ruff.py <repo> [<repo> ...]`.
+Results are checked into `bench/`.
+
+## Development
+
+```console
+$ pip install -e ".[test]"
+$ pytest
+$ ruff check .
+```

failroute-0.3.0/pyproject.toml

@@ -0,0 +1,50 @@
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "failroute"
+version = "0.3.0"
+description = "Static detection of failure-routing anti-patterns in Python"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "fc" }]
+keywords = ["static-analysis", "exception-handling", "code-quality", "llm", "reliability"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Quality Assurance",
+]
+dependencies = []
+
+[project.optional-dependencies]
+test = ["pytest>=7"]
+
+[project.scripts]
+failroute = "failroute.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+addopts = "-q"
+
+[tool.ruff]
+target-version = "py39"
+line-length = 110
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP"]
+
+[tool.ruff.lint.per-file-ignores]
+# Corpus fixtures are intentionally synthetic snippets with undefined names.
+"tests/corpus/*" = ["F821", "B017"]

failroute-0.3.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

failroute-0.3.0/src/failroute/__init__.py

@@ -0,0 +1,27 @@
+"""failroute — static detection of failure-routing anti-patterns in Python.
+
+Failure-routing is the practice of converting an underlying failure into a
+*success-like* outcome at the wrong layer: swallowing an exception and
+returning a default truthy/"no error" value, transforming a metric failure
+into a 0.0/False score, or logging-and-continuing where the caller is
+contractually entitled to know the operation failed.
+
+This module implements an AST-based scanner that flags those patterns, so
+tests / CI can treat them like the correctness bugs they are.
+"""
+
+from failroute.analyzer import FailureMode, Finding, scan_path, scan_repo, scan_source
+from failroute.cli import main
+from failroute.sarif import to_sarif, to_sarif_json
+
+__all__ = [
+    "FailureMode",
+    "Finding",
+    "scan_source",
+    "scan_path",
+    "scan_repo",
+    "main",
+    "to_sarif",
+    "to_sarif_json",
+]
+__version__ = "0.3.0"

failroute-0.3.0/src/failroute/__main__.py

@@ -0,0 +1,6 @@
+"""Allow ``python -m failroute`` invocation."""
+
+from failroute.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())