debian-gleam 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- debian_gleam-0.1.0/LICENSE +21 -0
- debian_gleam-0.1.0/PKG-INFO +134 -0
- debian_gleam-0.1.0/README.md +104 -0
- debian_gleam-0.1.0/pyproject.toml +60 -0
- debian_gleam-0.1.0/setup.cfg +4 -0
- debian_gleam-0.1.0/src/debian_gleam.egg-info/PKG-INFO +134 -0
- debian_gleam-0.1.0/src/debian_gleam.egg-info/SOURCES.txt +104 -0
- debian_gleam-0.1.0/src/debian_gleam.egg-info/dependency_links.txt +1 -0
- debian_gleam-0.1.0/src/debian_gleam.egg-info/entry_points.txt +2 -0
- debian_gleam-0.1.0/src/debian_gleam.egg-info/requires.txt +6 -0
- debian_gleam-0.1.0/src/debian_gleam.egg-info/top_level.txt +1 -0
- debian_gleam-0.1.0/src/gleam/__init__.py +9 -0
- debian_gleam-0.1.0/src/gleam/__main__.py +23 -0
- debian_gleam-0.1.0/src/gleam/core/__init__.py +0 -0
- debian_gleam-0.1.0/src/gleam/core/config/__init__.py +74 -0
- debian_gleam-0.1.0/src/gleam/core/config/binder.py +529 -0
- debian_gleam-0.1.0/src/gleam/core/config/env.py +152 -0
- debian_gleam-0.1.0/src/gleam/core/config/spec.py +649 -0
- debian_gleam-0.1.0/src/gleam/core/model/__init__.py +66 -0
- debian_gleam-0.1.0/src/gleam/core/model/client.py +289 -0
- debian_gleam-0.1.0/src/gleam/core/model/parse.py +237 -0
- debian_gleam-0.1.0/src/gleam/core/model/protocol.py +417 -0
- debian_gleam-0.1.0/src/gleam/core/model/strictjson.py +39 -0
- debian_gleam-0.1.0/src/gleam/core/package_version.py +31 -0
- debian_gleam-0.1.0/src/gleam/core/result.py +210 -0
- debian_gleam-0.1.0/src/gleam/core/sources/__init__.py +7 -0
- debian_gleam-0.1.0/src/gleam/core/sources/bts/__init__.py +67 -0
- debian_gleam-0.1.0/src/gleam/core/sources/bts/challenge.py +177 -0
- debian_gleam-0.1.0/src/gleam/core/sources/bts/client.py +886 -0
- debian_gleam-0.1.0/src/gleam/core/sources/bts/errors.py +36 -0
- debian_gleam-0.1.0/src/gleam/core/sources/bts/messages.py +478 -0
- debian_gleam-0.1.0/src/gleam/core/sources/bts/policy.py +182 -0
- debian_gleam-0.1.0/src/gleam/core/sources/github/__init__.py +100 -0
- debian_gleam-0.1.0/src/gleam/core/sources/github/client.py +375 -0
- debian_gleam-0.1.0/src/gleam/core/sources/github/policy.py +286 -0
- debian_gleam-0.1.0/src/gleam/core/sources/github/shapes.py +163 -0
- debian_gleam-0.1.0/src/gleam/core/sources/repology.py +734 -0
- debian_gleam-0.1.0/src/gleam/core/sources/searxng/__init__.py +75 -0
- debian_gleam-0.1.0/src/gleam/core/sources/searxng/client.py +377 -0
- debian_gleam-0.1.0/src/gleam/core/sources/searxng/lifecycle.py +816 -0
- debian_gleam-0.1.0/src/gleam/core/sources/searxng/normalize.py +162 -0
- debian_gleam-0.1.0/src/gleam/core/sources/searxng/transport.py +80 -0
- debian_gleam-0.1.0/src/gleam/core/sources/searxng/validate.py +261 -0
- debian_gleam-0.1.0/src/gleam/core/sources/udd/__init__.py +37 -0
- debian_gleam-0.1.0/src/gleam/core/sources/udd/catalog.py +673 -0
- debian_gleam-0.1.0/src/gleam/core/sources/udd/client.py +415 -0
- debian_gleam-0.1.0/src/gleam/core/sources/udd/pgdriver.py +377 -0
- debian_gleam-0.1.0/src/gleam/core/storage/__init__.py +81 -0
- debian_gleam-0.1.0/src/gleam/core/storage/inference.py +172 -0
- debian_gleam-0.1.0/src/gleam/core/storage/runstore.py +516 -0
- debian_gleam-0.1.0/src/gleam/core/transport/__init__.py +42 -0
- debian_gleam-0.1.0/src/gleam/core/transport/extract.py +463 -0
- debian_gleam-0.1.0/src/gleam/core/transport/net.py +548 -0
- debian_gleam-0.1.0/src/gleam/core/transport/outcomes.py +46 -0
- debian_gleam-0.1.0/src/gleam/core/transport/safety.py +157 -0
- debian_gleam-0.1.0/src/gleam/core/user_agent.py +10 -0
- debian_gleam-0.1.0/src/gleam/core/verbose.py +102 -0
- debian_gleam-0.1.0/src/gleam/data/searxng-settings.yml +58 -0
- debian_gleam-0.1.0/src/gleam/entries/__init__.py +7 -0
- debian_gleam-0.1.0/src/gleam/entries/_human.py +54 -0
- debian_gleam-0.1.0/src/gleam/entries/bts.py +239 -0
- debian_gleam-0.1.0/src/gleam/entries/check.py +426 -0
- debian_gleam-0.1.0/src/gleam/entries/help.py +143 -0
- debian_gleam-0.1.0/src/gleam/entries/investigate.py +3171 -0
- debian_gleam-0.1.0/src/gleam/entries/repology.py +268 -0
- debian_gleam-0.1.0/src/gleam/entries/searxng.py +406 -0
- debian_gleam-0.1.0/src/gleam/entries/udd.py +274 -0
- debian_gleam-0.1.0/src/gleam/entries/version.py +78 -0
- debian_gleam-0.1.0/src/gleam/router.py +61 -0
- debian_gleam-0.1.0/tests/test_config_keys.py +551 -0
- debian_gleam-0.1.0/tests/test_determinism.py +73 -0
- debian_gleam-0.1.0/tests/test_entry_bts.py +260 -0
- debian_gleam-0.1.0/tests/test_entry_check.py +651 -0
- debian_gleam-0.1.0/tests/test_entry_investigate.py +2870 -0
- debian_gleam-0.1.0/tests/test_entry_meta.py +298 -0
- debian_gleam-0.1.0/tests/test_entry_repology.py +234 -0
- debian_gleam-0.1.0/tests/test_entry_searxng.py +727 -0
- debian_gleam-0.1.0/tests/test_entry_udd.py +282 -0
- debian_gleam-0.1.0/tests/test_env_behavior.py +508 -0
- debian_gleam-0.1.0/tests/test_env_e2e.py +201 -0
- debian_gleam-0.1.0/tests/test_faults.py +573 -0
- debian_gleam-0.1.0/tests/test_golden.py +541 -0
- debian_gleam-0.1.0/tests/test_kind_matrix.py +476 -0
- debian_gleam-0.1.0/tests/test_model.py +582 -0
- debian_gleam-0.1.0/tests/test_model_api_key.py +152 -0
- debian_gleam-0.1.0/tests/test_result.py +241 -0
- debian_gleam-0.1.0/tests/test_searxng_lifecycle.py +841 -0
- debian_gleam-0.1.0/tests/test_searxng_settings.py +33 -0
- debian_gleam-0.1.0/tests/test_searxng_validate.py +461 -0
- debian_gleam-0.1.0/tests/test_sources_bts.py +1068 -0
- debian_gleam-0.1.0/tests/test_sources_github.py +465 -0
- debian_gleam-0.1.0/tests/test_sources_repology.py +483 -0
- debian_gleam-0.1.0/tests/test_sources_searxng.py +305 -0
- debian_gleam-0.1.0/tests/test_sources_udd.py +731 -0
- debian_gleam-0.1.0/tests/test_sources_udd_pgdriver.py +390 -0
- debian_gleam-0.1.0/tests/test_storage.py +595 -0
- debian_gleam-0.1.0/tests/test_transport.py +814 -0
- debian_gleam-0.1.0/tests/test_verbose.py +305 -0
- debian_gleam-0.1.0/tests/test_verbose_bts.py +256 -0
- debian_gleam-0.1.0/tests/test_verbose_check.py +193 -0
- debian_gleam-0.1.0/tests/test_verbose_dump.py +252 -0
- debian_gleam-0.1.0/tests/test_verbose_fetcher.py +289 -0
- debian_gleam-0.1.0/tests/test_verbose_investigate.py +423 -0
- debian_gleam-0.1.0/tests/test_verbose_repology.py +227 -0
- debian_gleam-0.1.0/tests/test_verbose_searxng.py +392 -0
- debian_gleam-0.1.0/tests/test_verbose_udd.py +170 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Haolin Xue <tzbkk@outlook.com>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: debian-gleam
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Debian QA evidence-investigation infrastructure with bounded probes, an append-only evidence store, and a local-model investigation agent; upstream archaeology is the first built-in workflow
|
|
5
|
+
Author-email: Haolin Xue <tzbkk@outlook.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tzbkk/gleam
|
|
8
|
+
Project-URL: Repository, https://github.com/tzbkk/gleam
|
|
9
|
+
Project-URL: Issues, https://github.com/tzbkk/gleam/issues
|
|
10
|
+
Keywords: debian,qa,evidence,investigation
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: markdown-it-py==4.0.0
|
|
25
|
+
Requires-Dist: pg8000==1.31.6
|
|
26
|
+
Requires-Dist: trafilatura==2.3.1
|
|
27
|
+
Requires-Dist: verbatimeter==0.1.3
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# Gleam
|
|
32
|
+
|
|
33
|
+
[](https://pypi.org/project/debian-gleam/)
|
|
34
|
+
|
|
35
|
+
Gleam is Debian QA evidence-investigation infrastructure. It collects verifiable facts about Debian packages, keeps every observation in an append-only evidence store, and runs bounded, tool-calling investigations driven by a local model on top of that store. Upstream archaeology, tracing where an orphaned Debian package's development went, is the first built-in workflow. The report it produces is a proposal for human review: nothing is confirmed automatically and no action is ever executed.
|
|
36
|
+
|
|
37
|
+
## Quick start
|
|
38
|
+
|
|
39
|
+
The CLI has eight modules: `udd`, `bts`, `repology`, `check`, `investigate`, `searxng`, `help`, and `version`. Install the Python dependencies (trafilatura for HTML body text, markdown-it-py for README markdown, verbatimeter for quote verification, pg8000 for the UDD database connection; everything else is the standard library), then try the offline commands:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
cd /path/to/gleam
|
|
43
|
+
uv venv .venv && uv pip install -r requirements.txt # or: uv sync
|
|
44
|
+
|
|
45
|
+
python -m gleam help # one-page overview of all eight modules
|
|
46
|
+
python -m gleam help investigate # one module's self-contained help
|
|
47
|
+
python -m gleam version # print the gleam version
|
|
48
|
+
python -m gleam udd --list-datasets # the 18-dataset catalog, no database contact
|
|
49
|
+
python -m gleam check --store runs # validate an evidence store (read-only)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Examples that need a live backend (documented here, but they will not run offline):
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
# Requires network (queries the public UDD mirror)
|
|
56
|
+
python -m gleam udd --package wizznic --dataset sources --json
|
|
57
|
+
# Requires network (fetches a bug's discussion from bugs.debian.org)
|
|
58
|
+
python -m gleam bts --bug-id 1000000 --json
|
|
59
|
+
# Requires network (queries the Repology cross-distro version tracker)
|
|
60
|
+
python -m gleam repology --project ncdu --json
|
|
61
|
+
# Requires a local model endpoint and network (see docs/configuration.md)
|
|
62
|
+
python -m gleam investigate --package wizznic --store runs \
|
|
63
|
+
--model my-model --model-base http://127.0.0.1:8000/v1
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`investigate` rejects startup without a model name and endpoint (exit 2) before it creates any run directory.
|
|
67
|
+
|
|
68
|
+
## The eight modules
|
|
69
|
+
|
|
70
|
+
### udd
|
|
71
|
+
|
|
72
|
+
Query live package facts from the public read-only UDD mirror (`postgresql://udd-mirror.debian.net/udd`) through the built-in pg8000 driver, a pure-Python PostgreSQL wire-protocol client with no native library and no external database tools, running a fixed read-only template set per dataset. Runs are read-only and write nothing to disk.
|
|
73
|
+
|
|
74
|
+
Flags: `--package NAME`, `--dataset NAME` (repeatable), `--release NAME`, `--limit N` (1..50 per dataset), `--dsn URL`, `--timeout SECONDS` (query deadline: socket deadline plus a per-statement budget), `--list-datasets`, `--json`, `--verbose`, `--env-file PATH`.
|
|
75
|
+
|
|
76
|
+
Two modes. `--list-datasets` prints the 18-dataset catalog without touching the database. Query mode requires `--package` and at least one `--dataset`; the UDD data appears only then. An honest empty result (zero rows) is a success, truncation is reported as partial, and failures are reported as failures, never masked.
|
|
77
|
+
|
|
78
|
+
### bts
|
|
79
|
+
|
|
80
|
+
Fetch one Debian BTS bug's complete public discussion. Flags: `--bug-id ID` (1..9999999, no leading zeros), `--source auto|soap|mbox` (default `auto`), `--json`, `--verbose`, `--env-file PATH`. The default path uses the documented SOAP machine interface first and falls back to the official mbox view only on transport-class errors and only when robots.txt explicitly allows it. Rate limits, unknown bugs, and budget exhaustion never fall back, and there is no bypass switch.
|
|
81
|
+
|
|
82
|
+
### repology
|
|
83
|
+
|
|
84
|
+
Query the Repology cross-distro version tracker (default base `https://repology.org`) through the public fetcher safety layer (SSRF guards, DNS pinning, byte caps, timeouts, request budget). Flags: `--project SLUG` (version ladder for one project) or `--search QUERY` (resolve a name to project slugs; exactly one of the two is required, otherwise exit 2), `--json`, `--verbose`, `--env-file PATH`, plus the policy flags `--repology-base`, `--repology-max-requests` (default 6), `--repology-min-interval` (default 1.1s; the API terms require <=1 req/s), `--repology-timeout` (default 15s). HTTP 200 with an empty entry list is the honest "no such project" answer; failures are failures (unknown), never "no results". The client carries a tool-intrinsic robots.txt exemption for the host of its configured base: Repology's own API terms authorize programmatic access on that surface, so the tool is the specific permission; manual `fetch_url` visits of the same host stay governed by the operator exemption file. Read-only: never writes to disk.
|
|
85
|
+
|
|
86
|
+
### check
|
|
87
|
+
|
|
88
|
+
Validate an evidence store for structural integrity. Flags: `--store DIR` (default `runs`), `--run DIR`, `--json`, `--verbose`, `--env-file PATH`. `--store` takes the store root, the directory whose children are run directories: a single run's own directory, or a directory that holds no runs, is rejected as a usage error (exit 2) instead of being misread as a store full of broken runs. `--run DIR` validates exactly one run directory instead (it must carry `run.json` or the run's evidence directories; a configured non-default `--store` cannot be combined with it). The scan is strictly read-only: nothing under the store is created, modified, or removed, and there is no repair function. An empty or missing store is a valid store with zero counts.
|
|
89
|
+
|
|
90
|
+
### investigate
|
|
91
|
+
|
|
92
|
+
Run a bounded, tool-calling investigation of one package with a local model over an OpenAI-compatible endpoint. The model sees the investigation tools (`fetch_url`, `search_web`, `github`, `fetch_bug`, `fetch_udd`, `read_evidence`, `repology`, `inspect_report_state`, `validate_report_draft`, and `finish_investigation`), results come back by `tool_call_id`, and every tool executes serially through the same safety layer. There is zero prefetch: no UDD query runs before the loop starts, and the package name is only policy-validated. The final report is anti-forgery checked against the evidence actually exposed to the model: candidate URLs must come from cited observations, quotes must be verbatim substrings of returned text, GitHub repositories observed in search results or fetched-page links must be covered by a `github` call or a repository-page fetch before the report is accepted. The model can inspect that ledger and repeatedly validate revised drafts before final submission.
|
|
93
|
+
|
|
94
|
+
Flags, grouped:
|
|
95
|
+
|
|
96
|
+
- Inherited query fields (same meaning as udd): `--package`, `--dataset`, `--release`, `--limit`, `--dsn`, `--timeout`.
|
|
97
|
+
- Endpoints: `--model NAME` (required), `--model-base URL` (required), `--model-api-key KEY` (optional bearer token; unset = unauthenticated), `--search-base URL` (default `http://127.0.0.1:8888`), `--github-api-base URL` (default `https://api.github.com`), `--repology-base URL` (default `https://repology.org`), `--store DIR` (default `runs`).
|
|
98
|
+
- Model policy: `--model-max-tokens` (default 8192, cap 65536), `--model-timeout` (default 120s, cap 600s), `--search-timeout` (default 10s, cap 60s).
|
|
99
|
+
- Budgets: `--max-rounds` (default 48, cap 64), `--max-tool-calls` (default 48, cap 512; `finish_investigation` is never counted), `--max-http-requests` (default 52, one shared budget for fetch, robots, search reservations, BTS requests, GitHub client usage, and Repology requests), `--max-body-chars` (default 40000), `--max-links` (default 120), `--max-context-chars` (default 320000; overflow halves the longest tool results inside a legal JSON envelope with double disclosure), `--udd-max-calls` (default 6, cap 50), `--udd-max-rows` (default 200, cap 2000), `--repology-max-requests` (default 6, per-run cap), `--repology-min-interval` (default 1.1s; API terms require <=1 req/s), `--repology-timeout` (default 15s), `--max-report-repairs` (default 2).
|
|
100
|
+
- Fetch policy: `--fetch-max-bytes` (default 300000), `--fetch-timeout` (default 15s), `--fetch-max-redirects` (default 3), `--fetch-min-interval` (default 1s per host), `--robots-exempt-file` (default `robots-exempt.txt` relative to the working directory; the operator-owned robots.txt exemption hostname list, `GLEAM_ROBOTS_EXEMPT_FILE`).
|
|
101
|
+
- Tool toggles: `--no-search-web`, `--no-github`, `--no-bts`, `--no-udd` (fetching stays disabled, the dataset catalog stays available), `--no-repology`.
|
|
102
|
+
- Output: `--json`, `--verbose`, `--env-file PATH`.
|
|
103
|
+
|
|
104
|
+
### searxng
|
|
105
|
+
|
|
106
|
+
Manage the self-hosted SearXNG instance behind the `search_web` tool: one guarded, loopback-bound Docker deployment, addressed by exactly one action per invocation (`--start`, `--stop`, or `--validate`; any other combination is a usage error, exit 2). The only requirement is a working Docker CLI with daemon access, and only when `search_web` is used.
|
|
107
|
+
|
|
108
|
+
Flags: `--image REF` (default `searxng/searxng:latest`), `--name NAME` (default `gleam-searxng`), `--port N` (default 8888, 1..65535), `--bind ADDRESS` (default `127.0.0.1`), `--pull` (force an image update before bring-up), `--search-base URL` (default `http://127.0.0.1:8888`; the same trusted base investigate uses, `GLEAM_SEARCH_BASE`), `--json`, `--verbose`, `--env-file PATH`. The module adds no `.env` key of its own and never writes a file anywhere.
|
|
109
|
+
|
|
110
|
+
| Action | Semantics |
|
|
111
|
+
|---|---|
|
|
112
|
+
| `--validate` | Side-effect-free end-to-end probe: docker availability, the gleam-named container, its healthz endpoint, and one real search query through the same client investigate uses. PASS exits 0; FAIL and BLOCKER exit 1. It never starts anything. |
|
|
113
|
+
| `--start` | Pre-validates first: an already-valid deployment is reused as-is (exit 0, no bring-up at all). Otherwise it brings the configured image up, pulling only when the image is missing (`--pull` forces an update) and refusing any image or same-named container that does not verify as SearXNG, then must pass the same probe before it reports success. |
|
|
114
|
+
| `--stop` | Stops and removes only the gleam-named container: an absent container is a success no-op, an unreachable docker daemon is a failure (never a silent no-op), and a present container is stopped gracefully then removed, never force-removed. Unrelated containers are untouched. |
|
|
115
|
+
|
|
116
|
+
Deployment shape: the official `docker.io/searxng/searxng` image, bound to `127.0.0.1:8888` by default, no host networking, no privileged mode, no docker socket mounted. The shipped `searxng-settings.yml` (package data under `src/gleam/data/`, mounted read-only at bring-up) enables the JSON format and disables the limiter for local-only use; its loopback-only `secret_key` is a deliberate public constant, replaced only if you ever expose the service beyond loopback.
|
|
117
|
+
|
|
118
|
+
### help
|
|
119
|
+
|
|
120
|
+
Print the overview (`gleam help`) or one module's self-contained documentation (`gleam help MODULE`). The module name is a positional argument, not a flag. There are no `-h`/`--help`/`--version` builtins anywhere in gleam: unknown flags, unknown targets, and stray arguments are usage errors (exit 2).
|
|
121
|
+
|
|
122
|
+
### version
|
|
123
|
+
|
|
124
|
+
Print one version line. The single source of truth is `pyproject.toml`; the installed package metadata and the source-tree file always agree. Any argument at all is a usage error (exit 2).
|
|
125
|
+
|
|
126
|
+
## Documentation
|
|
127
|
+
|
|
128
|
+
- [Exit codes](docs/exit-codes.md)
|
|
129
|
+
- [Result JSON](docs/result-json.md)
|
|
130
|
+
- [Configuration (.env)](docs/configuration.md)
|
|
131
|
+
- [Security boundaries](docs/security-boundaries.md)
|
|
132
|
+
- [Evidence store](docs/evidence-store.md)
|
|
133
|
+
- [Investigation stack (SearXNG)](docs/investigation-stack-searxng.md)
|
|
134
|
+
- [Release process](docs/release.md)
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# Gleam
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/debian-gleam/)
|
|
4
|
+
|
|
5
|
+
Gleam is Debian QA evidence-investigation infrastructure. It collects verifiable facts about Debian packages, keeps every observation in an append-only evidence store, and runs bounded, tool-calling investigations driven by a local model on top of that store. Upstream archaeology, tracing where an orphaned Debian package's development went, is the first built-in workflow. The report it produces is a proposal for human review: nothing is confirmed automatically and no action is ever executed.
|
|
6
|
+
|
|
7
|
+
## Quick start
|
|
8
|
+
|
|
9
|
+
The CLI has eight modules: `udd`, `bts`, `repology`, `check`, `investigate`, `searxng`, `help`, and `version`. Install the Python dependencies (trafilatura for HTML body text, markdown-it-py for README markdown, verbatimeter for quote verification, pg8000 for the UDD database connection; everything else is the standard library), then try the offline commands:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
cd /path/to/gleam
|
|
13
|
+
uv venv .venv && uv pip install -r requirements.txt # or: uv sync
|
|
14
|
+
|
|
15
|
+
python -m gleam help # one-page overview of all eight modules
|
|
16
|
+
python -m gleam help investigate # one module's self-contained help
|
|
17
|
+
python -m gleam version # print the gleam version
|
|
18
|
+
python -m gleam udd --list-datasets # the 18-dataset catalog, no database contact
|
|
19
|
+
python -m gleam check --store runs # validate an evidence store (read-only)
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Examples that need a live backend (documented here, but they will not run offline):
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
# Requires network (queries the public UDD mirror)
|
|
26
|
+
python -m gleam udd --package wizznic --dataset sources --json
|
|
27
|
+
# Requires network (fetches a bug's discussion from bugs.debian.org)
|
|
28
|
+
python -m gleam bts --bug-id 1000000 --json
|
|
29
|
+
# Requires network (queries the Repology cross-distro version tracker)
|
|
30
|
+
python -m gleam repology --project ncdu --json
|
|
31
|
+
# Requires a local model endpoint and network (see docs/configuration.md)
|
|
32
|
+
python -m gleam investigate --package wizznic --store runs \
|
|
33
|
+
--model my-model --model-base http://127.0.0.1:8000/v1
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
`investigate` rejects startup without a model name and endpoint (exit 2) before it creates any run directory.
|
|
37
|
+
|
|
38
|
+
## The eight modules
|
|
39
|
+
|
|
40
|
+
### udd
|
|
41
|
+
|
|
42
|
+
Query live package facts from the public read-only UDD mirror (`postgresql://udd-mirror.debian.net/udd`) through the built-in pg8000 driver, a pure-Python PostgreSQL wire-protocol client with no native library and no external database tools, running a fixed read-only template set per dataset. Runs are read-only and write nothing to disk.
|
|
43
|
+
|
|
44
|
+
Flags: `--package NAME`, `--dataset NAME` (repeatable), `--release NAME`, `--limit N` (1..50 per dataset), `--dsn URL`, `--timeout SECONDS` (query deadline: socket deadline plus a per-statement budget), `--list-datasets`, `--json`, `--verbose`, `--env-file PATH`.
|
|
45
|
+
|
|
46
|
+
Two modes. `--list-datasets` prints the 18-dataset catalog without touching the database. Query mode requires `--package` and at least one `--dataset`; the UDD data appears only then. An honest empty result (zero rows) is a success, truncation is reported as partial, and failures are reported as failures, never masked.
|
|
47
|
+
|
|
48
|
+
### bts
|
|
49
|
+
|
|
50
|
+
Fetch one Debian BTS bug's complete public discussion. Flags: `--bug-id ID` (1..9999999, no leading zeros), `--source auto|soap|mbox` (default `auto`), `--json`, `--verbose`, `--env-file PATH`. The default path uses the documented SOAP machine interface first and falls back to the official mbox view only on transport-class errors and only when robots.txt explicitly allows it. Rate limits, unknown bugs, and budget exhaustion never fall back, and there is no bypass switch.
|
|
51
|
+
|
|
52
|
+
### repology
|
|
53
|
+
|
|
54
|
+
Query the Repology cross-distro version tracker (default base `https://repology.org`) through the public fetcher safety layer (SSRF guards, DNS pinning, byte caps, timeouts, request budget). Flags: `--project SLUG` (version ladder for one project) or `--search QUERY` (resolve a name to project slugs; exactly one of the two is required, otherwise exit 2), `--json`, `--verbose`, `--env-file PATH`, plus the policy flags `--repology-base`, `--repology-max-requests` (default 6), `--repology-min-interval` (default 1.1s; the API terms require <=1 req/s), `--repology-timeout` (default 15s). HTTP 200 with an empty entry list is the honest "no such project" answer; failures are failures (unknown), never "no results". The client carries a tool-intrinsic robots.txt exemption for the host of its configured base: Repology's own API terms authorize programmatic access on that surface, so the tool is the specific permission; manual `fetch_url` visits of the same host stay governed by the operator exemption file. Read-only: never writes to disk.
|
|
55
|
+
|
|
56
|
+
### check
|
|
57
|
+
|
|
58
|
+
Validate an evidence store for structural integrity. Flags: `--store DIR` (default `runs`), `--run DIR`, `--json`, `--verbose`, `--env-file PATH`. `--store` takes the store root, the directory whose children are run directories: a single run's own directory, or a directory that holds no runs, is rejected as a usage error (exit 2) instead of being misread as a store full of broken runs. `--run DIR` validates exactly one run directory instead (it must carry `run.json` or the run's evidence directories; a configured non-default `--store` cannot be combined with it). The scan is strictly read-only: nothing under the store is created, modified, or removed, and there is no repair function. An empty or missing store is a valid store with zero counts.
|
|
59
|
+
|
|
60
|
+
### investigate
|
|
61
|
+
|
|
62
|
+
Run a bounded, tool-calling investigation of one package with a local model over an OpenAI-compatible endpoint. The model sees the investigation tools (`fetch_url`, `search_web`, `github`, `fetch_bug`, `fetch_udd`, `read_evidence`, `repology`, `inspect_report_state`, `validate_report_draft`, and `finish_investigation`), results come back by `tool_call_id`, and every tool executes serially through the same safety layer. There is zero prefetch: no UDD query runs before the loop starts, and the package name is only policy-validated. The final report is anti-forgery checked against the evidence actually exposed to the model: candidate URLs must come from cited observations, quotes must be verbatim substrings of returned text, GitHub repositories observed in search results or fetched-page links must be covered by a `github` call or a repository-page fetch before the report is accepted. The model can inspect that ledger and repeatedly validate revised drafts before final submission.
|
|
63
|
+
|
|
64
|
+
Flags, grouped:
|
|
65
|
+
|
|
66
|
+
- Inherited query fields (same meaning as udd): `--package`, `--dataset`, `--release`, `--limit`, `--dsn`, `--timeout`.
|
|
67
|
+
- Endpoints: `--model NAME` (required), `--model-base URL` (required), `--model-api-key KEY` (optional bearer token; unset = unauthenticated), `--search-base URL` (default `http://127.0.0.1:8888`), `--github-api-base URL` (default `https://api.github.com`), `--repology-base URL` (default `https://repology.org`), `--store DIR` (default `runs`).
|
|
68
|
+
- Model policy: `--model-max-tokens` (default 8192, cap 65536), `--model-timeout` (default 120s, cap 600s), `--search-timeout` (default 10s, cap 60s).
|
|
69
|
+
- Budgets: `--max-rounds` (default 48, cap 64), `--max-tool-calls` (default 48, cap 512; `finish_investigation` is never counted), `--max-http-requests` (default 52, one shared budget for fetch, robots, search reservations, BTS requests, GitHub client usage, and Repology requests), `--max-body-chars` (default 40000), `--max-links` (default 120), `--max-context-chars` (default 320000; overflow halves the longest tool results inside a legal JSON envelope with double disclosure), `--udd-max-calls` (default 6, cap 50), `--udd-max-rows` (default 200, cap 2000), `--repology-max-requests` (default 6, per-run cap), `--repology-min-interval` (default 1.1s; API terms require <=1 req/s), `--repology-timeout` (default 15s), `--max-report-repairs` (default 2).
|
|
70
|
+
- Fetch policy: `--fetch-max-bytes` (default 300000), `--fetch-timeout` (default 15s), `--fetch-max-redirects` (default 3), `--fetch-min-interval` (default 1s per host), `--robots-exempt-file` (default `robots-exempt.txt` relative to the working directory; the operator-owned robots.txt exemption hostname list, `GLEAM_ROBOTS_EXEMPT_FILE`).
|
|
71
|
+
- Tool toggles: `--no-search-web`, `--no-github`, `--no-bts`, `--no-udd` (fetching stays disabled, the dataset catalog stays available), `--no-repology`.
|
|
72
|
+
- Output: `--json`, `--verbose`, `--env-file PATH`.
|
|
73
|
+
|
|
74
|
+
### searxng
|
|
75
|
+
|
|
76
|
+
Manage the self-hosted SearXNG instance behind the `search_web` tool: one guarded, loopback-bound Docker deployment, addressed by exactly one action per invocation (`--start`, `--stop`, or `--validate`; any other combination is a usage error, exit 2). The only requirement is a working Docker CLI with daemon access, and only when `search_web` is used.
|
|
77
|
+
|
|
78
|
+
Flags: `--image REF` (default `searxng/searxng:latest`), `--name NAME` (default `gleam-searxng`), `--port N` (default 8888, 1..65535), `--bind ADDRESS` (default `127.0.0.1`), `--pull` (force an image update before bring-up), `--search-base URL` (default `http://127.0.0.1:8888`; the same trusted base investigate uses, `GLEAM_SEARCH_BASE`), `--json`, `--verbose`, `--env-file PATH`. The module adds no `.env` key of its own and never writes a file anywhere.
|
|
79
|
+
|
|
80
|
+
| Action | Semantics |
|
|
81
|
+
|---|---|
|
|
82
|
+
| `--validate` | Side-effect-free end-to-end probe: docker availability, the gleam-named container, its healthz endpoint, and one real search query through the same client investigate uses. PASS exits 0; FAIL and BLOCKER exit 1. It never starts anything. |
|
|
83
|
+
| `--start` | Pre-validates first: an already-valid deployment is reused as-is (exit 0, no bring-up at all). Otherwise it brings the configured image up, pulling only when the image is missing (`--pull` forces an update) and refusing any image or same-named container that does not verify as SearXNG, then must pass the same probe before it reports success. |
|
|
84
|
+
| `--stop` | Stops and removes only the gleam-named container: an absent container is a success no-op, an unreachable docker daemon is a failure (never a silent no-op), and a present container is stopped gracefully then removed, never force-removed. Unrelated containers are untouched. |
|
|
85
|
+
|
|
86
|
+
Deployment shape: the official `docker.io/searxng/searxng` image, bound to `127.0.0.1:8888` by default, no host networking, no privileged mode, no docker socket mounted. The shipped `searxng-settings.yml` (package data under `src/gleam/data/`, mounted read-only at bring-up) enables the JSON format and disables the limiter for local-only use; its loopback-only `secret_key` is a deliberate public constant, replaced only if you ever expose the service beyond loopback.
|
|
87
|
+
|
|
88
|
+
### help
|
|
89
|
+
|
|
90
|
+
Print the overview (`gleam help`) or one module's self-contained documentation (`gleam help MODULE`). The module name is a positional argument, not a flag. There are no `-h`/`--help`/`--version` builtins anywhere in gleam: unknown flags, unknown targets, and stray arguments are usage errors (exit 2).
|
|
91
|
+
|
|
92
|
+
### version
|
|
93
|
+
|
|
94
|
+
Print one version line. The single source of truth is `pyproject.toml`; the installed package metadata and the source-tree file always agree. Any argument at all is a usage error (exit 2).
|
|
95
|
+
|
|
96
|
+
## Documentation
|
|
97
|
+
|
|
98
|
+
- [Exit codes](docs/exit-codes.md)
|
|
99
|
+
- [Result JSON](docs/result-json.md)
|
|
100
|
+
- [Configuration (.env)](docs/configuration.md)
|
|
101
|
+
- [Security boundaries](docs/security-boundaries.md)
|
|
102
|
+
- [Evidence store](docs/evidence-store.md)
|
|
103
|
+
- [Investigation stack (SearXNG)](docs/investigation-stack-searxng.md)
|
|
104
|
+
- [Release process](docs/release.md)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "debian-gleam"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Debian QA evidence-investigation infrastructure with bounded probes, an append-only evidence store, and a local-model investigation agent; upstream archaeology is the first built-in workflow"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
keywords = ["debian", "qa", "evidence", "investigation"]
|
|
7
|
+
classifiers = [
|
|
8
|
+
"Development Status :: 3 - Alpha",
|
|
9
|
+
"Environment :: Console",
|
|
10
|
+
"Intended Audience :: Developers",
|
|
11
|
+
"License :: OSI Approved :: MIT License",
|
|
12
|
+
"Operating System :: OS Independent",
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
18
|
+
]
|
|
19
|
+
requires-python = ">=3.11"
|
|
20
|
+
license = {text = "MIT"}
|
|
21
|
+
authors = [
|
|
22
|
+
{name = "Haolin Xue", email = "tzbkk@outlook.com"},
|
|
23
|
+
]
|
|
24
|
+
# Runtime third-party dependencies are deliberately minimal: trafilatura
|
|
25
|
+
# (HTML body text), markdown-it-py (README markdown -> text), verbatimeter
|
|
26
|
+
# (deterministic word-level quote verification) and pg8000 (pure-Python
|
|
27
|
+
# PostgreSQL wire-protocol driver, so UDD access stays platform-neutral
|
|
28
|
+
# with no psql binary and no native libpq).
|
|
29
|
+
dependencies = [
|
|
30
|
+
"markdown-it-py==4.0.0",
|
|
31
|
+
"pg8000==1.31.6",
|
|
32
|
+
"trafilatura==2.3.1",
|
|
33
|
+
"verbatimeter==0.1.3",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/tzbkk/gleam"
|
|
38
|
+
Repository = "https://github.com/tzbkk/gleam"
|
|
39
|
+
Issues = "https://github.com/tzbkk/gleam/issues"
|
|
40
|
+
|
|
41
|
+
[project.scripts]
|
|
42
|
+
gleam = "gleam.router:dispatch"
|
|
43
|
+
|
|
44
|
+
[project.optional-dependencies]
|
|
45
|
+
dev = []
|
|
46
|
+
|
|
47
|
+
[build-system]
|
|
48
|
+
requires = ["setuptools>=68"]
|
|
49
|
+
build-backend = "setuptools.build_meta"
|
|
50
|
+
|
|
51
|
+
[tool.setuptools.packages.find]
|
|
52
|
+
where = ["src"]
|
|
53
|
+
include = ["gleam*"]
|
|
54
|
+
|
|
55
|
+
[tool.setuptools.package-data]
|
|
56
|
+
gleam = ["data/*.yml"]
|
|
57
|
+
|
|
58
|
+
[tool.uv]
|
|
59
|
+
# Dependency management and venv only; no publishing, no frameworks
|
|
60
|
+
package = true
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: debian-gleam
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Debian QA evidence-investigation infrastructure with bounded probes, an append-only evidence store, and a local-model investigation agent; upstream archaeology is the first built-in workflow
|
|
5
|
+
Author-email: Haolin Xue <tzbkk@outlook.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/tzbkk/gleam
|
|
8
|
+
Project-URL: Repository, https://github.com/tzbkk/gleam
|
|
9
|
+
Project-URL: Issues, https://github.com/tzbkk/gleam/issues
|
|
10
|
+
Keywords: debian,qa,evidence,investigation
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: markdown-it-py==4.0.0
|
|
25
|
+
Requires-Dist: pg8000==1.31.6
|
|
26
|
+
Requires-Dist: trafilatura==2.3.1
|
|
27
|
+
Requires-Dist: verbatimeter==0.1.3
|
|
28
|
+
Provides-Extra: dev
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# Gleam
|
|
32
|
+
|
|
33
|
+
[](https://pypi.org/project/debian-gleam/)
|
|
34
|
+
|
|
35
|
+
Gleam is Debian QA evidence-investigation infrastructure. It collects verifiable facts about Debian packages, keeps every observation in an append-only evidence store, and runs bounded, tool-calling investigations driven by a local model on top of that store. Upstream archaeology, tracing where an orphaned Debian package's development went, is the first built-in workflow. The report it produces is a proposal for human review: nothing is confirmed automatically and no action is ever executed.
|
|
36
|
+
|
|
37
|
+
## Quick start
|
|
38
|
+
|
|
39
|
+
The CLI has eight modules: `udd`, `bts`, `repology`, `check`, `investigate`, `searxng`, `help`, and `version`. Install the Python dependencies (trafilatura for HTML body text, markdown-it-py for README markdown, verbatimeter for quote verification, pg8000 for the UDD database connection; everything else is the standard library), then try the offline commands:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
cd /path/to/gleam
|
|
43
|
+
uv venv .venv && uv pip install -r requirements.txt # or: uv sync
|
|
44
|
+
|
|
45
|
+
python -m gleam help # one-page overview of all eight modules
|
|
46
|
+
python -m gleam help investigate # one module's self-contained help
|
|
47
|
+
python -m gleam version # print the gleam version
|
|
48
|
+
python -m gleam udd --list-datasets # the 18-dataset catalog, no database contact
|
|
49
|
+
python -m gleam check --store runs # validate an evidence store (read-only)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Examples that need a live backend (documented here, but they will not run offline):
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
# Requires network (queries the public UDD mirror)
|
|
56
|
+
python -m gleam udd --package wizznic --dataset sources --json
|
|
57
|
+
# Requires network (fetches a bug's discussion from bugs.debian.org)
|
|
58
|
+
python -m gleam bts --bug-id 1000000 --json
|
|
59
|
+
# Requires network (queries the Repology cross-distro version tracker)
|
|
60
|
+
python -m gleam repology --project ncdu --json
|
|
61
|
+
# Requires a local model endpoint and network (see docs/configuration.md)
|
|
62
|
+
python -m gleam investigate --package wizznic --store runs \
|
|
63
|
+
--model my-model --model-base http://127.0.0.1:8000/v1
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`investigate` rejects startup without a model name and endpoint (exit 2) before it creates any run directory.
|
|
67
|
+
|
|
68
|
+
## The eight modules
|
|
69
|
+
|
|
70
|
+
### udd
|
|
71
|
+
|
|
72
|
+
Query live package facts from the public read-only UDD mirror (`postgresql://udd-mirror.debian.net/udd`) through the built-in pg8000 driver, a pure-Python PostgreSQL wire-protocol client with no native library and no external database tools, running a fixed read-only template set per dataset. Runs are read-only and write nothing to disk.
|
|
73
|
+
|
|
74
|
+
Flags: `--package NAME`, `--dataset NAME` (repeatable), `--release NAME`, `--limit N` (1..50 per dataset), `--dsn URL`, `--timeout SECONDS` (query deadline: socket deadline plus a per-statement budget), `--list-datasets`, `--json`, `--verbose`, `--env-file PATH`.
|
|
75
|
+
|
|
76
|
+
Two modes. `--list-datasets` prints the 18-dataset catalog without touching the database. Query mode requires `--package` and at least one `--dataset`; the UDD data appears only then. An honest empty result (zero rows) is a success, truncation is reported as partial, and failures are reported as failures, never masked.
|
|
77
|
+
|
|
78
|
+
### bts
|
|
79
|
+
|
|
80
|
+
Fetch one Debian BTS bug's complete public discussion. Flags: `--bug-id ID` (1..9999999, no leading zeros), `--source auto|soap|mbox` (default `auto`), `--json`, `--verbose`, `--env-file PATH`. The default path uses the documented SOAP machine interface first and falls back to the official mbox view only on transport-class errors and only when robots.txt explicitly allows it. Rate limits, unknown bugs, and budget exhaustion never fall back, and there is no bypass switch.
|
|
81
|
+
|
|
82
|
+
### repology
|
|
83
|
+
|
|
84
|
+
Query the Repology cross-distro version tracker (default base `https://repology.org`) through the public fetcher safety layer (SSRF guards, DNS pinning, byte caps, timeouts, request budget). Flags: `--project SLUG` (version ladder for one project) or `--search QUERY` (resolve a name to project slugs; exactly one of the two is required, otherwise exit 2), `--json`, `--verbose`, `--env-file PATH`, plus the policy flags `--repology-base`, `--repology-max-requests` (default 6), `--repology-min-interval` (default 1.1s; the API terms require <=1 req/s), `--repology-timeout` (default 15s). HTTP 200 with an empty entry list is the honest "no such project" answer; failures are failures (unknown), never "no results". The client carries a tool-intrinsic robots.txt exemption for the host of its configured base: Repology's own API terms authorize programmatic access on that surface, so the tool is the specific permission; manual `fetch_url` visits of the same host stay governed by the operator exemption file. Read-only: never writes to disk.
|
|
85
|
+
|
|
86
|
+
### check
|
|
87
|
+
|
|
88
|
+
Validate an evidence store for structural integrity. Flags: `--store DIR` (default `runs`), `--run DIR`, `--json`, `--verbose`, `--env-file PATH`. `--store` takes the store root, the directory whose children are run directories: a single run's own directory, or a directory that holds no runs, is rejected as a usage error (exit 2) instead of being misread as a store full of broken runs. `--run DIR` validates exactly one run directory instead (it must carry `run.json` or the run's evidence directories; a configured non-default `--store` cannot be combined with it). The scan is strictly read-only: nothing under the store is created, modified, or removed, and there is no repair function. An empty or missing store is a valid store with zero counts.
|
|
89
|
+
|
|
90
|
+
### investigate
|
|
91
|
+
|
|
92
|
+
Run a bounded, tool-calling investigation of one package with a local model over an OpenAI-compatible endpoint. The model sees the investigation tools (`fetch_url`, `search_web`, `github`, `fetch_bug`, `fetch_udd`, `read_evidence`, `repology`, `inspect_report_state`, `validate_report_draft`, and `finish_investigation`), results come back by `tool_call_id`, and every tool executes serially through the same safety layer. There is zero prefetch: no UDD query runs before the loop starts, and the package name is only policy-validated. The final report is anti-forgery checked against the evidence actually exposed to the model: candidate URLs must come from cited observations, quotes must be verbatim substrings of returned text, GitHub repositories observed in search results or fetched-page links must be covered by a `github` call or a repository-page fetch before the report is accepted. The model can inspect that ledger and repeatedly validate revised drafts before final submission.
|
|
93
|
+
|
|
94
|
+
Flags, grouped:
|
|
95
|
+
|
|
96
|
+
- Inherited query fields (same meaning as udd): `--package`, `--dataset`, `--release`, `--limit`, `--dsn`, `--timeout`.
|
|
97
|
+
- Endpoints: `--model NAME` (required), `--model-base URL` (required), `--model-api-key KEY` (optional bearer token; unset = unauthenticated), `--search-base URL` (default `http://127.0.0.1:8888`), `--github-api-base URL` (default `https://api.github.com`), `--repology-base URL` (default `https://repology.org`), `--store DIR` (default `runs`).
|
|
98
|
+
- Model policy: `--model-max-tokens` (default 8192, cap 65536), `--model-timeout` (default 120s, cap 600s), `--search-timeout` (default 10s, cap 60s).
|
|
99
|
+
- Budgets: `--max-rounds` (default 48, cap 64), `--max-tool-calls` (default 48, cap 512; `finish_investigation` is never counted), `--max-http-requests` (default 52, one shared budget for fetch, robots, search reservations, BTS requests, GitHub client usage, and Repology requests), `--max-body-chars` (default 40000), `--max-links` (default 120), `--max-context-chars` (default 320000; overflow halves the longest tool results inside a legal JSON envelope with double disclosure), `--udd-max-calls` (default 6, cap 50), `--udd-max-rows` (default 200, cap 2000), `--repology-max-requests` (default 6, per-run cap), `--repology-min-interval` (default 1.1s; API terms require <=1 req/s), `--repology-timeout` (default 15s), `--max-report-repairs` (default 2).
|
|
100
|
+
- Fetch policy: `--fetch-max-bytes` (default 300000), `--fetch-timeout` (default 15s), `--fetch-max-redirects` (default 3), `--fetch-min-interval` (default 1s per host), `--robots-exempt-file` (default `robots-exempt.txt` relative to the working directory; the operator-owned robots.txt exemption hostname list, `GLEAM_ROBOTS_EXEMPT_FILE`).
|
|
101
|
+
- Tool toggles: `--no-search-web`, `--no-github`, `--no-bts`, `--no-udd` (fetching stays disabled, the dataset catalog stays available), `--no-repology`.
|
|
102
|
+
- Output: `--json`, `--verbose`, `--env-file PATH`.
|
|
103
|
+
|
|
104
|
+
### searxng
|
|
105
|
+
|
|
106
|
+
Manage the self-hosted SearXNG instance behind the `search_web` tool: one guarded, loopback-bound Docker deployment, addressed by exactly one action per invocation (`--start`, `--stop`, or `--validate`; any other combination is a usage error, exit 2). The only requirement is a working Docker CLI with daemon access, and only when `search_web` is used.
|
|
107
|
+
|
|
108
|
+
Flags: `--image REF` (default `searxng/searxng:latest`), `--name NAME` (default `gleam-searxng`), `--port N` (default 8888, 1..65535), `--bind ADDRESS` (default `127.0.0.1`), `--pull` (force an image update before bring-up), `--search-base URL` (default `http://127.0.0.1:8888`; the same trusted base investigate uses, `GLEAM_SEARCH_BASE`), `--json`, `--verbose`, `--env-file PATH`. The module adds no `.env` key of its own and never writes a file anywhere.
|
|
109
|
+
|
|
110
|
+
| Action | Semantics |
|
|
111
|
+
|---|---|
|
|
112
|
+
| `--validate` | Side-effect-free end-to-end probe: docker availability, the gleam-named container, its healthz endpoint, and one real search query through the same client investigate uses. PASS exits 0; FAIL and BLOCKER exit 1. It never starts anything. |
|
|
113
|
+
| `--start` | Pre-validates first: an already-valid deployment is reused as-is (exit 0, no bring-up at all). Otherwise it brings the configured image up, pulling only when the image is missing (`--pull` forces an update) and refusing any image or same-named container that does not verify as SearXNG, then must pass the same probe before it reports success. |
|
|
114
|
+
| `--stop` | Stops and removes only the gleam-named container: an absent container is a success no-op, an unreachable docker daemon is a failure (never a silent no-op), and a present container is stopped gracefully then removed, never force-removed. Unrelated containers are untouched. |
|
|
115
|
+
|
|
116
|
+
Deployment shape: the official `docker.io/searxng/searxng` image, bound to `127.0.0.1:8888` by default, no host networking, no privileged mode, no docker socket mounted. The shipped `searxng-settings.yml` (package data under `src/gleam/data/`, mounted read-only at bring-up) enables the JSON format and disables the limiter for local-only use; its loopback-only `secret_key` is a deliberate public constant, replaced only if you ever expose the service beyond loopback.
|
|
117
|
+
|
|
118
|
+
### help
|
|
119
|
+
|
|
120
|
+
Print the overview (`gleam help`) or one module's self-contained documentation (`gleam help MODULE`). The module name is a positional argument, not a flag. There are no `-h`/`--help`/`--version` builtins anywhere in gleam: unknown flags, unknown targets, and stray arguments are usage errors (exit 2).
|
|
121
|
+
|
|
122
|
+
### version
|
|
123
|
+
|
|
124
|
+
Print one version line. The single source of truth is `pyproject.toml`; the installed package metadata and the source-tree file always agree. Any argument at all is a usage error (exit 2).
|
|
125
|
+
|
|
126
|
+
## Documentation
|
|
127
|
+
|
|
128
|
+
- [Exit codes](docs/exit-codes.md)
|
|
129
|
+
- [Result JSON](docs/result-json.md)
|
|
130
|
+
- [Configuration (.env)](docs/configuration.md)
|
|
131
|
+
- [Security boundaries](docs/security-boundaries.md)
|
|
132
|
+
- [Evidence store](docs/evidence-store.md)
|
|
133
|
+
- [Investigation stack (SearXNG)](docs/investigation-stack-searxng.md)
|
|
134
|
+
- [Release process](docs/release.md)
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/debian_gleam.egg-info/PKG-INFO
|
|
5
|
+
src/debian_gleam.egg-info/SOURCES.txt
|
|
6
|
+
src/debian_gleam.egg-info/dependency_links.txt
|
|
7
|
+
src/debian_gleam.egg-info/entry_points.txt
|
|
8
|
+
src/debian_gleam.egg-info/requires.txt
|
|
9
|
+
src/debian_gleam.egg-info/top_level.txt
|
|
10
|
+
src/gleam/__init__.py
|
|
11
|
+
src/gleam/__main__.py
|
|
12
|
+
src/gleam/router.py
|
|
13
|
+
src/gleam/core/__init__.py
|
|
14
|
+
src/gleam/core/package_version.py
|
|
15
|
+
src/gleam/core/result.py
|
|
16
|
+
src/gleam/core/user_agent.py
|
|
17
|
+
src/gleam/core/verbose.py
|
|
18
|
+
src/gleam/core/config/__init__.py
|
|
19
|
+
src/gleam/core/config/binder.py
|
|
20
|
+
src/gleam/core/config/env.py
|
|
21
|
+
src/gleam/core/config/spec.py
|
|
22
|
+
src/gleam/core/model/__init__.py
|
|
23
|
+
src/gleam/core/model/client.py
|
|
24
|
+
src/gleam/core/model/parse.py
|
|
25
|
+
src/gleam/core/model/protocol.py
|
|
26
|
+
src/gleam/core/model/strictjson.py
|
|
27
|
+
src/gleam/core/sources/__init__.py
|
|
28
|
+
src/gleam/core/sources/repology.py
|
|
29
|
+
src/gleam/core/sources/bts/__init__.py
|
|
30
|
+
src/gleam/core/sources/bts/challenge.py
|
|
31
|
+
src/gleam/core/sources/bts/client.py
|
|
32
|
+
src/gleam/core/sources/bts/errors.py
|
|
33
|
+
src/gleam/core/sources/bts/messages.py
|
|
34
|
+
src/gleam/core/sources/bts/policy.py
|
|
35
|
+
src/gleam/core/sources/github/__init__.py
|
|
36
|
+
src/gleam/core/sources/github/client.py
|
|
37
|
+
src/gleam/core/sources/github/policy.py
|
|
38
|
+
src/gleam/core/sources/github/shapes.py
|
|
39
|
+
src/gleam/core/sources/searxng/__init__.py
|
|
40
|
+
src/gleam/core/sources/searxng/client.py
|
|
41
|
+
src/gleam/core/sources/searxng/lifecycle.py
|
|
42
|
+
src/gleam/core/sources/searxng/normalize.py
|
|
43
|
+
src/gleam/core/sources/searxng/transport.py
|
|
44
|
+
src/gleam/core/sources/searxng/validate.py
|
|
45
|
+
src/gleam/core/sources/udd/__init__.py
|
|
46
|
+
src/gleam/core/sources/udd/catalog.py
|
|
47
|
+
src/gleam/core/sources/udd/client.py
|
|
48
|
+
src/gleam/core/sources/udd/pgdriver.py
|
|
49
|
+
src/gleam/core/storage/__init__.py
|
|
50
|
+
src/gleam/core/storage/inference.py
|
|
51
|
+
src/gleam/core/storage/runstore.py
|
|
52
|
+
src/gleam/core/transport/__init__.py
|
|
53
|
+
src/gleam/core/transport/extract.py
|
|
54
|
+
src/gleam/core/transport/net.py
|
|
55
|
+
src/gleam/core/transport/outcomes.py
|
|
56
|
+
src/gleam/core/transport/safety.py
|
|
57
|
+
src/gleam/data/searxng-settings.yml
|
|
58
|
+
src/gleam/entries/__init__.py
|
|
59
|
+
src/gleam/entries/_human.py
|
|
60
|
+
src/gleam/entries/bts.py
|
|
61
|
+
src/gleam/entries/check.py
|
|
62
|
+
src/gleam/entries/help.py
|
|
63
|
+
src/gleam/entries/investigate.py
|
|
64
|
+
src/gleam/entries/repology.py
|
|
65
|
+
src/gleam/entries/searxng.py
|
|
66
|
+
src/gleam/entries/udd.py
|
|
67
|
+
src/gleam/entries/version.py
|
|
68
|
+
tests/test_config_keys.py
|
|
69
|
+
tests/test_determinism.py
|
|
70
|
+
tests/test_entry_bts.py
|
|
71
|
+
tests/test_entry_check.py
|
|
72
|
+
tests/test_entry_investigate.py
|
|
73
|
+
tests/test_entry_meta.py
|
|
74
|
+
tests/test_entry_repology.py
|
|
75
|
+
tests/test_entry_searxng.py
|
|
76
|
+
tests/test_entry_udd.py
|
|
77
|
+
tests/test_env_behavior.py
|
|
78
|
+
tests/test_env_e2e.py
|
|
79
|
+
tests/test_faults.py
|
|
80
|
+
tests/test_golden.py
|
|
81
|
+
tests/test_kind_matrix.py
|
|
82
|
+
tests/test_model.py
|
|
83
|
+
tests/test_model_api_key.py
|
|
84
|
+
tests/test_result.py
|
|
85
|
+
tests/test_searxng_lifecycle.py
|
|
86
|
+
tests/test_searxng_settings.py
|
|
87
|
+
tests/test_searxng_validate.py
|
|
88
|
+
tests/test_sources_bts.py
|
|
89
|
+
tests/test_sources_github.py
|
|
90
|
+
tests/test_sources_repology.py
|
|
91
|
+
tests/test_sources_searxng.py
|
|
92
|
+
tests/test_sources_udd.py
|
|
93
|
+
tests/test_sources_udd_pgdriver.py
|
|
94
|
+
tests/test_storage.py
|
|
95
|
+
tests/test_transport.py
|
|
96
|
+
tests/test_verbose.py
|
|
97
|
+
tests/test_verbose_bts.py
|
|
98
|
+
tests/test_verbose_check.py
|
|
99
|
+
tests/test_verbose_dump.py
|
|
100
|
+
tests/test_verbose_fetcher.py
|
|
101
|
+
tests/test_verbose_investigate.py
|
|
102
|
+
tests/test_verbose_repology.py
|
|
103
|
+
tests/test_verbose_searxng.py
|
|
104
|
+
tests/test_verbose_udd.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
gleam
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""Gleam: Debian QA evidence-investigation infrastructure.
|
|
2
|
+
|
|
3
|
+
Seven entry modules (udd, bts, repology, check, investigate, help,
|
|
4
|
+
version) over a shared core (config, transport, storage, sources,
|
|
5
|
+
model). Every business outcome is a unified Result; storage is
|
|
6
|
+
append-only and read paths never mutate the store. The package version
|
|
7
|
+
lives in pyproject.toml (see gleam.entries.version); this module
|
|
8
|
+
carries no state.
|
|
9
|
+
"""
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
from gleam.router import dispatch
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def _reconfigure_streams() -> None:
|
|
7
|
+
"""Pin stdout/stderr to UTF-8 with LF-only line endings.
|
|
8
|
+
|
|
9
|
+
Byte determinism for ``--json`` output and no cp1252
|
|
10
|
+
UnicodeEncodeError in human output: on Windows both streams default
|
|
11
|
+
to the locale encoding and LF->CRLF translation; ``reconfigure`` is
|
|
12
|
+
a no-op elsewhere. Skipped when a stream lacks ``reconfigure``
|
|
13
|
+
(e.g. a replaced/None stream under an embedding host).
|
|
14
|
+
"""
|
|
15
|
+
for stream in (sys.stdout, sys.stderr):
|
|
16
|
+
reconfigure = getattr(stream, "reconfigure", None)
|
|
17
|
+
if reconfigure is not None:
|
|
18
|
+
reconfigure(encoding="utf-8", newline="\n")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
if __name__ == "__main__":
|
|
22
|
+
_reconfigure_streams()
|
|
23
|
+
sys.exit(dispatch(sys.argv[1:]))
|
|
File without changes
|