doc-drift-detector 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- doc_drift_detector-1.0.0/PKG-INFO +276 -0
- doc_drift_detector-1.0.0/README.md +263 -0
- doc_drift_detector-1.0.0/ai/__init__.py +18 -0
- doc_drift_detector-1.0.0/ai/ollama_client.py +117 -0
- doc_drift_detector-1.0.0/ai/prose_rewriter.py +81 -0
- doc_drift_detector-1.0.0/ai/semantic_verifier.py +96 -0
- doc_drift_detector-1.0.0/detectors/__init__.py +63 -0
- doc_drift_detector-1.0.0/detectors/default_value.py +135 -0
- doc_drift_detector-1.0.0/detectors/feature_remove.py +143 -0
- doc_drift_detector-1.0.0/detectors/flag_rename.py +152 -0
- doc_drift_detector-1.0.0/detectors/param_change.py +234 -0
- doc_drift_detector-1.0.0/doc_drift_detector.egg-info/PKG-INFO +276 -0
- doc_drift_detector-1.0.0/doc_drift_detector.egg-info/SOURCES.txt +23 -0
- doc_drift_detector-1.0.0/doc_drift_detector.egg-info/dependency_links.txt +1 -0
- doc_drift_detector-1.0.0/doc_drift_detector.egg-info/entry_points.txt +2 -0
- doc_drift_detector-1.0.0/doc_drift_detector.egg-info/requires.txt +5 -0
- doc_drift_detector-1.0.0/doc_drift_detector.egg-info/top_level.txt +4 -0
- doc_drift_detector-1.0.0/drift_detector.py +487 -0
- doc_drift_detector-1.0.0/pyproject.toml +31 -0
- doc_drift_detector-1.0.0/setup.cfg +4 -0
- doc_drift_detector-1.0.0/tests/test_basic.py +378 -0
- doc_drift_detector-1.0.0/utils/__init__.py +19 -0
- doc_drift_detector-1.0.0/utils/diff_parser.py +241 -0
- doc_drift_detector-1.0.0/utils/doc_finder.py +560 -0
- doc_drift_detector-1.0.0/utils/reporter.py +338 -0
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: doc-drift-detector
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Catch documentation drift in Pull Requests β static + AI hybrid
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Keywords: documentation,drift,git,pr,devops,ci
|
|
7
|
+
Requires-Python: >=3.9
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: colorama>=0.4.6
|
|
10
|
+
Provides-Extra: dev
|
|
11
|
+
Requires-Dist: pytest>=7.4.0; extra == "dev"
|
|
12
|
+
Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
|
|
13
|
+
|
|
14
|
+
# π§ Documentation Drift Detector (DocDrift)
|
|
15
|
+
|
|
16
|
+
[]()
|
|
17
|
+
[]()
|
|
18
|
+
[]()
|
|
19
|
+
|
|
20
|
+
> **Automated Documentation Drift Detection for Pull Requests.**
|
|
21
|
+
> Catches outdated prose, config tables, default values, renamed flags, and removed features *before* they merge into production.
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## π₯ The Problem
|
|
26
|
+
|
|
27
|
+
Documentation silently drifts whenever code changes in a Pull Request:
|
|
28
|
+
* An engineer updates `DEFAULT_TIMEOUT = 30` to `DEFAULT_TIMEOUT = 60` in `settings.py`.
|
|
29
|
+
* The documentation in `docs/config.md` still reads:
|
|
30
|
+
> *"The default timeout is 30 seconds."*
|
|
31
|
+
* Existing validation tools (`python -m doctest`, `mktestdocs`, `Doc Detective`) **only validate executable code blocks**. They **CANNOT** detect whether explanatory prose, config tables, or tutorial instructions still match reality.
|
|
32
|
+
|
|
33
|
+
**DocDrift** solves this by performing semantic diff analysis on code changes in a PR, locating affected documentation, and isolating the **exact sentence** that has driftedβwith actionable fix suggestions.
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
## β‘ Key Capabilities
|
|
38
|
+
|
|
39
|
+
1. **Semantic PR Diff Parsing:** Analyzes git diffs (`base...head`) across Python, JavaScript/TypeScript, Go, Rust, Java, YAML, TOML, JSON, and `.env` files.
|
|
40
|
+
2. **Four Specialized Semantic Detectors:**
|
|
41
|
+
- π’ **Changed Default Values:** Constants (`DEFAULT_TIMEOUT`), dict defaults, function kwargs, YAML/TOML keys, `.env` settings.
|
|
42
|
+
- π© **Renamed Flags & Config Keys:** CLI options (`--old-flag` $\to$ `--new-flag` in argparse, Click, Typer) and renamed configuration keys.
|
|
43
|
+
- βοΈ **Changed Function Parameters:** Added/removed parameters and modified default arguments via AST analysis.
|
|
44
|
+
- ποΈ **Removed Features:** Deleted functions or classes verified to no longer exist in the codebase.
|
|
45
|
+
3. **Smart Natural Language Normalization:** Bridges code and human writing (`DEFAULT_TIMEOUT` $\equiv$ `default_timeout` $\equiv$ `default timeout` $\equiv$ `default-timeout` $\equiv$ `defaultTimeout`).
|
|
46
|
+
4. **False-Alarm Mitigation:** For default value changes, it **only** flags when the **old value** (e.g. `30` or `30s`) actually appears in the documentation sentence.
|
|
47
|
+
5. **Exact Sentence Highlighting:** Extracts and pins the precise faulty sentence and table row.
|
|
48
|
+
6. **First-Class GitHub Actions Workflow:** Generates inline PR diff annotations (`::error`, `::warning`) and PR Step Summaries.
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## π Project Structure
|
|
53
|
+
|
|
54
|
+
```text
|
|
55
|
+
doc-drift-detector/
|
|
56
|
+
βββ drift_detector.py # Main CLI entrypoint
|
|
57
|
+
βββ ai/
|
|
58
|
+
β βββ __init__.py # AI package exports
|
|
59
|
+
β βββ ollama_client.py # Zero-dependency local Ollama REST client
|
|
60
|
+
β βββ semantic_verifier.py # Layer 2: LLM reasoning verification
|
|
61
|
+
β βββ prose_rewriter.py # Layer 3: Natural prose rewriting
|
|
62
|
+
βββ detectors/
|
|
63
|
+
β βββ __init__.py # CodeChange data model & detector exports
|
|
64
|
+
β βββ default_value.py # Detects changed default values & constants
|
|
65
|
+
β βββ flag_rename.py # Detects renamed CLI flags & config keys
|
|
66
|
+
β βββ param_change.py # Detects changed function signatures & defaults
|
|
67
|
+
β βββ feature_remove.py # Detects deleted functions & classes
|
|
68
|
+
βββ utils/
|
|
69
|
+
β βββ __init__.py
|
|
70
|
+
β βββ diff_parser.py # Unified git diff parser with line tracking
|
|
71
|
+
β βββ doc_finder.py # Doc discovery, name normalizer, drift & auto-fix engine
|
|
72
|
+
β βββ reporter.py # Colored terminal, JSON & GitHub action reporter
|
|
73
|
+
βββ .github/workflows/
|
|
74
|
+
β βββ doc-drift.yml # Ready-to-use GitHub Action workflow
|
|
75
|
+
βββ requirements.txt # Dependencies
|
|
76
|
+
βββ README.md # Documentation
|
|
77
|
+
βββ tests/
|
|
78
|
+
βββ test_basic.py # Complete test suite (15 passing tests)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## π Quickstart
|
|
84
|
+
|
|
85
|
+
### 1. Installation
|
|
86
|
+
|
|
87
|
+
DocDrift is built primarily using Python's standard library for zero-friction setup.
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
cd doc-drift-detector
|
|
91
|
+
pip install -r requirements.txt
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### 2. Analyze a Pull Request Diff
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
# Compare base branch (main) with current branch (HEAD)
|
|
98
|
+
python drift_detector.py --base main --head HEAD
|
|
99
|
+
|
|
100
|
+
# Analyze an offline diff file
|
|
101
|
+
python drift_detector.py --diff-file my_pr.diff
|
|
102
|
+
|
|
103
|
+
# Output JSON report
|
|
104
|
+
python drift_detector.py --base main --head HEAD --json
|
|
105
|
+
|
|
106
|
+
# Save report to a file
|
|
107
|
+
python drift_detector.py --base main --head HEAD --output report.json
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## π CLI Options
|
|
113
|
+
|
|
114
|
+
| Option | Type | Default | Description |
|
|
115
|
+
| :--- | :--- | :--- | :--- |
|
|
116
|
+
| `--base` | `str` | `None` | Base branch or commit SHA (e.g., `main`, `origin/main`, `HEAD~1`) |
|
|
117
|
+
| `--head` | `str` | `None` | Head branch or commit SHA (e.g., `HEAD`) |
|
|
118
|
+
| `--diff-file` | `path` | `None` | Path to pre-generated `.diff` or `.patch` file |
|
|
119
|
+
| `--repo-dir` | `path` | `.` | Root directory of the repository to scan |
|
|
120
|
+
| `--docs-dir` | `path` | `.` | Directory containing documentation files (`.md`, `.rst`, `.adoc`, `.txt`) |
|
|
121
|
+
| `--format` | `choice` | `text` | Output format: `text`, `json`, `github`, or `markdown` |
|
|
122
|
+
| `--json` | `flag` | `False` | Shorthand for `--format json` |
|
|
123
|
+
| `--output` | `path` | `None` | File path to write the formatted report |
|
|
124
|
+
| `--ai` | `flag` | `False` | Enable local Ollama AI hybrid verification & prose rewriting |
|
|
125
|
+
| `--ai-model` | `str` | `auto` | Ollama model to use (e.g. `llama3.2:1b`, `llama3.2`, `mistral`) |
|
|
126
|
+
| `--fix`, `--autofix` | `flag` | `False` | Automatically rewrite outdated documentation files on disk |
|
|
127
|
+
| `--min-confidence` | `choice` | `medium`| Minimum issue confidence to report: `high`, `medium`, or `low` |
|
|
128
|
+
| `--fail-on` | `choice` | `high` | Exit with code `1` if drift of this level is found: `high`, `medium`, `any`, `none` |
|
|
129
|
+
| `-v`, `--verbose` | `flag` | `False` | Enable step-by-step pipeline logging |
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
## π How It Works (The 5-Step Pipeline)
|
|
134
|
+
|
|
135
|
+
```mermaid
|
|
136
|
+
flowchart TD
|
|
137
|
+
A["PR Git Diff<br>(git diff base...head)"] --> B["Step 1: Diff Parser<br>(Track +/- lines & code files)"]
|
|
138
|
+
B --> C["Step 2: Semantic Detectors<br>(Defaults, Flags, Params, Features)"]
|
|
139
|
+
C --> D["Step 3: Doc Finder<br>(Normalize names across .md, .rst, .adoc)"]
|
|
140
|
+
D --> E["Step 4: Smart Drift Engine<br>(Verify old value in text & score confidence)"]
|
|
141
|
+
E --> F["Step 5: Multi-Format Reporter<br>(CLI Boxes, JSON, GitHub Annotations)"]
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### Step 1: Parse PR Diff (`utils/diff_parser.py`)
|
|
145
|
+
- Executes `git diff <base>...<head> --unified=0`.
|
|
146
|
+
- Isolates code files (`.py`, `.js`, `.ts`, `.go`, `.rs`, `.java`, `.yaml`, `.yml`, `.toml`, `.json`, `.env`, `.cfg`, `.ini`).
|
|
147
|
+
- Extracts exact removed (`-`) and added (`+`) lines with source line numbers.
|
|
148
|
+
|
|
149
|
+
### Step 2: Semantic Code Change Detection (`detectors/`)
|
|
150
|
+
- **Default Values (`default_value.py`):** Pairs removed `VAR = OLD` with added `VAR = NEW`.
|
|
151
|
+
- **Flag Renames (`flag_rename.py`):** Pairs removed `--old-flag` with added `--new-flag`, or renamed config keys with identical values.
|
|
152
|
+
- **Param Changes (`param_change.py`):** Uses Python AST / regex to detect signature modifications and modified default arguments.
|
|
153
|
+
- **Feature Removals (`feature_remove.py`):** Detects deleted functions or classes, verifying they don't exist elsewhere in the repo.
|
|
154
|
+
|
|
155
|
+
### Step 3: Documentation Search (`utils/doc_finder.py`)
|
|
156
|
+
- Scans `.md`, `.rst`, `.adoc`, and `.txt` files (excluding `.git`, `node_modules`, `venv`, etc.).
|
|
157
|
+
- Normalizes names into searchable variants (`DEFAULT_TIMEOUT` $\to$ `"default timeout"`, `default-timeout`, etc.).
|
|
158
|
+
- Extracts the exact sentence using punctuation boundaries and table row delimitation.
|
|
159
|
+
|
|
160
|
+
### Step 4: Smart Drift Decision Engine
|
|
161
|
+
- **DEFAULT_VALUE_CHANGED:** Flags **only** if the old value (e.g. `30` or `30s`) appears in the matched doc sentence. If the identifier is mentioned without the old value, it is skipped to prevent false alarms.
|
|
162
|
+
- **FLAG_RENAMED:** Flags if the old flag name appears in documentation.
|
|
163
|
+
- **PARAM_CHANGED:** Flags if removed parameter or old default is in documentation.
|
|
164
|
+
- **FEATURE_REMOVED:** Flags if documentation references the deleted feature.
|
|
165
|
+
|
|
166
|
+
#### Confidence Levels
|
|
167
|
+
- π΄ **HIGH (Merge Blocker):** Exact name match + old value confirmed in doc sentence.
|
|
168
|
+
- π‘ **MEDIUM (Warning):** Normalized match or function signature change.
|
|
169
|
+
- βͺ **LOW (Info):** Weak / partial match.
|
|
170
|
+
|
|
171
|
+
### Step 5: Multi-Channel Reporting (`utils/reporter.py`)
|
|
172
|
+
- **Human-Readable CLI:** Rich colored badges, exact file:line references, quotes, and suggested fixes.
|
|
173
|
+
- **JSON Schema:** Machine-readable output for dashboards and metrics.
|
|
174
|
+
- **GitHub PR Annotations:** Inline warnings directly in the GitHub PR review interface.
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## π₯οΈ Example CLI Output
|
|
179
|
+
|
|
180
|
+
```text
|
|
181
|
+
======================================================================
|
|
182
|
+
DOCUMENTATION DRIFT DETECTOR β PR ANALYSIS
|
|
183
|
+
======================================================================
|
|
184
|
+
Code Changes Analyzed: 3 | Doc Files Scanned: 4
|
|
185
|
+
Drift Issues Found: 2 (2 HIGH, 0 MEDIUM, 0 LOW)
|
|
186
|
+
----------------------------------------------------------------------
|
|
187
|
+
|
|
188
|
+
Issue #1: DEFAULT VALUE DRIFT [HIGH - MERGE BLOCKER]
|
|
189
|
+
Doc Location : docs/configuration.md:42
|
|
190
|
+
Wrong Sentence: "The default timeout is 30 seconds before reconnecting."
|
|
191
|
+
Code Culprit : src/config.py:12 [DEFAULT_TIMEOUT] (30 -> 60)
|
|
192
|
+
Suggested Fix : Update default value from '30' to '60'
|
|
193
|
+
|
|
194
|
+
Issue #2: FLAG / CONFIG RENAME DRIFT [HIGH - MERGE BLOCKER]
|
|
195
|
+
Doc Location : docs/cli_reference.md:88
|
|
196
|
+
Wrong Sentence: "Run `worker --old-flag` to enable legacy processing mode."
|
|
197
|
+
Code Culprit : src/cli.py:54 [--old-flag] (--old-flag -> --new-flag)
|
|
198
|
+
Suggested Fix : Rename flag/key '--old-flag' to '--new-flag'
|
|
199
|
+
|
|
200
|
+
======================================================================
|
|
201
|
+
FAILED: 2 high-confidence drift issue(s) require documentation updates.
|
|
202
|
+
======================================================================
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
---
|
|
206
|
+
|
|
207
|
+
## π€ GitHub Actions Integration
|
|
208
|
+
|
|
209
|
+
Add `.github/workflows/doc-drift.yml` to your repository:
|
|
210
|
+
|
|
211
|
+
```yaml
|
|
212
|
+
name: Documentation Drift Detection
|
|
213
|
+
|
|
214
|
+
on:
|
|
215
|
+
pull_request:
|
|
216
|
+
branches: [ main, master ]
|
|
217
|
+
|
|
218
|
+
jobs:
|
|
219
|
+
check-drift:
|
|
220
|
+
runs-on: ubuntu-latest
|
|
221
|
+
steps:
|
|
222
|
+
- uses: actions/checkout@v4
|
|
223
|
+
with:
|
|
224
|
+
fetch-depth: 0
|
|
225
|
+
|
|
226
|
+
- uses: actions/setup-python@v5
|
|
227
|
+
with:
|
|
228
|
+
python-version: '3.11'
|
|
229
|
+
|
|
230
|
+
- name: Run DocDrift
|
|
231
|
+
run: |
|
|
232
|
+
python3 doc-drift-detector/drift_detector.py \
|
|
233
|
+
--base "${{ github.event.pull_request.base.sha }}" \
|
|
234
|
+
--head "${{ github.event.pull_request.head.sha }}" \
|
|
235
|
+
--format github
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
Inline GitHub PR annotations will appear directly on the changed documentation files, blocking merge until documentation is updated!
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## π§ͺ Running Tests
|
|
243
|
+
|
|
244
|
+
The test suite covers all units, detectors, normalization logic, and end-to-end integration:
|
|
245
|
+
|
|
246
|
+
```bash
|
|
247
|
+
python3 -m pytest tests/test_basic.py -v
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Output:
|
|
251
|
+
```text
|
|
252
|
+
tests/test_basic.py::test_diff_parser_filters_code_files PASSED
|
|
253
|
+
tests/test_basic.py::test_diff_parser_tracks_lines PASSED
|
|
254
|
+
tests/test_basic.py::test_detect_default_value_python_constant PASSED
|
|
255
|
+
tests/test_basic.py::test_detect_default_value_yaml_json PASSED
|
|
256
|
+
tests/test_basic.py::test_detect_default_value_env PASSED
|
|
257
|
+
tests/test_basic.py::test_detect_flag_renames_cli PASSED
|
|
258
|
+
tests/test_basic.py::test_detect_flag_renames_config_key PASSED
|
|
259
|
+
tests/test_basic.py::test_detect_param_change_added_parameter PASSED
|
|
260
|
+
tests/test_basic.py::test_detect_param_change_modified_default PASSED
|
|
261
|
+
tests/test_basic.py::test_detect_feature_removal PASSED
|
|
262
|
+
tests/test_basic.py::test_name_variants_normalization PASSED
|
|
263
|
+
tests/test_basic.py::test_extract_exact_sentence PASSED
|
|
264
|
+
tests/test_basic.py::test_drift_default_value_smart_filter PASSED
|
|
265
|
+
tests/test_basic.py::test_end_to_end_pipeline PASSED
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
---
|
|
269
|
+
|
|
270
|
+
## π Hackathon Highlights
|
|
271
|
+
|
|
272
|
+
- **Zero-Dependency Core:** Runs everywhere without heavy ML or external network API calls.
|
|
273
|
+
- **Precision First:** Smart filtering eliminates false alarms by validating value occurrences in context.
|
|
274
|
+
- **Universal Code Support:** Works across Python, JS/TS, Go, Rust, Java, YAML, TOML, JSON, and ENV files.
|
|
275
|
+
- **Developer Experience:** Clear, colored terminal UI and automated GitHub Action PR annotations.
|
|
276
|
+
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
# π§ Documentation Drift Detector (DocDrift)
|
|
2
|
+
|
|
3
|
+
[]()
|
|
4
|
+
[]()
|
|
5
|
+
[]()
|
|
6
|
+
|
|
7
|
+
> **Automated Documentation Drift Detection for Pull Requests.**
|
|
8
|
+
> Catches outdated prose, config tables, default values, renamed flags, and removed features *before* they merge into production.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## π₯ The Problem
|
|
13
|
+
|
|
14
|
+
Documentation silently drifts whenever code changes in a Pull Request:
|
|
15
|
+
* An engineer updates `DEFAULT_TIMEOUT = 30` to `DEFAULT_TIMEOUT = 60` in `settings.py`.
|
|
16
|
+
* The documentation in `docs/config.md` still reads:
|
|
17
|
+
> *"The default timeout is 30 seconds."*
|
|
18
|
+
* Existing validation tools (`python -m doctest`, `mktestdocs`, `Doc Detective`) **only validate executable code blocks**. They **CANNOT** detect whether explanatory prose, config tables, or tutorial instructions still match reality.
|
|
19
|
+
|
|
20
|
+
**DocDrift** solves this by performing semantic diff analysis on code changes in a PR, locating affected documentation, and isolating the **exact sentence** that has driftedβwith actionable fix suggestions.
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
## β‘ Key Capabilities
|
|
25
|
+
|
|
26
|
+
1. **Semantic PR Diff Parsing:** Analyzes git diffs (`base...head`) across Python, JavaScript/TypeScript, Go, Rust, Java, YAML, TOML, JSON, and `.env` files.
|
|
27
|
+
2. **Four Specialized Semantic Detectors:**
|
|
28
|
+
- π’ **Changed Default Values:** Constants (`DEFAULT_TIMEOUT`), dict defaults, function kwargs, YAML/TOML keys, `.env` settings.
|
|
29
|
+
- π© **Renamed Flags & Config Keys:** CLI options (`--old-flag` $\to$ `--new-flag` in argparse, Click, Typer) and renamed configuration keys.
|
|
30
|
+
- βοΈ **Changed Function Parameters:** Added/removed parameters and modified default arguments via AST analysis.
|
|
31
|
+
- ποΈ **Removed Features:** Deleted functions or classes verified to no longer exist in the codebase.
|
|
32
|
+
3. **Smart Natural Language Normalization:** Bridges code and human writing (`DEFAULT_TIMEOUT` $\equiv$ `default_timeout` $\equiv$ `default timeout` $\equiv$ `default-timeout` $\equiv$ `defaultTimeout`).
|
|
33
|
+
4. **False-Alarm Mitigation:** For default value changes, it **only** flags when the **old value** (e.g. `30` or `30s`) actually appears in the documentation sentence.
|
|
34
|
+
5. **Exact Sentence Highlighting:** Extracts and pins the precise faulty sentence and table row.
|
|
35
|
+
6. **First-Class GitHub Actions Workflow:** Generates inline PR diff annotations (`::error`, `::warning`) and PR Step Summaries.
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
## π Project Structure
|
|
40
|
+
|
|
41
|
+
```text
|
|
42
|
+
doc-drift-detector/
|
|
43
|
+
βββ drift_detector.py # Main CLI entrypoint
|
|
44
|
+
βββ ai/
|
|
45
|
+
β βββ __init__.py # AI package exports
|
|
46
|
+
β βββ ollama_client.py # Zero-dependency local Ollama REST client
|
|
47
|
+
β βββ semantic_verifier.py # Layer 2: LLM reasoning verification
|
|
48
|
+
β βββ prose_rewriter.py # Layer 3: Natural prose rewriting
|
|
49
|
+
βββ detectors/
|
|
50
|
+
β βββ __init__.py # CodeChange data model & detector exports
|
|
51
|
+
β βββ default_value.py # Detects changed default values & constants
|
|
52
|
+
β βββ flag_rename.py # Detects renamed CLI flags & config keys
|
|
53
|
+
β βββ param_change.py # Detects changed function signatures & defaults
|
|
54
|
+
β βββ feature_remove.py # Detects deleted functions & classes
|
|
55
|
+
βββ utils/
|
|
56
|
+
β βββ __init__.py
|
|
57
|
+
β βββ diff_parser.py # Unified git diff parser with line tracking
|
|
58
|
+
β βββ doc_finder.py # Doc discovery, name normalizer, drift & auto-fix engine
|
|
59
|
+
β βββ reporter.py # Colored terminal, JSON & GitHub action reporter
|
|
60
|
+
βββ .github/workflows/
|
|
61
|
+
β βββ doc-drift.yml # Ready-to-use GitHub Action workflow
|
|
62
|
+
βββ requirements.txt # Dependencies
|
|
63
|
+
βββ README.md # Documentation
|
|
64
|
+
βββ tests/
|
|
65
|
+
βββ test_basic.py # Complete test suite (15 passing tests)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
---
|
|
69
|
+
|
|
70
|
+
## π Quickstart
|
|
71
|
+
|
|
72
|
+
### 1. Installation
|
|
73
|
+
|
|
74
|
+
DocDrift is built primarily using Python's standard library for zero-friction setup.
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
cd doc-drift-detector
|
|
78
|
+
pip install -r requirements.txt
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
### 2. Analyze a Pull Request Diff
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
# Compare base branch (main) with current branch (HEAD)
|
|
85
|
+
python drift_detector.py --base main --head HEAD
|
|
86
|
+
|
|
87
|
+
# Analyze an offline diff file
|
|
88
|
+
python drift_detector.py --diff-file my_pr.diff
|
|
89
|
+
|
|
90
|
+
# Output JSON report
|
|
91
|
+
python drift_detector.py --base main --head HEAD --json
|
|
92
|
+
|
|
93
|
+
# Save report to a file
|
|
94
|
+
python drift_detector.py --base main --head HEAD --output report.json
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## π CLI Options
|
|
100
|
+
|
|
101
|
+
| Option | Type | Default | Description |
|
|
102
|
+
| :--- | :--- | :--- | :--- |
|
|
103
|
+
| `--base` | `str` | `None` | Base branch or commit SHA (e.g., `main`, `origin/main`, `HEAD~1`) |
|
|
104
|
+
| `--head` | `str` | `None` | Head branch or commit SHA (e.g., `HEAD`) |
|
|
105
|
+
| `--diff-file` | `path` | `None` | Path to pre-generated `.diff` or `.patch` file |
|
|
106
|
+
| `--repo-dir` | `path` | `.` | Root directory of the repository to scan |
|
|
107
|
+
| `--docs-dir` | `path` | `.` | Directory containing documentation files (`.md`, `.rst`, `.adoc`, `.txt`) |
|
|
108
|
+
| `--format` | `choice` | `text` | Output format: `text`, `json`, `github`, or `markdown` |
|
|
109
|
+
| `--json` | `flag` | `False` | Shorthand for `--format json` |
|
|
110
|
+
| `--output` | `path` | `None` | File path to write the formatted report |
|
|
111
|
+
| `--ai` | `flag` | `False` | Enable local Ollama AI hybrid verification & prose rewriting |
|
|
112
|
+
| `--ai-model` | `str` | `auto` | Ollama model to use (e.g. `llama3.2:1b`, `llama3.2`, `mistral`) |
|
|
113
|
+
| `--fix`, `--autofix` | `flag` | `False` | Automatically rewrite outdated documentation files on disk |
|
|
114
|
+
| `--min-confidence` | `choice` | `medium`| Minimum issue confidence to report: `high`, `medium`, or `low` |
|
|
115
|
+
| `--fail-on` | `choice` | `high` | Exit with code `1` if drift of this level is found: `high`, `medium`, `any`, `none` |
|
|
116
|
+
| `-v`, `--verbose` | `flag` | `False` | Enable step-by-step pipeline logging |
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
## π How It Works (The 5-Step Pipeline)
|
|
121
|
+
|
|
122
|
+
```mermaid
|
|
123
|
+
flowchart TD
|
|
124
|
+
A["PR Git Diff<br>(git diff base...head)"] --> B["Step 1: Diff Parser<br>(Track +/- lines & code files)"]
|
|
125
|
+
B --> C["Step 2: Semantic Detectors<br>(Defaults, Flags, Params, Features)"]
|
|
126
|
+
C --> D["Step 3: Doc Finder<br>(Normalize names across .md, .rst, .adoc)"]
|
|
127
|
+
D --> E["Step 4: Smart Drift Engine<br>(Verify old value in text & score confidence)"]
|
|
128
|
+
E --> F["Step 5: Multi-Format Reporter<br>(CLI Boxes, JSON, GitHub Annotations)"]
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
### Step 1: Parse PR Diff (`utils/diff_parser.py`)
|
|
132
|
+
- Executes `git diff <base>...<head> --unified=0`.
|
|
133
|
+
- Isolates code files (`.py`, `.js`, `.ts`, `.go`, `.rs`, `.java`, `.yaml`, `.yml`, `.toml`, `.json`, `.env`, `.cfg`, `.ini`).
|
|
134
|
+
- Extracts exact removed (`-`) and added (`+`) lines with source line numbers.
|
|
135
|
+
|
|
136
|
+
### Step 2: Semantic Code Change Detection (`detectors/`)
|
|
137
|
+
- **Default Values (`default_value.py`):** Pairs removed `VAR = OLD` with added `VAR = NEW`.
|
|
138
|
+
- **Flag Renames (`flag_rename.py`):** Pairs removed `--old-flag` with added `--new-flag`, or renamed config keys with identical values.
|
|
139
|
+
- **Param Changes (`param_change.py`):** Uses Python AST / regex to detect signature modifications and modified default arguments.
|
|
140
|
+
- **Feature Removals (`feature_remove.py`):** Detects deleted functions or classes, verifying they don't exist elsewhere in the repo.
|
|
141
|
+
|
|
142
|
+
### Step 3: Documentation Search (`utils/doc_finder.py`)
|
|
143
|
+
- Scans `.md`, `.rst`, `.adoc`, and `.txt` files (excluding `.git`, `node_modules`, `venv`, etc.).
|
|
144
|
+
- Normalizes names into searchable variants (`DEFAULT_TIMEOUT` $\to$ `"default timeout"`, `default-timeout`, etc.).
|
|
145
|
+
- Extracts the exact sentence using punctuation boundaries and table row delimitation.
|
|
146
|
+
|
|
147
|
+
### Step 4: Smart Drift Decision Engine
|
|
148
|
+
- **DEFAULT_VALUE_CHANGED:** Flags **only** if the old value (e.g. `30` or `30s`) appears in the matched doc sentence. If the identifier is mentioned without the old value, it is skipped to prevent false alarms.
|
|
149
|
+
- **FLAG_RENAMED:** Flags if the old flag name appears in documentation.
|
|
150
|
+
- **PARAM_CHANGED:** Flags if removed parameter or old default is in documentation.
|
|
151
|
+
- **FEATURE_REMOVED:** Flags if documentation references the deleted feature.
|
|
152
|
+
|
|
153
|
+
#### Confidence Levels
|
|
154
|
+
- π΄ **HIGH (Merge Blocker):** Exact name match + old value confirmed in doc sentence.
|
|
155
|
+
- π‘ **MEDIUM (Warning):** Normalized match or function signature change.
|
|
156
|
+
- βͺ **LOW (Info):** Weak / partial match.
|
|
157
|
+
|
|
158
|
+
### Step 5: Multi-Channel Reporting (`utils/reporter.py`)
|
|
159
|
+
- **Human-Readable CLI:** Rich colored badges, exact file:line references, quotes, and suggested fixes.
|
|
160
|
+
- **JSON Schema:** Machine-readable output for dashboards and metrics.
|
|
161
|
+
- **GitHub PR Annotations:** Inline warnings directly in the GitHub PR review interface.
|
|
162
|
+
|
|
163
|
+
---
|
|
164
|
+
|
|
165
|
+
## π₯οΈ Example CLI Output
|
|
166
|
+
|
|
167
|
+
```text
|
|
168
|
+
======================================================================
|
|
169
|
+
DOCUMENTATION DRIFT DETECTOR β PR ANALYSIS
|
|
170
|
+
======================================================================
|
|
171
|
+
Code Changes Analyzed: 3 | Doc Files Scanned: 4
|
|
172
|
+
Drift Issues Found: 2 (2 HIGH, 0 MEDIUM, 0 LOW)
|
|
173
|
+
----------------------------------------------------------------------
|
|
174
|
+
|
|
175
|
+
Issue #1: DEFAULT VALUE DRIFT [HIGH - MERGE BLOCKER]
|
|
176
|
+
Doc Location : docs/configuration.md:42
|
|
177
|
+
Wrong Sentence: "The default timeout is 30 seconds before reconnecting."
|
|
178
|
+
Code Culprit : src/config.py:12 [DEFAULT_TIMEOUT] (30 -> 60)
|
|
179
|
+
Suggested Fix : Update default value from '30' to '60'
|
|
180
|
+
|
|
181
|
+
Issue #2: FLAG / CONFIG RENAME DRIFT [HIGH - MERGE BLOCKER]
|
|
182
|
+
Doc Location : docs/cli_reference.md:88
|
|
183
|
+
Wrong Sentence: "Run `worker --old-flag` to enable legacy processing mode."
|
|
184
|
+
Code Culprit : src/cli.py:54 [--old-flag] (--old-flag -> --new-flag)
|
|
185
|
+
Suggested Fix : Rename flag/key '--old-flag' to '--new-flag'
|
|
186
|
+
|
|
187
|
+
======================================================================
|
|
188
|
+
FAILED: 2 high-confidence drift issue(s) require documentation updates.
|
|
189
|
+
======================================================================
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
---
|
|
193
|
+
|
|
194
|
+
## π€ GitHub Actions Integration
|
|
195
|
+
|
|
196
|
+
Add `.github/workflows/doc-drift.yml` to your repository:
|
|
197
|
+
|
|
198
|
+
```yaml
|
|
199
|
+
name: Documentation Drift Detection
|
|
200
|
+
|
|
201
|
+
on:
|
|
202
|
+
pull_request:
|
|
203
|
+
branches: [ main, master ]
|
|
204
|
+
|
|
205
|
+
jobs:
|
|
206
|
+
check-drift:
|
|
207
|
+
runs-on: ubuntu-latest
|
|
208
|
+
steps:
|
|
209
|
+
- uses: actions/checkout@v4
|
|
210
|
+
with:
|
|
211
|
+
fetch-depth: 0
|
|
212
|
+
|
|
213
|
+
- uses: actions/setup-python@v5
|
|
214
|
+
with:
|
|
215
|
+
python-version: '3.11'
|
|
216
|
+
|
|
217
|
+
- name: Run DocDrift
|
|
218
|
+
run: |
|
|
219
|
+
python3 doc-drift-detector/drift_detector.py \
|
|
220
|
+
--base "${{ github.event.pull_request.base.sha }}" \
|
|
221
|
+
--head "${{ github.event.pull_request.head.sha }}" \
|
|
222
|
+
--format github
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Inline GitHub PR annotations will appear directly on the changed documentation files, blocking merge until documentation is updated!
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
## π§ͺ Running Tests
|
|
230
|
+
|
|
231
|
+
The test suite covers all units, detectors, normalization logic, and end-to-end integration:
|
|
232
|
+
|
|
233
|
+
```bash
|
|
234
|
+
python3 -m pytest tests/test_basic.py -v
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
Output:
|
|
238
|
+
```text
|
|
239
|
+
tests/test_basic.py::test_diff_parser_filters_code_files PASSED
|
|
240
|
+
tests/test_basic.py::test_diff_parser_tracks_lines PASSED
|
|
241
|
+
tests/test_basic.py::test_detect_default_value_python_constant PASSED
|
|
242
|
+
tests/test_basic.py::test_detect_default_value_yaml_json PASSED
|
|
243
|
+
tests/test_basic.py::test_detect_default_value_env PASSED
|
|
244
|
+
tests/test_basic.py::test_detect_flag_renames_cli PASSED
|
|
245
|
+
tests/test_basic.py::test_detect_flag_renames_config_key PASSED
|
|
246
|
+
tests/test_basic.py::test_detect_param_change_added_parameter PASSED
|
|
247
|
+
tests/test_basic.py::test_detect_param_change_modified_default PASSED
|
|
248
|
+
tests/test_basic.py::test_detect_feature_removal PASSED
|
|
249
|
+
tests/test_basic.py::test_name_variants_normalization PASSED
|
|
250
|
+
tests/test_basic.py::test_extract_exact_sentence PASSED
|
|
251
|
+
tests/test_basic.py::test_drift_default_value_smart_filter PASSED
|
|
252
|
+
tests/test_basic.py::test_end_to_end_pipeline PASSED
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
---
|
|
256
|
+
|
|
257
|
+
## π Hackathon Highlights
|
|
258
|
+
|
|
259
|
+
- **Zero-Dependency Core:** Runs everywhere without heavy ML or external network API calls.
|
|
260
|
+
- **Precision First:** Smart filtering eliminates false alarms by validating value occurrences in context.
|
|
261
|
+
- **Universal Code Support:** Works across Python, JS/TS, Go, Rust, Java, YAML, TOML, JSON, and ENV files.
|
|
262
|
+
- **Developer Experience:** Clear, colored terminal UI and automated GitHub Action PR annotations.
|
|
263
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ai
|
|
3
|
+
==
|
|
4
|
+
AI-Powered Semantic Verification and Prose Rewriting module.
|
|
5
|
+
Integrates with local Ollama daemon (e.g. Llama 3.2, Mistral).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .ollama_client import OllamaClient
|
|
9
|
+
from .semantic_verifier import SemanticVerifier, VerificationResult
|
|
10
|
+
from .prose_rewriter import ProseRewriter
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"OllamaClient",
|
|
14
|
+
"SemanticVerifier",
|
|
15
|
+
"VerificationResult",
|
|
16
|
+
"ProseRewriter",
|
|
17
|
+
]
|
|
18
|
+
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""
|
|
2
|
+
ai/ollama_client.py
|
|
3
|
+
===================
|
|
4
|
+
Lightweight, zero-dependency REST client for local Ollama instance.
|
|
5
|
+
Communicates directly with http://localhost:11434 (or OLLAMA_HOST).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import urllib.error
|
|
11
|
+
import urllib.request
|
|
12
|
+
from typing import Any, Dict, List, Optional
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
DEFAULT_OLLAMA_HOST = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
|
|
16
|
+
PREFERRED_MODELS = [
|
|
17
|
+
"llama3.2", "llama3.2:1b", "llama3.2:3b", "llama3.1",
|
|
18
|
+
"llama3", "mistral", "qwen2.5", "qwen2.5-coder", "phi3"
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class OllamaClient:
|
|
23
|
+
"""Zero-dependency HTTP client for interacting with a local Ollama daemon."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, host: str = DEFAULT_OLLAMA_HOST, timeout: float = 15.0):
|
|
26
|
+
self.host = host.rstrip("/")
|
|
27
|
+
self.timeout = timeout
|
|
28
|
+
self._cached_available: Optional[bool] = None
|
|
29
|
+
self._cached_models: Optional[List[str]] = None
|
|
30
|
+
|
|
31
|
+
def is_available(self) -> bool:
|
|
32
|
+
"""Checks if the local Ollama server is running and responsive."""
|
|
33
|
+
try:
|
|
34
|
+
req = urllib.request.Request(
|
|
35
|
+
f"{self.host}/api/tags",
|
|
36
|
+
headers={"User-Agent": "DocDrift/1.0"}
|
|
37
|
+
)
|
|
38
|
+
with urllib.request.urlopen(req, timeout=2.0) as resp:
|
|
39
|
+
if resp.status == 200:
|
|
40
|
+
self._cached_available = True
|
|
41
|
+
return True
|
|
42
|
+
except Exception:
|
|
43
|
+
pass
|
|
44
|
+
self._cached_available = False
|
|
45
|
+
return False
|
|
46
|
+
|
|
47
|
+
def list_models(self) -> List[str]:
|
|
48
|
+
"""Returns list of model names downloaded in local Ollama."""
|
|
49
|
+
if not self.is_available():
|
|
50
|
+
return []
|
|
51
|
+
try:
|
|
52
|
+
req = urllib.request.Request(f"{self.host}/api/tags")
|
|
53
|
+
with urllib.request.urlopen(req, timeout=3.0) as resp:
|
|
54
|
+
data = json.loads(resp.read().decode("utf-8"))
|
|
55
|
+
models = [m.get("name", "") for m in data.get("models", [])]
|
|
56
|
+
self._cached_models = models
|
|
57
|
+
return models
|
|
58
|
+
except Exception:
|
|
59
|
+
return []
|
|
60
|
+
|
|
61
|
+
def get_preferred_model(self) -> Optional[str]:
|
|
62
|
+
"""
|
|
63
|
+
Determines the best available model on the user's Ollama instance.
|
|
64
|
+
Prioritizes fast models: llama3.2 -> llama3 -> mistral -> first available.
|
|
65
|
+
"""
|
|
66
|
+
installed = self.list_models()
|
|
67
|
+
if not installed:
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
# Check preferred list
|
|
71
|
+
for preferred in PREFERRED_MODELS:
|
|
72
|
+
for inst in installed:
|
|
73
|
+
if inst == preferred or inst.startswith(f"{preferred}:"):
|
|
74
|
+
return inst
|
|
75
|
+
|
|
76
|
+
# Return first installed model as fallback
|
|
77
|
+
return installed[0]
|
|
78
|
+
|
|
79
|
+
def chat(
|
|
80
|
+
self,
|
|
81
|
+
messages: List[Dict[str, str]],
|
|
82
|
+
model: Optional[str] = None,
|
|
83
|
+
json_format: bool = False,
|
|
84
|
+
temperature: float = 0.1
|
|
85
|
+
) -> Optional[str]:
|
|
86
|
+
"""
|
|
87
|
+
Executes a chat completion request to Ollama /api/chat.
|
|
88
|
+
"""
|
|
89
|
+
active_model = model or self.get_preferred_model()
|
|
90
|
+
if not active_model:
|
|
91
|
+
return None
|
|
92
|
+
|
|
93
|
+
payload: Dict[str, Any] = {
|
|
94
|
+
"model": active_model,
|
|
95
|
+
"messages": messages,
|
|
96
|
+
"stream": False,
|
|
97
|
+
"options": {
|
|
98
|
+
"temperature": temperature,
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if json_format:
|
|
102
|
+
payload["format"] = "json"
|
|
103
|
+
|
|
104
|
+
try:
|
|
105
|
+
req_data = json.dumps(payload).encode("utf-8")
|
|
106
|
+
req = urllib.request.Request(
|
|
107
|
+
f"{self.host}/api/chat",
|
|
108
|
+
data=req_data,
|
|
109
|
+
headers={"Content-Type": "application/json"}
|
|
110
|
+
)
|
|
111
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as resp:
|
|
112
|
+
result = json.loads(resp.read().decode("utf-8"))
|
|
113
|
+
message = result.get("message", {})
|
|
114
|
+
return message.get("content", "").strip()
|
|
115
|
+
except Exception:
|
|
116
|
+
return None
|
|
117
|
+
|