flagcheck 0.2.0a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,27 @@
1
+ name: CI
2
+ on: [push, pull_request]
3
+ permissions:
4
+ contents: read
5
+ jobs:
6
+ tests:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ os: [ubuntu-latest]
12
+ python-version: ['3.10', '3.11', '3.12', '3.13']
13
+ include:
14
+ - os: windows-latest
15
+ python-version: '3.12'
16
+ - os: macos-latest
17
+ python-version: '3.12'
18
+ steps:
19
+ - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
20
+ with:
21
+ persist-credentials: false
22
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
23
+ with:
24
+ python-version: ${{ matrix.python-version }}
25
+ - run: python -m pip install .
26
+ - run: python -m unittest discover -s tests -v
27
+ - run: flagcheck --version
@@ -0,0 +1,70 @@
1
+ name: Prepare or publish approved alpha
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ inputs:
6
+ registry:
7
+ description: Target registry (no upload until its environment is approved)
8
+ type: choice
9
+ options: [testpypi, pypi]
10
+ default: testpypi
11
+ tag:
12
+ description: Exact approved release tag, e.g. v0.2.0a1
13
+ type: string
14
+ required: true
15
+ publish:
16
+ description: Explicitly authorize uploading the tested artifacts
17
+ type: boolean
18
+ default: false
19
+
20
+ permissions:
21
+ contents: read
22
+
23
+ jobs:
24
+ build:
25
+ if: startsWith(github.ref, 'refs/tags/v') && inputs.tag == github.ref_name
26
+ runs-on: ubuntu-latest
27
+ steps:
28
+ - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
29
+ with:
30
+ persist-credentials: false
31
+ - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
32
+ with:
33
+ python-version: '3.12'
34
+ - name: Verify tag and version
35
+ env:
36
+ RELEASE_TAG: ${{ inputs.tag }}
37
+ run: |
38
+ python - <<'PY'
39
+ import os, pathlib, tomllib
40
+ project = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']
41
+ assert os.environ['RELEASE_TAG'] == 'v' + project['version']
42
+ PY
43
+ - run: python -m pip install build twine .
44
+ - run: python -m unittest discover -s tests -v
45
+ - run: python -m build
46
+ - run: python -m twine check --strict dist/*
47
+ - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
48
+ with:
49
+ name: python-distributions
50
+ path: dist/
51
+ if-no-files-found: error
52
+
53
+ publish:
54
+ if: inputs.publish == true
55
+ needs: build
56
+ runs-on: ubuntu-latest
57
+ environment:
58
+ name: ${{ inputs.registry }}
59
+ url: ${{ inputs.registry == 'testpypi' && 'https://test.pypi.org/project/flagcheck/' || 'https://pypi.org/project/flagcheck/' }}
60
+ permissions:
61
+ id-token: write
62
+ contents: read
63
+ steps:
64
+ - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6
65
+ with:
66
+ name: python-distributions
67
+ path: dist/
68
+ - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1, pinned
69
+ with:
70
+ repository-url: ${{ inputs.registry == 'testpypi' && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }}
@@ -0,0 +1,7 @@
1
+ - id: flagcheck
2
+ name: flagcheck
3
+ description: Check long options against a reviewed schema
4
+ entry: flagcheck scan
5
+ language: python
6
+ files: '(^|/)([Mm]akefile|GNUmakefile)$|\.mk$|\.(sh|bash|zsh|md|markdown)$|(^|/)\.github/workflows/.*\.ya?ml$'
7
+ require_serial: true
@@ -0,0 +1,12 @@
1
+ # Changelog
2
+
3
+ ## 0.2.0a1 — first public alpha candidate (unreleased)
4
+
5
+ - Static long-option validation with an explicit, reviewed JSON command schema.
6
+ - Non-executing extraction of literal argparse declarations, implicit help and reviewed custom action arities.
7
+ - Shell, Makefile, Markdown examples and GitHub Actions run-step support; limited literal project runner forms.
8
+ - `--summary` exposes partial coverage counters on stderr; `--require-match` detects a scan in which no configured invocation was recognized.
9
+ - JSON output, configurable warning threshold, Python packaging and a pre-commit hook.
10
+ - Reproducible, pinned-source benchmark for xml2rfc and three unrelated CLI projects. Results are bounded checks, not claims of universal accuracy.
11
+
12
+ Known limitations are listed in the README. No automated fixes, network requests, execution of scanned commands or arbitrary CLI semantics are claimed.
@@ -0,0 +1,14 @@
1
+ # Contributing
2
+
3
+ Flagcheck is an alpha project focused on catching drift between command-line option declarations and literal invocations in source-controlled files. Real reproductions are especially helpful.
4
+
5
+ For a bug, include a minimal command, the reviewed schema, expected output, actual output and Flagcheck/Python versions. Redact credentials and private filenames. Please discuss broad features or rewrites before starting them.
6
+
7
+ For a fix:
8
+
9
+ 1. Add a regression test that fails before the change.
10
+ 2. Keep the change focused; run `python -m unittest discover -s tests -v` after installing the package.
11
+ 3. Explain limitations and manual verification. Avoid speculative or mass-generated changes that add review burden.
12
+ 4. AI-assisted contributions are welcome when the human contributor understands, verifies and takes responsibility for the whole change. Disclose material AI assistance. Do not submit unattended agent PRs or comments.
13
+
14
+ Treat other contributors respectfully. Good-faith questions and useful reproductions matter more than contribution counts.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Parth Agrawal (aparth9100)
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND MERCHANTABILITY AND FITNESS FOR A
18
+ PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
19
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
20
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
21
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,7 @@
1
+ include README.md LICENSE CHANGELOG.md CONTRIBUTING.md SECURITY.md
2
+ include VALIDATION.md PHASE2_VALIDATION.md
3
+ include .pre-commit-hooks.yaml
4
+ recursive-include examples *.json *.py *.sh
5
+ recursive-include tests *.py
6
+ recursive-include benchmarks *.py
7
+ recursive-include .github/workflows *.yml
@@ -0,0 +1,61 @@
1
+ # Flagcheck 0.2.0 — Phase 2 validation (22 November 2026)
2
+
3
+ ## Executive result
4
+
5
+ Three unrelated open-source argparse CLIs were evaluated. Flagcheck discovered weaknesses in its own extractor and scanner, corrected those weaknesses, and preserved the original xml2rfc result. No upstream change, repository creation, publication, download count or adoption is claimed.
6
+
7
+ | Project | Reviewed flag spellings | Long options | Unchanged files | Deliberately modified real option tokens detected |
8
+ | --- | ---: | ---: | ---: | ---: |
9
+ | PyPA Build | 27 | 17 | 8 | 1/1 |
10
+ | pip-audit | 36 | 26 | 7 | 11/11 |
11
+ | Microsoft MarkItDown | 28 | 19 | 3 | 12/12 |
12
+ | Total | 91 | 62 | 18 | 24/24 |
13
+
14
+ No findings on the 18 unmodified files. The 24 manually enumerated exact long-option occurrences in supported README contexts were individually replaced with an invented invalid spelling; all were reported on the expected line. These were controlled local mutations, not actual upstream bugs. The remainder of each file was unchanged for every mutation. This only establishes coverage for those 24 reviewed occurrences; unsupported shell constructs, commands and unenumerated examples may be skipped.
15
+
16
+ ## Changes made after Phase 1
17
+
18
+ - Fixed a false warning on pip-audit's legitimate `--help`: direct argparse constructors implicitly add help unless disabled. Manual confirmation is supported for indirect constructors.
19
+ - PyPA Build's custom `NegativeCountAction` could not be extracted automatically. Added an explicit, reviewer-provided arity mapping; no custom code is executed. Verified its `nargs=0` in the pinned source.
20
+ - Handle simple optional option arguments, known short-option clusters and attached values; do not lint unknown short flags.
21
+ - Recognize literal `pipenv run`, `uv run` and `uvx --from PACKAGE COMMAND` forms; arbitrary wrapper arguments are not parsed.
22
+ - Fixed a path filter that mistakenly skipped explicitly requested files located below a directory named `build`.
23
+ - Added 12 regression tests (52 total, versus 40 in Phase 1).
24
+
25
+ ## Option spelling oracle
26
+
27
+ The three inventories were manually transcribed from their parser declarations, including implicit help and the reviewed Build custom action. The extraction result was asserted equal to each inventory, including option arity. Separately, standard Python `argparse` parsers with these known long option names resolved exact names, all proper long-name prefixes, and invented unknown names. Those parsers deliberately model name-resolution only, using boolean actions to isolate spelling behavior; they do not model actual package side effects, value validation or mutual constraints. Each case was evaluated in shell, Makefile, Markdown, and workflow contexts, including both Build invocation aliases.
28
+
29
+ | Expected class | Cases | Correct |
30
+ | --- | ---: | ---: |
31
+ | Exact / clean | 316 | 316 |
32
+ | Unique abbreviation | 2,188 | 2,188 |
33
+ | Ambiguous abbreviation | 260 | 260 |
34
+ | Unknown | 316 | 316 |
35
+ | Total | 3,080 | 3,080 |
36
+
37
+ Observed false positives: 0 among 316 clean synthetic cases. Observed missed findings: 0 among 2,764 positive synthetic cases. These are systematic, generated test cases with known labels, **not 3,080 independently discovered real bugs or a population-level accuracy estimate**. Phase 2 files were also used to improve the implementation, so the result is not a held-out benchmark. The original Phase 1 regression remained 65 before / 0 after.
38
+
39
+ ## Reproducibility
40
+
41
+ Working copies of upstream inputs are not redistributed. Obtain the named files from the following exact commits; put the source in `cli.py` and the root README in `README.md` under each local dataset subdirectory. Preserve `.github/workflows/` paths. No upstream code is imported, installed, run or modified by the benchmark.
42
+
43
+ | Dataset directory | Pinned commit | Parser path |
44
+ | --- | --- | --- |
45
+ | `build` | [pypa/build@c0b4f18](https://github.com/pypa/build/tree/c0b4f18acc16971d9856a273cd1ddd2233de9774) | `src/build/__main__.py` |
46
+ | `audit` | [pypa/pip-audit@864264b](https://github.com/pypa/pip-audit/tree/864264baeb9a9e30115a0f2838fe4769cf5a0840) | `pip_audit/_cli.py` |
47
+ | `markitdown` | [microsoft/markitdown@ed17895](https://github.com/microsoft/markitdown/tree/ed178952a6672abdfd35b5f9e76a80b6cefc086e) | `packages/markitdown/src/markitdown/__main__.py` |
48
+
49
+ Workflow filenames are listed in `benchmarks/independent.py` and form an explicit 15-file manifest (7 Build, 6 pip-audit, 2 MarkItDown). Three README files complete the 18 natural scan inputs. Run from the Flagcheck package root after installation:
50
+
51
+ ```sh
52
+ python benchmarks/independent.py /path/to/dataset
53
+ ```
54
+
55
+ The benchmark asserts completeness of the manual option inventories, all expected inputs, exact mutation locations, natural findings, and generated classifications. A mismatch exits nonzero. For the earlier regression, use `benchmarks/xml2rfc.py` as documented in `VALIDATION.md`.
56
+
57
+ ## Release gate
58
+
59
+ Verified locally on Linux, Python 3.12.14, PyYAML 6.0.3: the 0.2.0 wheel built offline using the existing build dependencies, installed into a separate virtual environment with system site packages providing PyYAML, reported version `0.2.0`, passed all 52 unit tests, and reproduced both benchmarks from the installed package. CI and the pre-commit hook are configured but have not been run on GitHub; Python 3.10, 3.11 and 3.13 were not exercised in this environment.
60
+
61
+ The project is a credible alpha, not a production-ready universal shell linter. Before publishing, review existing competing tools and distribution-name availability; validate additional held-out projects, especially those with complex commands; check support for Python versions declared in CI; ask maintainers or developers for real usability feedback. General OSS program eligibility is not established by these local results alone.
@@ -0,0 +1,107 @@
1
+ Metadata-Version: 2.4
2
+ Name: flagcheck
3
+ Version: 0.2.0a1
4
+ Summary: Conservative, schema-driven command-line flag linter
5
+ Author: Parth Agrawal
6
+ License-Expression: MIT
7
+ Project-URL: Repository, https://github.com/aparth9100/flagcheck
8
+ Project-URL: Issues, https://github.com/aparth9100/flagcheck/issues
9
+ Project-URL: Changelog, https://github.com/aparth9100/flagcheck/blob/main/CHANGELOG.md
10
+ Keywords: argparse,cli,lint,makefile,pre-commit
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Software Development :: Quality Assurance
20
+ Requires-Python: >=3.10
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: PyYAML>=6
24
+ Dynamic: license-file
25
+
26
+ # Flagcheck — CLI option drift linter
27
+
28
+ **0.2.0a1 — experimental alpha.**
29
+
30
+ Flagcheck detects brittle long-option abbreviations, unknown long options, and explicitly marked deprecated options in static shell commands. It checks against a schema you review, not against a guessed universal list of CLI flags.
31
+
32
+ It is intended for maintainers who control a CLI and want their own Makefile, CI and documentation examples to use stable option names. The scanner never runs a scanned command, imports a target package or makes network requests.
33
+
34
+ ## Quick start
35
+
36
+ Python 3.10+ and PyYAML are required. Install from this source directory:
37
+
38
+ ```sh
39
+ python -m pip install .
40
+ ```
41
+
42
+ After the alpha is available on PyPI, the equivalent registry installation is `python -m pip install --pre flagcheck`.
43
+
44
+ Use the included example schema:
45
+
46
+ ```sh
47
+ flagcheck scan --schema examples/schema.json --cmd 'mytool --verb --output result.txt'
48
+ ```
49
+
50
+ Expected output: a warning suggesting `--verbose`. The command exits with code 1 because the default threshold includes warnings.
51
+
52
+ ```sh
53
+ flagcheck scan --schema examples/schema.json --format json --summary --require-match examples/sample.sh
54
+ ```
55
+
56
+ Exit codes: 0 = no finding at the selected severity; 1 = finding; 2 = input/configuration error. `--fail-on error` permits warnings without failing CI. An exit code of 0 is **not** proof that all commands are valid.
57
+
58
+ `--summary` writes JSON counters to stderr, keeping findings-only JSON on stdout intact. It reports supported logical commands, recognized invocations, checked long options, lexical failures, unmatched segments and invocations whose scanning stopped early. It is partial coverage, not a measure of all unsupported syntax. `--require-match` returns status 2 when no configured command was recognized; use it on a specific file or path expected to invoke your CLI.
59
+
60
+ ## Generate a schema from source
61
+
62
+ ```sh
63
+ flagcheck schema-from-source examples/cli_example.py --command mytool --receiver parser --complete
64
+ ```
65
+
66
+ The command emits JSON to stdout. Save it into a schema file after inspecting it. Repeat `--receiver` for explicitly identified argument groups. The AST extractor recognizes literal `add_argument` declarations, aliases, common actions, value counts and literal deprecation metadata. It never imports the Python file. It recognizes default `-h`/`--help` in a direct `argparse.ArgumentParser` constructor unless disabled; use `--implicit-help` only after verifying an indirect constructor. For a custom action whose arity you have reviewed, specify, for example, `--action-arity NegativeCountAction=0`. For parsers with abbreviation disabled, pass `--no-abbrev`. It cannot infer dynamically constructed arguments, inherited parsers, plugin options or arbitrary runtime behavior. It combines conditional declarations. Only use `--complete` when you have independently verified the parser's complete option inventory; omissions could cause incorrect reports.
67
+
68
+ An example schema:
69
+
70
+ ```json
71
+ {
72
+ "mytool": {
73
+ "complete": true,
74
+ "allow_abbrev": true,
75
+ "flags": {
76
+ "--verbose": {"nargs": 0},
77
+ "--output": {"nargs": 1},
78
+ "--legacy": {"nargs": 0, "deprecated": true, "replacement": "--verbose"}
79
+ }
80
+ }
81
+ }
82
+ ```
83
+
84
+ Each command can also specify `subcommands`, a mapping of subcommand names to command-schema objects. A subcommand has its own options, with no implicit inheritance. `complete` defaults to false: incomplete schemas only report explicitly known deprecated flags and stop at unrecognized options. `allow_abbrev` controls whether a unique prefix is a warning or an invalid-option error. `nargs` supports nonnegative integer counts and `?`, `*`, `+`, `...`; optional values are consumed when statically recognizable, and the scanner stops after variadic options rather than guessing their extent.
85
+
86
+ ## Supported input
87
+
88
+ - Shell scripts (`.sh`, `.bash`, `.zsh`): simple literal command lines, common wrappers, chains, pipelines and backslash continuations. Limited literal `pipenv run`, `uv run` and `uvx [--from PACKAGE] COMMAND` forms are recognized.
89
+ - Makefiles and `.mk`: tab-prefixed recipes, simple loops, continued lines, literal `bash -c`/`-cx` wrappers, automatic filename variables.
90
+ - GitHub Actions workflows: `jobs.*.steps.*.run` scalar values for explicit `bash`/`sh` or default Ubuntu/macOS runners; literal multiline blocks.
91
+ - Markdown: explicitly shell-tagged fenced snippets; console fences with `$ ` prompts.
92
+
93
+ The scanner intentionally skips unsupported formats, unrecognized commands and ambiguous dynamic arguments. It is not a shell parser, a CLI semantic validator, or a substitute for actual CI. It consumes recognized argparse-style short-option clusters and attached values to avoid false long-option matches, but does not lint short option spellings. It does not support custom Make recipe prefixes, arbitrary Make `$(shell ...)` expansions, folded YAML blocks, PowerShell, fish, shell functions, aliases, dynamically constructed commands or all possible nested shell syntax. A finding's line refers to the beginning of the logical command; it can be the first line of a continuation. Treat results as review prompts, not automatic edits.
94
+
95
+ ## Testing
96
+
97
+ ```sh
98
+ python -m unittest discover -s tests -v
99
+ ```
100
+
101
+ The test suite uses the standard library `unittest`. The included CI workflow targets Python 3.10–3.13 but has not been run on GitHub. The pre-commit hook is configured but is not publicly installable yet. When a repository and release tag exist, configure the hook with `args: [--schema, path/to/schema.json]`. For now you can invoke the CLI directly or use a local pre-commit hook passing that schema.
102
+
103
+ ## Evidence and next milestone
104
+
105
+ See `VALIDATION.md` for the original xml2rfc benchmark and `PHASE2_VALIDATION.md` for validation against PyPA Build, pip-audit and Microsoft MarkItDown. The Phase 2 measurements cover reviewed literal option inventories, original files, deliberate mutations and a Python argparse option-resolution oracle. They are not a general-accuracy claim or independent external adoption evidence. The next milestone is review of a release candidate, check for existing similar tools and distribution-name availability, and decide whether to release. The distribution name is provisional; registry availability is not verified.
106
+
107
+ MIT licensed; source, docs and tests prepared with AI assistance for human review. No publication or external adoption is claimed.
@@ -0,0 +1,82 @@
1
+ # Flagcheck — CLI option drift linter
2
+
3
+ **0.2.0a1 — experimental alpha.**
4
+
5
+ Flagcheck detects brittle long-option abbreviations, unknown long options, and explicitly marked deprecated options in static shell commands. It checks against a schema you review, not against a guessed universal list of CLI flags.
6
+
7
+ It is intended for maintainers who control a CLI and want their own Makefile, CI and documentation examples to use stable option names. The scanner never runs a scanned command, imports a target package or makes network requests.
8
+
9
+ ## Quick start
10
+
11
+ Python 3.10+ and PyYAML are required. Install from this source directory:
12
+
13
+ ```sh
14
+ python -m pip install .
15
+ ```
16
+
17
+ After the alpha is available on PyPI, the equivalent registry installation is `python -m pip install --pre flagcheck`.
18
+
19
+ Use the included example schema:
20
+
21
+ ```sh
22
+ flagcheck scan --schema examples/schema.json --cmd 'mytool --verb --output result.txt'
23
+ ```
24
+
25
+ Expected output: a warning suggesting `--verbose`. The command exits with code 1 because the default threshold includes warnings.
26
+
27
+ ```sh
28
+ flagcheck scan --schema examples/schema.json --format json --summary --require-match examples/sample.sh
29
+ ```
30
+
31
+ Exit codes: 0 = no finding at the selected severity; 1 = finding; 2 = input/configuration error. `--fail-on error` permits warnings without failing CI. An exit code of 0 is **not** proof that all commands are valid.
32
+
33
+ `--summary` writes JSON counters to stderr, keeping findings-only JSON on stdout intact. It reports supported logical commands, recognized invocations, checked long options, lexical failures, unmatched segments and invocations whose scanning stopped early. It is partial coverage, not a measure of all unsupported syntax. `--require-match` returns status 2 when no configured command was recognized; use it on a specific file or path expected to invoke your CLI.
34
+
35
+ ## Generate a schema from source
36
+
37
+ ```sh
38
+ flagcheck schema-from-source examples/cli_example.py --command mytool --receiver parser --complete
39
+ ```
40
+
41
+ The command emits JSON to stdout. Save it into a schema file after inspecting it. Repeat `--receiver` for explicitly identified argument groups. The AST extractor recognizes literal `add_argument` declarations, aliases, common actions, value counts and literal deprecation metadata. It never imports the Python file. It recognizes default `-h`/`--help` in a direct `argparse.ArgumentParser` constructor unless disabled; use `--implicit-help` only after verifying an indirect constructor. For a custom action whose arity you have reviewed, specify, for example, `--action-arity NegativeCountAction=0`. For parsers with abbreviation disabled, pass `--no-abbrev`. It cannot infer dynamically constructed arguments, inherited parsers, plugin options or arbitrary runtime behavior. It combines conditional declarations. Only use `--complete` when you have independently verified the parser's complete option inventory; omissions could cause incorrect reports.
42
+
43
+ An example schema:
44
+
45
+ ```json
46
+ {
47
+ "mytool": {
48
+ "complete": true,
49
+ "allow_abbrev": true,
50
+ "flags": {
51
+ "--verbose": {"nargs": 0},
52
+ "--output": {"nargs": 1},
53
+ "--legacy": {"nargs": 0, "deprecated": true, "replacement": "--verbose"}
54
+ }
55
+ }
56
+ }
57
+ ```
58
+
59
+ Each command can also specify `subcommands`, a mapping of subcommand names to command-schema objects. A subcommand has its own options, with no implicit inheritance. `complete` defaults to false: incomplete schemas only report explicitly known deprecated flags and stop at unrecognized options. `allow_abbrev` controls whether a unique prefix is a warning or an invalid-option error. `nargs` supports nonnegative integer counts and `?`, `*`, `+`, `...`; optional values are consumed when statically recognizable, and the scanner stops after variadic options rather than guessing their extent.
60
+
61
+ ## Supported input
62
+
63
+ - Shell scripts (`.sh`, `.bash`, `.zsh`): simple literal command lines, common wrappers, chains, pipelines and backslash continuations. Limited literal `pipenv run`, `uv run` and `uvx [--from PACKAGE] COMMAND` forms are recognized.
64
+ - Makefiles and `.mk`: tab-prefixed recipes, simple loops, continued lines, literal `bash -c`/`-cx` wrappers, automatic filename variables.
65
+ - GitHub Actions workflows: `jobs.*.steps.*.run` scalar values for explicit `bash`/`sh` or default Ubuntu/macOS runners; literal multiline blocks.
66
+ - Markdown: explicitly shell-tagged fenced snippets; console fences with `$ ` prompts.
67
+
68
+ The scanner intentionally skips unsupported formats, unrecognized commands and ambiguous dynamic arguments. It is not a shell parser, a CLI semantic validator, or a substitute for actual CI. It consumes recognized argparse-style short-option clusters and attached values to avoid false long-option matches, but does not lint short option spellings. It does not support custom Make recipe prefixes, arbitrary Make `$(shell ...)` expansions, folded YAML blocks, PowerShell, fish, shell functions, aliases, dynamically constructed commands or all possible nested shell syntax. A finding's line refers to the beginning of the logical command; it can be the first line of a continuation. Treat results as review prompts, not automatic edits.
69
+
70
+ ## Testing
71
+
72
+ ```sh
73
+ python -m unittest discover -s tests -v
74
+ ```
75
+
76
+ The test suite uses the standard library `unittest`. The included CI workflow targets Python 3.10–3.13 but has not been run on GitHub. The pre-commit hook is configured but is not publicly installable yet. When a repository and release tag exist, configure the hook with `args: [--schema, path/to/schema.json]`. For now you can invoke the CLI directly or use a local pre-commit hook passing that schema.
77
+
78
+ ## Evidence and next milestone
79
+
80
+ See `VALIDATION.md` for the original xml2rfc benchmark and `PHASE2_VALIDATION.md` for validation against PyPA Build, pip-audit and Microsoft MarkItDown. The Phase 2 measurements cover reviewed literal option inventories, original files, deliberate mutations and a Python argparse option-resolution oracle. They are not a general-accuracy claim or independent external adoption evidence. The next milestone is review of a release candidate, check for existing similar tools and distribution-name availability, and decide whether to release. The distribution name is provisional; registry availability is not verified.
81
+
82
+ MIT licensed; source, docs and tests prepared with AI assistance for human review. No publication or external adoption is claimed.
@@ -0,0 +1,7 @@
1
+ # Security
2
+
3
+ Flagcheck does not intentionally execute or upload scanned commands. It reads files locally and uses a safe YAML loader for workflow syntax. It is not a sandbox, a complete shell parser or a general security scanner. It should not be used as proof that a script is safe to run.
4
+
5
+ Never share authentication material or confidential source in a public issue or benchmark. For a potential security vulnerability, use the repository's private vulnerability reporting feature once it is enabled. If private reporting is unavailable, wait for a private contact channel rather than posting exploitable details publicly.
6
+
7
+ Only the latest alpha release is intended to receive fixes. No security support SLA is promised.
@@ -0,0 +1,39 @@
1
+ # Phase 1 validation — 22 November 2026
2
+
3
+ ## Local execution results
4
+
5
+ - Runtime: Linux, CPython 3.12.14, PyYAML 6.0.3.
6
+ - `PYTHONPATH=src python -m unittest discover -s tests -v`: 40 tests passed.
7
+ - `python -m pip wheel --no-deps --no-build-isolation --no-index ... .`: wheel built successfully (`flagcheck-0.1.0-py3-none-any.whl`).
8
+ - Installed the wheel into a separate virtual environment with the preinstalled PyYAML dependency available through system site packages, without downloading packages. All 40 tests passed again from the installed package.
9
+ - Installed CLI smoke test: `flagcheck --version` returned `0.1.0`; source extraction emitted the expected schema; a deliberately abbreviated option produced one warning and status 1.
10
+ - GitHub Actions and pre-commit are configured, not executed. No claim of Python 3.10, 3.11 or 3.13 runtime verification is made.
11
+
12
+ ## Reproducible targeted benchmark
13
+
14
+ The test uses exact source files retrieved from GitHub, not synthetic clones or guessed flags.
15
+
16
+ - Base: [ietf-tools/xml2rfc at 8f07f9e](https://github.com/ietf-tools/xml2rfc/tree/8f07f9ec5876bb0fd90f7f686238ed201e52e52f).
17
+ - Reviewed patch: [PR #1363](https://github.com/ietf-tools/xml2rfc/pull/1363), [fork commit b7e56f4](https://github.com/aparth9100/xml2rfc/commit/b7e56f49b2b8bd76745490871cc413a2693fc2e4).
18
+ - Inputs: base `xml2rfc/run.py`; base and patched `Makefile` and `.github/workflows/pdfa-verification.yml`.
19
+ - 98 literal flag spellings extracted from 11 selected parser/argument group receivers, with zero extractor notes. The complete option list was assumed for this pinned benchmark after comparing the literal declarations.
20
+
21
+ | File | Original findings | Patched findings |
22
+ | --- | ---: | ---: |
23
+ | Makefile | 64 | 0 |
24
+ | PDF/A workflow | 1 | 0 |
25
+ | Total | 65 | 0 |
26
+
27
+ All 65 findings were the seven abbreviations addressed by the PR: `--skip-config` (39), `--base` (7), `--prep` (6), `--no-inline-version` (6), `--exp` (5), `--doc` (1), `--man` (1). No other findings were observed in the supported scan contexts. The three commented examples are intentionally skipped; Make's `$(shell ...)` expressions are not covered. This result does not establish general precision or recall; the original benchmark informed the implementation.
28
+
29
+ To reproduce locally after separately obtaining the pinned files (they are not redistributed in this bundle):
30
+
31
+ ```sh
32
+ python benchmarks/xml2rfc.py run.py before/Makefile before/pdfa-verification.yml after/Makefile after/pdfa-verification.yml
33
+ ```
34
+
35
+ The script asserts the exact finding counts and absence of findings in the patched files. It never executes upstream files.
36
+
37
+ ## Next stage
38
+
39
+ Validate at least two unrelated projects before release. Pin the commits and tool versions, build complete or explicitly partial schemas, compare findings with actual parser semantics, manually classify false positives and missed issues, and publish the reproducible benchmark methodology. Check packaging-name availability and policies before publication. No external repository, issue, PR or package was created during Phase 1.
@@ -0,0 +1,164 @@
1
+ """Phase 2 benchmark against pinned, separately downloaded source files.
2
+
3
+ The reviewed inventories below come from the actual parser declarations, not
4
+ from Flagcheck's extractor. No upstream code is imported or executed. The
5
+ oracle uses Python argparse to test option-spelling resolution only.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ import contextlib
11
+ import io
12
+ from collections import Counter
13
+ from pathlib import Path
14
+
15
+ from flagcheck.schema import _command, extract_argparse
16
+ from flagcheck.scanner import scan_text
17
+
18
+
19
+ def inventory(zero: str, one: str, optional: str = "") -> dict[str, dict]:
20
+ return {name: {"nargs": arity} for names, arity in ((zero, 0), (one, 1), (optional, "?"))
21
+ for name in names.split()}
22
+
23
+
24
+ PROJECTS = {
25
+ "build": {
26
+ "receivers": {"parser", "global_group", "verbosity_exclusive_group", "build_group", "config_exclusive_group", "install_group", "env_exclusive_group"},
27
+ "actions": {"NegativeCountAction": 0},
28
+ "commands": ("build", "pyproject-build"),
29
+ "flags": inventory(
30
+ "-h --help -V --version -q --quiet -v --verbose -s --sdist -w --wheel --metadata -n --no-isolation -x --skip-dependency-check",
31
+ "-o --outdir --sdist-extract-dir --report -C --config-setting --config-json --installer --env-dir --dependency-constraints-txt"),
32
+ },
33
+ "audit": {
34
+ "receivers": {"parser", "dep_source_args"}, "actions": {}, "commands": ("pip-audit",),
35
+ "flags": inventory(
36
+ "-h --help -V --version -l --local --locked -d --dry-run -S --strict -v --verbose --fix --require-hashes --skip-editable --no-deps --disable-pip",
37
+ "-r --requirement -f --format -s --vulnerability-service --osv-url --cache-dir --progress-spinner --timeout --path --index-url --extra-index-url -o --output --ignore-vuln",
38
+ "--desc --aliases"),
39
+ },
40
+ "markitdown": {
41
+ "receivers": {"parser", "cloud_group"}, "actions": {}, "commands": ("markitdown",),
42
+ "flags": inventory(
43
+ "-h --help -v --version -d --use-docintel --use-cu --use-content-understanding -p --use-plugins --list-plugins --keep-data-uris",
44
+ "-o --output -x --extension -m --mime-type -c --charset -e --endpoint --cu-endpoint --cu-analyzer --cu-file-types --docintel-model-id --docintel-query-fields --docintel-features"),
45
+ },
46
+ }
47
+
48
+ # Manually reviewed exact long-option occurrences in supported README shell
49
+ # examples. Each tuple is (physical line, logical command start, option, count).
50
+ # The count handles repeated literal options at the same location.
51
+ REAL_OPTIONS = {
52
+ "build": [(79, 79, "--installer", 1)],
53
+ "audit": [(132,132,"--help",1), (303,303,"--locked",1), (324,324,"--aliases",1),
54
+ (335,335,"--desc",1), (408,408,"--fix",1), (443,443,"--ignore-vuln",1),
55
+ (454,454,"--ignore-vuln",2), (475,475,"--local",1), (488,488,"--no-deps",1),
56
+ (491,491,"--require-hashes",1)],
57
+ "markitdown": [(120,120,"--list-plugins",1), (126,126,"--use-plugins",1),
58
+ (189,189,"--use-cu",1), (189,189,"--cu-endpoint",1), (196,196,"--use-cu",1),
59
+ (268,268,"--use-docintel",1), (269,268,"--endpoint",1),
60
+ (270,268,"--docintel-model-id",1), (271,268,"--docintel-features",1),
61
+ (272,268,"--docintel-query-fields",1), (415,415,"--list-plugins",1),
62
+ (416,416,"--use-plugins",1)],
63
+ }
64
+
65
+ WORKFLOWS = {
66
+ 'build': ('cd.yml', 'pre-release.yml', 'reusable-change-detection.yml', 'reusable-docs.yml', 'reusable-pytest.yml', 'reusable-type.yml', 'test.yml'),
67
+ 'audit': ('ci.yml', 'docs.yml', 'lint.yml', 'release.yml', 'scorecards.yml', 'zizmor.yml'),
68
+ 'markitdown': ('pre-commit.yml', 'tests.yml'),
69
+ }
70
+
71
+
72
+ def oracle(parser: argparse.ArgumentParser, candidate: str) -> tuple[str | None, str | None]:
73
+ with contextlib.redirect_stderr(io.StringIO()):
74
+ try:
75
+ ns, rest = parser.parse_known_args([candidate])
76
+ except SystemExit:
77
+ return "ambiguous", None
78
+ if rest:
79
+ return "unknown", None
80
+ matched = next(k for k, value in vars(ns).items() if value)
81
+ return (None, None) if candidate == matched else ("abbreviation", matched)
82
+
83
+
84
+ def format_inputs(command: str):
85
+ yield "shell", command
86
+ yield "make", "all:\n\t@" + command + "\n"
87
+ yield "markdown", "```bash\n" + command + "\n```\n"
88
+ yield "workflow", "jobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - run: |\n " + command + "\n"
89
+
90
+
91
+ def main() -> int:
92
+ parser = argparse.ArgumentParser(description=__doc__)
93
+ parser.add_argument("data", type=Path, help="directory with build/, audit/, markitdown/ inputs")
94
+ data = parser.parse_args().data
95
+ schemas = {}
96
+ for name, cfg in PROJECTS.items():
97
+ flags, notes = extract_argparse((data/name/"cli.py").read_text(), cfg["receivers"], action_arities=cfg["actions"])
98
+ assert not notes, (name, notes)
99
+ assert flags == cfg["flags"], (name, set(flags) ^ set(cfg["flags"]), {k:(flags[k],cfg["flags"][k]) for k in flags.keys() & cfg["flags"].keys() if flags[k]!=cfg["flags"][k]})
100
+ print(name, "reviewed spellings", len(flags), "long options", sum(f.startswith('--') for f in flags))
101
+ for command in cfg["commands"]:
102
+ schemas[command] = _command({"flags": flags, "complete": True})
103
+
104
+ natural = Counter()
105
+ real_mutations = Counter()
106
+ for name in PROJECTS:
107
+ files = [(data/name/"README.md", "markdown"), *((data/name/".github/workflows"/p,"workflow") for p in WORKFLOWS[name])]
108
+ for path, kind in files:
109
+ findings = scan_text(path.read_text(), schemas, kind, str(path))
110
+ natural["files"] += 1
111
+ natural["false_positive_findings"] += len(findings)
112
+ assert not findings, (path, findings)
113
+ lines = (data/name/"README.md").read_text().splitlines(keepends=True)
114
+ for physical, logical, option, count in REAL_OPTIONS[name]:
115
+ original = lines[physical-1]
116
+ assert original.count(option) == count, (name, physical, option, original)
117
+ start = 0
118
+ for _ in range(count):
119
+ offset = original.index(option, start)
120
+ changed = lines.copy()
121
+ changed[physical-1] = original[:offset] + '--flagcheck-benchmark-invalid' + original[offset+len(option):]
122
+ findings = scan_text(''.join(changed), schemas, "markdown")
123
+ real_mutations["cases"] += 1
124
+ if [(f.flag,f.code,f.line) for f in findings] == [("--flagcheck-benchmark-invalid","unknown",logical)]:
125
+ real_mutations["detected"] += 1
126
+ else:
127
+ print("MISSED OR EXTRA", name, physical, option, findings)
128
+ start = offset + len(option)
129
+ print("Natural files:", dict(natural))
130
+ print("Mutation of real README option tokens:", dict(real_mutations))
131
+
132
+ synthetic = Counter()
133
+ for name, cfg in PROJECTS.items():
134
+ long = {f: meta for f,meta in cfg["flags"].items() if f.startswith('--')}
135
+ ref = argparse.ArgumentParser(add_help=False, allow_abbrev=True)
136
+ for flag in sorted(long):
137
+ ref.add_argument(flag, action='store_true', dest=flag)
138
+ names = set(long)
139
+ candidates = names | {f[:n] for f in names for n in range(3,len(f))} | {f+'-flagcheck-test-invalid' for f in names}
140
+ for candidate in sorted(candidates):
141
+ expected, suggestion = oracle(ref, candidate)
142
+ canonical = candidate if expected is None else suggestion
143
+ token = candidate + ('=sample' if canonical and long[canonical]['nargs'] != 0 else '')
144
+ for command in cfg['commands']:
145
+ command = 'python -m build' if command == 'build' else command
146
+ for kind, text in format_inputs(command + ' ' + token):
147
+ findings = scan_text(text, schemas, kind)
148
+ actual = (findings[0].code,findings[0].suggestion) if len(findings)==1 else (None,None) if not findings else ('multiple',None)
149
+ synthetic['cases'] += 1
150
+ synthetic['expected_' + (expected or 'clean')] += 1
151
+ if actual == (expected,suggestion):
152
+ synthetic['correct'] += 1
153
+ else:
154
+ synthetic['mismatches'] += 1
155
+ print('MISMATCH', name, kind, candidate, (expected,suggestion), actual)
156
+ print("Synthetic argparse oracle:", dict(synthetic))
157
+ assert real_mutations['cases'] == real_mutations['detected']
158
+ assert natural['files'] == 18 and real_mutations['cases'] == 24
159
+ assert synthetic['cases'] == synthetic['correct']
160
+ return 0
161
+
162
+
163
+ if __name__ == '__main__':
164
+ raise SystemExit(main())