artie-cli 0.7.0__tar.gz

artie_cli-0.7.0/.artie.toml.example

@@ -0,0 +1,32 @@
+# .artie.toml — per-project configuration for artie-cli.
+#
+# Copy this file to `.artie.toml` in your repository root. artie discovers it
+# automatically by searching the working directory and its parents; you can
+# also point at one explicitly with `artie check --config path/to/.artie.toml`.
+#
+# Command-line flags always override what is set here. Every section is
+# optional. See docs/scoring.md for what the thresholds mean.
+
+# Checks excluded from the pass/fail gate. They still run and appear in the
+# report — they just cannot fail the build. Use this for checks that don't
+# apply to your API rather than disabling the gate entirely.
+ignore = ["Schema Complexity"]
+
+[defaults]
+# Default values for CLI flags, so CI doesn't need a long command line.
+with_generation = false        # run the opt-in Sample Generation check
+differential = false           # differential mode (implies with_generation)
+model = "claude-sonnet-4-6"     # model for Sample Generation
+fail_under = 7                  # global gate threshold for checks without
+                                # a per-check threshold below (0-10)
+
+[thresholds]
+# Per-check minimum scores. A check scoring below its threshold fails the
+# run (exit code 1). These take precedence over defaults.fail_under and over
+# the --fail-under CLI flag. Names must match the check names exactly.
+"Format Efficiency" = 9
+"Endpoint Completeness" = 8
+"Example Coverage" = 7
+"Error Documentation" = 7
+"Auth Clarity" = 7
+"Parameter Naming" = 8

artie_cli-0.7.0/.claude/settings.local.json

@@ -0,0 +1,25 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(python3 -c \"import sys; sys.path.insert\\(0,'src'\\); import tomllib\")",
+      "Bash(python3 -c ' *)",
+      "Bash(python3 -m venv .venv)",
+      "Bash(.venv/bin/pip install *)",
+      "Bash(.venv/bin/python -c \"from artie import SCORING_VERSION; print\\('scoring', SCORING_VERSION\\)\")",
+      "Bash(.venv/bin/artie check *)",
+      "Read(//tmp/**)",
+      "Bash(/home/edgrz/lib/artie-cli/.venv/bin/artie check *)",
+      "Bash(echo \"exit=$?\")",
+      "Bash(python3 -c \"import json; d=json.load\\(open\\('/tmp/sample.json'\\)\\); print\\('scoring_version:', d.get\\('scoring_version'\\)\\); print\\('checks:', [\\(c['name'],c['score'],c.get\\('informational'\\)\\) for c in d['checks']]\\)\")",
+      "Bash(echo \"gate exit=$?\")",
+      "Bash(rm /tmp/.artie.toml)",
+      "Bash(.venv/bin/python -m pytest -q)",
+      "Bash(.venv/bin/artie --version)",
+      "Bash(env -u ANTHROPIC_API_KEY .venv/bin/artie check examples/sample-openapi.yaml --with-generation)",
+      "Bash(python3 -c \"import json,sys; d=json.load\\(sys.stdin\\); sg=[c for c in d['checks'] if c['name']=='Sample Generation'][0]; print\\('top keys:', sorted\\(d.keys\\(\\)\\)\\); print\\('sample-gen informational/evaluable/score:', sg['informational'], sg['evaluable'], sg['score']\\)\")",
+      "Bash(git checkout *)",
+      "Bash(git add *)",
+      "Bash(.venv/bin/python -m pytest --co -q)"
+    ]
+  }
+}

artie_cli-0.7.0/.github/workflows/ci.yml

@@ -0,0 +1,37 @@
+name: CI
+
+# Run the test suite on every push and pull request. The suite is fully
+# offline — no check here calls the Anthropic API.
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package and dev dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest
+
+      - name: Smoke-check the CLI against the sample specs
+        run: |
+          artie check examples/bookclub-openapi.yaml
+          artie check examples/sample-openapi.yaml
+          artie check examples/broken-openapi.yaml

artie_cli-0.7.0/.gitignore

@@ -0,0 +1,23 @@
+# Python bytecode
+__pycache__/
+*.py[cod]
+
+# Virtual environments
+.venv/
+venv/
+
+# Packaging / build
+build/
+dist/
+*.egg-info/
+
+# Test and tooling caches
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Local artie config (the example template is tracked; a real one is not)
+.artie.toml
+
+# Editor noise
+.DS_Store

artie_cli-0.7.0/ARCHITECTURE.md

@@ -0,0 +1,83 @@
+# Architecture
+
+artie-cli scores API documentation for AI readiness. It accepts either a local file or an HTTP URL, detects the format, parses what it can, runs a set of checks against the parsed content, and renders a report. Each check is an independent measurement; there is no aggregate score.
+
+## Layout
+
+```
+src/artie/
+├── __init__.py        Package version and SCORING_VERSION.
+├── __main__.py        Lets `python -m artie` run the CLI.
+├── cli.py             Typer-based CLI. Owns the request flow and the gate.
+├── config.py          Loads .artie.toml: per-check thresholds, ignored
+│                      checks, default flags.
+├── baseline.py        Loads a saved JSON report for run-over-run deltas.
+├── fetcher.py         HTTP fetcher using stdlib urllib. Returns raw content
+│                      plus a format hint derived from Content-Type and URL.
+├── generator.py       Minimal Anthropic Messages API client (stdlib urllib).
+│                      Used by the Sample Generation check. Implements retry
+│                      logic for transient errors (429, 503, 529).
+├── parsers/
+│   ├── __init__.py    Format detection (detect_format) and the public
+│   │                  parse() entry point.
+│   ├── openapi.py     OpenAPI YAML and JSON parsing. Also extracts embedded
+│   │                  OpenAPI specs from markdown code fences.
+│   └── types.py       Endpoint and ParsedDocs dataclasses. Shared by all
+│                      checks that need a structured view of the spec.
+├── checks/
+│   ├── __init__.py    The ALL_CHECKS registry.
+│   ├── base.py        BaseCheck, CheckResult, Severity. The check protocol.
+│   ├── format_efficiency.py
+│   ├── endpoint_completeness.py
+│   ├── example_coverage.py
+│   ├── error_documentation.py
+│   ├── auth_clarity.py
+│   ├── parameter_naming.py
+│   ├── schema_complexity.py
+│   └── generation_quality.py  SampleGenerationCheck (opt-in, unscored).
+└── reporters/
+    ├── __init__.py
+    ├── terminal.py    Rich-based human-readable report.
+    └── json_report.py Machine-readable JSON for CI integration.
+```
+
+## Request flow
+
+The CLI does the same thing in every invocation:
+
+1. Parse arguments. Resolve target as either a file path or an HTTP URL.
+2. Load configuration. `config.load_config` reads a `.artie.toml` (an explicit `--config` path, or auto-discovered from the working directory upward). CLI flags override config defaults; a `None` flag means "not passed". If `--baseline` is given, `baseline.load_baseline` reads the prior JSON report.
+3. Read content. For files, read directly. For URLs, call `fetcher.fetch`, which sends an Accept header preferring structured formats and returns raw bytes, a Content-Type, and a format hint.
+4. Detect format. `parsers.detect_format` combines the URL suffix, the Content-Type hint, and content inspection to return one of: `openapi-yaml`, `openapi-json`, `yaml`, `json`, `markdown`, `html`, `unknown`.
+5. Parse. `parsers.parse` produces a `ParsedDocs`. For OpenAPI inputs this contains the raw dict, the extracted endpoints, and the components map. For markdown, parse scans code fences for an embedded OpenAPI spec; if found, the ParsedDocs is fully populated and `extracted_from` is set to `"markdown"`. For HTML or other unstructured inputs, raw is None and `parse_error` explains why.
+6. Run checks. The CLI instantiates each check in `ALL_CHECKS`, plus `SampleGenerationCheck` constructed with the resolved flags, and calls `check.run(content, format_type, parsed)` on each. Checks decide for themselves whether they can run. If they cannot, they return a not-evaluable result.
+7. Render. Either terminal output via Rich or JSON via `json_report.render`, both passed the baseline for delta reporting.
+8. Apply the gate. `_gate_failed` exits non-zero if any scored, non-ignored check is below its threshold (per-check from config, else the global `--fail-under` or `defaults.fail_under`).
+
+## The check protocol
+
+Every check subclasses `BaseCheck` and implements `run(content, format_type, parsed) -> CheckResult`. A check returns one of three things:
+
+- An evaluable result with a score 0-10 and a severity computed from the score.
+- A not-evaluable result, created by calling `self.not_evaluable(reason)`. Use this when the input does not provide enough information to score honestly. Examples: schema complexity has no `components.schemas` section, endpoint completeness was given markdown with no embedded spec, generation quality has no API key in the environment.
+- An evaluable result with score 0 if the check ran but found nothing worth scoring above zero. Used sparingly. Prefer not-evaluable when the issue is missing information rather than missing quality.
+
+`CheckResult.metadata` is a free-form dict. Checks use it to carry machine-readable data that does not belong in findings or recommendations: token counts, evaluation criteria booleans, generated code blocks, baseline scores in differential mode, and so on. The JSON reporter passes metadata through verbatim. The terminal reporter has special-case rendering for `metadata.code`, which it shows as a syntax-highlighted Python block.
+
+## Notable design decisions
+
+**No aggregate score.** Each check stands alone. We do not currently have empirical evidence for the right weights to combine them. See `docs/scoring.md` for every threshold, its evidence basis, and the rationale for not aggregating.
+
+**Stdlib urllib instead of the anthropic SDK.** The generator does one kind of call: a single Messages request, no streaming, no batching. Adding httpx, pydantic, and the SDK would roughly double cold-start time under uvx. The fetcher uses the same approach for the same reason.
+
+**Embedded OpenAPI extraction.** Modern docs sites (Mintlify, Fern, ReadMe, Stainless, Scalar) embed the OpenAPI spec inside markdown code fences alongside prose and SDK examples. When the input is markdown, `parsers.openapi._parse_markdown` scans for the first fenced block whose contents parse as OpenAPI and uses that as the spec. The structured checks then run against the embedded spec while Format Efficiency still scores against the full markdown.
+
+**Sample Generation is a separate kind of check, and unscored.** It calls an external API, costs money, and produces results that depend partly on the model rather than entirely on the docs. Because of that it is opt-in (`--with-generation`) and deliberately produces no 0-10 score: its `CheckResult` has `informational=True` and `score=None`. The generated code is the deliverable; the five structural criteria are reported as commentary. It accepts a `console` parameter so it can render a spinner during the LLM call, and a `differential` flag that triggers a second baseline call. When that baseline is structurally strong it emits a contamination warning. It is the most complex check and the least empirically grounded. Treat it accordingly.
+
+**Informational is a third result state.** Alongside scored and not-evaluable, a `CheckResult` can be `informational`: the check ran and has findings but assigns no score. It is excluded from `is_evaluable` and from the pass/fail gate. Only Sample Generation uses it.
+
+**Not-evaluable is a first-class state, not an error.** A check returning N/A is honest: this measurement does not apply to this input. The terminal reporter renders N/A with its own symbol and severity color. The JSON reporter exposes `evaluable: false`. Both are intentional design choices to avoid pressuring checks into producing fake scores when they cannot honestly produce real ones.
+
+## Adding a check
+
+See `CONTRIBUTING.md`.

artie_cli-0.7.0/CONTRIBUTING.md

@@ -0,0 +1,142 @@
+# Contributing
+
+## Development setup
+
+```bash
+git clone https://github.com/grzetich/artie-cli
+cd artie-cli
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e ".[dev]"
+artie check examples/bookclub-openapi.yaml
+pytest
+```
+
+The static checks and the test suite are fully offline. If you want to exercise the opt-in Sample Generation check, export an Anthropic API key and pass `--with-generation`:
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+artie check examples/bookclub-openapi.yaml --with-generation
+```
+
+Read `ARCHITECTURE.md` before touching the code. It explains the request flow, the check protocol, and the design decisions that constrain implementation choices.
+
+## Running examples
+
+Three samples live in `examples/`:
+
+- `bookclub-openapi.yaml` is the gold standard. Every check should score 10/10 against it. If a change makes it score lower, the change is suspect.
+- `sample-openapi.yaml` is a minimal spec. Endpoint Completeness scores well, Example Coverage and Auth Clarity do not. Useful for checking middle-of-the-road behavior.
+- `broken-openapi.yaml` is deliberately bad. Most checks score zero or near zero, with named operations and properties in the recommendations. Useful for verifying that low scores produce actionable findings.
+
+Run each before and after any scoring change to make sure the change does what you expect.
+
+## Writing a new check
+
+Subclass `BaseCheck`, implement `run`, register in `checks/__init__.py`.
+
+```python
+from typing import Any
+
+from artie.checks.base import BaseCheck, CheckResult
+from artie.parsers.types import ParsedDocs
+
+
+class MyNewCheck(BaseCheck):
+    name = "My New Check"
+    description = "What this check measures."
+
+    def run(
+        self, content: str, format_type: str, parsed: Any = None
+    ) -> CheckResult:
+        if not isinstance(parsed, ParsedDocs) or not parsed.is_openapi:
+            return self.not_evaluable("This check needs a parsed OpenAPI spec.")
+
+        # Compute the score on a 0-10 scale.
+        score = self._compute(parsed)
+        severity = self.severity_for(score)
+
+        return CheckResult(
+            name=self.name,
+            description=self.description,
+            score=score,
+            max_score=self.max_score,
+            severity=severity,
+            findings=["What we observed."],
+            recommendations=["What to do about it."],
+            metadata={"raw_data": "for JSON consumers"},
+        )
+
+    def _compute(self, parsed: ParsedDocs) -> int:
+        ...
+```
+
+Then add it to `ALL_CHECKS` in `checks/__init__.py`. Order matters because it determines display order in the report.
+
+Things to honor in your check:
+
+- Return `not_evaluable` when the check cannot honestly score the input. Do not return a fake zero or a fake ten just to have a number on the report.
+- Use `self.severity_for(score)` to map the score to a severity bucket. Do not invent your own mapping.
+- Findings are observations. Recommendations are advice. Keep them distinct.
+- Findings should be specific to the input being scored. "5 of 14 endpoints" beats "many endpoints." When pointing at specific problems, name the operations or properties (up to about five examples, truncated with an ellipsis).
+- Metadata is for machine consumers. Put structured data there instead of synthesizing it from finding strings.
+
+## Style notes
+
+Write the way the existing code reads. Concrete observations, named items, no hedging. Avoid these:
+
+- Em or en dashes. Use a comma or split the sentence.
+- "Leverage," "seamless," "robust," "fundamentally," "paradigm," "ecosystem" as metaphor, and the rest of the AI-tells list.
+- Guided-tour phrasings like "Let's look at." Just look at it.
+- Long bolded thesis sentences that summarize a paragraph. Let the paragraph carry its own weight.
+- Sentences clustered on "This." More than two in a row signals laziness.
+
+For docstrings, one sentence is usually enough. Longer when the function does something non-obvious. Module docstrings should explain why the module exists and what it produces, not narrate every function.
+
+## Scoring conventions
+
+Scores are 0-10 integers. The mapping to severity, in `BaseCheck.severity_for`:
+
+- 9-10: excellent
+- 7-8: good
+- 4-6: needs work
+- 0-3: poor
+- None: not evaluable
+
+If you find yourself wanting a different bucket size or a different scale, that is a sign to add a new check rather than redefine the scale. The convention is shared across all checks so users can read across them.
+
+When a check has multiple sub-signals, average them and scale the result to 0-10. When sub-signals do not apply (no path parameters, no request bodies), drop them from the average rather than scoring them zero.
+
+Document threshold choices in code comments at the top of the check. Future calibration work needs to know what each threshold currently corresponds to.
+
+## Tests
+
+Tests live under `tests/` and use pytest. Run them with `pytest` (the
+configuration in `pyproject.toml` handles the source path). The suite is
+fully offline — no test calls the Anthropic API.
+
+Current coverage:
+
+- Each check's scoring math and the Schema Complexity threshold buckets, including not-evaluable edge cases (`test_checks_scoring.py`).
+- The format detector across all supported formats (`test_format_detector.py`).
+- The OpenAPI parser including embedded extraction from markdown (`test_parser.py`).
+- Sample Generation internals: the gap-marker regex, the identifier builder, and code evaluation (`test_generation_eval.py`, `test_sample_generation.py`).
+- Config loading, baseline deltas, the pass/fail gate, and scoring-version sync (`test_config.py`, `test_baseline.py`, `test_gate.py`, `test_scoring_version.py`).
+
+Still uncovered and worth adding: the fetcher's content-type hint logic and
+the generator's retry behavior, both with mocked HTTP responses.
+
+`.github/workflows/ci.yml` runs the suite on every push and pull request
+across Python 3.10–3.12.
+
+## Versioning
+
+Bump the version in `pyproject.toml` and `src/artie/__init__.py` together. They must match. Use semver. A change to the scoring rubric is a minor version bump because it changes user-visible behavior even when code APIs do not change.
+
+The scoring rubric is versioned separately from the package. `artie.SCORING_VERSION` is the runtime source of truth; `pyproject.toml [tool.artie] scoring_version` mirrors it, and `tests/test_scoring_version.py` fails if the two drift. Bump it whenever a threshold, bucket, or weighting changes, and update `docs/scoring.md` in the same commit. It is emitted in JSON output so downstream tooling can pin against it.
+
+## Pull requests
+
+Keep them focused. One check, one bug fix, one CI feature per PR. If you find yourself wanting to do unrelated cleanups, open a separate PR for those.
+
+Run the three example specs before and after your change. Include the diff in the PR description when scores change. If a sample's score moves, explain why the new score is more correct than the old one.

artie_cli-0.7.0/PKG-INFO

@@ -0,0 +1,169 @@
+Metadata-Version: 2.4
+Name: artie-cli
+Version: 0.7.0
+Summary: Score your API documentation for AI-readiness. Based on the Tokens Not Jokin' research.
+Project-URL: Homepage, https://artie.fm
+Project-URL: Repository, https://github.com/grzetich/artie-cli
+Project-URL: Research, https://leanpub.com/tokensnotjokin
+Author-email: Ed Grzetich <ed@grzeti.ch>
+License: MIT
+Keywords: ai,api,developer-tools,documentation,llm,openapi
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Documentation
+Classifier: Topic :: Software Development :: Documentation
+Requires-Python: >=3.10
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: tiktoken>=0.7.0
+Requires-Dist: tomli>=2.0; python_version < '3.11'
+Requires-Dist: typer>=0.12.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# artie-cli
+
+Score your API documentation for AI-readiness.
+
+artie reads your API docs the way an AI agent would, then reports how well your documentation actually supports code generation. The checks are derived from [*Tokens Not Jokin'*](https://leanpub.com/tokensnotjokin), a 21,462-test empirical study comparing four AI models against four documentation formats.
+
+artie measures and reports. It does not convert, clean, or modify your documentation.
+
+## Quickstart
+
+```bash
+# No install required
+uvx artie-cli check ./openapi.yaml
+uvx artie-cli check https://api.example.com/openapi.yaml
+
+# Install with pipx
+pipx install artie-cli
+artie check ./openapi.yaml
+```
+
+By default artie runs the seven static checks, entirely offline. The optional Sample Generation check is opt-in: pass `--with-generation` and set an Anthropic API key.
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+artie check ./openapi.yaml --with-generation
+```
+
+Without `--with-generation`, Sample Generation shows as N/A with a note on how to enable it.
+
+## What it checks
+
+**Seven static checks**, derived from the TNJ research:
+
+- **Format Efficiency**: how token-efficient your documentation format is
+- **Endpoint Completeness**: descriptions, operationIds, and documented path parameters
+- **Example Coverage**: request and response body examples
+- **Error Documentation**: 4xx and 5xx responses with meaningful descriptions and content schemas
+- **Auth Clarity**: security schemes defined, described, and applied to operations
+- **Parameter Naming**: consistent naming convention across parameters and schema properties
+- **Schema Complexity**: nesting depth across component schemas
+
+Each static check scores 0–10. Every threshold and weighting is documented in [docs/scoring.md](docs/scoring.md), along with which numbers are empirically grounded and which are heuristics awaiting calibration. artie deliberately reports no aggregate score.
+
+**One optional, opt-in check**, applied to any input format:
+
+- **Sample Generation**: an AI model is given your docs and asked to write a Python function that calls the API. The generated code is the deliverable — artie includes it in the report so you can see exactly what an agent writes from your docs, and comments on its structure (valid syntax, HTTP client import, error handling, request construction, response handling). It is **not scored**: one generation against one model is a sample, not a measurement. Enable it with `--with-generation`.
+
+The static checks tell you what specifically to fix. Sample Generation shows you a concrete example of what an agent produces. Static checks return N/A when the input isn't OpenAPI; Sample Generation runs against anything.
+
+## Inputs
+
+artie accepts either a local file or an HTTP/HTTPS URL:
+
+```bash
+artie check ./openapi.yaml
+artie check https://api.example.com/openapi.yaml
+artie check https://docs.example.com/getting-started
+```
+
+Supported formats: OpenAPI YAML, OpenAPI JSON, plain YAML, plain JSON, Markdown, HTML.
+
+When the input is markdown, artie scans for an OpenAPI spec embedded in a code fence and runs the structured checks against that spec. Modern docs sites (Mintlify, Fern, ReadMe, Stainless) embed the spec inline alongside prose and SDK examples, and artie picks it up automatically.
+
+When fetching URLs, artie sends an `Accept` header that requests structured formats first. If the server honors content negotiation (some major docs sites do, including parts of AWS and most Mintlify-hosted sites), you may receive a different format than the URL suggests. The report calls this out.
+
+## Output formats
+
+```bash
+# Pretty terminal output (default); static checks only, fully offline
+artie check ./openapi.yaml
+
+# JSON for CI pipelines (includes scoring_version)
+artie check ./openapi.yaml --output json
+
+# Fail the build if any check scores below 7
+artie check ./openapi.yaml --fail-under 7
+
+# Compare against a previously saved JSON report and show per-check deltas
+artie check ./openapi.yaml --baseline previous.json
+
+# Run the opt-in Sample Generation check (uses the Anthropic API)
+artie check ./openapi.yaml --with-generation
+
+# Differential mode: flag training-data contamination (implies --with-generation, doubles cost)
+artie check ./openapi.yaml --differential
+
+# Use a different model for Sample Generation
+artie check ./openapi.yaml --with-generation --model claude-opus-4-7
+```
+
+## Configuration
+
+Drop a `.artie.toml` in your repository root to pin per-check pass/fail thresholds, exclude checks from the gate, and set default flags. artie discovers it automatically from the working directory upward. See [`.artie.toml.example`](.artie.toml.example) for the full schema and [docs/scoring.md](docs/scoring.md) for what the thresholds mean.
+
+## About Sample Generation and training contamination
+
+Sample Generation has a real limitation worth understanding. When the model has seen an API during training (which is the case for AWS, Stripe, GitHub, and most public APIs with Python SDKs on PyPI), it can write working code from training alone, regardless of how complete the docs you're testing are. Good-looking code for a famous API tells you the model knows the API, not that the docs are good.
+
+This is exactly why the check is unscored: a grade would imply a measurement the method can't honestly deliver. Instead artie shows you the generated code and comments on its structure.
+
+The prompt instructs the model to use only information from the docs and to flag gaps in inline code comments. artie detects those gap comments automatically and reports them as evidence of real documentation deficiencies.
+
+`--differential` mode adds a second API call with no docs body, measuring what the model produces from training alone. When that baseline is already strong, artie prints a contamination warning: the docs-informed sample reflects model capability, not docs quality. Differential mode is most informative for novel or internal APIs the model has not seen.
+
+## Cost
+
+Sample Generation makes one Anthropic API call per run (two with `--differential`). On Claude Sonnet 4.6 (the default), a typical docs page costs roughly $0.02 to $0.05 per run at retail pricing. Because the check is opt-in, CI runs stay free and offline unless you explicitly ask for it.
+
+## Examples
+
+```bash
+artie check examples/bookclub-openapi.yaml
+artie check examples/broken-openapi.yaml
+artie check examples/sample-openapi.yaml
+```
+
+## Privacy
+
+The static checks run entirely locally. Sample Generation sends your documentation content to Anthropic's API — but it is opt-in, so nothing leaves your machine unless you pass `--with-generation`.
+
+## Why a CLI
+
+artie is a checker, not a converter. CLI tools live where the docs do: in the repo, in the pipeline, next to the spec. You get a score, you act on it, the static-check data never leaves your machine.
+
+## Research
+
+artie's checks are grounded in empirical findings published in *Tokens Not Jokin'*. Key results:
+
+- YAML uses up to 80% fewer tokens than OpenAPI 3.0 JSON
+- Documentation format explains more than 10x the variance in generated code quality than model choice
+- Disciplined error documentation produces dramatically better error handling in generated code
+
+The Sample Generation check uses the same methodology as TNJ, applied per-spec: ask an AI to write code from these docs, then show what it produced.
+
+Buy the book: [leanpub.com/tokensnotjokin](https://leanpub.com/tokensnotjokin)
+
+## License
+
+MIT

artie_cli-0.7.0/README.md

@@ -0,0 +1,139 @@
+# artie-cli
+
+Score your API documentation for AI-readiness.
+
+artie reads your API docs the way an AI agent would, then reports how well your documentation actually supports code generation. The checks are derived from [*Tokens Not Jokin'*](https://leanpub.com/tokensnotjokin), a 21,462-test empirical study comparing four AI models against four documentation formats.
+
+artie measures and reports. It does not convert, clean, or modify your documentation.
+
+## Quickstart
+
+```bash
+# No install required
+uvx artie-cli check ./openapi.yaml
+uvx artie-cli check https://api.example.com/openapi.yaml
+
+# Install with pipx
+pipx install artie-cli
+artie check ./openapi.yaml
+```
+
+By default artie runs the seven static checks, entirely offline. The optional Sample Generation check is opt-in: pass `--with-generation` and set an Anthropic API key.
+
+```bash
+export ANTHROPIC_API_KEY=sk-ant-...
+artie check ./openapi.yaml --with-generation
+```
+
+Without `--with-generation`, Sample Generation shows as N/A with a note on how to enable it.
+
+## What it checks
+
+**Seven static checks**, derived from the TNJ research:
+
+- **Format Efficiency**: how token-efficient your documentation format is
+- **Endpoint Completeness**: descriptions, operationIds, and documented path parameters
+- **Example Coverage**: request and response body examples
+- **Error Documentation**: 4xx and 5xx responses with meaningful descriptions and content schemas
+- **Auth Clarity**: security schemes defined, described, and applied to operations
+- **Parameter Naming**: consistent naming convention across parameters and schema properties
+- **Schema Complexity**: nesting depth across component schemas
+
+Each static check scores 0–10. Every threshold and weighting is documented in [docs/scoring.md](docs/scoring.md), along with which numbers are empirically grounded and which are heuristics awaiting calibration. artie deliberately reports no aggregate score.
+
+**One optional, opt-in check**, applied to any input format:
+
+- **Sample Generation**: an AI model is given your docs and asked to write a Python function that calls the API. The generated code is the deliverable — artie includes it in the report so you can see exactly what an agent writes from your docs, and comments on its structure (valid syntax, HTTP client import, error handling, request construction, response handling). It is **not scored**: one generation against one model is a sample, not a measurement. Enable it with `--with-generation`.
+
+The static checks tell you what specifically to fix. Sample Generation shows you a concrete example of what an agent produces. Static checks return N/A when the input isn't OpenAPI; Sample Generation runs against anything.
+
+## Inputs
+
+artie accepts either a local file or an HTTP/HTTPS URL:
+
+```bash
+artie check ./openapi.yaml
+artie check https://api.example.com/openapi.yaml
+artie check https://docs.example.com/getting-started
+```
+
+Supported formats: OpenAPI YAML, OpenAPI JSON, plain YAML, plain JSON, Markdown, HTML.
+
+When the input is markdown, artie scans for an OpenAPI spec embedded in a code fence and runs the structured checks against that spec. Modern docs sites (Mintlify, Fern, ReadMe, Stainless) embed the spec inline alongside prose and SDK examples, and artie picks it up automatically.
+
+When fetching URLs, artie sends an `Accept` header that requests structured formats first. If the server honors content negotiation (some major docs sites do, including parts of AWS and most Mintlify-hosted sites), you may receive a different format than the URL suggests. The report calls this out.
+
+## Output formats
+
+```bash
+# Pretty terminal output (default); static checks only, fully offline
+artie check ./openapi.yaml
+
+# JSON for CI pipelines (includes scoring_version)
+artie check ./openapi.yaml --output json
+
+# Fail the build if any check scores below 7
+artie check ./openapi.yaml --fail-under 7
+
+# Compare against a previously saved JSON report and show per-check deltas
+artie check ./openapi.yaml --baseline previous.json
+
+# Run the opt-in Sample Generation check (uses the Anthropic API)
+artie check ./openapi.yaml --with-generation
+
+# Differential mode: flag training-data contamination (implies --with-generation, doubles cost)
+artie check ./openapi.yaml --differential
+
+# Use a different model for Sample Generation
+artie check ./openapi.yaml --with-generation --model claude-opus-4-7
+```
+
+## Configuration
+
+Drop a `.artie.toml` in your repository root to pin per-check pass/fail thresholds, exclude checks from the gate, and set default flags. artie discovers it automatically from the working directory upward. See [`.artie.toml.example`](.artie.toml.example) for the full schema and [docs/scoring.md](docs/scoring.md) for what the thresholds mean.
+
+## About Sample Generation and training contamination
+
+Sample Generation has a real limitation worth understanding. When the model has seen an API during training (which is the case for AWS, Stripe, GitHub, and most public APIs with Python SDKs on PyPI), it can write working code from training alone, regardless of how complete the docs you're testing are. Good-looking code for a famous API tells you the model knows the API, not that the docs are good.
+
+This is exactly why the check is unscored: a grade would imply a measurement the method can't honestly deliver. Instead artie shows you the generated code and comments on its structure.
+
+The prompt instructs the model to use only information from the docs and to flag gaps in inline code comments. artie detects those gap comments automatically and reports them as evidence of real documentation deficiencies.
+
+`--differential` mode adds a second API call with no docs body, measuring what the model produces from training alone. When that baseline is already strong, artie prints a contamination warning: the docs-informed sample reflects model capability, not docs quality. Differential mode is most informative for novel or internal APIs the model has not seen.
+
+## Cost
+
+Sample Generation makes one Anthropic API call per run (two with `--differential`). On Claude Sonnet 4.6 (the default), a typical docs page costs roughly $0.02 to $0.05 per run at retail pricing. Because the check is opt-in, CI runs stay free and offline unless you explicitly ask for it.
+
+## Examples
+
+```bash
+artie check examples/bookclub-openapi.yaml
+artie check examples/broken-openapi.yaml
+artie check examples/sample-openapi.yaml
+```
+
+## Privacy
+
+The static checks run entirely locally. Sample Generation sends your documentation content to Anthropic's API — but it is opt-in, so nothing leaves your machine unless you pass `--with-generation`.
+
+## Why a CLI
+
+artie is a checker, not a converter. CLI tools live where the docs do: in the repo, in the pipeline, next to the spec. You get a score, you act on it, the static-check data never leaves your machine.
+
+## Research
+
+artie's checks are grounded in empirical findings published in *Tokens Not Jokin'*. Key results:
+
+- YAML uses up to 80% fewer tokens than OpenAPI 3.0 JSON
+- Documentation format explains more than 10x the variance in generated code quality than model choice
+- Disciplined error documentation produces dramatically better error handling in generated code
+
+The Sample Generation check uses the same methodology as TNJ, applied per-spec: ask an AI to write code from these docs, then show what it produced.
+
+Buy the book: [leanpub.com/tokensnotjokin](https://leanpub.com/tokensnotjokin)
+
+## License
+
+MIT