code2okf 0.1.0__tar.gz

code2okf-0.1.0/.coderabbit.yaml

@@ -0,0 +1,168 @@
+# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json
+#
+# CodeRabbit configuration for code2okf.
+# Anything not listed here uses CodeRabbit's own default.
+language: en-US
+tone_instructions: "Be concise and focus on critical issues only. Do not comment on trivial matters, nitpicks, or stylistic preferences. Only flag issues that could cause bugs, security vulnerabilities, or significant maintainability problems."
+early_access: false
+enable_free_tier: true
+reviews:
+  profile: quiet
+  request_changes_workflow: false
+  high_level_summary: true
+  high_level_summary_in_walkthrough: false
+  review_status: true
+  commit_status: true
+  fail_commit_status: false
+  collapse_walkthrough: true
+  changed_files_summary: true
+  sequence_diagrams: false
+  estimate_code_review_effort: true
+  assess_linked_issues: true
+  related_issues: true
+  related_prs: true
+  suggested_labels: true
+  auto_apply_labels: false
+  suggested_reviewers: true
+  auto_assign_reviewers: false
+  in_progress_fortune: false
+  poem: false
+  abort_on_close: true
+  # Generated, gitignored, or manually maintained content that is not reviewed:
+  # the wiki's input and output, the lockfile, the optional PDF step, and
+  # SPEC.md, which is the upstream OKF revision this repo builds against
+  # rather than code of ours to review.
+  path_filters:
+    - "!SPEC.md"
+    - "!uv.lock"
+    - "!md/**"
+    - "!okf/**/*.md"
+    - "!pdf2md/**"
+    - "!.cursor/mcp.json"
+    - "!.cursor/plans/**"
+    - "!.claude/plans/**"
+  path_instructions:
+    - path: "**/*.sh"
+      instructions: |
+        Apply robust Bash practices: strict error handling, quoted expansions,
+        arrays over constructed command strings, and no silently masked
+        failures. Resolve paths independently of the caller's working
+        directory, validate prerequisites early and exit non-zero with a clear
+        message on stderr, clean up temporary resources with traps, keep
+        secrets out of arguments and logs, and require ShellCheck-clean code.
+    - path: "**/spec.yaml"
+      instructions: |
+        For Docker Sandbox Kit specs, treat schema conformance as load-bearing:
+        flag renamed or restructured keys that a schema check would reject, and
+        say so rather than asserting the spec is valid. Keep capabilities
+        least-privilege and network access default-deny with a stated reason
+        for every allowed host, keep credentials proxy-managed and real secrets
+        out of the file, pin installed versions, make setup commands
+        idempotent, and bound retries and timeouts.
+    - path: ".github/workflows/**"
+      instructions: |
+        Apply GitHub Actions best practices: pin third-party actions and
+        downloaded binaries to a version, keep workflow permissions
+        least-privilege, avoid running untrusted input with elevated
+        privileges, fail loudly on download and script errors, and never echo
+        secrets into logs. Keep duplicated trigger configuration in sync.
+    - path: "**/*.md"
+      instructions: |
+        Review documentation against the rest of the diff rather than for prose
+        style: flag commands, paths, options, and examples that this change has
+        made wrong or stale, and instructions that contradict each other. Do
+        not raise wording, grammar, or line-length nitpicks.
+    - path: "**/*.py"
+      instructions: |
+        Review Python code for:
+        - PEP 8, type hints, docstrings, error handling, and performance. Flag mutable default arguments.
+        - Blocking calls inside async functions (use asyncio equivalents)
+        - API keys or secrets accidentally logged or exposed in error messages
+        - Missing context managers for network/file resources
+        - Unbounded retries or missing timeouts on external calls
+        - User-controlled strings passed unsanitized to shell commands or LLM prompts
+    - path: "**/tests/**/*.py"
+      instructions: |
+        Review test code for:
+        - Check pytest best practices, fixture usage, test isolation, and edge case coverage.
+        - Async test functions must have @pytest.mark.asyncio decorator
+        - Tests creating files/directories must have cleanup fixtures
+        - Prefer tmp_path over manual temp directories, and monkeypatch over
+          direct mutation of module state, so tests stay isolated.
+        - Tests must not perform real network I/O; HTTP belongs behind
+          httpx.MockTransport.
+  auto_review:
+    enabled: true
+    auto_incremental_review: true
+    ignore_title_keywords:
+      - "DO NOT REVIEW"
+      - "WIP"
+    drafts: false
+  # The web2md scraper (web2md/src/) and its pytest suite (web2md/tests/) are the
+  # only first-party code here.
+  finishing_touches:
+    docstrings:
+      enabled: true
+    unit_tests:
+      enabled: true
+  pre_merge_checks:
+    docstrings:
+      mode: "off"
+    title:
+      mode: warning
+      requirements: "Start with an imperative verb, name the area touched, and keep it under 72 characters."
+    description:
+      mode: warning
+    issue_assessment:
+      mode: warning
+  # Only deliberate overrides are listed. Every other tool is left at its
+  # default so it starts working on its own if this repo grows that language.
+  tools:
+    shellcheck:
+      enabled: true
+    checkmake:
+      enabled: true
+    yamllint:
+      enabled: true
+    actionlint:
+      enabled: true
+    markdownlint:
+      enabled: true
+    gitleaks:
+      enabled: true
+    checkov:
+      enabled: true
+    semgrep:
+      enabled: true
+    osvScanner:
+      enabled: true
+    # Grammar and style findings are exactly what tone_instructions above tells
+    # the reviewer not to raise, so leave this off rather than filtering it.
+    languagetool:
+      enabled: false
+    github-checks:
+      enabled: true
+      timeout_ms: 900000
+    ruff:
+      enabled: true
+      config_file: "web2md/pyproject.toml"
+chat:
+  art: false
+  auto_reply: true
+knowledge_base:
+  opt_out: false
+  web_search:
+    enabled: true
+  code_guidelines:
+    enabled: true
+  learnings:
+    scope: auto
+  issues:
+    scope: auto
+  pull_requests:
+    scope: auto
+issue_enrichment:
+  auto_enrich:
+    enabled: true
+  planning:
+    enabled: true

code2okf-0.1.0/.cspell.json

@@ -0,0 +1,103 @@
+{
+  "version": "0.2",
+  "language": "en,en-GB",
+  "ignorePaths": [
+    "md/**",
+    ".claude/**",
+    ".cursor/**",
+    "SPEC.md",
+    "CLAUDE.md",
+    "uv.lock",
+    "web2md/tests/**",
+    "scripts/inspectmd/tests/**",
+    "scripts/inspectokf/tests/**",
+    "scripts/sizeokf/tests/**",
+    "scripts/merkleokf/tests/**",
+    "tests/**"
+  ],
+  "words": [
+    "okf",
+    "frontmatter",
+    "deepseek",
+    "litellm",
+    "gemini",
+    "thisismydesign",
+    "MarkerMarkdown",
+    "OpenRouter",
+    "Ollama",
+    "sbx",
+    "sbxenv",
+    "npmjs",
+    "maxsockets",
+    "earendil",
+    "passwordless",
+    "sandboxed",
+    "shellcheck",
+    "nullglob",
+    "nolog",
+    "pytest",
+    "pyproject",
+    "pipefail",
+    "psutil",
+    "fdfind",
+    "venv",
+    "pycache",
+    "httpx",
+    "pythonpath",
+    "testpaths",
+    "minversion",
+    "addopts",
+    "coderabbit",
+    "pycodestyle",
+    "pyflakes",
+    "pydocstyle",
+    "docstring",
+    "docstrings",
+    "conftest",
+    "monkeypatch",
+    "argparse",
+    "workbench",
+    "qwen",
+    "resumability",
+    "resumable",
+    "deepinfra",
+    "xhigh",
+    "gemma",
+    "idempotently",
+    "misordered",
+    "scannable",
+    "writable",
+    "inspectmd",
+    "uvx",
+    "hatchling",
+    "sdist",
+    "pypi",
+    "stdlib",
+    "context7",
+    "upstash",
+    "inspectokf",
+    "sizeokf",
+    "merkleokf",
+    "Merkle",
+    "worldbuilding",
+    "retcon",
+    "retconned",
+    "retcons",
+    "uncited",
+    "wikilinks",
+    "outlinks",
+    "dataview",
+    "throughline",
+    "plannable",
+    "leyline",
+    "Dragonsteel",
+    "Holocron",
+    "Dramatica",
+    "Egri",
+    "usermod",
+    "newgrp",
+    "okfctl",
+    "okflintrc",
+    "cwest"
+  ]
+}

code2okf-0.1.0/.editorconfig

@@ -0,0 +1,13 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.md]
+trim_trailing_whitespace = false
+
+[Makefile]
+indent_style = tab

code2okf-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,184 @@
+---
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+# The jobs only read the checked-out tree.
+permissions:
+  contents: read
+
+# Cancel superseded runs
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Host-side shell tests for state mount selection and relocation. The Linux
+  # runner must exercise the real bind-mount and idempotency cases.
+  test-shell:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Test state mounts
+        run: make test-shell CODE2OKF_REQUIRE_BIND=1
+
+  # Lint Markdown, JSON, YAML, shell, spelling, and Python. Reuses `make lint`
+  # so the lint definition lives in one place (the Makefile). markdownlint runs
+  # via npx; yamllint and ruff run ephemerally at a pinned version via
+  # `uv tool run`, belonging to no project, so this job installs neither
+  # project's dependencies. cspell runs via npx. shellcheck and jq are both
+  # pre-installed on ubuntu-latest. The JSON check is syntax only (`jq empty`)
+  # — no schema validation.
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # No git operations after checkout, so don't leave the job token in
+          # .git/config where a later third-party step could read it.
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Lint (markdownlint, jq, yamllint, shellcheck, cspell, ruff)
+        run: make lint MARKDOWNLINT='npx --yes markdownlint-cli2'
+
+  # Unit-test the web2md scraper. Reuses `make test-web2md` so the definition
+  # stays in the Makefile. web2md is its own project, so its lockfile holds
+  # only the scraper's own deps plus pytest — marker-pdf / torch cannot reach
+  # it and there is nothing to exclude. The suite is offline
+  # (httpx.MockTransport), so this job needs no network beyond the install.
+  test-web2md:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # No git operations after checkout, so don't leave the job token in
+          # .git/config where a later third-party step could read it.
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Test (pytest)
+        run: make test-web2md
+
+  # Unit-test the four host CLIs (inspectmd, inspectokf, sizeokf, merkleokf).
+  # Reuses `make test-clis`. Each CLI is its own project and lockfile;
+  # stdlib-only subjects under test (inspectokf mocks tree). Offline: no
+  # network beyond the install.
+  test-clis:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # No git operations after checkout, so don't leave the job token in
+          # .git/config where a later third-party step could read it.
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Test (pytest)
+        run: make test-clis
+
+  # Unit-test the code2okf driver itself. Reuses `make test-code2okf`. Offline:
+  # every sbx call goes through the one seam in code2okf.sandbox, which the
+  # suite replaces with a fake rather than shelling out to a real sandbox.
+  test-code2okf:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # No git operations after checkout, so don't leave the job token in
+          # .git/config where a later third-party step could read it.
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Test (pytest)
+        run: make test-code2okf
+
+  # Build the distribution and prove it works away from the checkout. Reuses
+  # `make dist` so the definition stays in the Makefile. `uv build` builds the
+  # wheel from the sdist, so an sdist missing kits/, SPEC.md or
+  # scripts/<cli>/ breaks here rather than after a release; the smoke run then
+  # exercises the *bundled* assets, which nothing run from the repository can
+  # do -- resources.py finds the real tree and every lookup succeeds either
+  # way. Offline apart from the installs: --dry-run creates no sandbox.
+  build-package:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # No git operations after checkout, so don't leave the job token in
+          # .git/config where a later third-party step could read it.
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Build and smoke-test the distribution
+        run: make dist
+
+  # Validate kits/code2okf/spec.yaml against the current Sandbox Kit schema.
+  # Static check — no Docker or login; only the CLI download needs the network.
+  validate-kit:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # No git operations after checkout, so don't leave the job token in
+          # .git/config where a later third-party step could read it.
+          persist-credentials: false
+
+      # Install the latest sbx CLI. "latest" is intentional: the schema is
+      # bundled in the binary, so this surfaces schema drift as soon as it ships.
+      # Install steps mirror docker/sbx-kits-contrib .github/workflows/tck.yml.
+      - name: Install sbx CLI
+        run: |
+          set -euo pipefail
+          curl -fsSL \
+            "https://github.com/docker/sbx-releases/releases/latest/download/DockerSandboxes-linux.tar.gz" \
+            -o /tmp/DockerSandboxes-linux.tar.gz
+          tar xzf /tmp/DockerSandboxes-linux.tar.gz -C /tmp
+          sudo PREFIX="${HOME}/.docker/sbx" /tmp/docker-sbx/install.sh
+          echo "${HOME}/.docker/sbx/bin" >> "$GITHUB_PATH"
+
+      - name: Verify sbx
+        run: sbx version
+
+      - name: Validate Pi kit spec
+        run: make validate

code2okf-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,209 @@
+---
+name: Release
+
+# A tag push is the release: pushing vX.Y.Z verifies the repository agrees with
+# itself, builds the distribution once, publishes it to PyPI, and then creates a
+# GitHub Release whose notes are the matching section of CHANGELOG.md and whose
+# assets are those same artifacts.
+#
+# The job order is deliberate and must stay a chain, not a fan-out. Two jobs
+# that both merely `needs: [verify]` would run concurrently, and the one
+# attaching assets could reach a Release that does not exist yet. Publishing
+# first also means a tag on the Releases page always corresponds to a version
+# that is actually installable.
+#
+# One build feeds both publishers, which is what makes a re-run safe: PyPI
+# treats an upload of a byte-identical file as idempotent, so re-running after
+# a partial failure re-uploads the same bytes rather than colliding with them.
+#
+# All the version and notes logic lives in scripts/, so `make lint`
+# shellchecks the helpers and they can be rehearsed locally.
+
+on:
+  push:
+    # Digit-only, so vX.Y.Z-rc1 and other shapes cannot trigger a release.
+    # Matches the pattern scripts/check-release-tag.sh enforces.
+    tags: ["v[0-9]+.[0-9]+.[0-9]+"]
+
+# Narrowed further per job: github-release writes to the repository,
+# publish-pypi mints an OIDC token, and verify and build need neither.
+permissions:
+  contents: read
+
+# Never cancel a release in flight.
+concurrency:
+  group: release-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  # Gate the release on the repo agreeing with itself before anything is
+  # announced. `make lint` is the substance here: it already fails when VERSION
+  # and CHANGELOG.md's latest release heading disagree, so a tag whose version
+  # is inconsistent cannot create a Release.
+  verify:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    outputs:
+      version: ${{ steps.version.outputs.version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # No git operations after checkout, so don't leave the job token in
+          # .git/config where a later third-party step could read it.
+          persist-credentials: false
+
+      # The tag is authoritative and must agree with VERSION. check-release-tag.sh
+      # writes version= on fd 3, which is why it is redirected rather than
+      # piped: its human output must not reach $GITHUB_OUTPUT.
+      - name: Resolve version
+        id: version
+        run: ./scripts/check-release-tag.sh "${GITHUB_REF_NAME}" 3>>"$GITHUB_OUTPUT"
+
+      # Fail an empty changelog section here, before a Release is created.
+      # `make lint` cannot catch this: it checks only that the latest
+      # `## [X.Y.Z]` heading exists and agrees with VERSION, and a heading with
+      # no body passes that. The github-release job runs the same script again
+      # for the notes themselves, so there is one definition of "non-empty".
+      - name: Check the release notes are not empty
+        env:
+          RELEASE_VERSION: ${{ steps.version.outputs.version }}
+        run: ./scripts/release-notes.sh "${RELEASE_VERSION}" >/dev/null
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Lint (includes the version-agreement check)
+        run: make lint MARKDOWNLINT='npx --yes markdownlint-cli2'
+
+  # Build the wheel and the sdist once, and hand the same bytes to both
+  # publishers below. `make dist` is the same target CI runs on every pull
+  # request: it builds both artifacts, rebuilds the wheel from the sdist in a
+  # scratch directory, and smoke-tests the result from outside the checkout, so
+  # a release cannot ship an sdist that cannot rebuild itself.
+  build:
+    needs: [verify]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # No git operations after checkout, so don't leave the job token in
+          # .git/config where a later third-party step could read it.
+          persist-credentials: false
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: true
+
+      - name: Build and smoke-test the distribution
+        run: make dist
+
+      # `overwrite` defaults to false, which fails the upload outright when an
+      # artifact of this name already exists in the run. Re-running the whole
+      # workflow after a partial failure would then die here, before reaching
+      # PyPI — so overwrite rather than collide, matching the re-run safety the
+      # rest of this workflow is built for.
+      - name: Upload the distribution
+        uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+          overwrite: true
+
+  # Publish to PyPI by trusted publishing: no token exists to leak or rotate,
+  # and the `pypi` environment is what PyPI's publisher configuration is keyed
+  # to. A separate job from the build on purpose — the OIDC credential is never
+  # present in the job that produced the artifacts.
+  publish-pypi:
+    needs: [build]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment:
+      name: pypi
+      url: https://pypi.org/p/code2okf
+    permissions:
+      # Mint the OIDC token PyPI exchanges for an upload token. Nothing else.
+      id-token: write
+    steps:
+      - name: Install uv
+        uses: astral-sh/setup-uv@v9.0.0
+        with:
+          enable-cache: false
+
+      - name: Download the distribution
+        uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+
+      # `always` rather than uv's default `automatic`: with no credentials
+      # configured the default silently falls back and the upload fails later
+      # as an opaque 403, while `always` fails here, saying the OIDC exchange
+      # is what went wrong.
+      # The artifacts are named rather than left to uv's default `dist/*`:
+      # `uv build` also drops a .gitignore in there, and naming the two files
+      # keeps anything else that appears out of the upload.
+      - name: Publish to PyPI
+        run: uv publish --trusted-publishing always dist/*.tar.gz dist/*.whl
+
+  # Create the GitHub Release. A tag and a Release are different objects and
+  # pushing a tag never creates one, so without this the Releases page stays
+  # empty however many versions ship. The notes are the changelog's own section,
+  # so there is no second description of a release to drift.
+  github-release:
+    # verify for the version output, publish-pypi for the ordering: the assets
+    # attached here are the ones already on PyPI.
+    needs: [verify, publish-pypi]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      # The only job in this workflow that writes to the repository.
+      contents: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # Needed for CHANGELOG.md and the script. gh authenticates with
+          # GH_TOKEN below, not with the checkout credential.
+          persist-credentials: false
+
+      - name: Download the distribution
+        uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+
+      - name: Create the GitHub Release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG: ${{ github.ref_name }}
+          RELEASE_VERSION: ${{ needs.verify.outputs.version }}
+        run: |
+          set -euo pipefail
+          # Re-running a release is normal, so converge rather than fail: an
+          # existing Release keeps its notes -- they may have been edited by
+          # hand -- and only its assets are refreshed, which is what a re-run
+          # after a half-finished upload needs.
+          if gh release view "${TAG}" >/dev/null 2>&1; then
+            echo "NOTICE: release ${TAG} already exists — refreshing its assets only"
+            gh release upload "${TAG}" dist/* --clobber
+            exit 0
+          fi
+          # Read the notes first and only then create, rather than piping one
+          # into the other. In a pipeline both sides start together, so a
+          # failing extractor would hand gh an empty stdin and the Release
+          # would be created empty before pipefail reported the failure — and
+          # the existing-release check above would then skip it forever. The
+          # verify job makes that unreachable in practice; this makes it
+          # impossible.
+          notes="$(./scripts/release-notes.sh "${RELEASE_VERSION}")"
+          # --verify-tag refuses to invent a tag that does not exist, turning a
+          # typo into a failure instead of a stray tag.
+          printf '%s\n' "${notes}" |
+            gh release create "${TAG}" --title "${TAG}" --verify-tag --notes-file - dist/*