lang-parsing-substrate 0.5.1__tar.gz

lang_parsing_substrate-0.5.1/.github/workflows/build.yml

@@ -0,0 +1,103 @@
+name: Build and Test
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*.*.*'
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    name: Build and Test
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Extract version from Cargo.toml
+        id: version
+        run: |
+          VERSION=$(grep '^version = ' Cargo.toml | head -1 | sed 's/version = "\(.*\)"/\1/')
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          echo "Extracted version: $VERSION"
+
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy, rustfmt
+
+      - name: Verify Rust installation
+        run: |
+          rustc --version
+          cargo --version
+
+      - name: Cache cargo registry
+        uses: actions/cache@v4
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Cache cargo index
+        uses: actions/cache@v4
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Cache target directory
+        uses: actions/cache@v4
+        with:
+          path: target
+          key: ${{ runner.os }}-target-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Security Audit
+        run: |
+          cargo install cargo-audit
+          cargo audit
+        continue-on-error: true
+
+      - name: Clippy (all features)
+        run: cargo clippy --all-features -- -D warnings
+
+      - name: Format check
+        run: cargo fmt --check
+
+      - name: Build (all features)
+        run: cargo build --all-features
+
+      - name: Build (subset — C/C++ only)
+        run: cargo build --no-default-features --features lang-c,lang-cpp
+
+      - name: Build (no features)
+        run: cargo build --no-default-features
+
+      - name: Test (all features)
+        run: cargo test --all-features
+
+      - name: Test (subset — C/C++ only)
+        run: cargo test --no-default-features --features lang-c,lang-cpp
+
+      - name: Install cargo-about
+        run: cargo install cargo-about --features cli
+
+      - name: Generate Third-Party Licenses
+        run: cargo about generate about.hbs -o THIRD_PARTY_LICENSES.txt
+
+      - name: Install cargo-cyclonedx
+        run: cargo install cargo-cyclonedx
+
+      - name: Generate SBOM (CycloneDX)
+        run: cargo cyclonedx --format json --spec-version 1.5
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: lang-parsing-substrate-${{ steps.version.outputs.VERSION }}
+          path: |
+            THIRD_PARTY_LICENSES.txt
+            lang-parsing-substrate.cdx.json
+          retention-days: 30

lang_parsing_substrate-0.5.1/.github/workflows/wheels.yml

@@ -0,0 +1,121 @@
+name: Wheels (PyPI)
+
+# Build per-platform abi3 wheels + an sdist for the `lang-parsing-substrate`
+# pyo3 bindings with maturin, and publish them to PyPI via Trusted Publishing
+# (OIDC, no API token) on a v*.*.* tag.
+#
+# abi3-py310 (see Cargo.toml's pyo3 dependency) means ONE wheel per platform
+# covers every CPython 3.10+, not one per minor version — no Python-version
+# axis needed in the build matrix, unlike a non-abi3 pyo3 crate would need.
+#
+# Validate the matrix without publishing by running this workflow manually
+# (Actions tab -> "Wheels (PyPI)" -> Run workflow); publish only happens on tags.
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: build (${{ matrix.platform.id }})
+    runs-on: ${{ matrix.platform.runner }}
+    env:
+      # Passed into the manylinux/musl Docker containers via `--env CFLAGS`
+      # below — same fix build.yml already carries for its own native
+      # cargo-test builds. _BSD_SOURCE enables le16toh/be16toh on glibc 2.17
+      # (manylinux2014 predates _DEFAULT_SOURCE). -std=c11 is what
+      # tree-sitter-fortran's parser.c actually needs: manylinux's aarch64
+      # cross-compiler defaults to C89 and rejects its C99 `for`-loop
+      # declarations — confirmed live (aarch64 build failure, x86_64 fine) on
+      # the first run of this workflow before this was added.
+      CFLAGS: -D_BSD_SOURCE -std=c11
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          # `id` must be unique per row — macOS and Linux share the bare
+          # x86_64/aarch64 target labels, which would otherwise collide as
+          # artifact names (upload-artifact@v4 rejects duplicate names).
+          - { id: linux-x86_64,  runner: ubuntu-22.04, target: x86_64, manylinux: auto }
+          - { id: linux-aarch64, runner: ubuntu-22.04, target: aarch64, manylinux: auto }
+          - { id: musl-x86_64,   runner: ubuntu-22.04, target: x86_64-unknown-linux-musl, manylinux: musllinux_1_2 }
+          - { id: musl-aarch64,  runner: ubuntu-22.04, target: aarch64-unknown-linux-musl, manylinux: musllinux_1_2 }
+          # Both macOS arches build on the Apple-Silicon runner; x86_64 is
+          # cross-compiled. Avoids the deprecated/scarce macos-13 Intel runners.
+          - { id: macos-x86_64,  runner: macos-14, target: x86_64-apple-darwin }
+          - { id: macos-aarch64, runner: macos-14, target: aarch64 }
+          - { id: windows-x64,   runner: windows-2022, target: x64 }
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          command: build
+          target: ${{ matrix.platform.target }}
+          # No --features flag needed: pyproject.toml's [tool.maturin] already
+          # declares features = ["pyo3"], matching knots' own wheels.yml
+          # convention of letting pyproject.toml be the single source of the
+          # build's feature set.
+          args: --release --out dist
+          manylinux: ${{ matrix.platform.manylinux }}
+          sccache: 'true'
+          # Inherit CFLAGS from the job-level env (see above). Passing by name
+          # (no =value) lets docker copy it from the runner's environment,
+          # avoiding shell-quoting issues with multi-word values. Only
+          # containerized (linux) builds use docker-options; macOS/Windows
+          # ignore it, but the env var is harmless there too.
+          docker-options: --env CFLAGS
+
+      - name: Upload wheel artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ matrix.platform.id }}
+          path: dist/*.whl
+
+  sdist:
+    name: sdist
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+
+      - name: Upload sdist artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+
+  publish:
+    name: Publish to PyPI
+    needs: [build, sdist]
+    runs-on: ubuntu-22.04
+    if: startsWith(github.ref, 'refs/tags/v')
+    environment: release
+    permissions:
+      id-token: write          # required for PyPI Trusted Publishing (OIDC)
+    steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          merge-multiple: true   # flatten wheel-*/ and sdist/ into dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist
+          # Idempotent re-runs: a version that partially published (e.g. one
+          # platform's wheel failed) skips the files already on PyPI on retry
+          # instead of erroring on "file already exists".
+          skip-existing: true

lang_parsing_substrate-0.5.1/.gitignore

@@ -0,0 +1,6 @@
+/target
+Cargo.lock
+todo.db
+.todo-sqlite-cli
+__pycache__/
+*.pyc

lang_parsing_substrate-0.5.1/.pre-commit-config.yaml

@@ -0,0 +1,55 @@
+# Pre-commit hooks for lang-parsing-substrate
+# Install: pre-commit install
+# Run manually: pre-commit run --all-files
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+      - id: mixed-line-ending
+        args: [--fix=lf]
+
+  - repo: local
+    hooks:
+      - id: cargo-fmt
+        name: cargo fmt (auto-fix)
+        description: Format Rust code with rustfmt
+        entry: bash -c 'cargo fmt --all 2>&1 && echo "Formatting complete"'
+        language: system
+        pass_filenames: false
+        types: [rust]
+        verbose: true
+
+      - id: cargo-clippy
+        name: cargo clippy (all features)
+        description: Lint Rust code with Clippy - deny warnings (all targets, all 16 language features)
+        entry: bash -c 'cargo clippy --all-targets --all-features -- -D warnings 2>&1'
+        language: system
+        pass_filenames: false
+        types: [rust]
+        verbose: true
+
+      - id: cargo-test
+        name: cargo test (all features + lang-c,lang-cpp subset)
+        description: Run the test suite under the full feature set and the minimal C/C++ subset, matching CI
+        entry: bash -c 'cargo test --all-features 2>&1 && cargo test --no-default-features --features lang-c,lang-cpp 2>&1'
+        language: system
+        pass_filenames: false
+        types: [rust]
+        verbose: true
+
+  # knots complexity gate — prebuilt-wheel edition (language: python, pulls
+  # the `knots` wheel from PyPI instead of building from source). The default
+  # `knots` hook ships args [--aird-threshold=85], so no override is needed
+  # here. To tighten or add metrics, set `args:` (pre-commit replaces them
+  # wholesale), e.g. args: [--aird-threshold=85, --cognitive-threshold=20]
+  - repo: https://github.com/brandon-arrendondo/knots-pre-commit
+    rev: v1.13.1
+    hooks:
+      - id: knots
+        types: [rust]

lang_parsing_substrate-0.5.1/CLAUDE.md

@@ -0,0 +1,67 @@
+# lang-parsing-substrate — developer guide for Claude
+
+Shared Rust **library crate** — the common parsing substrate for knots, moldy, and tools_sqc.
+Provides language detection, tree-sitter grammar dispatch, and `LanguageInfo` registry across
+16 languages, compiled in at build time via Cargo feature flags.
+
+## Repository layout
+
+| Path | Purpose |
+|------|---------|
+| `src/lib.rs` | Crate root — module wiring, cfg-gated grammar re-exports (`pub use tree_sitter_*`) |
+| `src/registry.rs` | `LanguageInfo`, `SlocMode`, `languages()`, `language_for_file()`, `language_for_header_content()`, `language_info_for_file()`, `sloc_mode_for_file()`, `is_source_extension`, `is_parseable_extension`, `supported_languages_report()` |
+| `src/cpp_header.rs` | `looks_like_cpp()` — best-effort content sniff for `.h` (C vs C++), gated on `lang-c` + `lang-cpp` |
+| `Cargo.toml` | 16 optional `lang-*` features + `all-languages` + `default = ["all-languages"]` |
+| `tasks.py` | `invoke build / test / check / bump-version / publish / clean` |
+| `todo.db` | Task tracking — run `todo-sqlite-cli list` to see open work |
+
+## Key invariants
+
+- **`language_for_file` returns `Option<Language>`** — never a fallback. If a language feature is disabled, its extensions return `None`. It is extension-only (no I/O): `.h` always resolves to the C grammar regardless of content. Callers that already have the file's bytes and want best-effort C-vs-C++ disambiguation for `.h` should use `language_for_header_content(path, source)` instead (see `src/cpp_header.rs`) — it defers to `language_for_file` for every other extension and only overrides `.h` when the content contains an unambiguous C++-only construct.
+- **`languages()` is runtime-constructed** via `OnceLock<Vec<LanguageInfo>>`. It cannot be a `const` because its contents vary by compiled feature set. Do not attempt to make it `const`.
+- **Feature flags are the language gate** — every language is an optional Cargo dep. The `all-languages` feature enables all 16. Consumers use `default-features = false` to opt into a subset (e.g. tools_sqc only needs `lang-c,lang-cpp`).
+- **Grammar re-exports are cfg-gated** — `pub use tree_sitter_rust` is `#[cfg(feature = "lang-rust")]`. Consumers reach grammars transitively without their own direct deps.
+
+## Adding a new language
+
+1. **`Cargo.toml`** — add `tree-sitter-<lang> = { version = "...", optional = true }` and a `lang-<name>` feature under `[features]`. Add it to `all-languages`.
+2. **`src/registry.rs` — `languages()`** — add a `#[cfg(feature = "lang-<name>")] v.push(LanguageInfo { ... })` block.
+3. **`src/registry.rs` — `language_for_file()`** — add a `#[cfg(feature = "lang-<name>")] Some("ext" | ...) => Some(...LANGUAGE.into())` arm. Keep C last (it matches `.h` which could shadow other languages if placed first).
+4. **`src/lib.rs`** — add `#[cfg(feature = "lang-<name>")] pub use tree_sitter_<name>;`
+
+## Three-tier consumer model
+
+| Tool | Execution model | Cross-file features | Storage need |
+|------|----------------|--------------------|----|
+| knots | pre-commit or `--recursive` | OFF in single-file; ON in recursive | in-memory |
+| moldy | pre-commit or `--recursive` | OFF in single-file; ON in recursive | in-memory |
+| tools_sqc | always full-scan | always ON | SQLite (path+mtime keyed) |
+
+Cross-file features (Tier 2: import graph, call graph; Tier 3: CFG; Tier 4: pattern matching)
+are not yet implemented — see `todo.db` for open tasks.
+
+## Substrate capability tiers
+
+| Tier | Content | Status |
+|------|---------|--------|
+| 1 | Parse layer — language detection, tree-sitter dispatch, source bytes | **Done** |
+| 2 | Graph layer — import graph (Ce/Ca/Instability), call graph | Pending |
+| 3 | Control flow / basic blocks | Pending |
+| 4 | Pattern matching — generalized rule engine (from tools_sqc) | Pending |
+| 5 | Fingerprinting / similarity (code dedup) | Speculative |
+
+## Related projects
+
+- `../knots/` — complexity metrics tool; CLAUDE.md there is the knots developer guide
+- `../moldy/` — formatting tool
+- `../tools_sqc/` — CERT-C compliance tool; has the rule engine that becomes Tier 4
+
+## Fixed-form Fortran dependency note
+
+Fixed-form Fortran (`.f`/`.for`/`.f77`) support was **dropped from `lang-fortran`** ahead of the
+v0.1.0 crates.io release — `cargo publish` rejects any manifest containing a `git` dependency,
+even on a fully optional feature, and `tree-sitter-fixed-form-fortran` only exists as a git dep on
+the `brandon-arrendondo/tree-sitter-fixed-form-fortran` fork. `lang-fortran` now covers only
+free-form Fortran (`tree-sitter-fortran`, a real crates.io dep). Re-add fixed-form once
+`stadelmanma/tree-sitter-fixed-form-fortran` merges PR #4 (real crates.io version) or the fork is
+itself published to crates.io.