codecaliper 0.1.0__py3-none-any.whl

codecaliper/spec/rulings/index.toml

@@ -0,0 +1,54 @@
+# The versioned metric-to-syntax mapping specification (ARCHITECTURE.md §3).
+# SPEC semver is independent of the package version:
+#   patch = editorial; minor = new rulings, no existing number changes;
+#   major = any change that alters a number for any existing corpus case
+#           (including a grammar pin bump that does so).
+[spec]
+version = "1.1.0"
+
+[[changelog]]
+version = "1.1.0"
+summary = """
+Completions, no existing corpus value changes (MINOR): COG-JAVA-0001 added
+(labeled break/continue = flat +1 in both modes — the whitepaper's fundamental
+increment was silently missing for Java); TOK-ALL-0007 added (anonymous
+word tokens outside the keyword table are identifiers — Python `__future__`
+and Java contextual keywords previously fell through to OPERATOR, except
+`yield`, which the Java keyword table wrongly reserved, contradicting
+BW-JAVA-0001, so it lexed as KEYWORD); TOK-PY-0002 supersedes TOK-PY-0001
+(`concatenated_string` is descended: implicit-concatenation parts are separate
+STRING tokens and an interleaved comment is a COMMENT token — the superseded
+ruling swallowed such comments into one string token, undercounting comment
+lines); TOK-JAVA-0002 added (Java `_` is an IDENTIFIER token in every
+position — `underscore_pattern` declarator uses previously lexed as OTHER,
+while pattern/lambda uses already lexed as identifiers); TOK-PY-0003 added
+(Python `...`, the named `ellipsis` leaf, lexes as an OPERATOR token and is
+therefore visible to Halstead — previously it fell through to OTHER,
+undercounting `.pyi`-shaped stub bodies); the lloc statement
+tables were completed to match LOC-ALL-0004's stated scope (Python:
+future_import_statement, type_alias_statement, legacy print/exec; Java:
+package/import/interface/enum/record/annotation-type/constant declarations,
+static initializers, compact constructors, explicit constructor invocations,
+module declarations and directives). Numeric changes on inputs containing
+those constructs only (including a Halstead change on `...`-bearing inputs);
+every new ruling ships with its corpus case.
+"""
+
+[[changelog]]
+version = "1.0.0"
+summary = """
+First calibrated evolution, arbitrated by the pre-registered BW faithfulness
+experiment (validation/bw_faithfulness/derived/arbitration_report.md):
+TOK-ALL-0006 supersedes TOK-ALL-0004 (indentation tab = 8, was 1 — under
+tab=1 the indentation/readability correlation contradicted BW Fig. 9);
+BW-ALL-0007 added (BW token-family features use the full lexical stream,
+ERROR regions included — the construct is lexical; coverage on the original
+dataset 29/100 clean parses -> 100/100 full token streams). The operator-class
+arbitration returned a NULL result: BW-ALL-0006 unchanged. Numeric changes:
+indentation features on tab-indented lines; BW token features on parse-error
+inputs. Every active ruling is corpus-covered (the 1.0 gate).
+"""
+
+[[changelog]]
+version = "0.1.0"
+summary = "Seed ruling set: core policies, tokenization, CC/COG/HAL/MI/LOC, BW feature delimitation for Python + Java."

codecaliper/spec/rulings/loc.toml

@@ -0,0 +1,50 @@
+# LOC-*: line counting.
+
+[[ruling]]
+id = "LOC-ALL-0001"
+metric = "loc"
+language = "all"
+title = "Physical and blank lines"
+statement = """
+physical_lines = number of lines of the normalized text (a trailing final
+newline does not create an extra empty line). blank_lines = lines whose content
+strips to empty.
+"""
+examples = ["py-multiline-string-001", "py-blank-lines-001"]
+
+[[ruling]]
+id = "LOC-ALL-0002"
+metric = "loc"
+language = "all"
+title = "sloc: lines covered by at least one code token"
+statement = """
+sloc counts lines covered by the span of at least one non-comment token. A
+multi-line string is code: every line it spans counts. (This is where the old
+regex lane provably fails — ARCHITECTURE.md §6.)
+"""
+examples = ["py-multiline-string-001"]
+
+[[ruling]]
+id = "LOC-ALL-0003"
+metric = "loc"
+language = "all"
+title = "comment_lines: lines covered by at least one comment token"
+statement = """
+comment_lines counts lines covered by the span of at least one comment token;
+a multi-line block comment counts on EVERY line it spans. A line containing
+both code and a trailing comment counts toward both sloc and comment_lines
+(the two are coverage measures, not a partition).
+"""
+examples = ["java-block-comment-001", "py-multiline-string-001"]
+
+[[ruling]]
+id = "LOC-ALL-0004"
+metric = "loc"
+language = "all"
+title = "lloc: statement-classified nodes"
+statement = """
+lloc counts nodes in the adapter's statement table (simple statements plus
+compound-statement headers plus declarations). The table is part of the spec
+surface and validated against the grammar.
+"""
+examples = ["py-cc-boolop-001", "java-elseif-001"]

codecaliper/spec/rulings/mi.toml

@@ -0,0 +1,15 @@
+# MI-*: maintainability index.
+
+[[ruling]]
+id = "MI-ALL-0001"
+metric = "maintainability_index"
+language = "all"
+title = "MI variant: no comment term, clamped to 0-100"
+statement = """
+MI = clamp_0_100((171 - 5.2*ln(V) - 0.23*CC - 16.2*ln(SLOC)) * 100/171), the
+Visual Studio / radon convention without the comment term. MI is DERIVED from
+Halstead volume, cyclomatic complexity and SLOC — it is not independent of CC
+(ARCHITECTURE.md §13); every MI value carries a typed derived_from and a standing
+mi-contains-cc diagnostic.
+"""
+examples = ["py-cc-boolop-001"]

codecaliper/spec/rulings/tokenization.toml

@@ -0,0 +1,191 @@
+# TOK-*: input normalization and the unified lexical token stream.
+# These change emitted numbers, so they are spec, not implementation detail.
+
+[[ruling]]
+id = "TOK-ALL-0001"
+metric = "tokenization"
+language = "all"
+title = "Input is decoded as UTF-8 with replacement"
+statement = """
+Byte input is decoded as UTF-8; undecodable bytes are replaced (U+FFFD) and an
+encoding-replaced diagnostic is attached. str input is used as-is.
+"""
+examples = ["py-tok-normalize-001"]
+
+[[ruling]]
+id = "TOK-ALL-0002"
+metric = "tokenization"
+language = "all"
+title = "A leading UTF-8 BOM is stripped"
+statement = """
+A leading U+FEFF is stripped before measurement (with a bom-stripped
+diagnostic); otherwise it would pollute line-1 character and length features.
+"""
+examples = ["py-tok-normalize-001"]
+
+[[ruling]]
+id = "TOK-ALL-0003"
+metric = "tokenization"
+language = "all"
+title = "Line endings are normalized to LF"
+statement = """
+CRLF and lone CR are normalized to LF before measurement, so character-level
+features are platform-stable.
+"""
+examples = ["py-tok-normalize-001"]
+
+[[ruling]]
+id = "TOK-ALL-0004"
+metric = "tokenization"
+language = "all"
+title = "A tab counts as one indentation character (provisional)"
+status = "superseded"
+superseded_by = "TOK-ALL-0006"
+statement = """
+Indentation is measured as the count of leading whitespace characters; a tab
+counts as 1. PROVISIONAL: the BW faithfulness pipeline (§6.3) will arbitrate
+tab semantics empirically; any change supersedes this ruling.
+"""
+
+[[ruling]]
+id = "TOK-ALL-0005"
+metric = "tokenization"
+language = "all"
+title = "Token line attribution"
+statement = """
+Every lexical token is attributed to the physical line of its start position.
+A multi-line token (string, block comment) is ONE token on its start line for
+token-count features; line-coverage features (sloc, comment lines) use its full
+span instead.
+"""
+
+[[ruling]]
+id = "TOK-PY-0001"
+metric = "tokenization"
+language = "python"
+title = "Python atomic tokens: strings are consumed whole"
+status = "superseded"
+superseded_by = "TOK-PY-0002"
+statement = """
+`string` and `concatenated_string` subtrees are consumed as single STRING
+tokens. F-string interpolation contents therefore do NOT contribute token-level
+features (identifiers/operators inside interpolations are invisible). This
+diverges from CPython 3.12+ tokenize (PEP 701), which tokenizes interpolation
+contents; the divergence vs the stdlib reference extractor is classified, and
+uniformity with Java strings is preferred.
+"""
+node_types = ["string", "concatenated_string"]
+
+[[ruling]]
+id = "TOK-PY-0002"
+metric = "tokenization"
+language = "python"
+title = "Python atomic tokens: strings are consumed whole; implicit concatenation is descended"
+since_spec = "1.1.0"
+statement = """
+`string` subtrees are consumed as single STRING tokens. F-string interpolation
+contents therefore do NOT contribute token-level features (identifiers and
+operators inside interpolations are invisible); this diverges from CPython
+3.12+ tokenize (PEP 701), the divergence vs the stdlib reference extractor is
+classified, and uniformity with Java strings is preferred. A
+`concatenated_string` (implicit concatenation) is NOT atomic: it is descended,
+so each string part is its own STRING token and a comment interleaved between
+parts is an ordinary COMMENT token — matching CPython tokenize. Supersedes
+TOK-PY-0001, which consumed `concatenated_string` whole and thereby silently
+swallowed interleaved comments (undercounting comment lines and counting
+comment-only lines inside the concatenation as code).
+"""
+node_types = ["string", "concatenated_string"]
+examples = ["py-concat-string-001", "py-multiline-string-001"]
+
+[[ruling]]
+id = "TOK-PY-0003"
+metric = "tokenization"
+language = "python"
+title = "Python `...` is an operator token"
+since_spec = "1.1.0"
+statement = """
+The `...` literal (the grammar's named `ellipsis` leaf) lexes as an OPERATOR
+token. This matches CPython tokenize (which emits an OP token for `...`) and
+Java's varargs `...` (also OPERATOR), and keeps `...` visible to Halstead as
+an operator — consistent with Python's other singleton literals (None/True/
+False count as Halstead operators via keyword_leaf_types). Before 1.1.0 the
+named `ellipsis` leaf fell through to OTHER and was invisible to Halstead
+(systematically undercounting `.pyi` stub bodies, which are mostly `...`). BW
+token-family features are unaffected either way: `...` matches none of the
+counted operator spellings.
+"""
+node_types = ["ellipsis"]
+examples = ["py-ellipsis-001"]
+
+[[ruling]]
+id = "TOK-JAVA-0001"
+metric = "tokenization"
+language = "java"
+title = "Java atomic tokens: string and character literals"
+statement = """
+`string_literal` and `character_literal` subtrees are consumed as single STRING
+tokens (text blocks included).
+"""
+node_types = ["string_literal", "character_literal"]
+
+[[ruling]]
+id = "TOK-JAVA-0002"
+metric = "tokenization"
+language = "java"
+title = "Java `_` is an identifier token in every position"
+since_spec = "1.1.0"
+statement = """
+The reserved word `_` (JLS §3.9, reserved since SE 9) lexes as an IDENTIFIER
+token in every syntactic position. The grammar already yields `identifier`
+nodes for `_` in switch patterns and lambda parameters; the
+`underscore_pattern` node it yields for unnamed variable declarators is
+classified identically, so the same source token cannot silently change token
+class with syntactic position (before 1.1.0 it lexed as OTHER there —
+invisible to BW token-family features and Halstead). Mirrors BW-PY-0001,
+which counts Python's soft-keyword `_` as an identifier. BW-JAVA-0001's
+keyword table deliberately omits `_`.
+"""
+node_types = ["underscore_pattern"]
+examples = ["java-underscore-001"]
+
+[[ruling]]
+id = "TOK-ALL-0007"
+metric = "tokenization"
+language = "all"
+title = "Anonymous word tokens outside the keyword table are identifiers"
+since_spec = "1.1.0"
+statement = """
+A grammar-anonymous leaf token whose text is word-shaped (letter/underscore
+start, word characters, internal hyphens allowed) and not in the language's
+keyword table is an IDENTIFIER token: Python's `__future__`, Java's contextual
+keywords (record, sealed, non-sealed, permits, when, yield, module directives).
+This matches the reference tokenizers, which yield NAME for all of them, and
+keeps the keyword tables exactly the reserved-word sets (BW-PY-0001,
+BW-JAVA-0001). Before 1.1.0 these tokens fell through to OPERATOR — except
+Java's `yield`, which was wrongly in the keyword table (contradicting
+BW-JAVA-0001's stated set) and lexed as KEYWORD.
+"""
+examples = ["py-future-import-001", "java-contextual-001"]
+
+[[ruling]]
+id = "TOK-ALL-0006"
+metric = "tokenization"
+language = "all"
+title = "A tab counts as 8 indentation characters (arbitrated)"
+since_spec = "1.0.0"
+statement = """
+Indentation features count leading whitespace with a tab as 8 characters and a
+space as 1. Supersedes TOK-ALL-0004 (tab = 1), whose PROVISIONAL marker was
+resolved by the pre-registered 32-cell BW faithfulness arbitration
+(validation/bw_faithfulness/derived/arbitration_report.md, 2026-07): under
+tab = 1 the avg_indentation correlation with the human readability scores is
+zero (rho +0.001), contradicting the paper's Fig. 9; any expansion >= 2 fixes
+the sign in both extraction modes, and 8 won the pre-registered AUC tie-break
+(honest caveat, per the independent verification: 8-vs-4 is a deterministic
+convention pick within statistical noise at n=100 — both deliver the
+Fig. 9-matching rho ~ -0.25; 2 does not). Scope: indentation features only;
+line-length and space-count features remain raw character counts (a candidate
+for future arbitration, noted in the report).
+"""
+examples = ["py-tok-normalize-001"]

codecaliper/spec/validated_grammars.toml

@@ -0,0 +1,6 @@
+# The grammar versions this release is calibrated against (ARCHITECTURE.md §10).
+# A deviating installed version still runs, but every report is stamped
+# GrammarInfo.validated=false plus an unvalidated-grammar diagnostic.
+[grammars]
+python = "0.25.0"
+java = "0.23.5"

codecaliper/syntax/_treesitter.py

@@ -0,0 +1,179 @@
+"""The tree-sitter quarantine — the ONLY module that imports ``tree_sitter``.
+
+py-tree-sitter breaks its API across 0.x minors (0.22 removed
+``Language(path, name)``; 0.23 removed the Parser setters; 0.24/0.25 moved
+query execution to ``QueryCursor`` and deprecated ``Language.version`` for
+``abi_version``). Confining the import here bounds every future breakage to one
+file (ARCHITECTURE.md §4). The MVP deliberately avoids the Query API entirely —
+cursor walks suffice.
+"""
+
+from __future__ import annotations
+
+import importlib
+import importlib.metadata
+from collections.abc import Iterator
+from typing import Any
+
+from tree_sitter import Language, Node, Parser, Tree
+
+from codecaliper.errors import GrammarLoadError
+
+__all__ = ["Language", "Node", "Parser", "Tree", "load_language", "make_parser",
+           "walk", "leaves", "node_kinds", "count_error_nodes"]
+
+
+def load_language(grammar_module: str) -> tuple[Language, str, int]:
+    """Import a grammar wheel and wrap its PyCapsule.
+
+    Returns ``(language, installed_version, abi_version)``.
+    """
+    try:
+        mod = importlib.import_module(grammar_module)
+        lang = Language(mod.language())
+    except Exception as exc:  # noqa: BLE001 — surface as our typed error
+        raise GrammarLoadError(f"cannot load grammar module {grammar_module!r}: {exc}") from exc
+    dist = grammar_module.replace("_", "-")
+    try:
+        version = importlib.metadata.version(dist)
+    except importlib.metadata.PackageNotFoundError:
+        version = "unknown"
+    # version -> abi_version shim (abi_version since py-tree-sitter 0.25)
+    abi: int | None = getattr(lang, "abi_version", None)
+    if abi is None:  # pragma: no cover - older binding shim
+        abi = int(getattr(lang, "version", 0))
+    return lang, version, int(abi)
+
+
+def make_parser(language: Language) -> Parser:
+    return Parser(language)
+
+
+def walk(tree: Tree) -> Iterator[tuple[Node, int]]:
+    """Cursor-based preorder ``(node, depth)`` — O(1) memory, no recursion limit,
+    deterministic order."""
+    cursor = tree.walk()
+    depth = 0
+    while True:
+        node = cursor.node
+        assert node is not None  # a fresh walk cursor always sits on a node
+        yield node, depth
+        if cursor.goto_first_child():
+            depth += 1
+            continue
+        while True:
+            if cursor.goto_next_sibling():
+                break
+            if not cursor.goto_parent():
+                return
+            depth -= 1
+
+
+def is_opaque(node: Node) -> bool:
+    """ERROR/MISSING subtrees are opaque to all measurement (CORE-ALL-0002)."""
+    return node.type == "ERROR" or node.is_missing
+
+
+def leaves(tree: Tree, atomic_types: frozenset[str]) -> Iterator[Node]:
+    """Terminal-token stream: descend to leaves, but consume node types in
+    ``atomic_types`` (e.g. Python ``string`` with its start/content/end
+    children) as single opaque tokens (TOK-PY-0002 / TOK-JAVA-0001).
+
+    ERROR/MISSING subtrees yield nothing and are not descended into
+    (CORE-ALL-0002): their tokens must not feed Halstead/LOC or (on clean
+    parses) BW. On parse errors, BW token-family features instead use
+    :func:`leaves_lexical`, which descends into ERROR subtrees (BW-ALL-0007).
+    """
+    cursor = tree.walk()
+    while True:
+        node = cursor.node
+        assert node is not None  # a fresh walk cursor always sits on a node
+        descend = False
+        if not is_opaque(node):
+            if node.child_count == 0 or node.type in atomic_types:
+                yield node
+            else:
+                descend = True
+        if descend and cursor.goto_first_child():
+            continue
+        while True:
+            if cursor.goto_next_sibling():
+                break
+            if not cursor.goto_parent():
+                return
+
+
+def leaves_lexical(tree: Tree, atomic_types: frozenset[str]) -> Iterator[Node]:
+    """Full lexical leaf stream: like :func:`leaves`, but DESCENDS into ERROR
+    subtrees, yielding the tokens tree-sitter recovered inside them — the BW
+    construct is lexical and must see fragment tokens (BW-ALL-0007). MISSING
+    nodes are zero-width fabrications and still yield nothing; a childless
+    ERROR node (raw untokenized bytes) also yields nothing."""
+    cursor = tree.walk()
+    while True:
+        node = cursor.node
+        assert node is not None  # a fresh walk cursor always sits on a node
+        descend = False
+        if not node.is_missing:
+            if node.type == "ERROR":
+                descend = node.child_count > 0
+            elif node.child_count == 0 or node.type in atomic_types:
+                yield node
+            else:
+                descend = True
+        if descend and cursor.goto_first_child():
+            continue
+        while True:
+            if cursor.goto_next_sibling():
+                break
+            if not cursor.goto_parent():
+                return
+
+
+def node_kinds(language: Language) -> frozenset[str]:
+    """Grammar introspection: every node-kind string of the COMPILED grammar.
+
+    Used by tests/test_grammar_integrity.py to validate adapter tables and
+    ruling node_types against the actually-installed grammar, so a grammar node
+    rename fails loudly instead of silently zeroing metrics.
+    """
+    count: int = language.node_kind_count
+    kinds = set()
+    for i in range(count):
+        kind = language.node_kind_for_id(i)
+        if kind is not None:
+            kinds.add(kind)
+    return frozenset(kinds)
+
+
+def count_error_nodes(tree: Tree) -> int:
+    """Number of ERROR/MISSING nodes (parse-error policy CORE-ALL-0002)."""
+    n = 0
+    for node, _ in walk(tree):
+        if node.type == "ERROR" or node.is_missing:
+            n += 1
+    return n
+
+
+def node_text(node: Node, source: bytes) -> str:
+    return source[node.start_byte : node.end_byte].decode("utf-8", errors="replace")
+
+
+def field_name_of(node: Node) -> str | None:
+    """The field name this node occupies in its parent, if any."""
+    parent = node.parent
+    if parent is None:
+        return None
+    # tree-sitter exposes this via the parent's field iteration; the portable
+    # way across binding versions is child_by_field_name comparison.
+    for field in ("alternative", "consequence", "condition", "operator", "body", "name"):
+        try:
+            if parent.child_by_field_name(field) == node:
+                return field
+        except Exception:  # pragma: no cover - defensive against binding churn
+            return None
+    return None
+
+
+def language_of_parser(parser: Parser) -> Any:  # pragma: no cover - debugging aid
+    return parser.language

codecaliper/syntax/grammars.py

@@ -0,0 +1,45 @@
+"""Grammar loading and the calibration check (ARCHITECTURE.md §10).
+
+A deviating installed grammar version never refuses to run — it is stamped
+``GrammarInfo.validated=False`` and every report carries an
+``unvalidated-grammar`` diagnostic. Run and label, never refuse, never silently
+trust.
+"""
+
+from __future__ import annotations
+
+import sys
+from functools import cache, lru_cache
+from importlib import resources
+
+from codecaliper.model import GrammarInfo
+from codecaliper.syntax import _treesitter as ts
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:  # pragma: no cover - py3.10 fallback
+    import tomli as tomllib
+
+_GRAMMAR_MODULES = {"python": "tree_sitter_python", "java": "tree_sitter_java"}
+
+
+@lru_cache(maxsize=1)
+def validated_versions() -> dict[str, str]:
+    ref = resources.files("codecaliper.spec") / "validated_grammars.toml"
+    with ref.open("rb") as f:
+        return dict(tomllib.load(f)["grammars"])
+
+
+@cache
+def load(language_name: str) -> tuple[ts.Language, ts.Parser, GrammarInfo]:
+    module = _GRAMMAR_MODULES[language_name]
+    lang, version, abi = ts.load_language(module)
+    parser = ts.make_parser(lang)
+    info = GrammarInfo(
+        language=language_name,
+        package=module.replace("_", "-"),
+        version=version,
+        abi_version=abi,
+        validated=(validated_versions().get(language_name) == version),
+    )
+    return lang, parser, info