codeecho 1.0.0__py3-none-any.whl

codeecho/normalizer.py

@@ -0,0 +1,519 @@
+"""
+Token extraction and Type-2 normalisation for code fragments.
+
+Tokens are extracted from the raw fragment source text using a language-aware
+regex-based tokeniser.  This avoids direct tree-sitter AST node traversal,
+which is unstable across tree-sitter 0.26+ Python bindings.
+
+Identifiers and literals are replaced with sequential placeholders
+(``ID_0``, ``ID_1``, ``LIT_0``, …) so that two fragments with the same structure
+but different variable names still produce the same normalised token sequence.
+
+:author: Ron Webb
+:since: 1.0.0
+"""
+
+import re
+import logging
+from dataclasses import dataclass, field
+
+from tree_sitter import Node
+
+_logger = logging.getLogger("codeecho.normalizer")
+
+
+@dataclass
+class NormalisationProfile:
+    """Language-specific sets of identifier and literal node types."""
+
+    identifier_types: frozenset[str] = field(default_factory=frozenset)
+    literal_types: frozenset[str] = field(default_factory=frozenset)
+
+
+_PROFILES: dict[str, NormalisationProfile] = {
+    "Python": NormalisationProfile(
+        identifier_types=frozenset({"identifier", "type_identifier"}),
+        literal_types=frozenset(
+            {"string", "integer", "float", "true", "false", "none"}
+        ),
+    ),
+    "JavaScript": NormalisationProfile(
+        identifier_types=frozenset(
+            {"identifier", "property_identifier", "shorthand_property_identifier"}
+        ),
+        literal_types=frozenset(
+            {"string", "number", "true", "false", "null", "template_string"}
+        ),
+    ),
+    "TypeScript": NormalisationProfile(
+        identifier_types=frozenset(
+            {
+                "identifier",
+                "property_identifier",
+                "type_identifier",
+                "shorthand_property_identifier",
+            }
+        ),
+        literal_types=frozenset(
+            {"string", "number", "true", "false", "null", "template_string"}
+        ),
+    ),
+    "Java": NormalisationProfile(
+        identifier_types=frozenset({"identifier", "type_identifier"}),
+        literal_types=frozenset(
+            {
+                "string_literal",
+                "decimal_integer_literal",
+                "decimal_floating_point_literal",
+                "true",
+                "false",
+                "null_literal",
+            }
+        ),
+    ),
+    "Gosu": NormalisationProfile(
+        identifier_types=frozenset({"identifier", "type_identifier"}),
+        literal_types=frozenset(
+            {
+                "string_literal",
+                "decimal_integer_literal",
+                "decimal_floating_point_literal",
+                "true",
+                "false",
+                "null_literal",
+            }
+        ),
+    ),
+    "Go": NormalisationProfile(
+        identifier_types=frozenset(
+            {"identifier", "type_identifier", "field_identifier"}
+        ),
+        literal_types=frozenset(
+            {
+                "interpreted_string_literal",
+                "raw_string_literal",
+                "int_literal",
+                "float_literal",
+                "true",
+                "false",
+                "nil",
+            }
+        ),
+    ),
+}
+
+_FALLBACK_PROFILE: NormalisationProfile = NormalisationProfile()
+
+
+# ── Regex-based tokeniser ───────────────────────────────────────────────────
+# Matches identifiers/keywords, number literals, quoted strings, and single
+# non-whitespace characters (operators, punctuation).
+_TOKEN_RE: re.Pattern[str] = re.compile(
+    r'"(?:[^"\\]|\\.)*"'  # double-quoted string
+    r"|'(?:[^'\\]|\\.)*'"  # single-quoted string
+    r"|`(?:[^`\\]|\\.)*`"  # backtick string (JS/Go)
+    r"|//[^\n]*"  # single-line comment
+    r"|/\*.*?\*/"  # multi-line comment (non-greedy)
+    r"|[0-9]+(?:\.[0-9]+)?"  # integer or float literal
+    r"|[a-zA-Z_$][a-zA-Z0-9_$]*"  # identifier / keyword
+    r"|[^\s]",  # any other single non-whitespace char
+    re.DOTALL,
+)
+
+# Per-language keyword sets used to distinguish identifiers from keywords
+_KEYWORDS: dict[str, frozenset[str]] = {
+    "Python": frozenset(
+        {
+            "False",
+            "None",
+            "True",
+            "and",
+            "as",
+            "assert",
+            "async",
+            "await",
+            "break",
+            "class",
+            "continue",
+            "def",
+            "del",
+            "elif",
+            "else",
+            "except",
+            "finally",
+            "for",
+            "from",
+            "global",
+            "if",
+            "import",
+            "in",
+            "is",
+            "lambda",
+            "nonlocal",
+            "not",
+            "or",
+            "pass",
+            "raise",
+            "return",
+            "try",
+            "while",
+            "with",
+            "yield",
+        }
+    ),
+    "JavaScript": frozenset(
+        {
+            "break",
+            "case",
+            "catch",
+            "class",
+            "const",
+            "continue",
+            "debugger",
+            "default",
+            "delete",
+            "do",
+            "else",
+            "export",
+            "extends",
+            "false",
+            "finally",
+            "for",
+            "function",
+            "if",
+            "import",
+            "in",
+            "instanceof",
+            "let",
+            "new",
+            "null",
+            "return",
+            "static",
+            "super",
+            "switch",
+            "this",
+            "throw",
+            "true",
+            "try",
+            "typeof",
+            "undefined",
+            "var",
+            "void",
+            "while",
+            "with",
+            "yield",
+        }
+    ),
+    "TypeScript": frozenset(
+        {
+            "abstract",
+            "any",
+            "as",
+            "asserts",
+            "async",
+            "await",
+            "break",
+            "case",
+            "catch",
+            "class",
+            "const",
+            "continue",
+            "debugger",
+            "declare",
+            "default",
+            "delete",
+            "do",
+            "else",
+            "enum",
+            "export",
+            "extends",
+            "false",
+            "finally",
+            "for",
+            "from",
+            "function",
+            "if",
+            "implements",
+            "import",
+            "in",
+            "infer",
+            "instanceof",
+            "interface",
+            "is",
+            "keyof",
+            "let",
+            "module",
+            "namespace",
+            "never",
+            "new",
+            "null",
+            "of",
+            "override",
+            "private",
+            "protected",
+            "public",
+            "readonly",
+            "return",
+            "satisfies",
+            "static",
+            "super",
+            "switch",
+            "this",
+            "throw",
+            "true",
+            "try",
+            "type",
+            "typeof",
+            "undefined",
+            "unique",
+            "unknown",
+            "var",
+            "void",
+            "while",
+            "with",
+            "yield",
+        }
+    ),
+    "Java": frozenset(
+        {
+            "abstract",
+            "assert",
+            "boolean",
+            "break",
+            "byte",
+            "case",
+            "catch",
+            "char",
+            "class",
+            "const",
+            "continue",
+            "default",
+            "do",
+            "double",
+            "else",
+            "enum",
+            "extends",
+            "final",
+            "finally",
+            "float",
+            "for",
+            "goto",
+            "if",
+            "implements",
+            "import",
+            "instanceof",
+            "int",
+            "interface",
+            "long",
+            "native",
+            "new",
+            "null",
+            "package",
+            "private",
+            "protected",
+            "public",
+            "return",
+            "short",
+            "static",
+            "strictfp",
+            "super",
+            "switch",
+            "synchronized",
+            "this",
+            "throw",
+            "throws",
+            "transient",
+            "true",
+            "try",
+            "var",
+            "void",
+            "volatile",
+            "while",
+        }
+    ),
+    "Gosu": frozenset(
+        {
+            "abstract",
+            "as",
+            "assert",
+            "block",
+            "break",
+            "case",
+            "catch",
+            "class",
+            "classpath",
+            "continue",
+            "default",
+            "do",
+            "else",
+            "enum",
+            "erases",
+            "eval",
+            "exists",
+            "extends",
+            "false",
+            "final",
+            "finally",
+            "for",
+            "foreach",
+            "function",
+            "hiding",
+            "if",
+            "implements",
+            "import",
+            "in",
+            "index",
+            "interface",
+            "new",
+            "null",
+            "override",
+            "package",
+            "property",
+            "protected",
+            "public",
+            "readonly",
+            "return",
+            "static",
+            "super",
+            "switch",
+            "this",
+            "throw",
+            "throws",
+            "transient",
+            "true",
+            "try",
+            "unless",
+            "using",
+            "var",
+            "void",
+            "where",
+            "while",
+        }
+    ),
+    "Go": frozenset(
+        {
+            "break",
+            "case",
+            "chan",
+            "const",
+            "continue",
+            "default",
+            "defer",
+            "else",
+            "fallthrough",
+            "false",
+            "for",
+            "func",
+            "go",
+            "goto",
+            "if",
+            "import",
+            "interface",
+            "map",
+            "nil",
+            "package",
+            "range",
+            "return",
+            "select",
+            "struct",
+            "switch",
+            "true",
+            "type",
+            "var",
+        }
+    ),
+}
+
+# Patterns that indicate the token is a string/number literal
+_STRING_RE: re.Pattern[str] = re.compile(r"""^["'`0-9]""")
+
+
+def get_profile(language_name: str) -> NormalisationProfile:
+    """Return the :class:`NormalisationProfile` for *language_name*, falling back to empty sets."""
+    return _PROFILES.get(language_name, _FALLBACK_PROFILE)
+
+
+def _tokenise_text(text: str) -> list[str]:
+    """Split *text* into a list of raw tokens using :data:`_TOKEN_RE`."""
+    return _TOKEN_RE.findall(text)
+
+
+def _node_text(node: Node, source_bytes: bytes) -> str:
+    """Return the UTF-8 decoded source text for the byte range covered by *node*."""
+    return source_bytes[node.start_byte : node.end_byte].decode(
+        "utf-8", errors="replace"
+    )
+
+
+def tokenise_and_normalise(
+    text: str, language_name: str
+) -> tuple[list[str], list[str]]:
+    """Return ``(raw_tokens, normalised_tokens)`` for a fragment source *text* string.
+
+    Args:
+        text: Raw fragment source text (UTF-8 decoded).
+        language_name: Language whose keyword set governs normalisation.
+
+    Returns:
+        Raw token list and normalised token list where identifiers become
+        ``ID_N`` and literals become ``LIT_N``.
+    """
+    raw_tokens = _tokenise_text(text)
+    keywords = _KEYWORDS.get(language_name, frozenset())
+    return raw_tokens, _normalise(raw_tokens, keywords)
+
+
+def extract_tokens(node: Node, source_bytes: bytes) -> list[str]:
+    """Return a flat list of raw tokens for the source text covered by *node*.
+
+    Uses a regex-based tokeniser on the raw source slice — no tree traversal.
+    """
+    return _tokenise_text(_node_text(node, source_bytes))
+
+
+def extract_and_normalise(
+    node: Node, source_bytes: bytes, language_name: str
+) -> tuple[list[str], list[str]]:
+    """Return ``(raw_tokens, normalised_tokens)`` for the source covered by *node*.
+
+    Args:
+        node: Tree-sitter node; only its byte range is used (no child traversal).
+        source_bytes: Full source bytes of the file.
+        language_name: Language whose keyword set governs normalisation.
+
+    Returns:
+        Raw token list and normalised token list where identifiers become
+        ``ID_N`` and literals become ``LIT_N``.
+    """
+    return tokenise_and_normalise(_node_text(node, source_bytes), language_name)
+
+
+def _normalise(raw_tokens: list[str], keywords: frozenset[str]) -> list[str]:
+    """Build a normalised token list from *raw_tokens* using *keywords* for classification."""
+    norm_tokens: list[str] = []
+    id_map: dict[str, str] = {}
+    lit_count = 0
+
+    for token in raw_tokens:
+        if _is_comment(token):
+            continue  # skip comments
+        if _is_identifier(token) and token not in keywords:
+            placeholder = id_map.setdefault(token, f"ID_{len(id_map)}")
+            norm_tokens.append(placeholder)
+        elif _STRING_RE.match(token):
+            norm_tokens.append(f"LIT_{lit_count}")
+            lit_count += 1
+        else:
+            norm_tokens.append(token)
+
+    return norm_tokens
+
+
+def _is_identifier(token: str) -> bool:
+    """Return True if *token* looks like an identifier (letter/underscore start)."""
+    return bool(token) and (token[0].isalpha() or token[0] == "_")
+
+
+def _is_comment(token: str) -> bool:
+    """Return True if *token* is a C-style single-line (``//``) or multi-line (``/* */``) comment."""
+    return token.startswith("//") or (token.startswith("/*") and token.endswith("*/"))

codeecho/parser.py

@@ -0,0 +1,94 @@
+"""
+Tree-sitter parser wrapper providing one cached :class:`tree_sitter.Parser` per language.
+
+Gosu (``.gs``) files are parsed using the Java grammar because Gosu is syntactically
+Java-like (methods, classes, blocks).
+
+:author: Ron Webb
+:since: 1.0.0
+"""
+
+import logging
+from functools import lru_cache
+
+from tree_sitter import Language, Parser, Tree
+
+_logger = logging.getLogger("codeecho.parser")
+
+
+def _build_language(
+    language_name: str,
+) -> Language | None:  # pylint: disable=too-many-return-statements
+    """Instantiate and return the tree-sitter :class:`Language` for *language_name*."""
+    match language_name:
+        case "Python":
+            import tree_sitter_python as m  # pylint: disable=import-outside-toplevel
+
+            return Language(m.language())
+        case "JavaScript":
+            import tree_sitter_javascript as m  # pylint: disable=import-outside-toplevel
+
+            return Language(m.language())
+        case "TypeScript" | "TSX":
+            import tree_sitter_typescript as m  # pylint: disable=import-outside-toplevel
+
+            return Language(m.language_typescript())
+        case "Java" | "Gosu":
+            import tree_sitter_java as m  # pylint: disable=import-outside-toplevel
+
+            return Language(m.language())
+        case "Go":
+            import tree_sitter_go as m  # pylint: disable=import-outside-toplevel
+
+            return Language(m.language())
+        case _:
+            return None
+
+
+@lru_cache(maxsize=None)
+def get_language(language_name: str) -> Language | None:
+    """Return a cached :class:`tree_sitter.Language` for *language_name*, or ``None`` on failure."""
+    try:
+        lang = _build_language(language_name)
+    except Exception as exc:  # pylint: disable=broad-exception-caught
+        _logger.warning("Failed to load grammar for %s: %s", language_name, exc)
+        return None
+    if lang is None:
+        _logger.warning("No tree-sitter grammar for language: %s", language_name)
+    return lang
+
+
+@lru_cache(maxsize=None)
+def get_parser(language_name: str) -> Parser | None:
+    """Return a cached :class:`tree_sitter.Parser` for *language_name*, or ``None`` on failure.
+
+    .. note::
+        The returned parser is NOT used directly for parsing; call :func:`parse` instead,
+        which creates a fresh :class:`~tree_sitter.Parser` instance per call to avoid
+        internal state corruption across files in tree-sitter 0.26+.
+    """
+    lang = get_language(language_name)
+    if lang is None:
+        return None
+    return Parser(lang)
+
+
+def parse(source_bytes: bytes, language_name: str) -> Tree | None:
+    """Parse *source_bytes* with the grammar for *language_name*.
+
+    A new :class:`~tree_sitter.Parser` is created for each call so that the
+    internal state of the C extension cannot carry over between files.
+
+    :param source_bytes: UTF-8-encoded source code.
+    :param language_name: Name as returned by :data:`codeecho.scanner.EXTENSION_TO_LANGUAGE`.
+    :returns: Parsed :class:`tree_sitter.Tree`, or ``None`` if the grammar is unavailable.
+    """
+    lang = get_language(language_name)
+    if lang is None:
+        _logger.warning("Skipping parse — no grammar for %s", language_name)
+        return None
+    fresh_parser = Parser(lang)
+    tree = fresh_parser.parse(source_bytes)
+    if tree.root_node.has_error:
+        _logger.debug("Parse errors detected in %s source.", language_name)
+    return tree

codeecho/reporter/__init__.py

@@ -0,0 +1,6 @@
+"""
+Reporter sub-package for codeecho: JSON and HTML output from session database results.
+
+:author: Ron Webb
+:since: 1.0.0
+"""