token-savior-recall 2.0.0__py3-none-any.whl

token_savior/__init__.py

@@ -0,0 +1,3 @@
+"""Token Savior — structural code indexer with MCP server for AI-assisted development."""
+
+__version__ = "3.0.0"

token_savior/annotator.py

@@ -0,0 +1,113 @@
+"""Dispatch layer that selects the appropriate annotator by file type."""
+
+from token_savior.c_annotator import annotate_c
+from token_savior.conf_annotator import annotate_conf
+from token_savior.dockerfile_annotator import annotate_dockerfile
+from token_savior.csharp_annotator import annotate_csharp
+from token_savior.env_annotator import annotate_env
+from token_savior.generic_annotator import annotate_generic
+from token_savior.go_annotator import annotate_go
+from token_savior.hcl_annotator import annotate_hcl
+from token_savior.ini_annotator import annotate_ini
+from token_savior.json_annotator import annotate_json
+from token_savior.models import AnnotatorProtocol, StructuralMetadata
+from token_savior.python_annotator import annotate_python
+from token_savior.rust_annotator import annotate_rust
+from token_savior.text_annotator import annotate_text
+from token_savior.toml_annotator import annotate_toml
+from token_savior.typescript_annotator import annotate_typescript
+from token_savior.xml_annotator import annotate_xml
+from token_savior.yaml_annotator import annotate_yaml
+
+_EXTENSION_MAP: dict[str, str] = {
+    ".c": "c",
+    ".h": "c",
+    ".glsl": "c",
+    ".vert": "c",
+    ".frag": "c",
+    ".comp": "c",
+    ".py": "python",
+    ".pyw": "python",
+    ".md": "text",
+    ".txt": "text",
+    ".rst": "text",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".go": "go",
+    ".rs": "rust",
+    ".cs": "csharp",
+    ".json": "json",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".toml": "toml",
+    ".ini": "ini",
+    ".cfg": "ini",
+    ".properties": "ini",
+    ".env": "env",
+    ".xml": "xml",
+    ".plist": "xml",
+    ".svg": "xml",
+    ".xhtml": "xml",
+    ".hcl": "hcl",
+    ".tf": "hcl",
+    ".conf": "conf",
+}
+
+
+_ANNOTATOR_MAP: dict[str, AnnotatorProtocol] = {
+    "c": annotate_c,
+    "python": annotate_python,
+    "text": annotate_text,
+    "typescript": annotate_typescript,
+    "javascript": annotate_typescript,
+    "go": annotate_go,
+    "rust": annotate_rust,
+    "csharp": annotate_csharp,
+    "json": annotate_json,
+    "yaml": annotate_yaml,
+    "ini": annotate_ini,
+    "xml": annotate_xml,
+    "hcl": annotate_hcl,
+    "toml": annotate_toml,
+    "env": annotate_env,
+    "conf": annotate_conf,
+    "dockerfile": annotate_dockerfile,
+}
+
+
+def annotate(
+    text: str,
+    source_name: str = "<source>",
+    file_type: str | None = None,
+) -> StructuralMetadata:
+    """Annotate text with structural metadata.
+
+    Dispatch rules:
+    - file_type overrides extension-based detection
+    - Extension -> _EXTENSION_MAP -> language name -> _ANNOTATOR_MAP -> annotator
+    - Dockerfile detected by filename pattern
+    - Otherwise -> generic annotator (line-only)
+    """
+    if file_type is None:
+        import os as _os
+
+        _basename = _os.path.basename(source_name)
+        if (
+            _basename == "Dockerfile"
+            or _basename.lower() == "dockerfile"
+            or _basename.startswith("Dockerfile.")
+            or _basename.lower().endswith(".dockerfile")
+        ):
+            file_type = "dockerfile"
+        else:
+            dot_idx = source_name.rfind(".")
+            if dot_idx >= 0:
+                ext = source_name[dot_idx:].lower()
+                file_type = _EXTENSION_MAP.get(ext)
+
+    annotator = _ANNOTATOR_MAP.get(file_type, annotate_generic)
+    if not callable(annotator):
+        raise TypeError(f"Annotator for '{file_type}' is not callable: {annotator!r}")
+    return annotator(text, source_name)

token_savior/brace_matcher.py

@@ -0,0 +1,367 @@
+"""Shared brace-matching helpers for C-family annotators.
+
+Each function finds the 0-based line where the outermost ``{`` closes,
+correctly skipping language-specific strings, char literals, and comments.
+The implementations are intentionally separate because the lexical rules
+differ significantly between languages (nested comments in Rust, verbatim
+strings in C#, backtick raw strings in Go, etc.).
+"""
+
+from __future__ import annotations
+
+
+def find_brace_end_c(lines: list[str], start_line_0: int) -> int:
+    """Find the 0-based line where the outermost brace closes,
+    skipping strings, char literals, and comments."""
+    depth = 0
+    found_open = False
+    in_block_comment = False
+    for idx in range(start_line_0, len(lines)):
+        line = lines[idx]
+        i = 0
+        while i < len(line):
+            ch = line[i]
+            # Block comment handling (C does NOT nest /* */)
+            if in_block_comment:
+                if ch == "*" and i + 1 < len(line) and line[i + 1] == "/":
+                    in_block_comment = False
+                    i += 2
+                    continue
+                i += 1
+                continue
+            # Line comment
+            if ch == "/" and i + 1 < len(line):
+                if line[i + 1] == "/":
+                    break  # rest is line comment
+                if line[i + 1] == "*":
+                    in_block_comment = True
+                    i += 2
+                    continue
+            # String literal
+            if ch == '"':
+                i += 1
+                while i < len(line):
+                    if line[i] == "\\":
+                        i += 2
+                        continue
+                    if line[i] == '"':
+                        i += 1
+                        break
+                    i += 1
+                continue
+            # Char literal
+            if ch == "'":
+                i += 1
+                if i < len(line) and line[i] == "\\":
+                    i += 2  # skip escaped char
+                elif i < len(line):
+                    i += 1  # skip char
+                if i < len(line) and line[i] == "'":
+                    i += 1
+                continue
+            if ch == "{":
+                depth += 1
+                found_open = True
+            elif ch == "}":
+                depth -= 1
+                if found_open and depth == 0:
+                    return idx
+            i += 1
+    return len(lines) - 1
+
+
+def find_brace_end_csharp(lines: list[str], start_line_0: int) -> int:
+    """Find the 0-based line where the outermost brace closes,
+    skipping strings, verbatim strings, interpolated strings, char literals, and comments."""
+    depth = 0
+    found_open = False
+    in_block_comment = False
+    for idx in range(start_line_0, len(lines)):
+        line = lines[idx]
+        i = 0
+        while i < len(line):
+            ch = line[i]
+            # Block comment handling
+            if in_block_comment:
+                if ch == "*" and i + 1 < len(line) and line[i + 1] == "/":
+                    in_block_comment = False
+                    i += 2
+                    continue
+                i += 1
+                continue
+            # Line comment
+            if ch == "/" and i + 1 < len(line):
+                if line[i + 1] == "/":
+                    break  # rest is line comment
+                if line[i + 1] == "*":
+                    in_block_comment = True
+                    i += 2
+                    continue
+            # Verbatim/interpolated strings: $@"...", @$"...", @"...", $"..."
+            if ch in ("@", "$") and i + 1 < len(line):
+                # Check for $@" or @$" (interpolated verbatim)
+                if (
+                    ch == "$" and line[i + 1] == "@" and i + 2 < len(line) and line[i + 2] == '"'
+                ) or (
+                    ch == "@" and line[i + 1] == "$" and i + 2 < len(line) and line[i + 2] == '"'
+                ):
+                    # Interpolated verbatim string — "" for escaped quote
+                    i += 3
+                    while i < len(line):
+                        if line[i] == '"':
+                            if i + 1 < len(line) and line[i + 1] == '"':
+                                i += 2
+                                continue
+                            i += 1
+                            break
+                        i += 1
+                    else:
+                        # Multi-line verbatim string
+                        idx += 1
+                        while idx < len(lines):
+                            line = lines[idx]
+                            i = 0
+                            while i < len(line):
+                                if line[i] == '"':
+                                    if i + 1 < len(line) and line[i + 1] == '"':
+                                        i += 2
+                                        continue
+                                    i += 1
+                                    break
+                                i += 1
+                            else:
+                                idx += 1
+                                continue
+                            break
+                        else:
+                            return len(lines) - 1
+                    continue
+                # Verbatim string: @"..."
+                if ch == "@" and line[i + 1] == '"':
+                    i += 2
+                    while i < len(line):
+                        if line[i] == '"':
+                            if i + 1 < len(line) and line[i + 1] == '"':
+                                i += 2
+                                continue
+                            i += 1
+                            break
+                        i += 1
+                    else:
+                        # Multi-line verbatim
+                        idx += 1
+                        while idx < len(lines):
+                            line = lines[idx]
+                            i = 0
+                            while i < len(line):
+                                if line[i] == '"':
+                                    if i + 1 < len(line) and line[i + 1] == '"':
+                                        i += 2
+                                        continue
+                                    i += 1
+                                    break
+                                i += 1
+                            else:
+                                idx += 1
+                                continue
+                            break
+                        else:
+                            return len(lines) - 1
+                    continue
+                # Interpolated string: $"..."
+                if ch == "$" and line[i + 1] == '"':
+                    i += 2
+                    while i < len(line):
+                        if line[i] == "\\":
+                            i += 2
+                            continue
+                        if line[i] == '"':
+                            i += 1
+                            break
+                        i += 1
+                    continue
+            # Regular string
+            if ch == '"':
+                i += 1
+                while i < len(line):
+                    if line[i] == "\\":
+                        i += 2
+                        continue
+                    if line[i] == '"':
+                        i += 1
+                        break
+                    i += 1
+                continue
+            # Char literal
+            if ch == "'" and i + 1 < len(line):
+                if i + 2 < len(line) and line[i + 1] == "\\":
+                    end = line.find("'", i + 2)
+                    if end >= 0 and end <= i + 4:
+                        i = end + 1
+                        continue
+                elif i + 2 < len(line) and line[i + 2] == "'":
+                    i += 3
+                    continue
+            if ch == "{":
+                depth += 1
+                found_open = True
+            elif ch == "}":
+                depth -= 1
+                if found_open and depth == 0:
+                    return idx
+            i += 1
+    return len(lines) - 1
+
+
+def find_brace_end_rust(lines: list[str], start_line_0: int) -> int:
+    """Find the 0-based line where the outermost brace closes,
+    skipping strings, raw strings, char literals, and comments."""
+    depth = 0
+    found_open = False
+    in_block_comment = 0  # nesting depth for /* */
+    for idx in range(start_line_0, len(lines)):
+        line = lines[idx]
+        i = 0
+        while i < len(line):
+            ch = line[i]
+            # Block comment handling (Rust supports nested /* */)
+            if in_block_comment > 0:
+                if ch == "/" and i + 1 < len(line) and line[i + 1] == "*":
+                    in_block_comment += 1
+                    i += 2
+                    continue
+                if ch == "*" and i + 1 < len(line) and line[i + 1] == "/":
+                    in_block_comment -= 1
+                    i += 2
+                    continue
+                i += 1
+                continue
+            # Line comment
+            if ch == "/" and i + 1 < len(line):
+                if line[i + 1] == "/":
+                    break  # rest is line comment
+                if line[i + 1] == "*":
+                    in_block_comment += 1
+                    i += 2
+                    continue
+            # Raw string: r#"..."#, r##"..."##, etc.
+            if ch == "r" and i + 1 < len(line) and line[i + 1] in ('"', "#"):
+                hash_count = 0
+                j = i + 1
+                while j < len(line) and line[j] == "#":
+                    hash_count += 1
+                    j += 1
+                if j < len(line) and line[j] == '"':
+                    j += 1
+                    # Find closing "###
+                    closing = '"' + "#" * hash_count
+                    while True:
+                        pos = line.find(closing, j)
+                        if pos >= 0:
+                            i = pos + len(closing)
+                            break
+                        # Span to next line
+                        idx += 1
+                        if idx >= len(lines):
+                            return len(lines) - 1
+                        line = lines[idx]
+                        j = 0
+                    continue
+            # Regular string
+            if ch == '"':
+                i += 1
+                while i < len(line):
+                    if line[i] == "\\":
+                        i += 2
+                        continue
+                    if line[i] == '"':
+                        i += 1
+                        break
+                    i += 1
+                continue
+            # Char literal (skip 'a', '\n', etc. but not lifetime 'a)
+            if ch == "'" and i + 1 < len(line):
+                # Lifetime check: 'a where next is alpha and followed by non-'
+                # Char literal: 'x' or '\n'
+                if i + 2 < len(line) and line[i + 1] == "\\":
+                    # Escaped char literal like '\n'
+                    end = line.find("'", i + 2)
+                    if end >= 0 and end <= i + 4:
+                        i = end + 1
+                        continue
+                elif i + 2 < len(line) and line[i + 2] == "'":
+                    # Simple char literal like 'a'
+                    i += 3
+                    continue
+                # Otherwise it's a lifetime, skip
+            if ch == "{":
+                depth += 1
+                found_open = True
+            elif ch == "}":
+                depth -= 1
+                if found_open and depth == 0:
+                    return idx
+            i += 1
+    return len(lines) - 1
+
+
+def find_brace_end_go(lines: list[str], start_line_0: int) -> int:
+    """Find the 0-based line where the outermost brace closes, skipping strings/comments."""
+    depth = 0
+    found_open = False
+    in_block_comment = False
+    for idx in range(start_line_0, len(lines)):
+        line = lines[idx]
+        i = 0
+        while i < len(line):
+            ch = line[i]
+            if in_block_comment:
+                if ch == "*" and i + 1 < len(line) and line[i + 1] == "/":
+                    in_block_comment = False
+                    i += 2
+                    continue
+                i += 1
+                continue
+            if ch == "/" and i + 1 < len(line):
+                if line[i + 1] == "/":
+                    break  # rest is line comment
+                if line[i + 1] == "*":
+                    in_block_comment = True
+                    i += 2
+                    continue
+            if ch == '"':
+                i += 1
+                while i < len(line) and line[i] != '"':
+                    if line[i] == "\\":
+                        i += 1
+                    i += 1
+                i += 1
+                continue
+            if ch == "`":
+                # raw string can span lines - scan to end
+                i += 1
+                while True:
+                    while i < len(line):
+                        if line[i] == "`":
+                            i += 1
+                            break
+                        i += 1
+                    else:
+                        # continue to next line
+                        idx += 1
+                        if idx >= len(lines):
+                            return len(lines) - 1
+                        line = lines[idx]
+                        i = 0
+                        continue
+                    break
+                continue
+            if ch == "{":
+                depth += 1
+                found_open = True
+            elif ch == "}":
+                depth -= 1
+                if found_open and depth == 0:
+                    return idx
+            i += 1
+    return len(lines) - 1