codecrate 0.1.0__py3-none-any.whl

codecrate/packer.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from dataclasses import replace
+from pathlib import Path
+
+from .ids import stable_body_hash
+from .model import ClassRef, DefRef, FilePack, PackResult
+from .parse import module_name_for, parse_symbols
+from .stubber import stub_file_text
+
+
+def _extract_canonical_source(text: str, d: DefRef) -> str:
+    lines = text.splitlines(keepends=True)
+    i0 = max(0, d.decorator_start - 1)
+    i1 = min(len(lines), d.end_line)
+    return "".join(lines[i0:i1]).rstrip() + "\n"
+
+
+def _line_count(text: str) -> int:
+    return text.count("\n") + 1 if text else 0
+
+
+def pack_repo(
+    root: Path,
+    files: list[Path],
+    keep_docstrings: bool = True,
+    dedupe: bool = False,
+) -> tuple[PackResult, dict[str, str]]:
+    filepacks: list[FilePack] = []
+    all_defs: list[DefRef] = []
+    all_classes: list[ClassRef] = []
+
+    local_canon: dict[str, str] = {}
+
+    for path in files:
+        text = path.read_text(encoding="utf-8", errors="replace")
+
+        if path.suffix.lower() == ".py":
+            classes, defs = parse_symbols(path=path, root=root, text=text)
+            file_module = module_name_for(path, root)
+
+            for d in defs:
+                local_canon[d.local_id] = _extract_canonical_source(text, d)
+
+            stubbed = stub_file_text(text, defs, keep_docstrings=keep_docstrings)
+        else:
+            # Non-Python files are included verbatim (no symbol parsing / stubbing).
+            classes, defs = [], []
+            file_module = ""
+            stubbed = text
+
+        fp = FilePack(
+            path=path,
+            module=file_module,
+            original_text=text,
+            stubbed_text=stubbed,
+            line_count=_line_count(text),
+            classes=classes,
+            defs=defs,
+        )
+        filepacks.append(fp)
+        all_defs.extend(defs)
+        all_classes.extend(classes)
+
+    canonical_sources: dict[str, str] = {}
+    if not dedupe:
+        canonical_sources = {d.local_id: local_canon[d.local_id] for d in all_defs}
+    else:
+        seen_by_hash: dict[str, str] = {}
+        remapped_defs: list[DefRef] = []
+
+        for d in all_defs:
+            code = local_canon[d.local_id]
+            h = stable_body_hash(code)
+            cid = seen_by_hash.get(h)
+            if cid is None:
+                cid = d.local_id
+                seen_by_hash[h] = cid
+                canonical_sources[cid] = code
+            remapped_defs.append(replace(d, id=cid))
+
+        all_defs = remapped_defs
+
+        defs_by_file: dict[Path, list[DefRef]] = {}
+        for d in all_defs:
+            defs_by_file.setdefault(d.path, []).append(d)
+
+        filepacks2: list[FilePack] = []
+        for fp in filepacks:
+            defs2 = defs_by_file.get(fp.path, [])
+            stubbed2 = stub_file_text(
+                fp.original_text, defs2, keep_docstrings=keep_docstrings
+            )
+            filepacks2.append(
+                FilePack(
+                    path=fp.path,
+                    module=fp.module,
+                    original_text=fp.original_text,
+                    stubbed_text=stubbed2,
+                    line_count=fp.line_count,
+                    classes=fp.classes,
+                    defs=defs2,
+                )
+            )
+        filepacks = filepacks2
+
+    pack = PackResult(root=root, files=filepacks, classes=all_classes, defs=all_defs)
+    return pack, canonical_sources

codecrate/parse.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+from .ids import stable_location_id
+from .model import ClassRef, DefRef
+
+
+def module_name_for(path: Path, root: Path) -> str:
+    rel = path.resolve().relative_to(root.resolve())
+    parts = list(rel.parts)
+    if parts and parts[0] == "src":
+        parts = parts[1:]
+    if parts and parts[-1].endswith(".py"):
+        parts[-1] = parts[-1][:-3]
+    if parts and parts[-1] == "__init__":
+        parts = parts[:-1]
+    return ".".join(parts)
+
+
+class _Visitor(ast.NodeVisitor):
+    def __init__(self, path: Path, root: Path):
+        self.path = path
+        self.root = root
+        self.module = module_name_for(path, root)
+        self.stack: list[str] = []
+        self.defs: list[DefRef] = []
+        self.classes: list[ClassRef] = []
+
+    def visit_ClassDef(self, node: ast.ClassDef):
+        self._add_class(node)
+        self.stack.append(node.name)
+        self.generic_visit(node)
+        self.stack.pop()
+
+    def visit_FunctionDef(self, node: ast.FunctionDef):
+        self._add_def(node, kind="function")
+        self.stack.append(node.name)
+        self.generic_visit(node)
+        self.stack.pop()
+
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef):
+        self._add_def(node, kind="async_function")
+        self.stack.append(node.name)
+        self.generic_visit(node)
+        self.stack.pop()
+
+    def _decorator_start(self, node: ast.AST, default_line: int) -> int:
+        start = default_line
+        for d in getattr(node, "decorator_list", []) or []:
+            if hasattr(d, "lineno"):
+                start = min(start, int(d.lineno))
+        return start
+
+    def _add_class(self, node: ast.ClassDef) -> None:
+        qual = ".".join(self.stack + [node.name]) if self.stack else node.name
+        class_line = int(getattr(node, "lineno", 1))
+        end_line = int(getattr(node, "end_lineno", class_line))
+        decorator_start = self._decorator_start(node, class_line)
+
+        rel_path = self.path.resolve().relative_to(self.root.resolve())
+        cid = stable_location_id(rel_path, f"class:{qual}", class_line)
+
+        self.classes.append(
+            ClassRef(
+                path=self.path,
+                module=self.module,
+                qualname=qual,
+                id=cid,
+                decorator_start=decorator_start,
+                class_line=class_line,
+                end_line=end_line,
+            )
+        )
+
+    def _add_def(self, node: ast.AST, kind: str) -> None:
+        name = getattr(node, "name", "<anon>")
+        qual = ".".join(self.stack + [name]) if self.stack else name
+
+        def_line = int(getattr(node, "lineno", 1))
+        end_line = int(getattr(node, "end_lineno", def_line))
+        decorator_start = self._decorator_start(node, def_line)
+
+        body = getattr(node, "body", []) or []
+        body_start = def_line
+        doc_start: int | None = None
+        doc_end: int | None = None
+
+        if body:
+            body_start = int(getattr(body[0], "lineno", def_line))
+            if (
+                isinstance(body[0], ast.Expr)
+                and isinstance(getattr(body[0], "value", None), ast.Constant)
+                and isinstance(getattr(body[0].value, "value", None), str)
+            ):
+                doc_start = int(getattr(body[0], "lineno", body_start))
+                doc_end = int(getattr(body[0], "end_lineno", doc_start))
+        else:
+            body_start = end_line
+
+        is_single_line = def_line == end_line
+
+        rel_path = self.path.resolve().relative_to(self.root.resolve())
+        local_id = stable_location_id(rel_path, qual, def_line)
+        canonical_id = local_id
+
+        self.defs.append(
+            DefRef(
+                path=self.path,
+                module=self.module,
+                qualname=qual,
+                id=canonical_id,
+                local_id=local_id,
+                kind=kind,
+                decorator_start=decorator_start,
+                def_line=def_line,
+                body_start=body_start,
+                end_line=end_line,
+                doc_start=doc_start,
+                doc_end=doc_end,
+                is_single_line=is_single_line,
+            )
+        )
+
+
+def parse_symbols(
+    path: Path, root: Path, text: str
+) -> tuple[list[ClassRef], list[DefRef]]:
+    tree = ast.parse(text)
+    v = _Visitor(path=path, root=root)
+    v.visit(tree)
+    return v.classes, v.defs

codecrate/stubber.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import io
+import tokenize
+
+from .model import DefRef
+
+
+def _indent_of(line: str) -> str:
+    return line[: len(line) - len(line.lstrip(" \t"))]
+
+
+def _rewrite_single_line_def(line: str, marker: str) -> list[str]:
+    src = line if line.endswith("\n") else line + "\n"
+    tokens = list(tokenize.generate_tokens(io.StringIO(src).readline))
+    colon_col = None
+    for tok in tokens:
+        if tok.type == tokenize.OP and tok.string == ":":
+            colon_col = tok.end[1]
+    if colon_col is None:
+        return [line]
+    head = line[:colon_col].rstrip()
+    return [f"{head} ...  {marker}\n"]
+
+
+def _replacement_lines(indent: str, marker: str) -> list[str]:
+    # Compact stub: single placeholder line.
+    return [f"{indent}...  {marker}\n"]
+
+
+def stub_file_text(text: str, defs: list[DefRef], keep_docstrings: bool = True) -> str:
+    lines = text.splitlines(keepends=True)
+    # IMPORTANT: Do not stub defs that are nested inside other defs.
+    # When using compact stubs, stubbing an inner def first can shift line
+    # positions and cause later outer-def replacements to overrun and truncate
+    # subsequent code (a common issue with nested helper functions).
+    outer_defs: list[DefRef] = []
+    stack: list[int] = []
+    for d in sorted(defs, key=lambda d: (d.decorator_start, -d.end_line)):
+        while stack and d.decorator_start > stack[-1]:
+            stack.pop()
+        if stack and d.end_line <= stack[-1]:
+            continue
+        outer_defs.append(d)
+        stack.append(d.end_line)
+
+    defs_sorted = sorted(
+        outer_defs, key=lambda d: (d.def_line, d.body_start, d.end_line), reverse=True
+    )
+
+    for d in defs_sorted:
+        marker = f"# ↪ FUNC:{d.local_id}"
+
+        if d.is_single_line:
+            i = d.def_line - 1
+            if 0 <= i < len(lines):
+                lines[i : i + 1] = _rewrite_single_line_def(lines[i], marker)
+            continue
+
+        start_line = d.body_start
+        if keep_docstrings and d.doc_end is not None:
+            start_line = d.doc_end + 1
+
+        i0 = max(0, start_line - 1)
+        i1 = min(len(lines), d.end_line)
+        if i0 >= i1:
+            # No body lines to replace.
+            # If we kept the docstring, annotate the docstring closing line instead.
+            if keep_docstrings and d.doc_end is not None:
+                idx = d.doc_end - 1
+                if 0 <= idx < len(lines):
+                    ln = lines[idx]
+                    base = ln[:-1] if ln.endswith("\n") else ln
+                    if marker not in base:
+                        lines[idx] = base + f"  {marker}\n"
+            continue
+
+        sample = lines[i0] if 0 <= i0 < len(lines) else ""
+        indent = _indent_of(sample) if sample else " " * 4
+        lines[i0:i1] = _replacement_lines(indent, marker)
+
+    return "".join(lines)