codna 0.1.0__py3-none-any.whl

codna/cli_errors.py

@@ -0,0 +1,50 @@
+"""Helpers for rendering engine/SDK failures in CLI-friendly form."""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def _pick(mapping: dict[str, Any] | None, *keys: str) -> Any:
+    if not isinstance(mapping, dict):
+        return None
+    for key in keys:
+        value = mapping.get(key)
+        if value not in (None, ""):
+            return value
+    return None
+
+
+def _details_text(details: Any) -> str | None:
+    if details in (None, "", {}, []):
+        return None
+    if isinstance(details, str):
+        return details
+    try:
+        return json.dumps(details, sort_keys=True)
+    except TypeError:
+        return repr(details)
+
+
+def format_cli_error(exc: BaseException) -> str:
+    """Render a structured exception without depending on a specific SDK version."""
+    error_obj = getattr(exc, "error", None)
+    head = _pick(error_obj, "message") or str(exc).strip() or exc.__class__.__name__
+    code = getattr(exc, "error_code", None) or getattr(exc, "code", None) or _pick(error_obj, "code")
+    request_id = getattr(exc, "request_id", None) or _pick(error_obj, "request_id")
+    status = getattr(exc, "status_code", None) or getattr(exc, "http_status", None)
+    details = getattr(exc, "details", None)
+    if details in (None, "", {}, []):
+        details = _pick(error_obj, "details")
+
+    parts = [head]
+    if code:
+        parts.append(f"code={code}")
+    if request_id:
+        parts.append(f"request_id={request_id}")
+    if status not in (None, ""):
+        parts.append(f"status={status}")
+    details_text = _details_text(details)
+    if details_text:
+        parts.append(f"details={details_text}")
+    return " | ".join(parts)

codna/cline_agent.py

@@ -0,0 +1,90 @@
+"""Drive codna's vendored Cline agent (agent-core) to remediate a repo in place.
+
+This is the agentic-fix executor behind `codna secure --fix`: it shells out to `bun` running the
+vendored Cline SDK, which edits files in `working_dir`. The engine seam is optional — with only an
+ANTHROPIC_API_KEY the agent runs as a plain coding agent (no HTTP engine/sidecar required).
+"""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import tempfile
+from pathlib import Path
+
+
+class ClineAgentError(Exception):
+    pass
+
+
+def _agent_core_dir() -> Path:
+    """Locate the vendored Cline SDK root (agent-core/vendor/cline)."""
+    env = os.environ.get("CODNA_AGENT_CORE_DIR")
+    if env:
+        return Path(env)
+    for parent in Path(__file__).resolve().parents:
+        cand = parent / "agent-core" / "vendor" / "cline"
+        if (cand / "algenta" / "adapter.ts").exists():
+            return cand
+    raise ClineAgentError(
+        "could not locate agent-core/vendor/cline; set CODNA_AGENT_CORE_DIR to its path."
+    )
+
+
+def _ensure_cline_data_dir(env: dict[str, str]) -> str:
+    configured = env.get("CLINE_DATA_DIR", "").strip()
+    if configured:
+        data_dir = Path(configured).expanduser()
+    else:
+        data_dir = Path(tempfile.gettempdir()) / "codna-cline-data"
+        env["CLINE_DATA_DIR"] = str(data_dir)
+    data_dir.mkdir(parents=True, exist_ok=True)
+    return str(data_dir)
+
+
+def run_cline_fix(
+    working_dir: str,
+    prompt: str,
+    *,
+    task_kind: str = "fix",
+    provider: str = "anthropic",
+    model: str | None = None,
+    max_iterations: int = 20,
+    timeout_s: int = 360,
+) -> dict:
+    """Run the Cline agent against `working_dir` (it edits files in place). Returns the agent's
+    result summary ({status, iterations, model, text, usage}). Raises ClineAgentError on failure."""
+    core = _agent_core_dir()
+    if not (core / "algenta" / "fix-runner.ts").exists():
+        raise ClineAgentError(f"fix-runner.ts not found under {core}")
+    if not (core / "node_modules").is_dir():
+        raise ClineAgentError(f"vendored SDK not built (no node_modules) under {core}")
+    if task_kind not in {"fix", "triage"}:
+        raise ClineAgentError(f"unsupported cline task_kind '{task_kind}'")
+
+    env = dict(os.environ)
+    _ensure_cline_data_dir(env)
+    env["FIX_WORKING_DIR"] = str(working_dir)
+    env["FIX_PROMPT"] = prompt
+    env["FIX_TASK_KIND"] = task_kind
+    env["FIX_PROVIDER"] = provider
+    env["FIX_MAX_ITERATIONS"] = str(max_iterations)
+    if model:
+        env["FIX_MODEL"] = model
+
+    try:
+        proc = subprocess.run(
+            ["bun", "algenta/fix-runner.ts"],
+            cwd=str(core), env=env, capture_output=True, text=True, timeout=timeout_s,
+        )
+    except FileNotFoundError as exc:
+        raise ClineAgentError("`bun` not found on PATH — required to run the Cline agent.") from exc
+    except subprocess.TimeoutExpired as exc:
+        raise ClineAgentError(f"cline agent timed out after {timeout_s}s") from exc
+
+    for line in proc.stdout.splitlines():
+        if line.startswith("RESULT "):
+            return json.loads(line[len("RESULT "):])
+    raise ClineAgentError(
+        f"cline agent produced no RESULT (rc={proc.returncode}); stderr tail: {proc.stderr[-300:]}"
+    )

codna/codeunits.py

@@ -0,0 +1,221 @@
+"""Extract a repository into structural code units for Codna's local code memory.
+
+Codna has no symbol extractor of its own — the Algenta engine does that server-side. For the
+in-process ``codna memory`` capability we ship a small, dependency-free extractor: day-1 is Python
+(stdlib ``ast``) producing module / class / function / method / test units. Other languages plug in
+via :func:`register_extractor` — keeping multi-language support additive.
+
+A :class:`CodeUnit` is a structural unit (not a fixed-size chunk): structure gives cleaner retrieval
+and provenance. ``id`` is a stable symbol path so a changed symbol re-indexes with ``upsert`` (no
+duplicate row) and a removed symbol can be ``delete``d.
+"""
+from __future__ import annotations
+
+import ast
+import os
+from dataclasses import dataclass
+from typing import Iterator, Protocol
+
+# The filter columns the memory collection declares. ``CodeUnit.metadata()`` returns EXACTLY these
+# (plus the physical partition key ``scope_key``), so Telys' ingest never KeyErrors on a missing column.
+FILTER_COLUMNS = ["repo_id", "service", "language", "path", "symbol_type", "commit_sha", "branch"]
+
+# Symbol types emitted day-1 (a subset of INTEGRATION-CODNA §4; error-strings/config/routes come later).
+SYMBOL_TYPES = ("module", "class", "function", "method", "test")
+
+_MAX_UNIT_CHARS = 2000  # bound the indexed text slice per symbol
+
+_DEFAULT_IGNORE = (
+    ".git", ".hg", ".svn", ".venv", "venv", "env", "node_modules", "__pycache__",
+    ".mypy_cache", ".pytest_cache", ".ruff_cache", "dist", "build", ".codna-memory",
+)
+
+
+@dataclass(frozen=True)
+class CodeUnit:
+    """One structural code unit with provenance."""
+
+    id: str            # "<repo_id>:<relpath>:<Qualified.Name>" — stable -> upsert/delete by id
+    text: str          # what gets embedded: qualname + docstring + a bounded source slice
+    path: str          # repo-relative path
+    language: str
+    symbol_type: str   # one of SYMBOL_TYPES
+    qualname: str
+    service: str | None = None
+
+    def metadata(self, repo_id: str, scope: str, *, commit_sha: str = "", branch: str = "") -> dict:
+        """Exactly the collection key + ``FILTER_COLUMNS`` (Telys reads each by name)."""
+        return {
+            "scope_key": scope,
+            "repo_id": repo_id,
+            "service": self.service or "",
+            "language": self.language,
+            "path": self.path,
+            "symbol_type": self.symbol_type,
+            "commit_sha": commit_sha,
+            "branch": branch,
+        }
+
+
+class LanguageExtractor(Protocol):
+    extensions: tuple[str, ...]
+    language: str
+
+    def extract(self, repo_id: str, relpath: str, source: str) -> Iterator[CodeUnit]:
+        ...
+
+
+@dataclass
+class ExtractStats:
+    files: int = 0       # source files scanned for the requested languages
+    units: int = 0       # structural units produced
+    skipped: int = 0     # files that failed to parse (e.g. SyntaxError) — counted, not fatal
+
+
+class UnsupportedLanguageError(ValueError):
+    """Raised when a caller explicitly requests a language with no registered extractor."""
+
+
+def _is_test(relpath: str, name: str) -> bool:
+    base = os.path.basename(relpath)
+    return base.startswith("test_") or base.endswith("_test.py") or name.startswith("test_")
+
+
+class PythonExtractor:
+    """Python structural units via stdlib ``ast`` — no third-party parser."""
+
+    extensions = (".py",)
+    language = "python"
+
+    def extract(self, repo_id: str, relpath: str, source: str) -> Iterator[CodeUnit]:
+        tree = ast.parse(source)   # may raise SyntaxError; extract_repo counts the skip (non-fatal)
+        doc = ast.get_docstring(tree)
+        if doc:
+            yield self._unit(repo_id, relpath, relpath, f"{relpath}\n{doc}", "module")
+        yield from self._walk(repo_id, relpath, source, tree.body, prefix="")
+
+    def _walk(self, repo_id, relpath, source, body, prefix) -> Iterator[CodeUnit]:
+        for node in body:
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                qual = f"{prefix}{node.name}"
+                if _is_test(relpath, node.name):
+                    sym = "test"
+                elif prefix:
+                    sym = "method"
+                else:
+                    sym = "function"
+                yield self._unit(repo_id, relpath, qual, self._body_text(source, node, qual), sym)
+            elif isinstance(node, ast.ClassDef):
+                qual = f"{prefix}{node.name}"
+                yield self._unit(repo_id, relpath, qual, self._header_text(source, node, qual), "class")
+                yield from self._walk(repo_id, relpath, source, node.body, prefix=f"{qual}.")
+
+    def _body_text(self, source, node, qual) -> str:
+        seg = ast.get_source_segment(source, node) or ""
+        doc = ast.get_docstring(node) or ""
+        return f"{qual}\n{doc}\n{seg}".strip()[:_MAX_UNIT_CHARS]
+
+    def _header_text(self, source, node, qual) -> str:
+        # for a class, prefer the signature line + docstring over the whole body
+        seg = ast.get_source_segment(source, node) or ""
+        first = seg.splitlines()[0] if seg else ""
+        doc = ast.get_docstring(node) or ""
+        return f"{qual}\n{first}\n{doc}".strip()[:_MAX_UNIT_CHARS]
+
+    def _unit(self, repo_id, relpath, qual, text, symbol_type) -> CodeUnit:
+        return CodeUnit(
+            id=f"{repo_id}:{relpath}:{qual}",
+            text=text or qual,
+            path=relpath,
+            language=self.language,
+            symbol_type=symbol_type,
+            qualname=qual,
+        )
+
+
+_EXTRACTORS: dict[str, LanguageExtractor] = {}
+_EXTRACTOR_ERRORS: dict[str, str] = {}
+
+
+def register_extractor(extractor: LanguageExtractor) -> None:
+    """Register a language extractor by its file extensions (the seam for Go/TS/… later)."""
+    for ext in extractor.extensions:
+        _EXTRACTORS[ext] = extractor
+
+
+def registered_languages() -> tuple[str, ...]:
+    """Return every language currently backed by a registered extractor."""
+    return tuple(sorted({extractor.language for extractor in _EXTRACTORS.values()}))
+
+
+def _record_extractor_error(language: str, message: str) -> None:
+    _EXTRACTOR_ERRORS[language] = message
+
+
+def resolve_languages(languages: tuple[str, ...] | None) -> tuple[str, ...]:
+    """Resolve and validate the requested extractor languages."""
+    available = set(registered_languages())
+    resolved = tuple(registered_languages() if languages is None else languages)
+    missing = sorted(set(resolved) - available)
+    if missing:
+        available_text = ", ".join(sorted(available)) if available else "none"
+        missing_text = ", ".join(missing)
+        diagnostics = [f"{lang}: {_EXTRACTOR_ERRORS[lang]}" for lang in missing if lang in _EXTRACTOR_ERRORS]
+        if "tree-sitter" in _EXTRACTOR_ERRORS:
+            diagnostics.append(f"tree-sitter: {_EXTRACTOR_ERRORS['tree-sitter']}")
+        details = f" Registration failures: {'; '.join(diagnostics)}." if diagnostics else ""
+        raise UnsupportedLanguageError(
+            f"unsupported language extractor(s): {missing_text}. "
+            f"Registered languages: {available_text}.{details} "
+            "Install `codna[memory-languages]` for JS/TS/Go/Rust/Java/C/C++/C#/PHP/Ruby."
+        )
+    return resolved
+
+
+register_extractor(PythonExtractor())
+
+# Optional multi-language support (additive): plug tree-sitter grammars into the same seam when the
+# `tree_sitter` wheels are installed. Falls back to Python-only (ast) if they're absent — non-breaking.
+try:
+    from .codeunits_treesitter import register_treesitter
+
+    register_treesitter(register_extractor, CodeUnit, record_error=_record_extractor_error)
+except Exception as exc:  # noqa: BLE001 — tree-sitter is optional, but the failure must remain inspectable.
+    _record_extractor_error("tree-sitter", f"{type(exc).__name__}: {exc}")
+
+
+def extract_repo(repo_path: str, repo_id: str, *, languages: tuple[str, ...] | None = None,
+                 ignore: tuple[str, ...] = _DEFAULT_IGNORE) -> tuple[list[CodeUnit], ExtractStats]:
+    """Walk ``repo_path`` and return (units, stats). Files that fail to parse are skipped + counted."""
+    langs = set(resolve_languages(languages))
+    units: list[CodeUnit] = []
+    stats = ExtractStats()
+    ignore_set = set(ignore)
+    for root, dirs, files in os.walk(repo_path):
+        dirs[:] = [d for d in dirs if d not in ignore_set and not d.startswith(".")]
+        for fn in files:
+            ext = os.path.splitext(fn)[1]
+            ex = _EXTRACTORS.get(ext)
+            if ex is None or ex.language not in langs:
+                continue
+            full = os.path.join(root, fn)
+            relpath = os.path.relpath(full, repo_path)
+            try:
+                with open(full, encoding="utf-8", errors="replace") as f:
+                    source = f.read()
+            except OSError:
+                continue
+            if "\x00" in source:
+                # Some source files carry embedded NUL bytes (binary-ish blobs, generated code, certain repos
+                # — caveman/tailwindcss/grpc/Ventoy/TrafficMonitor). The on-device embedder rejects NUL, so
+                # strip it here at extraction (a NUL is never semantically meaningful in source text).
+                source = source.replace("\x00", "")
+            stats.files += 1
+            try:
+                got = list(ex.extract(repo_id, relpath, source))
+            except SyntaxError:
+                stats.skipped += 1
+                continue
+            units.extend(got)
+            stats.units += len(got)
+    return units, stats

codna/codeunits_treesitter.py

@@ -0,0 +1,148 @@
+"""Optional multi-language structural extraction via tree-sitter — ONE generic walk for every grammar.
+
+Codna's day-1 extractor is stdlib-``ast`` Python only (:mod:`codna.codeunits`). This module plugs the
+other languages into the same ``register_extractor`` seam, so ``codna memory`` indexes any repo the
+Algenta engine already understands. It is **additive + optional**: if ``tree_sitter`` and the grammar
+wheels aren't installed, :func:`register_treesitter` is a no-op and codna stays Python-only.
+
+There is NO per-language logic: a node is a unit when its grammar type *contains* a function/type hint
+and has a name; the doc is the leading comment sibling(s), else a leading string literal in the body
+(docstring languages). Validated across 12 languages on 200+ public repos (recall@10 0.60, and the
+WordLlama precision rerank lifts it to 0.66 — broadly, not just Python)."""
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import re
+from functools import lru_cache
+from typing import Callable
+
+_MAX = 2000
+
+# grammar -> (pip module, capsule accessor). Adding a language = one entry + ``pip install tree-sitter-<x>``.
+# Python is intentionally absent: the stdlib-``ast`` PythonExtractor stays the (tested) path for ``.py``.
+GRAMMAR_MOD = {
+    "javascript": ("tree_sitter_javascript", "language"),
+    "typescript": ("tree_sitter_typescript", "language_typescript"),
+    "tsx": ("tree_sitter_typescript", "language_tsx"),
+    "go": ("tree_sitter_go", "language"), "rust": ("tree_sitter_rust", "language"),
+    "java": ("tree_sitter_java", "language"), "c": ("tree_sitter_c", "language"),
+    "cpp": ("tree_sitter_cpp", "language"), "csharp": ("tree_sitter_c_sharp", "language"),
+    "php": ("tree_sitter_php", "language_php"), "ruby": ("tree_sitter_ruby", "language"),
+}
+EXT_LANG = {
+    ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript",
+    ".ts": "typescript", ".tsx": "tsx", ".go": "go", ".rs": "rust", ".java": "java",
+    ".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp", ".cxx": "cpp", ".hpp": "cpp", ".hh": "cpp",
+    ".cs": "csharp", ".php": "php", ".rb": "ruby",
+}
+_FUNC = ("function", "method", "constructor", "subroutine", "fn_", "_fn", "def")
+_TYPE = ("class", "struct", "interface", "trait", "impl", "enum", "module", "namespace", "object_definition")
+_NOISE = re.compile(r"SPDX-License|Copyright\s*[\(©]|Licensed under|All rights reserved|"
+                    r"GNU General Public|Permission is hereby granted|Redistribution and use", re.I)
+
+
+@lru_cache(maxsize=64)
+def _parser(lang):
+    from tree_sitter import Language, Parser
+    mod, acc = GRAMMAR_MOD[lang]
+    return Parser(Language(getattr(importlib.import_module(mod), acc)()))
+
+
+def _txt(node, src): return src[node.start_byte:node.end_byte].decode("utf-8", "replace")
+def _is_func(t): return any(h in t for h in _FUNC)
+def _is_type(t): return any(h in t for h in _TYPE)
+
+
+def _name_of(node, src):
+    n = node.child_by_field_name("name")
+    if n is not None:
+        return _txt(n, src)
+    for c in node.children:
+        if "identifier" in c.type or c.type.endswith("name"):
+            return _txt(c, src)
+    return ""
+
+
+def _leading_doc(node, src):
+    parts, sib = [], node.prev_sibling
+    while sib is not None and "comment" in sib.type:
+        parts.append(_txt(sib, src))
+        sib = sib.prev_sibling
+    if parts:
+        return "\n".join(reversed(parts))
+    body = node.child_by_field_name("body")
+    if body is not None:
+        for c in list(body.children)[:3]:
+            if "string" in c.type:
+                return _txt(c, src)
+            for g in list(c.children)[:2]:
+                if "string" in g.type:
+                    return _txt(g, src)
+    return ""
+
+
+def _iter(node, src, prefix, depth, out):
+    if depth > 6:
+        return
+    for c in node.children:
+        t = c.type
+        if _is_func(t) or _is_type(t):
+            name = _name_of(c, src)
+            if name and re.match(r"^[A-Za-z_][\w]*$", name):
+                qual = f"{prefix}{name}"
+                doc = _leading_doc(c, src)
+                if _NOISE.search(doc):
+                    doc = ""
+                is_type = _is_type(t) and not _is_func(t)
+                out.append((qual, "class" if is_type else ("method" if prefix else "function"),
+                            doc, _txt(c, src)[:_MAX]))
+                if is_type:  # descend type bodies for methods, not fn bodies
+                    _iter(c.child_by_field_name("body") or c, src, f"{qual}.", depth + 1, out)
+        else:
+            _iter(c, src, prefix, depth + 1, out)
+
+
+def _iter_symbols(lang, source):
+    src = source.encode("utf-8", "replace")
+    out = []
+    _iter(_parser(lang).parse(src).root_node, src, "", 0, out)
+    return out
+
+
+def _make_extractor(CodeUnit, lang, exts):
+    class _TS:
+        extensions = tuple(exts)
+        language = lang
+        def extract(self, repo_id, relpath, source):
+            for qual, sym, doc, body in _iter_symbols(lang, source):
+                text = f"{qual}\n{doc}\n{body}".strip()[:_MAX]
+                yield CodeUnit(id=f"{repo_id}:{relpath}:{qual}", text=text or qual, path=relpath,
+                               language=lang, symbol_type=sym, qualname=qual)
+    return _TS()
+
+
+def register_treesitter(register_extractor, CodeUnit, *, record_error: Callable[[str, str], None] | None = None) -> set:
+    """Register a tree-sitter extractor for every grammar whose wheel is installed. No-op + returns an
+    empty set if ``tree_sitter`` itself is missing (codna then stays Python-only). Returns the live set."""
+    try:
+        import tree_sitter  # noqa: F401
+    except Exception:
+        return set()
+    by_lang: dict[str, list] = {}
+    for ext, lang in EXT_LANG.items():
+        by_lang.setdefault(lang, []).append(ext)
+    live = set()
+    for lang, exts in by_lang.items():
+        mod, _acc = GRAMMAR_MOD[lang]
+        if importlib.util.find_spec(mod) is None:
+            continue
+        try:
+            _parser(lang)  # confirm the grammar wheel loads before registering
+        except Exception as exc:  # noqa: BLE001 - installed grammar failed; record it for explicit requests.
+            if record_error is not None:
+                record_error(lang, f"{mod}: {type(exc).__name__}: {exc}")
+            continue
+        register_extractor(_make_extractor(CodeUnit, lang, exts))
+        live.add(lang)
+    return live