codepulse-engine 0.1.0__py3-none-any.whl

codepulse/parser/python.py

@@ -0,0 +1,331 @@
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+from tree_sitter_languages import get_parser
+
+from codepulse.models import CallReference, ImportItem, ImportStatement, Symbol, SymbolType
+from codepulse.parser.base import LanguageParser
+
+
+class PythonParser(LanguageParser):
+    """AST parser for Python source code using Tree-sitter."""
+
+    def __init__(self) -> None:
+        self._parser = get_parser("python")
+
+    @property
+    def language(self) -> str:
+        return "python"
+
+    def extract_symbols(self, source_code: str, file_path: str = "") -> list[Symbol]:
+        source_bytes = source_code.encode("utf-8")
+        tree = self._parser.parse(source_bytes)
+        symbols: list[Symbol] = []
+
+        def walk(node: Any, parent_class: str | None = None) -> None:
+            if node.type in ("function_definition", "async_function_definition"):
+                sym = self._parse_function(node, source_bytes, file_path, parent_class)
+                if sym:
+                    symbols.append(sym)
+                # Continue walking into function body to support nested functions/classes
+                body_node = node.child_by_field_name("body")
+                if body_node:
+                    for child in body_node.children:
+                        walk(child, parent_class=sym.name if sym else parent_class)
+                return
+
+            elif node.type == "class_definition":
+                sym = self._parse_class(node, source_bytes, file_path, parent_class)
+                if sym:
+                    symbols.append(sym)
+                body_node = node.child_by_field_name("body")
+                if body_node:
+                    for child in body_node.children:
+                        walk(child, parent_class=sym.name if sym else parent_class)
+                return
+
+            for child in node.children:
+                walk(child, parent_class)
+
+        walk(tree.root_node)
+        return symbols
+
+    def _parse_function(
+        self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
+    ) -> Symbol | None:
+        name_node = node.child_by_field_name("name")
+        if not name_node:
+            return None
+
+        name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
+        start_line = node.start_point[0] + 1
+        end_line = node.end_point[0] + 1
+        body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        docstring = self._extract_docstring(node, source_bytes)
+        signature = self._extract_signature(node, source_bytes)
+        complexity = self._calculate_complexity(node)
+
+        symbol_type = SymbolType.METHOD if parent_class else SymbolType.FUNCTION
+
+        return Symbol(
+            name=name,
+            type=symbol_type,
+            file_path=file_path,
+            start_line=start_line,
+            end_line=end_line,
+            signature=signature,
+            docstring=docstring,
+            body=body_text,
+            body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
+            language="python",
+            parent_symbol=parent_class,
+            complexity=complexity,
+        )
+
+    def _parse_class(
+        self, node: Any, source_bytes: bytes, file_path: str, parent_class: str | None
+    ) -> Symbol | None:
+        name_node = node.child_by_field_name("name")
+        if not name_node:
+            return None
+
+        name = source_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="replace")
+        start_line = node.start_point[0] + 1
+        end_line = node.end_point[0] + 1
+        body_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        docstring = self._extract_docstring(node, source_bytes)
+        signature = self._extract_signature(node, source_bytes)
+        base_classes = self._extract_base_classes(node, source_bytes)
+
+        return Symbol(
+            name=name,
+            type=SymbolType.CLASS,
+            file_path=file_path,
+            start_line=start_line,
+            end_line=end_line,
+            signature=signature,
+            docstring=docstring,
+            body=body_text,
+            body_hash=hashlib.sha256(body_text.strip().encode("utf-8")).hexdigest(),
+            language="python",
+            parent_symbol=parent_class,
+            complexity=1,
+            base_classes=base_classes,
+        )
+
+    def _extract_base_classes(self, node: Any, source_bytes: bytes) -> list[str]:
+        """
+        Phase 4.1 (roadmap §37 item 4.1: "Base-class/inheritance
+        extraction (Python + JS/TS)"). `class_definition`'s
+        `superclasses` field is the `argument_list` in `class Foo(Base,
+        pkg.Mixin, metaclass=Meta):` -- confirmed via a direct tree-
+        sitter grammar probe during this milestone's own design audit.
+
+        Keeps `identifier` (`Base`) and `attribute` (`pkg.Mixin`, a
+        dotted/qualified name) nodes -- the only two node types that can
+        actually name a base class in this grammar. Deliberately
+        EXCLUDES `keyword_argument` (e.g. `metaclass=Meta`) -- not a
+        base class -- and punctuation/commas. Returns the raw text
+        exactly as written; RESOLVING each name to a real symbol is a
+        separate step performed by `CallGraphBuilder`, not here (see
+        `Symbol.base_classes`'s own docstring for why raw and resolved
+        are deliberately kept apart).
+        """
+        superclasses = node.child_by_field_name("superclasses")
+        if not superclasses:
+            return []
+
+        bases: list[str] = []
+        for child in superclasses.children:
+            if child.type in ("identifier", "attribute"):
+                bases.append(
+                    source_bytes[child.start_byte:child.end_byte].decode("utf-8", errors="replace")
+                )
+        return bases
+
+    def _extract_docstring(self, node: Any, source_bytes: bytes) -> str | None:
+        body_node = node.child_by_field_name("body")
+        if not body_node or not body_node.children:
+            return None
+
+        first_stmt = body_node.children[0]
+        if first_stmt.type == "expression_statement" and first_stmt.children:
+            expr = first_stmt.children[0]
+            if expr.type == "string":
+                doc = source_bytes[expr.start_byte:expr.end_byte].decode("utf-8", errors="replace")
+                # Strip quotes (triple or single)
+                for q in ('"""', "'''", '"', "'"):
+                    if doc.startswith(q) and doc.endswith(q) and len(doc) >= 2 * len(q):
+                        return doc[len(q):-len(q)].strip()
+                return doc.strip()
+        return None
+
+    def _extract_signature(self, node: Any, source_bytes: bytes) -> str:
+        body_node = node.child_by_field_name("body")
+        if body_node:
+            sig_bytes = source_bytes[node.start_byte:body_node.start_byte]
+            sig_text = sig_bytes.decode("utf-8", errors="replace").strip()
+            if sig_text.endswith(":"):
+                sig_text = sig_text[:-1].strip()
+            return " ".join(sig_text.split())
+        
+        # Fallback to first line
+        full_text = source_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="replace")
+        return full_text.splitlines()[0].strip().rstrip(":")
+
+    def _calculate_complexity(self, node: Any) -> int:
+        complexity = 1
+        branching_types = {
+            "if_statement", "elif_clause", "for_statement", "while_statement",
+            "except_clause", "with_statement", "boolean_operator", "conditional_expression"
+        }
+
+        def count_branches(n: Any) -> None:
+            nonlocal complexity
+            if n.type in branching_types:
+                complexity += 1
+            for child in n.children:
+                count_branches(child)
+
+        count_branches(node)
+        return complexity
+
+    def extract_imports(self, source_code: str, file_path: str = "") -> list[ImportStatement]:
+        source_bytes = source_code.encode("utf-8")
+        tree = self._parser.parse(source_bytes)
+        imports: list[ImportStatement] = []
+
+        def walk(node: Any) -> None:
+            if node.type == "import_statement":
+                # e.g., import math, sys as s
+                line_num = node.start_point[0] + 1
+                for child in node.children:
+                    if child.type == "dotted_name":
+                        mod_name = source_bytes[child.start_byte:child.end_byte].decode("utf-8")
+                        imports.append(
+                            ImportStatement(
+                                module=mod_name,
+                                items=[],
+                                is_relative=False,
+                                level=0,
+                                line_number=line_num,
+                                file_path=file_path,
+                            )
+                        )
+                    elif child.type == "aliased_import":
+                        name_n = child.child_by_field_name("name")
+                        alias_n = child.child_by_field_name("alias")
+                        if name_n:
+                            mod_name = source_bytes[name_n.start_byte:name_n.end_byte].decode("utf-8")
+                            alias = source_bytes[alias_n.start_byte:alias_n.end_byte].decode("utf-8") if alias_n else None
+                            imports.append(
+                                ImportStatement(
+                                    module=mod_name,
+                                    items=[ImportItem(name=mod_name, alias=alias)],
+                                    is_relative=False,
+                                    level=0,
+                                    line_number=line_num,
+                                    file_path=file_path,
+                                )
+                            )
+
+            elif node.type == "import_from_statement":
+                # e.g., from os.path import join as j, exists
+                # e.g., from ..utils import helper
+                line_num = node.start_point[0] + 1
+                module_name = ""
+                level = 0
+                is_relative = False
+
+                module_node = node.child_by_field_name("module_name")
+                if module_node:
+                    if module_node.type == "relative_import":
+                        is_relative = True
+                        # Count dots
+                        dot_str = ""
+                        for sub in module_node.children:
+                            if sub.type == "import_prefix":
+                                dot_str += source_bytes[sub.start_byte:sub.end_byte].decode("utf-8")
+                            elif sub.type == "dotted_name":
+                                module_name = source_bytes[sub.start_byte:sub.end_byte].decode("utf-8")
+                        level = dot_str.count(".")
+                    else:
+                        module_name = source_bytes[module_node.start_byte:module_node.end_byte].decode("utf-8")
+
+                items: list[ImportItem] = []
+                for child in node.children:
+                    if child.type == "dotted_name" and child != module_node:
+                        item_name = source_bytes[child.start_byte:child.end_byte].decode("utf-8")
+                        items.append(ImportItem(name=item_name))
+                    elif child.type == "aliased_import":
+                        name_n = child.child_by_field_name("name")
+                        alias_n = child.child_by_field_name("alias")
+                        if name_n:
+                            item_name = source_bytes[name_n.start_byte:name_n.end_byte].decode("utf-8")
+                            alias = source_bytes[alias_n.start_byte:alias_n.end_byte].decode("utf-8") if alias_n else None
+                            items.append(ImportItem(name=item_name, alias=alias))
+
+                imports.append(
+                    ImportStatement(
+                        module=module_name,
+                        items=items,
+                        is_relative=is_relative,
+                        level=level,
+                        line_number=line_num,
+                        file_path=file_path,
+                    )
+                )
+
+            for child in node.children:
+                walk(child)
+
+        walk(tree.root_node)
+        return imports
+
+    def extract_calls(
+        self, source_code: str, file_path: str = "", symbols: list[Symbol] | None = None
+    ) -> list[CallReference]:
+        if symbols is None:
+            symbols = self.extract_symbols(source_code, file_path=file_path)
+
+        source_bytes = source_code.encode("utf-8")
+        tree = self._parser.parse(source_bytes)
+        calls: list[CallReference] = []
+
+        def find_enclosing_symbol(line: int) -> str:
+            # Match most specific / deepest symbol containing line
+            matching = [s for s in symbols if s.start_line <= line <= s.end_line]
+            if not matching:
+                return "<module>"
+            # Sort by smallest line span
+            matching.sort(key=lambda s: (s.end_line - s.start_line))
+            return matching[0].name
+
+        def walk(node: Any) -> None:
+            if node.type == "call":
+                func_node = node.child_by_field_name("function")
+                if func_node:
+                    callee_name = source_bytes[func_node.start_byte:func_node.end_byte].decode("utf-8", errors="replace")
+                    # If attribute call like obj.method(), extract method name
+                    if func_node.type == "attribute":
+                        attr_node = func_node.child_by_field_name("attribute")
+                        if attr_node:
+                            callee_name = source_bytes[attr_node.start_byte:attr_node.end_byte].decode("utf-8", errors="replace")
+                    
+                    line_num = node.start_point[0] + 1
+                    caller = find_enclosing_symbol(line_num)
+                    calls.append(
+                        CallReference(
+                            caller_symbol=caller,
+                            callee_name=callee_name,
+                            line_number=line_num,
+                            file_path=file_path,
+                        )
+                    )
+
+            for child in node.children:
+                walk(child)
+
+        walk(tree.root_node)
+        return calls

codepulse/rate_limit.py

@@ -0,0 +1,142 @@
+"""
+Phase 1.4 — In-process token-bucket rate limiter (roadmap §37 item 1.4;
+§6 "Recommended Architecture": rate limits live in the control plane /
+API Gateway layer, alongside auth and central authorization; §35's
+architecture diagram places "Rate limits" directly beside "Auth (API
+keys)" and "Central authz"; backlog B11).
+
+Scope, precisely (do not infer requirements from later milestones):
+  - A classic token-bucket algorithm (capacity + refill rate), entirely
+    in-process (a plain in-memory dict) -- no Redis, no database
+    persistence, no distributed state. State resets on process restart;
+    this project runs as one process, matching "in-process" literally
+    (roadmap §37/§45: "rate limiting (in-process token bucket, no Redis
+    yet)").
+  - One bucket per client identity: the raw `X-API-Key` header value if
+    present (and non-empty), else the client's IP address as a fallback
+    for unauthenticated requests -- so an unauthenticated flood doesn't
+    get an unlimited exemption, but distinct callers don't share one
+    bucket either. This does NOT verify the key (no DB lookup, no import
+    of `codepulse.auth`/`codepulse.api.security` here) -- the bucket is
+    keyed on the raw string alone, kept deliberately cheap and
+    independent; validity checking remains exactly `require_api_key`'s
+    job (Phase 1.2), unchanged and un-duplicated.
+  - Exceeding the bucket's capacity is reported as HTTP 429 with a real
+    `Retry-After` value (backlog B11's literal DoD: "Excess requests
+    receive 429 with retry-after"), computed as the actual number of
+    seconds until at least one token will be available -- never a
+    fabricated/estimated value.
+
+Deliberately NOT in scope for Phase 1.4 (do not infer these belong here):
+  - Per-role/per-plan differentiated limits (roadmap §15's RBAC roles are
+    Phase 7 / B25 scope, not modeled at all yet).
+  - Distributed/shared rate limiting across multiple processes or
+    workers (explicitly deferred -- "no Redis yet").
+  - A structured `{"error": {...}}` response envelope for the 429 body --
+    Phase 1.6. The middleware in `codepulse/api/app.py` that uses this
+    module returns a plain `{"detail": "..."}` body, matching FastAPI's
+    own default `HTTPException` shape and every other error path already
+    in `api/app.py`.
+  - Ingestion-specific safety limits (path allow-listing, SSRF checks,
+    size caps) -- Phase 1.5 / B10, an entirely separate concern from
+    request-rate limiting.
+
+Default parameters (`CodePulseConfig.rate_limit_requests_per_second`,
+`.rate_limit_burst`) are a reasonable *starting* configuration for
+local-first, single-developer use -- NOT a benchmarked/measured figure,
+matching roadmap §19's own caution against presenting unvalidated
+numbers as fact. Tune via config for a given workload.
+
+This module lives outside `codepulse/indexer/` and `codepulse/api/`
+deliberately -- a top-level module, mirroring `codepulse/auth.py` and
+`codepulse/jobs.py`'s placement, per the roadmap's own "control plane"
+grouping (§6) of auth/rate-limits/job-tracking as one layer, reusable
+identically by a future CLI or MCP surface (though today only the REST
+app installs it, as middleware).
+"""
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass
+
+
+@dataclass
+class _Bucket:
+    tokens: float
+    last_refill: float
+
+
+class TokenBucketLimiter:
+    """
+    A plain in-memory token-bucket rate limiter, one bucket per identity
+    string. Thread-safe via a single lock guarding the bucket dict --
+    cheap and sufficient for this project's single-process, in-process
+    scale (no Redis, no multi-process coordination).
+    """
+
+    def __init__(self, requests_per_second: float, burst: int) -> None:
+        self.requests_per_second = requests_per_second
+        self.burst = burst
+        self._buckets: dict[str, _Bucket] = {}
+        self._lock = threading.Lock()
+
+    def check(self, identity: str) -> tuple[bool, float]:
+        """
+        Attempts to consume one token for `identity`. Returns
+        `(allowed, retry_after_seconds)`: `retry_after_seconds` is `0.0`
+        when `allowed` is True, or the real number of seconds until at
+        least one token will be available when it is False.
+        """
+        now = time.monotonic()
+        with self._lock:
+            bucket = self._buckets.get(identity)
+            if bucket is None:
+                bucket = _Bucket(tokens=float(self.burst), last_refill=now)
+                self._buckets[identity] = bucket
+
+            elapsed = max(0.0, now - bucket.last_refill)
+            bucket.tokens = min(float(self.burst), bucket.tokens + elapsed * self.requests_per_second)
+            bucket.last_refill = now
+
+            if bucket.tokens >= 1.0:
+                bucket.tokens -= 1.0
+                return True, 0.0
+
+            missing = 1.0 - bucket.tokens
+            retry_after = missing / self.requests_per_second if self.requests_per_second > 0 else float("inf")
+            return False, retry_after
+
+    def reset(self) -> None:
+        """Clears all bucket state -- primarily for test isolation."""
+        with self._lock:
+            self._buckets.clear()
+
+
+_DEFAULT_LIMITER: TokenBucketLimiter | None = None
+
+
+def get_limiter() -> TokenBucketLimiter:
+    """
+    Returns the process-wide default `TokenBucketLimiter`, constructed
+    from the currently active `CodePulseConfig` on first use (and cached
+    -- call `reset_limiter()` after changing rate-limit settings, e.g. in
+    tests, matching the same pattern `codepulse.config.reset_settings()`
+    already establishes for the settings singleton itself).
+    """
+    global _DEFAULT_LIMITER
+    if _DEFAULT_LIMITER is None:
+        from codepulse.config import get_settings
+
+        settings = get_settings()
+        _DEFAULT_LIMITER = TokenBucketLimiter(
+            requests_per_second=settings.rate_limit_requests_per_second,
+            burst=settings.rate_limit_burst,
+        )
+    return _DEFAULT_LIMITER
+
+
+def reset_limiter() -> None:
+    """Resets the module-level default limiter -- primarily for test isolation."""
+    global _DEFAULT_LIMITER
+    _DEFAULT_LIMITER = None

codepulse/repository_identity.py

@@ -0,0 +1,83 @@
+"""
+One repository, one identity, however its path is spelled
+(CODEPULSE_FUTURE_CHANGES.md item 2; user-approved 2026-10-14).
+
+A repository is identified by `RepositoryModel.url` (unique). For a repository
+indexed from a local directory that string was whatever the caller typed, so the
+same directory spelled with backslashes vs forward slashes, a trailing
+separator, `..` segments, or a lowercase vs uppercase Windows drive letter
+(VS Code reports `c:\\x`, a CLI index stores `C:\\x`) became two repositories.
+Re-indexing then created a duplicate with a new id and, because a differently
+spelled path looked brand-new to the "re-indexing needs `indexer`" check, a
+viewer-only key could bypass it.
+
+`canonical_repository_url` is the one definition of identity:
+  - a LOCAL PATH becomes absolute, with symlinks and `..` resolved
+    (`os.path.realpath`, which on Windows also returns the true case of every
+    component and normalizes slashes), an uppercase drive letter on Windows (also
+    for a path that no longer exists, where realpath cannot fix the case), and no
+    trailing separator;
+  - anything that is not a local path -- a git URL, `https://...`, the synthetic
+    `file:///...` ids some tests use -- is returned exactly as given.
+
+`find_repository` is the one lookup: it matches the canonical form first, then
+any row stored under another spelling (a database not yet migrated, or a
+duplicate that predates this fix), so a lookup never creates a duplicate just
+because an old row has an old spelling. When several rows match it returns the
+lowest id, deterministically; it never merges or deletes anything.
+
+Limit: identity follows the filesystem. A mapped network drive resolves to its
+UNC path while the mapping exists; a path on a filesystem that is
+case-insensitive but where `realpath` cannot report the true case (a deleted
+directory, macOS) can still be spelled in more than one way.
+"""
+from __future__ import annotations
+
+import os
+import re
+
+from sqlalchemy.orm import Session
+
+from codepulse.storage.db import RepositoryModel
+
+# `scheme://...` is a URL, not a path. A Windows drive (`C:\\x`, `C:/x`) has no `//` right after the colon.
+_URL_SCHEME = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*://")
+
+
+def is_local_path(url: str) -> bool:
+    return bool(url) and not _URL_SCHEME.match(url)
+
+
+def canonical_repository_url(url: str) -> str:
+    """The one canonical identity for `url`. Non-paths are returned unchanged; see the module docstring."""
+    if not is_local_path(url):
+        return url
+    resolved = os.path.realpath(os.path.abspath(url))
+    if os.name == "nt" and len(resolved) >= 2 and resolved[1] == ":":
+        resolved = resolved[0].upper() + resolved[1:]
+    return resolved
+
+
+def identity_key(url: str) -> str:
+    """The canonical form, case-folded where the filesystem is case-insensitive, for comparing two spellings."""
+    return os.path.normcase(canonical_repository_url(url)) if is_local_path(url) else url
+
+
+def find_repository(session: Session, url: str) -> RepositoryModel | None:
+    """
+    The repository `url` names, under any spelling of the same directory, or None.
+    Prefers an exact canonical row, then an exact row, then any row whose canonical
+    identity matches (lowest id first).
+    """
+    canonical = canonical_repository_url(url)
+    exact = session.query(RepositoryModel).filter(RepositoryModel.url == canonical).first()
+    if exact is not None or not is_local_path(url):
+        return exact
+    raw = session.query(RepositoryModel).filter(RepositoryModel.url == url).first()
+    if raw is not None:
+        return raw
+    wanted = identity_key(url)
+    for row in session.query(RepositoryModel).order_by(RepositoryModel.id).all():
+        if is_local_path(row.url) and identity_key(row.url) == wanted:
+            return row
+    return None