attocode-code-intel 0.1.0__py3-none-any.whl

attocode_intel/storage/provenance_store.py

@@ -0,0 +1,137 @@
+"""Provenance write path for derived code-intel artifacts.
+
+Normalizes a ``Provenance`` dataclass (from the shared ``artifacts/``
+package) into a row in the server-side ``provenance`` table.
+
+- :func:`write_provenance_rows` takes a list of dicts so batched writes
+  (one per embedding chunk in the same upsert) collapse into a single
+  ``session.add_all`` call.
+- Schema is loose: keys that don't match ``Provenance`` columns flow
+  into the ``extra`` JSONB bucket, so callers can tack on contextual
+  metadata without a migration.
+- No commit — the caller owns the session lifecycle.
+"""
+
+from __future__ import annotations
+
+import logging
+import socket
+import time
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+
+# Columns that are first-class on the Provenance ORM (everything else
+# in the input dict goes into ``extra``).
+_TOP_LEVEL_COLS: frozenset[str] = frozenset({
+    "action_hash",
+    "artifact_type",
+    "input_blob_oid",
+    "input_tree_oid",
+    "indexer_name",
+    "indexer_version",
+    "config_digest",
+    "producer_service",
+    "producer_user_id",
+    "producer_job_id",
+    "producer_host",
+})
+
+
+def _split_known(data: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
+    """Partition a raw dict into (top-level, extra) based on ORM columns."""
+    top: dict[str, Any] = {}
+    extra: dict[str, Any] = {}
+    for k, v in data.items():
+        if k in _TOP_LEVEL_COLS:
+            top[k] = v
+        else:
+            extra[k] = v
+    return top, extra
+
+
+async def write_provenance_rows(
+    session: AsyncSession,
+    rows: list[dict[str, Any]],
+) -> int:
+    """Insert provenance rows for a batch of derived artifacts.
+
+    Each ``row`` is a dict whose keys are either Provenance ORM column
+    names or arbitrary extras (which land in the ``extra`` JSONB bucket).
+    Missing required fields are filled from sensible defaults so callers
+    can write a provenance row with just an ``action_hash`` +
+    ``input_blob_oid`` if that's all they know.
+
+    Returns the number of rows inserted. Does NOT commit — the caller
+    is expected to be inside a larger unit of work that commits at its
+    own boundary.
+    """
+    if not rows:
+        return 0
+
+    from attocode_intel.db.models import Provenance
+
+    default_host = socket.gethostname() or "unknown"
+    _ = time  # keep import alive for future use (e.g. produced_at override)
+
+    orm_rows: list[Provenance] = []
+    for raw in rows:
+        top, extra = _split_known(dict(raw))
+        orm_rows.append(
+            Provenance(
+                action_hash=str(top.get("action_hash", "")),
+                artifact_type=str(top.get("artifact_type", "unknown")),
+                input_blob_oid=str(top.get("input_blob_oid", "")),
+                input_tree_oid=top.get("input_tree_oid"),
+                indexer_name=str(top.get("indexer_name", "unknown")),
+                indexer_version=str(top.get("indexer_version", "0")),
+                config_digest=str(top.get("config_digest", "")),
+                producer_service=str(
+                    top.get("producer_service", "attocode-server"),
+                ),
+                producer_user_id=top.get("producer_user_id"),
+                producer_job_id=top.get("producer_job_id"),
+                producer_host=str(top.get("producer_host") or default_host),
+                extra=extra,
+            )
+        )
+
+    session.add_all(orm_rows)
+    return len(orm_rows)
+
+
+def embedding_provenance_dict(
+    *,
+    action_hash: str,
+    content_sha: str,
+    model_name: str,
+    model_version: str,
+    dimension: int | None,
+    indexer_name: str = "attocode-embedding-store",
+    indexer_version: str = "1",
+    config_digest: str = "",
+    **extra: Any,
+) -> dict[str, Any]:
+    """Build the dict that gets passed to :func:`write_provenance_rows`
+    for an embedding chunk.
+
+    Factored out so :class:`EmbeddingStore` doesn't have to hand-assemble
+    the same dict in every insert path.
+    """
+    row: dict[str, Any] = {
+        "action_hash": action_hash,
+        "artifact_type": "embedding_blob_v1",
+        "input_blob_oid": f"sha256:{content_sha}",
+        "indexer_name": indexer_name,
+        "indexer_version": indexer_version,
+        "config_digest": config_digest,
+        "model_name": model_name,
+        "model_version": model_version,
+        "dimension": dimension,
+    }
+    row.update(extra)
+    return row

attocode_intel/storage/security_scanner_db.py

@@ -0,0 +1,164 @@
+"""DB-backed security scanning — runs regex patterns on ContentStore content.
+
+For remote repos without a local clone, this provides security scanning
+by reading file content from the database and applying the same patterns
+used by the local scanner.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import TYPE_CHECKING
+
+from attocode_intel._internal.integrations.security.matcher import iter_pattern_matches
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+# Scannable file extensions (same as local scanner)
+_SCANNABLE_EXTENSIONS = {
+    ".py", ".pyi", ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs",
+    ".rs", ".go", ".java", ".kt", ".rb", ".php", ".swift", ".cs",
+    ".c", ".cpp", ".h", ".hpp", ".lua", ".sh", ".bash", ".zsh",
+    ".yaml", ".yml", ".toml", ".json", ".env", ".cfg", ".ini", ".conf",
+}
+
+# Max file size to scan (200KB)
+_MAX_SCAN_SIZE = 200 * 1024
+
+# Max files to scan per run
+_MAX_SCAN_FILES = 500
+
+
+def _detect_language(path: str) -> str:
+    """Detect language from file extension."""
+    import os
+    ext = os.path.splitext(path)[1].lower()
+    lang_map = {
+        ".py": "python", ".pyi": "python",
+        ".js": "javascript", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript",
+        ".ts": "typescript", ".tsx": "typescript",
+        ".go": "go", ".rs": "rust", ".java": "java", ".kt": "kotlin",
+        ".rb": "ruby", ".php": "php", ".swift": "swift", ".cs": "csharp",
+        ".c": "c", ".cpp": "cpp", ".h": "c", ".hpp": "cpp",
+    }
+    return lang_map.get(ext, "")
+
+
+def _is_scannable(path: str) -> bool:
+    """Check if a file path is scannable by extension or filename."""
+    import os
+    basename = os.path.basename(path)
+    ext = os.path.splitext(path)[1].lower()
+    # Handle dotfiles like .env (ext is ".env", basename is ".env")
+    if not ext and basename.startswith("."):
+        ext = basename.lower()
+    return ext in _SCANNABLE_EXTENSIONS
+
+
+async def db_security_scan(
+    session: AsyncSession,
+    branch_id: uuid.UUID,
+    manifest: dict[str, str],
+    *,
+    mode: str = "full",
+    path_filter: str = "",
+) -> dict:
+    """Run security scan on DB-stored file contents.
+
+    Args:
+        session: Async DB session.
+        branch_id: Branch UUID for scoping.
+        manifest: Resolved manifest (path → content_sha).
+        mode: Scan mode — "quick" (secrets only), "full" (secrets + patterns),
+              "secrets", "patterns".
+        path_filter: Optional path prefix filter.
+
+    Returns:
+        Dict with findings, total_findings, summary.
+    """
+    from attocode_intel.storage.content_store import ContentStore
+
+    # Import patterns
+    from attocode_intel._internal.integrations.security.patterns import (
+        ANTI_PATTERNS,
+        SECRET_PATTERNS,
+    )
+
+    content_store = ContentStore(session)
+
+    # Select which patterns to use based on mode
+    patterns = []
+    if mode in ("full", "quick", "secrets"):
+        patterns.extend(SECRET_PATTERNS)
+    if mode in ("full", "patterns"):
+        patterns.extend(ANTI_PATTERNS)
+
+    # Filter and limit files
+    scannable: list[tuple[str, str]] = []  # (path, sha)
+    for path, sha in sorted(manifest.items()):
+        if path_filter and not path.startswith(path_filter):
+            continue
+        if not _is_scannable(path):
+            continue
+        scannable.append((path, sha))
+        if len(scannable) >= _MAX_SCAN_FILES:
+            break
+
+    findings: list[dict] = []
+    files_scanned = 0
+    severity_counts = {"critical": 0, "high": 0, "medium": 0, "low": 0}
+
+    for path, sha in scannable:
+        content_bytes = await content_store.get(sha)
+        if content_bytes is None:
+            continue
+        if len(content_bytes) > _MAX_SCAN_SIZE:
+            continue
+
+        try:
+            text = content_bytes.decode("utf-8", errors="replace")
+        except Exception:
+            continue
+
+        files_scanned += 1
+        file_lang = _detect_language(path)
+
+        for line_no, line, pat in iter_pattern_matches(text, patterns, file_lang):
+            sev = str(pat.severity)
+            severity_counts[sev] = severity_counts.get(sev, 0) + 1
+            findings.append({
+                "pattern": pat.name,
+                "severity": sev,
+                "category": str(pat.category),
+                "cwe_id": pat.cwe_id,
+                "file": path,
+                "line": line_no,
+                "message": pat.message,
+                "recommendation": pat.recommendation,
+                "snippet": line.strip()[:200],
+            })
+
+    # Compute compliance score
+    score = 100 - (
+        severity_counts.get("critical", 0) * 20
+        + severity_counts.get("high", 0) * 10
+        + severity_counts.get("medium", 0) * 3
+        + severity_counts.get("low", 0) * 1
+    )
+    score = max(0, min(100, score))
+
+    return {
+        "mode": mode,
+        "path": path_filter,
+        "findings": findings,
+        "total_findings": len(findings),
+        "summary": {
+            "files_scanned": files_scanned,
+            "compliance_score": score,
+            **severity_counts,
+        },
+    }

attocode_intel/storage/symbol_store.py

@@ -0,0 +1,160 @@
+"""Symbol storage — content-addressed symbol indexing with branch resolution."""
+
+from __future__ import annotations
+
+import uuid
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+
+class SymbolStore:
+    """Content-addressed symbol storage.
+
+    Symbols are keyed by content SHA, so identical files share symbol data.
+    Branch-aware queries resolve through BranchOverlay.
+    """
+
+    def __init__(self, session: AsyncSession) -> None:
+        self._session = session
+
+    async def has_symbols(self, content_sha: str) -> bool:
+        """Check if symbols already exist for a content SHA. Used for dedup gating."""
+        from sqlalchemy import select
+
+        from attocode_intel.db.models import Symbol
+
+        result = await self._session.execute(
+            select(Symbol.id).where(Symbol.content_sha == content_sha).limit(1)
+        )
+        return result.scalar_one_or_none() is not None
+
+    async def upsert_symbols(
+        self,
+        content_sha: str,
+        symbols: list[dict],
+        skip_if_exists: bool = False,
+    ) -> int:
+        """Insert or update symbols for a content hash.
+
+        Each symbol dict should have: name, kind, line_start, line_end,
+        signature (optional), exported (optional).
+
+        If skip_if_exists=True, returns 0 immediately when SHA already has symbols.
+        This is the content-hash dedup optimization — same content always produces
+        the same symbols.
+
+        Returns count of symbols stored.
+        """
+        from attocode_intel.db.models import Symbol
+
+        if skip_if_exists and await self.has_symbols(content_sha):
+            return 0
+
+        # Bulk DELETE — O(N) individual ORM deletes are too slow here.
+        from sqlalchemy import delete
+        await self._session.execute(
+            delete(Symbol).where(Symbol.content_sha == content_sha)
+        )
+
+        # Insert new
+        for s in symbols:
+            sym = Symbol(
+                content_sha=content_sha,
+                name=s["name"],
+                kind=s["kind"],
+                line_start=s.get("line_start"),
+                line_end=s.get("line_end"),
+                signature=s.get("signature"),
+                exported=s.get("exported", False),
+                metadata_=s.get("metadata", {}),
+            )
+            self._session.add(sym)
+
+        await self._session.flush()
+        return len(symbols)
+
+    async def search_symbols(
+        self,
+        repo_id: uuid.UUID,
+        branch_id: uuid.UUID,
+        pattern: str,
+        limit: int = 50,
+    ) -> list[dict]:
+        """Search symbols by name pattern within a branch context.
+
+        Resolves through the branch overlay to find relevant content SHAs.
+        """
+        from sqlalchemy import select
+
+        from attocode_intel.db.models import Symbol
+        from attocode_intel.storage.branch_overlay import BranchOverlay
+
+        overlay = BranchOverlay(self._session)
+        manifest = await overlay.resolve_manifest(branch_id)
+        content_shas = set(manifest.values())
+
+        if not content_shas:
+            return []
+
+        result = await self._session.execute(
+            select(Symbol)
+            .where(
+                Symbol.content_sha.in_(content_shas),
+                Symbol.name.ilike(f"%{pattern}%"),
+            )
+            .limit(limit)
+        )
+
+        # Build reverse map: sha → path
+        sha_to_path = {sha: path for path, sha in manifest.items()}
+
+        symbols = []
+        for sym in result.scalars():
+            symbols.append({
+                "name": sym.name,
+                "kind": sym.kind,
+                "file": sha_to_path.get(sym.content_sha, "unknown"),
+                "line_start": sym.line_start,
+                "line_end": sym.line_end,
+                "signature": sym.signature,
+                "exported": sym.exported,
+            })
+        return symbols
+
+    async def get_symbols_for_branch(
+        self,
+        branch_id: uuid.UUID,
+    ) -> list[dict]:
+        """Get all symbols for a branch, resolved through overlay."""
+        from sqlalchemy import select
+
+        from attocode_intel.db.models import Symbol
+        from attocode_intel.storage.branch_overlay import BranchOverlay
+
+        overlay = BranchOverlay(self._session)
+        manifest = await overlay.resolve_manifest(branch_id)
+        content_shas = set(manifest.values())
+
+        if not content_shas:
+            return []
+
+        sha_to_path = {sha: path for path, sha in manifest.items()}
+
+        result = await self._session.execute(
+            select(Symbol).where(Symbol.content_sha.in_(content_shas))
+        )
+
+        symbols = []
+        for sym in result.scalars():
+            symbols.append({
+                "name": sym.name,
+                "kind": sym.kind,
+                "file": sha_to_path.get(sym.content_sha, "unknown"),
+                "line_start": sym.line_start,
+                "line_end": sym.line_end,
+                "signature": sym.signature,
+                "exported": sym.exported,
+            })
+        return symbols

attocode_intel/structural_evidence.py

@@ -0,0 +1,144 @@
+"""Syntax-local evidence groups; this does not resolve data flow or earlier exits."""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+LANGUAGES = {'.py': 'python', '.js': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript',
+             '.ts': 'typescript', '.tsx': 'tsx', '.jsx': 'javascript'}
+CONTROLS = {'if_statement', 'elif_clause', 'else_clause', 'for_statement', 'for_in_statement',
+            'while_statement', 'with_statement', 'try_statement', 'except_clause',
+            'catch_clause', 'finally_clause', 'switch_statement', 'switch_case', 'switch_default'}
+BLOCKS = {'block', 'statement_block', 'switch_body'}
+LEAVES = {'identifier', 'property_identifier', 'string', 'integer', 'float', 'number',
+          'true', 'false', 'none', 'null'}
+
+
+def evidence_terms(value):
+    from attocode_intel.focused_evidence import terms
+    return terms(value) | set(re.findall(r'--[\w-]*|(?<!\w)\d+(?!\w)', value))
+
+
+def structural_excerpts(source, definition, hint, preview_end):
+    """Return None when structural context is unavailable, otherwise ranked groups.
+
+    Each list member owns its context so the response budget drops it atomically.
+    Context ranges describe enclosing syntax, not complete executable reachability.
+    """
+    language = LANGUAGES.get(Path(definition.file_path).suffix)
+    if not language:
+        return None
+    from attocode_intel._internal.integrations.context.ts_parser import _get_parser
+    parser = _get_parser(language)
+    if parser is None:
+        return None
+    raw = '\n'.join(source).encode()
+    tree = parser.parse(raw)  # No awaits between parse and traversal of this local tree.
+    if tree.root_node.has_error:
+        return None
+    from attocode_intel.focused_evidence import task_terms
+    positive, _ = task_terms(hint)
+    query = positive | {term for term in evidence_terms(hint)
+                        if term.startswith('--') or term.isdigit()}
+    first, last = definition.start_line, min(definition.end_line, len(source))
+
+    def bounds(node):
+        return node.start_point.row + 1, node.end_point.row + bool(node.end_point.column)
+
+    def header(node):
+        bodies = [c for c in node.children if c.type in BLOCKS]
+        for field in ('consequence', 'body'):
+            child = node.child_by_field_name(field)
+            if child is not None:
+                bodies.append(child)
+        a, b = bounds(node)
+        if bodies:
+            body = min(bodies, key=lambda c: c.start_byte)
+            # Keep a same-line statement verbatim; do not invent sliced source lines.
+            b = max(a, body.start_point.row + (body.type == 'statement_block'))
+        return a, b
+
+    def controls(node):
+        ranges = []
+        child, parent = node, node.parent
+        while parent is not None and bounds(parent)[0] >= first:
+            if parent.type in CONTROLS:
+                ranges.append(header(parent))
+                # Python elif/else are sibling clauses, so include preceding guards.
+                for sibling in parent.named_children:
+                    if sibling.start_byte >= child.start_byte:
+                        break
+                    if sibling.type == 'elif_clause':
+                        ranges.append(header(sibling))
+            child, parent = parent, parent.parent
+        return ranges
+
+    def tokens(node, interval=None):
+        if interval and (bounds(node)[1] < interval[0] or bounds(node)[0] > interval[1]):
+            return set()
+        if node.type in {'comment', 'comment_block'}:
+            return set()
+        # A bare Python string statement is documentation, not an executable literal.
+        if node.type == 'expression_statement' and node.named_children and node.named_children[0].type == 'string':
+            return set()
+        if node.type in LEAVES:
+            return evidence_terms(raw[node.start_byte:node.end_byte].decode())
+        result = set()
+        for child in node.named_children:
+            result.update(tokens(child, interval))
+        return result
+
+    candidates = []
+    stack = [tree.root_node]
+    while stack:
+        node = stack.pop()
+        a, b = bounds(node)
+        if b < first or a > last:
+            continue
+        stack.extend(reversed(node.named_children))
+        is_control = node.type in CONTROLS
+        is_statement = node.type.endswith('_statement') or node.type in {'lexical_declaration', 'variable_declaration'}
+        if not (is_control or is_statement) or a < first or b > last:
+            continue
+        # Compound statements contribute a header or a small complete branch.
+        if is_control:
+            effect = (a, b) if b - a < 18 and node.type in {'if_statement', 'elif_clause', 'else_clause'} else header(node)
+        else:
+            if any(c.type in BLOCKS for c in node.named_children):
+                continue
+            effect = (a, b)
+        if effect[1] <= preview_end:
+            continue
+        hits = tokens(node, effect) & query
+        if not hits:
+            continue
+        intervals = sorted(set([effect, *controls(node)]))
+        merged = []
+        for begin, end in intervals:
+            if merged and (begin <= merged[-1][1] + 1 or end - merged[-1][0] < 12):
+                merged[-1] = (merged[-1][0], max(end, merged[-1][1]))
+            else:
+                merged.append((begin, end))
+        # Prefer one small contiguous excerpt. Large gaps stay explicitly linked.
+        if merged[-1][1] - merged[0][0] < 12:
+            merged = [(merged[0][0], merged[-1][1])]
+        main = next(r for r in merged if r[0] <= effect[0] and r[1] >= effect[1])
+        context = [r for r in merged if r != main]
+        candidates.append((len(hits) + int(effect == (a, b)), hits, main, context))
+
+    def excerpt(interval):
+        a, b = interval
+        return {'file_path': definition.file_path, 'start_line': a, 'end_line': b,
+                'text': '\n'.join(source[a-1:b])}
+
+    selected, covered, occupied = [], set(), set()
+    while candidates and len(selected) < 3:
+        score, hits, main, context = max(candidates, key=lambda r: (r[0] + 4 * len(r[1] - covered), -r[2][0]))
+        selected.append({**excerpt(main), 'matched_terms': sorted(hits),
+                         'context_ranges': [excerpt(r) for r in context],
+                         'context_relation': 'enclosing branch and preceding alternative guards',
+                         'context_incomplete': False})
+        covered.update(hits)
+        occupied.update(range(main[0], main[1] + 1))
+        candidates = [r for r in candidates if not occupied.intersection(range(r[2][0], r[2][1]+1))]
+    return selected