entrygraph 0.1.0__py3-none-any.whl

entrygraph/data/sinks/ruby.toml

@@ -0,0 +1,102 @@
+# Built-in Ruby sink/source catalog.
+# Patterns are brace-globs matched against canonical callee qnames:
+#   unresolved bare calls      -> "rb:system"
+#   unresolved attribute calls -> "rb:*.execute"
+#   fuzzy/import project calls  -> the project qname (not matched here)
+
+[[sink]]
+id = "rb.command-exec.kernel"
+category = "command_exec"
+severity = "high"
+callee = "rb:{system,exec,spawn,fork,syscall}"
+description = "Kernel command execution"
+
+[[sink]]
+id = "rb.command-exec.attr"
+category = "command_exec"
+severity = "high"
+callee = "rb:*.{system,exec,spawn}"
+description = "Command execution on a receiver (Open3, Process, etc.)"
+
+[[sink]]
+id = "rb.code-eval"
+category = "code_eval"
+severity = "critical"
+callee = "rb:{eval,instance_eval,class_eval,module_eval,define_method}"
+description = "Dynamic code evaluation"
+
+[[sink]]
+id = "rb.code-eval.attr"
+category = "code_eval"
+severity = "critical"
+callee = "rb:*.{eval,instance_eval,class_eval,module_eval}"
+description = "Dynamic code evaluation on a receiver"
+
+[[sink]]
+id = "rb.sql-execute"
+category = "sql"
+severity = "medium"
+callee = "rb:*.{execute,find_by_sql,exec_query,select_all}"
+description = "SQL statement execution (receiver-agnostic)"
+
+[[source]]
+id = "rb.sinatra.params"
+category = "http_input"
+callee = "rb:*.params*"
+description = "Sinatra/Rails request params"
+
+# --- C5 deepening ---
+# NOTE: `X::Y.method` receivers canonicalize to `rb:*.method` when unresolved,
+# so receiver-agnostic patterns (with arg hints) are used for stdlib sinks.
+[[sink]]
+id = "rb.command-exec.open3"
+category = "command_exec"
+severity = "high"
+callee = "rb:*.{capture2,capture2e,capture3,popen2,popen3,popen2e}"
+description = "Open3 command execution"
+
+[[sink]]
+id = "rb.deserialization"
+category = "deserialization"
+severity = "high"
+callee = "rb:*.{load,restore,unsafe_load,object_load}"
+require_arg_hint = "params|\\#\\{|@|\\("
+description = "Unsafe deserialization (Marshal/YAML/Oj)"
+
+[[sink]]
+id = "rb.code-eval.send"
+category = "code_eval"
+severity = "high"
+callee = "rb:*.{send,public_send,constantize,const_get}"
+require_arg_hint = "params|\\#\\{|@"
+description = "Dynamic dispatch / constant resolution from input"
+
+[[sink]]
+id = "rb.path-traversal"
+category = "path_traversal"
+severity = "medium"
+callee = "rb:*.{read,open,write,binread}"
+require_arg_hint = "params|\\#\\{|\\+"
+description = "File access with a non-literal path"
+
+[[sink]]
+id = "rb.ssrf"
+category = "ssrf"
+severity = "medium"
+callee = "rb:*.{get_response,post_form}"
+description = "Net::HTTP outbound request"
+
+[[sink]]
+id = "rb.template-injection"
+category = "template_injection"
+severity = "high"
+callee = "rb:*.{new,result}"
+require_arg_hint = "\\#\\{"
+description = "ERB template built from interpolated input"
+
+[[source]]
+id = "rb.env"
+category = "env_input"
+callee = "rb:*.{fetch}"
+require_arg_hint = "ENV"
+description = "Environment variable input"

entrygraph/data/sinks/rust.toml

@@ -0,0 +1,68 @@
+# Built-in Rust sink/source catalog.
+# Patterns are brace-globs matched against canonical callee qnames. Rust qnames
+# normalize `::` -> `.`, so:
+#   import-expanded external calls -> "rs:std.process.Command.new"
+#   unresolved method calls        -> "rs:*.Command.new"
+#   macro invocations              -> "rs:sqlx.query" (the trailing `!` is stripped)
+# `use std::fs` binds `fs`, so file sinks list both `std.fs.*` and `fs.*` aliases.
+
+[[sink]]
+id = "rust.command-exec"
+category = "command_exec"
+severity = "high"
+callee = "rs:{std.process.Command.new,Command.new,*.Command.new}"
+description = "Subprocess construction via std::process::Command (sink on the builder, not spawn/output)"
+
+[[sink]]
+id = "rust.sql-query"
+category = "sql"
+severity = "medium"
+callee = "rs:{sqlx.query,sqlx.query_as,sqlx.query_scalar,sqlx.raw_sql,diesel.sql_query,*.sql_query}"
+description = "SQL execution (matches both fn calls and sqlx!/diesel! macros after `!`-stripping)"
+
+[[sink]]
+id = "rust.path-traversal"
+category = "path_traversal"
+severity = "medium"
+callee = "rs:std.fs.{write,read,read_to_string,remove_file,remove_dir_all,copy,rename}"
+description = "Filesystem access via std::fs (verify the path is not attacker-controlled)"
+
+[[sink]]
+id = "rust.file-alias"
+category = "path_traversal"
+severity = "medium"
+callee = "rs:fs.{write,read,read_to_string,remove_file}"
+description = "Filesystem access via a `use std::fs` alias"
+
+[[sink]]
+id = "rust.ssrf"
+category = "ssrf"
+severity = "low"
+callee = "rs:{reqwest.get,reqwest.blocking.get}"
+description = "Outbound HTTP request via reqwest"
+
+[[sink]]
+id = "rust.dynamic-load"
+category = "dynamic_load"
+severity = "medium"
+callee = "rs:{libloading.Library.new,*.Library.new}"
+description = "Dynamic library / shared object load via libloading"
+
+[[sink]]
+id = "rust.deserialization"
+category = "deserialization"
+severity = "medium"
+callee = "rs:{bincode.deserialize,serde_yaml.from_str}"
+description = "Deserialization of untrusted input"
+
+[[source]]
+id = "rust.env"
+category = "env_input"
+callee = "rs:{std.env.args,env.args,std.env.var,env.var}"
+description = "Process arguments / environment variable input"
+
+[[source]]
+id = "rust.stdin"
+category = "stdin_input"
+callee = "rs:{std.io.stdin,io.stdin}"
+description = "Standard input"

entrygraph/db/engine.py

@@ -0,0 +1,34 @@
+"""Engine construction with SQLite tuned for bulk-write and read-heavy workloads."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from sqlalchemy import Engine, create_engine, event
+from sqlalchemy.orm import Session, sessionmaker
+
+_PRAGMAS = (
+    "PRAGMA journal_mode=WAL",
+    "PRAGMA synchronous=NORMAL",
+    "PRAGMA foreign_keys=ON",
+    "PRAGMA temp_store=MEMORY",
+    "PRAGMA cache_size=-64000",  # 64 MiB page cache
+    "PRAGMA mmap_size=268435456",  # 256 MiB
+)
+
+
+def make_engine(db_path: str | Path) -> Engine:
+    engine = create_engine(f"sqlite:///{Path(db_path)}")
+
+    @event.listens_for(engine, "connect")
+    def _set_pragmas(dbapi_conn, _record) -> None:  # pragma: no cover - exercised via tests
+        cursor = dbapi_conn.cursor()
+        for pragma in _PRAGMAS:
+            cursor.execute(pragma)
+        cursor.close()
+
+    return engine
+
+
+def make_session_factory(engine: Engine) -> sessionmaker[Session]:
+    return sessionmaker(engine, expire_on_commit=False)

entrygraph/db/meta.py

@@ -0,0 +1,70 @@
+"""Schema-version bookkeeping.
+
+The database is a rebuildable cache of the repository, not a system of record:
+instead of migrations, a version mismatch either raises (read paths) or drops
+and recreates every table (index paths).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from sqlalchemy import Engine, select
+from sqlalchemy.orm import Session
+
+from entrygraph.db.models import Base, Meta
+from entrygraph.errors import SchemaMismatchError
+
+SCHEMA_VERSION = 2
+
+
+def create_schema(engine: Engine) -> None:
+    """Create all tables and stamp the current schema version."""
+    from entrygraph import __version__
+
+    Base.metadata.create_all(engine)
+    with Session(engine) as session:
+        existing = session.get(Meta, "schema_version")
+        if existing is None:
+            session.add_all(
+                [
+                    Meta(key="schema_version", value=str(SCHEMA_VERSION)),
+                    Meta(key="entrygraph_version", value=__version__),
+                    Meta(key="created_at", value=datetime.now(timezone.utc).isoformat()),
+                ]
+            )
+            session.commit()
+
+
+def stored_schema_version(engine: Engine) -> int | None:
+    """Return the version stamped in the db, or None if the meta table is absent/empty."""
+    with Session(engine) as session:
+        try:
+            row = session.execute(select(Meta.value).where(Meta.key == "schema_version")).scalar()
+        except Exception:
+            return None
+    return int(row) if row is not None else None
+
+
+def check_schema(engine: Engine) -> None:
+    """Raise SchemaMismatchError unless the db matches SCHEMA_VERSION."""
+    version = stored_schema_version(engine)
+    if version != SCHEMA_VERSION:
+        raise SchemaMismatchError(
+            f"database schema version {version!r} != expected {SCHEMA_VERSION}; "
+            "re-run `entrygraph index` to rebuild the index"
+        )
+
+
+def ensure_schema(engine: Engine) -> bool:
+    """Make the db usable for indexing: create if empty, rebuild if mismatched.
+
+    Returns True if the schema was (re)created from scratch.
+    """
+    version = stored_schema_version(engine)
+    if version == SCHEMA_VERSION:
+        return False
+    if version is not None:
+        Base.metadata.drop_all(engine)
+    create_schema(engine)
+    return True

entrygraph/db/models.py

@@ -0,0 +1,176 @@
+"""SQLAlchemy 2.0 typed ORM models — the single definition of the on-disk schema.
+
+Bump ``meta.SCHEMA_VERSION`` whenever anything here changes; the database is a
+rebuildable cache, so a version mismatch triggers a full re-index rather than a
+migration.
+
+There are deliberately no relationship() constructs: all writes go through the
+bulk-insert writer with app-assigned PKs, which inserts tables in dependency
+order itself (repositories -> files -> symbols -> edges/entrypoints). Without
+relationships the unit of work cannot order cross-table inserts within one
+flush, so ad-hoc sessions must flush between dependency levels.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import (
+    BigInteger,
+    Enum,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Text,
+    UniqueConstraint,
+    text,
+)
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+from entrygraph.kinds import EdgeKind, EntrypointKind, SymbolKind
+
+
+def _values(enum_cls: type) -> list[str]:
+    return [m.value for m in enum_cls]
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class Meta(Base):
+    """Key/value schema metadata; checked on every open()."""
+
+    __tablename__ = "meta"
+
+    key: Mapped[str] = mapped_column(String(64), primary_key=True)
+    value: Mapped[str] = mapped_column(Text)
+
+
+class Repository(Base):
+    __tablename__ = "repositories"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    root_path: Mapped[str] = mapped_column(Text, unique=True)
+    indexed_at: Mapped[datetime | None]
+    # Bumped once per index run; drives adjacency-cache invalidation.
+    index_generation: Mapped[int] = mapped_column(Integer, default=0)
+    file_count: Mapped[int] = mapped_column(Integer, default=0)
+    symbol_count: Mapped[int] = mapped_column(Integer, default=0)
+
+
+class File(Base):
+    __tablename__ = "files"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)  # app-assigned
+    repo_id: Mapped[int] = mapped_column(ForeignKey("repositories.id", ondelete="CASCADE"))
+    path: Mapped[str] = mapped_column(Text)  # repo-relative, posix separators
+    language: Mapped[str | None] = mapped_column(String(32))
+    content_hash: Mapped[str] = mapped_column(String(64))  # blake2b-128 hex
+    size_bytes: Mapped[int] = mapped_column(BigInteger)
+    mtime_ns: Mapped[int] = mapped_column(BigInteger)
+    generation: Mapped[int] = mapped_column(Integer)  # index_generation at last (re)index
+    skip_reason: Mapped[str | None] = mapped_column(String(32))  # too_large/binary/minified/parse_error
+
+    __table_args__ = (
+        UniqueConstraint("repo_id", "path", name="uq_files_repo_path"),
+        Index("ix_files_repo_lang", "repo_id", "language"),
+    )
+
+
+class Symbol(Base):
+    __tablename__ = "symbols"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)  # app-assigned
+    # NULL for kind=external placeholder symbols (no defining file in the repo).
+    file_id: Mapped[int | None] = mapped_column(ForeignKey("files.id", ondelete="CASCADE"))
+    kind: Mapped[SymbolKind] = mapped_column(
+        Enum(SymbolKind, native_enum=False, length=16, values_callable=_values)
+    )
+    name: Mapped[str] = mapped_column(Text)  # unqualified: "run"
+    qname: Mapped[str] = mapped_column(Text)  # "pkg.mod.Class.run" / external: "py:subprocess.run"
+    parent_id: Mapped[int | None] = mapped_column(ForeignKey("symbols.id", ondelete="CASCADE"))
+    start_line: Mapped[int] = mapped_column(Integer, default=0)
+    end_line: Mapped[int] = mapped_column(Integer, default=0)
+    start_col: Mapped[int] = mapped_column(Integer, default=0)
+    signature: Mapped[str | None] = mapped_column(Text)
+    docstring: Mapped[str | None] = mapped_column(Text)
+    is_exported: Mapped[bool] = mapped_column(default=True)
+
+    __table_args__ = (
+        Index("ix_symbols_file", "file_id"),
+        Index("ix_symbols_qname", "qname"),
+        Index("ix_symbols_kind_name", "kind", "name"),
+        Index("ix_symbols_parent", "parent_id"),
+    )
+
+
+class Edge(Base):
+    __tablename__ = "edges"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)  # app-assigned
+    kind: Mapped[EdgeKind] = mapped_column(
+        Enum(EdgeKind, native_enum=False, length=16, values_callable=_values)
+    )
+    src_symbol_id: Mapped[int] = mapped_column(ForeignKey("symbols.id", ondelete="CASCADE"))
+    # SET NULL so a deleted target degrades the edge to "unresolved" instead of
+    # deleting it; dst_qname keeps the textual target for later re-resolution.
+    dst_symbol_id: Mapped[int | None] = mapped_column(ForeignKey("symbols.id", ondelete="SET NULL"))
+    dst_qname: Mapped[str] = mapped_column(Text)
+    # Denormalized so incremental re-index can wipe a file's edges in one statement.
+    src_file_id: Mapped[int] = mapped_column(ForeignKey("files.id", ondelete="CASCADE"))
+    line: Mapped[int] = mapped_column(Integer, default=0)
+    confidence: Mapped[int] = mapped_column(Integer, default=0)  # kinds.Confidence value
+    arg_preview: Mapped[str | None] = mapped_column(Text)
+    sink_id: Mapped[str | None] = mapped_column(String(64))  # pre-tagged at index time
+    # Edge provenance for edges not resolved by the direct import/scope pass:
+    # "cha" (class-hierarchy candidate), "dynamic" (getattr/computed call),
+    # "reexport" (chased through a barrel file). NULL for directly-resolved edges.
+    via: Mapped[str | None] = mapped_column(String(12))
+
+    __table_args__ = (
+        Index("ix_edges_src_kind", "src_symbol_id", "kind"),
+        Index("ix_edges_dst_kind", "dst_symbol_id", "kind"),
+        Index("ix_edges_srcfile", "src_file_id"),
+        Index("ix_edges_unresolved", "dst_qname", sqlite_where=text("dst_symbol_id IS NULL")),
+        Index("ix_edges_sink", "sink_id", sqlite_where=text("sink_id IS NOT NULL")),
+        Index("ix_edges_via", "via", sqlite_where=text("via IS NOT NULL")),
+    )
+
+
+class Entrypoint(Base):
+    __tablename__ = "entrypoints"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)  # app-assigned
+    kind: Mapped[EntrypointKind] = mapped_column(
+        Enum(EntrypointKind, native_enum=False, length=24, values_callable=_values)
+    )
+    framework: Mapped[str | None] = mapped_column(String(32))
+    symbol_id: Mapped[int] = mapped_column(ForeignKey("symbols.id", ondelete="CASCADE"))
+    route: Mapped[str | None] = mapped_column(Text)
+    http_method: Mapped[str | None] = mapped_column(String(16))
+    extra: Mapped[str | None] = mapped_column(Text)  # JSON blob: decorator args etc.
+
+    __table_args__ = (
+        Index("ix_entrypoints_kind_fw", "kind", "framework"),
+        Index("ix_entrypoints_symbol", "symbol_id"),
+    )
+
+
+class Detection(Base):
+    """Per-repo detected languages and frameworks."""
+
+    __tablename__ = "detections"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    repo_id: Mapped[int] = mapped_column(ForeignKey("repositories.id", ondelete="CASCADE"))
+    category: Mapped[str] = mapped_column(String(16))  # "language" | "framework"
+    name: Mapped[str] = mapped_column(String(48))
+    version: Mapped[str | None] = mapped_column(String(32))
+    confidence: Mapped[float] = mapped_column(default=1.0)
+    evidence: Mapped[str | None] = mapped_column(Text)  # JSON: file counts, manifest paths
+
+    __table_args__ = (
+        UniqueConstraint("repo_id", "category", "name", name="uq_detections"),
+    )

entrygraph/db/queries.py

@@ -0,0 +1,155 @@
+"""SELECT builders and row -> DTO conversion.
+
+Globs use ``*`` and ``?`` (translated to SQL LIKE with escaping); a pattern
+without glob characters is an exact match.
+"""
+
+from __future__ import annotations
+
+import json
+
+from sqlalchemy import Select, select
+from sqlalchemy.orm import Session
+
+from entrygraph.db import models
+from entrygraph.kinds import EntrypointKind, SymbolKind
+from entrygraph.results import Entrypoint, FileInfo, Symbol
+
+_LIKE_ESCAPE = "\\"
+
+
+def glob_to_like(pattern: str) -> str:
+    escaped = (
+        pattern.replace(_LIKE_ESCAPE, _LIKE_ESCAPE * 2)
+        .replace("%", _LIKE_ESCAPE + "%")
+        .replace("_", _LIKE_ESCAPE + "_")
+    )
+    return escaped.replace("*", "%").replace("?", "_")
+
+
+def _match(column, pattern: str):
+    if "*" in pattern or "?" in pattern:
+        return column.like(glob_to_like(pattern), escape=_LIKE_ESCAPE)
+    return column == pattern
+
+
+def symbol_to_dto(row: models.Symbol, file_path: str | None) -> Symbol:
+    return Symbol(
+        id=row.id,
+        kind=row.kind.value,
+        name=row.name,
+        qname=row.qname,
+        file=file_path,
+        start_line=row.start_line,
+        end_line=row.end_line,
+        signature=row.signature,
+        docstring=row.docstring,
+        is_exported=row.is_exported,
+    )
+
+
+def _symbol_select() -> Select:
+    return select(models.Symbol, models.File.path).join(
+        models.File, models.Symbol.file_id == models.File.id, isouter=True
+    )
+
+
+def select_symbols(
+    session: Session,
+    *,
+    kind: str | SymbolKind | None = None,
+    name: str | None = None,
+    qname: str | None = None,
+    file: str | None = None,
+    include_external: bool = False,
+    limit: int | None = None,
+    offset: int | None = None,
+) -> list[Symbol]:
+    stmt = _symbol_select().order_by(models.Symbol.qname)
+    if kind is not None:
+        stmt = stmt.where(models.Symbol.kind == SymbolKind(kind))
+    elif not include_external:
+        stmt = stmt.where(models.Symbol.kind != SymbolKind.EXTERNAL)
+    if name is not None:
+        stmt = stmt.where(_match(models.Symbol.name, name))
+    if qname is not None:
+        stmt = stmt.where(_match(models.Symbol.qname, qname))
+    if file is not None:
+        stmt = stmt.where(_match(models.File.path, file))
+    if limit is not None:
+        stmt = stmt.limit(limit)
+    if offset is not None:
+        stmt = stmt.offset(offset)
+    return [symbol_to_dto(sym, path) for sym, path in session.execute(stmt)]
+
+
+def symbols_by_ids(session: Session, ids: set[int]) -> dict[int, Symbol]:
+    if not ids:
+        return {}
+    stmt = _symbol_select().where(models.Symbol.id.in_(ids))
+    return {sym.id: symbol_to_dto(sym, path) for sym, path in session.execute(stmt)}
+
+
+def symbol_ids_matching(session: Session, pattern: str) -> set[int]:
+    """Symbol ids whose qname matches a glob (or exact qname)."""
+    return set(
+        session.execute(
+            select(models.Symbol.id).where(_match(models.Symbol.qname, pattern))
+        ).scalars()
+    )
+
+
+def select_files(
+    session: Session, *, language: str | None = None, path: str | None = None
+) -> list[FileInfo]:
+    stmt = select(models.File).order_by(models.File.path)
+    if language is not None:
+        stmt = stmt.where(models.File.language == language)
+    if path is not None:
+        stmt = stmt.where(_match(models.File.path, path))
+    return [
+        FileInfo(
+            id=f.id,
+            path=f.path,
+            language=f.language,
+            size_bytes=f.size_bytes,
+            skip_reason=f.skip_reason,
+        )
+        for f in session.execute(stmt).scalars()
+    ]
+
+
+def select_entrypoints(
+    session: Session,
+    *,
+    kind: str | EntrypointKind | None = None,
+    framework: str | None = None,
+    route: str | None = None,
+    limit: int | None = None,
+) -> list[Entrypoint]:
+    stmt = (
+        select(models.Entrypoint, models.Symbol, models.File.path)
+        .join(models.Symbol, models.Entrypoint.symbol_id == models.Symbol.id)
+        .join(models.File, models.Symbol.file_id == models.File.id, isouter=True)
+        .order_by(models.Entrypoint.id)
+    )
+    if kind is not None:
+        stmt = stmt.where(models.Entrypoint.kind == EntrypointKind(kind))
+    if framework is not None:
+        stmt = stmt.where(models.Entrypoint.framework == framework)
+    if route is not None:
+        stmt = stmt.where(_match(models.Entrypoint.route, route))
+    if limit is not None:
+        stmt = stmt.limit(limit)
+    return [
+        Entrypoint(
+            id=ep.id,
+            kind=ep.kind.value,
+            framework=ep.framework,
+            symbol=symbol_to_dto(sym, path),
+            route=ep.route,
+            http_method=ep.http_method,
+            extra=json.loads(ep.extra) if ep.extra else {},
+        )
+        for ep, sym, path in session.execute(stmt)
+    ]

entrygraph/detect/entrypoints/__init__.py

@@ -0,0 +1,22 @@
+"""Entrypoint rule registry. Importing this package registers built-in rules."""
+
+import importlib
+
+from entrygraph.detect.entrypoints.base import (
+    EntrypointRule,
+    all_rules,
+    register,
+    rules_for,
+)
+
+# Importing each rule module registers its rules. Done defensively so a
+# language still under construction doesn't break the whole registry.
+for _mod in ("python", "javascript", "golang", "java", "ruby", "csharp", "php", "rust"):
+    _full = f"entrygraph.detect.entrypoints.{_mod}"
+    try:
+        importlib.import_module(_full)
+    except ModuleNotFoundError as _exc:  # noqa: PERF203
+        if _exc.name != _full:
+            raise
+
+__all__ = ["EntrypointRule", "all_rules", "register", "rules_for"]