commiter-cli 0.3.0__py3-none-any.whl

commiter/extractors/base.py

@@ -0,0 +1,40 @@
+"""Base class for all documentation extractors."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from tree_sitter import Tree
+    from commiter.models import FileClassification
+
+
+class BaseExtractor(ABC):
+    """Base class that all extractors inherit from.
+
+    Each extractor is responsible for finding a specific type of documentation
+    artifact (endpoints, dependencies, DB operations, etc.) from parsed files.
+    """
+
+    name: str = "base"
+
+    @abstractmethod
+    def can_handle(self, file_path: str, language: str | None) -> bool:
+        """Whether this extractor should run on the given file."""
+        ...
+
+    @abstractmethod
+    def extract(self, file_path: str, repo_name: str, tree: "Tree | None" = None, language: str | None = None) -> list[Any]:
+        """Extract documentation artifacts from a file.
+
+        Args:
+            file_path: Absolute path to the file.
+            repo_name: Name of the repository this file belongs to.
+            tree: Pre-parsed Tree-sitter tree (None for non-AST extractors).
+            language: Detected language of the file.
+
+        Returns:
+            A list of model instances (APIEndpoint, Dependency, etc.).
+        """
+        ...

commiter/extractors/db_operations.py

@@ -0,0 +1,69 @@
+"""Extract database operations from code using ORM/DB adapters."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from commiter.adapters.base import DBAdapter
+from commiter.adapters.supabase import SupabaseAdapter
+from commiter.adapters.prisma import PrismaAdapter
+from commiter.adapters.sqlalchemy import SQLAlchemyAdapter
+from commiter.adapters.raw_sql import RawSQLAdapter
+from commiter.extractors.base import BaseExtractor
+from commiter.models import DBOperation
+from commiter.parser import get_source
+
+if TYPE_CHECKING:
+    from tree_sitter import Tree
+
+_DB_ADAPTERS: list[DBAdapter] = [
+    SupabaseAdapter(),
+    PrismaAdapter(),
+    SQLAlchemyAdapter(),
+    RawSQLAdapter(),
+]
+
+
+class DBOperationExtractor(BaseExtractor):
+    name = "db_operations"
+
+    def can_handle(self, file_path: str, language: str | None) -> bool:
+        return language in ("python", "javascript", "typescript", "tsx")
+
+    def extract(self, file_path: str, repo_name: str, tree: Tree | None = None, language: str | None = None) -> list[DBOperation]:
+        if tree is None:
+            return []
+
+        source = get_source(file_path)
+        operations = []
+
+        for adapter in _DB_ADAPTERS:
+            # Check if the source contains hints of this ORM
+            if not self._source_has_orm_hints(source, adapter):
+                continue
+
+            matches = adapter.find_db_operations(tree, source)
+            for match in matches:
+                operations.append(DBOperation(
+                    repo=repo_name,
+                    file_path=file_path,
+                    line=match.line,
+                    operation_type=match.operation_type,
+                    table_name=match.table_name,
+                    orm_library=adapter.orm_name,
+                    filters=match.filters,
+                ))
+
+        return operations
+
+    def _source_has_orm_hints(self, source: bytes, adapter: DBAdapter) -> bool:
+        """Quick check if the source file likely uses this ORM."""
+        hints = {
+            "supabase": (b"supabase", b".table(", b".from(", b".from_("),
+            "prisma": (b"prisma.", b"@prisma"),
+            "sqlalchemy": (b"session.", b"sqlalchemy", b".query(", b"Model.query"),
+            "raw_sql": (b"cursor.execute", b".execute(", b"pool.query", b"client.query",
+                        b"knex.raw", b"db.query", b"connection.query", b".raw("),
+        }
+        orm_hints = hints.get(adapter.orm_name, ())
+        return any(hint in source for hint in orm_hints)

commiter/extractors/dependencies.py

@@ -0,0 +1,219 @@
+"""Extract dependencies from manifest files (package.json, requirements.txt, pyproject.toml, etc.)."""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+from tree_sitter import Tree
+
+from commiter.extractors.base import BaseExtractor
+from commiter.models import Dependency
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    import tomli as tomllib
+
+# Files this extractor handles
+MANIFEST_FILES = {
+    "package.json",
+    "requirements.txt",
+    "pyproject.toml",
+    "setup.cfg",
+    "Pipfile",
+    "go.mod",
+    "Cargo.toml",
+    "Gemfile",
+    "composer.json",
+}
+
+
+class DependencyExtractor(BaseExtractor):
+    name = "dependencies"
+
+    def can_handle(self, file_path: str, language: str | None) -> bool:
+        return Path(file_path).name in MANIFEST_FILES
+
+    def extract(self, file_path: str, repo_name: str, tree: Tree | None = None, language: str | None = None) -> list[Dependency]:
+        filename = Path(file_path).name
+        try:
+            content = Path(file_path).read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            return []
+
+        dispatch = {
+            "package.json": self._parse_package_json,
+            "requirements.txt": self._parse_requirements_txt,
+            "pyproject.toml": self._parse_pyproject_toml,
+            "setup.cfg": self._parse_setup_cfg,
+            "Pipfile": self._parse_pipfile,
+            "go.mod": self._parse_go_mod,
+            "Cargo.toml": self._parse_cargo_toml,
+            "Gemfile": self._parse_gemfile,
+            "composer.json": self._parse_composer_json,
+        }
+
+        parser = dispatch.get(filename)
+        if parser is None:
+            return []
+        return parser(content, file_path, repo_name)
+
+    def _parse_package_json(self, content: str, file_path: str, repo_name: str) -> list[Dependency]:
+        deps = []
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError:
+            return []
+
+        for name, version in data.get("dependencies", {}).items():
+            deps.append(Dependency(repo=repo_name, name=name, version_constraint=version, source_file=file_path, dev_only=False))
+        for name, version in data.get("devDependencies", {}).items():
+            deps.append(Dependency(repo=repo_name, name=name, version_constraint=version, source_file=file_path, dev_only=True))
+        return deps
+
+    def _parse_requirements_txt(self, content: str, file_path: str, repo_name: str) -> list[Dependency]:
+        deps = []
+        for line in content.splitlines():
+            line = line.strip()
+            if not line or line.startswith("#") or line.startswith("-"):
+                continue
+            # Handle name==version, name>=version, name~=version, bare name
+            match = re.match(r"^([a-zA-Z0-9_.-]+)\s*([><=!~]+.+)?", line)
+            if match:
+                name = match.group(1)
+                version = match.group(2) or "*"
+                deps.append(Dependency(repo=repo_name, name=name, version_constraint=version.strip(), source_file=file_path))
+        return deps
+
+    def _parse_pyproject_toml(self, content: str, file_path: str, repo_name: str) -> list[Dependency]:
+        deps = []
+        try:
+            data = tomllib.loads(content)
+        except Exception:
+            return []
+
+        for dep_str in data.get("project", {}).get("dependencies", []):
+            name, version = self._split_pep508(dep_str)
+            deps.append(Dependency(repo=repo_name, name=name, version_constraint=version, source_file=file_path))
+
+        for group_deps in data.get("project", {}).get("optional-dependencies", {}).values():
+            for dep_str in group_deps:
+                name, version = self._split_pep508(dep_str)
+                deps.append(Dependency(repo=repo_name, name=name, version_constraint=version, source_file=file_path, dev_only=True))
+        return deps
+
+    def _parse_setup_cfg(self, content: str, file_path: str, repo_name: str) -> list[Dependency]:
+        deps = []
+        in_install_requires = False
+        for line in content.splitlines():
+            stripped = line.strip()
+            if stripped == "install_requires =":
+                in_install_requires = True
+                continue
+            if in_install_requires:
+                if not stripped or (not line[0].isspace() and "=" in stripped):
+                    break
+                name, version = self._split_pep508(stripped)
+                deps.append(Dependency(repo=repo_name, name=name, version_constraint=version, source_file=file_path))
+        return deps
+
+    def _parse_pipfile(self, content: str, file_path: str, repo_name: str) -> list[Dependency]:
+        deps = []
+        section = None
+        for line in content.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("["):
+                section = stripped.strip("[]").lower()
+                continue
+            if section in ("packages", "dev-packages") and "=" in stripped:
+                name, _, version = stripped.partition("=")
+                name = name.strip().strip('"')
+                version = version.strip().strip('"')
+                deps.append(Dependency(
+                    repo=repo_name, name=name, version_constraint=version,
+                    source_file=file_path, dev_only=(section == "dev-packages"),
+                ))
+        return deps
+
+    def _parse_go_mod(self, content: str, file_path: str, repo_name: str) -> list[Dependency]:
+        deps = []
+        in_require = False
+        for line in content.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("require ("):
+                in_require = True
+                continue
+            if in_require:
+                if stripped == ")":
+                    in_require = False
+                    continue
+                parts = stripped.split()
+                if len(parts) >= 2:
+                    deps.append(Dependency(repo=repo_name, name=parts[0], version_constraint=parts[1], source_file=file_path))
+            elif stripped.startswith("require "):
+                parts = stripped.split()
+                if len(parts) >= 3:
+                    deps.append(Dependency(repo=repo_name, name=parts[1], version_constraint=parts[2], source_file=file_path))
+        return deps
+
+    def _parse_cargo_toml(self, content: str, file_path: str, repo_name: str) -> list[Dependency]:
+        deps = []
+        try:
+            data = tomllib.loads(content)
+        except Exception:
+            return []
+
+        for section_key in ("dependencies", "dev-dependencies"):
+            dev = section_key == "dev-dependencies"
+            for name, val in data.get(section_key, {}).items():
+                if isinstance(val, str):
+                    version = val
+                elif isinstance(val, dict):
+                    version = val.get("version", "*")
+                else:
+                    version = "*"
+                deps.append(Dependency(repo=repo_name, name=name, version_constraint=version, source_file=file_path, dev_only=dev))
+        return deps
+
+    def _parse_gemfile(self, content: str, file_path: str, repo_name: str) -> list[Dependency]:
+        deps = []
+        for line in content.splitlines():
+            stripped = line.strip()
+            match = re.match(r"""gem\s+['"]([^'"]+)['"](?:\s*,\s*['"]([^'"]+)['"])?""", stripped)
+            if match:
+                name = match.group(1)
+                version = match.group(2) or "*"
+                deps.append(Dependency(repo=repo_name, name=name, version_constraint=version, source_file=file_path))
+        return deps
+
+    def _parse_composer_json(self, content: str, file_path: str, repo_name: str) -> list[Dependency]:
+        deps = []
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError:
+            return []
+
+        for name, version in data.get("require", {}).items():
+            if name == "php":
+                continue
+            deps.append(Dependency(repo=repo_name, name=name, version_constraint=version, source_file=file_path))
+        for name, version in data.get("require-dev", {}).items():
+            deps.append(Dependency(repo=repo_name, name=name, version_constraint=version, source_file=file_path, dev_only=True))
+        return deps
+
+    @staticmethod
+    def _split_pep508(dep_str: str) -> tuple[str, str]:
+        """Split a PEP 508 dependency string into (name, version_constraint)."""
+        match = re.match(r"^([a-zA-Z0-9_.-]+)\s*(.*)", dep_str.strip())
+        if match:
+            name = match.group(1)
+            rest = match.group(2).strip().rstrip(";").strip()
+            # Remove environment markers
+            if ";" in rest:
+                rest = rest[:rest.index(";")].strip()
+            return name, rest if rest else "*"
+        return dep_str.strip(), "*"

commiter/generic_resolver.py

@@ -0,0 +1,204 @@
+"""Resolve generic and utility types to concrete field lists.
+
+Handles:
+  Pick<User, "id" | "name">  → User's fields filtered to id, name
+  Omit<User, "password">     → User's fields minus password
+  Partial<User>              → User's fields all optional
+  Required<User>             → User's fields all required
+  ApiResponse<User>          → ApiResponse's fields with T substituted by User
+  User[]                     → Recognized as array of User
+"""
+
+from __future__ import annotations
+
+from copy import copy
+from typing import Callable, TYPE_CHECKING
+
+from commiter.parser import parse_generic_type
+
+if TYPE_CHECKING:
+    from commiter.parser import TypeField
+
+# Built-in utility types that we can resolve
+UTILITY_TYPES = {"Pick", "Omit", "Partial", "Required", "Readonly"}
+
+# Wrapper types that we unwrap to get the inner type
+WRAPPER_TYPES = {"Array", "Promise", "Set"}
+
+
+def resolve_generic_type(
+    type_str: str,
+    type_lookup: Callable[[str], "list[TypeField] | None"],
+    generic_def_lookup: Callable[[str], "tuple[list[TypeField], list[str]] | None"] = None,
+) -> "list[TypeField] | None":
+    """Resolve a generic type string to concrete fields.
+
+    Args:
+        type_str: The type string (e.g. "Pick<User, \\"id\\" | \\"name\\">").
+        type_lookup: Callback to look up a type name and get its fields.
+        generic_def_lookup: Callback to look up a generic type definition,
+                            returns (fields, generic_params) or None.
+
+    Returns:
+        Resolved list of TypeField, or None if not resolvable.
+    """
+    parsed = parse_generic_type(type_str)
+    if not parsed:
+        return None
+
+    base, args = parsed
+    if not args:
+        return None
+
+    # Built-in utility types
+    if base in UTILITY_TYPES:
+        return _resolve_utility_type(base, args, type_lookup)
+
+    # Wrapper types — unwrap to inner type
+    if base in WRAPPER_TYPES:
+        inner_type = args[0]
+        return type_lookup(inner_type)
+
+    # Custom generic type: ApiResponse<User>
+    if generic_def_lookup:
+        return _resolve_custom_generic(base, args, type_lookup, generic_def_lookup)
+
+    return None
+
+
+def _resolve_utility_type(
+    utility: str,
+    args: list[str],
+    type_lookup: Callable[[str], "list[TypeField] | None"],
+) -> "list[TypeField] | None":
+    """Resolve a built-in utility type."""
+
+    if utility == "Pick" and len(args) >= 2:
+        return _resolve_pick(args[0], args[1], type_lookup)
+
+    elif utility == "Omit" and len(args) >= 2:
+        return _resolve_omit(args[0], args[1], type_lookup)
+
+    elif utility == "Partial" and len(args) >= 1:
+        return _resolve_partial(args[0], type_lookup)
+
+    elif utility == "Required" and len(args) >= 1:
+        return _resolve_required(args[0], type_lookup)
+
+    elif utility == "Readonly" and len(args) >= 1:
+        # Readonly doesn't change field structure, just marks them readonly
+        return type_lookup(args[0])
+
+    return None
+
+
+def _resolve_pick(
+    base_type: str,
+    selector: str,
+    type_lookup: Callable[[str], "list[TypeField] | None"],
+) -> "list[TypeField] | None":
+    """Resolve Pick<User, "id" | "name"> to filtered fields."""
+    fields = type_lookup(base_type)
+    if not fields:
+        return None
+
+    selected = _parse_field_selector(selector)
+    return [f for f in fields if f.name in selected]
+
+
+def _resolve_omit(
+    base_type: str,
+    selector: str,
+    type_lookup: Callable[[str], "list[TypeField] | None"],
+) -> "list[TypeField] | None":
+    """Resolve Omit<User, "password" | "email"> to fields minus excluded."""
+    fields = type_lookup(base_type)
+    if not fields:
+        return None
+
+    excluded = _parse_field_selector(selector)
+    return [f for f in fields if f.name not in excluded]
+
+
+def _resolve_partial(
+    base_type: str,
+    type_lookup: Callable[[str], "list[TypeField] | None"],
+) -> "list[TypeField] | None":
+    """Resolve Partial<User> to all fields marked optional."""
+    fields = type_lookup(base_type)
+    if not fields:
+        return None
+
+    result = []
+    for f in fields:
+        new_f = copy(f)
+        new_f.optional = True
+        result.append(new_f)
+    return result
+
+
+def _resolve_required(
+    base_type: str,
+    type_lookup: Callable[[str], "list[TypeField] | None"],
+) -> "list[TypeField] | None":
+    """Resolve Required<User> to all fields marked required."""
+    fields = type_lookup(base_type)
+    if not fields:
+        return None
+
+    result = []
+    for f in fields:
+        new_f = copy(f)
+        new_f.optional = False
+        result.append(new_f)
+    return result
+
+
+def _resolve_custom_generic(
+    base: str,
+    args: list[str],
+    type_lookup: Callable[[str], "list[TypeField] | None"],
+    generic_def_lookup: Callable[[str], "tuple[list[TypeField], list[str]] | None"],
+) -> "list[TypeField] | None":
+    """Resolve a custom generic like ApiResponse<User>.
+
+    Looks up ApiResponse's definition (which has generic_params ["T"]),
+    then substitutes T → User in all field type_str values.
+    """
+    definition = generic_def_lookup(base)
+    if not definition:
+        return None
+
+    def_fields, generic_params = definition
+
+    if not generic_params or len(generic_params) != len(args):
+        # Param count mismatch — return fields as-is
+        return list(def_fields)
+
+    # Build substitution map: T → User
+    subs = dict(zip(generic_params, args))
+
+    result = []
+    for f in def_fields:
+        new_f = copy(f)
+        # Substitute generic params in type_str
+        new_type = new_f.type_str
+        for param, replacement in subs.items():
+            if new_type == param:
+                new_type = replacement
+            elif param in new_type:
+                new_type = new_type.replace(param, replacement)
+        new_f.type_str = new_type
+        result.append(new_f)
+
+    return result
+
+
+def _parse_field_selector(selector: str) -> set[str]:
+    """Parse a field selector like '"id" | "name"' into a set of field names."""
+    names = set()
+    for part in selector.split("|"):
+        name = part.strip().strip("'\"")
+        if name:
+            names.add(name)
+    return names

commiter/handler_index.py

@@ -0,0 +1,97 @@
+"""Cross-file handler type index: maps class controller methods to their parameter types.
+
+Resolves the body type when a route uses a class-based handler like:
+  router.post("/", projectController.create)
+
+where the type annotation Request<{}, {}, CreateProjectBody> lives in the
+controller's class definition in a separate file.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+from commiter.parser import (
+    find_ts_class_methods,
+    find_ts_class_instances,
+    ClassMethod,
+    ClassInstance,
+)
+
+if TYPE_CHECKING:
+    from tree_sitter import Tree
+
+
+class HandlerIndex:
+    """Index of class method type annotations across all files.
+
+    Built during Pass 1, queried during Pass 2 to resolve body types
+    for class-based route handlers.
+
+    Usage:
+        idx = HandlerIndex()
+        idx.index_file(path, tree, source, "typescript")
+        body_type = idx.get_body_type("projectController.create")
+        # Returns "CreateProjectBody"
+    """
+
+    def __init__(self) -> None:
+        # "ClassName.methodName" -> ClassMethod
+        self._methods: dict[str, ClassMethod] = {}
+        # "varName" -> "ClassName" (e.g. "projectController" -> "ProjectController")
+        self._instances: dict[str, str] = {}
+
+    def index_file(self, file_path: str, tree: "Tree", source: bytes, language: str) -> None:
+        """Scan a file for class method signatures and instance variables."""
+        if language not in ("javascript", "typescript", "tsx"):
+            return
+
+        abs_path = os.path.abspath(file_path)
+
+        # Index class methods
+        methods = find_ts_class_methods(tree.root_node, source, file_path=abs_path)
+        for method in methods:
+            key = f"{method.class_name}.{method.method_name}"
+            self._methods[key] = method
+
+        # Index instance variables: const x = new ClassName()
+        instances = find_ts_class_instances(tree.root_node, source, file_path=abs_path)
+        for inst in instances:
+            self._instances[inst.var_name] = inst.class_name
+
+    def get_body_type(self, handler_ref: str) -> str | None:
+        """Resolve a handler reference like 'projectController.create' to its body type.
+
+        Steps:
+        1. Split into var_name='projectController', method='create'
+        2. Look up var_name in _instances -> 'ProjectController'
+        3. Look up 'ProjectController.create' in _methods
+        4. Return request_body_type
+        """
+        if "." not in handler_ref:
+            return None
+
+        var_name, method_name = handler_ref.rsplit(".", 1)
+
+        # Resolve variable to class name
+        class_name = self._instances.get(var_name)
+        if not class_name:
+            # Try using the variable name as-is (might already be the class name)
+            class_name = var_name
+
+        # Look up the method
+        key = f"{class_name}.{method_name}"
+        method = self._methods.get(key)
+        if method:
+            return method.request_body_type
+
+        return None
+
+    @property
+    def method_count(self) -> int:
+        return len(self._methods)
+
+    @property
+    def instance_count(self) -> int:
+        return len(self._instances)

commiter/lib.py

@@ -0,0 +1,63 @@
+"""
+Public library API for `commiter-cli`.
+
+Programmatic callers (e.g. the Commiter backend running enrich on behalf of
+users) should import from this module. Everything else under `commiter.cli`,
+`commiter.scanner`, `commiter.report.*` etc. is considered internal and is
+not a stable surface — it may change between releases.
+"""
+
+import json
+from pathlib import Path
+from typing import Union
+
+from commiter.scanner import scan_repos_full
+from commiter.report.architecture import generate_architecture
+
+
+PathLike = Union[str, Path]
+
+
+def generate_snapshot(
+    repo_paths: list[PathLike],
+    *,
+    extra_excludes: list[str] | None = None,
+) -> dict:
+    """
+    Scan one or more local repository directories and return the architecture
+    snapshot as a Python dict — the same shape the CLI POSTs to
+    `/v1/architecture/upload` on commiter.dev.
+
+    Does *not* upload. The caller is responsible for persisting the result.
+
+    Args:
+        repo_paths: One or more on-disk paths, each pointing at the root of a
+            repository (i.e. a directory containing the project's source).
+        extra_excludes: Additional path-substring exclusions forwarded to the
+            scanner. Useful if the caller wants to skip vendored dirs beyond
+            the scanner's built-in list.
+
+    Returns:
+        A dict with top-level keys `nodes`, `edges`, `fileTree`,
+        `nodeAnalysis`, `nodeHashes`, and `repoFullNames`. See the upload
+        endpoint's contract for field-level details.
+
+    Raises:
+        FileNotFoundError: if any of the supplied paths doesn't exist.
+    """
+    if not repo_paths:
+        raise ValueError("repo_paths must contain at least one path")
+
+    str_paths: list[str] = []
+    for p in repo_paths:
+        path = Path(p)
+        if not path.exists():
+            raise FileNotFoundError(f"Repository path does not exist: {path}")
+        str_paths.append(str(path))
+
+    results = scan_repos_full(str_paths, extra_excludes or [])
+    json_str = generate_architecture(results)
+    return json.loads(json_str)
+
+
+__all__ = ["generate_snapshot"]