commiter-cli 0.3.0__py3-none-any.whl

commiter/correlator.py

@@ -0,0 +1,208 @@
+"""Cross-repo correlator: matches frontend API calls to backend endpoints."""
+
+from __future__ import annotations
+
+import re
+
+from commiter.models import (
+    APICall,
+    APIEndpoint,
+    RepoDocumentation,
+    ServiceRelationship,
+)
+
+
+def correlate(docs: list[RepoDocumentation]) -> list[ServiceRelationship]:
+    """Match frontend API calls to backend endpoints across all repos.
+
+    Returns a list of ServiceRelationship objects describing connections.
+    """
+    all_endpoints: list[APIEndpoint] = []
+    all_calls: list[APICall] = []
+
+    for doc in docs:
+        all_endpoints.extend(doc.endpoints)
+        all_calls.extend(doc.api_calls)
+
+    relationships = []
+
+    for call in all_calls:
+        best_match = _find_best_endpoint_match(call, all_endpoints)
+        if best_match:
+            endpoint, confidence = best_match
+            relationships.append(ServiceRelationship(
+                source_repo=call.repo,
+                target_repo=endpoint.repo,
+                connection_type="api_call",
+                source_file=f"{call.file_path}:{call.line}",
+                target_endpoint=f"{endpoint.http_method} {endpoint.route_pattern}",
+                confidence=confidence,
+            ))
+
+    # Detect shared dependencies between repos
+    relationships.extend(_find_shared_dependencies(docs))
+
+    # Detect shared database tables between repos
+    relationships.extend(_find_shared_db_tables(docs))
+
+    return relationships
+
+
+def _find_best_endpoint_match(
+    call: APICall, endpoints: list[APIEndpoint]
+) -> tuple[APIEndpoint, float] | None:
+    """Find the best matching backend endpoint for a frontend API call."""
+    best: tuple[APIEndpoint, float] | None = None
+
+    call_path = _normalize_url(call.url_pattern)
+    call_method = call.http_method.upper()
+
+    for ep in endpoints:
+        ep_path = _normalize_route(ep.route_pattern)
+        ep_method = ep.http_method.upper()
+
+        # Method must match (or one is ALL)
+        if call_method != ep_method and call_method != "ALL" and ep_method != "ALL":
+            continue
+
+        confidence = _path_similarity(call_path, ep_path)
+        if confidence > 0 and (best is None or confidence > best[1]):
+            best = (ep, confidence)
+
+    return best
+
+
+def _normalize_url(url: str) -> str:
+    """Normalize a frontend URL for comparison.
+
+    Strips protocol, host, base path variables, and normalizes parameters.
+    """
+    # Remove protocol + host
+    url = re.sub(r"^https?://[^/]+", "", url)
+    # Remove template literal expressions like ${API_URL}
+    url = re.sub(r"\$\{[^}]+\}", "", url)
+    # Remove env variable references
+    url = re.sub(r"process\.env\.\w+", "", url)
+    # Strip leading/trailing slashes and normalize
+    url = url.strip("/")
+    # Replace template literal params ${id} with :param placeholder
+    url = re.sub(r"\$\{(\w+)\}", r":\1", url)
+    return url
+
+
+def _normalize_route(route: str) -> str:
+    """Normalize a backend route pattern for comparison.
+
+    Converts all param syntaxes to a common format.
+    """
+    route = route.strip("/")
+    # Flask: <param> or <int:param>
+    route = re.sub(r"<(?:\w+:)?(\w+)>", r":\1", route)
+    # FastAPI: {param}
+    route = re.sub(r"\{(\w+)\}", r":\1", route)
+    # Next.js: [param]
+    route = re.sub(r"\[(\w+)\]", r":\1", route)
+    return route
+
+
+def _path_similarity(call_path: str, endpoint_path: str) -> float:
+    """Calculate similarity between a frontend call URL and backend route.
+
+    Returns 0.0 (no match) to 1.0 (exact match).
+    """
+    call_parts = [p for p in call_path.split("/") if p]
+    ep_parts = [p for p in endpoint_path.split("/") if p]
+
+    if not call_parts or not ep_parts:
+        return 0.0
+
+    # Different lengths — check if one is a suffix of the other
+    if len(call_parts) != len(ep_parts):
+        # Try suffix matching (frontend might include /api prefix that backend doesn't)
+        shorter, longer = (call_parts, ep_parts) if len(call_parts) <= len(ep_parts) else (ep_parts, call_parts)
+        for offset in range(len(longer) - len(shorter) + 1):
+            if _segments_match(shorter, longer[offset:offset + len(shorter)]):
+                return 0.7  # partial match
+        return 0.0
+
+    if _segments_match(call_parts, ep_parts):
+        return 1.0
+
+    return 0.0
+
+
+def _segments_match(a: list[str], b: list[str]) -> bool:
+    """Check if two path segment lists match, treating :params as wildcards."""
+    if len(a) != len(b):
+        return False
+    for sa, sb in zip(a, b):
+        if sa.startswith(":") or sb.startswith(":"):
+            continue  # parameter — always matches
+        if sa != sb:
+            return False
+    return True
+
+
+def _find_shared_dependencies(docs: list[RepoDocumentation]) -> list[ServiceRelationship]:
+    """Find packages used by multiple repos."""
+    relationships = []
+    if len(docs) < 2:
+        return relationships
+
+    # Build dep name -> list of repos
+    dep_repos: dict[str, list[str]] = {}
+    for doc in docs:
+        for dep in doc.dependencies:
+            if dep.dev_only:
+                continue
+            dep_repos.setdefault(dep.name, []).append(doc.repo_name)
+
+    for dep_name, repos in dep_repos.items():
+        if len(repos) > 1:
+            for i in range(len(repos)):
+                for j in range(i + 1, len(repos)):
+                    relationships.append(ServiceRelationship(
+                        source_repo=repos[i],
+                        target_repo=repos[j],
+                        connection_type="shared_package",
+                        source_file=dep_name,
+                        target_endpoint=dep_name,
+                        confidence=0.5,
+                    ))
+
+    return relationships
+
+
+def _find_shared_db_tables(docs: list[RepoDocumentation]) -> list[ServiceRelationship]:
+    """Find database tables accessed by multiple repos (shared data layer)."""
+    relationships = []
+    if len(docs) < 2:
+        return relationships
+
+    # Build table_name -> list of (repo_name, first_file_reference)
+    table_repos: dict[str, list[tuple[str, str]]] = {}
+    for doc in docs:
+        # Collect unique tables per repo (normalized to lowercase)
+        repo_tables: dict[str, str] = {}
+        for op in doc.db_operations:
+            table_lower = op.table_name.lower()
+            if table_lower not in repo_tables:
+                repo_tables[table_lower] = f"{op.file_path}:{op.line}"
+
+        for table, first_ref in repo_tables.items():
+            table_repos.setdefault(table, []).append((doc.repo_name, first_ref))
+
+    for table_name, repos in table_repos.items():
+        if len(repos) > 1:
+            for i in range(len(repos)):
+                for j in range(i + 1, len(repos)):
+                    relationships.append(ServiceRelationship(
+                        source_repo=repos[i][0],
+                        target_repo=repos[j][0],
+                        connection_type="shared_db",
+                        source_file=repos[i][1],
+                        target_endpoint=f"table:{table_name}",
+                        confidence=0.8,
+                    ))
+
+    return relationships

commiter/extractors/api_calls.py

@@ -0,0 +1,91 @@
+"""Extract frontend API calls (fetch, axios, etc.) from component files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from commiter.adapters.react import ReactAdapter
+from commiter.extractors.base import BaseExtractor
+from commiter.models import APICall
+from commiter.parser import get_source, build_constants_from_file
+
+if TYPE_CHECKING:
+    from tree_sitter import Tree
+
+_FRONTEND_ADAPTER = ReactAdapter()
+
+
+class APICallExtractor(BaseExtractor):
+    name = "api_calls"
+    _env_vars: dict[str, str] = {}
+
+    def set_env_vars(self, env_vars: dict[str, str]) -> None:
+        """Set environment variables from .env files (called by scanner)."""
+        self._env_vars = env_vars
+
+    def can_handle(self, file_path: str, language: str | None) -> bool:
+        if language not in ("javascript", "typescript", "tsx"):
+            return False
+        # Focus on frontend-like files (components, pages, hooks, lib)
+        lower = file_path.lower()
+        frontend_hints = (
+            "components/", "pages/", "src/", "app/", "hooks/",
+            "lib/", "utils/", "services/", "features/",
+        )
+        # Exclude API route files (backend)
+        if "/pages/api/" in lower or "/route." in Path(file_path).name.lower():
+            return False
+        return any(hint in lower for hint in frontend_hints) or language in ("tsx",)
+
+    def extract(self, file_path: str, repo_name: str, tree: Tree | None = None, language: str | None = None) -> list[APICall]:
+        if tree is None:
+            return []
+
+        source = get_source(file_path)
+
+        # Build per-file constants for URL resolution
+        constants = build_constants_from_file(tree.root_node, source, self._env_vars)
+        matches = _FRONTEND_ADAPTER.find_api_calls(tree, source, file_path=file_path, constants=constants)
+
+        # Derive component/page name from file path
+        component_name = self._infer_component_name(file_path)
+
+        calls = []
+        for match in matches:
+            calls.append(APICall(
+                repo=repo_name,
+                file_path=file_path,
+                line=match.line,
+                component_or_page=component_name,
+                http_method=match.http_method,
+                url_pattern=match.url_pattern,
+                client_library=self._detect_library(match),
+                response_type=match.response_type,
+                body_type=match.body_type,
+            ))
+
+        return calls
+
+    def _infer_component_name(self, file_path: str) -> str:
+        """Derive a component/page name from the file path."""
+        path = Path(file_path)
+        # Use parent dir + stem for index files
+        if path.stem in ("index", "page"):
+            return path.parent.name
+        return path.stem
+
+    def _detect_library(self, match) -> str:
+        """Detect which HTTP library was used."""
+        # Based on the adapter that found it — for now just check the call text
+        # ReactAdapter already distinguishes fetch vs axios
+        if hasattr(match, "call_node"):
+            from commiter.parser import node_text
+            # Simple heuristic based on node content
+            try:
+                source = match.call_node.text
+                if source and b"axios" in source:
+                    return "axios"
+            except (AttributeError, TypeError):
+                pass
+        return "fetch"

commiter/extractors/api_endpoints.py

@@ -0,0 +1,354 @@
+"""Extract API endpoint definitions from backend files."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from commiter.adapters.base import BaseAdapter
+from commiter.adapters.flask import FlaskAdapter
+from commiter.adapters.fastapi import FastAPIAdapter
+from commiter.adapters.express import ExpressAdapter
+from commiter.adapters.nextjs import NextJSAdapter
+from commiter.adapters.django_rest import DjangoRESTAdapter
+from commiter.extractors.base import BaseExtractor
+from commiter.models import APIEndpoint, Param, ParamSource
+from commiter.parser import (
+    get_source, find_ts_interface_fields, find_ts_enum_declarations,
+    find_ts_type_aliases, find_ts_const_objects, find_py_class_fields,
+    parse_generic_type, TypeField,
+)
+from commiter.generic_resolver import resolve_generic_type
+
+if TYPE_CHECKING:
+    from tree_sitter import Tree
+
+# Map framework names to adapter instances
+_ADAPTERS: dict[str, BaseAdapter] = {
+    "flask": FlaskAdapter(),
+    "fastapi": FastAPIAdapter(),
+    "express": ExpressAdapter(),
+    "nextjs": NextJSAdapter(),
+    "django-rest": DjangoRESTAdapter(),
+}
+
+
+def _register_adapter(name: str, adapter: BaseAdapter) -> None:
+    """Register an adapter (used by later phases to add Express, Next.js, etc.)."""
+    _ADAPTERS[name] = adapter
+
+
+def get_adapters() -> dict[str, BaseAdapter]:
+    """Return all registered adapters."""
+    return dict(_ADAPTERS)
+
+
+class APIEndpointExtractor(BaseExtractor):
+    name = "api_endpoints"
+
+    def __init__(self) -> None:
+        self._type_index = None
+        self._prefix_index = None
+        self._middleware_index = None
+        self._handler_index = None
+
+    def set_type_index(self, type_index) -> None:
+        """Set the cross-file type index (called by scanner after Pass 1)."""
+        self._type_index = type_index
+
+    def set_prefix_index(self, prefix_index) -> None:
+        """Set the cross-file prefix index (called by scanner after Pass 1)."""
+        self._prefix_index = prefix_index
+
+    def set_middleware_index(self, middleware_index) -> None:
+        """Set the cross-file middleware index (called by scanner after Pass 1)."""
+        self._middleware_index = middleware_index
+
+    def set_handler_index(self, handler_index) -> None:
+        """Set the cross-file handler type index (called by scanner after Pass 1)."""
+        self._handler_index = handler_index
+
+    def can_handle(self, file_path: str, language: str | None) -> bool:
+        if language is None:
+            return False
+        # Only handle languages we have adapters for
+        return language in ("python", "javascript", "typescript", "tsx")
+
+    def extract(self, file_path: str, repo_name: str, tree: Tree | None = None, language: str | None = None) -> list[APIEndpoint]:
+        if tree is None or language is None:
+            return []
+
+        source = get_source(file_path)
+        endpoints = []
+
+        # Build type definitions for resolution (same-file + cross-file fallback)
+        type_defs = self._build_type_definitions(tree, source, language, file_path)
+
+        # Detect which framework the file actually uses (from imports/requires)
+        file_frameworks = self._detect_file_framework(source, language)
+
+        # Languages that JS adapters should also handle
+        js_family = {"javascript", "typescript", "tsx"}
+
+        # Try each adapter that matches the language
+        for adapter in _ADAPTERS.values():
+            # Match adapter language to file language
+            adapter_matches = (
+                adapter.language == language
+                or (adapter.language in js_family and language in js_family)
+            )
+            if not adapter_matches:
+                continue
+
+            # Skip adapters that don't match the file's actual framework
+            if file_frameworks is not None and adapter.framework_name not in file_frameworks:
+                continue
+
+            # NextJS adapter needs file_path for convention-based routing
+            if isinstance(adapter, NextJSAdapter):
+                routes = adapter.find_route_definitions(tree, source, file_path=file_path)
+            else:
+                routes = adapter.find_route_definitions(tree, source)
+            for route in routes:
+                # Resolve prefix from blueprint/router mounting
+                if self._prefix_index and route.router_var:
+                    prefix = self._prefix_index.get_prefix_for_file(file_path, route.router_var)
+                    if prefix:
+                        route.route_pattern = _join_prefix(prefix, route.route_pattern)
+
+                # Extract details, passing type definitions for resolution
+                path_param_names = adapter.extract_path_params(route.route_pattern)
+                auth = adapter.extract_auth_info(route, source)
+
+                # Pass type_defs to adapters that support it
+                if hasattr(adapter.extract_request_body_fields, '__code__') and \
+                   'type_definitions' in adapter.extract_request_body_fields.__code__.co_varnames:
+                    body_fields = adapter.extract_request_body_fields(route, source, type_definitions=type_defs)
+                else:
+                    body_fields = adapter.extract_request_body_fields(route, source)
+
+                response_fields = adapter.extract_response_fields(route, source)
+
+                params = [
+                    Param(name=p, source=ParamSource.PATH, type_hint=route.param_types.get(p))
+                    for p in path_param_names
+                ]
+
+                # Pick up type names from route.param_types (set by adapters)
+                request_body_type = route.param_types.get("_request_body_type")
+                response_type = route.param_types.get("_response_type")
+
+                # For class-based handlers: look up body type from handler index
+                if not request_body_type and self._handler_index and "." in route.handler_name:
+                    request_body_type = self._handler_index.get_body_type(route.handler_name)
+                    # If we found a body type, also resolve its fields
+                    if request_body_type and request_body_type in type_defs:
+                        body_fields = []
+                        for tf in type_defs[request_body_type]:
+                            opt = "?" if tf.optional else ""
+                            body_fields.append(f"{tf.name}: {tf.type_str}{opt}")
+
+                # Query middleware index for middleware covering this route
+                mw = []
+                if self._middleware_index:
+                    mw = self._middleware_index.get_middleware_for_route(
+                        file_path, route.route_pattern, route.router_var, route.line,
+                    )
+
+                endpoints.append(APIEndpoint(
+                    repo=repo_name,
+                    file_path=file_path,
+                    line=route.line,
+                    http_method=route.http_method,
+                    route_pattern=route.route_pattern,
+                    handler_name=route.handler_name,
+                    framework=adapter.framework_name,
+                    parameters=params,
+                    request_body_fields=body_fields,
+                    response_fields=response_fields,
+                    auth_decorators=auth,
+                    request_body_type=request_body_type,
+                    response_type=response_type,
+                    middleware=mw,
+                ))
+
+        return endpoints
+
+    @staticmethod
+    def _detect_file_framework(source: bytes, language: str) -> set[str] | None:
+        """Detect which framework a file uses based on its imports.
+
+        Returns a set of framework names, or None if we can't determine (allow all).
+        """
+        text = source.decode("utf-8", errors="replace")
+
+        if language == "python":
+            frameworks = set()
+            if "from flask" in text or "import flask" in text:
+                frameworks.add("flask")
+            if "from fastapi" in text or "import fastapi" in text:
+                frameworks.add("fastapi")
+            if "from django" in text or "from rest_framework" in text:
+                frameworks.add("django-rest")
+            return frameworks if frameworks else None
+
+        if language in ("javascript", "typescript", "tsx"):
+            frameworks = set()
+            if "express" in text and ("require(" in text or "from " in text):
+                frameworks.add("express")
+            if "next" in text or "/pages/api/" in text or "/app/" in text:
+                frameworks.add("nextjs")
+            # If we found any backend indicators, return them; if none found,
+            # return empty set to prevent backend adapters from matching frontend files
+            if frameworks:
+                return frameworks
+            # Check if this looks like a backend file at all (has route-like patterns)
+            has_route_patterns = ("router." in text or "app." in text) and ("require" in text or "express" in text)
+            if not has_route_patterns:
+                return set()  # empty = no backend adapters should run
+            return None
+
+        return None
+
+    def _build_type_definitions(self, tree, source: bytes, language: str, file_path: str) -> dict[str, list[TypeField]]:
+        """Build a dict of type definitions for resolution.
+
+        Combines same-file definitions with cross-file lookups from the TypeIndex.
+        """
+        type_defs: dict[str, list[TypeField]] = {}
+
+        if language in ("typescript", "tsx"):
+            type_defs = find_ts_interface_fields(tree.root_node, source)
+            type_defs.update(find_ts_enum_declarations(tree.root_node, source))
+            type_defs.update(find_ts_type_aliases(tree.root_node, source))
+            type_defs.update(find_ts_const_objects(tree.root_node, source))
+        elif language == "python":
+            import re
+            text = source.decode("utf-8", errors="replace")
+            for match in re.finditer(r'class\s+(\w+)\s*\([^)]*(?:BaseModel|Schema|TypedDict)', text):
+                class_name = match.group(1)
+                fields = find_py_class_fields(tree.root_node, source, class_name)
+                if fields:
+                    type_defs[class_name] = fields
+
+        # Wrap with cross-file fallback if TypeIndex is available
+        if self._type_index is not None:
+            return _TypeDefsWithFallback(type_defs, self._type_index, file_path)
+
+        return type_defs
+
+
+class _TypeDefsWithFallback(dict):
+    """Dict-like that falls back to TypeIndex for missing keys."""
+
+    def __init__(self, local_defs: dict, type_index, file_path: str):
+        super().__init__(local_defs)
+        self._type_index = type_index
+        self._file_path = file_path
+
+    def __bool__(self):
+        # Always truthy so `if type_definitions and ...` doesn't short-circuit
+        return True
+
+    def __contains__(self, key):
+        if super().__contains__(key):
+            return True
+        # Check cross-file index
+        typedef = self._type_index.resolve(key, None, self._file_path)
+        if typedef is not None:
+            return True
+        # Check if it's a generic type we can resolve
+        if "<" in key or key.endswith("[]"):
+            resolved = self._try_generic_resolve(key)
+            if resolved is not None:
+                return True
+        return False
+
+    def __getitem__(self, key):
+        try:
+            fields = super().__getitem__(key)
+        except KeyError:
+            typedef = self._type_index.resolve(key, None, self._file_path)
+            if typedef is not None:
+                fields = typedef.fields
+            else:
+                # Try generic resolution: Pick<User, "id">, Partial<User>, etc.
+                if "<" in key or key.endswith("[]"):
+                    resolved = self._try_generic_resolve(key)
+                    if resolved is not None:
+                        return self._resolve_type_refs(resolved)
+                raise
+        # Resolve any __type_ref__ markers (from intersections/unions)
+        return self._resolve_type_refs(fields)
+
+    def _try_generic_resolve(self, type_str: str) -> list | None:
+        """Try to resolve a generic type like Pick<User, "id"> to fields."""
+        def type_lookup(name):
+            try:
+                return list(self[name])  # recursive — uses our own __getitem__
+            except KeyError:
+                return None
+
+        def generic_def_lookup(name):
+            """Look up a generic type definition and return (fields, generic_params)."""
+            # Try local
+            try:
+                fields = list(super(_TypeDefsWithFallback, self).__getitem__(name))
+                # Check type_index for generic_params
+                typedef = self._type_index.resolve(name, None, self._file_path)
+                params = typedef.generic_params if typedef else []
+                return (fields, params)
+            except KeyError:
+                pass
+            # Try cross-file
+            typedef = self._type_index.resolve(name, None, self._file_path)
+            if typedef:
+                return (list(typedef.fields), typedef.generic_params)
+            return None
+
+        return resolve_generic_type(type_str, type_lookup, generic_def_lookup)
+
+    def _resolve_type_refs(self, fields: list, depth: int = 0) -> list:
+        """Recursively resolve __type_ref__ and __generic__ markers to actual fields."""
+        if depth > 5:
+            return fields  # guard against circular refs
+
+        resolved = []
+        for field in fields:
+            if field.type_str == "__type_ref__":
+                ref_fields = self._lookup_type(field.name)
+                if ref_fields:
+                    resolved.extend(self._resolve_type_refs(ref_fields, depth + 1))
+                else:
+                    resolved.append(field)
+            elif field.type_str == "__generic__" and field.value:
+                # Generic type alias: Pick<User, "id">, Partial<User>, ApiResponse<User>
+                generic_fields = self._try_generic_resolve(field.value)
+                if generic_fields:
+                    resolved.extend(self._resolve_type_refs(generic_fields, depth + 1))
+                else:
+                    resolved.append(field)
+            else:
+                resolved.append(field)
+        return resolved
+
+    def _lookup_type(self, type_name: str) -> list | None:
+        """Look up a type by name in local defs or cross-file index."""
+        # Try local first
+        try:
+            return list(super().__getitem__(type_name))
+        except KeyError:
+            pass
+        # Try cross-file
+        typedef = self._type_index.resolve(type_name, None, self._file_path)
+        if typedef is not None:
+            return list(typedef.fields)
+        return None
+
+
+def _join_prefix(prefix: str, route: str) -> str:
+    """Join a URL prefix with a route pattern, avoiding double slashes."""
+    prefix = prefix.rstrip("/")
+    if not route.startswith("/"):
+        route = "/" + route
+    return prefix + route

commiter/extractors/backend_files.py

@@ -0,0 +1,33 @@
+"""Classify files as backend, frontend, config, test, etc."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from commiter.extractors.base import BaseExtractor
+from commiter.models import FileClassification
+from commiter.utils.file_classifier import classify_file
+
+if TYPE_CHECKING:
+    from tree_sitter import Tree
+
+
+class BackendFileExtractor(BaseExtractor):
+    """Classifies files by role. Returns FileClassification objects.
+
+    Note: The scanner already classifies files, so this extractor is mainly
+    useful for enriching classifications with AST-based hints (e.g., detecting
+    that a .py file imports Flask and is therefore a backend file).
+    """
+
+    name = "backend_files"
+
+    def can_handle(self, file_path: str, language: str | None) -> bool:
+        # We handle all files — classification is universal
+        return True
+
+    def extract(self, file_path: str, repo_name: str, tree: Tree | None = None, language: str | None = None) -> list[FileClassification]:
+        # The scanner already does basic classification. This extractor could
+        # enrich it with AST-based detection in the future (e.g., checking imports).
+        # For now, return empty to avoid duplicates — the scanner handles it.
+        return []