compos-cli 0.0.0__py3-none-any.whl

compos/cli/analyzers/python.py

@@ -0,0 +1,604 @@
+"""Python-specific detection."""
+
+from __future__ import annotations
+
+import ast
+import logging
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from pathlib import Path
+
+from compos.cli.analyzers.base import AnalysisResult
+from compos.schema.models import (
+    Component,
+    ComponentType,
+    ObjectStatus,
+    Provenance,
+    ProvenanceSource,
+    Relationship,
+    RelationshipPattern,
+    RelationshipType,
+)
+
+logger = logging.getLogger(__name__)
+
+_SKIP_DIRS = frozenset(
+    {
+        ".venv",
+        "venv",
+        "node_modules",
+        "__pycache__",
+        ".git",
+        ".compos",
+        "site-packages",
+        ".tox",
+        ".mypy_cache",
+        ".worktrees",
+        "worktrees",
+        "tests",
+        "test",
+    }
+)
+
+_FRAMEWORK_CALLS: dict[str, str] = {
+    "FastAPI": "FastAPI application",
+    "Flask": "Flask application",
+    "Django": "Django application",
+}
+
+# call-name → responsibility label
+_DB_CALLS: dict[str, str] = {
+    "create_engine": "SQLAlchemy database engine",
+    "create_async_engine": "SQLAlchemy async database engine",
+    "declarative_base": "SQLAlchemy declarative base",
+    "MongoClient": "MongoDB client",
+    "AsyncIOMotorClient": "MongoDB async client (Motor)",
+    "Redis": "Redis client",
+    "StrictRedis": "Redis client",
+}
+
+# Module-gated DB calls: only match when imported from specific modules
+_DB_MODULE_CALLS: dict[str, set[str]] = {
+    "connect": {"psycopg2", "asyncpg"},
+}
+
+# call-name → (ComponentType, responsibility)
+_QUEUE_CALLS: dict[str, tuple[ComponentType, str]] = {
+    "Celery": (ComponentType.WORKER, "Celery task worker"),
+    "KafkaProducer": (ComponentType.QUEUE, "Kafka producer"),
+    "KafkaConsumer": (ComponentType.QUEUE, "Kafka consumer"),
+    "BlockingConnection": (ComponentType.QUEUE, "RabbitMQ connection (pika)"),
+    "SelectConnection": (ComponentType.QUEUE, "RabbitMQ connection (pika)"),
+}
+
+
+_IGNORE_PACKAGE_DIRS = _SKIP_DIRS | frozenset(
+    {"tests", "test", "docs", "scripts", "bin", "migrations", "alembic"}
+)
+
+
+def _is_package(path: Path) -> bool:
+    """Return True if *path* is a directory containing ``__init__.py``."""
+    return path.is_dir() and (path / "__init__.py").is_file()
+
+
+def _find_source_root(project_root: Path) -> Path | None:
+    """Find the main Python package root under *project_root*.
+
+    For src-layout projects (``src/<pkg>/``), returns the package dir under
+    ``src/``.  For flat-layout (``<pkg>/``), returns the first package dir at
+    root.  Returns ``None`` if no packages are found.
+    """
+    # src-layout: look inside src/ first
+    src_dir = project_root / "src"
+    if src_dir.is_dir():
+        for child in sorted(src_dir.iterdir()):
+            if child.name.startswith("."):
+                continue
+            if child.name in _IGNORE_PACKAGE_DIRS:
+                continue
+            if _is_package(child):
+                return child
+
+    # flat-layout: first qualifying package at project root
+    for child in sorted(project_root.iterdir()):
+        if child.name.startswith("."):
+            continue
+        if child.name in _IGNORE_PACKAGE_DIRS:
+            continue
+        if _is_package(child):
+            return child
+
+    return None
+
+
+def _extract_init_docstring(package_dir: Path) -> str | None:
+    """Extract the first-line docstring from ``__init__.py``, or None."""
+    init_file = package_dir / "__init__.py"
+    if not init_file.is_file():
+        return None
+    try:
+        tree = ast.parse(init_file.read_text())
+    except SyntaxError:
+        return None
+    docstring = ast.get_docstring(tree)
+    if docstring:
+        return docstring.split("\n")[0]
+    return None
+
+
+def _detect_packages(source_root: Path, project_root: Path) -> list[Component]:
+    """Detect direct child packages of *source_root* as LIBRARY components."""
+    parent_name = source_root.name
+    components: list[Component] = []
+    for child in sorted(source_root.iterdir()):
+        if child.name.startswith("."):
+            continue
+        if child.name in _IGNORE_PACKAGE_DIRS:
+            continue
+        if not _is_package(child):
+            continue
+        child_name = child.name
+        docstring = _extract_init_docstring(child)
+        responsibility = docstring if docstring else "Python package"
+        components.append(
+            Component(
+                id=f"pypkg-{parent_name}.{child_name}",
+                name=child_name,
+                responsibility=responsibility,
+                type=ComponentType.LIBRARY,
+                detection_confidence=0.70,
+                status=ObjectStatus.CANDIDATE,
+                provenance=_make_provenance(),
+                paths=(str(child.relative_to(project_root)) + "/",),
+            )
+        )
+    return components
+
+
+def _make_provenance() -> Provenance:
+    return Provenance(
+        source=ProvenanceSource.STATIC_ANALYSIS,
+        tool="cli-analyze",
+        timestamp=datetime.now(UTC),
+    )
+
+
+def _file_to_module_id(file: Path, root: Path, prefix: str) -> str:
+    rel = file.relative_to(root).with_suffix("")
+    module = str(rel).replace("/", ".").replace("\\", ".")
+    return f"{prefix}-{module}"
+
+
+def _display_name(file: Path, root: Path) -> str:
+    """Human-readable name: parent/stem for disambiguation, stem if unique."""
+    rel = file.relative_to(root).with_suffix("")
+    parts = rel.parts
+    if len(parts) >= 2:
+        return f"{parts[-2]}/{parts[-1]}"
+    return parts[-1] if parts else file.stem
+
+
+def _iter_python_files(root: Path) -> Iterator[Path]:
+    for p in sorted(root.rglob("*.py")):
+        if any(part in _SKIP_DIRS for part in p.parts):
+            continue
+        yield p
+
+
+def _get_call_name(node: ast.Call) -> str | None:
+    """Extract the function/method name from a Call node."""
+    if isinstance(node.func, ast.Name):
+        return node.func.id
+    if isinstance(node.func, ast.Attribute):
+        return node.func.attr
+    return None
+
+
+def _collect_imports(tree: ast.Module) -> dict[str, str]:
+    """Map imported names to their source modules.
+
+    e.g. ``from psycopg2 import connect`` → {"connect": "psycopg2"}
+    """
+    mapping: dict[str, str] = {}
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ImportFrom) and node.module:
+            top_module = node.module.split(".")[0]
+            for alias in node.names:
+                mapping[alias.asname or alias.name] = top_module
+    return mapping
+
+
+def _detect_framework_apps(
+    file: Path,
+    root: Path,
+    tree: ast.Module,
+) -> list[Component]:
+    components: list[Component] = []
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        func_name = _get_call_name(node)
+        if func_name and func_name in _FRAMEWORK_CALLS:
+            components.append(
+                Component(
+                    id=_file_to_module_id(file, root, "pyapp"),
+                    name=_display_name(file, root),
+                    responsibility=_FRAMEWORK_CALLS[func_name],
+                    type=ComponentType.SERVICE,
+                    detection_confidence=0.85,
+                    status=ObjectStatus.CANDIDATE,
+                    provenance=_make_provenance(),
+                    paths=(str(file.relative_to(root)),),
+                )
+            )
+            break  # one component per file
+    return components
+
+
+def _detect_database_clients(
+    file: Path,
+    root: Path,
+    tree: ast.Module,
+) -> list[Component]:
+    """Detect database client instantiation — one component per file."""
+    imports = _collect_imports(tree)
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        func_name = _get_call_name(node)
+        if func_name is None:
+            continue
+        # Direct match (e.g. create_engine, MongoClient, Redis)
+        if func_name in _DB_CALLS:
+            return [
+                Component(
+                    id=_file_to_module_id(file, root, "pydb"),
+                    name=_display_name(file, root),
+                    responsibility=_DB_CALLS[func_name],
+                    type=ComponentType.DATABASE,
+                    detection_confidence=0.80,
+                    status=ObjectStatus.CANDIDATE,
+                    provenance=_make_provenance(),
+                    paths=(str(file.relative_to(root)),),
+                )
+            ]
+        # Module-gated match (e.g. psycopg2.connect)
+        if func_name in _DB_MODULE_CALLS:
+            source_module = imports.get(func_name)
+            if source_module and source_module in _DB_MODULE_CALLS[func_name]:
+                return [
+                    Component(
+                        id=_file_to_module_id(file, root, "pydb"),
+                        name=_display_name(file, root),
+                        responsibility=f"{source_module} database connection",
+                        type=ComponentType.DATABASE,
+                        detection_confidence=0.80,
+                        status=ObjectStatus.CANDIDATE,
+                        provenance=_make_provenance(),
+                        paths=(str(file.relative_to(root)),),
+                    )
+                ]
+    return []
+
+
+def _detect_queue_clients(
+    file: Path,
+    root: Path,
+    tree: ast.Module,
+) -> list[Component]:
+    """Detect queue/worker client instantiation — one component per file."""
+    for node in ast.walk(tree):
+        if not isinstance(node, ast.Call):
+            continue
+        func_name = _get_call_name(node)
+        if func_name and func_name in _QUEUE_CALLS:
+            comp_type, responsibility = _QUEUE_CALLS[func_name]
+            return [
+                Component(
+                    id=_file_to_module_id(file, root, "pyq"),
+                    name=_display_name(file, root),
+                    responsibility=responsibility,
+                    type=comp_type,
+                    detection_confidence=0.75,
+                    status=ObjectStatus.CANDIDATE,
+                    provenance=_make_provenance(),
+                    paths=(str(file.relative_to(root)),),
+                )
+            ]
+    return []
+
+
+def _build_module_index(components: list[Component]) -> dict[str, str]:
+    """Map module dotted paths to component IDs.
+
+    Registers both the filesystem-relative path and the importable path.
+    For src-layout projects (path starts with ``src.``), the importable
+    path strips the ``src.`` prefix so ``from compos.db.session import …``
+    matches ``src/compos/db/session.py``.
+
+    Only indexes non-package (file-level) components.
+    """
+    index: dict[str, str] = {}
+    for comp in components:
+        if comp.id.startswith("pypkg-"):
+            continue  # package components use _build_package_module_index
+        parts = comp.id.split("-", 1)
+        if len(parts) == 2:
+            module_path = parts[1]
+            index[module_path] = comp.id
+            # src-layout: also register the importable path
+            if module_path.startswith("src."):
+                importable = module_path[4:]  # strip "src."
+                index[importable] = comp.id
+    return index
+
+
+def _build_package_module_index(
+    package_components: list[Component],
+    source_root: Path,
+    project_root: Path,
+) -> dict[str, str]:
+    """Map dotted import paths to package-level component IDs.
+
+    For each package component ``pypkg-{parent}.{child}``, registers:
+    - The filesystem-relative path: ``src.{parent}.{child}`` (if src-layout)
+    - The importable path: ``{parent}.{child}``
+
+    These entries match both exact imports and sub-module imports via
+    prefix matching in ``_check_import``.
+    """
+    index: dict[str, str] = {}
+    parent_name = source_root.name
+    # Determine if this is src-layout
+    is_src_layout = source_root.parent.name == "src"
+
+    for comp in package_components:
+        # id format: pypkg-{parent}.{child}
+        child_name = comp.id.split(".")[-1]
+        # Register importable path: {parent}.{child}
+        importable = f"{parent_name}.{child_name}"
+        index[importable] = comp.id
+        # Register filesystem-relative path (if src-layout)
+        if is_src_layout:
+            fs_path = f"src.{parent_name}.{child_name}"
+            index[fs_path] = comp.id
+    return index
+
+
+def _check_import(
+    module_name: str,
+    source_component_id: str,
+    module_index: dict[str, str],
+    seen_targets: set[str],
+    rels: list[Relationship],
+    root: Path,
+    file: Path,
+) -> None:
+    """Check if an import resolves to a known component and emit a relationship."""
+    for mod_path, target_id in module_index.items():
+        if target_id == source_component_id:
+            continue  # skip self-references
+        if module_name == mod_path or module_name.startswith(mod_path + "."):
+            if target_id not in seen_targets:
+                seen_targets.add(target_id)
+                rels.append(
+                    Relationship(
+                        id=f"rel-{source_component_id}-{target_id}",
+                        source=source_component_id,
+                        target=target_id,
+                        type=RelationshipType.DEPENDENCY,
+                        pattern=RelationshipPattern.SYNCHRONOUS,
+                        description="Python import dependency",
+                        detection_confidence=0.70,
+                        status=ObjectStatus.CANDIDATE,
+                        provenance=_make_provenance(),
+                    )
+                )
+                break  # found a match, no need to check others
+
+
+def _detect_import_relationships(
+    file: Path,
+    root: Path,
+    tree: ast.Module,
+    source_component_id: str,
+    module_index: dict[str, str],
+    seen_targets: set[str] | None = None,
+) -> list[Relationship]:
+    """Walk AST for imports and emit DEPENDENCY relationships to known components."""
+    rels: list[Relationship] = []
+    if seen_targets is None:
+        seen_targets = set()
+    for node in ast.walk(tree):
+        module_name: str | None = None
+        if isinstance(node, ast.ImportFrom) and node.module:
+            module_name = node.module
+        elif isinstance(node, ast.Import):
+            for alias in node.names:
+                if alias.name:
+                    _check_import(
+                        alias.name,
+                        source_component_id,
+                        module_index,
+                        seen_targets,
+                        rels,
+                        root,
+                        file,
+                    )
+            continue
+        if module_name is not None:
+            _check_import(
+                module_name,
+                source_component_id,
+                module_index,
+                seen_targets,
+                rels,
+                root,
+                file,
+            )
+    return rels
+
+
+def _is_subpath(path: Path, directory: Path) -> bool:
+    """Check if *path* is inside *directory* (non-strict)."""
+    try:
+        path.relative_to(directory)
+        return True
+    except ValueError:
+        return False
+
+
+class PythonAnalyzer:
+    name = "python"
+
+    def can_analyze(self, project_root: Path) -> bool:
+        return any(True for _ in _iter_python_files(project_root))
+
+    def analyze(self, project_root: Path) -> AnalysisResult:
+        components: list[Component] = []
+        warnings: list[str] = []
+        # Track which file produced which component(s) and its parsed AST
+        file_components: dict[Path, list[Component]] = {}
+        parsed_trees: dict[Path, ast.Module] = {}
+
+        # Pass 0: detect package-level components
+        source_root = _find_source_root(project_root)
+        # Map package directory -> package component for merge lookups
+        pkg_dir_to_comp: dict[Path, Component] = {}
+        if source_root is not None:
+            pkg_components = _detect_packages(source_root, project_root)
+            for pkg_comp in pkg_components:
+                # Derive the package directory from the component id
+                # id format: pypkg-{parent}.{child} -> source_root / child
+                child_name = pkg_comp.id.split(".")[-1]
+                pkg_dir = source_root / child_name
+                pkg_dir_to_comp[pkg_dir] = pkg_comp
+            components.extend(pkg_components)
+
+        # Pass 1: detect file-level components, merging into packages
+        for py_file in _iter_python_files(project_root):
+            try:
+                tree = ast.parse(py_file.read_text())
+            except SyntaxError:
+                warnings.append(f"Skipped unparseable file: {py_file}")
+                continue
+            parsed_trees[py_file] = tree
+            file_comps: list[Component] = []
+            file_comps.extend(_detect_framework_apps(py_file, project_root, tree))
+            file_comps.extend(_detect_database_clients(py_file, project_root, tree))
+            file_comps.extend(_detect_queue_clients(py_file, project_root, tree))
+            if file_comps:
+                # Check if this file belongs to a detected package
+                owning_pkg_dir: Path | None = None
+                for pkg_dir in pkg_dir_to_comp:
+                    if _is_subpath(py_file, pkg_dir):
+                        owning_pkg_dir = pkg_dir
+                        break
+
+                if owning_pkg_dir is not None:
+                    # Merge: upgrade the package component instead of
+                    # creating standalone file-level components.
+                    # Pick the highest-confidence file detection to upgrade with.
+                    best = max(file_comps, key=lambda c: c.detection_confidence)
+                    existing_pkg = pkg_dir_to_comp[owning_pkg_dir]
+                    # Only upgrade if the file detection has higher confidence
+                    # or a more specific type than LIBRARY
+                    if (
+                        existing_pkg.type == ComponentType.LIBRARY
+                        or best.detection_confidence > existing_pkg.detection_confidence
+                    ):
+                        # Preserve the docstring-based responsibility if it
+                        # was extracted from __init__.py (not the generic
+                        # fallback).  Only use the detection label when the
+                        # package had no docstring.
+                        responsibility = (
+                            existing_pkg.responsibility
+                            if existing_pkg.responsibility != "Python package"
+                            else best.responsibility
+                        )
+                        upgraded = Component(
+                            id=existing_pkg.id,
+                            name=existing_pkg.name,
+                            responsibility=responsibility,
+                            type=best.type,
+                            detection_confidence=best.detection_confidence,
+                            status=existing_pkg.status,
+                            provenance=existing_pkg.provenance,
+                            paths=existing_pkg.paths,
+                        )
+                        # Replace in components list and tracking dict
+                        components = [
+                            upgraded if c.id == existing_pkg.id else c
+                            for c in components
+                        ]
+                        pkg_dir_to_comp[owning_pkg_dir] = upgraded
+                    # Don't add file-level components — they're merged
+                else:
+                    file_components[py_file] = file_comps
+                    components.extend(file_comps)
+
+        # Pass 2: detect import relationships between components.
+        #
+        # Two kinds of components need scanning:
+        # (a) Package components — scan ALL parsed files inside their directory
+        # (b) Standalone file components — scan their subtree (old logic)
+        #
+        # Build a combined module index from both package and file components.
+        pkg_components_list = [c for c in components if c.id.startswith("pypkg-")]
+        pkg_module_index: dict[str, str] = {}
+        if source_root is not None and pkg_components_list:
+            pkg_module_index = _build_package_module_index(
+                pkg_components_list, source_root, project_root
+            )
+        file_module_index = _build_module_index(components)
+        combined_index = {**file_module_index, **pkg_module_index}
+
+        relationships: list[Relationship] = []
+
+        # (a) Package components: scan all parsed files in their directory subtree
+        for pkg_dir, pkg_comp in pkg_dir_to_comp.items():
+            seen: set[str] = set()
+            for py_file, tree in parsed_trees.items():
+                if _is_subpath(py_file, pkg_dir):
+                    relationships.extend(
+                        _detect_import_relationships(
+                            py_file,
+                            project_root,
+                            tree,
+                            pkg_comp.id,
+                            combined_index,
+                            seen_targets=seen,
+                        )
+                    )
+
+        # (b) Standalone file components (outside packages): old subtree logic
+        all_component_files = set(file_components.keys())
+        for py_file, file_comps in file_components.items():
+            comp_dir = py_file.parent
+            other_component_files = all_component_files - {py_file}
+            package_files = [
+                f
+                for f in parsed_trees
+                if (f == py_file or _is_subpath(f, comp_dir))
+                and f not in other_component_files
+            ]
+            for comp in file_comps:
+                seen_standalone: set[str] = set()
+                for pkg_file in package_files:
+                    relationships.extend(
+                        _detect_import_relationships(
+                            pkg_file,
+                            project_root,
+                            parsed_trees[pkg_file],
+                            comp.id,
+                            combined_index,
+                            seen_targets=seen_standalone,
+                        )
+                    )
+
+        return AnalysisResult(
+            components=tuple(components),
+            relationships=tuple(relationships),
+            warnings=tuple(warnings),
+        )