codemble 0.3.0__py3-none-any.whl

codemble/__init__.py

@@ -0,0 +1,4 @@
+"""Codemble — a learning game that turns the code AI wrote for you into a galaxy
+you light up by understanding it. See CLAUDE.md for the agent brief and roadmap."""
+
+__version__ = "0.3.0"

codemble/adapters/__init__.py

@@ -0,0 +1,38 @@
+"""Language adapters turn parser evidence into Codemble's graph contract."""
+
+from codemble.adapters.base import (
+    AdapterParseError,
+    ConceptAnnotation,
+    Edge,
+    Graph,
+    LanguageAdapter,
+    Node,
+)
+from codemble.adapters.project import (
+    ProjectIntake,
+    ProjectParseError,
+    ProjectParser,
+    ProjectScaleError,
+)
+from codemble.adapters.python_ast import PythonAstAdapter, PythonParseError
+from codemble.adapters.typescript_tree_sitter import (
+    JavaScriptTypeScriptAdapter,
+    JavaScriptTypeScriptParseError,
+)
+
+__all__ = [
+    "AdapterParseError",
+    "ConceptAnnotation",
+    "Edge",
+    "Graph",
+    "LanguageAdapter",
+    "Node",
+    "JavaScriptTypeScriptAdapter",
+    "JavaScriptTypeScriptParseError",
+    "PythonAstAdapter",
+    "PythonParseError",
+    "ProjectIntake",
+    "ProjectParseError",
+    "ProjectParser",
+    "ProjectScaleError",
+]

codemble/adapters/base.py

@@ -0,0 +1,199 @@
+"""Language-neutral graph contracts for Codemble parsers.
+
+Adapters are the only layer allowed to derive structure from source text.  The
+rest of Codemble consumes this render-ready, deterministic representation.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Literal, Protocol
+
+NodeKind = Literal["module", "class", "function"]
+EdgeKind = Literal["import", "call"]
+
+
+class AdapterParseError(ValueError):
+    """A supported language could not be mapped without guessing."""
+
+
+@dataclass(frozen=True, slots=True)
+class Node:
+    """A parser-proven source structure.
+
+    ``partial`` is true only for module nodes whose source could not be fully
+    parsed.  Keeping that node makes parse failures visible without inventing
+    any structure for the unreadable file.
+    """
+
+    id: str
+    kind: NodeKind
+    name: str
+    language: str
+    file: str
+    lineno: int
+    end_lineno: int
+    loc: int
+    region: str
+    centrality: int = 0
+    entrypoint_rank: int | None = None
+    understood: bool = False
+    partial: bool = False
+    system_x: float = 0.0
+    system_y: float = 0.0
+    system_z: float = 0.0
+
+
+@dataclass(frozen=True, slots=True)
+class Edge:
+    """A parser-observed relationship between source structures.
+
+    A destination beginning with ``external:`` is intentionally not a graph
+    node.  ``external`` lets consumers render or filter those observations
+    without pretending third-party or dynamically-resolved structure exists.
+    """
+
+    src: str
+    dst: str
+    kind: EdgeKind
+    certain: bool
+    lineno: int
+    external: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class ConceptAnnotation:
+    """A language construct proven by an adapter at an exact source span."""
+
+    node_id: str
+    language: str
+    concept: str
+    lineno: int
+    end_lineno: int
+    snippet: str
+
+
+@dataclass(frozen=True, slots=True)
+class Region:
+    """A render-ready star system derived from parser-proven nodes."""
+
+    id: str
+    language: str
+    loc: int
+    centrality: int
+    node_count: int
+    understood: bool
+    home: bool
+    x: float
+    y: float
+    z: float
+
+
+@dataclass(frozen=True, slots=True)
+class RegionEdge:
+    """An aggregated import route between two project regions."""
+
+    src: str
+    dst: str
+    weight: int
+    certain: bool
+
+
+@dataclass(frozen=True, slots=True)
+class Graph:
+    """A deterministic, language-tagged project graph."""
+
+    nodes: tuple[Node, ...]
+    edges: tuple[Edge, ...]
+    entrypoint_candidates: tuple[str, ...]
+    project_root: str
+    file_hashes: dict[str, str]
+    selected_entrypoint: str | None = None
+    concept_annotations: tuple[ConceptAnnotation, ...] = ()
+    regions: tuple[Region, ...] = ()
+    region_edges: tuple[RegionEdge, ...] = ()
+    partial_files: tuple[str, ...] = ()
+    schema_version: int = field(default=4, init=False)
+
+    def to_dict(self) -> dict[str, object]:
+        """Return a JSON-ready representation in canonical collection order."""
+
+        nodes = sorted(self.nodes, key=lambda node: node.id)
+        edges = sorted(
+            self.edges,
+            key=lambda edge: (
+                edge.src,
+                edge.dst,
+                edge.kind,
+                edge.lineno,
+                edge.certain,
+                edge.external,
+            ),
+        )
+        return {
+            "schema_version": self.schema_version,
+            "nodes": [asdict(node) for node in nodes],
+            "edges": [asdict(edge) for edge in edges],
+            "entrypoint_candidates": list(self.entrypoint_candidates),
+            "selected_entrypoint": self.selected_entrypoint,
+            "project_root": self.project_root,
+            "file_hashes": dict(sorted(self.file_hashes.items())),
+            "concept_annotations": [
+                asdict(annotation)
+                for annotation in sorted(
+                    self.concept_annotations,
+                    key=lambda item: (
+                        item.language,
+                        item.node_id,
+                        item.lineno,
+                        item.concept,
+                        item.end_lineno,
+                    ),
+                )
+            ],
+            "regions": [asdict(region) for region in sorted(self.regions, key=lambda item: item.id)],
+            "region_edges": [
+                asdict(edge)
+                for edge in sorted(self.region_edges, key=lambda item: (item.src, item.dst))
+            ],
+            "partial_files": sorted(self.partial_files),
+        }
+
+    def to_json(self) -> str:
+        """Serialize with stable formatting so identical input yields bytes."""
+
+        return json.dumps(self.to_dict(), indent=2, sort_keys=True, ensure_ascii=False) + "\n"
+
+    def write_json(self, destination: Path) -> None:
+        """Write canonical graph JSON to ``destination``."""
+
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_text(self.to_json(), encoding="utf-8")
+
+
+class LanguageAdapter(Protocol):
+    """The seam every supported language implements."""
+
+    language: str
+    file_extensions: frozenset[str]
+    ignored_directories: frozenset[str]
+
+    def discover(self, path: Path) -> tuple[Path, tuple[Path, ...]]:
+        """Return the exact root and supported files this adapter will parse."""
+
+    def parse(self, path: Path, *, entrypoint: str | None = None) -> Graph:
+        """Parse ``path`` into a graph without inventing source structure."""
+
+    def parse_files(
+        self,
+        project_root: Path,
+        files: tuple[Path, ...],
+        *,
+        entrypoint: str | None = None,
+    ) -> Graph:
+        """Parse files already discovered as owned by this adapter."""
+
+    def concepts(self, node: Node, source: str) -> list[ConceptAnnotation]:
+        """Return only language constructs proven present in ``source``."""

codemble/adapters/discovery.py

@@ -0,0 +1,190 @@
+"""Shared, deterministic source discovery for language adapters."""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+_IGNORED_DIRECTORIES = {"venv", ".venv", "node_modules", "__pycache__"}
+
+
+class SourceDiscoveryError(ValueError):
+    """The requested source scope does not exist or is not readable as a scope."""
+
+
+@dataclass(frozen=True, slots=True)
+class SourceDiscovery:
+    """A normalized project root and its supported source files."""
+
+    root: Path
+    files: tuple[Path, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class SourceOwnership:
+    """One adapter's source extensions and directory exclusions."""
+
+    owner: str
+    extensions: frozenset[str]
+    ignored_directories: frozenset[str]
+
+
+@dataclass(frozen=True, slots=True)
+class OwnedSourceFiles:
+    """Files assigned to one adapter during project discovery."""
+
+    owner: str
+    files: tuple[Path, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class ProjectSourceDiscovery:
+    """One normalized project root with all adapter ownership resolved."""
+
+    root: Path
+    ownership: tuple[OwnedSourceFiles, ...]
+
+    @property
+    def files(self) -> tuple[Path, ...]:
+        return tuple(
+            sorted({file for owned in self.ownership for file in owned.files})
+        )
+
+
+def discover_source_files(
+    requested: Path,
+    extensions: frozenset[str],
+    *,
+    ignored_directories: frozenset[str] = frozenset(),
+) -> SourceDiscovery:
+    """Discover matching files while honoring the project's root ``.gitignore``."""
+    owner = "source"
+    project = discover_project_sources(
+        requested,
+        (SourceOwnership(owner, extensions, ignored_directories),),
+    )
+    return SourceDiscovery(project.root, project.ownership[0].files)
+
+
+def discover_project_sources(
+    requested: Path,
+    ownership: tuple[SourceOwnership, ...],
+) -> ProjectSourceDiscovery:
+    """Discover every adapter's files in one deterministic filesystem walk."""
+
+    normalized = requested.expanduser().resolve()
+    if not normalized.exists():
+        raise SourceDiscoveryError(f"path does not exist: {normalized}")
+    if not ownership:
+        raise ValueError("project source discovery requires at least one owner")
+    discovered: dict[str, list[Path]] = {rule.owner: [] for rule in ownership}
+    if normalized.is_file():
+        for rule in ownership:
+            if normalized.suffix.lower() in rule.extensions:
+                discovered[rule.owner].append(normalized)
+        return ProjectSourceDiscovery(
+            normalized.parent,
+            tuple(
+                OwnedSourceFiles(rule.owner, tuple(discovered[rule.owner]))
+                for rule in ownership
+            ),
+        )
+    if not normalized.is_dir():
+        raise SourceDiscoveryError(f"expected a source file or directory: {normalized}")
+
+    ignore_rules = _load_gitignore(normalized)
+    for current, directory_names, file_names in os.walk(normalized):
+        current_path = Path(current)
+        directory_names[:] = sorted(
+            directory_name
+            for directory_name in directory_names
+            if not _ignore_project_directory(
+                (current_path / directory_name).relative_to(normalized),
+                ignore_rules,
+                ownership,
+            )
+        )
+        for file_name in sorted(file_names):
+            candidate = current_path / file_name
+            relative = candidate.relative_to(normalized)
+            if _matches_gitignore(relative, False, ignore_rules):
+                continue
+            for rule in ownership:
+                if candidate.suffix.lower() not in rule.extensions:
+                    continue
+                if any(part in rule.ignored_directories for part in relative.parts):
+                    continue
+                discovered[rule.owner].append(candidate)
+    return ProjectSourceDiscovery(
+        normalized,
+        tuple(
+            OwnedSourceFiles(rule.owner, tuple(sorted(discovered[rule.owner])))
+            for rule in ownership
+        ),
+    )
+
+
+def _load_gitignore(root: Path) -> tuple[tuple[str, bool], ...]:
+    gitignore = root / ".gitignore"
+    if not gitignore.is_file():
+        return ()
+    rules: list[tuple[str, bool]] = []
+    for raw_line in gitignore.read_text(encoding="utf-8", errors="replace").splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        negated = line.startswith("!")
+        pattern = line[1:] if negated else line
+        rules.append((pattern, negated))
+    return tuple(rules)
+
+
+def _ignore_project_directory(
+    relative: Path,
+    rules: tuple[tuple[str, bool], ...],
+    ownership: tuple[SourceOwnership, ...],
+) -> bool:
+    if any(part.startswith(".") for part in relative.parts):
+        return True
+    if any(part in _IGNORED_DIRECTORIES for part in relative.parts):
+        return True
+    if _matches_gitignore(relative, True, rules):
+        return True
+    return all(
+        any(part in rule.ignored_directories for part in relative.parts)
+        for rule in ownership
+    )
+
+
+def _matches_gitignore(
+    relative: Path,
+    is_directory: bool,
+    rules: tuple[tuple[str, bool], ...],
+) -> bool:
+    path = relative.as_posix()
+    ignored = False
+    for raw_pattern, negated in rules:
+        directory_only = raw_pattern.endswith("/")
+        pattern = raw_pattern.rstrip("/").lstrip("/")
+        if directory_only and not is_directory and not path.startswith(f"{pattern}/"):
+            continue
+        if "/" in pattern:
+            matched = fnmatch.fnmatch(path, pattern) or path.startswith(f"{pattern}/")
+        else:
+            matched = any(fnmatch.fnmatch(part, pattern) for part in relative.parts)
+        if matched:
+            ignored = not negated
+    return ignored
+
+
+__all__ = [
+    "OwnedSourceFiles",
+    "ProjectSourceDiscovery",
+    "SourceDiscovery",
+    "SourceDiscoveryError",
+    "SourceOwnership",
+    "discover_project_sources",
+    "discover_source_files",
+]

codemble/adapters/project.py

@@ -0,0 +1,206 @@
+"""Language-neutral project parsing and graph composition."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+from codemble.adapters.base import AdapterParseError, Graph, LanguageAdapter, Node
+from codemble.adapters.discovery import (
+    OwnedSourceFiles,
+    SourceDiscoveryError,
+    SourceOwnership,
+    discover_project_sources,
+)
+from codemble.graph.finalize import GraphFinalizationError, finalize_graph
+
+
+class ProjectParseError(AdapterParseError):
+    """A project cannot be composed into one honest graph."""
+
+
+class ProjectScaleError(ProjectParseError):
+    """A discovered project needs a smaller learner-selected scope."""
+
+    def __init__(self, intake: ProjectIntake, scale_cap: int) -> None:
+        self.intake = intake
+        self.scale_cap = scale_cap
+        super().__init__(
+            f"found {len(intake.files)} supported source files; Codemble is capped at "
+            f"{scale_cap}. Re-run with `codemble --path PATH` to choose a project "
+            "subdirectory."
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class ProjectIntake:
+    """One supported project scope with adapter ownership resolved once."""
+
+    path: Path
+    root: Path
+    files: tuple[Path, ...]
+    _ownership: tuple[OwnedSourceFiles, ...]
+
+    def _files_for(self, language: str) -> tuple[Path, ...]:
+        return next(
+            (owned.files for owned in self._ownership if owned.owner == language),
+            (),
+        )
+
+    def scope_counts(self) -> tuple[tuple[str, int], ...]:
+        """Count supported files per top-level directory, busiest first."""
+
+        counts: dict[str, int] = {}
+        for file in self.files:
+            relative = file.relative_to(self.root)
+            directory = relative.parts[0] if len(relative.parts) > 1 else "."
+            counts[directory] = counts.get(directory, 0) + 1
+        return tuple(
+            sorted(counts.items(), key=lambda item: (-item[1], item[0]))
+        )
+
+
+class ProjectParser:
+    """Discover supported languages and compose their graphs behind one interface."""
+
+    def __init__(self, adapters: Iterable[LanguageAdapter] | None = None) -> None:
+        if adapters is None:
+            from codemble.adapters.python_ast import PythonAstAdapter
+            from codemble.adapters.typescript_tree_sitter import (
+                JavaScriptTypeScriptAdapter,
+            )
+
+            adapters = (PythonAstAdapter(), JavaScriptTypeScriptAdapter())
+        self._adapters = tuple(adapters)
+        if not self._adapters:
+            raise ValueError("ProjectParser requires at least one language adapter")
+        languages = [adapter.language for adapter in self._adapters]
+        if len(languages) != len(set(languages)):
+            raise ValueError("ProjectParser adapter languages must be unique")
+
+    @property
+    def languages(self) -> tuple[str, ...]:
+        """Return supported language identifiers in stable registry order."""
+
+        return tuple(adapter.language for adapter in self._adapters)
+
+    scale_cap = 300
+
+    def intake(self, path: Path, *, explicit: bool = False) -> ProjectIntake:
+        """Resolve one project scope and every adapter's owned files."""
+
+        normalized = path.expanduser().resolve()
+        extensions = frozenset().union(*(adapter.file_extensions for adapter in self._adapters))
+        ownership = tuple(
+            SourceOwnership(
+                owner=adapter.language,
+                extensions=adapter.file_extensions,
+                ignored_directories=adapter.ignored_directories,
+            )
+            for adapter in self._adapters
+        )
+        try:
+            discovery = discover_project_sources(normalized, ownership)
+        except SourceDiscoveryError as error:
+            raise ProjectParseError(str(error)) from error
+        files = discovery.files
+        if not files:
+            expected = ", ".join(sorted(extensions))
+            raise ProjectParseError(
+                f"no supported source files found under: {normalized} "
+                f"(expected {expected})"
+            )
+        intake = ProjectIntake(
+            path=normalized,
+            root=discovery.root,
+            files=files,
+            _ownership=discovery.ownership,
+        )
+        if not explicit and len(files) > self.scale_cap:
+            raise ProjectScaleError(intake, self.scale_cap)
+        return intake
+
+    def discover(self, path: Path) -> tuple[Path, tuple[Path, ...]]:
+        """Return all files accepted by the registered language adapters."""
+
+        intake = self.intake(path)
+        return intake.root, intake.files
+
+    def parse(
+        self,
+        source: Path | ProjectIntake,
+        *,
+        entrypoint: str | None = None,
+        explicit: bool = False,
+    ) -> Graph:
+        """Parse every detected language and return one deterministic graph."""
+
+        intake = (
+            source
+            if isinstance(source, ProjectIntake)
+            else self.intake(source, explicit=explicit)
+        )
+        graphs: list[Graph] = []
+        for adapter in self._adapters:
+            files = intake._files_for(adapter.language)
+            if not files:
+                continue
+            try:
+                graphs.append(adapter.parse_files(intake.root, files))
+            except AdapterParseError as error:
+                raise ProjectParseError(str(error)) from error
+        return _compose_graphs(tuple(graphs), intake.root, entrypoint)
+
+def _compose_graphs(
+    graphs: tuple[Graph, ...],
+    project_root: Path,
+    entrypoint: str | None,
+) -> Graph:
+    nodes: list[Node] = []
+    edges = []
+    annotations = []
+    partial_files: set[str] = set()
+    file_hashes: dict[str, str] = {}
+    node_ids: set[str] = set()
+
+    for graph in graphs:
+        for node in graph.nodes:
+            if node.id in node_ids:
+                raise ProjectParseError(
+                    f"language adapters produced the same node ID: {node.id}"
+                )
+            node_ids.add(node.id)
+            nodes.append(node)
+        edges.extend(graph.edges)
+        annotations.extend(graph.concept_annotations)
+        partial_files.update(graph.partial_files)
+        for file, digest in graph.file_hashes.items():
+            existing = file_hashes.get(file)
+            if existing is not None and existing != digest:
+                raise ProjectParseError(
+                    f"language adapters disagreed on the source hash for: {file}"
+                )
+            file_hashes[file] = digest
+
+    draft = Graph(
+        nodes=tuple(nodes),
+        edges=tuple(edges),
+        entrypoint_candidates=(),
+        project_root=str(project_root),
+        file_hashes=file_hashes,
+        concept_annotations=tuple(annotations),
+        partial_files=tuple(partial_files),
+    )
+    try:
+        return finalize_graph(draft, entrypoint=entrypoint)
+    except GraphFinalizationError as error:
+        raise ProjectParseError(str(error)) from error
+
+
+__all__ = [
+    "ProjectIntake",
+    "ProjectParseError",
+    "ProjectParser",
+    "ProjectScaleError",
+]