knowcode 0.1.0__py3-none-any.whl

structural_engine/parser/languages/python/adapter.py

@@ -0,0 +1,174 @@
+"""Python structural extractor.
+
+Walks a Python tree-sitter AST to extract entities, relationships,
+and raw calls.
+"""
+
+from __future__ import annotations
+
+import tree_sitter
+
+from structural_engine.parser.extractors.base import ExtractionResult, LanguageAdapter
+from structural_engine.parser.models import (
+    Entity,
+    EntityType,
+    FileInfo,
+    RawCall,
+    Relationship,
+    RelationshipType,
+)
+
+class PythonAdapter(LanguageAdapter):
+    """AST extractor for Python."""
+
+    def extract(
+        self, file_info: FileInfo, tree: tree_sitter.Tree, source_bytes: bytes
+    ) -> ExtractionResult:
+        result = ExtractionResult()
+
+        file_id = file_info.relative_path
+        file_entity = Entity(
+            id=file_id,
+            type=EntityType.FILE,
+            name=file_info.absolute_path.name,
+            path=file_info.relative_path,
+            parent_id=str(file_info.absolute_path.parent.relative_to(file_info.absolute_path.parents[len(file_info.relative_path.split("/")) - 1])) if "/" in file_info.relative_path else "repo", # Actually let's just make the parser build the directories later or assign properly. Let's simplify and make the caller build dir entities. Or build them here if needed.
+            start_line=1,
+            end_line=source_bytes.count(b"\n") + 1,
+        )
+        result.entities.append(file_entity)
+        
+        # Let's fix parent_id for file. The parser.py should probably assemble Repository and Directory entities.
+        # But we'll leave it as None for now and let the caller fix it, or we can just say parent_id is the directory.
+        parent_dir = file_info.relative_path.rsplit("/", 1)[0] if "/" in file_info.relative_path else "repo"
+        file_entity = Entity(
+            id=file_id,
+            type=EntityType.FILE,
+            name=file_info.absolute_path.name,
+            path=file_info.relative_path,
+            parent_id=parent_dir,
+            start_line=1,
+            end_line=source_bytes.count(b"\n") + 1,
+        )
+        result.entities[-1] = file_entity
+
+        def walk(node: tree_sitter.Node, current_parent_id: str, is_in_class: bool = False):
+            # Extract Imports
+            if node.type == "import_statement":
+                for child in node.children:
+                    if child.type == "dotted_name":
+                        imported_module = child.text.decode("utf-8")
+                        result.relationships.append(
+                            Relationship(
+                                source_id=file_id,
+                                target_id=imported_module, # This will be resolved conservatively later, but for now we record it as is. Wait, IMPORTS should target the module id.
+                                type=RelationshipType.IMPORTS,
+                            )
+                        )
+            elif node.type == "import_from_statement":
+                module_node = node.child_by_field_name("module_name")
+                if module_node:
+                    module_name = module_node.text.decode("utf-8")
+                    result.relationships.append(
+                        Relationship(
+                            source_id=file_id,
+                            target_id=module_name,
+                            type=RelationshipType.IMPORTS,
+                        )
+                    )
+
+            # Extract Classes
+            elif node.type == "class_definition":
+                name_node = node.child_by_field_name("name")
+                if name_node:
+                    name = name_node.text.decode("utf-8")
+                    class_id = f"{current_parent_id}::{name}"
+                    
+                    result.entities.append(
+                        Entity(
+                            id=class_id,
+                            type=EntityType.CLASS,
+                            name=name,
+                            path=file_info.relative_path,
+                            parent_id=current_parent_id,
+                            start_line=node.start_point[0] + 1,
+                            end_line=node.end_point[0] + 1,
+                        )
+                    )
+                    result.relationships.append(
+                        Relationship(
+                            source_id=current_parent_id,
+                            target_id=class_id,
+                            type=RelationshipType.CONTAINS,
+                        )
+                    )
+                    
+                    # Extract Base Classes (INHERITS)
+                    superclasses = node.child_by_field_name("superclasses")
+                    if superclasses:
+                        for child in superclasses.children:
+                            if child.type == "identifier" or child.type == "attribute":
+                                base_name = child.text.decode("utf-8")
+                                result.relationships.append(
+                                    Relationship(
+                                        source_id=class_id,
+                                        target_id=base_name, # Raw target, we may need to resolve it, but INHERITS is usually raw or we can use the same resolver.
+                                        type=RelationshipType.INHERITS,
+                                    )
+                                )
+
+                    for child in node.children:
+                        walk(child, class_id, is_in_class=True)
+                return
+
+            # Extract Functions/Methods
+            elif node.type == "function_definition":
+                name_node = node.child_by_field_name("name")
+                if name_node:
+                    name = name_node.text.decode("utf-8")
+                    func_id = f"{current_parent_id}::{name}"
+                    func_type = EntityType.METHOD if is_in_class else EntityType.FUNCTION
+                    
+                    result.entities.append(
+                        Entity(
+                            id=func_id,
+                            type=func_type,
+                            name=name,
+                            path=file_info.relative_path,
+                            parent_id=current_parent_id,
+                            start_line=node.start_point[0] + 1,
+                            end_line=node.end_point[0] + 1,
+                        )
+                    )
+                    result.relationships.append(
+                        Relationship(
+                            source_id=current_parent_id,
+                            target_id=func_id,
+                            type=RelationshipType.CONTAINS,
+                        )
+                    )
+                    for child in node.children:
+                        walk(child, func_id, is_in_class=False)
+                return
+
+            # Extract Calls
+            elif node.type == "call":
+                func_node = node.child_by_field_name("function")
+                if func_node:
+                    target_name = func_node.text.decode("utf-8")
+                    if "." in target_name:
+                        target_name = target_name.split(".")[-1]
+                    result.raw_calls.append(
+                        RawCall(
+                            caller_id=current_parent_id,
+                            target_name=target_name,
+                            source_file=file_info.relative_path,
+                            line=node.start_point[0] + 1,
+                        )
+                    )
+            
+            for child in node.children:
+                walk(child, current_parent_id, is_in_class)
+
+        walk(tree.root_node, file_id)
+        return result

structural_engine/parser/languages/typescript/adapter.py

@@ -0,0 +1,165 @@
+"""TypeScript structural extractor.
+
+Walks a TypeScript tree-sitter AST to extract entities, relationships,
+and raw calls.
+"""
+
+from __future__ import annotations
+
+import tree_sitter
+
+from structural_engine.parser.extractors.base import ExtractionResult, LanguageAdapter
+from structural_engine.parser.models import (
+    Entity,
+    EntityType,
+    FileInfo,
+    RawCall,
+    Relationship,
+    RelationshipType,
+)
+
+
+class TypeScriptAdapter(LanguageAdapter):
+    """AST extractor for TypeScript."""
+
+    def extract(
+        self, file_info: FileInfo, tree: tree_sitter.Tree, source_bytes: bytes
+    ) -> ExtractionResult:
+        result = ExtractionResult()
+
+        file_id = file_info.relative_path
+        parent_dir = file_info.relative_path.rsplit("/", 1)[0] if "/" in file_info.relative_path else "repo"
+        file_entity = Entity(
+            id=file_id,
+            type=EntityType.FILE,
+            name=file_info.absolute_path.name,
+            path=file_info.relative_path,
+            parent_id=parent_dir,
+            start_line=1,
+            end_line=source_bytes.count(b"\n") + 1,
+        )
+        result.entities.append(file_entity)
+
+        def walk(node: tree_sitter.Node, current_parent_id: str, is_in_class: bool = False):
+            # Extract Imports
+            if node.type == "import_statement":
+                source_node = node.child_by_field_name("source")
+                if source_node:
+                    imported_module = source_node.text.decode("utf-8").strip("'\"")
+                    result.relationships.append(
+                        Relationship(
+                            source_id=file_id,
+                            target_id=imported_module,
+                            type=RelationshipType.IMPORTS,
+                        )
+                    )
+
+            # Extract Classes / Interfaces
+            elif node.type in ("class_declaration", "interface_declaration"):
+                name_node = node.child_by_field_name("name")
+                if name_node:
+                    name = name_node.text.decode("utf-8")
+                    entity_id = f"{current_parent_id}::{name}"
+                    
+                    entity_type = EntityType.INTERFACE if node.type == "interface_declaration" else EntityType.CLASS
+
+                    result.entities.append(
+                        Entity(
+                            id=entity_id,
+                            type=entity_type,
+                            name=name,
+                            path=file_info.relative_path,
+                            parent_id=current_parent_id,
+                            start_line=node.start_point[0] + 1,
+                            end_line=node.end_point[0] + 1,
+                        )
+                    )
+                    result.relationships.append(
+                        Relationship(
+                            source_id=current_parent_id,
+                            target_id=entity_id,
+                            type=RelationshipType.CONTAINS,
+                        )
+                    )
+
+                    for child in node.children:
+                        if child.type == "class_heritage":
+                            for heritage_child in child.children:
+                                if heritage_child.type == "extends_clause":
+                                    for gchild in heritage_child.children:
+                                        if gchild.type == "identifier" or gchild.type == "type_identifier":
+                                            result.relationships.append(
+                                                Relationship(
+                                                    source_id=entity_id,
+                                                    target_id=gchild.text.decode("utf-8"),
+                                                    type=RelationshipType.INHERITS,
+                                                )
+                                            )
+                                elif heritage_child.type == "implements_clause":
+                                    for gchild in heritage_child.children:
+                                        if gchild.type == "type_identifier" or gchild.type == "identifier":
+                                            result.relationships.append(
+                                                Relationship(
+                                                    source_id=entity_id,
+                                                    target_id=gchild.text.decode("utf-8"),
+                                                    type=RelationshipType.IMPLEMENTS,
+                                                )
+                                            )
+
+                    for child in node.children:
+                        walk(child, entity_id, is_in_class=(entity_type == EntityType.CLASS))
+                return
+
+            # Extract Functions
+            elif node.type in ("function_declaration", "method_definition", "arrow_function"):
+                # Arrow functions might not have a direct 'name' field if assigned to variable,
+                # but let's stick to basics for V1
+                name_node = node.child_by_field_name("name")
+                if name_node:
+                    name = name_node.text.decode("utf-8")
+                    func_id = f"{current_parent_id}::{name}"
+                    func_type = EntityType.METHOD if is_in_class or node.type == "method_definition" else EntityType.FUNCTION
+                    
+                    result.entities.append(
+                        Entity(
+                            id=func_id,
+                            type=func_type,
+                            name=name,
+                            path=file_info.relative_path,
+                            parent_id=current_parent_id,
+                            start_line=node.start_point[0] + 1,
+                            end_line=node.end_point[0] + 1,
+                        )
+                    )
+                    result.relationships.append(
+                        Relationship(
+                            source_id=current_parent_id,
+                            target_id=func_id,
+                            type=RelationshipType.CONTAINS,
+                        )
+                    )
+                    for child in node.children:
+                        walk(child, func_id, is_in_class=False)
+                return
+
+            # Extract Calls
+            elif node.type == "call_expression":
+                func_node = node.child_by_field_name("function")
+                if func_node:
+                    target_name = func_node.text.decode("utf-8")
+                    if "." in target_name:
+                        target_name = target_name.split(".")[-1]
+                    result.raw_calls.append(
+                        RawCall(
+                            caller_id=current_parent_id,
+                            target_name=target_name,
+                            source_file=file_info.relative_path,
+                            line=node.start_point[0] + 1,
+                        )
+                    )
+            
+            for child in node.children:
+                walk(child, current_parent_id, is_in_class)
+
+        walk(tree.root_node, file_id)
+        return result

structural_engine/parser/models.py

@@ -0,0 +1,186 @@
+"""Parser domain models.
+
+All structural data types produced by the parser pipeline.
+
+These models are internal to the Structural Engine.  The Runtime never
+imports them directly — it only sees the ``StructuralSnapshot`` indirectly
+through result objects returned by the Engine's public interface.
+
+Invariants
+----------
+- All models are frozen (immutable).
+- ``Entity.id`` uses stable, path-based identifiers — never UUIDs.
+- ``StructuralSnapshot`` contains no metadata, revisions, or timestamps.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from pathlib import Path
+
+
+# ── Enums ─────────────────────────────────────────────────────────────
+
+
+class EntityType(Enum):
+    """Classification of structural entities discovered in a repository."""
+
+    REPOSITORY = auto()
+    DIRECTORY = auto()
+    FILE = auto()
+    CLASS = auto()
+    INTERFACE = auto()
+    FUNCTION = auto()
+    METHOD = auto()
+
+
+class RelationshipType(Enum):
+    """Classification of structural relationships between entities."""
+
+    CONTAINS = auto()
+    IMPORTS = auto()
+    INHERITS = auto()
+    IMPLEMENTS = auto()
+    CALLS = auto()
+
+
+class Language(Enum):
+    """Supported programming languages for structural parsing.
+
+    V1 supports Python, TypeScript, and JavaScript.
+    """
+
+    PYTHON = auto()
+    TYPESCRIPT = auto()
+    JAVASCRIPT = auto()
+
+
+# ── Extension → Language mapping ──────────────────────────────────────
+
+EXTENSION_LANGUAGE_MAP: dict[str, Language] = {
+    ".py": Language.PYTHON,
+    ".ts": Language.TYPESCRIPT,
+    ".tsx": Language.TYPESCRIPT,
+    ".js": Language.JAVASCRIPT,
+}
+
+
+# ── Data models ───────────────────────────────────────────────────────
+
+
+@dataclass(frozen=True)
+class FileInfo:
+    """A discovered source file with its resolved language.
+
+    Produced by file discovery, consumed by the tree-sitter parsing stage.
+    """
+
+    absolute_path: Path
+    """Absolute path to the source file on disk."""
+
+    relative_path: str
+    """Path relative to repository root, using forward slashes (e.g. ``src/auth.py``)."""
+
+    language: Language
+    """Detected programming language."""
+
+
+@dataclass(frozen=True)
+class Entity:
+    """A structural entity extracted from the repository.
+
+    Entities are identified by stable, path-based string IDs that are
+    deterministic across machines and runs.
+
+    ID Examples
+    -----------
+    ::
+
+        repo
+        src
+        src/auth.py
+        src/auth.py::verify_token
+        src/auth.py::UserService
+        src/auth.py::UserService::create_user
+    """
+
+    id: str
+    """Stable path-based identifier.  Never a UUID."""
+
+    type: EntityType
+    """Structural classification of this entity."""
+
+    name: str
+    """Human-readable name (e.g. ``verify_token``, ``UserService``)."""
+
+    path: str
+    """Relative file path within the repository (forward slashes)."""
+
+    parent_id: str | None
+    """ID of the containing entity, or *None* for the repository root."""
+
+    start_line: int
+    """1-indexed start line in the source file.  0 for non-file entities."""
+
+    end_line: int
+    """1-indexed end line in the source file.  0 for non-file entities."""
+
+
+@dataclass(frozen=True)
+class Relationship:
+    """A directed structural relationship between two entities.
+
+    Relationships are matched for diffing using the composite key
+    ``(source_id, target_id, type)``.
+    """
+
+    source_id: str
+    """ID of the originating entity."""
+
+    target_id: str
+    """ID of the target entity."""
+
+    type: RelationshipType
+    """Classification of this relationship."""
+
+
+@dataclass(frozen=True)
+class RawCall:
+    """An unresolved function/method call extracted from source code.
+
+    Raw calls are an intermediate representation produced by call
+    extraction and consumed by the call resolver.  They do not appear
+    in the final snapshot.
+    """
+
+    caller_id: str
+    """Stable ID of the entity that makes the call."""
+
+    target_name: str
+    """Unresolved name of the called function/method."""
+
+    source_file: str
+    """Relative path to the source file."""
+
+    line: int
+    """1-indexed line number of the call site."""
+
+
+@dataclass(frozen=True)
+class StructuralSnapshot:
+    """Complete structural truth of a repository at a point in time.
+
+    Contains all discovered entities and relationships.
+    No metadata, revisions, timestamps, or reports.
+
+    Sorting contract (guarantees deterministic output):
+    - ``entities`` sorted by ``id``
+    - ``relationships`` sorted by ``(source_id, target_id, type.name)``
+    """
+
+    entities: tuple[Entity, ...] = field(default_factory=tuple)
+    """All structural entities, sorted by ``id``."""
+
+    relationships: tuple[Relationship, ...] = field(default_factory=tuple)
+    """All structural relationships, sorted by composite key."""

structural_engine/parser/parser.py

@@ -0,0 +1,160 @@
+"""Parser — public interface.
+
+The Parser is the observation subsystem of the Structural Engine.
+It answers one question:
+
+    *What does the repository look like right now?*
+
+Contract
+--------
+::
+
+    parse(paths: RepositoryPaths) -> StructuralSnapshot
+
+Invariants
+----------
+- **Deterministic:** Identical repository states always produce identical
+  snapshots, regardless of machine, user, or OS.
+- **Stateless:** Never reads ``state.yaml``, previous snapshots, reports,
+  logs, or knowledge.  Current repository state is the only authority.
+- **Side-effect free:** Performs no writes, no filesystem mutations,
+  no revision generation, no persistence.
+- **No Brain awareness:** Accepts ``RepositoryPaths`` but only consumes
+  ``paths.repo_root``.  All other fields are ignored.
+
+Pipeline (Phase 2A stub — discovery only)
+-----------------------------------------
+::
+
+    RepositoryPaths
+        ↓
+    repo_root
+        ↓
+    File Discovery
+        ↓
+    Language Detection
+        ↓
+    [Tree-sitter Parse]       ← Phase 2B
+        ↓
+    [Entity Extraction]       ← Phase 2B
+        ↓
+    [Relationship Extraction] ← Phase 2B
+        ↓
+    [Call Resolution]         ← Phase 2B
+        ↓
+    StructuralSnapshot
+"""
+
+from __future__ import annotations
+
+import structlog
+
+from runtime.repository.models import RepositoryPaths
+from structural_engine.parser.discovery import discover_files
+from structural_engine.parser.models import StructuralSnapshot, Entity, Relationship, RawCall, Language
+
+logger = structlog.get_logger(__name__)
+
+
+def parse(paths: RepositoryPaths) -> StructuralSnapshot:
+    """Parse a repository and produce its structural snapshot.
+
+    This is the sole public entry point of the Parser subsystem.
+    Called by ``StructuralEngine.initialize()`` and
+    ``StructuralEngine.sync()``.
+
+    Parameters
+    ----------
+    paths : RepositoryPaths
+        Canonical 9-field path contract.  Only ``paths.repo_root`` is
+        consumed by the parser.
+
+    Returns
+    -------
+    StructuralSnapshot
+        Complete structural truth of the repository.
+        Entities sorted by ``id``, relationships sorted by composite key.
+
+    Notes
+    -----
+    **Phase 2A stub:** Currently performs file discovery and language
+    detection only.  Tree-sitter extraction, entity/relationship
+    building, and call resolution will be added in Phase 2B.
+    """
+    repo_root = paths.repo_root
+
+    logger.info("parser.started", repo_root=str(repo_root))
+
+    # ── Stage 1: File Discovery + Language Detection ──────────────
+    files = discover_files(repo_root)
+    logger.info(
+        "parser.discovery_complete",
+        files_found=len(files),
+        languages={
+            lang.name: sum(1 for f in files if f.language == lang)
+            for lang in set(f.language for f in files)
+        },
+    )
+
+    # ── Stages 2–7: Tree-sitter parsing, extraction, resolution ──
+    all_entities: list[Entity] = []
+    all_relationships: list[Relationship] = []
+    all_raw_calls: list[RawCall] = []
+
+    for file_info in files:
+        # 1. Get the language parser
+        try:
+            from structural_engine.parser.tree_sitter.registry import get_parser
+            ts_parser = get_parser(file_info.language)
+        except ValueError:
+            logger.warning("parser.unsupported_language", file=file_info.relative_path, language=file_info.language.name)
+            continue
+
+        # 2. Parse the source code
+        try:
+            source_bytes = file_info.absolute_path.read_bytes()
+            tree = ts_parser.parse(source_bytes)
+        except Exception as e:
+            logger.error("parser.parse_failed", file=file_info.relative_path, error=str(e))
+            continue
+
+        # 3. Extract structural facts
+        adapter = None
+        if file_info.language == Language.PYTHON:
+            from structural_engine.parser.languages.python.adapter import PythonAdapter
+            adapter = PythonAdapter()
+        elif file_info.language == Language.JAVASCRIPT:
+            from structural_engine.parser.languages.javascript.adapter import JavaScriptAdapter
+            adapter = JavaScriptAdapter()
+        elif file_info.language == Language.TYPESCRIPT:
+            from structural_engine.parser.languages.typescript.adapter import TypeScriptAdapter
+            adapter = TypeScriptAdapter()
+
+        if adapter:
+            result = adapter.extract(file_info, tree, source_bytes)
+            all_entities.extend(result.entities)
+            all_relationships.extend(result.relationships)
+            all_raw_calls.extend(result.raw_calls)
+
+    # 4. Resolve calls
+    from structural_engine.parser.resolvers.calls import resolve_calls
+    resolved_calls = resolve_calls(all_entities, all_relationships, all_raw_calls)
+    all_relationships.extend(resolved_calls)
+
+    # 5. Sort for deterministic output
+    all_entities.sort(key=lambda e: e.id)
+    all_relationships.sort(key=lambda r: (r.source_id, r.target_id, r.type.name))
+
+    snapshot = StructuralSnapshot(
+        entities=tuple(all_entities),
+        relationships=tuple(all_relationships)
+    )
+
+    logger.info(
+        "parser.complete",
+        entities=len(snapshot.entities),
+        relationships=len(snapshot.relationships),
+        resolved_calls=len(resolved_calls),
+    )
+
+    return snapshot