coding-os 0.3.2__py3-none-any.whl

core/graph_os/extractors/task_deps.py

@@ -0,0 +1,345 @@
+"""graph_os — task dependency extractor (I.3).
+
+DEPENDS:  core/thinking_os/task_parser.py (existing parser).
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import sys
+from pathlib import Path, PurePosixPath
+
+from ..types import GraphEdge, GraphNode
+from .md_links import (
+    ExtractionResult,
+    ParseError,
+    _normalize_path,
+    _promote_stubs,
+    _resolve_through_symlink,
+    emit_contains_spine,
+    file_uid as doc_file_uid,
+)
+
+logger = logging.getLogger("graph_os.extractors.task_deps")
+
+EXTRACTOR_ID = "task_deps@v1"
+
+# Canonical task id: TASK-NNN (zero-padded to at least 3 digits, like
+# `task_parser.extract_task_id_from_h1`).
+_TASK_ID_RE = re.compile(r"TASK-(?P<num>\d+)")
+_DOC_PATH_RE = re.compile(r"([A-Za-z0-9_./\-]+\.md)")
+_SCOPE_PATH_RE = re.compile(
+    r"([A-Za-z0-9_./\-]+\.(?:py|ts|tsx|js|jsx|sh|yaml|yml|go|rs|java|md|json|toml))"
+)
+
+
+def task_uid(task_id: str) -> str:
+    """Stable uid for a task node — normalised on the canonical form."""
+    return f"task:file:{_canonical_task_id(task_id)}"
+
+
+def _canonical_task_id(raw: str) -> str:
+    match = _TASK_ID_RE.search(raw)
+    if not match:
+        return raw.strip()
+    num = match.group("num")
+    # task_parser uses zero-padded-to-3 — keep the same for edge equality.
+    return f"TASK-{int(num):03d}"
+
+
+# ---------------------------------------------------------------------------
+# task_parser bridge
+# ---------------------------------------------------------------------------
+
+
+def _import_task_parser():
+    try:
+        import task_parser  # type: ignore
+
+        return task_parser
+    except ImportError:
+        here = Path(__file__).resolve()
+        thinking_os = here.parent.parent.parent / "thinking_os"
+        if thinking_os.exists() and str(thinking_os) not in sys.path:
+            sys.path.insert(0, str(thinking_os))
+        import task_parser  # type: ignore
+
+        return task_parser
+
+
+# ---------------------------------------------------------------------------
+# Main entry
+# ---------------------------------------------------------------------------
+
+
+def extract(path: str, content: str) -> ExtractionResult:
+    """Parse a task file → task node + dependency / ssot / read_first edges."""
+    result = ExtractionResult()
+    try:
+        parser = _import_task_parser()
+        parsed = parser.parse_task_file(content)
+        normalised_path = _normalize_path(path)
+
+        if parsed is None:
+            # Not a task file — emit a minimal node so cross-references
+            # still resolve.
+            unknown_uid = f"task:file:unknown:{normalised_path}"
+            result.nodes.append(
+                GraphNode(
+                    uid=unknown_uid,
+                    kind="task:file",
+                    label=PurePosixPath(normalised_path).name,
+                    file_path=normalised_path,
+                    lang="md",
+                    metadata={"parse_error": "not_a_task", "extractor": EXTRACTOR_ID},
+                )
+            )
+            result.parse_errors.append(ParseError(kind="not_a_task", detail=normalised_path))
+            # S3: spine still attaches so unknown task files show up
+            # under their folder in the tree-view.
+            emit_contains_spine(
+                file_path=path,
+                file_uid_=unknown_uid,
+                result=result,
+                extractor_id=EXTRACTOR_ID,
+            )
+            return result
+
+        task_node = GraphNode(
+            uid=task_uid(parsed.task_id),
+            kind="task:file",
+            label=parsed.raw_title,
+            file_path=normalised_path,
+            lang="md",
+            doc_blob=parsed.goal_text[:2000] if parsed.goal_text else None,
+            content_hash=parsed.content_hash,
+            metadata={
+                "task_id": parsed.task_id,
+                "domain": parsed.domain,
+                "extractor": EXTRACTOR_ID,
+            },
+        )
+        result.nodes.append(task_node)
+
+        # Dependency edges — depends_on + blocks (inverse). Blocks is the
+        # inverse view materialised so `_dependents` queries don't have
+        # to scan every edge in the graph.
+        for dep in parsed.dependencies:
+            dep_uid = task_uid(dep)
+            if dep_uid == task_node.uid:
+                # Self-dependency — nonsensical, skip with a parse_error.
+                result.parse_errors.append(
+                    ParseError(kind="self_dependency", detail=parsed.task_id)
+                )
+                continue
+            result.edges.append(
+                GraphEdge(
+                    source_uid=task_node.uid,
+                    target_uid=dep_uid,
+                    edge_type="depends_on",
+                    extractor=EXTRACTOR_ID,
+                    confidence=1.0,
+                    source_span=f"{normalised_path}:dependencies",
+                )
+            )
+            result.edges.append(
+                GraphEdge(
+                    source_uid=dep_uid,
+                    target_uid=task_node.uid,
+                    edge_type="blocks",
+                    extractor=EXTRACTOR_ID,
+                    confidence=1.0,
+                    source_span=f"{normalised_path}:dependencies",
+                )
+            )
+
+        # Doc references — Source of Truth + Read First sections hold
+        # doc paths; emit `references_doc` edges so Researcher/Implementer can jump
+        # task → authoritative spec.
+        for doc_ref in _extract_doc_paths(parsed.source_of_truth, normalised_path):
+            result.edges.append(
+                GraphEdge(
+                    source_uid=task_node.uid,
+                    target_uid=doc_file_uid(doc_ref),
+                    edge_type="references_doc",
+                    extractor=EXTRACTOR_ID,
+                    confidence=0.95,
+                    source_span=f"{normalised_path}:source_of_truth",
+                )
+            )
+        for doc_ref in _extract_doc_paths(parsed.read_first, normalised_path):
+            result.edges.append(
+                GraphEdge(
+                    source_uid=task_node.uid,
+                    target_uid=doc_file_uid(doc_ref),
+                    edge_type="references_doc",
+                    extractor=EXTRACTOR_ID,
+                    confidence=0.9,
+                    source_span=f"{normalised_path}:read_first",
+                )
+            )
+
+        for scope_path in _extract_scope_paths(parsed.scope_in):
+            kind_for_target = "produces_doc" if scope_path.endswith(".md") else "produces_code"
+            result.edges.append(
+                GraphEdge(
+                    source_uid=task_node.uid,
+                    target_uid=_target_uid_for_file(scope_path),
+                    edge_type=kind_for_target,
+                    extractor=EXTRACTOR_ID,
+                    confidence=0.9,
+                    source_span=f"{normalised_path}:scope_in",
+                )
+            )
+
+        # S3: Folder→...→File spine anchored at the task uid so tree-
+        # view can render tasks under their folder parent.
+        emit_contains_spine(
+            file_path=path,
+            file_uid_=task_node.uid,
+            result=result,
+            extractor_id=EXTRACTOR_ID,
+        )
+
+        _promote_stubs(result)
+        return result
+
+    except Exception as exc:
+        logger.debug("task_deps.extract(%s) fatal: %s", path, exc)
+        result.parse_errors.append(ParseError(kind="fatal", detail=str(exc)))
+        return result
+
+
+# ---------------------------------------------------------------------------
+# produces_code — git-derived; out of the per-file write path.
+# ---------------------------------------------------------------------------
+
+
+def produces_code_edges(
+    *,
+    task_id: str,
+    modified_files: list[str],
+) -> list[GraphEdge]:
+    """Emit `produces_code` edges for files a task touched."""
+    canonical = _canonical_task_id(task_id)
+    seen: set[str] = set()
+    edges: list[GraphEdge] = []
+    for raw in modified_files:
+        path = _normalize_path(raw.strip())
+        if not path or path in seen:
+            continue
+        seen.add(path)
+        edges.append(
+            GraphEdge(
+                source_uid=task_uid(canonical),
+                target_uid=_target_uid_for_file(path),
+                edge_type="produces_code",
+                extractor=EXTRACTOR_ID,
+                confidence=0.85,
+                source_span=f"git-log:{canonical}",
+            )
+        )
+    return edges
+
+
+def _target_uid_for_file(path: str) -> str:
+    """Infer the canonical uid for a produced artifact path.
+
+    Docs land on doc:file:..., code files get code:file:... — later
+    slices (I.4+) reuse the same uid so `produces_code` edges connect
+    to the real Python / TS nodes.
+    """
+    if path.endswith(".md"):
+        return f"doc:file:{path}"
+    return f"code:file:{path}"
+
+
+# ---------------------------------------------------------------------------
+# Doc path helpers
+# ---------------------------------------------------------------------------
+
+
+def _resolve_doc_ref(origin_path: str, ref: str) -> str | None:
+    """Resolve a doc ref (possibly `../`-relative) against the task file
+    dir to a clean repo-rooted path.
+
+    Returns None when the ref is malformed (escapes repo root, or holds a
+    backtick / whitespace) so the caller can drop it instead of minting a
+    `doc:file:../…` stub that orphans on every reindex (R4 X7 residual).
+    """
+    ref = ref.strip().strip("`").strip()
+    if not ref or "`" in ref or any(c.isspace() for c in ref):
+        return None
+    if "../" not in ref and "./" not in ref:
+        return _normalize_path(ref)
+    from posixpath import dirname
+
+    origin_dir = dirname(_normalize_path(origin_path))
+    parts: list[str] = []
+    escaped = False
+    for part in f"{origin_dir}/{ref}".split("/"):
+        if part in ("", "."):
+            continue
+        if part == "..":
+            if parts:
+                parts.pop()
+            else:
+                escaped = True  # ref points above repo root → malformed
+            continue
+        parts.append(part)
+    if escaped:
+        return None
+    return "/".join(parts)
+
+
+def _extract_doc_paths(bullets: list[str], origin_path: str = "") -> list[str]:
+    """Pull `docs/...md` paths out of task-section bullets."""
+    paths: list[str] = []
+    seen: set[str] = set()
+    for item in bullets:
+        for match in _DOC_PATH_RE.finditer(item):
+            candidate = _resolve_doc_ref(origin_path, match.group(1))
+            if candidate is None or candidate in seen:
+                continue
+            seen.add(candidate)
+            # References are reads — a path not on disk is noise, not a
+            # contract. The greedy regex also captures bare names
+            # (`SKILL.md`) and URL tails (`//github.com/...` — `:` is
+            # outside its class so the scheme is cut); minting those makes
+            # permanent stale doc:file stubs (existence gate: roadmap §6,
+            # cwd == project root like md_links' checks).
+            if not Path(candidate).is_file():
+                continue
+            # Symlink ref (CLAUDE.md → AGENTS.md) lands on the real file —
+            # walk_local never owns symlink paths (roadmap §6).
+            resolved = _resolve_through_symlink(candidate)
+            if not resolved:
+                continue
+            if resolved != candidate:
+                if resolved in seen:
+                    continue
+                seen.add(resolved)
+            paths.append(resolved)
+    return paths
+
+
+def _extract_scope_paths(bullets: list[str]) -> list[str]:
+    paths: list[str] = []
+    seen: set[str] = set()
+    for item in bullets:
+        for match in _SCOPE_PATH_RE.finditer(item):
+            candidate = _normalize_path(match.group(1))
+            if candidate in seen:
+                continue
+            seen.add(candidate)
+            paths.append(candidate)
+    return paths
+
+
+__all__ = [
+    "EXTRACTOR_ID",
+    "extract",
+    "produces_code_edges",
+    "task_uid",
+]

core/graph_os/groups/__init__.py

@@ -0,0 +1,22 @@
+"""graph_os repo groups (I.12)."""
+
+from .cross_repo import CrossRepoReport, infer_cross_repo_edges
+from .manifest import (
+    ConflictError,
+    GroupManifest,
+    GroupMember,
+    load_manifest,
+    register_member,
+    save_manifest,
+)
+
+__all__ = [
+    "ConflictError",
+    "CrossRepoReport",
+    "GroupManifest",
+    "GroupMember",
+    "infer_cross_repo_edges",
+    "load_manifest",
+    "register_member",
+    "save_manifest",
+]

core/graph_os/groups/cross_repo.py

@@ -0,0 +1,156 @@
+"""Cross-repo edge inference (I.12)."""
+
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from collections.abc import Iterable
+from dataclasses import dataclass, field
+
+from ..types import EvidenceSignal, GraphEdge
+from .manifest import GroupManifest, GroupMember
+
+logger = logging.getLogger("graph_os.groups.cross_repo")
+
+EXTRACTOR_ID = "groups.cross_repo@v1"
+
+
+@dataclass
+class MemberInputs:
+    """Normalised per-member contribution to the group."""
+
+    alias: str
+    routes: list[dict] = field(default_factory=list)  # {method, path, handler_uid?}
+    mcp_tools: list[str] = field(default_factory=list)
+    fetches: list[dict] = field(default_factory=list)  # {caller_uid, method, path}
+
+
+@dataclass
+class CrossRepoReport:
+    edges: list[GraphEdge] = field(default_factory=list)
+    ambiguous_routes: list[str] = field(default_factory=list)
+    unresolved_fetches: list[str] = field(default_factory=list)
+
+
+def infer_cross_repo_edges(
+    manifest: GroupManifest,
+    inputs: Iterable[MemberInputs],
+) -> CrossRepoReport:
+    """Build cross-repo edges from per-member extract data."""
+    report = CrossRepoReport()
+
+    # Index all routes for quick lookup by (method, path).
+    route_index: dict[tuple[str, str], list[tuple[str, dict]]] = defaultdict(list)
+    for mi in inputs:
+        for route in mi.routes:
+            key = (route.get("method", "get").lower(), route.get("path", ""))
+            route_index[key].append((mi.alias, route))
+
+    # MCP tool owners.
+    tool_owners: dict[str, str] = {}
+    for mi in inputs:
+        for tool in mi.mcp_tools:
+            tool_owners[tool] = mi.alias
+
+    inputs_list = list(inputs)
+
+    for mi in inputs_list:
+        for fetch in mi.fetches:
+            method = fetch.get("method", "get").lower()
+            path = fetch.get("path", "")
+            owners = _owning_members(manifest, path)
+            candidates = route_index.get((method, path), [])
+            if owners:
+                # Explicit ownership → confidence 0.95 to declared target.
+                for owner_alias in owners:
+                    target = next((r for alias, r in candidates if alias == owner_alias), None)
+                    if target is None:
+                        # Owner declared the route but extractor didn't pick it
+                        # up (could be a dynamic route). Still emit a
+                        # high-confidence edge to the owner's synthetic uid.
+                        target = {
+                            "method": method,
+                            "path": path,
+                            "handler_uid": f"cos:route:{method.upper()}:{path}",
+                        }
+                    report.edges.append(
+                        _build_edge(
+                            source_uid=fetch.get("caller_uid", f"code:unknown:{mi.alias}"),
+                            target_uid=target.get("handler_uid")
+                            or f"cos:route:{method.upper()}:{path}",
+                            edge_type="calls_contract",
+                            confidence=0.95,
+                            evidence=(
+                                EvidenceSignal("ownership_declared", 0.55, note=owner_alias),
+                                EvidenceSignal("route_match", 0.4),
+                            ),
+                        )
+                    )
+                continue
+
+            if candidates:
+                # No explicit ownership — inferred link to each plausible owner.
+                for alias, target in candidates:
+                    if alias == mi.alias:
+                        continue  # self-edge — skip
+                    report.edges.append(
+                        _build_edge(
+                            source_uid=fetch.get("caller_uid", f"code:unknown:{mi.alias}"),
+                            target_uid=target.get("handler_uid")
+                            or f"cos:route:{method.upper()}:{path}",
+                            edge_type="calls_contract",
+                            confidence=0.6,
+                            evidence=(EvidenceSignal("route_match_inferred", 0.6),),
+                        )
+                    )
+                    report.ambiguous_routes.append(path)
+            else:
+                report.unresolved_fetches.append(path)
+
+    # MCP tool edges — tools are unique strings so confidence 0.95.
+    for mi in inputs_list:
+        for fetch in mi.mcp_tools:
+            # no-op; MCP tool edges are emitted when someone imports
+            # `cos_graph_query` etc. — callers supply that in `fetches`
+            # with method="mcp". For now the simpler contract:
+            if fetch in tool_owners and tool_owners[fetch] != mi.alias:
+                report.edges.append(
+                    _build_edge(
+                        source_uid=f"code:unknown:{mi.alias}",
+                        target_uid=f"cos:mcp_tool:{fetch}",
+                        edge_type="calls_mcp_tool",
+                        confidence=0.95,
+                        evidence=(EvidenceSignal("mcp_tool_unique", 0.95),),
+                    )
+                )
+
+    return report
+
+
+def _owning_members(manifest: GroupManifest, path: str) -> list[str]:
+    return [m.alias for m in manifest.members if m.owns_route(path)]
+
+
+def _build_edge(
+    *,
+    source_uid: str,
+    target_uid: str,
+    edge_type: str,
+    confidence: float,
+    evidence: tuple[EvidenceSignal, ...] = (),
+) -> GraphEdge:
+    return GraphEdge(
+        source_uid=source_uid,
+        target_uid=target_uid,
+        edge_type=edge_type,
+        extractor=EXTRACTOR_ID,
+        confidence=confidence,
+        evidence=evidence,
+    )
+
+
+__all__ = [
+    "CrossRepoReport",
+    "MemberInputs",
+    "infer_cross_repo_edges",
+]

core/graph_os/groups/manifest.py

@@ -0,0 +1,141 @@
+"""Group manifest + membership.yaml handling (I.12)."""
+
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("graph_os.groups.manifest")
+
+
+class ConflictError(RuntimeError):
+    """Two members declare overlapping `owns` entries."""
+
+
+@dataclass
+class GroupMember:
+    alias: str
+    path: str  # repo root path
+    owned_routes: list[str] = field(default_factory=list)
+    owned_mcp_tools: list[str] = field(default_factory=list)
+    owned_event_topics: list[str] = field(default_factory=list)
+
+    def owns_route(self, path: str) -> bool:
+        return any(_match(path, p) for p in self.owned_routes)
+
+    def owns_event(self, topic: str) -> bool:
+        return any(_match(topic, p) for p in self.owned_event_topics)
+
+
+@dataclass
+class GroupManifest:
+    name: str
+    members: list[GroupMember] = field(default_factory=list)
+
+    def by_alias(self, alias: str) -> GroupMember | None:
+        for member in self.members:
+            if member.alias == alias:
+                return member
+        return None
+
+    def validate(self) -> None:
+        """Raise ConflictError if two members declare the same route."""
+        seen: dict[str, str] = {}
+        for member in self.members:
+            for route in member.owned_routes:
+                existing = seen.get(route)
+                if existing and existing != member.alias:
+                    raise ConflictError(
+                        f"route {route!r} declared by both {existing!r} and {member.alias!r}"
+                    )
+                seen[route] = member.alias
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "members": [
+                {
+                    "alias": m.alias,
+                    "path": m.path,
+                    "owns": {
+                        "http_routes": m.owned_routes,
+                        "mcp_tools": m.owned_mcp_tools,
+                        "event_topics": m.owned_event_topics,
+                    },
+                }
+                for m in self.members
+            ],
+        }
+
+
+def load_manifest(path: str | Path) -> GroupManifest:
+    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    members = []
+    for entry in data.get("members", []):
+        owns = entry.get("owns", {}) or {}
+        members.append(
+            GroupMember(
+                alias=entry["alias"],
+                path=entry["path"],
+                owned_routes=list(owns.get("http_routes", []) or []),
+                owned_mcp_tools=list(owns.get("mcp_tools", []) or []),
+                owned_event_topics=list(owns.get("event_topics", []) or []),
+            )
+        )
+    return GroupManifest(name=data["name"], members=members)
+
+
+def save_manifest(manifest: GroupManifest, path: str | Path) -> None:
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    Path(path).write_text(
+        json.dumps(manifest.to_dict(), indent=2, sort_keys=True), encoding="utf-8"
+    )
+
+
+def register_member(
+    manifest: GroupManifest,
+    *,
+    alias: str,
+    path: str,
+    owned_routes: list[str] | None = None,
+    owned_mcp_tools: list[str] | None = None,
+    owned_event_topics: list[str] | None = None,
+) -> GroupManifest:
+    member = GroupMember(
+        alias=alias,
+        path=path,
+        owned_routes=list(owned_routes or []),
+        owned_mcp_tools=list(owned_mcp_tools or []),
+        owned_event_topics=list(owned_event_topics or []),
+    )
+    members = [m for m in manifest.members if m.alias != alias]
+    members.append(member)
+    new_manifest = GroupManifest(name=manifest.name, members=members)
+    new_manifest.validate()
+    return new_manifest
+
+
+def _match(path: str, pattern: str) -> bool:
+    """Support simple wildcards (`/api/*`, `/internal/**`)."""
+    if pattern == path:
+        return True
+    if pattern.endswith("/**"):
+        prefix = pattern[:-3]
+        return path.startswith(prefix)
+    if pattern.endswith("/*"):
+        prefix = pattern[:-2]
+        return path.startswith(prefix) and "/" not in path[len(prefix) + 1 :]
+    return False
+
+
+__all__ = [
+    "ConflictError",
+    "GroupManifest",
+    "GroupMember",
+    "load_manifest",
+    "register_member",
+    "save_manifest",
+]

core/graph_os/ingest/__init__.py

@@ -0,0 +1,19 @@
+"""Ingestion flexibility — local folders, GitHub clones, ZIP archives.
+
+All three paths funnel into `core/graph_os/ingest/base.py::IngestPlan`
+so the orchestrator can treat any source uniformly.
+"""
+
+from .base import IngestError, IngestPlan, walk_local
+from .github import GithubSize, clone_github
+from .zip import ZipSize, extract_zip
+
+__all__ = [
+    "GithubSize",
+    "IngestError",
+    "IngestPlan",
+    "ZipSize",
+    "clone_github",
+    "extract_zip",
+    "walk_local",
+]