kc-cli 0.4.0__py3-none-any.whl

kc/paths.py

@@ -0,0 +1,126 @@
+"""Repository path resolution and traversal checks."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from kc.errors import KcError
+
+
+@dataclass(frozen=True)
+class KcPaths:
+    root: Path
+    data_dir: Path
+    state_dir: Path
+
+    @property
+    def config_path(self) -> Path:
+        return self.root / "kc.toml"
+
+    @property
+    def sources_jsonl(self) -> Path:
+        return self.data_dir / "sources.jsonl"
+
+    @property
+    def ranges_jsonl(self) -> Path:
+        return self.data_dir / "source_ranges.jsonl"
+
+    @property
+    def source_revisions_jsonl(self) -> Path:
+        return self.data_dir / "source_revisions.jsonl"
+
+    @property
+    def artifacts_jsonl(self) -> Path:
+        return self.data_dir / "artifacts.jsonl"
+
+    @property
+    def citation_edges_jsonl(self) -> Path:
+        return self.data_dir / "citation_edges.jsonl"
+
+    @property
+    def sqlite_path(self) -> Path:
+        return self.state_dir / "state.sqlite"
+
+    @property
+    def locks_dir(self) -> Path:
+        return self.state_dir / "locks"
+
+    @property
+    def plans_dir(self) -> Path:
+        return self.state_dir / "plans"
+
+    @property
+    def snapshots_dir(self) -> Path:
+        return self.state_dir / "snapshots"
+
+    @property
+    def tasks_dir(self) -> Path:
+        return self.state_dir / "tasks"
+
+    @property
+    def context_dir(self) -> Path:
+        return self.state_dir / "context"
+
+    @property
+    def operations_dir(self) -> Path:
+        return self.state_dir / "operations"
+
+    @property
+    def wiki_dir(self) -> Path:
+        return self.data_dir / "wiki"
+
+    @property
+    def log_path(self) -> Path:
+        return self.wiki_dir / "log.md"
+
+
+def current_paths() -> KcPaths:
+    return current_workspace().paths
+
+
+def current_workspace():
+    from kc.workspace import resolve_workspace
+
+    return resolve_workspace()
+
+
+def ensure_data_dir_exists() -> KcPaths:
+    paths = current_paths()
+    if not paths.data_dir.exists():
+        raise KcError(
+            code="KC_CONFIG_NOT_FOUND",
+            message=f"Knowledge data directory not found: {repo_relative(paths.data_dir)}",
+            details={"data_dir": repo_relative(paths.data_dir)},
+        )
+    return paths
+
+
+def ensure_under_root(path: Path, root: Path | None = None) -> Path:
+    root = root or current_paths().root
+    resolved = path.resolve()
+    try:
+        resolved.relative_to(root)
+    except ValueError as exc:
+        raise KcError(
+            code="KC_PATH_OUTSIDE_REPO",
+            message=f"Path is outside repository root: {path}",
+            details={"path": str(path), "repo_root": str(root)},
+        ) from exc
+    return resolved
+
+
+def resolve_repo_path(path: Path | str, root: Path | None = None) -> Path:
+    root = root or current_paths().root
+    candidate = Path(path).expanduser()
+    if not candidate.is_absolute():
+        candidate = root / candidate
+    return ensure_under_root(candidate.resolve(), root=root)
+
+
+def repo_relative(path: Path, root: Path | None = None) -> str:
+    root = root or current_paths().root
+    try:
+        return path.resolve().relative_to(root).as_posix()
+    except ValueError:
+        return path.as_posix()

kc/provenance/__init__.py

@@ -0,0 +1 @@
+"""Provenance and citation validation."""

kc/provenance/citations.py

@@ -0,0 +1,296 @@
+"""Citation token parsing and validation."""
+
+from __future__ import annotations
+
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import unquote
+
+from kc.fingerprints import raw_fingerprint
+from kc.ids import new_id
+from kc.models.citation import ArtifactLocator, CitationEdgeRecord, ParsedCitation
+from kc.models.source import SourceRecord
+from kc.models.source_range import SourceRangeRecord
+from kc.paths import resolve_repo_path
+from kc.store.jsonl import read_jsonl
+
+CITATION_RE = re.compile(
+    r"\[kc:(?P<source>src_[A-Za-z0-9_]+):"
+    r"(?:(?P<range>rng_[A-Za-z0-9_]+)(?::"
+    r"(?:(?:L(?P<v2_line_start>\d+)-L(?P<v2_line_end>\d+))|"
+    r"(?:JP:(?P<v2_pointer>[^\]]+))|"
+    r"(?:CSV:R(?P<v2_row_start>\d+)-R(?P<v2_row_end>\d+))))?|"
+    r"(?:(?:L(?P<line_start>\d+)-L(?P<line_end>\d+))|"
+    r"(?:JP:(?P<pointer>[^\]]+))|"
+    r"(?:CSV:R(?P<row_start>\d+)-R(?P<row_end>\d+))))\]"
+)
+KC_TOKEN_RE = re.compile(r"\[kc:[^\]]+\]")
+MARKER_RE = re.compile(r"\[kc:(inference|todo|uncited)\]")
+
+
+def parse_markdown_citations(text: str) -> list[ParsedCitation]:
+    parsed: list[ParsedCitation] = []
+    for line_no, line in enumerate(text.splitlines(), start=1):
+        for match in CITATION_RE.finditer(line):
+            range_id = match.group("range")
+            token_version = "v2" if range_id else "v1"
+            if range_id and not any(
+                match.group(name) is not None
+                for name in (
+                    "v2_line_start",
+                    "v2_pointer",
+                    "v2_row_start",
+                )
+            ):
+                parsed.append(
+                    ParsedCitation(
+                        token=match.group(0),
+                        source_id=match.group("source"),
+                        range_id=range_id,
+                        token_version=token_version,
+                        kind="line_range",
+                        line=line_no,
+                    )
+                )
+                continue
+            if (match.group("v2_line_start") or match.group("line_start")) is not None:
+                parsed.append(
+                    ParsedCitation(
+                        token=match.group(0),
+                        source_id=match.group("source"),
+                        range_id=range_id,
+                        token_version=token_version,
+                        kind="line_range",
+                        start_line=int(match.group("v2_line_start") or match.group("line_start")),
+                        end_line=int(match.group("v2_line_end") or match.group("line_end")),
+                        line=line_no,
+                    )
+                )
+                continue
+            if (match.group("v2_pointer") or match.group("pointer")) is not None:
+                parsed.append(
+                    ParsedCitation(
+                        token=match.group(0),
+                        source_id=match.group("source"),
+                        range_id=range_id,
+                        token_version=token_version,
+                        kind="json_pointer",
+                        pointer=unquote(match.group("v2_pointer") or match.group("pointer")),
+                        line=line_no,
+                    )
+                )
+                continue
+            parsed.append(
+                ParsedCitation(
+                    token=match.group(0),
+                    source_id=match.group("source"),
+                    range_id=range_id,
+                    token_version=token_version,
+                    kind="csv_row_range",
+                    start_row=int(match.group("v2_row_start") or match.group("row_start")),
+                    end_row=int(match.group("v2_row_end") or match.group("row_end")),
+                    line=line_no,
+                )
+            )
+    return parsed
+
+
+def invalid_markdown_citation_tokens(text: str) -> list[dict[str, Any]]:
+    invalid: list[dict[str, Any]] = []
+    for line_no, line in enumerate(text.splitlines(), start=1):
+        for match in KC_TOKEN_RE.finditer(line):
+            token = match.group(0)
+            if MARKER_RE.fullmatch(token) or CITATION_RE.fullmatch(token):
+                continue
+            invalid.append(
+                {
+                    "code": "KC_CITATION_INVALID_TOKEN",
+                    "message": f"Invalid kc citation token: {token}",
+                    "line": line_no,
+                    "token": token,
+                }
+            )
+    return invalid
+
+
+def has_citation_or_marker(text: str) -> bool:
+    return bool(CITATION_RE.search(text) or MARKER_RE.search(text))
+
+
+def find_range_for_token(
+    citation: ParsedCitation,
+    ranges: list[SourceRangeRecord],
+) -> SourceRangeRecord | None:
+    if citation.range_id:
+        for candidate in ranges:
+            if candidate.source_id != citation.source_id or candidate.range_id != citation.range_id:
+                continue
+            if citation.start_line is None and citation.pointer is None and citation.start_row is None:
+                return candidate
+            loc = candidate.locator
+            if loc.kind != citation.kind:
+                return None
+            if citation.kind == "line_range" and loc.start_line == citation.start_line and loc.end_line == citation.end_line:
+                return candidate
+            if citation.kind == "json_pointer" and loc.pointer == citation.pointer:
+                return candidate
+            if citation.kind == "csv_row_range" and loc.start_row == citation.start_row and loc.end_row == citation.end_row:
+                return candidate
+        return None
+    for candidate in ranges:
+        loc = candidate.locator
+        if candidate.source_id != citation.source_id or loc.kind != citation.kind:
+            continue
+        if citation.kind == "line_range" and loc.start_line == citation.start_line and loc.end_line == citation.end_line:
+            return candidate
+        if citation.kind == "json_pointer" and loc.pointer == citation.pointer:
+            return candidate
+        if citation.kind == "csv_row_range" and loc.start_row == citation.start_row and loc.end_row == citation.end_row:
+            return candidate
+    return None
+
+
+def _previous_valid_edges(
+    citation_edges_path: Path | None,
+    artifact_path: str,
+) -> dict[str, CitationEdgeRecord]:
+    if citation_edges_path is None:
+        return {}
+    return {
+        edge.citation_token: edge
+        for edge in read_jsonl(citation_edges_path, CitationEdgeRecord)
+        if edge.artifact_path == artifact_path and edge.status == "valid"
+    }
+
+
+def _current_source_fingerprint(source: SourceRecord) -> str | None:
+    original = source.metadata.get("original_path")
+    if not isinstance(original, str):
+        return None
+    path = resolve_repo_path(original)
+    if not path.exists():
+        return None
+    return raw_fingerprint(path)
+
+
+def validate_citations(
+    artifact_path: str,
+    artifact_text: str,
+    *,
+    sources_path: Path,
+    ranges_path: Path,
+    citation_edges_path: Path | None = None,
+    artifact_id: str | None = None,
+) -> tuple[list[CitationEdgeRecord], list[dict[str, Any]]]:
+    sources = read_jsonl(sources_path, SourceRecord)
+    ranges = read_jsonl(ranges_path, SourceRangeRecord)
+    source_by_id = {s.source_id: s for s in sources}
+    parsed = parse_markdown_citations(artifact_text)
+    previous_edges = _previous_valid_edges(citation_edges_path, artifact_path)
+    edges: list[CitationEdgeRecord] = []
+    problems: list[dict[str, Any]] = invalid_markdown_citation_tokens(artifact_text)
+    timestamp = datetime.now(UTC).isoformat()
+    for problem in problems:
+        token = str(problem.get("token", ""))
+        source_match = re.search(r"\[kc:(src_[A-Za-z0-9_]+)", token)
+        edges.append(
+            CitationEdgeRecord(
+                edge_id=new_id("cite"),
+                artifact_id=artifact_id,
+                artifact_path=artifact_path,
+                artifact_locator=ArtifactLocator(
+                    start_line=int(problem.get("line", 1)),
+                    end_line=int(problem.get("line", 1)),
+                ),
+                citation_token=token,
+                source_id=source_match.group(1) if source_match else "",
+                range_id=None,
+                source_fingerprint_at_validation=None,
+                validated_at=timestamp,
+                status="invalid_token",
+            )
+        )
+    for citation in parsed:
+        source = source_by_id.get(citation.source_id)
+        range_record = find_range_for_token(citation, ranges)
+        status = "valid"
+        if source is None:
+            status = "missing_source"
+            problems.append(
+                {
+                    "code": "KC_CITATION_SOURCE_MISSING",
+                    "message": f"Citation source does not exist: {citation.source_id}",
+                    "line": citation.line,
+                    "token": citation.token,
+                }
+            )
+        elif range_record is None:
+            status = "missing_range"
+            problems.append(
+                {
+                    "code": "KC_CITATION_RANGE_MISSING",
+                    "message": f"Citation range does not exist: {citation.token}",
+                    "line": citation.line,
+                    "token": citation.token,
+                }
+            )
+        elif range_record.source_fingerprint != source.fingerprint:
+            status = "stale_source"
+            problems.append(
+                {
+                    "code": "KC_CITATION_STALE_SOURCE",
+                    "message": f"Citation points to stale source fingerprint: {citation.token}",
+                    "line": citation.line,
+                    "token": citation.token,
+                }
+            )
+        else:
+            previous_edge = previous_edges.get(citation.token)
+            if (
+                citation.token_version == "v1"
+                and previous_edge is not None
+                and previous_edge.range_id
+                and previous_edge.range_id != range_record.range_id
+            ):
+                status = "stale_source"
+                problems.append(
+                    {
+                        "code": "KC_CITATION_STALE_SOURCE",
+                        "message": f"Legacy locator citation now resolves to different source text: {citation.token}",
+                        "line": citation.line,
+                        "token": citation.token,
+                        "previous_range_id": previous_edge.range_id,
+                        "current_range_id": range_record.range_id,
+                    }
+                )
+            current_fingerprint = _current_source_fingerprint(source)
+            if current_fingerprint is not None and current_fingerprint != source.fingerprint:
+                status = "stale_source"
+                problems.append(
+                    {
+                        "code": "KC_CITATION_STALE_SOURCE",
+                        "message": f"Citation source file fingerprint has changed: {citation.token}",
+                        "line": citation.line,
+                        "token": citation.token,
+                        "source_id": source.source_id,
+                        "registered_fingerprint": source.fingerprint,
+                        "current_fingerprint": current_fingerprint,
+                    }
+                )
+        edges.append(
+            CitationEdgeRecord(
+                edge_id=new_id("cite"),
+                artifact_id=artifact_id,
+                artifact_path=artifact_path,
+                artifact_locator=ArtifactLocator(start_line=citation.line, end_line=citation.line),
+                citation_token=citation.token,
+                source_id=citation.source_id,
+                range_id=range_record.range_id if range_record else None,
+                source_fingerprint_at_validation=source.fingerprint if source else None,
+                validated_at=timestamp,
+                status=status,  # type: ignore[arg-type]
+            )
+        )
+    return edges, problems

kc/search/__init__.py

@@ -0,0 +1 @@
+"""Search and extraction primitives."""

kc/search/extract.py

@@ -0,0 +1,268 @@
+"""Deterministic local source extraction."""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+import json
+import mimetypes
+import tomllib
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from kc.fingerprints import normalize_text, text_hash
+from kc.models.source_range import Locator, SourceRangeRecord
+
+TEXT_EXTENSIONS = {
+    ".md",
+    ".markdown",
+    ".txt",
+    ".rst",
+    ".py",
+    ".js",
+    ".ts",
+    ".json",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".csv",
+}
+
+
+def guess_media_type(path: Path) -> str:
+    guessed, _encoding = mimetypes.guess_type(path.name)
+    if guessed:
+        return guessed
+    if path.suffix.lower() in TEXT_EXTENSIONS:
+        return "text/plain"
+    return "application/octet-stream"
+
+
+def is_text_like(path: Path, media_type: str) -> bool:
+    return media_type.startswith("text/") or path.suffix.lower() in TEXT_EXTENSIONS
+
+
+def extract_ranges(
+    path: Path,
+    source_id: str,
+    source_fingerprint: str,
+    *,
+    revision_id: str | None = None,
+) -> list[SourceRangeRecord]:
+    media_type = guess_media_type(path)
+    if path.suffix.lower() == ".json":
+        try:
+            return _extract_structured_ranges(
+                json.loads(path.read_text(encoding="utf-8-sig")),
+                source_id,
+                source_fingerprint,
+                revision_id=revision_id,
+            )
+        except Exception:
+            return _extract_text_ranges(path, source_id, source_fingerprint, revision_id=revision_id)
+    if path.suffix.lower() in {".yaml", ".yml"}:
+        try:
+            return _extract_structured_ranges(
+                yaml.safe_load(path.read_text(encoding="utf-8-sig")),
+                source_id,
+                source_fingerprint,
+                revision_id=revision_id,
+            )
+        except Exception:
+            return _extract_text_ranges(path, source_id, source_fingerprint, revision_id=revision_id)
+    if path.suffix.lower() == ".toml":
+        try:
+            return _extract_structured_ranges(
+                tomllib.loads(path.read_text(encoding="utf-8-sig")),
+                source_id,
+                source_fingerprint,
+                revision_id=revision_id,
+            )
+        except Exception:
+            return _extract_text_ranges(path, source_id, source_fingerprint, revision_id=revision_id)
+    if path.suffix.lower() == ".csv":
+        try:
+            return _extract_csv_ranges(path, source_id, source_fingerprint, revision_id=revision_id)
+        except Exception:
+            return _extract_text_ranges(path, source_id, source_fingerprint, revision_id=revision_id)
+    if not is_text_like(path, media_type):
+        return []
+    return _extract_text_ranges(path, source_id, source_fingerprint, revision_id=revision_id)
+
+
+def _now() -> str:
+    return datetime.now(UTC).isoformat()
+
+
+def _estimate_tokens(text: str) -> int:
+    return max(1, len(text.split()))
+
+
+def _range_id(source_id: str, revision_id: str | None, locator: Locator, excerpt_hash: str) -> str:
+    digest = hashlib.sha256(
+        f"{source_id}:{revision_id or ''}:{locator.model_dump_json(exclude_none=True)}:{excerpt_hash}".encode()
+    ).hexdigest()
+    return f"rng_{digest[:26].upper()}"
+
+
+def _extract_text_ranges(
+    path: Path, source_id: str, source_fingerprint: str, *, revision_id: str | None = None
+) -> list[SourceRangeRecord]:
+    text = normalize_text(path.read_text(encoding="utf-8-sig"))
+    lines = text.split("\n")
+    chunks: list[tuple[int, int, list[str]]] = []
+    current_start: int | None = None
+    current_lines: list[str] = []
+    heading_path: list[str] = []
+    current_heading: list[str] = []
+
+    def flush(end_line: int) -> None:
+        nonlocal current_start, current_lines, current_heading
+        if current_start is None:
+            return
+        content = "\n".join(current_lines).strip()
+        if content:
+            chunks.append((current_start, end_line, list(current_heading)))
+        current_start = None
+        current_lines = []
+
+    for idx, raw_line in enumerate(lines, start=1):
+        line = raw_line.rstrip()
+        if path.suffix.lower() in {".md", ".markdown"} and line.startswith("#"):
+            flush(idx - 1)
+            depth = len(line) - len(line.lstrip("#"))
+            title = line.lstrip("#").strip()
+            if title:
+                heading_path = [*heading_path[: max(depth - 1, 0)], title]
+            current_heading = list(heading_path)
+            current_start = idx
+            current_lines = [raw_line]
+            continue
+
+        if not line.strip():
+            flush(idx - 1)
+            continue
+
+        if current_start is None:
+            current_start = idx
+            current_heading = list(heading_path)
+            current_lines = []
+        current_lines.append(raw_line)
+
+        if len(current_lines) >= 24:
+            flush(idx)
+
+    flush(len(lines))
+
+    if not chunks and text.strip():
+        chunks = [(1, len(lines), [])]
+
+    extracted_at = _now()
+    records: list[SourceRangeRecord] = []
+    for start_line, end_line, headings in chunks:
+        excerpt = "\n".join(lines[start_line - 1 : end_line]).strip()
+        if not excerpt:
+            continue
+        locator = Locator(kind="line_range", start_line=start_line, end_line=end_line)
+        excerpt_hash = text_hash(excerpt)
+        records.append(
+            SourceRangeRecord(
+                range_id=_range_id(source_id, revision_id, locator, excerpt_hash),
+                source_id=source_id,
+                revision_id=revision_id,
+                source_fingerprint=source_fingerprint,
+                locator=locator,
+                text_hash=excerpt_hash,
+                excerpt=excerpt,
+                tokens_estimate=_estimate_tokens(excerpt),
+                extracted_at=extracted_at,
+                metadata={"heading_path": headings},
+            )
+        )
+    return records
+
+
+def _extract_structured_ranges(
+    data: Any,
+    source_id: str,
+    source_fingerprint: str,
+    *,
+    revision_id: str | None = None,
+) -> list[SourceRangeRecord]:
+    records: list[SourceRangeRecord] = []
+    extracted_at = _now()
+
+    def visit(value: Any, pointer: str) -> None:
+        if isinstance(value, dict):
+            for key, child in value.items():
+                escaped = str(key).replace("~", "~0").replace("/", "~1")
+                visit(child, f"{pointer}/{escaped}")
+        elif isinstance(value, list):
+            for idx, child in enumerate(value):
+                visit(child, f"{pointer}/{idx}")
+        else:
+            excerpt = json.dumps(value, ensure_ascii=False, sort_keys=True) if not isinstance(value, str) else value
+            excerpt = excerpt.strip()
+            if not excerpt:
+                return
+            locator = Locator(kind="json_pointer", pointer=pointer or "/")
+            excerpt_hash = text_hash(excerpt)
+            records.append(
+                SourceRangeRecord(
+                    range_id=_range_id(source_id, revision_id, locator, excerpt_hash),
+                    source_id=source_id,
+                    revision_id=revision_id,
+                    source_fingerprint=source_fingerprint,
+                    locator=locator,
+                    text_hash=excerpt_hash,
+                    excerpt=excerpt,
+                    tokens_estimate=_estimate_tokens(excerpt),
+                    extracted_at=extracted_at,
+                    metadata={"heading_path": [pointer or "/"]},
+                )
+            )
+
+    visit(data, "")
+    return records
+
+
+def _extract_csv_ranges(
+    path: Path,
+    source_id: str,
+    source_fingerprint: str,
+    *,
+    revision_id: str | None = None,
+) -> list[SourceRangeRecord]:
+    text = path.read_text(encoding="utf-8-sig")
+    rows = list(csv.reader(text.splitlines()))
+    if not rows:
+        return []
+    header = [cell.strip() for cell in rows[0]]
+    records: list[SourceRangeRecord] = []
+    extracted_at = _now()
+    for row_index, row in enumerate(rows[1:], start=2):
+        values = {
+            header[index] if index < len(header) and header[index] else f"column_{index + 1}": value
+            for index, value in enumerate(row)
+        }
+        excerpt = json.dumps(values, ensure_ascii=False, sort_keys=True)
+        locator = Locator(kind="csv_row_range", start_row=row_index, end_row=row_index)
+        excerpt_hash = text_hash(excerpt)
+        records.append(
+            SourceRangeRecord(
+                range_id=_range_id(source_id, revision_id, locator, excerpt_hash),
+                source_id=source_id,
+                revision_id=revision_id,
+                source_fingerprint=source_fingerprint,
+                locator=locator,
+                text_hash=excerpt_hash,
+                excerpt=excerpt,
+                tokens_estimate=_estimate_tokens(excerpt),
+                extracted_at=extracted_at,
+                metadata={"heading_path": ["csv", f"row {row_index}"]},
+            )
+        )
+    return records