wikidelta 0.1.0__py3-none-any.whl

wikidelta/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

wikidelta/cli.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+
+from wikidelta.repository import Repository
+from wikidelta.service import add_source, apply_review, extract_effective, status_items, update_document, write_review
+
+
+app = typer.Typer(no_args_is_help=True)
+
+
+@app.callback()
+def main() -> None:
+    """Manage WikiDelta .wd lifecycle files."""
+
+
+def emit(payload: dict, *, as_json: bool) -> None:
+    if as_json:
+        typer.echo(json.dumps(payload, ensure_ascii=False, indent=2))
+    else:
+        typer.echo(payload.get("message", json.dumps(payload, ensure_ascii=False)))
+
+
+@app.command()
+def init(
+    workspace: Path = typer.Option(Path.cwd(), "--workspace", help="Workspace root."),
+    mode: str = typer.Option("llmwiki_project", "--mode", help="Repository mode."),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+) -> None:
+    repo = Repository(workspace)
+    repo.ensure_layout(mode=mode)
+    emit(
+        {
+            "ok": True,
+            "workspace": str(repo.root),
+            "mode": mode,
+            "config": str(repo.config_path),
+            "message": f"Initialized WikiDelta workspace at {repo.root}",
+        },
+        as_json=json_output,
+    )
+
+
+@app.command()
+def add(
+    target: str = typer.Argument(..., help="Local file path or URL to wrap as a .wd source."),
+    into: Path | None = typer.Option(None, "--into", help="Directory where the .wd file is written."),
+    workspace: Path = typer.Option(Path.cwd(), "--workspace", help="Workspace root."),
+    title: str | None = typer.Option(None, "--title", help="Knowledge title."),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+) -> None:
+    wd_path = add_source(target, workspace=workspace, into=into, title=title)
+    emit(
+        {
+            "ok": True,
+            "path": str(wd_path),
+            "message": f"Created {wd_path}",
+        },
+        as_json=json_output,
+    )
+
+
+@app.command()
+def update(
+    path: Path | None = typer.Argument(None, help=".wd file to update. Omit to update all .wd files in the workspace."),
+    workspace: Path = typer.Option(Path.cwd(), "--workspace", help="Workspace root."),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+) -> None:
+    if path is None:
+        repo = Repository(workspace)
+        items = [update_document(wd_path, workspace=repo.root) for wd_path in repo.iter_wd_files()]
+        emit({"ok": True, "workspace": str(repo.root), "items": items}, as_json=json_output)
+        return
+    if path.suffix != ".wd":
+        payload = {
+            "ok": False,
+            "error": {
+                "code": "INVALID_UPDATE_TARGET",
+                "message": "wd update only accepts .wd files. Run wd update path/to/file.wd, not the original source file.",
+            },
+        }
+        emit(payload, as_json=json_output)
+        raise typer.Exit(1)
+    emit(update_document(path, workspace=workspace), as_json=json_output)
+
+
+@app.command()
+def status(
+    workspace: Path = typer.Option(Path.cwd(), "--workspace", help="Workspace root."),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+) -> None:
+    items = status_items(workspace=workspace)
+    payload = {"ok": True, "workspace": str(workspace), "items": items}
+    emit(payload, as_json=json_output)
+    if any(item.get("state") in {"pending_review", "effective_changed"} for item in items):
+        raise typer.Exit(3)
+
+
+@app.command()
+def review(
+    path: Path = typer.Argument(..., help=".wd file to review."),
+    workspace: Path = typer.Option(Path.cwd(), "--workspace", help="Workspace root."),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+) -> None:
+    emit({"ok": True, "review": write_review(path, workspace=workspace)}, as_json=json_output)
+
+
+@app.command("apply")
+def apply_cmd(
+    path: Path = typer.Argument(..., help=".wd file to apply."),
+    workspace: Path = typer.Option(Path.cwd(), "--workspace", help="Workspace root."),
+    strategy: str = typer.Option("replace", "--strategy", help="Apply strategy."),
+    yes: bool = typer.Option(False, "--yes", help="Confirm lifecycle mutation."),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+) -> None:
+    emit(apply_review(path, workspace=workspace, strategy=strategy, yes=yes), as_json=json_output)
+
+
+@app.command()
+def ingest(
+    path: Path = typer.Argument(..., help=".wd file to extract for llmwiki."),
+    workspace: Path = typer.Option(Path.cwd(), "--workspace", help="Workspace root."),
+    json_output: bool = typer.Option(False, "--json", help="Emit JSON output."),
+) -> None:
+    emit(extract_effective(path, workspace=workspace), as_json=json_output)
+
+
+if __name__ == "__main__":
+    app()

wikidelta/document.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+from typing import Any
+
+import yaml
+
+from wikidelta.models import WdMeta
+
+
+FRONT_MATTER_RE = re.compile(r"\A---\n(.*?)\n---\n?", re.DOTALL)
+SECTION_RE = re.compile(r"<!-- wd:([a-z_]+) -->\n?(.*?)\n?<!-- /wd:\1 -->", re.DOTALL)
+
+
+class WdParseError(ValueError):
+    pass
+
+
+def content_hash(content: str) -> str:
+    return "sha256:" + hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+@dataclass
+class WdDocument:
+    meta: WdMeta
+    sections: dict[str, str]
+
+    @classmethod
+    def parse(cls, text: str) -> "WdDocument":
+        match = FRONT_MATTER_RE.match(text)
+        if not match:
+            raise WdParseError("Missing YAML front matter")
+
+        raw_meta = yaml.safe_load(match.group(1)) or {}
+        meta = WdMeta.model_validate(raw_meta)
+        body = text[match.end() :]
+        sections = {name: content.strip("\n") for name, content in SECTION_RE.findall(body)}
+
+        if "effective" not in sections:
+            raise WdParseError("Missing wd:effective section")
+
+        return cls(meta=meta, sections=sections)
+
+    def section(self, name: str, default: str | None = None) -> str | None:
+        return self.sections.get(name, default)
+
+    def set_section(self, name: str, content: str) -> None:
+        self.sections[name] = content.strip("\n")
+
+    def remove_section(self, name: str) -> None:
+        self.sections.pop(name, None)
+
+    def to_text(self) -> str:
+        meta_dict = self.meta.model_dump(mode="json", exclude_none=True)
+        front_matter = yaml.safe_dump(meta_dict, sort_keys=False, allow_unicode=True).strip()
+        section_names = ["effective"]
+        if "source_snapshot" in self.sections:
+            section_names.append("source_snapshot")
+        if "notes" in self.sections:
+            section_names.append("notes")
+        section_names.extend(name for name in self.sections if name not in section_names)
+
+        body_parts = []
+        for name in section_names:
+            body_parts.append(f"<!-- wd:{name} -->\n{self.sections[name].strip()}\n<!-- /wd:{name} -->")
+        return f"---\n{front_matter}\n---\n\n" + "\n\n".join(body_parts) + "\n"
+
+
+def render_wd(*, wd_id: str, title: str, source: dict[str, Any], content: str) -> str:
+    body = content.strip("\n")
+    hash_value = content_hash(body)
+    meta = WdMeta.model_validate(
+        {
+            "wd_version": 1,
+            "id": wd_id,
+            "title": title,
+            "status": "active",
+            "content_type": "markdown",
+            "tags": [],
+            "source": source,
+            "sync": {
+                "strategy": "review_before_apply",
+                "state": "up_to_date",
+                "source_hash": hash_value,
+                "snapshot_hash": None,
+                "effective_hash": hash_value,
+                "last_ingested_at": None,
+            },
+        }
+    )
+    doc = WdDocument(
+        meta=meta,
+        sections={
+            "effective": body,
+            "notes": "",
+        },
+    )
+    return doc.to_text()

wikidelta/models.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class SourceConfig(BaseModel):
+    fetcher: str
+    fetch: dict[str, Any] = Field(default_factory=dict)
+    transformer: str
+    transform: dict[str, Any] = Field(default_factory=dict)
+
+
+class SyncState(BaseModel):
+    state: str = "up_to_date"
+    strategy: str = "review_before_apply"
+    last_refreshed_at: str | None = None
+    source_hash: str | None = None
+    snapshot_hash: str | None = None
+    effective_hash: str | None = None
+    last_ingested_at: str | None = None
+
+
+class WdMeta(BaseModel):
+    wd_version: int = 1
+    id: str
+    title: str
+    status: str = "active"
+    content_type: str = "markdown"
+    tags: list[str] = Field(default_factory=list)
+    source: SourceConfig
+    sync: SyncState = Field(default_factory=SyncState)

wikidelta/repository.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+
+STATE_DIRS = ("cache", "reviews", "errors", "ingest")
+
+
+def find_workspace(start: Path | str) -> Path | None:
+    current = Path(start).resolve()
+    if current.is_file():
+        current = current.parent
+    for candidate in (current, *current.parents):
+        if (candidate / ".wikidelta").is_dir():
+            return candidate
+    return None
+
+
+@dataclass(frozen=True)
+class Repository:
+    root: Path
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "root", Path(self.root).resolve())
+
+    @property
+    def state_dir(self) -> Path:
+        return self.root / ".wikidelta"
+
+    @property
+    def config_path(self) -> Path:
+        return self.state_dir / "config.yaml"
+
+    def ensure_layout(self, *, mode: str = "llmwiki_project") -> None:
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+        for dirname in STATE_DIRS:
+            (self.state_dir / dirname).mkdir(parents=True, exist_ok=True)
+        if not self.config_path.exists():
+            self.config_path.write_text(yaml.safe_dump({"mode": mode}, sort_keys=False), encoding="utf-8")
+
+    def iter_wd_files(self) -> list[Path]:
+        files: list[Path] = []
+        for path in self.root.rglob("*.wd"):
+            if ".wikidelta" in path.relative_to(self.root).parts:
+                continue
+            files.append(path)
+        return sorted(files)

wikidelta/service.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import re
+import json
+import difflib
+from pathlib import Path
+
+from wikidelta.document import WdDocument, content_hash, render_wd
+from wikidelta.repository import Repository
+from wikidelta.sources import fetch_source, infer_source_config, transform_content
+
+
+def slugify(value: str) -> str:
+    slug = re.sub(r"[^a-zA-Z0-9]+", "-", value).strip("-").lower()
+    return slug or "document"
+
+
+def add_source(target: str, *, workspace: Path, into: Path | None = None, title: str | None = None) -> Path:
+    repo = Repository(workspace)
+    repo.ensure_layout()
+    source_config = infer_source_config(target)
+    raw = fetch_source(source_config["fetcher"], source_config["fetch"], workspace=repo.root, wd_id="new")
+    transformed = transform_content(source_config["transformer"], source_config.get("transform", {}), raw)
+
+    parsed_target = Path(target)
+    stem = parsed_target.stem if parsed_target.suffix else slugify(target.rstrip("/"))
+    wd_id = slugify(stem)
+    output_dir = into if into is not None else repo.root / "raw_source"
+    if not output_dir.is_absolute():
+        output_dir = repo.root / output_dir
+    output_dir.mkdir(parents=True, exist_ok=True)
+    wd_path = output_dir / f"{wd_id}.wd"
+    wd_path.write_text(
+        render_wd(
+            wd_id=wd_id,
+            title=title or stem.replace("-", " ").replace("_", " ").title(),
+            source=source_config,
+            content=transformed.content,
+        ),
+        encoding="utf-8",
+    )
+    return wd_path
+
+
+def update_document(path: Path, *, workspace: Path) -> dict:
+    repo = Repository(workspace)
+    doc = WdDocument.parse(path.read_text(encoding="utf-8"))
+    source = doc.meta.source
+    raw = fetch_source(source.fetcher, source.fetch, workspace=repo.root, wd_id=doc.meta.id)
+    transformed = transform_content(source.transformer, source.transform, raw)
+    snapshot = transformed.content.strip("\n")
+    effective = doc.section("effective", default="").strip("\n")
+    doc.meta.sync.source_hash = content_hash(raw.content)
+    doc.meta.sync.effective_hash = content_hash(effective)
+    if snapshot == effective:
+        doc.remove_section("source_snapshot")
+        doc.meta.sync.state = "up_to_date"
+        doc.meta.sync.snapshot_hash = None
+    else:
+        doc.set_section("source_snapshot", snapshot)
+        doc.meta.sync.state = "pending_review"
+        doc.meta.sync.snapshot_hash = content_hash(snapshot)
+    path.write_text(doc.to_text(), encoding="utf-8")
+    return {"ok": True, "path": str(path), "id": doc.meta.id, "snapshotHash": doc.meta.sync.snapshot_hash}
+
+
+def status_items(*, workspace: Path) -> list[dict]:
+    repo = Repository(workspace)
+    items: list[dict] = []
+    for path in repo.iter_wd_files():
+        try:
+            doc = WdDocument.parse(path.read_text(encoding="utf-8"))
+            effective_hash = content_hash(doc.section("effective", default="").strip("\n"))
+            snapshot = doc.section("source_snapshot", default=None)
+            snapshot_hash = content_hash(snapshot.strip("\n")) if snapshot is not None else None
+            state = "pending_review" if snapshot_hash is not None and effective_hash != snapshot_hash else "clean"
+            items.append(
+                {
+                    "path": str(path.relative_to(repo.root)),
+                    "id": doc.meta.id,
+                    "state": state,
+                    "effectiveChanged": effective_hash != doc.meta.sync.effective_hash,
+                    "sourceChanged": snapshot_hash is not None and effective_hash != snapshot_hash,
+                    "lastError": None,
+                }
+            )
+        except Exception as exc:  # noqa: BLE001 - status must report invalid files.
+            items.append({"path": str(path.relative_to(repo.root)), "id": None, "state": "invalid", "lastError": str(exc)})
+    return items
+
+
+def write_review(path: Path, *, workspace: Path) -> dict:
+    repo = Repository(workspace)
+    repo.ensure_layout()
+    doc = WdDocument.parse(path.read_text(encoding="utf-8"))
+    effective = doc.section("effective", default="").strip("\n")
+    maybe_snapshot = doc.section("source_snapshot", default=None)
+    if maybe_snapshot is None:
+        return {
+            "wdId": doc.meta.id,
+            "path": str(path.relative_to(repo.root) if path.is_relative_to(repo.root) else path),
+            "state": "clean",
+            "effectiveHash": content_hash(effective),
+            "snapshotHash": None,
+            "summary": {"addedLines": 0, "removedLines": 0},
+            "actions": [{"name": "update", "command": f"wd update {path}"}],
+        }
+    snapshot = maybe_snapshot.strip("\n")
+    patch = "".join(
+        difflib.unified_diff(
+            effective.splitlines(keepends=True),
+            snapshot.splitlines(keepends=True),
+            fromfile="effective",
+            tofile="source_snapshot",
+        )
+    )
+    review = {
+        "wdId": doc.meta.id,
+        "path": str(path.relative_to(repo.root) if path.is_relative_to(repo.root) else path),
+        "state": "pending_review" if effective != snapshot else "clean",
+        "effectiveHash": content_hash(effective),
+        "snapshotHash": content_hash(snapshot),
+        "summary": {
+            "addedLines": sum(1 for line in patch.splitlines() if line.startswith("+") and not line.startswith("+++")),
+            "removedLines": sum(1 for line in patch.splitlines() if line.startswith("-") and not line.startswith("---")),
+        },
+        "actions": [{"name": "replace_effective", "command": f"wd apply {path} --strategy replace --yes"}],
+    }
+    (repo.state_dir / "reviews" / f"{doc.meta.id}.json").write_text(json.dumps(review, ensure_ascii=False, indent=2), encoding="utf-8")
+    (repo.state_dir / "reviews" / f"{doc.meta.id}.patch").write_text(patch, encoding="utf-8")
+    return review
+
+
+def apply_review(path: Path, *, workspace: Path, strategy: str, yes: bool) -> dict:
+    if strategy != "replace":
+        raise ValueError("Only replace strategy is supported")
+    if not yes:
+        raise ValueError("--yes is required to apply changes")
+    doc = WdDocument.parse(path.read_text(encoding="utf-8"))
+    maybe_snapshot = doc.section("source_snapshot", default=None)
+    if maybe_snapshot is None:
+        raise ValueError("No source_snapshot to apply")
+    snapshot = maybe_snapshot.strip("\n")
+    doc.set_section("effective", snapshot)
+    doc.remove_section("source_snapshot")
+    doc.meta.sync.state = "up_to_date"
+    doc.meta.sync.effective_hash = content_hash(snapshot)
+    doc.meta.sync.snapshot_hash = None
+    path.write_text(doc.to_text(), encoding="utf-8")
+    return {"ok": True, "path": str(path), "id": doc.meta.id, "strategy": strategy}
+
+
+def extract_effective(path: Path, *, workspace: Path) -> dict:
+    repo = Repository(workspace)
+    doc = WdDocument.parse(path.read_text(encoding="utf-8"))
+    relative_path = str(path.relative_to(repo.root) if path.is_relative_to(repo.root) else path)
+    return {
+        "ok": True,
+        "items": [
+            {
+                "external_id": f"wd:{doc.meta.id}",
+                "id": doc.meta.id,
+                "title": doc.meta.title,
+                "path": relative_path,
+                "content_type": doc.meta.content_type,
+                "tags": doc.meta.tags,
+                "effective_hash": content_hash(doc.section("effective", default="").strip("\n")),
+                "content": doc.section("effective", default="").strip("\n"),
+            }
+        ],
+    }

wikidelta/sources.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from urllib.parse import urlparse
+
+import requests
+from bs4 import BeautifulSoup
+
+
+@dataclass
+class Artifact:
+    content: str
+    content_type: str
+    metadata: dict = field(default_factory=dict)
+
+
+def infer_source_config(target: str) -> dict:
+    parsed = urlparse(target)
+    if parsed.scheme in {"http", "https"}:
+        return {
+            "fetcher": "builtin.http",
+            "fetch": {"url": target},
+            "transformer": "builtin.html_to_markdown",
+            "transform": {},
+        }
+
+    suffix = Path(target).suffix.lower()
+    transformer = {
+        ".md": "builtin.markdown",
+        ".markdown": "builtin.markdown",
+        ".txt": "builtin.text",
+        ".html": "builtin.text",
+        ".htm": "builtin.text",
+        ".json": "builtin.json_to_markdown",
+        ".pdf": "builtin.pdf_to_markdown",
+    }.get(suffix, "builtin.text")
+    return {
+        "fetcher": "builtin.file",
+        "fetch": {"path": target},
+        "transformer": transformer,
+        "transform": {},
+    }
+
+
+def fetch_source(fetcher: str, config: dict, *, workspace: Path, wd_id: str) -> Artifact:
+    if fetcher == "builtin.file":
+        path = Path(config["path"])
+        if not path.is_absolute():
+            path = workspace / path
+        content = path.read_text(encoding="utf-8")
+        return Artifact(content=content, content_type=_content_type_for_path(path), metadata={"path": str(path)})
+
+    if fetcher == "builtin.http":
+        response = requests.get(config["url"], timeout=float(config.get("timeout", 20)))
+        response.raise_for_status()
+        return Artifact(
+            content=response.text,
+            content_type=response.headers.get("content-type", "text/html").split(";")[0],
+            metadata={"url": config["url"]},
+        )
+
+    if fetcher == "script.python":
+        payload = {
+            "kind": "fetch",
+            "wdId": wd_id,
+            "workspace": str(workspace),
+            "config": config.get("args", {}),
+        }
+        result = _run_python_script(config, payload, workspace=workspace)
+        return Artifact(
+            content=result["content"],
+            content_type=result.get("contentType", "text/plain"),
+            metadata=result.get("metadata", {}),
+        )
+
+    raise ValueError(f"Unsupported fetcher: {fetcher}")
+
+
+def transform_content(transformer: str, config: dict, raw: Artifact) -> Artifact:
+    if transformer == "builtin.markdown":
+        return Artifact(content=raw.content.strip(), content_type="text/markdown", metadata=raw.metadata)
+    if transformer == "builtin.text":
+        return Artifact(content=raw.content.strip(), content_type="text/markdown", metadata=raw.metadata)
+    if transformer == "builtin.html_to_markdown":
+        html = raw.content
+        selector = config.get("selector")
+        if selector:
+            soup = BeautifulSoup(html, "html.parser")
+            selected = soup.select_one(selector)
+            html = str(selected) if selected else ""
+        soup = BeautifulSoup(html, "html.parser")
+        for tag in soup(["script", "style"]):
+            tag.decompose()
+        return Artifact(content=soup.get_text("\n", strip=True), content_type="text/markdown", metadata=raw.metadata)
+    if transformer == "builtin.json_to_markdown":
+        parsed = json.loads(raw.content)
+        pretty = json.dumps(parsed, ensure_ascii=False, indent=2)
+        return Artifact(content=f"```json\n{pretty}\n```", content_type="text/markdown", metadata=raw.metadata)
+    if transformer == "builtin.pdf_to_markdown":
+        return Artifact(content=raw.content.strip(), content_type="text/markdown", metadata=raw.metadata)
+    if transformer == "script.python":
+        payload = {
+            "kind": "transform",
+            "wdId": config.get("wd_id", ""),
+            "workspace": config.get("workspace", ""),
+            "contentType": raw.content_type,
+            "content": raw.content,
+            "metadata": raw.metadata,
+            "config": config.get("args", {}),
+        }
+        workspace = Path(config.get("workspace") or ".").resolve()
+        result = _run_python_script(config, payload, workspace=workspace)
+        return Artifact(
+            content=result["content"],
+            content_type=result.get("contentType", "text/markdown"),
+            metadata=result.get("metadata", {}),
+        )
+    raise ValueError(f"Unsupported transformer: {transformer}")
+
+
+def _run_python_script(config: dict, payload: dict, *, workspace: Path) -> dict:
+    entry = Path(config["entry"])
+    if not entry.is_absolute():
+        entry = workspace / entry
+    completed = subprocess.run(
+        [sys.executable, str(entry)],
+        input=json.dumps(payload),
+        text=True,
+        capture_output=True,
+        timeout=float(config.get("timeout", 30)),
+        cwd=str(workspace),
+        check=False,
+    )
+    if completed.returncode != 0:
+        raise RuntimeError(completed.stderr.strip() or f"script exited with {completed.returncode}")
+    try:
+        result = json.loads(completed.stdout)
+    except json.JSONDecodeError as exc:
+        raise RuntimeError("script did not return JSON") from exc
+    if not result.get("ok"):
+        error = result.get("error", {})
+        raise RuntimeError(f"{error.get('code', 'SCRIPT_FAILED')}: {error.get('message', 'script failed')}")
+    return result
+
+
+def _content_type_for_path(path: Path) -> str:
+    return {
+        ".md": "text/markdown",
+        ".markdown": "text/markdown",
+        ".txt": "text/plain",
+        ".html": "text/html",
+        ".htm": "text/html",
+        ".json": "application/json",
+        ".pdf": "application/pdf",
+    }.get(path.suffix.lower(), "text/plain")

wikidelta-0.1.0.dist-info/METADATA

@@ -0,0 +1,375 @@
+Metadata-Version: 2.4
+Name: wikidelta
+Version: 0.1.0
+Summary: Lifecycle CLI for .wd llmwiki source files
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/jcb850/WikiDelta
+Project-URL: Repository, https://github.com/jcb850/WikiDelta
+Project-URL: Issues, https://github.com/jcb850/WikiDelta/issues
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pypdf>=4.2
+Requires-Dist: PyYAML>=6.0
+Requires-Dist: requests>=2.32
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest>=8.2; extra == "dev"
+Dynamic: license-file
+
+<p align="center">
+  <img src="./assets/wikidelta-icon.png" alt="WikiDelta icon" width="160">
+</p>
+
+# WikiDelta
+
+[中文说明](./README.zh-CN.md) | English
+
+WikiDelta is built for the llmwiki raw source layer. It introduces the `.wd` file format so raw sources can be managed as reviewable, agent-friendly knowledge units instead of loose files that are refreshed and ingested directly.
+
+A `.wd` file keeps the content llmwiki should ingest, the source configuration used to refresh it, and an optional candidate snapshot when upstream source content changes. This gives llmwiki raw source maintenance a clear lifecycle: fetch, compare, review, apply, and ingest.
+
+The first version follows one clear constraint:
+
+```text
+1 .wd file = 1 knowledge unit = 1 source = 1 effective content section
+```
+
+## Use Cases
+
+- The llmwiki raw source directory needs a durable source-file format that can be maintained continuously by people and agents.
+- Raw sources may originate from Markdown, text, HTML, JSON, PDF, local files, or web pages, but llmwiki should ingest only reviewed effective content.
+- You want an explicit "currently effective content" layer so source refreshes cannot directly pollute the knowledge base.
+- Agents need stable CLI commands and JSON output to inspect source state, generate diffs, and apply reviewed updates.
+
+## `.wd` File Structure
+
+A `.wd` file is essentially a Markdown file composed of YAML front matter and named content sections:
+
+```markdown
+---
+wd_version: 1
+id: pricing-policy
+title: Pricing Rules
+status: active
+content_type: markdown
+tags:
+  - pricing
+  - policy
+source:
+  fetcher: builtin.file
+  fetch:
+    path: ./pricing.md
+  transformer: builtin.markdown
+  transform: {}
+sync:
+  strategy: review_before_apply
+---
+
+<!-- wd:effective -->
+# Pricing Rules
+
+This is the currently effective content and will be imported into llmwiki.
+<!-- /wd:effective -->
+
+<!-- wd:notes -->
+Maintenance notes, review decisions, and why certain source changes were accepted or rejected.
+<!-- /wd:notes -->
+```
+
+When `wd update` finds source content that differs from `wd:effective`, WikiDelta adds a temporary candidate section:
+
+```markdown
+<!-- wd:source_snapshot -->
+# Pricing Rules
+
+This is the latest candidate content fetched and transformed from the source.
+<!-- /wd:source_snapshot -->
+```
+
+Core rules:
+
+- `wd:effective` is the only content imported into the knowledge base by default.
+- `wd:source_snapshot` is optional candidate content for review and diffing, and is not imported directly.
+- `wd:notes` contains maintenance notes and is not imported into llmwiki by default.
+- `id` is the stable identity of the knowledge unit, used for status, caching, review, and future upsert operations.
+- The first `wd add` writes only `effective`; later `wd update` creates `source_snapshot` only when source content differs.
+
+## Installation and Running
+
+This project is currently a Python CLI package. In a development environment, you can run it directly with `PYTHONPATH`:
+
+```bash
+PYTHONPATH=src python3 -m wikidelta.cli --help
+```
+
+Install it as a local executable command:
+
+```bash
+pip install -e .
+wd --help
+```
+
+## Quick Start
+
+Initialize a workspace:
+
+```bash
+wd init --mode llmwiki_project
+```
+
+Wrap a local Markdown file into a `.wd` file:
+
+```bash
+wd add ./policy.md --into raw_sources/policy
+```
+
+Refresh candidate snapshots after source files change. Without a path, WikiDelta updates every `.wd` file in the workspace:
+
+```bash
+wd update
+```
+
+Pass a path when you only want to update one `.wd` file:
+
+```bash
+wd update raw_sources/policy/policy.wd
+```
+
+Check status:
+
+```bash
+wd status --json
+```
+
+Generate review materials:
+
+```bash
+wd review raw_sources/policy/policy.wd --json
+```
+
+Accept the candidate content as the effective content:
+
+```bash
+wd apply raw_sources/policy/policy.wd --strategy replace --yes
+```
+
+Extract only `wd:effective` as an llmwiki-compatible bridge:
+
+```bash
+wd ingest raw_sources/policy/policy.wd --json
+```
+
+## Daily Workflow Example
+
+This is the workflow for maintaining an llmwiki raw source directory with `.wd` files.
+
+Start in the knowledge project directory:
+
+```bash
+cd /path/to/llmwiki-project
+wd init --mode llmwiki_project
+```
+
+Add source files. If `--into` is omitted, WikiDelta writes `.wd` files under `raw_source/` by default:
+
+```bash
+wd add ./policy.md
+wd add ./dashboard.html
+```
+
+Local HTML files are preserved as raw source text. That means the generated `.wd` keeps the original `<!doctype html>`, `<style>`, and other HTML text in `wd:effective`. Web URLs still use `builtin.html_to_markdown` by default.
+
+After source files change, update all `.wd` files:
+
+```bash
+wd update --json
+```
+
+If you only want to update one knowledge unit, pass the `.wd` path:
+
+```bash
+wd update raw_source/dashboard.wd --json
+```
+
+`wd update` intentionally accepts `.wd` files only. Do not pass the original source file:
+
+```bash
+# Wrong: this is the original source file
+wd update ./dashboard.html
+
+# Right: this is the lifecycle wrapper
+wd update raw_source/dashboard.wd
+```
+
+Check what needs review:
+
+```bash
+wd status --json
+```
+
+For every item with `state: pending_review`, generate review files:
+
+```bash
+wd review raw_source/dashboard.wd --json
+less .wikidelta/reviews/dashboard.patch
+cat .wikidelta/reviews/dashboard.json
+```
+
+If the candidate snapshot should become the new effective content:
+
+```bash
+wd apply raw_source/dashboard.wd --strategy replace --yes --json
+```
+
+After apply, confirm the workspace is clean:
+
+```bash
+wd status --json
+```
+
+Finally, inspect the content llmwiki should ingest:
+
+```bash
+wd ingest raw_source/dashboard.wd --json
+```
+
+`wd ingest` outputs only `wd:effective`; it does not include source config, `source_snapshot`, or `notes`.
+
+## CLI Commands
+
+```text
+wd init      Initialize the .wikidelta state directory
+wd add       Create a .wd file from a local file or URL
+wd update    Refresh all .wd files, or one .wd file when a path is provided
+wd status    Scan .wd status
+wd review    Generate review JSON and patch files
+wd apply     Apply candidate content to effective
+wd ingest    Extract effective as an llmwiki-compatible bridge
+```
+
+When `wd status` succeeds but pending review content exists, it returns exit code `3`. This is not an execution failure; it tells agents or scripts that pending review needs to be handled.
+
+## Built-in Sources and Transforms
+
+`wd add` automatically infers common source pipelines from the input:
+
+```text
+*.md        builtin.file + builtin.markdown
+*.txt       builtin.file + builtin.text
+*.html      builtin.file + builtin.text
+*.json      builtin.file + builtin.json_to_markdown
+*.pdf       builtin.file + builtin.pdf_to_markdown
+http(s)://  builtin.http + builtin.html_to_markdown
+```
+
+You can also maintain the source configuration manually in a `.wd` file:
+
+```yaml
+source:
+  fetcher: builtin.http
+  fetch:
+    url: https://example.com/policy
+  transformer: builtin.html_to_markdown
+  transform:
+    selector: main
+```
+
+## llmwiki Project Mode
+
+The recommended layout is to place `.wd` files directly inside the raw source file structure of an llmwiki project:
+
+```text
+llmwiki-project/
+  raw_sources/
+    pricing/pricing-policy.wd
+    policy/refund-policy.wd
+```
+
+In this mode, the project a `.wd` file belongs to is determined by directory context. You do not need to configure llmwiki `base_url` or `project` for every `.wd` file.
+
+The import rule is: llmwiki or a compatible bridge may only read `wd:effective`; it must not import the entire `.wd` file as ordinary Markdown, otherwise the source configuration, candidate snapshot, and notes will pollute the knowledge base.
+
+## Agent-Friendly Conventions
+
+Main commands support JSON output and an explicit workspace:
+
+```bash
+wd status --workspace /path/to/repo --json
+```
+
+Lifecycle operations that affect `effective` require explicit confirmation:
+
+```bash
+wd apply path/to/file.wd --strategy replace --yes
+```
+
+Recommended agent workflow:
+
+```text
+1. wd status --json
+2. Run wd review --json for files in pending_review
+3. Read .wikidelta/reviews/<id>.json and .patch
+4. Decide whether to accept the changes
+5. wd apply <file.wd> --strategy replace --yes
+6. wd ingest <file.wd> --json
+```
+
+## Script Extensions
+
+Complex sources can use `script.python`. The script reads JSON from stdin and writes JSON to stdout.
+
+`.wd` example:
+
+```yaml
+source:
+  fetcher: script.python
+  fetch:
+    entry: ./fetchers/feishu_doc.py
+    args:
+      doc_id: abc123
+  transformer: builtin.markdown
+  transform: {}
+```
+
+Successful output:
+
+```json
+{
+  "ok": true,
+  "contentType": "text/plain",
+  "content": "Fetched content",
+  "metadata": {}
+}
+```
+
+Failed output:
+
+```json
+{
+  "ok": false,
+  "error": {
+    "code": "AUTH_FAILED",
+    "message": "Missing token",
+    "retryable": false
+  }
+}
+```
+
+Credentials should be passed to scripts through environment variables and should not be written into `.wd` files.
+
+## Testing
+
+Run the full test suite:
+
+```bash
+pytest -v
+```
+
+Current tests cover `.wd` parsing, repository initialization, source inference, `wd add/update/status/review/apply/ingest`, the script protocol, and the end-to-end lifecycle.
+
+## License
+
+WikiDelta is licensed under the [Apache License 2.0](./LICENSE).

wikidelta-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+wikidelta/__init__.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
+wikidelta/cli.py,sha256=goScExr_VbBTEb4o-wwmknFXPEmPghRCVL3zTZNHTUI,4814
+wikidelta/document.py,sha256=PSKdoeCRJTACEkseITD8srDcRbEpFZKvCeB0-GOppOQ,3087
+wikidelta/models.py,sha256=tCawLDOASrwoxaNiIENly9vJcbgCw9eHxx0uDyPNKGY,848
+wikidelta/repository.py,sha256=mLTBWHFsDDsk6T2anD4-cYEz0K1p-acKUSWV0EcJYSg,1466
+wikidelta/service.py,sha256=3ueNx8PPmeaqxm6bVDdfSwjrjUZ4NVasCfbEJn3k4tY,7519
+wikidelta/sources.py,sha256=8gLhyb67IUt2Yy1u15rz8V4cjoZAhvORiSR3_R_4YRU,5821
+wikidelta-0.1.0.dist-info/licenses/LICENSE,sha256=wbnfEnXnafPbqwANHkV6LUsPKOtdpsd-SNw37rogLtc,11359
+wikidelta-0.1.0.dist-info/METADATA,sha256=mdiiQP3pCUkvJjiz76wO76EXrrqGXLKsJPJ-JPJXxl0,9885
+wikidelta-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+wikidelta-0.1.0.dist-info/entry_points.txt,sha256=TrVQBheCCPR0OGvMvLvYEeBGB7oQfpwfyKx5A-z099s,41
+wikidelta-0.1.0.dist-info/top_level.txt,sha256=glAayfAUZq3Kq_rdzFkuwOPN1mjD7y9S0MeNMAyazR8,10
+wikidelta-0.1.0.dist-info/RECORD,,

wikidelta-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

wikidelta-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+wd = wikidelta.cli:app

wikidelta-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

wikidelta-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+wikidelta