reindex-cli 0.3.0__py3-none-any.whl

reindex_cli/__init__.py

@@ -0,0 +1,6 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("reindex-cli")
+except PackageNotFoundError:  # Source-only execution without an installed distribution.
+    __version__ = "0.0.0+local"

reindex_cli/__main__.py

@@ -0,0 +1,3 @@
+from reindex_cli.cli import main
+
+raise SystemExit(main())

reindex_cli/api.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import httpx
+
+from reindex_cli.errors import ReIndexError
+
+
+class ApiClient:
+    def __init__(self, base_url: str, timeout: float = 1800.0) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.timeout = timeout
+
+    def health(self) -> dict:
+        with httpx.Client(base_url=self.base_url, timeout=30.0) as client:
+            return self._json(client.get("/health"))
+
+    def push(self, name: str, package: Path, sources: Path) -> dict:
+        with (
+            package.open("rb") as package_stream,
+            sources.open("rb") as source_stream,
+            httpx.Client(base_url=self.base_url, timeout=self.timeout) as client,
+        ):
+            response = client.post(
+                "/v1/push",
+                data={"name": name},
+                files={
+                    "package": ("package.zip", package_stream, "application/zip"),
+                    "sources": ("sources.zip", source_stream, "application/zip"),
+                },
+            )
+            return self._json(response)
+
+    def json(self, path: str, payload: dict) -> dict:
+        with httpx.Client(base_url=self.base_url, timeout=self.timeout) as client:
+            return self._json(client.post(path, json=payload))
+
+    def bytes(self, path: str, payload: dict) -> tuple[bytes, dict[str, str]]:
+        with httpx.Client(base_url=self.base_url, timeout=self.timeout) as client:
+            response = client.post(path, json=payload)
+            self._raise(response)
+            return response.content, dict(response.headers)
+
+    def _json(self, response: httpx.Response) -> dict:
+        self._raise(response)
+        try:
+            value = response.json()
+        except json.JSONDecodeError as error:
+            raise ReIndexError("Server returned invalid JSON") from error
+        if not isinstance(value, dict):
+            raise ReIndexError("Server returned an invalid response")
+        return value
+
+    @staticmethod
+    def _raise(response: httpx.Response) -> None:
+        if response.is_success:
+            return
+        try:
+            body = response.json()
+            detail = body.get("error", body)
+            message = detail.get("message") if isinstance(detail, dict) else None
+        except (json.JSONDecodeError, AttributeError):
+            message = None
+        raise ReIndexError(
+            f"Server request failed ({response.status_code}): "
+            f"{message or response.text or response.reason_phrase}"
+        )

reindex_cli/archives.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import tempfile
+import zipfile
+from pathlib import Path, PurePosixPath
+
+from reindex_cli.errors import ReIndexError
+from reindex_cli.package.cards import parse_card
+from reindex_cli.util import sha256_file
+
+
+def build_push_archives(context) -> tuple[Path, Path, tempfile.TemporaryDirectory]:
+    temporary = tempfile.TemporaryDirectory(prefix="rei-push-")
+    root = Path(temporary.name)
+    package_zip = root / "package.zip"
+    source_zip = root / "sources.zip"
+    with zipfile.ZipFile(package_zip, "w", zipfile.ZIP_DEFLATED) as bundle:
+        for file in sorted(context.output_dir.rglob("*")):
+            if file.is_file():
+                relative = Path(context.output_dir.name) / file.relative_to(
+                    context.output_dir
+                )
+                bundle.write(file, relative.as_posix())
+    references = raw_references(context.output_dir)
+    with zipfile.ZipFile(source_zip, "w", zipfile.ZIP_DEFLATED) as bundle:
+        for relative, expected in sorted(references.items()):
+            source = context.root / relative
+            if not source.is_file() or sha256_file(source) != expected:
+                temporary.cleanup()
+                raise ReIndexError(f"Missing or changed source: {relative}")
+            bundle.write(source, relative)
+    return package_zip, source_zip, temporary
+
+
+def raw_references(package: Path) -> dict[str, str]:
+    result: dict[str, str] = {}
+    for card_path in package.rglob("*.node.md"):
+        metadata, _body = parse_card(card_path)
+        values = [
+            metadata.get("source"),
+            metadata.get("content"),
+            *metadata.get("assets", []),
+        ]
+        for value in values:
+            if not isinstance(value, dict):
+                continue
+            uri = str(value.get("uri", ""))
+            if not uri.startswith("raw://"):
+                continue
+            relative = _safe(uri.removeprefix("raw://"))
+            digest = str(value.get("sha256", ""))
+            if relative in result and result[relative] != digest:
+                raise ReIndexError(f"Conflicting source hashes: {relative}")
+            result[relative] = digest
+    return result
+
+
+def extract_node_archive(content: bytes, target: Path) -> int:
+    target.mkdir(parents=True, exist_ok=True)
+    archive_path = target / ".download.zip"
+    archive_path.write_bytes(content)
+    count = 0
+    try:
+        with zipfile.ZipFile(archive_path) as bundle:
+            names: set[str] = set()
+            for item in bundle.infolist():
+                path = PurePosixPath(item.filename)
+                if (
+                    item.filename in names
+                    or path.is_absolute()
+                    or ".." in path.parts
+                    or not item.filename.endswith(".node.md")
+                    or (item.external_attr >> 16) & 0o170000 == 0o120000
+                ):
+                    raise ReIndexError(
+                        f"Unsafe or non-Node pull entry: {item.filename}"
+                    )
+                names.add(item.filename)
+                if item.is_dir():
+                    continue
+                destination = target.joinpath(*path.parts)
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                destination.write_bytes(bundle.read(item))
+                parse_card(destination)
+                count += 1
+    finally:
+        archive_path.unlink(missing_ok=True)
+    if not (target / "index.node.md").is_file():
+        raise ReIndexError("Pulled Node tree has no root index.node.md")
+    return count
+
+
+def _safe(value: str) -> str:
+    path = PurePosixPath(value)
+    if path.is_absolute() or ".." in path.parts or not path.parts:
+        raise ReIndexError(f"Unsafe raw path: {value}")
+    return path.as_posix()

reindex_cli/bundled_skills/reindex-create/SKILL.md

@@ -0,0 +1,12 @@
+---
+name: reindex-create
+description: Initialize a ReIndex project when the user asks to set up ReIndex or run rei init/create.
+---
+
+# ReIndex create
+
+1. Resolve the exact directory named by the user.
+2. Prefer `rei init <directory> --agent <current-agent>` for normal setup. It is idempotent and manages skills.
+3. Use `rei create <directory>` only when identity-only initialization was explicitly requested.
+4. Report the Collection name and whether it was created or reused. UUID is internal and normally omitted.
+5. Do not scan, push, delete source files, or replace `.rei/collection.json` during initialization.

reindex_cli/bundled_skills/reindex-data/SKILL.md

@@ -0,0 +1,22 @@
+---
+name: reindex-data
+description: Push, pull, search, and get exact ReIndex data when the user asks to publish or use remote ReIndex knowledge.
+---
+
+# ReIndex data
+
+## Publish
+
+1. Run `rei check <path>` before `rei push <path>`; scan first if stale.
+2. `rei push` synchronously sends the validated package and exactly referenced sources.
+3. Report the user-facing Collection name and ready status, not internal UUID details.
+
+## Find and fetch
+
+1. Use `rei search "<question>"` before downloading large files.
+2. Select the result whose Evidence supports the task.
+3. Use the result's Node path with `rei get <node-path> --target content`; use `source` only when original bytes are required.
+4. Use `rei get raw://<path>` for an explicitly named raw source.
+5. Answer from the fetched file and cite the Collection name and Node or raw path.
+
+`rei pull <name>` downloads the complete Node tree only. It intentionally excludes source, content and assets; fetch exact resources later with `rei get`.

reindex_cli/bundled_skills/reindex-scan/SKILL.md

@@ -0,0 +1,13 @@
+---
+name: reindex-scan
+description: Scan raw files into ReIndex when the user asks to scan, compile, ingest, or convert local data.
+---
+
+# ReIndex scan
+
+1. Run `rei inspect <path>` and review effective inputs, relationships and changes.
+2. Apply only evidence-backed manifest corrections; never delete raw files just to pass validation.
+3. Run `rei scan <path>` and review changes, warnings and generated Node cards.
+4. Edit Markdown card bodies only. The CLI owns YAML frontmatter.
+5. Run `rei check <path>` after manual card edits.
+6. Report the Collection name, Node count, warnings and package location. Passing checks is not human approval.

reindex_cli/cli.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import httpx
+
+from reindex_cli import __version__
+from reindex_cli.collection import create_collection, resolve_collection
+from reindex_cli.config import set_api_url
+from reindex_cli.errors import ReIndexError
+from reindex_cli.skills import AGENTS, manage_skills
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="rei", description="Build and use ReIndex knowledge Collections."
+    )
+    parser.add_argument("--version", action="version", version=__version__)
+    commands = parser.add_subparsers(dest="command", required=True)
+    init = commands.add_parser("init", help="Initialize ReIndex and Agent skills.")
+    init.add_argument("path", type=Path, nargs="?", default=Path.cwd())
+    init.add_argument("--name")
+    init.add_argument("--agent", choices=AGENTS, default="codex")
+    init.add_argument("--codex-home", type=Path)
+    create = commands.add_parser("create", help="Create only local identity state.")
+    create.add_argument("path", type=Path)
+    create.add_argument("--name")
+    rename = commands.add_parser("rename", help="Change the Collection name.")
+    rename.add_argument("path", type=Path)
+    rename.add_argument("name")
+    inspect = commands.add_parser("inspect", help="Inspect inputs without writing.")
+    inspect.add_argument("path", type=Path)
+    scan = commands.add_parser("scan", help="Compile a validated local package.")
+    scan.add_argument("path", type=Path)
+    scan.add_argument("--collection-root", type=Path)
+    check = commands.add_parser("check", help="Validate the current package.")
+    check.add_argument("path", type=Path)
+    skills = commands.add_parser("skills", help="Manage bundled Agent skills.")
+    skill_commands = skills.add_subparsers(dest="skill_command", required=True)
+    for name in ("install", "update"):
+        command = skill_commands.add_parser(name)
+        command.add_argument("--agent", choices=AGENTS, default="codex")
+        command.add_argument("--workspace-root", type=Path, default=Path.cwd())
+        command.add_argument("--codex-home", type=Path)
+        command.add_argument("--force", action="store_true")
+    api = commands.add_parser("set-api", help="Persist the default API URL.")
+    api.add_argument("url")
+    push = commands.add_parser(
+        "push", help="Synchronously publish package and sources."
+    )
+    push.add_argument("path", type=Path, nargs="?", default=Path.cwd())
+    push.add_argument("--api-url")
+    pull = commands.add_parser("pull", help="Download a Node-only ReIndex tree.")
+    pull.add_argument("name")
+    pull.add_argument("--output", type=Path)
+    pull.add_argument("--api-url")
+    pull.add_argument("--force", action="store_true")
+    search = commands.add_parser("search", help="Search a remote Collection.")
+    search.add_argument("query")
+    search.add_argument("--path", type=Path, default=Path.cwd())
+    search.add_argument("--remote")
+    search.add_argument("--api-url")
+    search.add_argument(
+        "--mode", choices=("lexical", "semantic", "hybrid"), default="lexical"
+    )
+    search.add_argument("--limit", type=int, default=10)
+    get = commands.add_parser("get", help="Reuse or fetch one exact resource.")
+    get.add_argument("reference")
+    get.add_argument("--path", type=Path, default=Path.cwd())
+    get.add_argument("--remote")
+    get.add_argument("--api-url")
+    get.add_argument("--target", choices=("card", "source", "content", "asset"))
+    get.add_argument("--asset-ordinal", type=int)
+    get.add_argument("--output", type=Path)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    try:
+        args = build_parser().parse_args(argv)
+        output = _execute(args)
+        print(json.dumps(output, ensure_ascii=False, indent=2))
+        return 0
+    except (ReIndexError, OSError, ValueError, httpx.HTTPError) as error:
+        print(
+            json.dumps({"status": "error", "error": str(error)}, ensure_ascii=False),
+            file=sys.stderr,
+        )
+        return 1
+
+
+def _execute(args) -> dict:
+    if args.command == "init":
+        root = args.path.expanduser().resolve()
+        collection = create_collection(root, args.name)
+        skills = manage_skills(
+            args.agent,
+            root,
+            update=True,
+            codex_home=args.codex_home,
+        )
+        return {
+            "status": "ready",
+            **collection,
+            "collection_id": collection["id"],
+            "skills": [vars(item) for item in skills],
+        }
+    if args.command in {"create", "rename"}:
+        name = args.name
+        result = create_collection(args.path.expanduser().resolve(), name)
+        return {
+            "status": "ready",
+            **result,
+            "collection_id": result["id"],
+            "renamed": args.command == "rename",
+        }
+    if args.command == "skills":
+        results = manage_skills(
+            args.agent,
+            args.workspace_root.expanduser().resolve(),
+            update=args.skill_command == "update",
+            force=args.force,
+            codex_home=args.codex_home,
+        )
+        return {"status": "ready", "skills": [vars(item) for item in results]}
+    if args.command == "set-api":
+        return {"status": "ready", "api_url": set_api_url(args.url)}
+    if args.command in {"push", "pull", "search"}:
+        from reindex_cli.remote_ops import (
+            pull_collection,
+            push_collection,
+            search_remote,
+        )
+
+    if args.command == "push":
+        return push_collection(args.path, args.api_url)
+    if args.command == "pull":
+        return pull_collection(
+            args.name,
+            args.output or Path.cwd() / args.name,
+            args.api_url,
+            force=args.force,
+        )
+    if args.command == "search":
+        return search_remote(
+            args.query, args.path, args.remote, args.api_url, args.mode, args.limit
+        )
+    if args.command == "get":
+        from reindex_cli.get_ops import get_resource
+
+        return get_resource(
+            args.reference,
+            args.path,
+            target=args.target,
+            asset_ordinal=args.asset_ordinal,
+            output=args.output,
+            remote=args.remote,
+            api_url=args.api_url,
+        )
+    from reindex_cli.pipeline.runner import (
+        check_collection,
+        inspect_collection,
+        run_scan,
+    )
+
+    context = resolve_collection(
+        args.path, args.collection_root if args.command == "scan" else None
+    )
+    if args.command == "inspect":
+        return inspect_collection(context)
+    if args.command == "scan":
+        return run_scan(context)
+    return check_collection(context)

reindex_cli/collection/__init__.py

@@ -0,0 +1,9 @@
+from .resolver import CollectionContext, resolve_collection
+from .state import create_collection, load_collection
+
+__all__ = [
+    "CollectionContext",
+    "create_collection",
+    "load_collection",
+    "resolve_collection",
+]

reindex_cli/collection/resolver.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from reindex_cli.collection.state import load_collection
+from reindex_cli.errors import ReIndexError
+
+
+@dataclass(frozen=True)
+class CollectionContext:
+    root: Path
+    scope: Path
+    scope_relative: str
+    collection_id: str
+    output_dir: Path
+    state: dict
+
+
+def resolve_collection(
+    path: Path, explicit_root: Path | None = None
+) -> CollectionContext:
+    requested = path.expanduser().resolve()
+    if not requested.exists():
+        raise ReIndexError(f"Scan path does not exist: {requested}")
+    root = (
+        explicit_root.expanduser().resolve() if explicit_root else _find_root(requested)
+    )
+    if root is None:
+        raise ReIndexError(f"No .rei/collection.json found above: {requested}")
+    if not root.is_dir():
+        raise ReIndexError(f"Collection root is not a directory: {root}")
+    try:
+        relative = requested.relative_to(root)
+    except ValueError as error:
+        raise ReIndexError(
+            f"Scan path is outside Collection root: {requested}"
+        ) from error
+    state = load_collection(root)
+    output = root / "reIndex" / state["output_dir"]
+    return CollectionContext(
+        root, requested, relative.as_posix(), state["id"], output, state
+    )
+
+
+def _find_root(requested: Path) -> Path | None:
+    start = requested if requested.is_dir() else requested.parent
+    for candidate in (start, *start.parents):
+        if (candidate / ".rei" / "collection.json").is_file():
+            return candidate
+    return None

reindex_cli/collection/state.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+from uuid import uuid4
+
+from reindex_cli.errors import ReIndexError
+from reindex_cli.util import atomic_json, load_json, slugify
+
+COLLECTION_SPEC = "reindex/collection@1.0"
+IDENTITY_FILE = "node-identities.json"
+
+
+def create_collection(directory: Path, name: str | None = None) -> dict:
+    root = directory.expanduser().resolve()
+    if not root.is_dir():
+        raise ReIndexError(f"Collection directory does not exist: {root}")
+    state_path = root / ".rei" / "collection.json"
+    if state_path.exists():
+        stored = load_json(state_path, {})
+        state = load_collection(root)
+        if name is not None:
+            state["name"] = collection_name(name)
+        if not isinstance(stored, dict) or stored.get("name") != state["name"]:
+            atomic_json(state_path, state)
+        return {**state, "created": False}
+    parent = _find_parent_collection(root.parent)
+    if parent is not None:
+        raise ReIndexError(f"Collection is nested inside existing Collection: {parent}")
+    collection_id = str(uuid4())
+    normalized_name = collection_name(name or root.name)
+    state = {
+        "spec": COLLECTION_SPEC,
+        "id": collection_id,
+        "name": normalized_name,
+        "created_at": datetime.now(UTC).isoformat(),
+        "output_dir": f"{collection_id}--{slugify(root.name, 'collection')}",
+    }
+    atomic_json(state_path, state)
+    atomic_json(
+        root / ".rei" / IDENTITY_FILE,
+        {"spec": "reindex/node-identities@1.0", "nodes": {}},
+    )
+    agent_path = root / ".rei" / "agent" / "collection.md"
+    agent_path.parent.mkdir(parents=True, exist_ok=True)
+    agent_path.write_text(
+        f"# {root.name}\n\nCollection ID: `{collection_id}`.\n\n"
+        "Review the root `reIndex.md` against real files before scanning.\n",
+        encoding="utf-8",
+        newline="\n",
+    )
+    return {**state, "created": True}
+
+
+def load_collection(root: Path) -> dict:
+    path = root / ".rei" / "collection.json"
+    try:
+        state = load_json(path, None)
+    except (OSError, ValueError) as error:
+        raise ReIndexError(f"Invalid Collection state: {path}") from error
+    if not isinstance(state, dict) or state.get("spec") != COLLECTION_SPEC:
+        raise ReIndexError(f"Unsupported Collection state: {path}")
+    if not all(
+        isinstance(state.get(key), str) and state[key] for key in ("id", "output_dir")
+    ):
+        raise ReIndexError(f"Incomplete Collection state: {path}")
+    state.setdefault("name", collection_name(root.name))
+    return state
+
+
+def collection_name(value: str) -> str:
+    normalized = slugify(value, "collection")[:80].rstrip("-")
+    if not normalized:
+        raise ReIndexError("Collection name must not be empty")
+    return normalized
+
+
+def identity_path(root: Path) -> Path:
+    current = root / ".rei" / IDENTITY_FILE
+    legacy = root / ".rei" / "identities.json"
+    return current if current.exists() or not legacy.exists() else legacy
+
+
+def _find_parent_collection(start: Path) -> Path | None:
+    for candidate in (start, *start.parents):
+        if (candidate / ".rei" / "collection.json").is_file():
+            return candidate
+    return None

reindex_cli/config.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+from reindex_cli.util import atomic_json
+
+DEFAULT_API_URL = "http://127.0.0.1:8000"
+
+
+def config_dir() -> Path:
+    if value := os.getenv("REINDEX_CONFIG_HOME"):
+        return Path(value).expanduser().resolve()
+    base = Path(os.getenv("XDG_CONFIG_HOME", Path.home() / ".config"))
+    return base.expanduser().resolve() / "reindex"
+
+
+def cache_dir() -> Path:
+    if value := os.getenv("REINDEX_CACHE_HOME"):
+        return Path(value).expanduser().resolve()
+    base = Path(os.getenv("XDG_CACHE_HOME", Path.home() / ".cache"))
+    return base.expanduser().resolve() / "reindex"
+
+
+def get_api_url(explicit: str | None = None) -> str:
+    if explicit:
+        return normalize_url(explicit)
+    if value := os.getenv("REINDEX_API_URL"):
+        return normalize_url(value)
+    path = config_dir() / "config.json"
+    if path.is_file():
+        value = json.loads(path.read_text(encoding="utf-8")).get("api_url")
+        if isinstance(value, str) and value.strip():
+            return normalize_url(value)
+    return DEFAULT_API_URL
+
+
+def set_api_url(value: str) -> str:
+    url = normalize_url(value)
+    atomic_json(config_dir() / "config.json", {"api_url": url})
+    return url
+
+
+def normalize_url(value: str) -> str:
+    result = value.strip().rstrip("/")
+    if not result.startswith(("http://", "https://")):
+        raise ValueError("API URL must start with http:// or https://")
+    return result

reindex_cli/errors.py

@@ -0,0 +1,10 @@
+class ReIndexError(RuntimeError):
+    """Expected CLI failure with a user-facing message."""
+
+
+class ManifestError(ReIndexError):
+    """The authoring manifest is invalid."""
+
+
+class PackageError(ReIndexError):
+    """The generated package violates the ReIndex protocol."""