ragjson 0.1.0__py3-none-any.whl

ragjson/__init__.py

@@ -0,0 +1,4 @@
+"""ragjson - 遵循RAG-Storage-Spec的知识库构建工具"""
+
+__version__ = "0.1.0"
+RAGJSON_VERSION = "2.0"

ragjson/__main__.py

@@ -0,0 +1,5 @@
+"""Allow running as: python -m ragjson"""
+
+from ragjson.cli import main
+
+main()

ragjson/atomic_ops.py

@@ -0,0 +1,118 @@
+"""Atomic write operations and directory-level file locking.
+
+Follows spec Section 3: Atomic Write Protocol.
+"""
+
+import fcntl
+import os
+from pathlib import Path
+
+
+class DirectoryLock:
+    """Context manager for directory-level exclusive file locking.
+
+    Uses flock(2) on the .rag/tmp/ directory file descriptor.
+    Per spec Section 3.3: same-directory operations are serialized,
+    different directories are fully isolated (no global lock).
+    """
+
+    def __init__(self, rag_dir: str | Path):
+        self.rag_dir = Path(rag_dir)
+        self.tmp_dir = self.rag_dir / "tmp"
+        self._fd: int | None = None
+
+    def __enter__(self) -> "DirectoryLock":
+        # Open the tmp directory for locking
+        self._fd = os.open(str(self.tmp_dir), os.O_RDONLY)
+        fcntl.flock(self._fd, fcntl.LOCK_EX)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        if self._fd is not None:
+            fcntl.flock(self._fd, fcntl.LOCK_UN)
+            os.close(self._fd)
+            self._fd = None
+
+
+def atomic_write(rag_dir: str | Path, filename: str, content: str) -> Path:
+    """Atomically write content to a .ragjson file.
+
+    Follows spec Section 3.1:
+    1. Write to .rag/tmp/{filename}.tmp
+    2. fsync to flush
+    3. os.rename() to .rag/{filename} — atomic on same filesystem
+
+    Args:
+        rag_dir: Path to the .rag directory
+        filename: Target filename (e.g., "readme.md.ragjson")
+        content: JSON string content to write
+
+    Returns:
+        Path to the final written file
+    """
+    rag_dir = Path(rag_dir)
+    tmp_dir = rag_dir / "tmp"
+    tmp_path = tmp_dir / f"{filename}.tmp"
+    final_path = rag_dir / filename
+
+    # Ensure directories exist
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+
+    # Step 1: Write to temporary file
+    with open(tmp_path, "w", encoding="utf-8") as f:
+        f.write(content)
+        # Step 2: fsync to ensure data is on disk
+        f.flush()
+        os.fsync(f.fileno())
+
+    # Step 3: Atomic rename
+    os.rename(tmp_path, final_path)
+
+    return final_path
+
+
+def clean_tmp_files(rag_dir: str | Path) -> list[Path]:
+    """Remove residual .tmp files from .rag/tmp/ directory.
+
+    Per spec Section 3.2: crash recovery cleans up incomplete writes.
+
+    Args:
+        rag_dir: Path to the .rag directory
+
+    Returns:
+        List of paths that were cleaned
+    """
+    rag_dir = Path(rag_dir)
+    tmp_dir = rag_dir / "tmp"
+    cleaned = []
+
+    if not tmp_dir.exists():
+        return cleaned
+
+    for tmp_file in tmp_dir.glob("*.tmp"):
+        try:
+            tmp_file.unlink()
+            cleaned.append(tmp_file)
+        except OSError:
+            pass  # File may already be gone
+
+    return cleaned
+
+
+def ensure_rag_dirs(path: str | Path) -> tuple[Path, Path]:
+    """Ensure .rag/ and .rag/tmp/ directories exist.
+
+    Args:
+        path: Source directory to create .rag structure in
+
+    Returns:
+        Tuple of (rag_dir, tmp_dir) paths
+    """
+    path = Path(path)
+    rag_dir = path / ".rag"
+    tmp_dir = rag_dir / "tmp"
+
+    rag_dir.mkdir(parents=True, exist_ok=True)
+    tmp_dir.mkdir(parents=True, exist_ok=True)
+
+    return rag_dir, tmp_dir

ragjson/cli.py

@@ -0,0 +1,178 @@
+"""CLI entry point for the `rag` command.
+
+Uses argparse with subcommands per spec Section 6.
+"""
+
+import argparse
+import os
+import sys
+
+from ragjson import __version__
+
+
+def main() -> None:
+    """Main entry point for the rag CLI tool."""
+    parser = argparse.ArgumentParser(
+        prog="rag",
+        description="RAG 知识库构建工具 — 遵循 RAG-Storage-Spec 规范",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # --- rag init <path> ---
+    init_parser = subparsers.add_parser(
+        "init",
+        help="Initialize .rag/ directory structure at the given path",
+    )
+    init_parser.add_argument(
+        "path",
+        help="Directory path to initialize",
+    )
+
+    # --- rag status [path] [-r] ---
+    status_parser = subparsers.add_parser(
+        "status",
+        help="Show knowledge base status with MD5 comparison",
+    )
+    status_parser.add_argument(
+        "path",
+        nargs="?",
+        default=".",
+        help="Directory path to check (default: current directory)",
+    )
+    status_parser.add_argument(
+        "-r", "--recursive",
+        action="store_true",
+        help="Recursively scan subdirectories",
+    )
+
+    # --- rag update [path] [-r] [-j N] [--splitter TYPE] [--parser TYPE] ---
+    update_parser = subparsers.add_parser(
+        "update",
+        help="Incremental update: process files with changed MD5",
+    )
+    update_parser.add_argument(
+        "path",
+        nargs="?",
+        default=".",
+        help="Directory path to update (default: current directory)",
+    )
+    update_parser.add_argument(
+        "-r", "--recursive",
+        action="store_true",
+        help="Recursively process subdirectories",
+    )
+    update_parser.add_argument(
+        "-j", "--jobs",
+        type=int,
+        default=0,
+        help="Number of parallel jobs (default: CPU core count)",
+    )
+    update_parser.add_argument(
+        "--splitter",
+        choices=["recursive", "markdown"],
+        default="recursive",
+        help="Text splitter type (default: recursive)",
+    )
+    update_parser.add_argument(
+        "--parser",
+        choices=["markitdown", "mineru"],
+        default="markitdown",
+        help="Document parser type (default: markitdown)",
+    )
+
+    # --- rag clean [path] [-r] ---
+    clean_parser = subparsers.add_parser(
+        "clean",
+        help="Clean residual .tmp files from .rag/tmp/",
+    )
+    clean_parser.add_argument(
+        "path",
+        nargs="?",
+        default=".",
+        help="Directory path to clean (default: current directory)",
+    )
+    clean_parser.add_argument(
+        "-r", "--recursive",
+        action="store_true",
+        help="Recursively clean subdirectories",
+    )
+
+    # --- rag cleanup [path] [-r] ---
+    cleanup_parser = subparsers.add_parser(
+        "cleanup",
+        help="Physically delete .ragjson files with DELETED status",
+    )
+    cleanup_parser.add_argument(
+        "path",
+        nargs="?",
+        default=".",
+        help="Directory path to cleanup (default: current directory)",
+    )
+    cleanup_parser.add_argument(
+        "-r", "--recursive",
+        action="store_true",
+        help="Recursively process subdirectories",
+    )
+
+    # --- rag export <path> <output> ---
+    export_parser = subparsers.add_parser(
+        "export",
+        help="Export all .ragjson files into a tar.gz archive",
+    )
+    export_parser.add_argument(
+        "path",
+        help="Source directory path to export from",
+    )
+    export_parser.add_argument(
+        "output",
+        help="Output archive path (e.g., export.tar.gz)",
+    )
+
+    # Parse and dispatch
+    args = parser.parse_args()
+
+    if args.command is None:
+        parser.print_help()
+        sys.exit(0)
+
+    # Dispatch to command implementations
+    from ragjson.commands import (
+        cmd_clean,
+        cmd_cleanup,
+        cmd_export,
+        cmd_init,
+        cmd_status,
+        cmd_update,
+    )
+
+    if args.command == "init":
+        cmd_init(args.path)
+    elif args.command == "status":
+        cmd_status(args.path, args.recursive)
+    elif args.command == "update":
+        cmd_update(
+            args.path,
+            recursive=args.recursive,
+            jobs=args.jobs,
+            splitter_type=args.splitter,
+            parser_type=args.parser,
+        )
+    elif args.command == "clean":
+        cmd_clean(args.path, args.recursive)
+    elif args.command == "cleanup":
+        cmd_cleanup(args.path, args.recursive)
+    elif args.command == "export":
+        cmd_export(args.path, args.output)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()