clm-kernel 0.0.0__py3-none-any.whl

clm/layer4/loader.py

@@ -0,0 +1,307 @@
+"""
+File loading and storage operations for MCard.
+
+This module handles the high-level logic of processing files and storing them
+into a CardCollection. It uses mcard.file_io for low-level file operations.
+"""
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from clm.cross_cutting.config.settings import settings
+from clm.layer0.model.card import MCard
+from clm.layer0.model.handle import HandleValidationError
+from clm.layer2 import file_io
+from clm.layer2.card_collection import CardCollection
+
+logger = logging.getLogger(__name__)
+
+
+def process_and_store_file(
+    file_path: str | Path,
+    collection: CardCollection,
+    *,
+    root_path: Path | None = None,
+    allow_problematic: bool = False,
+    max_bytes_on_problem: int | None = None,
+    metadata_only: bool = False,
+) -> dict[str, Any] | None:
+    """Process a file, create an MCard, and store it in the collection."""
+    # Convert to Path object for consistency
+    file_path = Path(file_path) if isinstance(file_path, str) else file_path
+
+    try:
+        # Check if the file is problematic before processing
+        if file_io.is_problematic_file(file_path):
+            if not allow_problematic:
+                logger.warning(f"Skipping problematic file: {file_path}")
+                return None
+            # Resolve environment-configured parameters
+            if max_bytes_on_problem is None:
+                max_bytes_on_problem = settings.file_processing.max_problem_text_bytes
+
+            # Extensions known to naturally contain very long lines
+            known_long_exts = (
+                ".json",
+                ".jsonl",
+                ".csv",
+                ".xml",
+                ".min.js",
+                ".min.css",
+                ".svg",
+            )
+            is_known_type = file_path.name.lower().endswith(known_long_exts)
+            wrap_width = (
+                settings.file_processing.wrap_width_known
+                if is_known_type
+                else settings.file_processing.wrap_width_default
+            )
+            # Process as safe text via streaming normalization
+            logger.warning(
+                f"Problematic file detected, processing as safe text with soft-wrap (cap {max_bytes_on_problem} bytes, wrap {wrap_width}): {file_path}"
+            )
+            try:
+                streamed = file_io.stream_read_normalized_text(
+                    file_path, byte_cap=max_bytes_on_problem, wrap_width=wrap_width
+                )
+                text = streamed["text"]
+                file_info = {
+                    "content": text,
+                    "filename": Path(file_path).name,
+                    "mime_type": "text/plain",
+                    "extension": Path(file_path).suffix.lower(),
+                    "is_binary": False,
+                    "size": len(text),
+                    "original_size": streamed["original_size"],
+                    "original_sha256_prefix": streamed["original_sha256_prefix"],
+                    "normalized": True,
+                    "wrap_width": wrap_width,
+                }
+            except Exception as _e:
+                # Last-resort fallback: capped binary
+                logger.warning(
+                    f"Safe text processing failed, falling back to capped binary ({max_bytes_on_problem} bytes): {file_path}",
+                    exc_info=True,
+                )
+                file_info = file_io.process_file_content(
+                    file_path,
+                    force_binary=True,
+                    allow_pathological=True,
+                    max_bytes=max_bytes_on_problem,
+                )
+        else:
+            logger.info(f"PROCESSING FILE: {file_path}")
+            # Process the file
+            logger.info(f"Reading file content: {file_path}")
+            file_info = file_io.process_file_content(file_path)
+            if not file_info:
+                logger.warning(f"No file info returned for: {file_path}")
+                return None
+            logger.info(
+                f"File processed - Type: {file_info.get('mime_type')}, Size: {file_info.get('size')} bytes"
+            )
+
+        mcard = None
+        # Optionally skip storing content for problematic files if metadata_only is requested
+        if metadata_only and file_io.is_problematic_file(file_path):
+            pass
+        else:
+            # Check for empty content (e.g., empty __init__.py files)
+            content = file_info.get("content")
+            if not content or (isinstance(content, bytes | str) and len(content) == 0):
+                logger.debug(
+                    f"Skipping empty file: {file_path} (empty files cannot be stored as MCards)"
+                )
+                return {
+                    "hash": None,
+                    "content_type": file_info.get("mime_type"),
+                    "is_binary": file_info.get("is_binary"),
+                    "filename": file_info.get("filename"),
+                    "size": 0,
+                    "file_path": str(file_path),
+                    "skipped": True,
+                    "skip_reason": "empty_file",
+                }
+
+            # Create MCard with just the content bytes
+            try:
+                logger.info(f"Creating MCard for: {file_path}")
+                mcard = MCard(content=content)
+                logger.info(f"Created MCard with hash: {mcard.get_hash()}")
+            except ValueError as e:
+                if "empty" in str(e).lower():
+                    logger.debug(f"Skipping file with empty content: {file_path}")
+                    return None
+                logger.error(
+                    f"Failed to create MCard for {file_path}: {e}", exc_info=True
+                )
+                return None
+            except Exception as _e:
+                logger.error(f"Failed to create MCard for {file_path}", exc_info=True)
+                return None
+            # Add to collection and Handle Registration
+            try:
+                logger.info(f"Adding MCard to collection for: {file_path}")
+                added_hash = collection.add(mcard)
+
+                # --- Handle Registration Logic ---
+                # 1. Try filename handle
+                handle = file_path.name
+                try:
+                    collection.engine.register_handle(handle, added_hash)
+                    logger.info(f"Registered handle '{handle}' for {file_path.name}")
+                except HandleValidationError:
+                    # 1b. Filename is invalid as handle (e.g., starts with number)
+                    # Generate alternative handle by prepending 'doc_'
+                    alt_handle = f"doc_{handle}"
+                    try:
+                        collection.engine.register_handle(alt_handle, added_hash)
+                        logger.info(
+                            f"Registered alternative handle '{alt_handle}' for {file_path.name}"
+                        )
+                    except ValueError:
+                        # Alternative handle also taken - skip handle registration
+                        logger.debug(
+                            f"Could not register handle for '{file_path.name}'. "
+                            f"MCard stored with hash {added_hash[:8]}..."
+                        )
+                except ValueError:
+                    # 2. Name taken: Try relative path handle
+                    if root_path:
+                        try:
+                            rel_path = str(file_path.relative_to(root_path))
+                            collection.engine.register_handle(rel_path, added_hash)
+                            logger.info(
+                                f"Registered handle '{rel_path}' (fallback) for {file_path.name}"
+                            )
+                        except ValueError:
+                            # Both name and path already registered - content still stored by hash
+                            logger.debug(
+                                f"Handle name '{handle}' already in use (common for files like README.md, LICENSE). "
+                                f"MCard stored successfully with hash {added_hash[:8]}... (accessible by hash, not by handle)"
+                            )
+                    else:
+                        logger.debug(
+                            f"Handle name '{handle}' already in use. "
+                            f"MCard stored successfully with hash {added_hash[:8]}... (accessible by hash, not by handle)"
+                        )
+                # ---------------------------------
+
+                logger.info(f"Successfully added MCard to collection for: {file_path}")
+            except Exception as _e:
+                logger.error(
+                    f"Failed to add MCard to collection for {file_path}",
+                    exc_info=True,
+                )
+                return None
+
+        # Prepare and return processing info
+        result = {
+            "hash": mcard.get_hash() if mcard else None,
+            "content_type": file_info.get("mime_type"),
+            "is_binary": file_info.get("is_binary"),
+            "filename": file_info.get("filename"),
+            "size": file_info.get("size"),
+            "file_path": str(file_path),
+        }
+        # Surface original bytes metadata if present
+        if "original_size" in file_info:
+            result["original_size"] = file_info["original_size"]
+        if "original_sha256_prefix" in file_info:
+            result["original_sha256_prefix"] = file_info["original_sha256_prefix"]
+        if metadata_only and file_io.is_problematic_file(file_path):
+            result["metadata_only"] = True
+
+        logger.info(f"COMPLETED processing file: {file_path}")
+        return result
+
+    except (TimeoutError, ValueError) as e:
+        logger.warning(f"Skipping problematic file {file_path}: {e}")
+        return None
+    except Exception as _e:
+        logger.error(f"Error processing {file_path}", exc_info=True)
+        return None
+
+
+def load_file_to_collection(
+    path: str | Path,
+    collection: CardCollection,
+    recursive: bool = False,
+    include_problematic: bool = False,
+    max_bytes_on_problem: int = 2 * 1024 * 1024,
+    metadata_only: bool = False,
+) -> dict[str, Any]:
+    """
+    Load a file or directory of files into the specified collection.
+
+    Returns:
+        Dict containing 'metrics' and 'results'.
+    """
+    path = Path(path).resolve() if isinstance(path, str) else path.resolve()
+    results = []
+
+    # Identify root for relative path calculation
+    root_path = path if path.is_dir() else path.parent
+
+    file_paths = []
+    if path.is_file():
+        file_paths = [path]
+    elif path.is_dir():
+        file_paths = file_io.list_files(path, recursive=recursive)
+    else:
+        raise FileNotFoundError(f"Path '{path}' does not exist or is not accessible")
+
+    # Metrics Calculation
+    unique_dirs = set()
+    max_depth = 0
+
+    for fp in file_paths:
+        try:
+            rel = fp.relative_to(root_path)
+            unique_dirs.add(fp.parent)
+            # Depth is number of parts in relative path (including filename? No, just dirs usually)
+            # "levels of directories" -> usually depth of hierarchy.
+            # path parts: 'a/b/c.txt' -> ('a', 'b', 'c.txt') len=3. Depth=2 (folders).
+            # If at root: 'c.txt' -> ('c.txt') len=1. Depth=0.
+            current_depth = len(rel.parts) - 1
+            if current_depth > max_depth:
+                max_depth = current_depth
+        except ValueError:
+            pass
+
+    metrics = {
+        "files_count": len(file_paths),
+        "directories_count": (
+            len(unique_dirs) if path.is_dir() else 0
+        ),  # Only count subdirs inside? Or all involved?
+        # unique_dirs includes root's subdirs. If path is file, dirs is just parent (which is outside scope usually).
+        # Let's count dirs traversed.
+        "directory_levels": max_depth,
+    }
+
+    logger.info(f"About to process {len(file_paths)} files")
+    for i, file_path in enumerate(file_paths):
+        logger.info(f"Processing file {i + 1}/{len(file_paths)}: {file_path}")
+        result = process_and_store_file(
+            file_path,
+            collection,
+            root_path=root_path,
+            allow_problematic=include_problematic,
+            max_bytes_on_problem=max_bytes_on_problem,
+            metadata_only=metadata_only,
+        )
+        if result:
+            results.append(result)
+        logger.info(f"Completed file {i + 1}/{len(file_paths)}: {file_path}")
+
+    return {"metrics": metrics, "results": results}
+
+
+# Register file loader with Layer 1 protocols (Dependency Inversion)
+try:
+    from clm.layer1.protocols import register_file_loader
+
+    register_file_loader(load_file_to_collection)
+except ImportError:
+    pass

clm/layer4/mcard_commands.py

@@ -0,0 +1,166 @@
+"""CLI Subcommands for MCard Ingestion, Inspection, and Export (Sprint 392 / DEL-392-03).
+
+Provides user-facing command-line operations:
+- clm mcard ingest <path> [--handle <name>] [--author <did>]
+- clm mcard inspect <uri> [--json]
+- clm mcard export <uri> <output_path>
+
+Acceptance Criteria:
+- AC-392-03-01: clm mcard ingest exits 0 and prints canonical URI to stdout.
+- AC-392-03-02: clm mcard inspect displays formatted YAML/JSON metadata and payload snippet.
+- AC-392-03-03: clm mcard export round-trips binary/text content bit-for-bit.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from typing import Any
+
+import yaml
+
+from clm.cross_cutting.clm_logger import get_logger
+from clm.layer2.ingest import export_mcard, get_default_fs, ingest_file
+from clm.layer2.mcard_fs import MCardFileSystem
+
+_logger = get_logger("clm.layer4.mcard_commands")
+
+
+def mcard_ingest_cmd(
+    path: str,
+    handle: str | None = None,
+    author: str = "did:clm:user",
+    fs: MCardFileSystem | None = None,
+) -> int:
+    """Ingest local file into CAS and print canonical mcard:// URI."""
+    try:
+        mcard = ingest_file(path=path, handle=handle, author=author, fs=fs)
+        # Format canonical user URI
+        uri_str = mcard.uri
+        if uri_str.startswith("mcard:"):
+            rest = uri_str[len("mcard:"):].lstrip("/")
+            canonical_print = f"mcard://{rest}"
+        else:
+            canonical_print = uri_str
+        print(canonical_print)
+        return 0
+    except Exception as e:
+        _logger.error(f"Error ingesting MCard from '{path}': {e}")
+        return 1
+
+
+def mcard_inspect_cmd(
+    uri: str,
+    as_json: bool = False,
+    fs: MCardFileSystem | None = None,
+) -> int:
+    """Inspect metadata and payload snippet of content-addressed MCard."""
+    try:
+        target_fs = fs or get_default_fs()
+        clean_uri = uri
+        if clean_uri.startswith("mcard://"):
+            clean_uri = "mcard:" + clean_uri[len("mcard://"):]
+        mcard = target_fs.read_file(clean_uri)
+
+        # Prepare inspect report
+        summary: dict[str, Any] = {
+            "uri": uri,
+            "hash": str(mcard.hash),
+            "author": str(mcard.author),
+            "sequence": mcard.sequence,
+            "metadata": mcard.metadata,
+        }
+
+        # Payload snippet
+        if isinstance(mcard.payload, str):
+            if len(mcard.payload) > 500:
+                summary["payload_snippet"] = mcard.payload[:500] + "... [truncated]"
+            else:
+                summary["payload_snippet"] = mcard.payload
+        elif isinstance(mcard.payload, bytes):
+            summary["payload_snippet"] = f"<bytes length={len(mcard.payload)}>"
+        else:
+            summary["payload_snippet"] = mcard.payload
+
+        if as_json:
+            print(json.dumps(summary, indent=2, default=str))
+        else:
+            print(yaml.dump(summary, sort_keys=False))
+        return 0
+    except Exception as e:
+        _logger.error(f"Error inspecting MCard '{uri}': {e}")
+        return 1
+
+
+def mcard_export_cmd(
+    uri: str,
+    output_path: str,
+    fs: MCardFileSystem | None = None,
+) -> int:
+    """Export an MCard from CAS back to the local filesystem."""
+    try:
+        clean_uri = uri
+        if clean_uri.startswith("mcard://"):
+            clean_uri = "mcard:" + clean_uri[len("mcard://"):]
+        out_p = export_mcard(clean_uri, output_path, fs=fs)
+        print(f"Exported {uri} -> {out_p}")
+        return 0
+    except Exception as e:
+        _logger.error(f"Error exporting MCard '{uri}' to '{output_path}': {e}")
+        return 1
+
+
+def handle_mcard_cli(args: list[str], fs: MCardFileSystem | None = None) -> int:
+    """CLI subparser dispatcher for 'clm mcard <subcommand>'."""
+    parser = argparse.ArgumentParser(
+        prog="clm mcard",
+        description="Universal MCard Ingestion and Inspection CLI",
+    )
+    subparsers = parser.add_subparsers(dest="subcommand", help="MCard operation")
+
+    # ingest
+    ingest_p = subparsers.add_parser("ingest", help="Ingest a file into universal MCard CAS")
+    ingest_p.add_argument("path", help="Path to local file")
+    ingest_p.add_argument("--handle", "-H", default=None, help="Optional handle name")
+    ingest_p.add_argument("--author", "-a", default="did:clm:user", help="Author DID")
+
+    # inspect
+    inspect_p = subparsers.add_parser("inspect", help="Inspect MCard metadata and content")
+    inspect_p.add_argument("uri", help="Canonical MCard URI (mcard://...)")
+    inspect_p.add_argument("--json", action="store_true", help="Output as JSON instead of YAML")
+
+    # export
+    export_p = subparsers.add_parser("export", help="Export an MCard from CAS to disk")
+    export_p.add_argument("uri", help="Canonical MCard URI (mcard://...)")
+    export_p.add_argument("output_path", help="Local destination file path")
+
+    # diff (DEL-396-01)
+    diff_p = subparsers.add_parser("diff", help="Compute semantic diff between two MCards")
+    diff_p.add_argument("uri1", help="First MCard URI")
+    diff_p.add_argument("uri2", help="Second MCard URI")
+    diff_p.add_argument("--json", action="store_true", help="Output diff as JSON")
+
+    # lineage (DEL-396-01)
+    lineage_p = subparsers.add_parser("lineage", help="Display causal provenance lineage of an MCard")
+    lineage_p.add_argument("uri", help="Canonical MCard URI")
+    lineage_p.add_argument("--mermaid", action="store_true", help="Output as Mermaid flowchart markdown")
+    lineage_p.add_argument("--dot", action="store_true", help="Output as Graphviz DOT")
+    lineage_p.add_argument("--json", action="store_true", help="Output as JSON graph")
+
+    parsed = parser.parse_args(args)
+
+    if parsed.subcommand == "ingest":
+        return mcard_ingest_cmd(parsed.path, handle=parsed.handle, author=parsed.author, fs=fs)
+    elif parsed.subcommand == "inspect":
+        return mcard_inspect_cmd(parsed.uri, as_json=parsed.json, fs=fs)
+    elif parsed.subcommand == "export":
+        return mcard_export_cmd(parsed.uri, parsed.output_path, fs=fs)
+    elif parsed.subcommand == "diff":
+        from clm.layer4.diff_commands import mcard_diff_cmd
+        return mcard_diff_cmd(parsed.uri1, parsed.uri2, as_json=parsed.json, fs=fs)
+    elif parsed.subcommand == "lineage":
+        from clm.layer4.diff_commands import mcard_lineage_cmd
+        return mcard_lineage_cmd(parsed.uri, as_mermaid=parsed.mermaid, as_dot=parsed.dot, as_json=parsed.json)
+    else:
+        parser.print_help()
+        return 1