clm-kernel 0.0.0__py3-none-any.whl

clm/layer2/file_io.py

@@ -0,0 +1,376 @@
+"""
+File I/O operations for MCard.
+
+This module handles safe file reading, streaming, and directory traversal.
+It is designed to be robust against large files, binary content, and pathological text.
+"""
+
+import hashlib
+import logging
+import threading
+import time
+from pathlib import Path
+from typing import Any
+
+from clm.cross_cutting.config.settings import settings
+from clm.layer0.model.interpreter import ContentTypeInterpreter
+from clm.layer0.model.utils.content_analyzer import (
+    _PATHOLOGICAL_NO_BREAK_MIN as READ_CHUNK_SIZE,
+)
+from clm.layer0.model.utils.content_analyzer import (
+    _PROBLEMATIC_MAX_KNOWN_TYPE as KNOWN_TYPE_SIZE_LIMIT,
+)
+from clm.layer0.model.utils.content_analyzer import (
+    _PROBLEMATIC_MIN_MULTI_POSITION as MULTI_POSITION_THRESHOLD,
+)
+from clm.layer0.model.utils.content_analyzer import (
+    _PROBLEMATIC_SAMPLE_SIZE as BINARY_CHECK_SAMPLE_SIZE,
+)
+from clm.layer0.model.utils.content_analyzer import ContentAnalyzer
+
+# File I/O specific constants
+MAX_FILE_SIZE = settings.file_processing.max_file_size  # Configurable via environment
+LARGE_FILE_THRESHOLD = 50 * 1024  # 50KB — above this, read in chunks
+CONTENT_DETECTION_SAMPLE_SIZE = 1024 * 1024  # 1MB cap for content-type detection
+
+logger = logging.getLogger(__name__)
+
+
+def soft_wrap_long_lines(text: str, max_line_length: int = 1000) -> str:
+    """Insert newlines into very long single lines to prevent pathological processing.
+    This is a safe text normalization step for minified or long-line content.
+    """
+    if max_line_length <= 0:
+        return text
+    out_lines: list[str] = []
+    for line in text.splitlines() or [text]:
+        if len(line) <= max_line_length:
+            out_lines.append(line)
+            continue
+        # Chunk the long line
+        for i in range(0, len(line), max_line_length):
+            out_lines.append(line[i : i + max_line_length])
+    # Preserve trailing newline if present in input
+    result = "\n".join(out_lines)
+    return result
+
+
+def stream_read_normalized_text(
+    file_path: Path, *, byte_cap: int, wrap_width: int
+) -> dict[str, Any]:
+    """Stream-read bytes up to byte_cap, decode with replacement, and insert soft wraps on the fly.
+    Returns dict with keys: text, original_size, original_sha256_prefix.
+    """
+    sha = hashlib.sha256()
+    total_size = 0
+    produced_chars: list[str] = []
+    current_len = 0
+    with open(file_path, "rb") as f:
+        remaining = byte_cap
+        while remaining > 0:
+            chunk = f.read(min(READ_CHUNK_SIZE, remaining))
+            if not chunk:
+                break
+            sha.update(chunk)
+            total_size += len(chunk)
+            remaining -= len(chunk)
+            try:
+                s = chunk.decode("utf-8", errors="replace")
+            except UnicodeDecodeError:
+                s = chunk.decode("latin-1", errors="replace")
+            for ch in s:
+                if ch == "\r":
+                    # Normalize CR to nothing; let CRLF become just LF via the subsequent '\n'
+                    continue
+                produced_chars.append(ch)
+                if ch == "\n":
+                    current_len = 0
+                else:
+                    current_len += 1
+                    if wrap_width > 0 and current_len >= wrap_width:
+                        produced_chars.append("\n")
+                        current_len = 0
+    return {
+        "text": "".join(produced_chars),
+        "original_size": total_size,
+        "original_sha256_prefix": sha.hexdigest()[:16],
+    }
+
+
+def read_file_safely(
+    file_path: str | Path,
+    allow_pathological: bool = False,
+    max_bytes: int | None = None,
+) -> bytes:
+    """
+    Read file content with timeout protection and size limits.
+
+    Args:
+        file_path: Path to the file to read
+        allow_pathological: If True, bypass long-line/pathological content checks
+        max_bytes: If set, cap the number of bytes read to this value
+
+    Returns:
+        File content as bytes
+
+    Raises:
+        TimeoutError: If reading takes too long
+        IOError: If file cannot be read
+    """
+    file_path = Path(file_path)
+
+    # Check file size first
+    try:
+        file_size = file_path.stat().st_size
+    except OSError as e:
+        raise OSError(f"Cannot access file {file_path}: {e}") from e
+
+    if file_size > MAX_FILE_SIZE:
+        raise OSError(f"File too large: {file_size} bytes (max {MAX_FILE_SIZE})")
+
+    # Timeout configurable with a small bias for larger files
+    base_timeout = settings.file_processing.read_timeout_secs
+    timeout_seconds = (
+        5.0 if file_size > LARGE_FILE_THRESHOLD and base_timeout >= 5 else base_timeout
+    )
+
+    result = [None]
+    exception = [None]
+
+    def read_with_timeout():
+        try:
+            with open(file_path, "rb") as file:
+                # For potentially problematic files, read in chunks with progress checking
+                if (
+                    file_size > LARGE_FILE_THRESHOLD or max_bytes is not None
+                ):  # Files > 50KB or when capping
+                    content = bytearray()
+                    chunk_size = READ_CHUNK_SIZE
+                    bytes_read = 0
+                    start_time = time.time()
+
+                    to_read = max_bytes if max_bytes is not None else file_size
+
+                    while bytes_read < to_read:
+                        if time.time() - start_time > timeout_seconds:
+                            raise TimeoutError(
+                                f"Reading timeout exceeded for {file_path}"
+                            )
+
+                        remaining = to_read - bytes_read
+                        chunk = file.read(min(chunk_size, remaining))
+                        if not chunk:
+                            break
+                        content.extend(chunk)
+                        bytes_read += len(chunk)
+
+                        # Check for pathological content patterns during reading unless allowed
+                        if (
+                            (not allow_pathological)
+                            and len(content) > 32768
+                            and b"\n" not in content
+                            and b"\r" not in content
+                        ):
+                            raise OSError(
+                                f"File appears to be a single massive line: {file_path}"
+                            )
+
+                    result[0] = bytes(content)
+                else:
+                    result[0] = file.read()
+        except Exception as e:
+            exception[0] = e
+
+    # Use threading for timeout since signal doesn't work well on all platforms
+    thread = threading.Thread(target=read_with_timeout)
+    thread.daemon = True
+    thread.start()
+    thread.join(timeout=timeout_seconds)
+
+    if thread.is_alive():
+        # Thread is still running, reading is taking too long
+        logger.warning(f"File reading timed out after {timeout_seconds}s: {file_path}")
+        raise TimeoutError(f"File reading timed out for {file_path}")
+
+    if exception[0]:
+        logger.error(f"Error reading file {file_path}: {exception[0]}")
+        raise exception[0]
+
+    if result[0] is None:
+        raise OSError(f"Failed to read file: {file_path}")
+
+    return result[0]
+
+
+def is_problematic_file(file_path: Path) -> bool:
+    """
+    Check if a file is likely to cause processing issues.
+    """
+    try:
+        file_size = file_path.stat().st_size
+
+        # Skip empty files
+        if file_size == 0:
+            return False
+
+        # If it's a known type that can have long lines, be more permissive
+        file_extension = file_path.suffix.lower()
+        is_known_type = ContentAnalyzer.is_known_long_line_extension(file_extension)
+
+        # For known types, still have size limits to prevent pathological cases
+        if (
+            is_known_type and file_size > KNOWN_TYPE_SIZE_LIMIT
+        ):  # 1MB limit for known types
+            logger.warning(
+                f"Skipping large file of known type: {file_path} ({file_size} bytes)"
+            )
+            return True
+
+        with open(file_path, "rb") as f:
+            # Sample from beginning, middle, and end for large files
+            sample_size = min(BINARY_CHECK_SAMPLE_SIZE, file_size)  # 32KB sample
+            samples = []
+
+            if (
+                file_size > MULTI_POSITION_THRESHOLD
+            ):  # For files > 100KB, check multiple positions
+                positions = [0, file_size // 2, max(0, file_size - sample_size)]
+            else:
+                positions = [0]
+
+            for pos in positions:
+                f.seek(pos)
+                sample = f.read(sample_size)
+                if sample:
+                    samples.append(sample)
+
+            for sample in samples:
+                # Check for binary content that's not structured
+                if ContentAnalyzer.is_unstructured_binary(sample):
+                    logger.warning(f"Skipping unstructured binary file: {file_path}")
+                    return True
+
+                # Check for pathological line patterns
+                if ContentAnalyzer.has_pathological_lines(sample, is_known_type):
+                    logger.warning(
+                        f"Skipping file with pathological line structure: {file_path}"
+                    )
+                    return True
+
+        return False
+
+    except OSError as e:
+        logger.warning(f"Error checking file {file_path}: {e}")
+        return True  # Skip files we can't check
+
+
+def list_files(directory: str | Path, recursive: bool = False) -> list[Path]:
+    """
+    Load all files from the specified directory.
+    Explicitly skips hidden files and directories (starting with .).
+    """
+    import os
+
+    dir_path = Path(directory) if isinstance(directory, str) else directory
+    if not dir_path.exists():
+        raise FileNotFoundError(f"Directory '{dir_path}' does not exist.")
+
+    safe_files = []
+
+    if recursive:
+        for root, dirs, filenames in os.walk(str(dir_path)):
+            # Modify dirs in-place to skip hidden directories and common massive dependencies
+            dirs[:] = [
+                d
+                for d in dirs
+                if not d.startswith(".")
+                and d
+                not in {
+                    "node_modules",
+                    "__pycache__",
+                    "venv",
+                    "env",
+                    "build",
+                    "dist",
+                    "target",
+                    "mcard-studio-worktree",
+                }
+            ]
+
+            for filename in filenames:
+                if filename.startswith("."):
+                    continue
+
+                file_path = Path(root) / filename
+                if not is_problematic_file(file_path):
+                    safe_files.append(file_path)
+    else:
+        # Non-recursive: just list top level
+        # os.walk matches this if we break, but simpler usually:
+        for item in dir_path.iterdir():
+            if item.name.startswith("."):
+                continue
+            if item.is_file():
+                if not is_problematic_file(item):
+                    safe_files.append(item)
+
+    logger.info(f"Found {len(safe_files)} safe files in {dir_path}")
+    return safe_files
+
+
+def process_file_content(
+    file_path: str | Path,
+    *,
+    force_binary: bool = False,
+    allow_pathological: bool = False,
+    max_bytes: int | None = None,
+) -> dict[str, Any]:
+    """
+    Process a file and return its metadata and content.
+    """
+    # Read file content as bytes
+    content = read_file_safely(
+        file_path, allow_pathological=allow_pathological, max_bytes=max_bytes
+    )
+
+    # Analyze the content to get MIME type and other metadata
+    # For very large files or files with extremely long lines, use a sample
+    content_sample = content
+    if (
+        len(content) > CONTENT_DETECTION_SAMPLE_SIZE
+    ):  # 1MB limit for content type detection
+        content_sample = content[:CONTENT_DETECTION_SAMPLE_SIZE]
+        logger.info(f"Using content sample for large file: {file_path}")
+
+    mime_type, extension = ContentTypeInterpreter.detect_content_type(content_sample)
+    is_binary = ContentTypeInterpreter.is_binary_content(content_sample)
+
+    # Force binary mode if requested
+    if force_binary:
+        is_binary = True
+        # If detection thought it's text, override to a safe default
+        if not mime_type or mime_type.startswith("text/"):
+            mime_type = "application/octet-stream"
+
+    # For text files, try to decode the content
+    if not is_binary and mime_type.startswith("text/"):
+        try:
+            # Decode the content as UTF-8 for text files
+            content = content.decode("utf-8")
+        except UnicodeDecodeError:
+            # If UTF-8 fails, try with error replacement
+            try:
+                content = content.decode("utf-8", errors="replace")
+            except UnicodeDecodeError as e:
+                logger.warning(
+                    f"Failed to decode content for {file_path} as UTF-8: {e}"
+                )
+
+    return {
+        "content": content,
+        "filename": Path(file_path).name,
+        "mime_type": mime_type,
+        "extension": extension,
+        "is_binary": is_binary,
+        "size": len(content),  # Use original content size, not sample
+    }

clm/layer2/file_registrar.py

@@ -0,0 +1,77 @@
+"""CLM File Registrar — File-to-MCard registration in the Tri-Database.
+
+Registers CLM manifest files as MCards, including AST normalization,
+classification, and dialect metadata handles.
+"""
+
+import json
+import os
+
+from clm.cross_cutting.clm_logger import get_logger
+from clm.layer0.utils import clm_hash_string, resolve_clm_path
+from clm.layer1.verifier import get_load_manifest
+from clm.layer2.storage import CLMDB, CLMRepository
+
+from .classifier import classify_clm, detect_dialect
+
+_logger = get_logger("clm.layer2.file_registrar")
+
+
+def register_file_mcard_py(clm_root: str, clm_file: str) -> None:
+    """
+    Registers a CLM manifest file as an MCard in the Tri-Database.
+
+    Args:
+        clm_root (str): The root directory of the CLM workspace.
+        clm_file (str): The CLM file to register.
+    """
+    load_normalized_manifest_fn = get_load_manifest()
+    clm_file = resolve_clm_path(clm_root, clm_file)
+    db = CLMDB(clm_root)
+    repo = CLMRepository(db, clm_root)
+    rel_path = os.path.relpath(clm_file, clm_root)
+    rel_path = rel_path.replace('\\', '/')
+    handle_name = f"file:{rel_path}"
+
+    with open(clm_file, encoding="utf-8") as f:
+        content = f.read()
+    norm_content = content.replace("\r\n", "\n").replace("\r", "\n").rstrip("\n")
+    content_hash = clm_hash_string(norm_content)
+
+    repo.register_handle(handle_name, norm_content)
+
+    json_ast = ""
+    try:
+        parsed = load_normalized_manifest_fn(clm_root, clm_file)
+        json_ast = json.dumps(parsed)
+    except Exception as e:
+        _logger.warning(f"[register_file_mcard_py] Warning: manifest normalization failed: {e}")
+
+    if json_ast:
+        classification = "Number"
+        try:
+            parsed_data = json.loads(json_ast)
+            classification = classify_clm(parsed_data)
+        except Exception as e:
+            _logger.warning(f"[register_file_mcard_py] Warning: classification failed: {e}")
+
+        dialect = "UNKNOWN"
+        if classification == "Function":
+            try:
+                parsed_data = json.loads(json_ast)
+                dialect = detect_dialect(parsed_data, classification)
+            except Exception as e:
+                _logger.warning(f"[register_file_mcard_py] Warning: dialect detection failed: {e}")
+
+        for key, val in [("ast", json_ast), ("classification", classification), ("dialect", dialect)]:
+            h_name = f"{key}:{content_hash}"
+            repo.register_handle(h_name, val)
+
+
+# Register file registrar hook with Layer 1 protocols (Dependency Inversion)
+try:
+    from clm.layer1.protocols import register_file_registrar_hook
+
+    register_file_registrar_hook(register_file_mcard_py)
+except ImportError:
+    pass

clm/layer2/ingest.py

@@ -0,0 +1,202 @@
+"""Universal MCard Ingestion Membrane (Sprint 392 / DEL-392-02).
+
+Provides universal ingestion of local files and raw byte streams into the
+content-addressed MCard virtual filesystem (mcard://), with automatic MIME detection,
+structural metadata extraction, and Zero-FS in-memory storage support.
+
+Acceptance Criteria:
+- AC-392-02-01: Ingesting any supported file returns valid MCard with deterministic mcard://<blake3_hash>.
+- AC-392-02-02: Automatic MIME detection without relying solely on file extensions.
+- AC-392-02-03: Structural metadata stored in card metadata.
+- AC-392-02-04: Zero-FS support (in-memory SQLite storage when CLM_ZERO_FS=true).
+"""
+
+from __future__ import annotations
+
+import base64
+import os
+import sqlite3
+from pathlib import Path
+
+from clm.cross_cutting.domain_models import MCard
+from clm.layer2.mcard_fs import MCardFileSystem
+from clm.layer2.mime_detector import detect_mime_and_format, extract_structural_metadata
+
+_SHARED_MEMORY_CONN: sqlite3.Connection | None = None
+_DEFAULT_FS: MCardFileSystem | None = None
+
+
+def get_default_fs() -> MCardFileSystem:
+    """Obtain the default MCardFileSystem instance for the current process.
+
+    Under CLM_ZERO_FS=true or default test mode, uses an in-memory SQLite instance.
+    """
+    global _SHARED_MEMORY_CONN, _DEFAULT_FS
+    if _DEFAULT_FS is not None:
+        return _DEFAULT_FS
+
+    # Check Zero-FS flag
+    zero_fs = os.environ.get("CLM_ZERO_FS", "true").lower() in ("true", "1", "yes")
+
+    if zero_fs:
+        if _SHARED_MEMORY_CONN is None:
+            _SHARED_MEMORY_CONN = sqlite3.connect(":memory:", check_same_thread=False)
+        _DEFAULT_FS = MCardFileSystem(conn=_SHARED_MEMORY_CONN)
+    else:
+        clm_root = os.environ.get("CLM_ROOT") or os.getcwd()
+        db_path = Path(clm_root) / "computable_knowledge" / "content_memory.db"
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        conn = sqlite3.connect(str(db_path), check_same_thread=False)
+        _DEFAULT_FS = MCardFileSystem(conn=conn)
+
+    return _DEFAULT_FS
+
+
+def reset_default_fs() -> None:
+    """Reset shared default filesystem (primarily for test isolation)."""
+    global _SHARED_MEMORY_CONN, _DEFAULT_FS
+    if _SHARED_MEMORY_CONN is not None:
+        try:
+            _SHARED_MEMORY_CONN.close()
+        except Exception:
+            pass
+        _SHARED_MEMORY_CONN = None
+    _DEFAULT_FS = None
+
+
+def ingest_bytes(
+    data: bytes,
+    format: str | None = None,
+    handle: str | None = None,
+    author: str = "did:clm:user",
+    fs: MCardFileSystem | None = None,
+    filename: str | None = None,
+) -> MCard:
+    """Ingest raw bytes into the MCard Virtual File System.
+
+    Args:
+        data: Raw byte payload.
+        format: Optional explicit format override. If None, sniffed automatically.
+        handle: Optional human-readable handle (e.g. 'customers.csv' or 'mcard://my_dataset').
+        author: DID string of the authoring entity.
+        fs: Optional MCardFileSystem instance. If None, uses default fs.
+        filename: Optional source filename to assist MIME sniffing.
+
+    Returns:
+        MCard instance registered in CAS.
+    """
+    target_fs = fs or get_default_fs()
+
+    # 1. Detect MIME and format
+    sniffed_mime, sniffed_format = detect_mime_and_format(data, filename=filename)
+    chosen_format = format or sniffed_format
+
+    # 2. Extract structural metadata
+    metadata = extract_structural_metadata(data, sniffed_mime, chosen_format)
+    if filename:
+        metadata["source_filename"] = os.path.basename(filename)
+
+    # 3. Format payload for canonical storage
+    is_textual = chosen_format in ("csv", "json", "yaml", "markdown", "text")
+    if is_textual:
+        try:
+            payload = data.decode("utf-8")
+            metadata["encoding"] = "utf-8"
+        except UnicodeDecodeError:
+            payload = base64.b64encode(data).decode("ascii")
+            metadata["encoding"] = "base64"
+    else:
+        payload = base64.b64encode(data).decode("ascii")
+        metadata["encoding"] = "base64"
+
+    # 4. Compute deterministic BLAKE3 hash of content
+    from clm.layer0.model.card_triad import compute_canonical_hash
+    digest = compute_canonical_hash(payload)
+
+    # 5. Resolve canonical URI
+    if handle:
+        clean_handle = handle.removeprefix("mcard://").removeprefix("mcard:").strip("/")
+        uri = f"mcard:{clean_handle}"
+    else:
+        uri = f"mcard:{digest}"
+
+    # 6. Store in filesystem
+    mcard = target_fs.write_file(
+        uri=uri,
+        payload=payload,
+        author=author,
+        metadata=metadata,
+    )
+    return mcard
+
+
+def ingest_file(
+    path: str | Path,
+    handle: str | None = None,
+    author: str = "did:clm:user",
+    format: str | None = None,
+    fs: MCardFileSystem | None = None,
+) -> MCard:
+    """Ingest a local filesystem file into the MCard Virtual File System.
+
+    Args:
+        path: Path to the local file.
+        handle: Optional handle name. If None, defaults to file's basename or hash.
+        author: DID string of the ingesting agent.
+        format: Optional format override.
+        fs: Optional MCardFileSystem instance.
+
+    Returns:
+        MCard instance registered in CAS.
+    """
+    file_path = Path(path).resolve()
+    if not file_path.is_file():
+        raise FileNotFoundError(f"File not found for ingestion: {path}")
+
+    data = file_path.read_bytes()
+    default_handle = handle or file_path.name
+
+    return ingest_bytes(
+        data=data,
+        format=format,
+        handle=default_handle,
+        author=author,
+        fs=fs,
+        filename=str(file_path),
+    )
+
+
+def export_mcard(
+    uri: str,
+    output_path: str | Path,
+    fs: MCardFileSystem | None = None,
+) -> Path:
+    """Export an MCard from CAS back to the local host filesystem.
+
+    Args:
+        uri: Handle or URI of the MCard (e.g. 'mcard://<hash>' or 'mcard:<handle>').
+        output_path: Local filesystem destination path.
+        fs: Optional MCardFileSystem instance.
+
+    Returns:
+        Path of written file.
+    """
+    target_fs = fs or get_default_fs()
+    mcard = target_fs.read_file(uri)
+
+    out_p = Path(output_path).resolve()
+    out_p.parent.mkdir(parents=True, exist_ok=True)
+
+    encoding = mcard.metadata.get("encoding", "utf-8")
+    if encoding == "base64" and isinstance(mcard.payload, str):
+        raw_bytes = base64.b64decode(mcard.payload)
+        out_p.write_bytes(raw_bytes)
+    elif isinstance(mcard.payload, str):
+        out_p.write_text(mcard.payload, encoding="utf-8")
+    elif isinstance(mcard.payload, bytes):
+        out_p.write_bytes(mcard.payload)
+    else:
+        import json
+        out_p.write_text(json.dumps(mcard.payload, indent=2), encoding="utf-8")
+
+    return out_p