clm-kernel 0.0.0__py3-none-any.whl

clm/layer2/mime.py

@@ -0,0 +1,244 @@
+import json
+import os
+import xml.etree.ElementTree as ET
+
+import yaml
+
+
+def detect_encoding(file_path: str) -> str:
+    """
+    Detects if a file is binary or text based on content sampling.
+
+    Args:
+        file_path (str): The file to analyze.
+
+    Returns:
+        str: 'binary' or 'text'.
+    """
+    if not os.path.exists(file_path):
+        return "binary"
+
+    try:
+        file_size = os.path.getsize(file_path)
+        if file_size == 0:
+            return "text"
+
+        with open(file_path, "rb") as f:
+            sample = f.read(4096)
+
+        if b"\x00" in sample:
+            return "binary"
+
+        # Count non-text characters (chars < 9 or 14-31 or 127+)
+        non_text_count = 0
+        for byte in sample:
+            if (byte < 9) or (13 < byte < 32) or (byte > 127):
+                non_text_count += 1
+
+        # Binary if non-text characters exceed 30% of sample size
+        if non_text_count > (len(sample) * 30 // 100):
+            return "binary"
+
+        return "text"
+    except Exception:
+        return "binary"
+
+def is_binary(file_path: str) -> bool:
+    """Checks if a file is binary."""
+    return detect_encoding(file_path) == "binary"
+
+def is_text(file_path: str) -> bool:
+    """Checks if a file is text."""
+    return detect_encoding(file_path) == "text"
+
+def get_magic_rules(clm_root: str) -> dict[str, str]:
+    """
+    Scans the types/ directory to dynamically compile magic byte rules.
+
+    Args:
+        clm_root (str): The CLM workspace root.
+
+    Returns:
+        Dict[str, str]: A dictionary mapping magic hex strings to MIME types.
+    """
+    # Scan types/ directory to dynamically compile magic rules from manifests
+    magic_rules = {}
+    types_dir = os.path.join(clm_root, "types")
+    if not os.path.exists(types_dir):
+        return magic_rules
+
+    for root, _, files in os.walk(types_dir):
+        for file in files:
+            if file.endswith(".yaml") or file.endswith(".clm"):
+                try:
+                    with open(os.path.join(root, file), encoding="utf-8") as f:
+                        data = yaml.safe_load(f)
+                    if not data or not isinstance(data, dict):
+                        continue
+                    inner = data.get("clm", data)
+                    concrete = inner.get("concrete_impl") or inner.get("concrete")
+                    if not concrete or not isinstance(concrete, dict):
+                        continue
+                    content_type = concrete.get("content_type")
+                    if not content_type or not isinstance(content_type, dict):
+                        continue
+
+                    mime = content_type.get("mime")
+                    magic_bytes = content_type.get("detection_rules", {}).get("magic_bytes")
+                    if mime and magic_bytes:
+                        magic_rules[magic_bytes.lower()] = mime
+                except Exception:
+                    pass
+    return magic_rules
+
+def detect_mime(clm_root: str, file_path: str, ext_hint: str | None = None) -> str:
+    """
+    Detects the MIME type of a file using magic bytes, extension, or encoding fallback.
+
+    Args:
+        clm_root (str): The CLM workspace root.
+        file_path (str): The file to analyze.
+        ext_hint (Optional[str]): An optional file extension hint.
+
+    Returns:
+        str: The detected MIME type.
+    """
+    if not os.path.exists(file_path) or not os.path.isfile(file_path):
+        return "application/octet-stream"
+
+    try:
+        # 1. Check magic bytes
+        with open(file_path, "rb") as f:
+            header = f.read(16)
+        hex_header = header.hex().lower()
+
+        # Load magic rules dynamically
+        magic_rules = get_magic_rules(clm_root)
+
+        # Add static defaults if not loaded or fallback
+        static_magic = {
+            "89504e470d0a1a0a": "image/png",
+            "ffd8ffe0": "image/jpeg",
+            "ffd8ffe1": "image/jpeg",
+            "ffd8ffe2": "image/jpeg",
+            "ffd8ffe3": "image/jpeg",
+            "ffd8ffe8": "image/jpeg",
+            "474946383761": "image/gif",
+            "474946383961": "image/gif",
+            "25504446": "application/pdf",
+            "504b0304": "application/zip",
+            "53514c69746520666f726d6174203300": "application/sqlite3"
+        }
+        for k, v in static_magic.items():
+            if k not in magic_rules:
+                magic_rules[k] = v
+
+        for magic_hex, mime in sorted(magic_rules.items(), key=lambda x: len(x[0]), reverse=True):
+            if hex_header.startswith(magic_hex):
+                return mime
+
+        # 2. Check extension hint
+        if ext_hint:
+            clean_hint = ext_hint.lstrip(".").lower()
+            ext_map = {
+                "md": "text/markdown",
+                "py": "text/x-python",
+                "sh": "text/x-shellscript",
+                "js": "application/javascript",
+                "yaml": "application/yaml",
+                "yml": "application/yaml",
+                "json": "application/json",
+                "xml": "text/xml",
+                "html": "text/html",
+                "htm": "text/html",
+                "svg": "image/svg+xml",
+                "stl": "model/stl",
+                "obj": "model/obj",
+                "dxf": "image/vnd.dxf",
+                "gltf": "model/gltf+json",
+                "tex": "text/x-tex",
+                "tikz": "text/x-tikz",
+                "tikzcd": "text/x-tikzcd",
+                "tikz-cd": "text/x-tikzcd",
+                "djvu": "image/vnd.djvu",
+                "djv": "image/vnd.djvu",
+                "geojson": "application/geo+json",
+                "kml": "application/vnd.google-earth.kml+xml",
+                "shp": "application/x-shapefile",
+                "gpkg": "application/geopackage+sqlite3"
+            }
+            if clean_hint in ext_map:
+                return ext_map[clean_hint]
+
+        # 3. Fallback based on encoding
+        if is_binary(file_path):
+            return "application/octet-stream"
+        else:
+            return "text/plain"
+    except Exception:
+        return "application/octet-stream"
+
+def validate(clm_root: str, file_path: str, mime: str | None = None) -> bool:
+    """
+    Validates a file's content against its detected or provided MIME type.
+
+    Args:
+        clm_root (str): The CLM workspace root.
+        file_path (str): The file to validate.
+        mime (Optional[str]): The expected MIME type.
+
+    Returns:
+        bool: True if valid, False otherwise.
+    """
+    if not os.path.exists(file_path) or not os.path.isfile(file_path):
+        return False
+
+    try:
+        file_size = os.path.getsize(file_path)
+        if file_size == 0:
+            return False
+
+        if not mime:
+            mime = detect_mime(clm_root, file_path)
+
+        if mime == "application/json":
+            with open(file_path, encoding="utf-8") as f:
+                json.load(f)
+            return True
+
+        elif mime in ("application/yaml", "application/x-yaml"):
+            with open(file_path, encoding="utf-8") as f:
+                yaml.safe_load(f)
+            return True
+
+        elif mime in ("text/xml", "application/xml", "image/svg+xml"):
+            with open(file_path, encoding="utf-8") as f:
+                ET.fromstring(f.read())
+            return True
+
+        elif mime.startswith("text/"):
+            return is_text(file_path)
+
+        else:
+            # Check magic bytes constraint from corresponding manifest
+            safe_mime = "".join(c if c.isalnum() else "_" for c in mime)
+            manifest_file = None
+            for folder in ("content", "transport", "meta"):
+                candidate = os.path.join(clm_root, "types", folder, f"{safe_mime}.yaml")
+                if os.path.exists(candidate):
+                    manifest_file = candidate
+                    break
+
+            if manifest_file:
+                with open(manifest_file, encoding="utf-8") as f:
+                    data = yaml.safe_load(f)
+                inner = data.get("clm", data)
+                magic_bytes = inner.get("concrete_impl", {}).get("content_type", {}).get("detection_rules", {}).get("magic_bytes", "")
+                if magic_bytes:
+                    with open(file_path, "rb") as f:
+                        header = f.read(len(magic_bytes) // 2)
+                    if header.hex().lower() != magic_bytes.lower():
+                        return False
+            return True
+    except Exception:
+        return False

clm/layer2/mime_detector.py

@@ -0,0 +1,245 @@
+"""MIME Detector and Structural Metadata Extractor (Sprint 392 / DEL-392-02).
+
+Provides robust MIME sniffing and structural metadata extraction for the Universal
+MCard Ingestion Membrane. Detects file formats without relying solely on file extensions.
+Extracts format-specific metadata (columns, keys, headers, dimensions, sizes).
+
+Layer 2 module (conforms to AST layer boundaries, only imports L0, L1, L2).
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+# Standard Magic Byte Signatures
+MAGIC_SIGNATURES: list[tuple[bytes, str, str]] = [
+    (b"\x89PNG\r\n\x1a\n", "image/png", "image"),
+    (b"\xff\xd8\xff", "image/jpeg", "image"),
+    (b"%PDF-", "application/pdf", "media"),
+    (b"GIF87a", "image/gif", "image"),
+    (b"GIF89a", "image/gif", "image"),
+    (b"glTF", "model/gltf-binary", "model3d"),
+    (b"\x4d\x4d\x00\x2a", "image/tiff", "image"),
+    (b"\x49\x49\x2a\x00", "image/tiff", "image"),
+    (b"\x4d\x4d", "model/x-3ds", "model3d"),
+]
+
+
+def _sniff_magic(data: bytes) -> tuple[str | None, str | None]:
+    """Check initial magic bytes for binary media formats."""
+    for magic, mime, fmt in MAGIC_SIGNATURES:
+        if data.startswith(magic):
+            return mime, fmt
+
+    # RIFF containers: WebP or WAV
+    if len(data) >= 12 and data[:4] == b"RIFF":
+        riff_type = data[8:12]
+        if riff_type == b"WEBP":
+            return "image/webp", "image"
+        elif riff_type == b"WAVE":
+            return "audio/wav", "audio"
+
+    # MP4 container: ftyp box in first 32 bytes
+    if len(data) >= 8 and (b"ftyp" in data[:32]):
+        return "video/mp4", "media"
+
+    # MP3: ID3 tag or sync word
+    if data.startswith(b"ID3") or (len(data) >= 2 and data[0] == 0xFF and (data[1] & 0xE0) == 0xE0):
+        return "audio/mpeg", "audio"
+
+    return None, None
+
+
+def _is_binary(data: bytes) -> bool:
+    """Heuristic check for binary content."""
+    if b"\x00" in data:
+        return True
+    # Count non-text bytes in first 4096 bytes
+    sample = data[:4096]
+    if not sample:
+        return False
+    non_text = sum(1 for b in sample if (b < 9) or (13 < b < 32) or (b > 127))
+    return (non_text / len(sample)) > 0.30
+
+
+def detect_mime_and_format(data: bytes, filename: str | None = None) -> tuple[str, str]:
+    """Detect MIME type and canonical CLM format for byte payload.
+
+    Returns:
+        (mime_type, clm_format) where clm_format is one of:
+        csv, json, yaml, markdown, text, binary, media, image, audio, model3d.
+    """
+    ext = Path(filename).suffix.lower() if filename else ""
+
+    # 1. Magic byte checks
+    magic_mime, magic_fmt = _sniff_magic(data)
+    if magic_mime and magic_fmt:
+        return magic_mime, magic_fmt
+
+    # 2. Binary vs Text check
+    if _is_binary(data):
+        # Specific binary 3D model formats
+        if ext in (".stl", ".obj"):
+            return "model/stl" if ext == ".stl" else "model/obj", "model3d"
+        return "application/octet-stream", "binary"
+
+    # 3. Text content inspection
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError:
+        try:
+            text = data.decode("latin-1")
+        except UnicodeDecodeError:
+            return "application/octet-stream", "binary"
+
+    stripped = text.strip()
+    if not stripped:
+        return "text/plain", "text"
+
+    # SVG check
+    if stripped.startswith("<svg") or (stripped.startswith("<?xml") and "<svg" in stripped[:500]):
+        return "image/svg+xml", "image"
+
+    # JSON check
+    if (stripped.startswith("{") and stripped.endswith("}")) or (stripped.startswith("[") and stripped.endswith("]")):
+        try:
+            parsed = json.loads(stripped)
+            # Check if GLTF JSON
+            if isinstance(parsed, dict) and "asset" in parsed and isinstance(parsed["asset"], dict) and "version" in parsed["asset"]:
+                return "model/gltf+json", "model3d"
+            return "application/json", "json"
+        except (json.JSONDecodeError, ValueError):
+            pass
+
+    # CSV check: multiple lines with consistent commas/delimiters
+    lines = [ln for ln in stripped.splitlines() if ln.strip()]
+    if len(lines) >= 2:
+        try:
+            # Sniff CSV dialect
+            sample = "\n".join(lines[:10])
+            sniffer = csv.Sniffer()
+            dialect = sniffer.sniff(sample, delimiters=",\t|;")
+            reader = csv.reader(io.StringIO(sample), dialect)
+            rows = list(reader)
+            if len(rows) >= 2 and all(len(r) == len(rows[0]) and len(r) >= 2 for r in rows):
+                return "text/csv", "csv"
+        except Exception:
+            pass
+
+    # Markdown check
+    has_md_heading = bool(re.search(r"^#{1,6}\s+\S+", text, re.MULTILINE))
+    has_md_table = bool(re.search(r"^\|.+\|\s*\n\|[-:| ]+\|\s*\n\|.+\|", text, re.MULTILINE))
+    has_md_links = bool(re.search(r"\[.+\]\(.+\)", text))
+    if has_md_heading or has_md_table or (has_md_links and ext in (".md", ".markdown")):
+        return "text/markdown", "markdown"
+
+    # YAML check
+    if ":" in stripped and not stripped.startswith("{"):
+        try:
+            parsed = yaml.safe_load(stripped)
+            if isinstance(parsed, (dict, list)):
+                return "application/x-yaml", "yaml"
+        except Exception:
+            pass
+
+    # Extension fallbacks if plain text
+    if ext in (".md", ".markdown"):
+        return "text/markdown", "markdown"
+    if ext == ".csv":
+        return "text/csv", "csv"
+    if ext in (".yaml", ".yml"):
+        return "application/x-yaml", "yaml"
+    if ext == ".json":
+        return "application/json", "json"
+    if ext == ".obj":
+        return "model/obj", "model3d"
+    if ext == ".stl":
+        return "model/stl", "model3d"
+
+    return "text/plain", "text"
+
+
+def extract_structural_metadata(data: bytes, mime_type: str, clm_format: str) -> dict[str, Any]:
+    """Extract structural metadata from byte content according to format."""
+    meta: dict[str, Any] = {
+        "byte_size": len(data),
+        "mime_type": mime_type,
+        "format": clm_format,
+    }
+
+    if clm_format == "csv":
+        try:
+            text = data.decode("utf-8")
+            lines = [ln for ln in text.splitlines() if ln.strip()]
+            if lines:
+                sniffer = csv.Sniffer()
+                sample = "\n".join(lines[:10])
+                try:
+                    dialect = sniffer.sniff(sample, delimiters=",\t|;")
+                    delim = dialect.delimiter
+                except Exception:
+                    delim = ","
+                reader = csv.reader(io.StringIO(text), delimiter=delim)
+                rows = list(reader)
+                if rows:
+                    meta["columns"] = rows[0]
+                    meta["column_count"] = len(rows[0])
+                    meta["row_count"] = len(rows) - 1
+                    meta["delimiter"] = delim
+        except Exception:
+            pass
+
+    elif clm_format == "json":
+        try:
+            parsed = json.loads(data.decode("utf-8"))
+            if isinstance(parsed, dict):
+                meta["json_type"] = "object"
+                meta["keys"] = list(parsed.keys())
+                meta["key_count"] = len(parsed)
+            elif isinstance(parsed, list):
+                meta["json_type"] = "array"
+                meta["item_count"] = len(parsed)
+        except Exception:
+            pass
+
+    elif clm_format == "yaml":
+        try:
+            parsed = yaml.safe_load(data.decode("utf-8"))
+            if isinstance(parsed, dict):
+                meta["yaml_type"] = "mapping"
+                meta["keys"] = list(parsed.keys())
+                meta["key_count"] = len(parsed)
+            elif isinstance(parsed, list):
+                meta["yaml_type"] = "sequence"
+                meta["item_count"] = len(parsed)
+        except Exception:
+            pass
+
+    elif clm_format == "markdown":
+        try:
+            text = data.decode("utf-8")
+            headings = re.findall(r"^(#{1,6})\s+(.+)$", text, re.MULTILINE)
+            meta["headings"] = [h[1].strip() for h in headings]
+            meta["heading_count"] = len(headings)
+            meta["line_count"] = len(text.splitlines())
+            meta["word_count"] = len(text.split())
+        except Exception:
+            pass
+
+    elif clm_format == "text":
+        try:
+            text = data.decode("utf-8")
+            meta["line_count"] = len(text.splitlines())
+            meta["char_count"] = len(text)
+            meta["word_count"] = len(text.split())
+        except Exception:
+            pass
+
+    return meta

clm/layer2/private_collection.py

@@ -0,0 +1,118 @@
+"""Private collection module for encrypted VCards and private execution logs."""
+
+import logging
+import os
+from typing import Any
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+from clm.cross_cutting.config.config_constants import DEFAULT_EXECUTION_LOG_PATH
+from clm.layer0.model.card import MCard
+from clm.layer2.card_collection import CardCollection
+
+logger = logging.getLogger(__name__)
+
+_VOLATILE_DEV_KEY = AESGCM.generate_key(256)
+
+
+class EncryptedStorageEngineWrapper:
+    """Wraps an existing SQLite/DuckDB engine to transparently encrypt/decrypt content."""
+
+    def __init__(self, engine: Any, aesgcm: AESGCM):
+        self._engine = engine
+        self._aesgcm = aesgcm
+
+    def _encrypt(self, data: bytes) -> bytes:
+        nonce = os.urandom(12)
+        ciphertext = self._aesgcm.encrypt(nonce, data, None)
+        return nonce + ciphertext
+
+    def _decrypt(self, data: bytes) -> bytes:
+        nonce = data[:12]
+        ciphertext = data[12:]
+        try:
+            return self._aesgcm.decrypt(nonce, ciphertext, None)
+        except InvalidTag as err:
+            logger.error(
+                "Failed to decrypt DB content. Key mismatch or data corruption."
+            )
+            raise ValueError(
+                "Decryption failed. Invalid key or corrupted data."
+            ) from err
+
+    def add(self, card: MCard) -> str:
+        # Create an encrypted version of the card, but maintain the plaintext hash for addressing
+        content_bytes = (
+            card.content
+            if isinstance(card.content, bytes)
+            else str(card.content).encode("utf-8")
+        )
+        encrypted_bytes = self._encrypt(content_bytes)
+
+        enc_card = MCard(encrypted_bytes)
+        enc_card.hash = card.hash
+        enc_card.g_time = card.g_time
+        return self._engine.add(enc_card)
+
+    def get(self, hash_value: str) -> MCard | None:
+        enc_card = self._engine.get(hash_value)
+        if not enc_card:
+            return None
+        plaintext_bytes = self._decrypt(enc_card.content)
+        return MCard(plaintext_bytes)
+
+    def delete(self, hash_value: str) -> bool:
+        return self._engine.delete(hash_value)
+
+    # Note: Search by content will not work over encrypted data with AES-GCM
+    # Search by string will only work over hash/g_time.
+    # We delegate other methods as needed, or raise NotImplementedError for full search
+    def __getattr__(self, name):
+        return getattr(self._engine, name)
+
+
+class PrivateCollection(CardCollection):
+    """A strictly isolated collection for VCards and execution traces.
+
+    Ensures that all content stored in this collection is AES-GCM encrypted
+    at rest. The encryption key should be provided via the `VCARD_ENCRYPTION_KEY`
+    environment variable as a hex string.
+    """
+
+    def __init__(self, db_path: str | None = None, key_hex: str | None = None):
+        """Initialize the private collection.
+
+        Args:
+            db_path: Path to the private database.
+            key_hex: Optional explicitly provided 256-bit AES key in hex.
+        """
+        if db_path is None:
+            db_path = DEFAULT_EXECUTION_LOG_PATH
+
+        # Resolve the private key
+        if not key_hex:
+            key_hex = os.getenv("VCARD_ENCRYPTION_KEY")
+
+        if key_hex:
+            try:
+                self.key = bytes.fromhex(key_hex)
+            except ValueError:
+                logger.error("VCARD_ENCRYPTION_KEY must be valid hex.")
+                raise
+        else:
+            logger.warning(
+                "No VCARD_ENCRYPTION_KEY found. Generating a volatile key for this session. "
+                "Data written will be unreadable after restart!"
+            )
+            self.key = _VOLATILE_DEV_KEY
+
+        self.aesgcm = AESGCM(self.key)
+
+        # Initialize the underlying standard CardCollection on the target db
+        super().__init__(db_path=db_path)
+
+        # Inject our encryption wrapper into the engine pipeline
+        self.engine = EncryptedStorageEngineWrapper(self.engine, self.aesgcm)
+
+    # The CardCollection methods will now safely call our wrapped engine's `add()` and `get()`

clm/layer2/shared_store.py

@@ -0,0 +1,11 @@
+"""Layer 2: Shared Store & Token State Space.
+
+Canonical Path: kernel/scripts/clm/layer2/shared_store.py
+Parity: Rust `src/layer2/storage.rs`, TypeScript `src/layer2/triDb.ts`
+"""
+
+from __future__ import annotations
+
+from clm.layer1.store import PTRSharedStore, create_store_from_context
+
+__all__ = ["PTRSharedStore", "create_store_from_context"]