clm-kernel 0.0.0__py3-none-any.whl

clm/layer0/model/hash/validator.py

@@ -0,0 +1,307 @@
+import hashlib
+import logging
+import os
+import struct
+
+from clm.layer0.model.hash.enums import HashAlgorithm
+
+# Set up basic logging for the module
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)  # Set to DEBUG to capture detailed messages
+
+
+# ── Zero-dependency pure Python BLAKE3 engine (INV-02 fallback) ─────────
+_B3_IV = (
+    0x6A09E667,
+    0xBB67AE85,
+    0x3C6EF372,
+    0xA54FF53A,
+    0x510E527F,
+    0x9B05688C,
+    0x1F83D9AB,
+    0x5BE0CD19,
+)
+_B3_MSG_PERM = (2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8)
+_B3_CHUNK_START, _B3_CHUNK_END, _B3_PARENT, _B3_ROOT = 1, 2, 4, 8
+
+
+def _b3_rotr(w: int, r: int) -> int:
+    return ((w >> r) | (w << (32 - r))) & 0xFFFFFFFF
+
+
+def _b3_g(s: list, a: int, b: int, c: int, d: int, mx: int, my: int) -> None:
+    s[a] = (s[a] + s[b] + mx) & 0xFFFFFFFF
+    s[d] = _b3_rotr(s[d] ^ s[a], 16)
+    s[c] = (s[c] + s[d]) & 0xFFFFFFFF
+    s[b] = _b3_rotr(s[b] ^ s[c], 12)
+    s[a] = (s[a] + s[b] + my) & 0xFFFFFFFF
+    s[d] = _b3_rotr(s[d] ^ s[a], 8)
+    s[c] = (s[c] + s[d]) & 0xFFFFFFFF
+    s[b] = _b3_rotr(s[b] ^ s[c], 7)
+
+
+def _b3_round(s: list, m: list) -> None:
+    _b3_g(s, 0, 4, 8, 12, m[0], m[1])
+    _b3_g(s, 1, 5, 9, 13, m[2], m[3])
+    _b3_g(s, 2, 6, 10, 14, m[4], m[5])
+    _b3_g(s, 3, 7, 11, 15, m[6], m[7])
+    _b3_g(s, 0, 5, 10, 15, m[8], m[9])
+    _b3_g(s, 1, 6, 11, 12, m[10], m[11])
+    _b3_g(s, 2, 7, 8, 13, m[12], m[13])
+    _b3_g(s, 3, 4, 9, 14, m[14], m[15])
+
+
+def _b3_compress(
+    cv: list, block_words: list, block_len: int, counter: int, flags: int
+) -> list:
+    s = (
+        list(cv)
+        + list(_B3_IV[:4])
+        + [counter & 0xFFFFFFFF, (counter >> 32) & 0xFFFFFFFF, block_len, flags]
+    )
+    m = list(block_words)
+    for r in range(7):
+        _b3_round(s, m)
+        if r < 6:
+            m = [m[_B3_MSG_PERM[i]] for i in range(16)]
+    out = [0] * 16
+    for i in range(8):
+        out[i] = (s[i] ^ s[i + 8]) & 0xFFFFFFFF
+        out[i + 8] = (s[i + 8] ^ cv[i]) & 0xFFFFFFFF
+    return out
+
+
+def _b3_words_from_bytes(b: bytearray) -> list:
+    words = [0] * 16
+    for i in range(min(16, (len(b) + 3) // 4)):
+        chunk = b[i * 4 : (i + 1) * 4]
+        words[i] = int.from_bytes(chunk, "little")
+    return words
+
+
+def _b3_words_to_hex(words: list) -> str:
+    return "".join(struct.pack("<I", w).hex() for w in words)
+
+
+class _B3ChunkState:
+    def __init__(self, counter: int, flags: int = 0):
+        self.cv = list(_B3_IV)
+        self.counter = counter
+        self.flags = flags
+        self.block = bytearray()
+        self.blocks_compressed = 0
+
+    def start_flag(self) -> int:
+        return _B3_CHUNK_START if self.blocks_compressed == 0 else 0
+
+    def update(self, data: bytes) -> None:
+        offset = 0
+        while offset < len(data):
+            if len(self.block) == 64:
+                words = _b3_words_from_bytes(self.block)
+                out = _b3_compress(
+                    self.cv, words, 64, self.counter, self.flags | self.start_flag()
+                )
+                self.cv = out[:8]
+                self.blocks_compressed += 1
+                self.block = bytearray()
+            take = min(64 - len(self.block), len(data) - offset)
+            self.block.extend(data[offset : offset + take])
+            offset += take
+
+    def output(self, is_root: bool = False) -> list:
+        words = _b3_words_from_bytes(self.block)
+        flags = self.flags | self.start_flag() | _B3_CHUNK_END
+        if is_root:
+            flags |= _B3_ROOT
+        return _b3_compress(self.cv, words, len(self.block), self.counter, flags)
+
+
+def _b3_parent_output(left_cv: list, right_cv: list, is_root: bool = False) -> list:
+    block_words = left_cv[:8] + right_cv[:8]
+    flags = _B3_PARENT
+    if is_root:
+        flags |= _B3_ROOT
+    return _b3_compress(_B3_IV, block_words, 64, 0, flags)
+
+
+def pure_python_blake3(data: bytes) -> str:
+    """Zero-dependency pure Python BLAKE3 hash computation."""
+    if isinstance(data, str):
+        data = data.encode("utf-8")
+    if len(data) <= 1024:
+        c = _B3ChunkState(0, 0)
+        c.update(data)
+        return _b3_words_to_hex(c.output(True)[:8])
+
+    stack = []
+    chunk_counter = 0
+    offset = 0
+    while offset + 1024 < len(data):
+        chunk_bytes = data[offset : offset + 1024]
+        c = _B3ChunkState(chunk_counter, 0)
+        c.update(chunk_bytes)
+        chunk_cv = c.output(False)[:8]
+        chunk_counter += 1
+        offset += 1024
+
+        total = chunk_counter
+        while (total & 1) == 0:
+            left_cv = stack.pop()
+            chunk_cv = _b3_parent_output(left_cv, chunk_cv, False)[:8]
+            total >>= 1
+        stack.append(chunk_cv)
+
+    final = _B3ChunkState(chunk_counter, 0)
+    final.update(data[offset:])
+    out = final.output(len(stack) == 0)[:8]
+    if len(stack) == 0:
+        return _b3_words_to_hex(out)
+
+    while stack:
+        left = stack.pop()
+        is_root = len(stack) == 0
+        out = _b3_parent_output(left, out, is_root)[:8]
+    return _b3_words_to_hex(out)
+
+
+class HashValidator:
+    """Validates and computes hash values for content."""
+
+    def __init__(
+        self,
+        content: str | bytes,
+        hash_algorithm: str | HashAlgorithm = HashAlgorithm.DEFAULT,
+    ):
+        self.content = content
+        self.hash_algorithm = hash_algorithm
+        self.hash_value = self.compute_hash(content, hash_algorithm)
+
+    @staticmethod
+    def compute_hash(
+        content: str | bytes,
+        hash_algorithm: str | HashAlgorithm = HashAlgorithm.DEFAULT,
+    ) -> str:
+        """Compute hash for given content using specified hash function"""
+        content_to_hash = (
+            content if isinstance(content, bytes) else content.encode("utf-8")
+        )  # Explicit encoding
+
+        # Handle HashAlgorithm conversion
+        if isinstance(hash_algorithm, str):
+            try:
+                hash_algorithm = HashAlgorithm(hash_algorithm.lower())
+            except ValueError as e:
+                raise ValueError(
+                    f"'{hash_algorithm}' is not a valid HashAlgorithm"
+                ) from e
+
+        if hash_algorithm == HashAlgorithm.CUSTOM:
+            raise ValueError(
+                "Custom hash function must be provided when using CUSTOM algorithm"
+            )
+
+        if hash_algorithm == HashAlgorithm.BLAKE3:
+            try:
+                import blake3
+
+                return blake3.blake3(content_to_hash).hexdigest()
+            except ImportError:
+                logger.warning(
+                    "Py-blake3 not found. Attempting OS fallback for BLAKE3."
+                )
+                candidates = [
+                    os.environ.get("CLM_RUST_CORE_BIN"),
+                    os.path.join(
+                        os.path.dirname(__file__),
+                        "../../../clm/kernel/clm_rust_core/target/release/clm_rust_core",
+                    ),
+                    os.path.join(
+                        os.path.dirname(__file__),
+                        "../../../clm/kernel/clm_rust_core/target/debug/clm_rust_core",
+                    ),
+                    "clm_rust_core",
+                    "b3sum",
+                ]
+                if os.environ.get("CLM_ZERO_FS", "").lower() == "true":
+                    return pure_python_blake3(content_to_hash)
+
+                for bin_path in candidates:
+                    if not bin_path:
+                        continue
+                    try:
+                        import subprocess
+                        # Assuming standard CLI argument structure for the tool
+                        cmd = (
+                            [bin_path, "hash"]
+                            if "clm_rust_core" in bin_path
+                            else [bin_path, "--no-names"]
+                        )
+                        res = subprocess.run(
+                            cmd,
+                            input=content_to_hash,
+                            capture_output=True,
+                            check=True,
+                        )
+                        out = res.stdout.decode("utf-8", errors="replace").strip()
+                        if out:
+                            # Attempt to extract the hash value cleanly
+                            return out.split()[0]
+                    except Exception as e:
+                        logger.debug(f"BLAKE3 fallback failed with {bin_path}: {e}")
+
+                # Fallback to pure Python if all external methods fail
+                logger.info("Falling back to pure Python implementation for BLAKE3.")
+                return pure_python_blake3(content_to_hash)
+
+        hash_functions = {
+            HashAlgorithm.MD5: hashlib.md5,
+            HashAlgorithm.SHA1: hashlib.sha1,
+            HashAlgorithm.SHA224: hashlib.sha224,
+            HashAlgorithm.SHA256: hashlib.sha256,
+            HashAlgorithm.SHA384: hashlib.sha384,
+            HashAlgorithm.SHA512: hashlib.sha512,
+        }
+
+        if hash_algorithm not in hash_functions:
+            raise ValueError(f"'{hash_algorithm}' is not a valid HashAlgorithm")
+
+        return hash_functions[hash_algorithm](content_to_hash).hexdigest()
+
+    def validate(self, expected_hash: str = None):
+        """Validate that the stored hash matches the expected hash"""
+        if expected_hash is not None:
+            self.hash_value = expected_hash
+
+        target = self.hash_value
+        algo = self.hash_algorithm
+
+        # Attempt to parse combined hash format (prefix:bare)
+        if isinstance(target, str) and ":" in target:
+            try:
+                prefix, bare = target.split(":", 1)
+                # Assuming HashAlgorithm.from_string exists and works if prefix is valid
+                if hasattr(HashAlgorithm, "from_string"):
+                    algo = HashAlgorithm.from_string(prefix)
+                    target = bare
+                else:
+                    logger.warning(
+                        "HashAlgorithm.from_string not available. Using default parsing."
+                    )
+            except ValueError:
+                pass  # Keep original algo/target if parsing fails
+
+        calculated_hash = self.compute_hash(self.content, algo)
+        logger.debug(
+            f"Validation Check: expected '{self.hash_value}', calculated '{calculated_hash}'"
+        )
+        logger.debug(
+            f"Hash calculation details: content='{self.content}', hash_function='{algo}'"
+        )
+        if calculated_hash != target:
+            raise ValueError(
+                f"Hash mismatch: expected '{self.hash_value}', got '{calculated_hash}'"
+            )
+
+    def __str__(self):
+        return self.hash_value

clm/layer0/model/interpreter.py

@@ -0,0 +1,275 @@
+"""
+Content type detection and validation service.
+
+This module provides a unified interface for content type detection and validation
+using a modular architecture with separate concerns for detection strategies,
+validation, and content analysis utilities.
+"""
+
+import json
+import logging
+
+try:
+    from defusedxml.ElementTree import fromstring as safe_fromstring
+except ImportError:
+    from xml.etree.ElementTree import fromstring as safe_fromstring  # nosec B314
+
+# Import modular components
+from .validators.validation_registry import ValidationError, validation_registry
+
+try:
+    import mcard_core_py  # type: ignore
+except ImportError:
+    LOGGER = logging.getLogger(__name__)
+    LOGGER.warning(
+        "mcard_core_py native extension not found, functionality may be degraded."
+    )
+
+# Module logger
+LOGGER = logging.getLogger(__name__)
+
+# Re-export ValidationError for backward compatibility
+__all__ = ["ContentTypeInterpreter", "ValidationError", "get_interpreter"]
+
+# Module-level singleton instance (lazy)
+_default_interpreter: "ContentTypeInterpreter | None" = None
+
+
+def get_interpreter() -> "ContentTypeInterpreter":
+    """Return the module-level ContentTypeInterpreter singleton."""
+    global _default_interpreter
+    if _default_interpreter is None:
+        _default_interpreter = ContentTypeInterpreter()
+    return _default_interpreter
+
+
+class ContentTypeInterpreter:
+    """
+    Unified service for content type detection and validation.
+
+    This consolidated class uses modular components internally while maintaining
+    the same external API for backward compatibility.
+    """
+
+    # ─── Load shared MIME data from external JSON ─────────────────────────
+    # The canonical data lives in mime_extensions.json at the repo root.
+    # Both Python and TypeScript load from it, keeping them in sync.
+    @staticmethod
+    def _load_shared_mime_data() -> tuple[frozenset, dict]:
+        """Load MIME data from the shared JSON file.
+
+        Returns a (text_mime_types frozenset, mime_to_extension dict) tuple.
+        Falls back to minimal built-in defaults if the file is not found.
+        """
+        from pathlib import Path
+
+        # Walk up from this file to find mime_extensions.json at repo root
+        search = Path(__file__).resolve().parent
+        for _ in range(5):
+            candidate = search / "mime_extensions.json"
+            if candidate.exists():
+                break
+            search = search.parent
+        else:
+            candidate = None
+
+        if candidate and candidate.exists():
+            try:
+                with open(candidate, encoding="utf-8") as f:
+                    data = json.load(f)
+                text_types = frozenset(data.get("textMimeTypes", []))
+                ext_map = dict(data.get("mimeToExtension", {}))
+                return text_types, ext_map
+            except Exception as e:
+                LOGGER.warning(
+                    "Failed to load mime_extensions.json: %s — using defaults", e
+                )
+
+        # Minimal fallback if JSON file is missing
+        return frozenset({"text/plain", "application/json"}), {
+            "text/plain": "txt",
+            "application/json": "json",
+        }
+
+    # Populated from the shared JSON (or fallback)
+    _text_types, _ext_map = _load_shared_mime_data.__func__()  # type: ignore[attr-defined]
+    TEXT_MIME_TYPES: frozenset = _text_types
+    _MIME_TO_EXTENSION: dict = _ext_map
+
+    @classmethod
+    def register_extension(cls, mime_type: str, extension: str) -> None:
+        """Dynamically register a new MIME-to-extension mapping.
+
+        Accepts extensions with or without a leading dot (e.g., '.custom' or 'custom').
+        The stored value will always be without a leading dot, consistent with the existing mapping.
+        """
+        if extension.startswith("."):
+            extension = extension[1:]
+        cls._MIME_TO_EXTENSION[mime_type] = extension
+
+    @classmethod
+    def register_extensions(cls, mapping: dict) -> None:
+        """Dynamically register a batch of MIME-to-extension mappings."""
+        for mime, ext in mapping.items():
+            cls.register_extension(mime, ext)
+
+    def __init__(self) -> None:
+        """
+        Initialize the interpreter.
+        """
+        pass
+
+    @staticmethod
+    def _decode_utf8(data: bytes) -> str:
+        """Decode bytes to UTF-8 string using a consistent policy."""
+        return data.decode("utf-8", errors="replace")
+
+    @staticmethod
+    def detect_content_type(
+        content: str | bytes, file_extension: str | None = None
+    ) -> tuple[str, str]:
+        """
+        Detect content type and suggested extension using Rust Core.
+        """
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+        return mcard_core_py.detect_content_type(content, file_extension)  # type: ignore
+
+    def detect_content_type_with_strategy(
+        self, content: str | bytes, file_extension: str | None = None
+    ) -> tuple[str, str]:
+        """
+        Detect content type using the configured strategy.
+
+        Args:
+            content: The content to analyze
+            file_extension: Optional file extension hint
+
+        Returns:
+            Tuple of (mime_type, extension)
+        """
+        return self.detect_content_type(content, file_extension)
+
+    def validate_content(self, content: str | bytes) -> None:
+        """
+        Validate content based on its detected type.
+
+        This method now delegates to the validation registry for better modularity.
+
+        Args:
+            content: The content to validate
+
+        Raises:
+            ValidationError: If content is invalid
+        """
+        mime_type, _ = self.detect_content_type(content)
+        validation_registry.validate(content, mime_type)
+
+    @staticmethod
+    def validate_content_static(content: str | bytes) -> None:
+        """
+        Static method for validating content (backward compatibility).
+
+        Args:
+            content: The content to validate
+
+        Raises:
+            ValidationError: If content is invalid
+        """
+        # Use default strategy for static method
+        mime_type, _ = ContentTypeInterpreter.detect_content_type(content)
+        validation_registry.validate(content, mime_type)
+
+    @staticmethod
+    def is_binary_content(content: str | bytes) -> bool:
+        """
+        Determine if content should be treated as binary using the Rust Core.
+        """
+        if isinstance(content, str):
+            content = content.encode("utf-8")
+        return mcard_core_py.is_binary_content(content)  # type: ignore
+
+    @staticmethod
+    def is_xml_content(content: str | bytes) -> bool:
+        """Check if content is valid XML."""
+        # Delegate to XMLDetector's is_valid_xml method
+        try:
+            if isinstance(content, str):
+                content = content.encode("utf-8")
+
+            # Try to parse the XML without requiring XML declaration
+            safe_fromstring(content)  # nosec B314
+            LOGGER.debug("Valid XML content detected.")
+            return True
+        except Exception as e:
+            LOGGER.debug("Invalid XML content detected: %s", str(e))
+            return False
+
+    @staticmethod
+    def is_svg_content(content: str | bytes) -> bool:
+        """Check if content is SVG."""
+        if isinstance(content, bytes):
+            content = content.decode("utf-8", errors="ignore")
+
+        # First check if it's valid XML
+        if not ContentTypeInterpreter.is_xml_content(content):
+            return False
+
+        try:
+            # Parse XML and check for SVG namespace
+            tree = safe_fromstring(content)  # nosec B314
+            return (
+                tree.tag == "svg"
+                or tree.tag.endswith("}svg")
+                or any(
+                    attr.endswith("xmlns") and "svg" in value
+                    for attr, value in tree.attrib.items()
+                )
+            )
+        except Exception:
+            return False
+
+    @staticmethod
+    def is_mermaid_content(content: str) -> bool:
+        """Check if content is Mermaid diagram."""
+        content = content.strip().lower()
+        mermaid_keywords = [
+            "graph ",
+            "sequencediagram",
+            "classDiagram",
+            "stateDiagram",
+            "erDiagram",
+            "gantt",
+            "pie",
+            "flowchart",
+            "journey",
+        ]
+        return any(content.startswith(keyword.lower()) for keyword in mermaid_keywords)
+
+    @staticmethod
+    def is_diagram_content(content: str) -> bool:
+        """Check if content is a diagram format."""
+        content = content.strip().lower()
+        # Check for PlantUML
+        if content.startswith("@startuml") and content.endswith("@enduml"):
+            return True
+        # Check for Graphviz
+        if content.startswith(("digraph", "graph", "strict")):
+            return True
+        # Check for Mermaid
+        return ContentTypeInterpreter.is_mermaid_content(content)
+
+    @staticmethod
+    def get_extension(mime_type: str) -> str:
+        """Get file extension from MIME type using Rust Core."""
+        ext = mcard_core_py.get_extension(mime_type)
+        if not ext:
+            ext = ContentTypeInterpreter.get_default_extension(mime_type)
+        return ext if ext is not None else ""
+
+    @staticmethod
+    def get_default_extension(mime_type: str) -> str:
+        """
+        Return the default file extension for a given MIME type.
+        """
+        return ContentTypeInterpreter._MIME_TO_EXTENSION.get(mime_type, "")

clm/layer0/model/pagination.py

@@ -0,0 +1,45 @@
+"""Pagination support for MCard collections."""
+
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+
+T = TypeVar("T")
+
+
+@dataclass
+class Page(Generic[T]):
+    """A page of search results."""
+
+    items: list[T]
+    page_number: int
+    page_size: int
+    total_items: int
+    total_pages: int
+    has_next: bool
+    has_previous: bool
+
+    @classmethod
+    def create(
+        cls, items: list, total_items: int, page_number: int, page_size: int
+    ) -> "Page":
+        """Factory that computes has_next, has_previous, and total_pages automatically."""
+        total_pages = (total_items + page_size - 1) // page_size if page_size > 0 else 0
+        return cls(
+            items=items,
+            total_items=total_items,
+            page_number=page_number,
+            page_size=page_size,
+            total_pages=total_pages,
+            has_next=total_items > page_number * page_size,
+            has_previous=page_number > 1,
+        )
+
+    @property
+    def next_page_number(self) -> int | None:
+        """Get the next page number, or None if this is the last page."""
+        return self.page_number + 1 if self.has_next else None
+
+    @property
+    def previous_page_number(self) -> int | None:
+        """Get the previous page number, or None if this is the first page."""
+        return self.page_number - 1 if self.has_previous else None