clm-kernel 0.0.0__py3-none-any.whl

clm/layer3/gateway.py

@@ -0,0 +1,353 @@
+import json
+import os
+import re
+import time
+from typing import Any
+
+import yaml
+
+from clm.cross_cutting.clm_logger import get_logger
+from clm.layer0.exceptions import GatewayError
+from clm.layer0.utils import clm_hash_string, prefixed_hash_raw, strip_hash_prefix
+
+_logger = get_logger("clm.layer3.gateway")
+
+
+def route_handle(clm_root: str, handle_name: str, fallback: str = "error", cache: str = "true") -> tuple[str, int]:
+    """
+    Routes and resolves a handle dynamically across local and remote mediums.
+
+    Args:
+        clm_root (str): The workspace root directory.
+        handle_name (str): The handle pointer to resolve.
+        fallback (str): Fallback behavior on resolution failure.
+        cache (str): Cache setting (true/false) for storing fetched content.
+
+    Returns:
+        Tuple[str, int]: The resolved final content and the status code.
+
+    Raises:
+        GatewayError: If handle is not found or circular reference is detected.
+    """
+    from clm.layer2.storage import CLMDB, CLMRepository
+    db = CLMDB(clm_root)
+    repo = CLMRepository(db, clm_root)
+    current_handle = handle_name
+    visited_handles = {handle_name}
+    final_content = ""
+    status = 0
+
+    while True:
+        content_hash = repo.resolve_handle_hash(current_handle, "content_memory")
+        if not content_hash:
+            if current_handle == handle_name:
+                raise GatewayError(f"Handle not found: {current_handle}")
+            else:
+                break
+
+        route_handle_name = f"route:{content_hash}"
+        route_content = repo.resolve_handle_content(route_handle_name, "content_memory")
+
+        medium = "local"
+        route_data = {}
+        if route_content:
+            try:
+                route_data = yaml.safe_load(route_content)
+                if isinstance(route_data, dict):
+                    medium = route_data.get("medium", "local")
+            except Exception as e:
+                _logger.error(f"[route_handle] Debug: route_content YAML load failed: {e}")
+
+        if medium == "file":
+            uri = route_data.get("uri", "")
+            path = uri.replace("file://", "", 1)
+            if not os.path.isabs(path) and not os.path.exists(path):
+                alt_path = os.path.join(clm_root, path)
+                if os.path.exists(alt_path):
+                    path = alt_path
+            try:
+                with open(path, encoding="utf-8") as f:
+                    final_content = f.read()
+                status = 0
+            except Exception as e:
+                _logger.warning(f"[route_handle] Warning: file medium resolution failed: {e}")
+                status = 2
+        elif medium in ("http", "ftp"):
+            uri = route_data.get("uri", "")
+            try:
+                import urllib.request
+                req = urllib.request.Request(
+                    uri,
+                    headers={'User-Agent': 'Mozilla/5.0'}
+                )
+                with urllib.request.urlopen(req, timeout=10) as response:
+                    final_content = response.read().decode('utf-8')
+                status = 0
+            except Exception as e:
+                _logger.warning(f"[route_handle] Warning: URL medium resolution failed: {e}")
+                status = 2
+        elif medium == "ipfs":
+            uri = route_data.get("uri", "")
+            cid = uri.replace("ipfs://", "", 1)
+            try:
+                import subprocess
+                proc = subprocess.Popen(["ipfs", "cat", cid], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+                try:
+                    stdout_data, _ = proc.communicate(timeout=15)
+                except subprocess.TimeoutExpired:
+                    proc.kill()
+                    proc.communicate()
+                    status = 2
+                else:
+                    if proc.returncode == 0:
+                        final_content = stdout_data
+                        status = 0
+                    else:
+                        status = 2
+            except Exception as e:
+                _logger.warning(f"[route_handle] Warning: IPFS medium resolution failed: {e}")
+                status = 2
+        elif medium == "indexeddb":
+            status = 2
+        else:
+            try:
+                final_content = repo.get_card_content(content_hash, "content_memory") or ""
+                status = 0
+            except Exception as e:
+                _logger.error(f"[route_handle] Error: get_card_content failed: {e}")
+                status = 2
+
+        if status == 0 and medium != "local":
+            cached_hash = route_data.get("cached_hash", "")
+            if cached_hash and cached_hash != "null":
+                actual_hash = clm_hash_string(final_content)
+                clean_expected = strip_hash_prefix(cached_hash)
+                if actual_hash != clean_expected:
+                    _logger.error(f"[route_handle] Error: Hash mismatch. Expected: {cached_hash}, Actual: {prefixed_hash_raw(actual_hash)}")
+                    status = 2
+
+        if status == 0 and cache == "true" and medium != "local":
+            final_hash = route_data.get("cached_hash", "")
+            if not final_hash or final_hash == "null":
+                actual_hash = clm_hash_string(final_content)
+                final_hash = prefixed_hash_raw(actual_hash)
+
+            clean_hash = strip_hash_prefix(final_hash)
+            ts = time.strftime("%Y-%m-%dT%H:%M:%S%z")
+            try:
+                repo.save_card(clean_hash, final_content, "content_memory", timestamp=ts, replace=False)
+            except Exception as e:
+                _logger.warning(f"[route_handle] Warning: Caching failed: {e}")
+
+        if status != 0:
+            if fallback == "stale":
+                cached_hash = route_data.get("cached_hash", "")
+                if cached_hash and cached_hash != "null":
+                    clean_hash = strip_hash_prefix(cached_hash)
+                    try:
+                        res = repo.get_card_content(clean_hash, "content_memory")
+                        if res:
+                            final_content = res
+                            status = 3
+                            _logger.warning(f"[route_handle] Warning: Stale fallback used for {current_handle}")
+                    except Exception as e:
+                        _logger.warning(f"[route_handle] Warning: Stale fallback fetch failed: {e}")
+
+                if status != 3:
+                    try:
+                        prev_hash = repo.get_latest_handle_history(current_handle, "content_memory")
+                        if prev_hash:
+                            res = repo.get_card_content(prev_hash, "content_memory")
+                            if res:
+                                final_content = res
+                                status = 3
+                                _logger.warning(f"[route_handle] Warning: Stale fallback (history) used for {current_handle}")
+                    except Exception as e:
+                        _logger.warning(f"[route_handle] Warning: Stale history fallback fetch failed: {e}")
+
+            if status != 3:
+                _logger.error(f"[route_handle] Error: Network error resolving medium {medium} for {current_handle}")
+                raise GatewayError(f"Network error resolving medium {medium}")
+
+        if len(final_content) <= 255 and re.match(r"^[a-zA-Z0-9_:-]+$", final_content):
+            check_hash = repo.resolve_handle_hash(final_content, "content_memory")
+            if check_hash:
+                if final_content in visited_handles:
+                    _logger.error(f"[route_handle] Error: Circular reference detected: {visited_handles} -> {final_content}")
+                    raise GatewayError("Circular reference detected")
+                visited_handles.add(final_content)
+                current_handle = final_content
+                continue
+
+        break
+
+    return final_content, status
+
+def remote_handle_resolver(
+    clm_root: str,
+    action: str | None = None,
+    handle: str | None = None,
+    peer_url: str | None = None,
+    locale_trust: str | None = None,
+    allowed_sponsors: str | None = None,
+    active_agent: str | None = None
+) -> tuple[dict[str, Any], int]:
+    """
+    Resolves a handle query over remote gateways or serves incoming peer resolution queries.
+    """
+    from clm.layer2.storage import CLMDB, CLMRepository
+    db = CLMDB(clm_root)
+    repo = CLMRepository(db, clm_root)
+
+    if action in ("serve", "server", "resolve_inbound"):
+        active_agent = active_agent or os.environ.get("ACTIVE_AGENT_DID") or os.environ.get("CLM_AGENT_DID") or ""
+        allowed_sponsors = allowed_sponsors or os.environ.get("CLM_RESOLVE_ALLOWED_SPONSORS") or ""
+
+        if allowed_sponsors and active_agent != "did:clm:bootstrap":
+            allowed_list = [s.strip() for s in allowed_sponsors.split(",") if s.strip()]
+            if active_agent not in allowed_list:
+                return {"status": "ERROR", "error": "Unauthorized agent"}, 1
+
+        if not handle:
+            return {"status": "ERROR", "error": "Missing handle parameter"}, 1
+
+        current_hash = ""
+        try:
+            current_hash = repo.resolve_handle_hash(handle, "content_memory")
+        except Exception as e:
+            _logger.warning(f"[remote_handle_resolver] Warning: resolve_handle_hash failed for {handle}: {e}")
+
+        if not current_hash and not handle.startswith("clm:"):
+            try:
+                current_hash = repo.resolve_handle_hash(f"clm:{handle}", "content_memory")
+            except Exception as e:
+                _logger.warning(f"[remote_handle_resolver] Warning: resolve_handle_hash failed for clm:{handle}: {e}")
+
+        if not current_hash:
+            return {"status": "ERROR", "error": "Handle not found"}, 1
+
+        yaml_content = ""
+        try:
+            yaml_content = repo.get_card_content(current_hash, "content_memory") or ""
+        except Exception as e:
+            _logger.warning(f"[remote_handle_resolver] Warning: get_card_content failed for {current_hash}: {e}")
+
+        if not yaml_content:
+            return {"status": "ERROR", "error": "Content not found in CAS"}, 1
+
+        locale_hash = os.environ.get("CLM_EXECUTION_LOCALE_HASH", "")
+        signature = ""
+        try:
+            parsed = yaml.safe_load(yaml_content)
+            if isinstance(parsed, dict):
+                inner = parsed.get("clm", parsed)
+                if isinstance(inner, dict):
+                    signature = inner.get("_meta", {}).get("signature", "")
+        except Exception as e:
+            _logger.error(f"[remote_handle_resolver] Debug: failed to parse signature from yaml_content: {e}")
+
+        return {
+            "status": "SUCCESS",
+            "hash": current_hash,
+            "content": yaml_content,
+            "locale_hash": locale_hash,
+            "signature": signature
+        }, 0
+
+    elif action in ("resolve", "client", "resolve_outbound"):
+        if not handle or not peer_url:
+            return {"status": "ERROR", "error": "Missing handle or peer_url parameters"}, 1
+
+        raw_response = None
+        if peer_url.startswith("http://") or peer_url.startswith("https://"):
+            try:
+                import urllib.error
+                import urllib.request
+                req = urllib.request.Request(
+                    peer_url,
+                    data=json.dumps({"action": "serve", "handle": handle}).encode("utf-8"),
+                    headers={"Content-Type": "application/json"}
+                )
+                with urllib.request.urlopen(req, timeout=10) as response:
+                    raw_response = response.read().decode("utf-8")
+            except Exception as e:
+                return {"status": "ERROR", "error": f"Transport connection failed: {e}"}, 1
+        elif peer_url == "mock" or peer_url == "mock_fail_hash":
+            mock_yaml = (
+                "clm:\n"
+                "  _meta:\n"
+                "    clm_root_hash: \"sha256:correct_hash\"\n"
+                "  content_hash: \"sha256:correct_hash\"\n"
+                "  g_time: \"2026-06-13T19:00:00Z\"\n"
+            )
+            correct_hash = clm_hash_string(mock_yaml)
+            mock_yaml = mock_yaml.replace("correct_hash", correct_hash)
+
+            h = correct_hash
+            if peer_url == "mock_fail_hash":
+                h = "sha256:bad_hash"
+
+            raw_response = json.dumps({
+                "status": "SUCCESS",
+                "hash": h,
+                "content": mock_yaml,
+                "locale_hash": "sha256:mock_locale",
+                "signature": "mock_sig"
+            })
+        else:
+            return {"status": "ERROR", "error": f"Unsupported peer URL: {peer_url}"}, 1
+
+        try:
+            resp_data = json.loads(raw_response)
+        except Exception:
+            return {"status": "ERROR", "error": "Invalid JSON response from peer"}, 1
+
+        status = resp_data.get("status", "ERROR")
+        if status != "SUCCESS":
+            error_msg = resp_data.get("error", "Unknown peer error")
+            return {"status": "ERROR", "error": f"Peer error: {error_msg}"}, 1
+
+        r_hash = resp_data.get("hash", "")
+        content = resp_data.get("content", "")
+        locale_hash = resp_data.get("locale_hash", "")
+        signature = resp_data.get("signature", "")
+
+        locale_trust = locale_trust or os.environ.get("CLM_LOCALE_TRUST") or ""
+        if locale_trust:
+            trusted_list = [t.strip() for t in locale_trust.split(",") if t.strip()]
+            if locale_hash not in trusted_list:
+                return {"status": "ERROR", "error": "Untrusted locale"}, 1
+
+        computed_root = ""
+        try:
+            parsed = yaml.safe_load(content)
+            if isinstance(parsed, dict):
+                inner = parsed.get("clm", parsed)
+                if isinstance(inner, dict):
+                    computed_root = inner.get("_meta", {}).get("clm_root_hash", "")
+        except Exception as e:
+            _logger.error(f"[remote_handle_resolver] Debug: yaml load failed for integrity check: {e}")
+
+        clean_computed = strip_hash_prefix(computed_root)
+        clean_returned = strip_hash_prefix(r_hash)
+        if clean_computed and clean_computed != clean_returned:
+            return {"status": "ERROR", "error": "Integrity hash mismatch"}, 1
+
+        clm_name = handle
+        if handle.startswith("clm:"):
+            clm_name = handle[4:]
+        remote_handle = f"remote:clm:{clm_name}"
+
+        try:
+            ts = time.strftime("%Y-%m-%dT%H:%M:%S%z")
+            repo.save_card_and_handle(remote_handle, clean_returned, content, "content_memory", timestamp=ts, replace=True)
+        except Exception as e:
+            _logger.warning(f"[remote_handle_resolver] Warning: Caching remote handle failed: {e}")
+
+        return {
+            "status": "SUCCESS",
+            "hash": r_hash,
+            "remote_handle": remote_handle
+        }, 0
+    else:
+        return {"status": "ERROR", "error": "Unknown action"}, 1

clm/layer3/open_interpreter_adapter.py

@@ -0,0 +1,159 @@
+import json
+import os
+import re
+import sys
+
+from clm.cross_cutting.clm_logger import get_logger
+
+_logger = get_logger("clm.layer3.open_interpreter_adapter")
+
+
+def evaluate_open_interpreter(clm_root, clm_file, block, extra_args):
+    """
+    Executes open_interpreter process payload, supporting sandbox isolation
+    and mock/simulated modes for offline testing.
+    """
+    payload = extra_args[0] if extra_args else ""
+
+    # 1. Parse manifest parameters
+    model = block.get("model", "gpt-4o")
+    temp = float(block.get("temperature", 0.1))
+    prompt_template = block.get("script", block.get("prompt", ""))
+
+    # Check for script_handle
+    script_handle = block.get("script_handle", "")
+    if script_handle:
+        from clm.layer2.storage import resolve_handle_content
+        prompt_template = resolve_handle_content(clm_root, script_handle)
+
+    if not prompt_template:
+        _logger.error("[PAL:open_interpreter] ❌ No script or prompt found.")
+        return 2
+
+    # 2. Interpolate prompt
+    prompt = interpolate_prompt(prompt_template, clm_file, payload)
+
+    # Check mock behavior first
+    mock_behavior = os.environ.get("CLM_MOCK_OI_BEHAVIOR") or os.environ.get("CLM_MOCK_LLM_BEHAVIOR")
+    if mock_behavior:
+        return handle_mock_behavior(mock_behavior, prompt, clm_root, payload)
+
+    # 3. Check for library availability
+    try:
+        from interpreter import interpreter
+
+        # Enforce Zero-FS Hermetic Isolation
+        zero_fs = os.environ.get("CLM_ZERO_FS", "").lower() == "true"
+        if zero_fs:
+            import shutil
+            import tempfile
+            temp_dir = tempfile.mkdtemp()
+            orig_cwd = os.getcwd()
+            os.chdir(temp_dir)
+
+        try:
+            interpreter.llm.model = model
+            interpreter.llm.temperature = temp
+            interpreter.auto_run = True
+
+            # Run the prompt
+            response = interpreter.chat(prompt)
+            print(response)
+            return 0
+        finally:
+            if zero_fs:
+                os.chdir(orig_cwd)
+                shutil.rmtree(temp_dir, ignore_errors=True)
+
+    except ImportError:
+        # Fallback to simulated execution if interpreter is not installed
+        _logger.warning("[PAL:open_interpreter] ⚠️ Open Interpreter library not installed. Running simulated sandbox.")
+        return run_simulated_execution(prompt, clm_root, payload)
+
+def interpolate_prompt(prompt_template, clm_file, payload):
+    context_str = os.environ.get("CLM_MLP_CONTEXT", "{}")
+    try:
+        context = json.loads(context_str)
+    except Exception:
+        context = {}
+
+    flat = {
+        "payload": context.get("context", {}).get("payload", "") or os.environ.get("CLM_PAYLOAD", "") or payload,
+        "clm_payload": context.get("context", {}).get("payload", "") or os.environ.get("CLM_PAYLOAD", "") or payload,
+        "operation": context.get("context", {}).get("operation", ""),
+        "clm_file": context.get("context", {}).get("clm_file", "") or clm_file,
+        "a_hash": context.get("context", {}).get("a_hash", ""),
+        "c_hash": context.get("context", {}).get("c_hash", ""),
+    }
+
+    interpolated = prompt_template
+    for k, v in flat.items():
+        interpolated = interpolated.replace("{{" + k + "}}", str(v))
+    return interpolated
+
+def handle_mock_behavior(behavior, prompt, clm_root, payload):
+    _logger.info(f"[PAL:open_interpreter] Mocking behavior: {behavior}")
+    if behavior == "success" or behavior == "react_success":
+        # Simulates successful file creation/command execution
+        match = re.search(r"create a file named (\S+) containing ['\"](.*?)['\"]", prompt, re.IGNORECASE)
+        if match:
+            filename, content = match.groups()
+            filepath = os.path.join(clm_root, filename)
+            with open(filepath, "w", encoding="utf-8") as f:
+                f.write(content)
+            print(f"Created file {filename} with content: {content}")
+        else:
+            print(json.dumps({"result": "success", "detail": "simulated action completed"}))
+        return 0
+    elif behavior == "failure" or behavior == "react_unauthorized":
+        _logger.info("[PAL:open_interpreter] Mocking execution failure / unauthorized tool access")
+        return 1
+    elif behavior == "malformed" or behavior == "react_infinite":
+        _logger.info("[PAL:open_interpreter] Mocking malformed output / infinite loop bounding")
+        return 2
+    else:
+        _logger.info(f"[PAL:open_interpreter] Unknown mock behavior {behavior}")
+        return 1
+
+def run_simulated_execution(prompt, clm_root, payload):
+    match = re.search(r"create a file named (\S+) containing ['\"](.*?)['\"]", prompt, re.IGNORECASE)
+    if match:
+        filename, content = match.groups()
+        filepath = os.path.join(clm_root, filename)
+        try:
+            with open(filepath, "w", encoding="utf-8") as f:
+                f.write(content)
+            print(f"Created file {filename} with content: {content}")
+            return 0
+        except Exception as e:
+            _logger.error(f"[PAL:open_interpreter] Simulation file write failed: {e}")
+            return 1
+
+    print(json.dumps({"result": "simulated_success", "payload": payload}))
+    return 0
+
+def main_cli():
+    if len(sys.argv) < 2:
+        _logger.info("Usage: open_interpreter_adapter.py <clm_file> [entry_point] [payload]")
+        return 1
+
+    clm_file = sys.argv[1]
+    sys.argv[2] if len(sys.argv) > 2 else ""
+    payload = sys.argv[3] if len(sys.argv) > 3 else ""
+
+    clm_root = os.environ.get("CLM_ROOT") or os.getcwd()
+
+    from clm.layer0.algebra import load_normalized_manifest
+    try:
+        yaml_data = load_normalized_manifest(clm_root, clm_file)
+    except Exception as e:
+        _logger.info(f"Failed to load CLM YAML: {e}")
+        return 1
+
+    inner = yaml_data.get("clm", yaml_data)
+    concrete = inner.get("concrete_impl", {})
+    process = concrete.get("process", {})
+
+    block = (process[0] if process else {}) if isinstance(process, list) else process
+
+    return evaluate_open_interpreter(clm_root, clm_file, block, [payload] if payload else [])

clm/layer3/rag/__init__.py

@@ -0,0 +1,70 @@
+"""
+MCard RAG (Retrieval-Augmented Generation) Package
+
+Provides vector search, semantic retrieval, and knowledge graph capabilities
+for MCard content using SQLite with sqlite-vec extension and Ollama embeddings.
+"""
+
+from .config import DEFAULT_RAG_CONFIG, RAGConfig
+from .engine import MCardRAGEngine, RAGResponse
+from .graph import (
+    Entity,
+    GraphExtractor,
+    GraphRAGEngine,
+    GraphRAGResponse,
+    GraphStore,
+    Relationship,
+)
+from .indexer import PersistentIndexer, get_indexer, index_mcard, semantic_search
+from .semantic_versioning import (
+    compare_versions_by_similarity,
+    find_most_similar_version,
+    get_handle_version_history,
+    get_semantic_evolution,
+    get_store,
+    get_store_info,
+    get_version_distances,
+    link_mcard_to_handle,
+    list_handles,
+    search_within_handle,
+)
+from .vector import (
+    HandleVectorStore,
+    HandleVersion,
+    VersionSimilarityResult,
+)
+
+__all__ = [
+    # Config
+    "RAGConfig",
+    "DEFAULT_RAG_CONFIG",
+    # Vector RAG
+    "MCardRAGEngine",
+    "RAGResponse",
+    # Persistent Indexer
+    "PersistentIndexer",
+    "get_indexer",
+    "semantic_search",
+    "index_mcard",
+    # GraphRAG
+    "GraphRAGEngine",
+    "GraphRAGResponse",
+    "GraphExtractor",
+    "GraphStore",
+    "Entity",
+    "Relationship",
+    # Semantic Versioning
+    "link_mcard_to_handle",
+    "get_handle_version_history",
+    "compare_versions_by_similarity",
+    "search_within_handle",
+    "get_version_distances",
+    "find_most_similar_version",
+    "get_semantic_evolution",
+    "list_handles",
+    "get_store_info",
+    "get_store",
+    "HandleVectorStore",
+    "HandleVersion",
+    "VersionSimilarityResult",
+]