vayrm 0.1.0__py3-none-any.whl

acr/__init__.py

@@ -0,0 +1,29 @@
+"""
+Adaptive Computing Resource (ACR)
+=================================
+
+ACR is a platform-agnostic, distributed computing runtime for heterogeneous clusters.
+It provides node intelligence, cluster knowledge exchange, adaptive scheduling,
+and closed-loop execution feedback.
+"""
+
+from acr.facade import ACRNode, ACRRoot
+from acr.workloads import Workload
+from acr.schedulers.adaptive import AdaptiveScheduler
+from acr.schedulers.decision import SchedulingDecision
+from acr.execution.manager import ExecutionManager
+from acr.execution.models import ExecutionAttempt
+from acr.communication.protocol import ExecutionObservationMessage as ExecutionObservation
+from acr.resources.snapshot import NodeSnapshot
+
+__all__ = [
+    "ACRNode",
+    "ACRRoot",
+    "Workload",
+    "AdaptiveScheduler",
+    "SchedulingDecision",
+    "ExecutionManager",
+    "ExecutionAttempt",
+    "ExecutionObservation",
+    "NodeSnapshot",
+]

acr/api/root_api.py

@@ -0,0 +1,87 @@
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+from acr.node.knowledge import NodeKnowledge
+
+@dataclass
+class WorkerRecord:
+    worker_key: str
+    endpoint: Optional[Dict]
+    paladin_node_id: str
+    session_id: Optional[str]
+    connection: str
+    connection_status: str
+    telemetry: Optional[dict]
+    knowledge: Optional[NodeKnowledge]
+
+class RootDashboardAPI:
+    """
+    API for retrieving WorkerRecords for the dashboard.
+    """
+    def __init__(self, root_brain, control_plane):
+        self._brain = root_brain
+        self._cp = control_plane
+
+    def get_workers(self) -> List[WorkerRecord]:
+        workers = []
+        all_k = self._brain.all_node_knowledge()
+        tracker = getattr(self._cp, "telemetry_tracker", None)
+        
+        # We also want to include nodes that the tracker knows about but have no knowledge yet
+        worker_keys = set(all_k.keys())
+        if tracker:
+            worker_keys.update(rec.node_id for rec in tracker.all_records())
+            
+        for worker_key in worker_keys:
+            # worker_key is usually "{paladin_node_id}_{session_id}"
+            if "_" in worker_key:
+                paladin_node_id, session_id = worker_key.split("_", 1)
+            else:
+                paladin_node_id, session_id = worker_key, None
+                
+            k = all_k.get(worker_key)
+            ep = None
+            conn_status = "UNKNOWN"
+            tel = None
+            
+            if tracker:
+                rec = tracker.get_record(worker_key)
+                if rec:
+                    tel = rec.__dict__
+                    if rec.current_mesh_ip:
+                        ep = {
+                            "mesh_ip": rec.current_mesh_ip,
+                            "service_port": rec.current_service_port
+                        }
+                    conn_status = rec.connection_status
+                    
+            if not ep and hasattr(self._cp, "get_node_endpoint"):
+                # fallback to paladin directly if tracker missed it
+                ep_data = self._cp.get_node_endpoint(paladin_node_id)
+                if ep_data:
+                    ep = {
+                        "mesh_ip": ep_data.get("mesh_ip"),
+                        "service_port": ep_data.get("service_port")
+                    }
+                    if conn_status == "UNKNOWN":
+                        conn_status = ep_data.get("connection_status", "UNKNOWN")
+            
+            workers.append(WorkerRecord(
+                worker_key=worker_key,
+                endpoint=ep,
+                paladin_node_id=paladin_node_id,
+                session_id=session_id,
+                connection="PALADIN",
+                connection_status=conn_status,
+                telemetry=tel,
+                knowledge=k
+            ))
+        
+        # Sort so order is stable
+        workers.sort(key=lambda w: w.worker_key)
+        return workers
+
+    def get_worker(self, worker_key: str) -> Optional[WorkerRecord]:
+        for w in self.get_workers():
+            if w.worker_key == worker_key:
+                return w
+        return None

acr/api/server.py

@@ -0,0 +1,70 @@
+"""
+acr/api/server.py
+FastAPI server exposing real-time state of the NodeBrain and WorkloadBrain.
+"""
+from fastapi import FastAPI, HTTPException
+import uuid
+import threading
+
+from acr.runtime.node_brain import NodeBrain
+from acr.runtime.workload_brain import WorkloadBrain
+from acr.runtime.monitoring import RuntimeMonitor
+from acr.workloads.pytorch.adapter import PyTorchWorkloadAdapter
+from acr.workloads.pytorch.checkpoint import PyTorchCheckpointer
+from acr.workloads.pytorch.runtime import run_training
+
+app = FastAPI(title="ACR Live Runtime API", version="1.0.0")
+
+# Global instances for the runtime
+run_id = str(uuid.uuid4())
+node_brain = NodeBrain(node_id="macos_node_1")
+workload_brain = WorkloadBrain(workload_id="pytorch_resnet_1")
+monitor = RuntimeMonitor(node_brain, workload_brain, run_id=run_id, interval_s=1.0)
+adapter = PyTorchWorkloadAdapter(workload_brain)
+checkpointer = PyTorchCheckpointer(f"checkpoints/{run_id}")
+
+training_thread = None
+
+@app.on_event("startup")
+def startup_event():
+    print("Starting Continuous Node Monitor...")
+    monitor.start()
+
+@app.on_event("shutdown")
+def shutdown_event():
+    print("Stopping runtime...")
+    monitor.stop()
+    adapter.stop()
+    if training_thread:
+        training_thread.join(timeout=2.0)
+
+@app.get("/api/v1/node")
+def get_node_state():
+    """Returns the current state and trends of the physical node."""
+    return node_brain.get_state_summary()
+
+@app.get("/api/v1/workload")
+def get_workload_state():
+    """Returns the current state and metrics of the running workload."""
+    return workload_brain.get_state_summary()
+
+@app.post("/api/v1/workload/start")
+def start_workload(device: str = "cpu", batch_size: int = 64):
+    """Start the real PyTorch workload in a background thread."""
+    global training_thread
+    if training_thread and training_thread.is_alive():
+        raise HTTPException(status_code=400, detail="Workload is already running")
+        
+    training_thread = threading.Thread(
+        target=run_training,
+        args=(adapter, checkpointer, device, batch_size),
+        daemon=True
+    )
+    training_thread.start()
+    return {"status": "started", "device": device, "batch_size": batch_size}
+
+@app.post("/api/v1/workload/stop")
+def stop_workload():
+    """Stop the running PyTorch workload."""
+    adapter.running = False
+    return {"status": "stopping"}

acr/cache/__init__.py

@@ -0,0 +1,20 @@
+"""
+acr/cache/__init__.py
+
+Local state cache management for ACR nodes.
+"""
+from acr.cache.policy import (
+    CacheEntry, EvictionPolicy, AdmissionPolicy,
+    LRUEvictionPolicy, LFUEvictionPolicy, SizeAwareEvictionPolicy,
+    AlwaysAdmitPolicy, SizeThresholdPolicy,
+)
+from acr.cache.manager import (
+    CacheTier, CacheEventType, CacheEvent, CacheStats, CacheManager,
+)
+
+__all__ = [
+    "CacheEntry", "EvictionPolicy", "AdmissionPolicy",
+    "LRUEvictionPolicy", "LFUEvictionPolicy", "SizeAwareEvictionPolicy",
+    "AlwaysAdmitPolicy", "SizeThresholdPolicy",
+    "CacheTier", "CacheEventType", "CacheEvent", "CacheStats", "CacheManager",
+]

acr/cache/manager.py

@@ -0,0 +1,394 @@
+"""
+acr/cache/manager.py
+
+CacheManager: local state cache for an ACR node.
+
+The CacheManager sits between the storage backend and the scheduler.
+It answers: "Is this chunk available locally? Can I serve it without a
+remote transfer?"
+
+Responsibilities
+----------------
+- Admission: decide whether to accept a new chunk into the cache
+- Eviction: free space when pressure requires it
+- Promotion: move a chunk to a higher-priority tier
+- Demotion: move a chunk to a lower-priority tier
+- Prefetch: proactively fetch chunks expected to be needed soon
+- Residency tracking: what chunks are cached and at what tier
+- Hit/miss accounting: maintain statistics for scheduling decisions
+
+Memory hierarchy (conceptual, represented as priority tiers)
+-------------------------------------------------------------
+    Tier 0: GPU / accelerator memory  (highest cost to reload)
+    Tier 1: RAM (system memory)
+    Tier 2: Local persistent storage
+    Tier 3: Remote (not cached — fetch required)
+
+The CacheManager tracks tiers 0–2. Tier 3 means "not here".
+The storage backend holds the actual bytes for tiers 1–2.
+GPU residency is metadata-only (ACR does not manage GPU memory directly).
+
+Design rules
+------------
+- CacheManager depends on StorageBackend (injected).
+- CacheManager depends on EvictionPolicy and AdmissionPolicy (injected).
+- All operations produce structured CacheEvent records.
+- Cache hits/misses are counted per chunk_id.
+- capacity_bytes is a soft limit: eviction is triggered when exceeded.
+- Eviction frees at least one slot; it may free more if under severe pressure.
+"""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+
+from acr.cache.policy import (
+    AdmissionPolicy, AlwaysAdmitPolicy, CacheEntry,
+    EvictionPolicy, LRUEvictionPolicy,
+)
+from acr.storage.backend import StorageBackend
+
+
+class CacheTier(Enum):
+    """
+    Conceptual memory tier.
+
+    GPU     : GPU / accelerator memory (fastest; metadata-only in ACR)
+    RAM     : System RAM (fast; managed via storage backend)
+    STORAGE : Local persistent storage (slower; managed via storage backend)
+    REMOTE  : Not locally cached
+    """
+    GPU     = 0
+    RAM     = 1
+    STORAGE = 2
+    REMOTE  = 3
+
+
+@dataclass
+class CacheEventType(Enum):
+    HIT       = "hit"
+    MISS      = "miss"
+    ADMITTED  = "admitted"
+    EVICTED   = "evicted"
+    PROMOTED  = "promoted"
+    DEMOTED   = "demoted"
+    REJECTED  = "rejected"    # admission policy said no
+    PREFETCH  = "prefetch"
+
+
+@dataclass
+class CacheEvent:
+    """A structured event emitted by the CacheManager for observability."""
+    event_type: CacheEventType
+    chunk_id: str
+    size_bytes: int | None
+    tier: CacheTier | None
+    timestamp: float = field(default_factory=time.monotonic)
+    reason: str = ""
+
+
+@dataclass
+class CacheStats:
+    """Accumulated hit/miss/eviction statistics."""
+    hits: int = 0
+    misses: int = 0
+    admissions: int = 0
+    evictions: int = 0
+    rejections: int = 0
+    promotions: int = 0
+    demotions: int = 0
+    bytes_evicted: int = 0
+    bytes_admitted: int = 0
+
+    @property
+    def hit_rate(self) -> float | None:
+        total = self.hits + self.misses
+        return self.hits / total if total > 0 else None
+
+    def to_dict(self) -> dict:
+        return {
+            "hits": self.hits, "misses": self.misses,
+            "admissions": self.admissions, "evictions": self.evictions,
+            "rejections": self.rejections,
+            "bytes_evicted": self.bytes_evicted,
+            "bytes_admitted": self.bytes_admitted,
+            "hit_rate": self.hit_rate,
+        }
+
+
+class CacheManager:
+    """
+    Local state cache for one ACR node.
+
+    Parameters
+    ----------
+    storage       : the storage backend that holds actual bytes
+    capacity_bytes: soft limit on total cache size; None = unlimited
+    eviction_policy  : which entry to evict when over capacity
+    admission_policy : whether to admit a new entry
+    default_tier  : which tier admitted entries land in (RAM or STORAGE)
+    max_events    : how many CacheEvents to retain in the event log
+    """
+
+    def __init__(
+        self,
+        storage: StorageBackend,
+        capacity_bytes: int | None = None,
+        eviction_policy: EvictionPolicy | None = None,
+        admission_policy: AdmissionPolicy | None = None,
+        default_tier: CacheTier = CacheTier.RAM,
+        max_events: int = 1000,
+    ) -> None:
+        self._storage = storage
+        self.capacity_bytes = capacity_bytes
+        self._eviction = eviction_policy or LRUEvictionPolicy()
+        self._admission = admission_policy or AlwaysAdmitPolicy()
+        self._default_tier = default_tier
+        self._max_events = max_events
+
+        # chunk_id → CacheEntry
+        self._entries: dict[str, CacheEntry] = {}
+        # chunk_id → CacheTier
+        self._tiers: dict[str, CacheTier] = {}
+        # Accumulated stats
+        self._stats = CacheStats()
+        # Event log (bounded)
+        self._events: list[CacheEvent] = []
+
+    # ------------------------------------------------------------------
+    # Core operations
+    # ------------------------------------------------------------------
+
+    def put(self, chunk_id: str, data: bytes, priority: int = 0) -> bool:
+        """
+        Admit a chunk into the cache.
+
+        Returns True if admitted, False if rejected by admission policy.
+        If the cache is over capacity after admission, eviction is triggered.
+
+        The data is written to the storage backend on admission.
+        """
+        size = len(data)
+
+        # Admission check
+        if not self._admission.should_admit(chunk_id, size):
+            self._stats.rejections += 1
+            self._emit(CacheEventType.REJECTED, chunk_id, size, None, "admission_policy")
+            return False
+
+        # If already present, update data and refresh entry
+        now = time.monotonic()
+        if chunk_id in self._entries:
+            old_size = self._entries[chunk_id].size_bytes
+            self._entries[chunk_id].size_bytes = size
+            self._eviction.on_access(self._entries[chunk_id], now)
+            key = self._storage_key(chunk_id)
+            self._storage.write(key, data)
+            self._stats.bytes_admitted += size - old_size
+            return True
+
+        # Write to storage backend
+        key = self._storage_key(chunk_id)
+        self._storage.write(key, data)
+
+        entry = CacheEntry(
+            chunk_id=chunk_id, size_bytes=size,
+            priority=priority, inserted_at=now,
+        )
+        self._eviction.on_insert(entry, now)
+        self._entries[chunk_id] = entry
+        self._tiers[chunk_id] = self._default_tier
+
+        self._stats.admissions += 1
+        self._stats.bytes_admitted += size
+        self._emit(CacheEventType.ADMITTED, chunk_id, size, self._default_tier)
+
+        # Evict if over capacity
+        self._enforce_capacity()
+        return True
+
+    def get(self, chunk_id: str) -> bytes | None:
+        """
+        Retrieve cached chunk bytes.
+        Returns None on cache miss (does NOT raise).
+        Records hit or miss in stats.
+        """
+        if chunk_id not in self._entries:
+            self._stats.misses += 1
+            self._emit(CacheEventType.MISS, chunk_id, None, None)
+            return None
+
+        entry = self._entries[chunk_id]
+        now = time.monotonic()
+        self._eviction.on_access(entry, now)
+        self._stats.hits += 1
+        tier = self._tiers.get(chunk_id, self._default_tier)
+        self._emit(CacheEventType.HIT, chunk_id, entry.size_bytes, tier)
+
+        key = self._storage_key(chunk_id)
+        try:
+            return self._storage.read(key)
+        except KeyError:
+            # Metadata says cached but bytes gone — treat as miss, clean up
+            self._remove_entry(chunk_id)
+            self._stats.hits -= 1
+            self._stats.misses += 1
+            return None
+
+    def contains(self, chunk_id: str) -> bool:
+        """Return True if chunk_id is in the cache (metadata check only)."""
+        return chunk_id in self._entries
+
+    def evict(self, chunk_id: str) -> bool:
+        """
+        Explicitly evict a specific chunk.
+        Returns True if it was in the cache, False otherwise.
+        """
+        if chunk_id not in self._entries:
+            return False
+        size = self._entries[chunk_id].size_bytes
+        tier = self._tiers.get(chunk_id)
+        self._remove_entry(chunk_id)
+        self._stats.evictions += 1
+        self._stats.bytes_evicted += size
+        self._emit(CacheEventType.EVICTED, chunk_id, size, tier, "explicit_eviction")
+        return True
+
+    def promote(self, chunk_id: str, target_tier: CacheTier) -> bool:
+        """
+        Promote a chunk to a higher (lower-number) tier.
+        Returns False if chunk not in cache or already at target tier.
+        GPU promotions are metadata-only (bytes stay in storage).
+        """
+        if chunk_id not in self._entries:
+            return False
+        current = self._tiers.get(chunk_id, self._default_tier)
+        if target_tier.value >= current.value:
+            return False  # not a promotion
+        self._tiers[chunk_id] = target_tier
+        self._stats.promotions += 1
+        size = self._entries[chunk_id].size_bytes
+        self._emit(CacheEventType.PROMOTED, chunk_id, size, target_tier,
+                   f"{current.name}→{target_tier.name}")
+        return True
+
+    def demote(self, chunk_id: str, target_tier: CacheTier) -> bool:
+        """
+        Demote a chunk to a lower (higher-number) tier.
+        Returns False if chunk not in cache or already at/below target.
+        """
+        if chunk_id not in self._entries:
+            return False
+        current = self._tiers.get(chunk_id, self._default_tier)
+        if target_tier.value <= current.value:
+            return False  # not a demotion
+        self._tiers[chunk_id] = target_tier
+        self._stats.demotions += 1
+        size = self._entries[chunk_id].size_bytes
+        self._emit(CacheEventType.DEMOTED, chunk_id, size, target_tier,
+                   f"{current.name}→{target_tier.name}")
+        return True
+
+    def prefetch(self, chunk_id: str, data: bytes, priority: int = 1) -> bool:
+        """
+        Proactively admit a chunk with a standard priority boost.
+        Delegates to put() with a structured event.
+        """
+        result = self.put(chunk_id, data, priority=priority)
+        if result:
+            # Relabel last admitted event as prefetch
+            self._emit(CacheEventType.PREFETCH, chunk_id, len(data), self._default_tier)
+        return result
+
+    # ------------------------------------------------------------------
+    # Pressure and capacity
+    # ------------------------------------------------------------------
+
+    def current_bytes(self) -> int:
+        """Total bytes in cache (from entry metadata, not storage scan)."""
+        return sum(e.size_bytes for e in self._entries.values())
+
+    def pressure(self) -> float | None:
+        """
+        Cache pressure (0.0–1.0).
+        Returns None if no capacity limit is set.
+        """
+        if self.capacity_bytes is None:
+            return None
+        return min(1.0, self.current_bytes() / self.capacity_bytes)
+
+    def entry_count(self) -> int:
+        return len(self._entries)
+
+    def cached_chunk_ids(self) -> set[str]:
+        return set(self._entries.keys())
+
+    def chunks_at_tier(self, tier: CacheTier) -> list[str]:
+        return [cid for cid, t in self._tiers.items() if t == tier]
+
+    def get_tier(self, chunk_id: str) -> CacheTier:
+        """Return the tier of a cached chunk, or REMOTE if not cached."""
+        return self._tiers.get(chunk_id, CacheTier.REMOTE)
+
+    # ------------------------------------------------------------------
+    # Stats and observability
+    # ------------------------------------------------------------------
+
+    @property
+    def stats(self) -> CacheStats:
+        return self._stats
+
+    def recent_events(self, n: int = 50) -> list[CacheEvent]:
+        return self._events[-n:]
+
+    def reset_stats(self) -> None:
+        self._stats = CacheStats()
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    def _storage_key(self, chunk_id: str) -> str:
+        return f"cache/{chunk_id}"
+
+    def _remove_entry(self, chunk_id: str) -> None:
+        """Remove entry metadata and backing storage."""
+        self._entries.pop(chunk_id, None)
+        self._tiers.pop(chunk_id, None)
+        key = self._storage_key(chunk_id)
+        self._storage.delete(key)
+
+    def _enforce_capacity(self) -> None:
+        """Evict entries until we are within capacity, or cache is empty."""
+        if self.capacity_bytes is None:
+            return
+        while self.current_bytes() > self.capacity_bytes and self._entries:
+            victim_id = self._eviction.choose_victim(self._entries)
+            if victim_id is None:
+                break
+            size = self._entries[victim_id].size_bytes
+            tier = self._tiers.get(victim_id)
+            self._remove_entry(victim_id)
+            self._stats.evictions += 1
+            self._stats.bytes_evicted += size
+            self._emit(CacheEventType.EVICTED, victim_id, size, tier, "capacity_pressure")
+
+    def _emit(
+        self,
+        event_type: CacheEventType,
+        chunk_id: str,
+        size_bytes: int | None,
+        tier: CacheTier | None,
+        reason: str = "",
+    ) -> None:
+        event = CacheEvent(
+            event_type=event_type,
+            chunk_id=chunk_id,
+            size_bytes=size_bytes,
+            tier=tier,
+            reason=reason,
+        )
+        self._events.append(event)
+        if len(self._events) > self._max_events:
+            self._events = self._events[-self._max_events:]