fleetmesh-node 0.2.0__py3-none-any.whl

node/planes/services/compute/jobs.py

@@ -0,0 +1,227 @@
+"""Execution dispatcher (run_job) and kind 30811 job receipts.
+
+Separate because this is the seam between executors and the wire/subsystem layers: compute/wire imports it without needing the Compute subsystem.
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+from typing import Any, Callable, Mapping, Optional
+
+
+from .types import ComputeReceiptPayload
+
+from fleetmesh_core.crypto import crypto, nip44
+from fleetmesh_core.identity import identity
+from fleetmesh_core.storage import store
+from fleetmesh_core.wire.interfaces import NostrEvent
+from .common import ComputeError, RECEIPT_KIND, DEFAULT_EXPIRATION, resolve_blob
+from .sandbox import ComputeLimits, clamp_limits, run_sandboxed
+from .executors import (
+    DEFAULT_REGISTRY,
+    ExecutorRegistry,
+    FfmpegTranscodeExecutor,
+    JobRunResult,
+    WasmExecutor,
+    _wasm_engine,
+    wasmtime_available,
+)
+from .executors.media import NODE_ONLY_INPUT_KEYS, requester_input
+
+# -------------------------------------------------------------------------
+# Execution Dispatcher
+# -------------------------------------------------------------------------
+def run_job(
+    job_payload: Mapping[str, Any],
+    custom_runner: Optional[Callable[[Mapping[str, Any]], bytes]] = None,
+    registry: Optional[ExecutorRegistry] = None,
+    limits: Optional[ComputeLimits] = None,
+    *,
+    allow_unsandboxed: bool = False,
+) -> JobRunResult:
+    """Execute a compute job deterministically or non-deterministically.
+
+    ``allow_unsandboxed`` lets the job run in-process on a host without
+    fork/rlimits (macOS spawn-only, Windows). Off by default: an operator who
+    sells compute must choose that explicitly rather than discover it.
+    """
+    fuel_max = int(job_payload.get('fuel_max', 1000000))
+    job_limits = clamp_limits(limits or ComputeLimits(
+        max_fuel=fuel_max,
+        max_memory_bytes=int(job_payload.get('max_memory_bytes', 1073741824)),
+        timeout_seconds=int(job_payload.get('timeout_seconds', 60)),
+    ))
+
+    # Blob stores hold sqlite handles/locks; keep them out of the forked child.
+    child_payload = {k: v for k, v in job_payload.items() if k != 'blob_store'}
+    # Executors size their own memory caps (wasm linear memory) under this.
+    child_payload['_memory_ceiling_bytes'] = job_limits.max_memory_bytes
+
+    if custom_runner is not None:
+        raw_output = run_sandboxed(lambda: custom_runner(child_payload), job_limits,
+                                   allow_unsandboxed=allow_unsandboxed)
+        # Compare the unclamped usage; clamping first made the check dead.
+        fuel_used = len(raw_output) * 10
+        if fuel_used > fuel_max:
+            raise ComputeError('job execution exceeded fuel ceiling (%d > %d)' % (fuel_used, fuel_max))
+        result_hash = crypto.sha256(raw_output)
+        return {
+            'result_hash': result_hash,
+            'raw_output': raw_output,
+            'fuel_used': fuel_used,
+            'engine': 'custom',
+            'wasi_profile_digest': '',
+            'counters': {},
+            'verification_mode': 'delivery',
+        }
+
+    reg = registry or DEFAULT_REGISTRY
+    executor = reg.select_for_job(child_payload)
+    bstore = job_payload.get('blob_store') or getattr(reg, 'blob_store', None)
+    if isinstance(executor, WasmExecutor):
+        # Load wasmtime's native library and build the engine in the parent:
+        # after fork the child runs under RLIMIT_AS, where mapping a ~20 MB
+        # shared object (and ctypes) can fail. The forked child inherits both.
+        if wasmtime_available():
+            _wasm_engine()
+        # Resolve module/input here: the blob store (sqlite handles) is not
+        # passed into the forked child, so the child could never read a
+        # module stored on this node. Pass the verified bytes instead.
+        for key in ('module', 'input'):
+            spec = job_payload.get(key)
+            if isinstance(spec, Mapping) and 'raw' not in spec:
+                data = resolve_blob(spec, blob_store=bstore,
+                                    blossom_servers=job_payload.get('blossom_servers'))
+                if data is not None:
+                    child_payload[key] = dict(spec, raw=data)
+    elif isinstance(executor, FfmpegTranscodeExecutor):
+        # The requester is remote: drop the fields only the node may set,
+        # then add this node's own source_path from its blob store.
+        for k in NODE_ONLY_INPUT_KEYS:
+            child_payload.pop(k, None)
+        inp = job_payload.get('input')
+        if isinstance(inp, Mapping):
+            inp = requester_input(inp)
+            child_payload['input'] = inp
+        if isinstance(inp, Mapping) and 'raw' not in inp:
+            src_sha = str(inp.get('source_sha256') or inp.get('sha256') or '').strip().lower()
+            if bstore is not None and len(src_sha) == 64:
+                p = None
+                if hasattr(bstore, 'get_blob_path'):
+                    p = bstore.get_blob_path(src_sha)
+                elif hasattr(bstore, '_blob_file_path'):
+                    p = bstore._blob_file_path(src_sha)
+                if p and os.path.isfile(p):
+                    child_payload['input'] = dict(inp, source_path=p)
+                else:
+                    data = (
+                        bstore.get_blob(src_sha)
+                        if hasattr(bstore, 'get_blob')
+                        else resolve_blob({'sha256': src_sha}, blob_store=bstore, blossom_servers=job_payload.get('blossom_servers'))
+                    )
+                    if data is not None:
+                        child_payload['input'] = dict(inp, raw=data)
+
+    if isinstance(executor, FfmpegTranscodeExecutor):
+        # FfmpegTranscodeExecutor spawns its own isolated `ffmpeg` subprocess
+        # with explicit timeout and nice priority; running directly avoids
+        # RLIMIT_AS breaking dynamic libavcodec mappings on Linux hosts.
+        if bstore is not None:
+            child_payload['_blob_store'] = bstore
+        res = executor.execute(child_payload, fuel_max)
+    else:
+        res = run_sandboxed(lambda: executor.execute(child_payload, fuel_max), job_limits,
+                            allow_unsandboxed=allow_unsandboxed)
+    blob_store = job_payload.get('blob_store')
+    result_cid = None
+    if blob_store is not None and res.raw_output and hasattr(blob_store, 'put') and not hasattr(blob_store, 'store_blob'):
+        desc = blob_store.put(res.raw_output)
+        result_cid = desc.get('cid') if isinstance(desc, dict) else str(desc)
+    else:
+        from node.protocols.nostr import blobstore
+        result_cid = blobstore.sha256_to_cid(res.result_hash)
+
+    return {
+        'result_hash': res.result_hash,
+        'result_cid': result_cid,
+        'raw_output': res.raw_output,
+        'fuel_used': res.fuel_used,
+        'engine': res.engine_id,
+        'wasi_profile_digest': res.wasi_profile_digest,
+        'counters': dict(res.counters),
+        'verification_mode': res.verification_mode,
+    }
+
+
+# -------------------------------------------------------------------------
+# Kind 30811: Compute Receipt
+# -------------------------------------------------------------------------
+def build_receipt(
+    seller_sk: str,
+    buyer_pk: str,
+    receipt_payload: Mapping[str, Any],
+    blinding: str,
+    expiration: Optional[int] = None,
+    now: Optional[int] = None,
+) -> NostrEvent:
+    """Build a signed kind 30811 compute job receipt event.
+
+    Encrypted under NIP-44 conversation key to the buyer, tagged with the
+    blinded job identifier.
+    """
+    now_val = int(now if now is not None else time.time())
+    exp_val = int(expiration if expiration is not None else now_val + DEFAULT_EXPIRATION)
+
+    d = identity.job_tag(seller_sk, buyer_pk)
+    canon_payload = crypto.canonical(receipt_payload)
+    commit = crypto.commit(canon_payload, blinding)
+
+    conv = nip44.conversation_key(seller_sk, buyer_pk)
+    ciphertext = nip44.encrypt(canon_payload.decode('utf-8'), conv)
+
+    tags: list[list[str]] = [
+        ['d', d],
+        ['commit', commit],
+        ['scheme', 'sha256-blind/1', 'nip44/2'],
+        ['expiration', str(exp_val)],
+    ]
+
+    ev = store.make_signed_event(seller_sk, now_val, RECEIPT_KIND, tags, ciphertext)
+    return ev
+
+
+def open_receipt(receipt: NostrEvent | Mapping[str, Any], buyer_sk: str) -> ComputeReceiptPayload:
+    """Decrypt and verify a kind 30811 compute job receipt."""
+    if receipt.get('kind') != RECEIPT_KIND:
+        raise ComputeError('not a kind 30811')
+
+    issuer = receipt.get('pubkey')
+    if not issuer:
+        raise ComputeError('receipt has no pubkey')
+
+    tags = {t[0]: t for t in receipt.get('tags', []) if t and isinstance(t, (list, tuple))}
+    d_tag = tags.get('d')
+    d_val = d_tag[1] if d_tag and len(d_tag) > 1 else None
+
+    if not d_val or d_val != identity.job_tag(buyer_sk, issuer):
+        raise ComputeError('d tag is not the blinded job identifier derived with this seller')
+
+    try:
+        conv = nip44.conversation_key(buyer_sk, issuer)
+        plain = nip44.decrypt(receipt.get('content', ''), conv)
+        payload = json.loads(plain)
+    except Exception as e:
+        raise ComputeError('failed to decrypt receipt ciphertext: %s' % (e,))
+
+    commit_tag = tags.get('commit')
+    commit_val = commit_tag[1] if commit_tag and len(commit_tag) > 1 else None
+    blinding = payload.get('blinding')
+    if not blinding:
+        # Fallback to empty if absent from payload
+        blinding = ''
+
+    if not commit_val or not crypto.open_commit(commit_val, crypto.canonical(payload), blinding):
+        raise ComputeError('commitment does not open to decrypted receipt payload')
+
+    return payload

node/planes/services/compute/load.py

@@ -0,0 +1,180 @@
+"""ComputeLoadMonitor: host CPU utilization sensing, worker slot accounting, and job reflection decisions."""
+from __future__ import annotations
+
+import contextlib
+import os
+import threading
+import time
+from typing import Callable, Iterator, Optional, TypedDict
+
+DEFAULT_CPU_REFLECT_CEILING: float = 0.75
+
+
+class ComputeLoadSnapshot(TypedDict):
+    cpu_utilization_ratio: float
+    cpu_load_pct: int
+    cpu_ceiling: float
+    active_jobs: int
+    max_concurrent_jobs: int
+    free_slots: int
+    should_reflect: bool
+    reflect_reason: str
+    jobs_reflected: int
+    total_fuel_used: int
+
+
+def _read_proc_stat_jiffies() -> Optional[tuple[int, int]]:
+    """Read `(idle_jiffies, total_jiffies)` from `/proc/stat` on Linux hosts."""
+    try:
+        with open('/proc/stat', 'r', encoding='utf-8') as f:
+            first_line = f.readline()
+        if not first_line.startswith('cpu '):
+            return None
+        parts = [int(x) for x in first_line.split()[1:]]
+        if len(parts) < 4:
+            return None
+        # user, nice, system, idle, iowait, irq, softirq, steal
+        idle = parts[3] + (parts[4] if len(parts) > 4 else 0)
+        total = sum(parts[:8])
+        return idle, total
+    except (OSError, ValueError, IndexError):
+        return None
+
+
+class ComputeLoadMonitor:
+    """Thread-safe CPU load sensor and active compute job slot tracker for a meshnode."""
+
+    def __init__(
+        self,
+        *,
+        max_concurrent_jobs: Optional[int] = None,
+        cpu_ceiling: float = DEFAULT_CPU_REFLECT_CEILING,
+        cpu_sampler: Optional[Callable[[], float]] = None,
+    ) -> None:
+        cores = max(1, os.cpu_count() or 2)
+        self.max_concurrent_jobs: int = max(1, int(max_concurrent_jobs if max_concurrent_jobs is not None else max(1, cores // 2)))
+        self.cpu_ceiling: float = max(0.05, min(0.99, float(cpu_ceiling)))
+        self._cpu_sampler: Optional[Callable[[], float]] = cpu_sampler
+        self._lock: threading.Lock = threading.Lock()
+        self.active_jobs: int = 0
+        self.jobs_reflected: int = 0
+        self.total_fuel_used: int = 0
+        self._last_sample_at: float = 0.0
+        self._last_cpu_ratio: float = 0.0
+        self._prev_jiffies: Optional[tuple[int, int]] = _read_proc_stat_jiffies()
+
+    def set_cpu_sampler(self, sampler: Optional[Callable[[], float]]) -> None:
+        with self._lock:
+            self._cpu_sampler = sampler
+
+    def cpu_utilization_ratio(self) -> float:
+        """Return current host CPU utilization ratio in `[0.0, 1.0]`."""
+        with self._lock:
+            if self._cpu_sampler is not None:
+                try:
+                    return max(0.0, min(1.0, float(self._cpu_sampler())))
+                except Exception:
+                    return 0.0
+
+            now = time.monotonic()
+            if now - self._last_sample_at < 0.75 and self._last_sample_at > 0.0:
+                return self._last_cpu_ratio
+
+            cur_jiffies = _read_proc_stat_jiffies()
+            if cur_jiffies is not None:
+                prev = self._prev_jiffies
+                self._prev_jiffies = cur_jiffies
+                if prev is not None:
+                    d_idle = cur_jiffies[0] - prev[0]
+                    d_total = cur_jiffies[1] - prev[1]
+                    if d_total > 0:
+                        ratio = max(0.0, min(1.0, 1.0 - (float(d_idle) / float(d_total))))
+                        self._last_cpu_ratio = round(ratio, 4)
+                        self._last_sample_at = now
+                        return self._last_cpu_ratio
+
+            if hasattr(os, 'getloadavg'):
+                try:
+                    load_1m = float(os.getloadavg()[0])
+                except OSError:
+                    self._last_sample_at = now
+                    return self._last_cpu_ratio
+                cores = max(1, os.cpu_count() or 1)
+                ratio = max(0.0, min(1.0, load_1m / float(cores)))
+                self._last_cpu_ratio = round(ratio, 4)
+                self._last_sample_at = now
+                return self._last_cpu_ratio
+
+            self._last_sample_at = now
+            return self._last_cpu_ratio
+
+    @property
+    def free_slots(self) -> int:
+        with self._lock:
+            return max(0, self.max_concurrent_jobs - self.active_jobs)
+
+    def should_reflect_job(self, cpu_ceiling: Optional[float] = None) -> tuple[bool, str]:
+        """Return `(should_reflect, reason)` based on active job slots and live CPU load."""
+        ceiling = float(cpu_ceiling if cpu_ceiling is not None else self.cpu_ceiling)
+        with self._lock:
+            active = self.active_jobs
+            max_jobs = self.max_concurrent_jobs
+        if active >= max_jobs:
+            return True, f'slots_full ({active}/{max_jobs})'
+        cpu = self.cpu_utilization_ratio()
+        if cpu >= ceiling:
+            return True, f'high_cpu ({int(round(cpu * 100))}% >= {int(round(ceiling * 100))}%)'
+        return False, 'local_capacity_available'
+
+    @contextlib.contextmanager
+    def job_slot(self) -> Iterator[None]:
+        """Acquire a local compute worker slot for the duration of a job."""
+        with self._lock:
+            self.active_jobs += 1
+        try:
+            yield
+        finally:
+            with self._lock:
+                self.active_jobs = max(0, self.active_jobs - 1)
+
+    def record_fuel(self, fuel_used: int) -> None:
+        with self._lock:
+            self.total_fuel_used += max(0, int(fuel_used))
+
+    def record_reflection(self) -> None:
+        with self._lock:
+            self.jobs_reflected += 1
+
+    def offer_limits(self, max_fuel: int = 50_000_000_000) -> list[list[str]]:
+        """Build Kind 11803 `['limit', ...]` tags advertising live CPU load and free worker slots."""
+        cpu_pct = int(round(self.cpu_utilization_ratio() * 100))
+        return [
+            ['fuel_max', str(int(max_fuel))],
+            ['cpu_load_pct', str(cpu_pct)],
+            ['free_slots', str(self.free_slots)],
+            ['concurrent', str(self.max_concurrent_jobs)],
+        ]
+
+    def snapshot(self) -> ComputeLoadSnapshot:
+        cpu = self.cpu_utilization_ratio()
+        with self._lock:
+            active = self.active_jobs
+            max_j = self.max_concurrent_jobs
+            reflected = self.jobs_reflected
+            fuel = self.total_fuel_used
+        reflect, reason = self.should_reflect_job()
+        return {
+            'cpu_utilization_ratio': round(cpu, 4),
+            'cpu_load_pct': int(round(cpu * 100)),
+            'cpu_ceiling': self.cpu_ceiling,
+            'active_jobs': active,
+            'max_concurrent_jobs': max_j,
+            'free_slots': max(0, max_j - active),
+            'should_reflect': reflect,
+            'reflect_reason': reason,
+            'jobs_reflected': reflected,
+            'total_fuel_used': fuel,
+        }
+
+
+__all__ = ['DEFAULT_CPU_REFLECT_CEILING', 'ComputeLoadMonitor', 'ComputeLoadSnapshot']