devtorch-core 3.0.1__py3-none-any.whl

devtorch_core/cloud/sync_bundle.py

@@ -0,0 +1,603 @@
+"""
+devtorch_core.cloud.sync_bundle
+================================
+Build, compress, chunk, merge, and apply sync bundles for bidirectional
+sync between local .GCC/ and the MCP server.
+
+A sync bundle is a JSON dict containing all 5 data types (commits, theta,
+learnings, sensitivities, topics). Bundles are gzip-compressed and
+base64-encoded for transport through JSON-RPC.
+"""
+from __future__ import annotations
+
+import base64
+import gzip
+import hashlib
+import json
+import logging
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+from devtorch_core.cloud.team_sync import _merge_learnings as _ts_merge_learnings
+
+logger = logging.getLogger("devtorch.cloud.sync_bundle")
+
+DEFAULT_CHUNK_SIZE = 500
+_MAX_SYNCED_IDS_FOR_HASH = 10000
+
+
+def compress(bundle: dict) -> str:
+    """Gzip + base64 compress a bundle for transport."""
+    raw = json.dumps(bundle, separators=(",", ":")).encode("utf-8")
+    compressed = gzip.compress(raw, compresslevel=6)
+    return base64.b64encode(compressed).decode("ascii")
+
+
+def decompress(encoded: str) -> dict:
+    """Decompress a bundle from transport."""
+    compressed = base64.b64decode(encoded)
+    raw = gzip.decompress(compressed)
+    return json.loads(raw.decode("utf-8"))
+
+
+def chunk(bundle: dict, chunk_size: int = DEFAULT_CHUNK_SIZE) -> list[dict]:
+    """Split a bundle into chunks of at most *chunk_size* items per data type."""
+    if chunk_size <= 0:
+        return [bundle]
+
+    chunks: list[dict] = []
+    for data_type, items in bundle.items():
+        if data_type in ("version", "synced_at"):
+            continue
+        if not isinstance(items, (list, dict)):
+            continue
+        if isinstance(items, list):
+            for i in range(0, len(items), chunk_size):
+                if len(chunks) <= i // chunk_size:
+                    chunks.append({"version": "1"})
+                chunks[i // chunk_size][data_type] = items[i : i + chunk_size]
+        elif isinstance(items, dict):
+            if len(chunks) == 0:
+                chunks.append({"version": "1"})
+            chunks[0][data_type] = items
+
+    return chunks if chunks else [bundle]
+
+
+def build_local_delta(gcc_dir: Path, remote_state: dict) -> dict:
+    """Build a delta bundle of items the server doesn't have (for push)."""
+    bundle: dict = {"version": "1"}
+    _add_commit_delta(gcc_dir, remote_state, bundle)
+    _add_theta_delta(gcc_dir, remote_state, bundle)
+    _add_learning_delta(gcc_dir, remote_state, bundle)
+    _add_sensitivity_delta(gcc_dir, remote_state, bundle)
+    _add_topic_delta(gcc_dir, remote_state, bundle)
+    return bundle
+
+
+def build_remote_delta(gcc_dir: Path, client_state: dict) -> dict:
+    """Build a delta bundle of items the client doesn't have (for pull, server-side)."""
+    bundle: dict = {"version": "1"}
+    _add_pull_commit_delta(gcc_dir, client_state, bundle)
+    _add_pull_theta_delta(gcc_dir, client_state, bundle)
+    _add_pull_learning_delta(gcc_dir, client_state, bundle)
+    _add_pull_sensitivity_delta(gcc_dir, client_state, bundle)
+    _add_pull_topic_delta(gcc_dir, client_state, bundle)
+    return bundle
+
+
+def build_server_state(gcc_dir: Path) -> dict:
+    """Return the server's current sync state (what it has)."""
+    return {
+        "commits": {"head": _get_max_commit_index(gcc_dir)},
+        "theta": _get_theta_per_concept_timestamps(gcc_dir),
+        "learnings": {"synced_ids": _get_learning_ids(gcc_dir), "count": _count_learnings(gcc_dir)},
+        "sensitivities": {"last_synced_ts": _get_latest_sensitivity_ts(gcc_dir)},
+        "topics": {"synced_custom_names": _get_custom_topic_names(gcc_dir)},
+    }
+
+
+def merge_pull(gcc_dir: Path, remote_delta: dict, conflict_log: Any) -> dict:
+    """Merge a remote delta into local .GCC/. Returns counts + conflicts.
+
+    *conflict_log* is a ``ConflictLog`` instance (or duck-typed equivalent).
+    """
+    counts: dict[str, int] = {}
+    conflicts: list[dict] = []
+
+    if "commits" in remote_delta:
+        c, cf = _merge_commits(gcc_dir, remote_delta["commits"], conflict_log)
+        counts["commits"] = c
+        conflicts.extend(cf)
+
+    if "theta" in remote_delta:
+        c, cf = _merge_theta(gcc_dir, remote_delta["theta"], conflict_log)
+        counts["theta"] = c
+        conflicts.extend(cf)
+
+    if "learnings" in remote_delta:
+        c, cf = _merge_learnings(gcc_dir, remote_delta["learnings"], conflict_log)
+        counts["learnings"] = c
+        conflicts.extend(cf)
+
+    if "sensitivities" in remote_delta:
+        c, cf = _merge_sensitivities(gcc_dir, remote_delta["sensitivities"], conflict_log)
+        counts["sensitivities"] = c
+        conflicts.extend(cf)
+
+    if "topics" in remote_delta:
+        c, cf = _merge_topics(gcc_dir, remote_delta["topics"], conflict_log)
+        counts["topics"] = c
+        conflicts.extend(cf)
+
+    return {"counts": counts, "conflicts": conflicts}
+
+
+def merge_push(gcc_dir: Path, local_delta: dict, conflict_log: Any) -> dict:
+    """Merge a local delta into server .GCC/. Returns counts + conflicts.
+
+    Same logic as merge_pull — the merge rules are symmetric.
+    """
+    return merge_pull(gcc_dir, local_delta, conflict_log)
+
+
+def _merge_commits(gcc_dir: Path, remote_commits: list[dict], conflict_log: Any) -> tuple[int, list[dict]]:
+    """Add new commits (skip if ID exists). Log ID collisions."""
+    commits_dir = gcc_dir / "commits"
+    commits_dir.mkdir(parents=True, exist_ok=True)
+    added = 0
+    conflicts = []
+    for commit in remote_commits:
+        cid = commit.get("id")
+        if not cid:
+            continue
+        path = commits_dir / f"{cid}.json"
+        if path.exists():
+            try:
+                existing = json.loads(path.read_text(encoding="utf-8"))
+                if existing.get("message") != commit.get("message") or existing.get("concepts") != commit.get("concepts"):
+                    conflict = _make_conflict("commit_id_collision", "high", "commits", cid,
+                                               {"message": existing.get("message"), "concepts": existing.get("concepts")},
+                                               {"message": commit.get("message"), "concepts": commit.get("concepts")},
+                                               "kept_local")
+                    if conflict_log:
+                        conflict["conflict_id"] = conflict_log.append(conflict)
+                    conflicts.append(conflict)
+            except (json.JSONDecodeError, OSError):
+                pass
+            continue
+        try:
+            _atomic_write(path, json.dumps(commit, indent=2) + "\n")
+            added += 1
+        except OSError as exc:
+            logger.warning("devtorch: failed to write commit %s — %s", cid, exc)
+    return added, conflicts
+
+
+def _merge_theta(gcc_dir: Path, remote_theta: dict, conflict_log: Any) -> tuple[int, list[dict]]:
+    """Merge remote theta using merge_theta rules. Log disclosure downgrades."""
+    from devtorch_core.theta import make_theta_store
+    theta_store = make_theta_store(gcc_dir)
+    theta = theta_store.load()
+    cv = theta.setdefault("coordination_vector", {})
+    conflicts = []
+
+    remote_cv = remote_theta.get("coordination_vector", {})
+    for concept, entry in remote_cv.items():
+        existing = cv.get(concept, {})
+        old_count = int(existing.get("event_count", 0))
+        new_count = int(entry.get("event_count", 0))
+        total = old_count + new_count
+
+        if total:
+            old_conf = float(existing.get("mean_confidence", entry["mean_confidence"]))
+            new_conf = float(entry["mean_confidence"])
+            blended = (old_conf * old_count + new_conf * new_count) / total
+        else:
+            blended = float(entry["mean_confidence"])
+
+        _LEVEL_RANK = {"PUBLIC": 0, "PROTECTED": 1, "PRIVATE": 2}
+        _RANK_LEVEL = {0: "PUBLIC", 1: "PROTECTED", 2: "PRIVATE"}
+        existing_rank = _LEVEL_RANK.get(existing.get("disclosure_level", "PUBLIC"), 0)
+        new_rank = _LEVEL_RANK.get(entry.get("disclosure_level", "PUBLIC"), 0)
+
+        if existing and existing_rank != new_rank and existing_rank > new_rank:
+            conflict = _make_conflict("theta_disclosure_downgrade", "low", "theta", concept,
+                                       {"disclosure_level": existing.get("disclosure_level")},
+                                       {"disclosure_level": entry.get("disclosure_level")},
+                                       "kept_max")
+            if conflict_log:
+                conflict["conflict_id"] = conflict_log.append(conflict)
+            conflicts.append(conflict)
+
+        merged_level = _RANK_LEVEL[max(existing_rank, new_rank)]
+        from datetime import datetime, timezone
+        cv[concept] = {
+            "mean_confidence": round(blended, 6),
+            "disclosure_level": merged_level,
+            "event_count": total,
+            "last_updated": datetime.now(timezone.utc).isoformat(),
+        }
+
+    theta_store.save(theta)
+    return len(remote_cv), conflicts
+
+
+def _merge_learnings(gcc_dir: Path, remote_learnings: list[dict], conflict_log: Any) -> tuple[int, list[dict]]:
+    """Add new learnings. For existing IDs: max confidence, union concepts, most restrictive validity. Log content divergence."""
+    from devtorch_core.reasoning_plus.learning.models import Learning
+    from devtorch_core.reasoning_plus.learning.store import LearningStore
+    store = LearningStore(gcc_dir)
+    added = 0
+    conflicts = []
+    for rl in remote_learnings:
+        rid = rl.get("id")
+        if not rid:
+            continue
+        existing_learning = store.get(rid)
+        if existing_learning:
+            if existing_learning.content != rl.get("content", ""):
+                conflict = _make_conflict("learning_content_divergence", "medium", "learnings", rid,
+                                           {"content": existing_learning.content, "confidence": existing_learning.confidence},
+                                           {"content": rl.get("content"), "confidence": rl.get("confidence")},
+                                           "kept_local")
+                if conflict_log:
+                    conflict["conflict_id"] = conflict_log.append(conflict)
+                conflicts.append(conflict)
+            existing_learning.confidence = max(existing_learning.confidence, float(rl.get("confidence", 0)))
+            existing_learning.trigger_concepts = list(set(existing_learning.trigger_concepts) | set(rl.get("trigger_concepts", [])))
+            valid_rank = {"active": 0, "stale": 1, "deprecated": 2}
+            if valid_rank.get(rl.get("validity", "active"), 0) > valid_rank.get(existing_learning.validity, 0):
+                existing_learning.validity = rl["validity"]
+            store.save(existing_learning, deduplicate=False)
+        else:
+            try:
+                learning = Learning.from_dict(rl)
+                store.save(learning, deduplicate=False)
+                added += 1
+            except Exception as exc:
+                logger.warning("devtorch: failed to save remote learning %s — %s", rid, exc)
+    return added, conflicts
+
+
+def _merge_sensitivities(gcc_dir: Path, remote_events: list[dict], conflict_log: Any) -> tuple[int, list[dict]]:
+    """Append new sensitivity events (dedup by SHA-256 hash)."""
+    path = gcc_dir / "sensitivities" / "events.jsonl"
+    path.parent.mkdir(parents=True, exist_ok=path.parent != gcc_dir and True or True)
+    path.parent.mkdir(parents=True, exist_ok=True)
+
+    existing_hashes = set()
+    if path.exists():
+        try:
+            for line in path.read_text(encoding="utf-8").splitlines():
+                if line.strip():
+                    existing_hashes.add(hashlib.sha256(line.encode("utf-8")).hexdigest())
+        except OSError:
+            pass
+
+    added = 0
+    with open(str(path), "a", encoding="utf-8") as f:
+        for event in remote_events:
+            line = json.dumps(event, separators=(",", ":"))
+            h = hashlib.sha256(line.encode("utf-8")).hexdigest()
+            if h in existing_hashes:
+                continue
+            existing_hashes.add(h)
+            f.write(line + "\n")
+            added += 1
+    return added, []
+
+
+def _merge_topics(gcc_dir: Path, remote_topics: dict, conflict_log: Any) -> tuple[int, list[dict]]:
+    """Merge custom topics (union by name, union concepts). Log delete vs update."""
+    from devtorch_core.topics import TopicStore
+    store = TopicStore(gcc_dir)
+    remote_custom = remote_topics.get("custom_topics", {})
+    added = 0
+    conflicts = []
+
+    local_full = store._read_full()
+    local_custom = local_full.setdefault("custom_topics", {})
+
+    for name, concepts in remote_custom.items():
+        if name in local_custom:
+            if set(local_custom[name]) != set(concepts):
+                conflict = _make_conflict("topic_delete_vs_update", "medium", "topics", name,
+                                           {"concepts": local_custom[name]},
+                                           {"concepts": concepts},
+                                           "kept_updated")
+                if conflict_log:
+                    conflict["conflict_id"] = conflict_log.append(conflict)
+                conflicts.append(conflict)
+                local_custom[name] = sorted(set(local_custom[name]) | set(concepts))
+        else:
+            local_custom[name] = sorted(set(concepts))
+            added += 1
+
+    store._write_full(local_full)
+    return added, conflicts
+
+
+def _make_conflict(
+    type_: str, severity: str, data_type: str, item_id: str,
+    local_value: dict, remote_value: dict, resolution: str,
+) -> dict:
+    from datetime import datetime, timezone
+    return {
+        "type": type_,
+        "severity": severity,
+        "data_type": data_type,
+        "item_id": item_id,
+        "local_value": local_value,
+        "remote_value": remote_value,
+        "resolution": resolution,
+        "resolved": False,
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    }
+
+
+def _add_commit_delta(gcc_dir: Path, remote_state: dict, bundle: dict) -> None:
+    commits_dir = gcc_dir / "commits"
+    if not commits_dir.exists():
+        return
+    remote_commits = remote_state.get("commits", {})
+    # Server state exposes commit progress via "head" (max commit id). Fall back
+    # to synced_ids for compatibility with older server states.
+    remote_head = remote_commits.get("head")
+    remote_synced = set(remote_commits.get("synced_ids", []))
+    delta = []
+    for path in sorted(commits_dir.glob("*.json")):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            cid = data.get("id")
+            if remote_head is not None and cid is not None:
+                if cid > remote_head:
+                    delta.append(data)
+            elif cid not in remote_synced:
+                delta.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    if delta:
+        bundle["commits"] = delta
+
+
+def _add_theta_delta(gcc_dir: Path, remote_state: dict, bundle: dict) -> None:
+    path = gcc_dir / "theta.json"
+    if not path.exists():
+        return
+    try:
+        theta = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return
+    cv = theta.get("coordination_vector", {})
+    remote_concepts = remote_state.get("theta", {})
+    delta_cv = {}
+    for concept, entry in cv.items():
+        remote_ts = remote_concepts.get(concept, {}).get("last_updated", "")
+        local_ts = entry.get("last_updated", "")
+        if local_ts > remote_ts:
+            delta_cv[concept] = entry
+    if delta_cv:
+        bundle["theta"] = {"version": 1, "coordination_vector": delta_cv}
+
+
+def _add_learning_delta(gcc_dir: Path, remote_state: dict, bundle: dict) -> None:
+    learn_dir = gcc_dir / "reasoning_learnings" / "learnings"
+    if not learn_dir.exists():
+        return
+    remote_ids = set(remote_state.get("learnings", {}).get("synced_ids", []))
+    delta = []
+    for path in learn_dir.glob("*.json"):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("id") not in remote_ids:
+                delta.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    if delta:
+        bundle["learnings"] = delta
+
+
+def _add_sensitivity_delta(gcc_dir: Path, remote_state: dict, bundle: dict) -> None:
+    path = gcc_dir / "sensitivities" / "events.jsonl"
+    if not path.exists():
+        return
+    remote_ts = remote_state.get("sensitivities", {}).get("last_synced_ts", "")
+    delta = []
+    try:
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            ev = json.loads(line)
+            payload = ev if "target_concept" in ev else ev.get("payload", {})
+            created = payload.get("created_at", "") or payload.get("timestamp", "")
+            if created > remote_ts:
+                delta.append(payload)
+    except (OSError, json.JSONDecodeError):
+        pass
+    if delta:
+        bundle["sensitivities"] = delta
+
+
+def _add_topic_delta(gcc_dir: Path, remote_state: dict, bundle: dict) -> None:
+    from devtorch_core.topics import TopicStore
+    store = TopicStore(gcc_dir)
+    remote_names = set(remote_state.get("topics", {}).get("synced_custom_names", []))
+    local_custom = store._read_custom()
+    delta = {name: concepts for name, concepts in local_custom.items() if name not in remote_names}
+    if delta:
+        bundle["topics"] = {"custom_topics": delta}
+
+
+def _add_pull_commit_delta(gcc_dir: Path, client_state: dict, bundle: dict) -> None:
+    commits_dir = gcc_dir / "commits"
+    if not commits_dir.exists():
+        return
+    client_commits = client_state.get("commits", {})
+    client_head = client_commits.get("remote_head")
+    client_ids = set(client_commits.get("synced_ids", []))
+    delta = []
+    for path in sorted(commits_dir.glob("*.json")):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            cid = data.get("id")
+            if client_head is not None and cid is not None:
+                if cid > client_head:
+                    delta.append(data)
+            elif cid not in client_ids:
+                delta.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    if delta:
+        bundle["commits"] = delta
+
+
+def _add_pull_theta_delta(gcc_dir: Path, client_state: dict, bundle: dict) -> None:
+    path = gcc_dir / "theta.json"
+    if not path.exists():
+        return
+    try:
+        theta = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return
+    cv = theta.get("coordination_vector", {})
+    client_concepts = client_state.get("theta", {})
+    delta_cv = {}
+    for concept, entry in cv.items():
+        client_ts = client_concepts.get(concept, {}).get("last_updated", "")
+        local_ts = entry.get("last_updated", "")
+        if local_ts > client_ts:
+            delta_cv[concept] = entry
+    if delta_cv:
+        bundle["theta"] = {"version": 1, "coordination_vector": delta_cv}
+
+
+def _add_pull_learning_delta(gcc_dir: Path, client_state: dict, bundle: dict) -> None:
+    learn_dir = gcc_dir / "reasoning_learnings" / "learnings"
+    if not learn_dir.exists():
+        return
+    client_ids = set(client_state.get("learnings", {}).get("synced_ids", []))
+    delta = []
+    for path in learn_dir.glob("*.json"):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("id") not in client_ids:
+                delta.append(data)
+        except (json.JSONDecodeError, OSError):
+            continue
+    if delta:
+        bundle["learnings"] = delta
+
+
+def _add_pull_sensitivity_delta(gcc_dir: Path, client_state: dict, bundle: dict) -> None:
+    path = gcc_dir / "sensitivities" / "events.jsonl"
+    if not path.exists():
+        return
+    client_ts = client_state.get("sensitivities", {}).get("last_synced_ts", "")
+    delta = []
+    try:
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            ev = json.loads(line)
+            payload = ev if "target_concept" in ev else ev.get("payload", {})
+            created = payload.get("created_at", "") or payload.get("timestamp", "")
+            if created > client_ts:
+                delta.append(payload)
+    except (OSError, json.JSONDecodeError):
+        pass
+    if delta:
+        bundle["sensitivities"] = delta
+
+
+def _add_pull_topic_delta(gcc_dir: Path, client_state: dict, bundle: dict) -> None:
+    from devtorch_core.topics import TopicStore
+    store = TopicStore(gcc_dir)
+    client_names = set(client_state.get("topics", {}).get("synced_custom_names", []))
+    local_custom = store._read_custom()
+    delta = {name: concepts for name, concepts in local_custom.items() if name not in client_names}
+    if delta:
+        bundle["topics"] = {"custom_topics": delta}
+
+
+def _get_max_commit_index(gcc_dir: Path) -> str:
+    commits_dir = gcc_dir / "commits"
+    if not commits_dir.exists():
+        return "0"
+    max_id = "0"
+    for path in commits_dir.glob("*.json"):
+        stem = path.stem
+        if stem > max_id:
+            max_id = stem
+    return max_id
+
+
+def _get_theta_per_concept_timestamps(gcc_dir: Path) -> dict[str, dict[str, str]]:
+    path = gcc_dir / "theta.json"
+    if not path.exists():
+        return {}
+    try:
+        theta = json.loads(path.read_text(encoding="utf-8"))
+    except (json.JSONDecodeError, OSError):
+        return {}
+    cv = theta.get("coordination_vector", {})
+    return {c: {"last_updated": e.get("last_updated", "")} for c, e in cv.items()}
+
+
+def _get_learning_ids(gcc_dir: Path) -> list[str]:
+    learn_dir = gcc_dir / "reasoning_learnings" / "learnings"
+    if not learn_dir.exists():
+        return []
+    ids = []
+    for path in learn_dir.glob("*.json"):
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+            if data.get("id"):
+                ids.append(data["id"])
+        except (json.JSONDecodeError, OSError):
+            continue
+    return ids
+
+
+def _count_learnings(gcc_dir: Path) -> int:
+    learn_dir = gcc_dir / "reasoning_learnings" / "learnings"
+    if not learn_dir.exists():
+        return 0
+    return len(list(learn_dir.glob("*.json")))
+
+
+def _get_latest_sensitivity_ts(gcc_dir: Path) -> str:
+    path = gcc_dir / "sensitivities" / "events.jsonl"
+    if not path.exists():
+        return ""
+    latest = ""
+    try:
+        for line in path.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            ev = json.loads(line)
+            ts = ev.get("created_at", "") or ev.get("timestamp", "")
+            if ts > latest:
+                latest = ts
+    except (OSError, json.JSONDecodeError):
+        pass
+    return latest
+
+
+def _get_custom_topic_names(gcc_dir: Path) -> list[str]:
+    from devtorch_core.topics import TopicStore
+    return list(TopicStore(gcc_dir)._read_custom().keys())
+
+
+def _atomic_write(path: Path, content: str) -> None:
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    try:
+        tmp.write_text(content, encoding="utf-8")
+        tmp.replace(path)
+    finally:
+        if tmp.exists():
+            try:
+                tmp.unlink()
+            except Exception:
+                pass