apsimo 1.3.4__tar.gz → 1.4.0__tar.gz

{apsimo-1.3.4 → apsimo-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apsimo
-Version: 1.3.4
+Version: 1.4.0
 Summary: Persistent identity, memory and autonomy for AI agents
 License-Expression: MIT
 Requires-Python: >=3.11
@@ -28,7 +28,7 @@ Requires-Dist: pandas>=2.0; extra == "vectors"
 Requires-Dist: pyarrow>=15.0; extra == "vectors"
 Requires-Dist: lancedb>=0.10; extra == "vectors"
 Provides-Extra: hermes
-Requires-Dist: apsimo-hermes<2,>=1.3.4; extra == "hermes"
+Requires-Dist: apsimo-hermes<2,>=1.4.0; extra == "hermes"
 Provides-Extra: mcp
 Requires-Dist: mcp[cli]<2,>=1.0; extra == "mcp"
 Provides-Extra: lancedb

{apsimo-1.3.4 → apsimo-1.4.0}/apsimo/api/routers/host.py

@@ -32,7 +32,10 @@ from apsimo.api.authority import (
     resolve_turn_person,
 )
 
+from apsimo.turns.tool_observations import ToolObservation
 from apsimo.api.schemas.host import (
+    HostIdentity,
+    HostTurnContext,
     HostConfigureRequest,
     HostConfigureResponse,
     ModelInfo,
@@ -3881,7 +3884,8 @@ async def forget_turn_sources(body: SourceForgetRequest, request: Request = None
                 tom_cleanup[name + '_cleanup'] = 'complete'
             except Exception:
                 logger.warning('source erasure %s cleanup is pending', name, exc_info=True)
-    vector_cleanup = 'unavailable'
+    vector_cleanup = ('disabled_not_checked' if os.environ.get('COLONY_EMBED_PROVIDER') == 'skip'
+                      else 'unavailable')
     from apsimo.vector import get_store
     vector_store = get_store()
     if vector_store is not None and getattr(vector_store, 'catalog', None) is not None:
@@ -3891,7 +3895,8 @@ async def forget_turn_sources(body: SourceForgetRequest, request: Request = None
             vector_cleanup = 'complete'
         except Exception:
             logger.warning('source erasure vector generation cleanup is pending', exc_info=True)
-    graph_cleanup = "unavailable" if _graph is None else "pending"
+    graph_cleanup = ('disabled_not_checked' if os.environ.get('COLONY_GRAPH_ENABLED', 'true').lower()
+                     in {'0', 'false', 'off'} else 'unavailable') if _graph is None else 'pending'
     if _graph is not None:
         try:
             await _graph.delete_source_memories(list(dict.fromkeys(result["source_ids"] + result["affected_source_ids"])))
@@ -3916,11 +3921,25 @@ async def forget_turn_sources(body: SourceForgetRequest, request: Request = None
         transport_cleanup = forget_sources(list(dict.fromkeys(result['source_ids'] + result['affected_source_ids'])))
     except Exception:
         logger.warning('Source erasure transport cleanup remains pending', exc_info=True)
+    communications_cleanup, communications_unlinked_rows = 'unavailable', None
+    if _comms_log is not None:
+        communications_cleanup = 'pending'
+        try:
+            _comms_log.purge_erased_sources(list(dict.fromkeys(result['source_ids'] + result['affected_source_ids'])),
+                                           contact_id=person)
+            communications_unlinked_rows = _comms_log.unlinked_summary_count(person)
+            communications_cleanup = 'complete'
+        except Exception:
+            logger.warning('Source erasure communication summary cleanup remains pending', exc_info=True)
     return {"source_erased": True, **result, "graph_cleanup": graph_cleanup,
             "shared_facts_cleanup": fact_cleanup, "vector_cleanup": vector_cleanup,
             "world_cleanup": world_cleanup, "transport_cleanup": transport_cleanup, **tom_cleanup,
+            "communications_cleanup": communications_cleanup,
+            "communications_scope": "source_linked_summaries_only",
+            "communications_unlinked_rows": communications_unlinked_rows,
             "scope": "canonical_turn_sources_and_linked_projections",
-            "host_reconciliation": "pending_until_each_host_connects"}
+            "host_reconciliation": "not_observed",
+            "host_reconciliation_detail": "The erasure feed is available at this watermark; this response does not measure which hosts have applied it."}
 
 
 async def _resolve_scoped_sender_contact(authority, gateway: str, address: str):
@@ -4177,6 +4196,37 @@ async def task_instruction_sync(turn_id: str, body: TurnSyncRequest, response: R
     return await turns_sync_v2(turn_id, body, response, request)
 
 
+class ToolObservationEnvelope(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    identity: HostIdentity
+    context: HostTurnContext
+    observation: ToolObservation
+
+
+@v2_router.put('/turns/source-observation/{turn_id:path}', response_model=TurnSyncResponse)
+def source_observation_sync(turn_id: str, body: ToolObservationEnvelope, response: Response, request: Request):
+    from apsimo.api.routers.executions import authorized_viewer
+    from apsimo.turns import get_turn_idempotency_ledger
+    from apsimo.turns.idempotency import SourceErased
+    from apsimo.turns.tool_observations import record
+    person, owner = authorized_viewer(request, body.context.contact_id, scope='turns:write')
+    if not owner:
+        raise HTTPException(403, detail='owner_observation_required')
+    if body.context.turn_id != turn_id:
+        raise HTTPException(422, detail='observation_identity_mismatch')
+    try:
+        created = record(get_turn_idempotency_ledger(get_state_dir()), body.observation,
+            contact_id=person, session_id=body.context.session_id, source_id=turn_id)
+    except SourceErased:
+        return TurnSyncResponse(accepted=False, source_recorded=False, continuity_updated=False,
+                                skipped_reason='source_erased')
+    except ValueError as exc:
+        raise HTTPException(409, detail=str(exc)) from exc
+    response.status_code = 201 if created else 200
+    return TurnSyncResponse(accepted=True, source_recorded=True, continuity_updated=False,
+                            skipped_reason='tool_observation_only')
+
+
 @v2_router.put("/turns/{turn_id:path}", response_model=TurnSyncResponse)
 async def turns_sync_v2(
     turn_id: str,
@@ -4633,16 +4683,17 @@ async def _process_turn_sync(
         # Cross-channel communication ledger: record this exchange under the
         # CONVERSATION's channel (group vs DM vs voice provenance), never the
         # contact's primary-handle gateway (which collapsed everything to one
-        # channel). System turns are recorded too, for ops visibility; they
-        # are excluded from every relationship surface.
+        # channel). New prose requires a canonical person source; transport
+        # receipts retain their separate metadata-only ingestion path.
         try:
-            if _comms_log is not None and body.context.contact_id:
+            if _comms_log is not None and body.context.contact_id and source_recorded:
                 _ch = body.context.channel_id or "direct"
                 _sess = body.context.session_id or ""
+                _lineage, _ = _comms_log.source_input(source_id, body.context.contact_id)
                 _comms_log.log(body.context.contact_id, channel=_ch,
                                direction="in",
                                summary=(body.summary or "")[:300],
-                               session_id=_sess)
+                               session_id=_sess, source_lineage=_lineage)
                 # Record the assistant's reply as an OUTBOUND exchange on the
                 # SAME resolved contact + conversation channel. Without this
                 # the ledger sees only half of every conversation, so
@@ -4653,7 +4704,7 @@ async def _process_turn_sync(
                 if _asst.strip():
                     _comms_log.log(body.context.contact_id, channel=_ch,
                                    direction="out",
-                                   summary=_asst[:300], session_id=_sess)
+                                   summary=_asst[:300], session_id=_sess, source_lineage=_lineage)
         except Exception:
             logger.debug("comms ledger log failed", exc_info=True)
     except Exception:

{apsimo-1.3.4 → apsimo-1.4.0}/apsimo/contacts/comms.py

@@ -14,8 +14,10 @@ import logging
 from pathlib import Path
 import sqlite3
 import uuid
+from contextlib import closing
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional
+from apsimo.tom.source_lineage import SourceLinkedStore
 
 logger = logging.getLogger(__name__)
 
@@ -34,11 +36,12 @@ def _parse(ts: Optional[str]) -> Optional[datetime]:
         return None
 
 
-class CommsLog:
+class CommsLog(SourceLinkedStore):
     """SQLite ledger of communications with each contact, across all channels."""
 
-    def __init__(self, db_path: str) -> None:
+    def __init__(self, db_path: str, *, source_ledger=None) -> None:
         self._db_path = str(db_path)
+        self._source_ledger = source_ledger
         self._conn = sqlite3.connect(str(db_path))
         self._conn.row_factory = sqlite3.Row
         self._conn.execute("PRAGMA journal_mode=WAL")
@@ -64,6 +67,7 @@ class CommsLog:
             "reaction": "TEXT",
             "receipt_ref": "TEXT",
             "outbound_ref": "TEXT",
+            "source_lineage_json": "TEXT",
         }.items():
             if name not in existing:
                 self._conn.execute(
@@ -71,10 +75,64 @@ class CommsLog:
         self._conn.execute(
             "CREATE INDEX IF NOT EXISTS idx_comms_reply_ref "
             "ON communications(contact_id,reply_to_ref,ts)")
+        self._conn.execute("""CREATE INDEX IF NOT EXISTS idx_comms_source_lineage
+            ON communications(contact_id,json_extract(source_lineage_json,'$.turn_id'))
+            WHERE source_lineage_json IS NOT NULL""")
         self._conn.commit()
         from .transport_ingress import ensure_schema
         ensure_schema(self._conn)
 
+    def _ledger(self):
+        # An offline copy can belong to another profile. Missing local source
+        # rows are not evidence that its linked communications were erased.
+        if self._source_ledger is None:
+            raise ValueError('communications_source_ledger_required')
+        return self._source_ledger
+
+    def _reconcile_erasure_ids(self, contact_id=None) -> None:
+        """Read existing tombstone IDs, not every original conversation body."""
+        where, params = (' AND contact_id=?', (contact_id,)) if contact_id else ('', ())
+        if self._conn.execute('SELECT 1 FROM communications WHERE source_lineage_json IS NOT NULL'
+                              + where + ' LIMIT 1', params).fetchone() is None:
+            return
+        condition = ' WHERE e.contact_id=?' if contact_id else ''
+        with closing(self._ledger()._connect()) as conn:
+            erased = conn.execute('''SELECT e.contact_id,coalesce(r.source_turn_id,e.turn_id)
+                FROM source_erasures e LEFT JOIN source_erasure_revisions r USING(sequence)'''
+                + condition + ''' UNION SELECT e.contact_id,p.turn_id FROM source_erasures e
+                LEFT JOIN source_erasure_revisions r USING(sequence)
+                JOIN source_projection_erasures p ON p.source_turn_id=coalesce(r.source_turn_id,e.turn_id)'''
+                + condition, params + params).fetchall()
+        with self._conn:
+            self._conn.executemany('''DELETE FROM communications WHERE contact_id=?
+                AND source_lineage_json IS NOT NULL
+                AND json_extract(source_lineage_json,'$.turn_id')=?''', [tuple(row) for row in erased])
+
+    def _summary_rows(self, query, params, fields, *, contact_id=None):
+        self._reconcile_erasure_ids(contact_id)
+        rows = self._conn.execute(query, params).fetchall()
+        invalid = self._invalid_sources([row for row in rows if row['source_lineage_json'] is not None])
+        rejected = {row['id'] for row in invalid}
+        with self._conn:
+            self._conn.executemany('DELETE FROM communications WHERE id=?', [(key,) for key in rejected])
+        return [{key: row[key] for key in fields} for row in rows if row['id'] not in rejected]
+
+    def purge_erased_sources(self, turn_ids=None, *, contact_id=None) -> int:
+        sql = ('SELECT id,contact_id,source_lineage_json FROM communications '
+               'WHERE source_lineage_json IS NOT NULL')
+        rows = self._conn.execute(sql + (' AND contact_id=?' if contact_id else ''),
+                                  (contact_id,) if contact_id else ()).fetchall()
+        invalid = self._invalid_sources(rows, turn_ids)
+        with self._conn:
+            self._conn.executemany('DELETE FROM communications WHERE id=?', [(row['id'],) for row in invalid])
+        return len(invalid)
+
+    def unlinked_summary_count(self, contact_id: str) -> int:
+        """Historical/unattributed prose cannot be erased by guessing its source."""
+        return self._conn.execute('''SELECT count(*) FROM communications WHERE contact_id=?
+            AND source_lineage_json IS NULL AND coalesce(summary,'')!=''
+            AND coalesce(receipt_ref,'')='' ''', (contact_id,)).fetchone()[0]
+
     def read_connection(self):
         """An independent read connection for projections running on a worker thread.
 
@@ -117,9 +175,12 @@ class CommsLog:
             summary: str = "", session_id: str = "",
             external_ref: str = "", reply_to_ref: str = "",
             reaction: str = "", receipt_ref: str = "",
-            ts: Optional[str] = None) -> None:
+            ts: Optional[str] = None, source_lineage=None) -> None:
         if not contact_id or direction not in ("in", "out"):
             return
+        if source_lineage is not None and not self._source_visible(contact_id, source_lineage):
+            from apsimo.turns.idempotency import SourceErased
+            raise SourceErased('source_erased')
         normalized_reaction = (reaction or "").strip().lower()
         if normalized_reaction not in {
                 "", "accepted", "acknowledged", "actioned", "negative",
@@ -128,39 +189,48 @@ class CommsLog:
         self._conn.execute(
             "INSERT INTO communications "
             "(id,contact_id,channel,direction,summary,session_id,ts,"
-            "external_ref,reply_to_ref,reaction,receipt_ref) "
-            "VALUES (?,?,?,?,?,?,?,?,?,?,?)",
+            "external_ref,reply_to_ref,reaction,receipt_ref,source_lineage_json) "
+            "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
             (uuid.uuid4().hex, contact_id, channel or "unknown", direction,
              (summary or "")[:500], session_id or "", ts or _now().isoformat(),
              (external_ref or "")[:512] or None,
              (reply_to_ref or "")[:512] or None,
              normalized_reaction or None,
-             (receipt_ref or "")[:512] or None),
+             (receipt_ref or "")[:512] or None,
+             json.dumps(source_lineage) if source_lineage is not None else None),
         )
         self._conn.commit()
+        # A source erasure can finish while this separate projection is written.
+        if source_lineage is not None and not self._source_visible(contact_id, source_lineage):
+            self.purge_erased_sources(contact_id=contact_id)
+            from apsimo.turns.idempotency import SourceErased
+            raise SourceErased('source_erased')
 
     def history(self, contact_id: str, limit: int = 15) -> List[Dict[str, Any]]:
-        rows = self._conn.execute(
-            "SELECT channel, direction, summary, ts FROM communications WHERE contact_id=?"
-            " ORDER BY ts DESC LIMIT ?", (contact_id, limit)).fetchall()
-        return [dict(r) for r in rows]
+        return self._summary_rows(
+            "SELECT id,contact_id,source_lineage_json,channel,direction,summary,ts "
+            "FROM communications WHERE contact_id=? ORDER BY ts DESC LIMIT ?",
+            (contact_id, limit), ('channel', 'direction', 'summary', 'ts'), contact_id=contact_id)
 
     def last_per_channel(self, contact_id: str) -> Dict[str, Dict[str, Any]]:
-        rows = self._conn.execute(
-            "SELECT channel, direction, summary, MAX(ts) AS ts FROM communications"
-            " WHERE contact_id=? GROUP BY channel", (contact_id,)).fetchall()
+        rows = self._summary_rows(
+            "SELECT id,contact_id,source_lineage_json,channel,direction,summary,MAX(ts) AS ts "
+            "FROM communications WHERE contact_id=? GROUP BY channel", (contact_id,),
+            ('channel', 'direction', 'summary', 'ts'), contact_id=contact_id)
         return {r["channel"]: {"direction": r["direction"], "summary": r["summary"], "ts": r["ts"]}
                 for r in rows}
 
     def last_outbound(self, contact_id: str) -> Optional[Dict[str, Any]]:
-        r = self._conn.execute(
-            "SELECT channel, summary, ts FROM communications WHERE contact_id=? AND direction='out'"
-            " ORDER BY ts DESC LIMIT 1", (contact_id,)).fetchone()
-        return dict(r) if r else None
+        rows = self._summary_rows(
+            "SELECT id,contact_id,source_lineage_json,channel,summary,ts "
+            "FROM communications WHERE contact_id=? AND direction='out' ORDER BY ts DESC LIMIT 1",
+            (contact_id,), ('channel', 'summary', 'ts'), contact_id=contact_id)
+        return rows[0] if rows else None
 
     def inbound_since(self, contact_id: str, since_iso: str) -> List[str]:
         """Timestamps of inbound rows from a contact since an ISO instant
         (selfhood benchmark: did the owner respond after a delivery)."""
+        self._reconcile_erasure_ids(contact_id)
         rows = self._conn.execute(
             "SELECT ts FROM communications WHERE contact_id=? AND"
             " direction='in' AND ts >= ? ORDER BY ts ASC LIMIT 5000",
@@ -176,19 +246,18 @@ class CommsLog:
         until_iso: str,
     ) -> List[Dict[str, Any]]:
         """Explicit inbound reactions tied to exact outbound references."""
-
         bounded = sorted({str(ref) for ref in refs if str(ref).strip()})[:5000]
         if not bounded:
             return []
         placeholders = ",".join("?" for _ in bounded)
-        rows = self._conn.execute(
-            "SELECT id,channel,summary,ts,external_ref,reply_to_ref,reaction,"
+        return self._summary_rows(
+            "SELECT id,contact_id,source_lineage_json,channel,summary,ts,external_ref,reply_to_ref,reaction,"
             "receipt_ref FROM communications WHERE contact_id=? "
             "AND direction='in' AND ts>=? AND ts<? AND reply_to_ref IN ("
             f"{placeholders}) AND reaction IS NOT NULL ORDER BY ts",
             [contact_id, since_iso, until_iso, *bounded],
-        ).fetchall()
-        return [dict(row) for row in rows]
+            ('id', 'channel', 'summary', 'ts', 'external_ref', 'reply_to_ref', 'reaction', 'receipt_ref'),
+            contact_id=contact_id)
 
     def match_reply(self, *, contact_id: str, outbound_ref: str, since_iso: str,
                     until_iso: Optional[str] = None, connection=None) -> Dict[str, Any]:
@@ -226,19 +295,18 @@ class CommsLog:
         require_receipt: bool = False,
     ) -> List[Dict[str, Any]]:
         """One auditable outbound cohort for benchmark denominators."""
-
         query = (
-            "SELECT id,channel,summary,ts,external_ref,receipt_ref "
+            "SELECT id,contact_id,source_lineage_json,channel,summary,ts,external_ref,receipt_ref "
             "FROM communications WHERE contact_id=? AND direction='out' "
             "AND ts>=? AND ts<?")
         if require_receipt:
             query += " AND receipt_ref IS NOT NULL AND receipt_ref!=''"
         query += " ORDER BY ts"
-        rows = self._conn.execute(
-            query, (contact_id, since_iso, until_iso)).fetchall()
-        return [dict(row) for row in rows]
+        return self._summary_rows(query, (contact_id, since_iso, until_iso),
+            ('id', 'channel', 'summary', 'ts', 'external_ref', 'receipt_ref'), contact_id=contact_id)
 
     def counts(self, contact_id: str) -> Dict[str, Any]:
+        self._reconcile_erasure_ids(contact_id)
         r = self._conn.execute(
             "SELECT SUM(direction='in') AS inbound, SUM(direction='out') AS outbound,"
             " COUNT(DISTINCT channel) AS channels FROM communications WHERE contact_id=?",
@@ -251,6 +319,7 @@ class CommsLog:
         the consumer shifts into the contact's timezone). Feeds the
         relationship profiler's approach guidance (preferred channel,
         best time to reach)."""
+        self._reconcile_erasure_ids(contact_id)
         rows = self._conn.execute(
             "SELECT channel, ts FROM communications"
             " WHERE contact_id=? AND ts >= datetime('now', ?)",
@@ -270,14 +339,15 @@ class CommsLog:
         first. The per-contact reads above answer 'how do I stand with X';
         this answers 'what has been flowing lately' for an ops ledger view."""
         limit = max(1, min(500, int(limit)))
-        rows = self._conn.execute(
-            "SELECT contact_id, channel, direction, summary, ts FROM communications"
-            " ORDER BY ts DESC LIMIT ?", (limit,)).fetchall()
-        return [dict(r) for r in rows]
+        return self._summary_rows(
+            "SELECT id,contact_id,source_lineage_json,channel,direction,summary,ts "
+            "FROM communications ORDER BY ts DESC LIMIT ?", (limit,),
+            ('contact_id', 'channel', 'direction', 'summary', 'ts'))
 
     def rollup(self, *, since_days: int = 30) -> Dict[str, Dict[str, int]]:
         """Inbound/outbound counts per channel over a window: the ledger's
         flow summary. ``{channel: {"in": n, "out": n}}``."""
+        self._reconcile_erasure_ids()
         rows = self._conn.execute(
             "SELECT channel, direction, COUNT(*) AS n FROM communications"
             " WHERE ts >= datetime('now', ?) GROUP BY channel, direction",

{apsimo-1.3.4 → apsimo-1.4.0}/apsimo/server.py

@@ -2069,7 +2069,8 @@ async def lifespan(app: FastAPI):
         logger.info("EngagementStore initialized (db=%s)", engagement_db)
 
         from apsimo.contacts.comms import CommsLog
-        comms_log = CommsLog(db_path=state_dir / "colony-comms.db")
+        comms_log = CommsLog(db_path=state_dir / "colony-comms.db", source_ledger=source_ledger)
+        comms_log.purge_erased_sources()
         set_comms_log(comms_log)
         logger.info("CommsLog initialized")
 

apsimo-1.4.0/apsimo/turns/tool_observations.py

@@ -0,0 +1,87 @@
+"""Explicit host-retained original tool evidence, using the canonical source ledger."""
+from contextlib import closing
+from datetime import datetime, timezone
+import hashlib
+import json
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+VERSION = 'native-tool-observation-v1'
+MAX_BYTES = 16384
+
+
+class NativeToolIdentity(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    profile_id: str = Field(pattern='^[0-9a-f]{64}$')
+    session_id: str = Field(min_length=1, max_length=256)
+    task_id: str = Field(min_length=1, max_length=256)
+    turn_id: str = Field(min_length=1, max_length=256)
+    tool_call_id: str = Field(min_length=1, max_length=256)
+    api_request_id: str = Field(min_length=1, max_length=256)
+    tool_name: str = Field(min_length=1, max_length=128)
+    message_id: int = Field(gt=0)
+    timestamp: float = Field(gt=0, allow_inf_nan=False)
+    result_sha256: str = Field(pattern='^[0-9a-f]{64}$')
+
+
+class ObservationSource(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    source_id: str = Field(min_length=1, max_length=256)
+    source_version: str = Field(pattern='^[0-9a-f]{64}$')
+
+
+class ToolObservation(BaseModel):
+    model_config = ConfigDict(extra='forbid')
+    native: NativeToolIdentity
+    content: str = Field(min_length=1, max_length=MAX_BYTES)
+    reason: str = Field(min_length=1, max_length=512)
+    origin: ObservationSource
+    sources: list[ObservationSource] = Field(default_factory=list, max_length=256)
+
+    @model_validator(mode='after')
+    def exact_result(self):
+        if (not self.content.strip() or not self.reason.strip()
+                or len(self.content.encode()) > MAX_BYTES
+                or hashlib.sha256(self.content.encode()).hexdigest() != self.native.result_sha256):
+            raise ValueError('invalid_original_tool_result')
+        return self
+
+
+def identity_id(native):
+    return 'native-observation:' + hashlib.sha256(json.dumps(native, sort_keys=True,
+        separators=(',', ':'), ensure_ascii=False).encode()).hexdigest()
+
+
+def record(ledger, observation, *, contact_id, session_id, source_id):
+    """The authenticated host supplies provenance; the model supplies only nomination."""
+    native = observation.native.model_dump()
+    if native['session_id'] != session_id or source_id != identity_id(native):
+        raise ValueError('observation_identity_mismatch')
+    origin = observation.origin.model_dump()
+    # The direct current owner instruction is already captured by the host.
+    # Tool results and machine task wrappers cannot stand in for this origin.
+    with closing(ledger._connect()) as conn:
+        row = conn.execute('SELECT session_id,messages_json FROM turn_sources WHERE turn_id=? AND contact_id=?',
+                           (origin['source_id'], contact_id)).fetchone()
+        from apsimo.turns.idempotency import canonical_turn_digest, SourceErased
+        if row is None:
+            if ledger.is_source_erased(origin['source_id'], contact_id):
+                raise SourceErased('observation_origin_erased')
+            raise ValueError('observation_origin_missing')
+        messages = json.loads(row['messages_json'])
+        if (row['session_id'] != session_id or len(messages) != 1 or messages[0].get('role') != 'user'
+                or canonical_turn_digest(messages) != origin['source_version']):
+            raise ValueError('observation_origin_mismatch')
+    refs = {}
+    for ref in [origin, *(s.model_dump() for s in observation.sources)]:
+        if ref['source_id'] in refs and refs[ref['source_id']] != ref:
+            raise ValueError('observation_source_revision_conflict')
+        refs[ref['source_id']] = ref
+    message = {'role': 'tool', 'content': observation.content, '_native_tool_observation': VERSION,
+        '_observation_sources': list(refs.values()),
+        'provenance': {'kind': VERSION, 'native': native,
+                       'selection': {'author': 'model', 'reason': observation.reason}}}
+    created = ledger.record_source(source_id, contact_id=contact_id, session_id=session_id,
+        messages=[message], occurred_at=datetime.fromtimestamp(native['timestamp'], timezone.utc).isoformat(),
+        derive_claims=False)
+    return created

{apsimo-1.3.4 → apsimo-1.4.0}/apsimo.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apsimo
-Version: 1.3.4
+Version: 1.4.0
 Summary: Persistent identity, memory and autonomy for AI agents
 License-Expression: MIT
 Requires-Python: >=3.11
@@ -28,7 +28,7 @@ Requires-Dist: pandas>=2.0; extra == "vectors"
 Requires-Dist: pyarrow>=15.0; extra == "vectors"
 Requires-Dist: lancedb>=0.10; extra == "vectors"
 Provides-Extra: hermes
-Requires-Dist: apsimo-hermes<2,>=1.3.4; extra == "hermes"
+Requires-Dist: apsimo-hermes<2,>=1.4.0; extra == "hermes"
 Provides-Extra: mcp
 Requires-Dist: mcp[cli]<2,>=1.0; extra == "mcp"
 Provides-Extra: lancedb

{apsimo-1.3.4 → apsimo-1.4.0}/apsimo.egg-info/SOURCES.txt

@@ -539,6 +539,7 @@ apsimo/turns/source_annotations.py
 apsimo/turns/source_attribution.py
 apsimo/turns/source_read.py
 apsimo/turns/source_vectors.py
+apsimo/turns/tool_observations.py
 apsimo/turns/video.py
 apsimo/util/autonomy_preset.py
 apsimo/util/instance.py
@@ -677,6 +678,7 @@ tests/test_commitment_resolution_downgrade.py
 tests/test_commitment_resolution_schema.py
 tests/test_commitment_work.py
 tests/test_commitments.py
+tests/test_comms_source_erasure.py
 tests/test_competence_reconciliation.py
 tests/test_compression.py
 tests/test_concern_resolution_receipts.py
@@ -808,6 +810,7 @@ tests/test_native_setup.py
 tests/test_native_source_image_read.py
 tests/test_native_source_read.py
 tests/test_native_source_video_read.py
+tests/test_native_tool_observations.py
 tests/test_neo4j_backend.py
 tests/test_operator_deck_read_truth.py
 tests/test_ordinary_context_briefings.py

{apsimo-1.3.4 → apsimo-1.4.0}/apsimo.egg-info/requires.txt

@@ -35,7 +35,7 @@ beautifulsoup4>=4.12
 neo4j>=5.18
 
 [hermes]
-apsimo-hermes<2,>=1.3.4
+apsimo-hermes<2,>=1.4.0
 
 [lancedb]
 lancedb>=0.6

{apsimo-1.3.4 → apsimo-1.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "apsimo"
-version = "1.3.4"
+version = "1.4.0"
 description = "Persistent identity, memory and autonomy for AI agents"
 readme = "README.md"
 license = "MIT"
@@ -29,7 +29,7 @@ dependencies = [
 graph = ["neo4j>=5.18"]
 neo4j = ["neo4j>=5.18"]
 vectors = ["sentence-transformers>=3.0", "pandas>=2.0", "pyarrow>=15.0", "lancedb>=0.10"]
-hermes = ["apsimo-hermes>=1.3.4,<2"]
+hermes = ["apsimo-hermes>=1.4.0,<2"]
 mcp = ["mcp[cli]>=1.0,<2"]
 lancedb = ["lancedb>=0.6", "pyarrow>=14.0", "pandas>=2.0"]
 postgres = ["asyncpg>=0.29"]

{apsimo-1.3.4 → apsimo-1.4.0}/tests/test_colony_memory_provider.py

@@ -131,6 +131,29 @@ def _make_provider(provider_mod, fake_httpx, monkeypatch):
     return p
 
 
+@pytest.mark.parametrize("reported", ["not_observed", "pending_until_each_host_connects", None])
+def test_source_remove_preserves_server_reconciliation_receipt(provider_mod, monkeypatch, reported):
+    payload = {"source_erased": True, "watermark": 7}
+    if reported is not None:
+        payload["host_reconciliation"] = reported
+    fake = _FakeHttpx({("POST", "/v1/host/memory/sources/forget"): payload})
+    monkeypatch.setattr(_FakeResponse, "is_success", property(lambda response: response.status_code < 300), raising=False)
+    provider = _make_provider(provider_mod, fake, monkeypatch)
+    provider._session_id = "source-session"
+    monkeypatch.setattr(provider, "_prefetch_contact", lambda: "cid-base")
+    provider._cached_context = "A recalled source."
+    provider._temporal_cache = (1.0, "Prior context.")
+    provider.on_memory_write("remove", "MEMORY.md", "", metadata={"old_text": "A recalled source."})
+    assert provider._last_erasure == {
+        "state": "source_erased", "scope": "canonical_turn_sources", "watermark": 7,
+        "host_reconciliation": reported or "not_observed",
+    }
+    assert provider._cached_context == "" and provider._temporal_cache == (0.0, "")
+    assert fake.requests[0]["json"] == {
+        "contact_id": "cid-base", "session_id": "source-session", "old_text": "A recalled source.",
+    }
+
+
 def test_native_setup_settings_survive_restart_and_profiles_stay_separate(
         provider_mod, monkeypatch, tmp_path):
     homes = [tmp_path / "profile-one", tmp_path / "profile-two"]