apsimo 1.3.0__py3-none-any.whl

apsimo/world_model/relationships.py

@@ -0,0 +1,25 @@
+"""World Model relationship dataclass."""
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+
+
+@dataclass
+class WorldRelationship:
+    """A directed relationship between two world model entities."""
+    id: str                           # wr-<timestamp>-<random7>
+    source_id: str
+    target_id: str
+    relationship_type: str            # WM_ prefixed type from constants
+    confidence: float = 0.5           # 0.0–1.0
+    valid_from: Optional[str] = None  # ISO8601; when relationship began
+    valid_to: Optional[str] = None    # ISO8601; None = currently active
+    properties: Dict[str, Any] = field(default_factory=dict)
+    source_observation_id: Optional[str] = None  # extraction provenance
+    created_at: Optional[str] = None
+    updated_at: Optional[str] = None
+
+    @property
+    def is_active(self) -> bool:
+        """True if the relationship has no end date (currently active)."""
+        return self.valid_to is None

apsimo/world_model/resolution/__init__.py

@@ -0,0 +1,13 @@
+"""Entity resolution and merge workflow."""
+from .entity_resolver import EntityResolver, ResolutionAction, ResolutionResult
+from .merge_workflow import MergeWorkflow, MergeProposal
+from .merge_audit import MergeAuditRecord
+
+__all__ = [
+    "EntityResolver",
+    "ResolutionAction",
+    "ResolutionResult",
+    "MergeWorkflow",
+    "MergeProposal",
+    "MergeAuditRecord",
+]

apsimo/world_model/resolution/entity_resolver.py

@@ -0,0 +1,232 @@
+"""Entity resolver: resolves extraction candidates against the world model.
+
+Resolution priority order (first match wins):
+1. Exact external ID match   → MERGE (confidence ≥ 0.98)
+2. Exact canonical name match (case-insensitive) → MERGE (confidence ≥ 0.90)
+3. Alias list match          → MERGE (confidence ≥ 0.85)
+4. High string similarity (≥ 0.92) + same type → PROPOSE MERGE
+5. Property match (domain, email, ticker) → MERGE (confidence ≥ 0.90)
+6. Moderate similarity (0.75–0.92) + same type → FLAG
+7. No match                  → CREATE NEW ENTITY
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import List, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..store import WorldModelStore
+from ..extraction.conversation_extractor import ExtractionCandidate
+from ..entities import BaseEntity
+
+
+def _jaro_winkler(s1: str, s2: str) -> float:
+    """Compute Jaro-Winkler similarity between two strings.
+
+    Returns a value in [0.0, 1.0].
+    Pure Python implementation; no external dependencies.
+    """
+    if s1 == s2:
+        return 1.0
+    if not s1 or not s2:
+        return 0.0
+
+    s1 = s1.lower()
+    s2 = s2.lower()
+    len1, len2 = len(s1), len(s2)
+
+    match_distance = max(len1, len2) // 2 - 1
+    match_distance = max(match_distance, 0)
+
+    s1_matches = [False] * len1
+    s2_matches = [False] * len2
+
+    matches = 0
+    transpositions = 0
+
+    for i in range(len1):
+        start = max(0, i - match_distance)
+        end = min(i + match_distance + 1, len2)
+        for j in range(start, end):
+            if s2_matches[j] or s1[i] != s2[j]:
+                continue
+            s1_matches[i] = True
+            s2_matches[j] = True
+            matches += 1
+            break
+
+    if matches == 0:
+        return 0.0
+
+    k = 0
+    for i in range(len1):
+        if not s1_matches[i]:
+            continue
+        while not s2_matches[k]:
+            k += 1
+        if s1[i] != s2[k]:
+            transpositions += 1
+        k += 1
+
+    jaro = (
+        matches / len1 + matches / len2 + (matches - transpositions / 2) / matches
+    ) / 3
+
+    # Winkler prefix bonus (up to 4 chars)
+    prefix = 0
+    for i in range(min(4, len1, len2)):
+        if s1[i] == s2[i]:
+            prefix += 1
+        else:
+            break
+
+    return jaro + prefix * 0.1 * (1 - jaro)
+
+
+class ResolutionAction(str, Enum):
+    CREATE = "create"           # no match found; create new entity
+    MERGE = "merge"             # confident match; merge immediately
+    PROPOSE_MERGE = "propose"   # uncertain match; queue for review
+    FLAG = "flag"               # possible match; log and continue as new
+
+
+@dataclass
+class ResolutionResult:
+    action: ResolutionAction
+    matched_entity_id: Optional[str]  # existing entity ID if matched
+    match_confidence: float
+    match_reason: str              # human-readable explanation
+    candidate_id: Optional[str] = None  # new entity ID if CREATE
+
+
+class EntityResolver:
+    """Resolves extraction candidates against existing world model entities."""
+
+    def __init__(
+        self,
+        store: "WorldModelStore",
+        auto_merge_threshold: float = 0.85,
+        propose_merge_threshold: float = 0.70,
+        similarity_auto_merge: float = 0.92,
+        similarity_propose: float = 0.75,
+    ) -> None:
+        self._store = store
+        self._auto_merge_threshold = auto_merge_threshold
+        self._propose_threshold = propose_merge_threshold
+        self._sim_auto = similarity_auto_merge
+        self._sim_propose = similarity_propose
+
+    async def resolve(
+        self,
+        candidate: ExtractionCandidate,
+        entity_type: str,
+    ) -> ResolutionResult:
+        """Attempt to resolve a candidate to an existing entity.
+
+        Runs the priority-ordered resolution algorithm.
+        """
+        text = candidate.text.strip()
+
+        # ── Step 1: External ID match ─────────────────────────────────────
+        if "@" in text:
+            existing = await self._store.get_entity_by_external_id("email", text)
+            if existing:
+                return ResolutionResult(
+                    action=ResolutionAction.MERGE,
+                    matched_entity_id=existing.id,
+                    match_confidence=0.98,
+                    match_reason="Exact external ID match (email)",
+                )
+
+        # ── Step 2: Exact canonical name match ────────────────────────────
+        candidates = await self._store.find_entities(
+            query=text, entity_type=entity_type, min_confidence=0.0, limit=50
+        )
+        for entity in candidates:
+            if entity.name.lower() == text.lower():
+                return ResolutionResult(
+                    action=ResolutionAction.MERGE,
+                    matched_entity_id=entity.id,
+                    match_confidence=0.90,
+                    match_reason="Exact canonical name match",
+                )
+
+        # ── Step 3: Alias list match ──────────────────────────────────────
+        for entity in candidates:
+            for alias in entity.aliases:
+                if alias.lower() == text.lower():
+                    return ResolutionResult(
+                        action=ResolutionAction.MERGE,
+                        matched_entity_id=entity.id,
+                        match_confidence=0.85,
+                        match_reason=f"Alias match: '{alias}'",
+                    )
+
+        # ── Steps 4 & 6: String similarity ───────────────────────────────
+        best_sim = 0.0
+        best_entity: Optional[BaseEntity] = None
+        for entity in candidates:
+            sim = await self.compute_string_similarity(text, entity.name)
+            if sim > best_sim:
+                best_sim = sim
+                best_entity = entity
+
+        if best_entity and best_sim >= self._sim_auto:
+            return ResolutionResult(
+                action=ResolutionAction.PROPOSE_MERGE,
+                matched_entity_id=best_entity.id,
+                match_confidence=best_sim,
+                match_reason=f"High string similarity ({best_sim:.2f})",
+            )
+
+        if best_entity and best_sim >= self._sim_propose:
+            return ResolutionResult(
+                action=ResolutionAction.FLAG,
+                matched_entity_id=best_entity.id,
+                match_confidence=best_sim,
+                match_reason=f"Moderate string similarity ({best_sim:.2f})",
+            )
+
+        # ── Step 7: No match — create new ────────────────────────────────
+        return ResolutionResult(
+            action=ResolutionAction.CREATE,
+            matched_entity_id=None,
+            match_confidence=0.0,
+            match_reason="No match found",
+        )
+
+    async def compute_string_similarity(self, a: str, b: str) -> float:
+        """Jaro-Winkler similarity between two name strings."""
+        return _jaro_winkler(a, b)
+
+    async def resolve_entity_against_store(
+        self,
+        entity: BaseEntity,
+    ) -> ResolutionResult:
+        """Resolve a full BaseEntity against the store.
+
+        More thorough than ExtractionCandidate resolution: checks external IDs,
+        domain, ticker, etc.
+        """
+        # External ID matches
+        for key, value in entity.external_ids.items():
+            existing = await self._store.get_entity_by_external_id(key, value)
+            if existing and existing.id != entity.id:
+                return ResolutionResult(
+                    action=ResolutionAction.MERGE,
+                    matched_entity_id=existing.id,
+                    match_confidence=0.98,
+                    match_reason=f"Exact external ID match ({key}={value})",
+                )
+
+        # Name-based resolution
+        from ..extraction.conversation_extractor import ExtractionCandidate
+        candidate = ExtractionCandidate(
+            text=entity.name,
+            entity_type=entity.entity_type,
+            start_char=0,
+            end_char=len(entity.name),
+            confidence=entity.confidence,
+            context_window="",
+        )
+        return await self.resolve(candidate, entity.entity_type)

apsimo/world_model/resolution/merge_audit.py

@@ -0,0 +1,16 @@
+"""Merge audit record dataclass."""
+
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class MergeAuditRecord:
+    id: str                   # ma-<timestamp>-<random7>
+    surviving_id: str         # entity that survived
+    retired_id: str           # entity that was merged away
+    relationships_repointed: int
+    properties_updated: int
+    executed_by: str          # "auto" | "owner_approved"
+    merge_proposal_id: Optional[str]
+    created_at: str

apsimo/world_model/resolution/merge_workflow.py

@@ -0,0 +1,117 @@
+"""Merge workflow: automatic and proposed entity merges."""
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..store import WorldModelStore
+
+
+@dataclass
+class MergeProposal:
+    id: str                          # mp-<timestamp>-<random7>
+    candidate_id: str                # the incoming entity
+    existing_id: str                 # the entity to merge into
+    match_confidence: float
+    match_reason: str
+    evidence: Dict[str, Any] = field(default_factory=dict)
+    status: str = "pending"          # "pending" | "approved" | "rejected" | "expired"
+    created_at: Optional[str] = None
+    resolved_at: Optional[str] = None
+
+
+class MergeWorkflow:
+    """Manages proposed and automatic entity merges."""
+
+    def __init__(self, store: "WorldModelStore") -> None:
+        self._store = store
+
+    async def propose_merge(
+        self,
+        candidate_id: str,
+        existing_id: str,
+        confidence: float,
+        reason: str,
+        evidence: Optional[Dict[str, Any]] = None,
+    ) -> MergeProposal:
+        """Create a merge proposal and enqueue for owner review."""
+        proposal_id = await self._store._backend.create_merge_proposal(
+            candidate_id=candidate_id,
+            existing_id=existing_id,
+            confidence=confidence,
+            reason=reason,
+            evidence=evidence,
+        )
+        return MergeProposal(
+            id=proposal_id,
+            candidate_id=candidate_id,
+            existing_id=existing_id,
+            match_confidence=confidence,
+            match_reason=reason,
+            evidence=evidence or {},
+        )
+
+    async def execute_merge(self, proposal_id: str) -> None:
+        """Execute an approved merge. The candidate entity is retired.
+
+        All relationships pointing to candidate_id are repointed to
+        existing_id. A merge audit record is created.
+        """
+        proposal = await self._store._backend.get_merge_proposal(proposal_id)
+        if not proposal:
+            raise ValueError(f"Merge proposal not found: {proposal_id}")
+        if proposal["status"] not in ("pending", "approved"):
+            raise ValueError(
+                f"Cannot execute merge in status '{proposal['status']}'"
+            )
+
+        await self._store._backend.execute_merge(
+            surviving_id=proposal["existing_id"],
+            retired_id=proposal["candidate_id"],
+            executed_by="owner_approved",
+            proposal_id=proposal_id,
+        )
+        await self._store._backend.update_merge_proposal_status(
+            proposal_id, "approved"
+        )
+
+    async def execute_auto_merge(
+        self,
+        surviving_id: str,
+        retired_id: str,
+    ) -> str:
+        """Execute an automatic high-confidence merge without owner review.
+
+        Returns the merge audit record ID.
+        """
+        return await self._store._backend.execute_merge(
+            surviving_id=surviving_id,
+            retired_id=retired_id,
+            executed_by="auto",
+            proposal_id=None,
+        )
+
+    async def reject_merge(self, proposal_id: str) -> None:
+        """Reject a merge proposal.
+
+        Both entities remain separate. The candidate's aliases list is
+        updated to avoid re-proposing the same merge.
+        """
+        proposal = await self._store._backend.get_merge_proposal(proposal_id)
+        if not proposal:
+            raise ValueError(f"Merge proposal not found: {proposal_id}")
+
+        await self._store._backend.update_merge_proposal_status(
+            proposal_id, "rejected"
+        )
+
+        # Add candidate's name as alias on existing to avoid re-proposing
+        candidate = await self._store.get_entity(proposal["candidate_id"])
+        if candidate:
+            await self._store.add_entity_alias(
+                proposal["existing_id"], candidate.name
+            )
+
+    async def get_pending_proposals(self) -> list:
+        """Return all pending merge proposals."""
+        return await self._store._backend.get_pending_merge_proposals()

apsimo/world_model/source_reports.py

@@ -0,0 +1,121 @@
+"""Existing world batch extraction driven by attributed canonical user sources."""
+from contextlib import closing
+from datetime import datetime, timedelta, timezone
+import hashlib
+import json
+import re
+
+from apsimo.turns.idempotency import canonical_turn_digest, source_message_hash
+from apsimo.turns.source_attribution import is_invalidated
+from .observations import canonical
+
+
+# This first current-state producer abstains on dated/modally qualified
+# messages. The canonical memory retains them for its existing valid-time
+# parser; receipt time must never turn a historical/future claim into now.
+_NONCURRENT = re.compile(r"\b(will|would|was|were|had|used to|tomorrow|yesterday|last|next|previously|formerly|"
+    r"soon|later|eventually|before|after|until|since|starting|ended|stopped|might|may|could|should|"
+    r"if|unless|not|never|isn't|aren't|doesn't|didn't|weren't|wasn't|"
+    r"monday|tuesday|wednesday|thursday|friday|saturday|sunday|"
+    r"january|february|march|april|june|july|august|september|october|november|december)\b|"
+    r"\b\d{4}-\d{2}-\d{2}\b|\?", re.I)
+_PRESENT = re.compile(r"\b(is|are|runs|uses|has|needs|depends on)\b", re.I)
+
+
+def current_source(ledger, source):
+    with closing(ledger._connect()) as conn:
+        row = conn.execute("SELECT contact_id,scope,messages_json FROM turn_sources WHERE turn_id=?",
+                           (source['source_id'],)).fetchone()
+        return bool(row and row['contact_id'] == source['contact_id'] and row['scope'] == 'person'
+                    and not is_invalidated(conn, source['source_id'])
+                    and canonical_turn_digest(json.loads(row['messages_json'])) == source['source_version'])
+
+
+def recent_batches(ledger, *, hours=24, limit=30):
+    """Same bounded batch job, with one participant per batch and exact lineage.
+
+    Only ordinary learning-admitted user sources are eligible. Historical
+    source-only imports and unattributed checkpoints remain source-only.
+    """
+    cutoff = (datetime.now(timezone.utc) - timedelta(hours=hours)).isoformat()
+    with closing(ledger._connect()) as conn:
+        rows = conn.execute('''SELECT s.* FROM turn_sources s JOIN source_claim_jobs j ON j.turn_id=s.turn_id
+            WHERE s.scope='person' AND julianday(s.ingested_at)>=julianday(?)
+            ORDER BY s.ingested_at DESC,s.turn_id LIMIT ?''', (cutoff, max(1, min(limit, 100)))).fetchall()
+        grouped = {}
+        for row in rows:
+            if is_invalidated(conn, row['turn_id']):
+                continue
+            messages = json.loads(row['messages_json'])
+            version = canonical_turn_digest(messages)
+            for message in messages:
+                if message.get('role') != 'user':
+                    continue
+                content = message.get('content', '')
+                if isinstance(content, list):
+                    content = '\n'.join(b.get('text', '') for b in content if isinstance(b, dict)
+                                        and b.get('type') in {'text', 'input_text'})
+                if not isinstance(content, str) or not content.strip():
+                    continue
+                grouped.setdefault(row['contact_id'], []).append(dict(content=content[:600],
+                    current_time_eligible=not bool(_NONCURRENT.search(content)),
+                    source_id=row['turn_id'], source_version=version, contact_id=row['contact_id'],
+                    message_hash=source_message_hash(row['session_id'], message),
+                    observed_at=row['occurred_at'] or row['ingested_at'],
+                    time_basis='occurred_at' if row['occurred_at'] else 'received_at'))
+        return [items[start:start+10] for items in grouped.values() for start in range(0, len(items), 10)]
+
+
+async def record_reports(extractor, proposals, sources, name_to_id, mode, report):
+    """Quote-qualified reports, never sensor observations or proven properties."""
+    if not sources or extractor._source_ledger is None or mode != 'live':
+        return
+    report.setdefault('property_observations', [])
+    report.setdefault('property_skipped', 0)
+    for item in proposals[:25] if isinstance(proposals, list) else []:
+        if not isinstance(item, dict):
+            continue
+        name, key, quote = str(item.get('entity', '')).strip(), str(item.get('property', '')).strip(), item.get('evidence')
+        value = item.get('value')
+        eid = name_to_id.get(name.lower())
+        matches = [s for s in sources if isinstance(quote, str) and len(quote) >= 8 and quote in s['content']
+                   and s['current_time_eligible'] and item.get('temporal_status') == 'current'
+                   and _PRESENT.search(quote) and name.lower() in quote.lower()
+                   and isinstance(value, str) and value.lower() in quote.lower()]
+        # Ambiguous identical quotations across messages need explicit source
+        # selection in a later extraction, rather than an invented attribution.
+        if not eid or not key or len(matches) != 1 or not current_source(extractor._source_ledger, matches[0]):
+            report['property_skipped'] += 1
+            continue
+        source = matches[0]
+        oid = 'wo-report-' + hashlib.sha256(canonical([source['source_id'], source['source_version'], source['contact_id'],
+            source['message_hash'], eid, key, quote]).encode()).hexdigest()
+        at = datetime.fromisoformat(source['observed_at'].replace('Z', '+00:00'))
+        try:
+            result = await extractor._store.record_property_observation(observation_id=oid, entity_id=eid,
+                property_key=key, value=value, kind='reported', producer=f"contact:{source['contact_id']}",
+                evidence_refs=[f"source:{source['source_id']}", f"message:{source['message_hash']}"],
+                source_refs=[dict(source_id=source['source_id'], source_version=source['source_version'])],
+                observed_at=at.isoformat(), fresh_until=(at + timedelta(days=1)).isoformat(),
+                subject_person_id=source['contact_id'], viewer_scope=f"person:{source['contact_id']}",
+                shareability='subject_private')
+            if not current_source(extractor._source_ledger, source):
+                await extractor._store.erase_property_evidence([f"source:{source['source_id']}"],
+                                                               subject_person_id=source['contact_id'])
+                report['property_skipped'] += 1
+                continue
+            report['property_observations'].append(dict(observation_id=result['observation_id'],
+                entity_id=eid, property_key=key, kind='reported', time_basis=source['time_basis']))
+            report['writes'] += 1
+        except (ValueError, NotImplementedError):
+            report['property_skipped'] += 1
+
+
+def validate_reports(records, ledger):
+    """Hydration checks close the cross-store correction/erasure race."""
+    result = []
+    for row in records:
+        refs = row.get('source_refs', [])
+        valid = all(current_source(ledger, ref | {'contact_id': row['subject_person_id']}) for ref in refs) if ledger else not refs
+        result.append(row if valid else row | {'invalidated': True, 'value': None})
+    return result

apsimo/world_model/sqlite/__init__.py

@@ -0,0 +1,4 @@
+"""SQLite backend for the Colony World Model."""
+from .backend import SQLiteBackend
+
+__all__ = ["SQLiteBackend"]