apsimo 1.3.0__py3-none-any.whl

apsimo/world_model/observations.py

@@ -0,0 +1,155 @@
+"""Qualified property observations in the existing world observation journal.
+
+Stable entities describe what a thing is. This projection describes what was
+observed, reported or hypothesized about it, when, and for which audience.
+Confidence is deliberately absent: repetition is not independent evidence.
+The host authenticates observation producers and resolves canonical references.
+"""
+from datetime import datetime, timezone
+import json
+import re
+
+from apsimo.self_model.situation import _safe_id, _validate_scope
+
+KINDS = ('observed', 'reported', 'hypothesis')
+SOURCE_PREFIX = 'world-property-v1:'
+
+
+def canonical(value):
+    return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False)
+
+
+def epoch(value):
+    if isinstance(value, bool):
+        raise ValueError('invalid_world_observation_time')
+    if isinstance(value, (int, float)):
+        parsed = datetime.fromtimestamp(value, timezone.utc)
+    else:
+        parsed = datetime.fromisoformat(str(value).replace('Z', '+00:00'))
+    if parsed.tzinfo is None:
+        raise ValueError('world_observation_time_requires_timezone')
+    return parsed.timestamp()
+
+
+def iso(value):
+    return datetime.fromtimestamp(epoch(value), timezone.utc).isoformat()
+
+
+def observation(*, observation_id, entity_id, property_key, value, kind, producer,
+                evidence_refs, observed_at, fresh_until, subject_person_id,
+                viewer_scope, shareability, valid_from=None, valid_to=None, source_refs=()):
+    for key, item in [('observation_id', observation_id), ('entity_id', entity_id), ('producer', producer)]:
+        _safe_id(item, key)
+    if not isinstance(property_key, str) or not re.fullmatch('[a-z][a-z0-9_.-]{0,63}', property_key):
+        raise ValueError('invalid_world_property_key')
+    if kind not in KINDS:
+        raise ValueError('invalid_world_observation_kind')
+    if not isinstance(evidence_refs, (tuple, list)) or not 1 <= len(evidence_refs) <= 40:
+        raise ValueError('world_observation_requires_evidence')
+    refs = sorted({_safe_id(ref, 'evidence_ref') for ref in evidence_refs})
+    if not isinstance(source_refs, (list, tuple)) or len(source_refs) > 40:
+        raise ValueError('invalid_world_source_refs')
+    sources = []
+    for source in source_refs:
+        if (not isinstance(source, dict) or set(source) != {'source_id', 'source_version'}
+                or not isinstance(source['source_version'], str)
+                or not re.fullmatch('[0-9a-f]{64}', source['source_version'])):
+            raise ValueError('invalid_world_source_refs')
+        _safe_id(source['source_id'], 'source_id')
+        if 'source:' + source['source_id'] not in refs:
+            raise ValueError('world_source_ref_requires_evidence')
+        sources.append(dict(source))
+    # Preserve existing source/receipt/media identifiers; do not invent text
+    # claims or upload referenced media into this journal.
+    subject, viewer, sharing = _validate_scope(subject_person_id, viewer_scope, shareability)
+    observed, fresh = iso(observed_at), iso(fresh_until)
+    start, end = iso(valid_from if valid_from is not None else observed_at), iso(valid_to) if valid_to is not None else None
+    if epoch(fresh) < epoch(observed) or epoch(fresh) - epoch(observed) > 366 * 86400:
+        raise ValueError('world_observation_freshness_must_be_bounded')
+    if end is not None and epoch(end) <= epoch(start):
+        raise ValueError('invalid_world_observation_validity')
+    if len(canonical(value).encode()) > 8192:
+        raise ValueError('world_observation_value_too_large')
+    return dict(schema='WorldPropertyObservationV1', observation_id=observation_id,
+        entity_id=entity_id, property_key=property_key, value=value, kind=kind, producer=producer,
+        evidence_refs=refs, source_refs=sorted(sources, key=lambda s: s['source_id']),
+        observed_at=observed, fresh_until=fresh, valid_from=start, valid_to=end,
+        subject_person_id=subject, viewer_scope=viewer, shareability=sharing)
+
+
+def project(records, *, entity_id, property_key, subject_person_id, viewer_scope,
+            shareability, as_of=None, coverage_limited=False):
+    """Observed-time view. Delayed arrival cannot win over a newer observation.
+
+    Each producer's latest applicable report is considered separately. Fresh
+    observations outrank reports, which outrank hypotheses, but disagreement
+    remains explicit. A stale latest observation cannot revive its older value.
+    ``as_of`` answers what held then using available evidence, not what the
+    system had already ingested then.
+    """
+    _validate_scope(subject_person_id, viewer_scope, shareability)
+    now = epoch(as_of if as_of is not None else datetime.now(timezone.utc).isoformat())
+    visible = []
+    for row in records:
+        if (row.get('schema') != 'WorldPropertyObservationV1' or row.get('entity_id') != entity_id
+                or row.get('property_key') != property_key):
+            continue
+        if (row['subject_person_id'], row['viewer_scope'], row['shareability']) != (subject_person_id, viewer_scope, shareability):
+            continue
+        if epoch(row['observed_at']) > now or epoch(row['valid_from']) > now:
+            continue
+        visible.append(row)
+    # Repackaging one receipt under fresh IDs cannot renew freshness or create
+    # independent corroboration. Keep its first observed occurrence per value.
+    dedup = {}
+    for row in sorted(visible, key=lambda r: (epoch(r['observed_at']), r['observation_id'])):
+        key = (row['producer'], row['kind'], tuple(row['evidence_refs']), canonical(row['value']))
+        dedup.setdefault(key, row)
+    latest = {}
+    for row in dedup.values():
+        key = (row['producer'], row['kind'])
+        moment = epoch(row['observed_at'])
+        if key not in latest or moment > epoch(latest[key][0]['observed_at']):
+            latest[key] = [row]
+        elif moment == epoch(latest[key][0]['observed_at']):
+            latest[key].append(row)
+    candidates = [dict(row, freshness='invalidated' if row.get('invalidated') else 'fresh' if epoch(row['fresh_until']) > now and
+        (row['valid_to'] is None or epoch(row['valid_to']) > now) else 'stale')
+        for rows in latest.values() for row in rows]
+    current = [row for row in candidates if row['freshness'] == 'fresh']
+    best = next(([r for r in current if r['kind'] == kind] for kind in KINDS
+                 if any(r['kind'] == kind for r in current)), [])
+    values = {canonical(row['value']) for row in best}
+    state = 'conflicted' if len(values) > 1 else 'current' if best else 'stale' if any(
+        r['freshness'] == 'stale' for r in candidates) else 'unknown'
+    if coverage_limited:
+        state = 'unknown'
+    selected = best if state == 'current' else []
+    value = selected[0]['value'] if selected else None
+    refs = sorted({ref for row in best for ref in row['evidence_refs']})
+    return dict(schema='WorldPropertyStateV1', entity_id=entity_id, property_key=property_key,
+        subject_person_id=subject_person_id, viewer_scope=viewer_scope, shareability=shareability,
+        state=state, value=value, kind=selected[0]['kind'] if selected else None,
+        as_of=datetime.fromtimestamp(now, timezone.utc).isoformat(), time_basis='observed_at',
+        evidence_refs=refs, observations=sorted(candidates, key=lambda r: (r['producer'], r['kind'], r['observation_id'])),
+        has_disagreement=len({canonical(r['value']) for r in current}) > 1,
+        coverage_limited=coverage_limited, authority_granted=False)
+
+
+def compact_situation(snapshot, *, limit=12):
+    """Scoped present-tense facts for context; stale values never become current."""
+    if snapshot is None:
+        return dict(schema='WorldSituationViewV1', state='unknown', facts=[], stale=[], authority_granted=False)
+    limit = max(1, min(int(limit), 32))
+    def compact(fact):
+        return {key: fact.public()[key] for key in ('observation_id', 'category', 'entity_id', 'state', 'active',
+            'observed_at', 'fresh_until', 'freshness', 'evidence_refs', 'attributes')}
+    facts = [compact(f) for f in snapshot.facts if f.freshness == 'fresh']
+    stale = [dict(entity_id=f.entity_id, category=f.category, state='unknown', last_observed_state=f.state,
+                  observed_at=f.observed_at, fresh_until=f.fresh_until, evidence_refs=list(f.evidence_refs))
+             for f in snapshot.facts if f.freshness != 'fresh']
+    return dict(schema='WorldSituationViewV1', snapshot_id=snapshot.snapshot_id, as_of=snapshot.as_of,
+        subject_person_id=snapshot.subject_person_id, viewer_scope=snapshot.viewer_scope,
+        shareability=snapshot.shareability, state='current' if facts else 'stale' if stale else 'unknown',
+        facts=facts[:limit], stale=stale[:limit], omitted=max(0, len(facts)-limit)+max(0, len(stale)-limit),
+        authority_granted=False)

apsimo/world_model/populator.py

@@ -0,0 +1,307 @@
+"""WorldModelPopulator -- turn conversation into structured world knowledge.
+
+Wires the (already-existing, rule-based, zero-LLM) ConversationExtractor to the
+world-model store so Colony actually learns the owner's people, companies,
+projects and products from what he says. This was the missing link that left
+the world model empty.
+
+Observability-first and reversible:
+  * mode "off"    -- do nothing.
+  * mode "shadow" -- extract + resolve + log EXACTLY what it WOULD populate,
+                     write nothing (default).
+  * mode "live"   -- upsert entities (dedup via EntityResolver) + light
+                     relationships into the local world-model store only.
+
+Every candidate is boundary-checked: if the owner set a directive to leave a
+subject alone, it is never added to the world model.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+from apsimo.world_model.extraction.conversation_extractor import (
+    ConversationExtractor, ExtractionCandidate,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def populate_mode() -> str:
+    from apsimo.util.autonomy_preset import resolve
+    return resolve("COLONY_WORLD_POPULATE_MODE",
+                   ("off", "shadow", "live"), "shadow")
+
+
+@dataclass
+class PopulationReport:
+    source_id: str = ""
+    mode: str = "shadow"
+    created: List[Dict[str, Any]] = field(default_factory=list)   # {name,type,confidence}
+    merged: List[Dict[str, Any]] = field(default_factory=list)    # {name,type,into}
+    proposed: List[Dict[str, Any]] = field(default_factory=list)
+    relationships: List[Dict[str, Any]] = field(default_factory=list)  # {source,rel,target}
+    skipped_boundary: List[str] = field(default_factory=list)
+
+    def total(self) -> int:
+        return len(self.created) + len(self.merged) + len(self.proposed)
+
+
+# Person <-> Company work link cues (person <cue> company).
+_WORK_CUE = re.compile(
+    r"\b(?:works?\s+(?:at|for)|employed\s+at|ceo\s+of|cto\s+of|founder\s+of|"
+    r"co-?founder\s+of|runs?|leads?|heads?|at)\b", re.IGNORECASE,
+)
+
+# Lowercase connectives that only appear inside a sentence, never in a real
+# entity NAME. Their presence marks a candidate as a sentence fragment.
+_FRAGMENT_WORDS = frozenset({
+    "the", "a", "an", "who", "whom", "that", "which", "and", "or", "but",
+    "at", "in", "on", "of", "for", "to", "with", "about", "met", "meet",
+    "works", "work", "working", "is", "are", "was", "were", "said", "says",
+    "from", "by", "joined", "join", "called", "told", "asked",
+})
+
+
+def _looks_like_fragment(name: str) -> bool:
+    """A real entity name is a few proper tokens, not a clause."""
+    toks = name.split()
+    if len(toks) > 5:
+        return True
+    lows = {t.lower().strip(".,") for t in toks}
+    return bool(lows & _FRAGMENT_WORDS)
+
+
+# Generic role/system words + calendar words that rule-based NER mis-tags as
+# people/places. Deployment-specific role names (the assistant persona, host
+# agent, etc.) are added from config/env at runtime -- never hardcoded here.
+_GENERIC_NOISE = frozenset({
+    "agent", "assistant", "system", "user", "bot", "ai",
+    "gpt", "model", "human", "owner",
+    "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
+    "january", "february", "march", "april", "may", "june", "july", "august",
+    "september", "october", "november", "december", "today", "tomorrow",
+    "yesterday", "morning", "evening", "tonight", "week", "month", "year",
+})
+
+
+def _noise_names() -> frozenset:
+    """Generic role/calendar noise + any deployment-specific role names from
+    env (persona/agent name, plus COLONY_WORLD_NOISE_NAMES)."""
+    extra = set()
+    for var in ("COLONY_PERSONA_NAME", "COLONY_AGENT_NAME"):
+        v = os.environ.get(var, "").strip().lower()
+        if v:
+            extra.add(v)
+    for t in os.environ.get("COLONY_WORLD_NOISE_NAMES", "").split(","):
+        t = t.strip().lower()
+        if t:
+            extra.add(t)
+    return _GENERIC_NOISE | extra
+
+
+# Common English nouns/verbs that never form part of a real person's name.
+# Title-cased operational phrases ("Root Cause", "Orphan Messages") are the
+# dominant person-noise class observed in live data.
+_NON_NAME_WORDS = frozenset({
+    "cause", "root", "message", "messages", "session", "sessions",
+    "initiative", "initiatives", "operation", "operations", "orphan",
+    "fresh", "spawn", "error", "errors", "task", "tasks", "queue", "memory",
+    "review", "reviews", "summary", "report", "reports", "update", "updates",
+    "status", "issue", "issues", "problem", "problems", "fix", "fixes",
+    "bug", "bugs", "test", "tests", "daily", "weekly", "briefing", "digest",
+    "reminder", "alert", "alerts", "notification", "delivery", "channel",
+    "gateway", "bridge", "worker", "workers", "pending", "stale", "self",
+    "improvement", "cleanup", "backlog", "audit", "check", "checks",
+})
+
+
+def _is_low_quality(name: str, etype: str) -> bool:
+    """High-precision gate for world-model population.
+
+    Rule-based NER is noisy on conversation; drop the dominant noise classes so
+    the world model is trustworthy rather than full of sentence-initial words.
+    """
+    low = name.strip().lower()
+    _noise = _noise_names()
+    if low in _noise:
+        return True
+    # URLs, paths, emails, and placeholder fragments are never entity NAMES
+    # (they belong in external_ids/properties, not as canonical names).
+    if ("://" in low or low.startswith(("http", "www."))
+            or low.endswith(("...", "/")) or "/" in name.strip()):
+        return True
+    toks = name.split()
+    # Any token being a role/system/greeting word taints the whole name
+    # (e.g. a greeting + the assistant's own name, or "Agent ...").
+    tok_lows = {t.lower().strip(".,") for t in toks}
+    if tok_lows & (_noise | {"hey", "hi", "hello", "yo", "ok", "okay", "thanks"}):
+        return True
+    # Persons/locations must be multi-token proper names (single capitalized
+    # words at sentence start are the main noise source). Companies/products
+    # come from structured signals (org-suffix, domain, url) and are kept.
+    if etype in ("person", "location") and len(toks) < 2:
+        return True
+    # A "person" whose name contains a common operational noun is a
+    # title-cased phrase, not a human ("Root Cause", "Orphan Messages").
+    if etype == "person" and tok_lows & _NON_NAME_WORDS:
+        return True
+    return False
+
+
+class WorldModelPopulator:
+    def __init__(
+        self,
+        store: Any,
+        directive_manager: Any = None,
+        mode: Optional[str] = None,
+        min_confidence: float = 0.30,
+    ) -> None:
+        self._store = store
+        self._directives = directive_manager
+        self._mode = mode or populate_mode()
+        self._min_conf = min_confidence
+        self._extractor = ConversationExtractor()
+        self._resolver = None
+        if store is not None:
+            try:
+                from apsimo.world_model.resolution.entity_resolver import EntityResolver
+                self._resolver = EntityResolver(store)
+            except Exception as exc:
+                logger.debug("EntityResolver unavailable: %s", exc)
+
+    @property
+    def mode(self) -> str:
+        return self._mode
+
+    def _boundary_ok(self, name: str) -> bool:
+        if self._directives is None:
+            # No directive manager configured: nothing to check against
+            # (distinct from "checked and allowed" — see log line).
+            logger.debug("world populate boundary UNCHECKED for %r "
+                         "(no directive manager)", name)
+            return True
+        try:
+            from apsimo.directives import Action
+            return self._directives.check(Action(kind="populate", text=name, target=name)).allowed
+        except Exception:
+            # Fail closed, matching directives.guard.boundary_fail_closed():
+            # an error while evaluating an owner boundary must refuse.
+            logger.warning("world populate boundary check errored for %r "
+                           "— failing closed", name, exc_info=True)
+            return False
+
+    async def populate_from_text(self, text: str, source_id: str) -> PopulationReport:
+        report = PopulationReport(source_id=source_id, mode=self._mode)
+        if self._mode == "off" or self._store is None or not text:
+            return report
+
+        try:
+            extraction = await self._extractor.extract(text, source_id)
+        except Exception as exc:
+            logger.debug("world populate extract failed: %s", exc)
+            return report
+
+        # Dedup within the message + confidence + boundary filter.
+        candidates: List[ExtractionCandidate] = []
+        seen = set()
+        for c in extraction.entities:
+            key = (c.text.strip().lower(), c.entity_type)
+            if not c.text.strip() or key in seen or c.confidence < self._min_conf:
+                continue
+            seen.add(key)
+            # Reject sentence-fragment "names" and low-quality/noise candidates
+            # that slip through rule-based NER (high-precision population).
+            if _looks_like_fragment(c.text) or _is_low_quality(c.text, c.entity_type):
+                continue
+            if not self._boundary_ok(c.text):
+                report.skipped_boundary.append(c.text)
+                continue
+            candidates.append(c)
+
+        name_to_type: Dict[str, str] = {}
+        for c in candidates:
+            name_to_type[c.text] = c.entity_type
+            await self._handle_candidate(c, report)
+
+        self._infer_relationships(text, candidates, report)
+        return report
+
+    async def _handle_candidate(self, c: ExtractionCandidate, report: PopulationReport) -> None:
+        action = "create"
+        matched = None
+        if self._resolver is not None:
+            try:
+                res = await self._resolver.resolve(c, c.entity_type)
+                action = getattr(res.action, "value", str(res.action))
+                matched = res.matched_entity_id
+            except Exception as exc:
+                logger.debug("resolve failed for %r: %s", c.text, exc)
+
+        rec = {"name": c.text, "type": c.entity_type, "confidence": round(c.confidence, 2)}
+        if action == "merge":
+            rec["into"] = matched
+            report.merged.append(rec)
+            if self._mode == "live" and matched:
+                try:
+                    await self._store.add_entity_alias(matched, c.text)
+                except Exception:
+                    pass
+                # Repeat mention (covers exact-match, external-id and alias
+                # resolves — they all arrive here as MERGE): the entity was
+                # just seen again, so touch last_seen / mention_count /
+                # confidence. Reinforcement only — a repeat-mention must never
+                # make an entity MORE prunable than a single mention.
+                try:
+                    await self._store.reinforce_entity(matched)
+                except Exception as exc:
+                    logger.debug("reinforce_entity failed for %r: %s", matched, exc)
+            return
+        if action == "propose":
+            rec["near"] = matched
+            report.proposed.append(rec)
+            # in live mode a proposal still creates the candidate + queues review,
+            # but to stay conservative we only create on a clean CREATE.
+            return
+        # CREATE
+        report.created.append(rec)
+        if self._mode == "live":
+            try:
+                from apsimo.world_model.entities import ENTITY_CLASS_MAP, BaseEntity
+                from apsimo.world_model.sqlite.backend import _generate_id
+                cls = ENTITY_CLASS_MAP.get(c.entity_type, BaseEntity)
+                ent = cls(id=_generate_id("we"), name=c.text,
+                          entity_type=c.entity_type, confidence=c.confidence)
+                await self._store.upsert_entity(ent)
+            except Exception as exc:
+                logger.debug("upsert_entity failed for %r: %s", c.text, exc)
+
+    def _infer_relationships(self, text: str, candidates: List[ExtractionCandidate],
+                             report: PopulationReport) -> None:
+        """Light, conservative relationship inference (name-level in the report).
+
+        Only person<->company WORKS_AT when a clear work cue links them, and
+        person<->person KNOWS co-occurrence at low confidence. Written in live
+        mode is deferred to LLM-assist; here we surface them for observability.
+        """
+        persons = [c for c in candidates if c.entity_type == "person"]
+        companies = [c for c in candidates if c.entity_type == "company"]
+        for p in persons:
+            for co in companies:
+                lo, hi = sorted([p.start_char, co.start_char])
+                between = text[lo:hi]
+                if _WORK_CUE.search(between) or _WORK_CUE.search(
+                    text[max(0, p.start_char - 20):p.start_char]
+                ):
+                    report.relationships.append(
+                        {"source": p.text, "rel": "WM_WORKS_AT", "target": co.text,
+                         "confidence": 0.5})
+        for i in range(len(persons)):
+            for j in range(i + 1, len(persons)):
+                report.relationships.append(
+                    {"source": persons[i].text, "rel": "WM_KNOWS",
+                     "target": persons[j].text, "confidence": 0.3})

apsimo/world_model/postgres/__init__.py

@@ -0,0 +1 @@
+# Postgres backend for World Model