mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/memory/cli.py

@@ -0,0 +1,207 @@
+"""`mobster memory`: see and edit what Mobster remembers (devtools.py creates its parser with fixed help).
+
+It reads and writes the same file as the Mac app (user_data_dir()/memory/memory.sqlite3), so a change here shows in
+Settings › Memory the next time it's opened. ``run(args)`` returns 0 done, 1 refused or not found, 2 usage, 3 memory
+can't be opened here (no home folder, or saved by a newer Mobster), 130 ctrl+c. It never raises.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+from ..api_errors import APIError
+from .store import CONFIRM
+
+
+def add_arguments(parser, helpers):
+    subs = parser.add_subparsers(dest="memory_command", metavar="<subcommand>")
+
+    def as_json(sub):
+        sub.add_argument("--json", action="store_true", help="print JSON")
+
+    listing = subs.add_parser("list", help="list what Mobster remembers (the default)",
+                              description="List what Mobster remembers: pinned things first, then the newest.")
+    # dest "bundle": `--app` before a command's name is the terminal UI's (__main__.check_placement reads args.app).
+    listing.add_argument("--app", dest="bundle", metavar="BUNDLE",
+                         help="only what Mobster remembers for this app (its bundle ID)")
+    listing.add_argument("--scope", metavar="SCOPE", help="only this scope: global, app:<bundle>, project:<id>, "
+                                                          "profile:<id>, workflow:<id>, "
+                                                          "profile:<id>/workflow:<id> or phone:<id>")
+    listing.add_argument("--kind", metavar="KIND", help="only this kind (path, place, value, preference, rule, "
+                                                        "structure, device, outcome, cost or repo)")
+    listing.add_argument("--new", action="store_true", help="only what Mobster learned since you last looked")
+    as_json(listing)
+
+    add = subs.add_parser("add", help="tell Mobster something to remember",
+                          description="Tell Mobster something to remember, in your words (up to 300 characters). "
+                                      "Pinned things go with eligible tasks in their scope; "
+                                      "the rest go with tasks that fit. "
+                                      "Mobster never remembers passwords, codes or card numbers.",
+                          epilog='examples:\n  mobster memory add "My gym is the one on 5th Street"\n'
+                                 '  mobster memory add "Sign my texts with – Sam" --app com.apple.MobileSMS --pin')
+    add.add_argument("text", metavar="TEXT", help="what to remember")
+    add.add_argument("--app", dest="bundle", metavar="BUNDLE", help="only for tasks in this app (its bundle ID)")
+    add.add_argument("--scope", metavar="SCOPE", help="where it applies: app:<bundle>, project:<id>, profile:<id>, "
+                                                      "workflow:<id>, profile:<id>/workflow:<id> or phone:<id> "
+                                                      "(default: everywhere)")
+    add.add_argument("--kind", metavar="KIND", help="what it is (default: value, or preference when pinned)")
+    add.add_argument("--pin", action="store_true",
+                     help="send it with every eligible task in its scope (up to 10 pinned)")
+    as_json(add)
+
+    remove = subs.add_parser("rm", aliases=["forget"], help="forget one thing",
+                             description="Forget one thing, by the ID `list` shows. Mobster won't learn it again "
+                                         "for 90 days.")
+    remove.add_argument("id", metavar="ID", help="the 12-character ID from `mobster memory list`")
+    keep = subs.add_parser("keep", help="keep something Mobster learned",
+                           description="Keep one thing Mobster learned: it stops showing as new and is never "
+                                       "offered for removal.")
+    keep.add_argument("id", metavar="ID", help="the 12-character ID from `mobster memory list`")
+
+    pin = subs.add_parser("pin", help="send one thing with every eligible task in its scope",
+                          description="Pin one thing for every eligible task in its scope (up to 10 pinned).")
+    pin.add_argument("id", metavar="ID", help="the 12-character ID from `mobster memory list`")
+    unpin = subs.add_parser("unpin", help="send one thing only with eligible tasks that fit",
+                            description="Unpin one thing: it goes only with eligible tasks that fit.")
+    unpin.add_argument("id", metavar="ID", help="the 12-character ID from `mobster memory list`")
+
+    clear = subs.add_parser("clear", help="delete everything Mobster remembers",
+                            description="Delete everything Mobster remembers, and its suggestions. Asks you to "
+                                        "type \"delete everything\" first, unless you add --yes.")
+    clear.add_argument("--yes", action="store_true", help="don't ask")
+
+    export = subs.add_parser("export", help="print everything Mobster remembers as Markdown",
+                             description="Print everything Mobster remembers as Markdown, or save it with --output.")
+    export.add_argument("--output", metavar="FILE", help="save it to this file instead")
+    export.add_argument("--json", action="store_true", help="export JSON with every fact's kind, source and scope")
+
+
+def _print(text=""):
+    try:
+        print(text, flush=True)
+    except BrokenPipeError:
+        pass
+
+
+def _fail(message, code=1):
+    print(f"mobster memory: {message}", file=sys.stderr, flush=True)
+    return code
+
+
+def _scope(args):
+    scope = getattr(args, "scope", None)
+    bundle = getattr(args, "bundle", None)
+    if scope and bundle:
+        raise APIError("Use --scope or --app, not both.", 400, "invalid_scope")
+    return scope or (f"app:{bundle}" if bundle else None)
+
+
+def _line(fact):
+    where = (f"In {fact['app']['name']}: " if fact.get("app")
+             else f"[{fact['scopeKind']} {fact['scopeId']}] " if fact["scopeKind"] != "global" else "")
+    marks = [mark for mark, on in (("pinned", fact["pinned"]), ("new", fact["new"]),
+                                   (f"from {fact['source']}", fact["source"] != "user")) if on]
+    kind = f"{fact['kind']:<10} " if fact["source"] != "user" else ""
+    return f"{fact['id']}  {kind}{where}{fact['text']}{'  (' + ', '.join(marks) + ')' if marks else ''}"
+
+
+def run(args):
+    from .store import default_store
+    command = getattr(args, "memory_command", None) or "list"
+    store = default_store()
+    if store is None:
+        return _fail("Mobster can't keep memory here: there's no home folder (HOME isn't set).", 3)
+    try:
+        return COMMANDS[command](store, args)
+    except APIError as error:
+        if error.status == 503:
+            return _fail(str(error), 3)
+        return _fail(str(error), 1)
+    except KeyboardInterrupt:
+        return 130
+
+
+def _list(store, args):
+    facts = store.list_facts(scope=_scope(args), kind=getattr(args, "kind", None), new=getattr(args, "new", False))
+    if getattr(args, "json", False):
+        _print(json.dumps({"facts": facts}, ensure_ascii=False))
+        return 0
+    if not facts:
+        _print("Mobster doesn't remember anything yet. Tell it something:\n"
+               '  mobster memory add "My gym is the one on 5th Street"')
+        return 0
+    for fact in facts:
+        _print(_line(fact))
+    count = len(facts)
+    _print(f"\n{count} {'thing' if count == 1 else 'things'}. Pinned ones go with eligible tasks in their scope; "
+           "the rest go with tasks that fit. Everything stays on this Mac.")
+    return 0
+
+
+def _add(store, args):
+    fact = store.add_fact(args.text, _scope(args) or "global", pinned=args.pin, kind=args.kind)
+    if args.json:
+        _print(json.dumps({"fact": fact}, ensure_ascii=False))
+    else:
+        _print(f"Mobster will remember: {fact['text']}  ({fact['id']})")
+    return 0
+
+
+def _remove(store, args):
+    fact = store.forget(args.id)
+    _print(f"Forgot: {fact['text']}")
+    return 0
+
+
+def _keep(store, args):
+    _print(f"Kept: {store.keep(args.id)['text']}")
+    return 0
+
+
+def _pin(pinned):
+    def change(store, args):
+        fact = store.update_fact(args.id, pinned=pinned)
+        _print(f"{'Pinned' if pinned else 'Unpinned'}: {fact['text']}")
+        return 0
+    return change
+
+
+def _clear(store, args):
+    count = store.count()
+    if not args.yes:
+        if not sys.stdin.isatty():
+            return _fail("Add --yes to delete everything Mobster remembers without being asked.", 2)
+        try:
+            typed = input(f'Delete all {count} {"thing" if count == 1 else "things"} Mobster remembers? '
+                          f'Type "{CONFIRM}" to confirm: ')
+        except EOFError:
+            typed = ""
+        if typed.strip().lower() != CONFIRM:
+            _print("Nothing was deleted.")
+            return 1
+    deleted = store.clear()
+    _print(f"Deleted everything Mobster remembered ({deleted['facts']} "
+           f"{'thing' if deleted['facts'] == 1 else 'things'}).")
+    return 0
+
+
+def _export(store, args):
+    text = (json.dumps(store.export_json(), ensure_ascii=False, indent=2) + "\n"
+            if args.json else store.export_markdown())
+    if args.output:
+        try:
+            Path(args.output).expanduser().write_text(text, encoding="utf-8")
+        except OSError as error:
+            return _fail(f"Couldn't write {args.output}: {error.strerror or error}.", 1)
+        _print(f"Saved to {args.output}")
+        return 0
+    try:
+        sys.stdout.write(text)
+        sys.stdout.flush()
+    except BrokenPipeError:
+        pass
+    return 0
+
+
+COMMANDS = {"list": _list, "add": _add, "rm": _remove, "forget": _remove, "keep": _keep, "pin": _pin(True),
+            "unpin": _pin(False), "clear": _clear, "export": _export}

mobile_agent/memory/extract.py

@@ -0,0 +1,191 @@
+"""Suggestions to remember something, read out of what the person wrote (SPEC §3.3 F3).
+
+Deterministic: no model call, so no prompt changes and no cost. It looks for the few ways people say something worth
+keeping, sentence by sentence:
+
+- "remember (that) …", "don't forget (that) …", "keep in mind …"  ->  the rest, as they said it
+- "my X is Y"                       ->  "My X is Y" (X up to four words; never a passing state such as "my battery
+                                        is low", never inside a question, never inside a task such as "text Sam that
+                                        my train is late")
+- "I always / never / usually / prefer …", "I'd rather …"  ->  as said (a preference, so it's suggested pinned)
+- "call me …"                       ->  "Call me …" (suggested pinned)
+
+A suggestion is only that: nothing is stored until the person accepts it (store.MemoryStore.resolve_proposal).
+"""
+
+import re
+from dataclasses import dataclass
+
+from ..secret_filter import is_secret
+
+MAX_PER_TEXT = 2
+MIN_CHARS = 8
+
+_BREAK = re.compile(r"[.!?;]+\s+|\n+")
+# The word right before a full stop. Searched in a short window before the break (the lookbehind still sees the text
+# before it), so a message full of "Dr. Dr. Dr." stays linear: a 4,000-character one took half a second before.
+_LAST_WORD = re.compile(r"(?<![\w.])([\w.]+)[.]$")
+_WORD_WINDOW = 24
+_ABBREVIATIONS = {"dr", "mr", "mrs", "ms", "mx", "prof", "st", "ave", "rd", "blvd", "apt", "no", "vs", "etc", "jr",
+                  "sr", "mt", "ft", "e.g", "i.e", "approx", "dept", "inc", "ltd", "co"}
+_REMEMBER = re.compile(r"^(?:(?:please|pls|and|also|oh|ok|okay|so|hey)[,\s]+)*"
+                       r"(?:(?:can|could|would|will) you\s+)?(?:please\s+)?"
+                       r"(?:remember|don'?t forget|don’t forget|do not forget|keep in mind)\b[,:]?\s*"
+                       r"(?:that\s+)?(?P<rest>.+)$", re.I)
+_MY_IS = re.compile(r"\bmy\s+(?P<thing>(?:[\w'’-]+\s+){0,3}?[\w'’-]+)\s+(?P<verb>is|are)\s+(?P<value>.+)$", re.I)
+_PREFERENCE = re.compile(r"\b(?P<rest>I(?:\s+(?:always|never|usually|prefer|would rather)|['’]d rather)\s+"
+                         r"(?P<next>[\w'’-]+).*)$", re.I)
+_CALL_ME = re.compile(r"\b(?i:call me)\s+(?P<name>[A-Z][\w'’-]*(?:\s+[A-Z][\w'’-]*)?)")
+
+_PASSING = {"battery", "phone", "iphone", "screen", "order", "package", "delivery", "flight", "train", "bus",
+            "ride", "uber", "car", "wifi", "wi-fi", "internet", "connection", "data", "storage", "volume",
+            "alarm", "timer", "meeting", "call", "inbox", "cart", "code", "password", "passcode", "pin", "otp",
+            "account", "app", "message", "email", "text", "reply", "question", "answer", "task", "request",
+            "food", "coffee", "lunch", "dinner", "breakfast", "day", "week", "morning", "night", "plan", "plans",
+            "schedule", "calendar", "location", "card", "balance", "bill", "payment", "verification", "turn",
+            "time", "guess", "point", "bad", "fault", "pleasure", "problem", "issue", "mistake", "head", "back"}
+_PASSING_VALUE = re.compile(r"^(?:low|dead|dying|charging|broken|late|early|running|on|off|full|empty|down|up|"
+                            r"out|here|there|ready|done|gone|lost|stuck|frozen|slow|fast|open|closed|busy|free|"
+                            r"almost|nearly|about|now|today|tomorrow|tonight|still|already|not|being|getting|"
+                            r"going|coming|at\s+\d|\d+\s*%|\d+\s*percent)\b", re.I)
+_PRONOUN_VALUE = re.compile(r"^(?:it|this|that|these|those|them|him|her)\b", re.I)
+_QUESTION_START = re.compile(r"^(?:what|when|where|who|whom|which|why|how|is|are|does|do|did|can|could|would|"
+                             r"should|will|whats|what's)\b", re.I)
+_COMMAND = re.compile(r"^(?:text|send|message|call|email|reply|tell|ask|book|buy|order|post|delete|open|find|check|"
+                      r"show|look|search|set|turn|add|remind|schedule|play|go|get|make|write|create|share|pay|move|"
+                      r"copy|read|say|let)\b", re.I)
+_PAST = re.compile(r"^(?:\w+ed|got|saw|heard|went|did|had|was|were|said|sent|made|knew|thought|told|took|came|"
+                   r"found|gave|left|read|ran|bought|paid|met|forgot|lost|won|ate|drank|wrote|spoke)$", re.I)
+_WHEN_WORDS = {"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", "tomorrow", "today",
+               "tonight", "later", "back", "asap", "when", "if", "after", "before", "at", "on", "in", "around"}
+
+
+@dataclass(frozen=True)
+class Candidate:
+    text: str
+    pin: bool = False       # a preference or a name: it helps in every task, so it's suggested pinned
+
+
+def _tidy(text):
+    text = " ".join(text.split()).strip(" ,;:-–—\"'“”‘’")
+    text = re.sub(r"[.!]+$", "", text).strip()
+    if text and text[0].islower():
+        text = text[0].upper() + text[1:]
+    return text
+
+
+def _inside_task(before):
+    """The words before a match make it part of a task or a quote ("Text Sam that …", "Tell her I always …")."""
+    before = before.strip(" ,")
+    if not before:
+        return False
+    words = re.sub(r"^(?:(?:please|pls|and|also|oh|ok|okay|so|hey|then|now)[,\s]+)*", "", before, flags=re.I)
+    return bool(_COMMAND.match(words) or re.search(r"\b(?:that|say|says|said|saying|tell|telling|asks?)$", before,
+                                                   re.I))
+
+
+def _sentences(text):
+    start = 0
+    for found in _BREAK.finditer(text):
+        word = _LAST_WORD.search(text, max(start, found.start() - _WORD_WINDOW), found.start() + 1)
+        if found.group().startswith(".") and word and (word.group(1).lower() in _ABBREVIATIONS
+                                                        or re.fullmatch(r"[A-Z]", word.group(1))):
+            continue
+        part = text[start:found.start() + (0 if found.group()[0] == "\n" else 1)].strip()
+        start = found.end()
+        if part:
+            yield part
+    part = text[start:].strip()
+    if part:
+        yield part
+
+
+def _remember(sentence):
+    found = _REMEMBER.match(sentence)
+    if not found:
+        return None
+    rest = found.group("rest")
+    # "remember to buy milk" is a reminder, not something about the person; a question asks, it doesn't tell.
+    if re.match(r"^to\s", rest, re.I) or sentence.rstrip().endswith("?"):
+        return None
+    return Candidate(_tidy(rest), pin=bool(_PREFERENCE.match(rest.strip()) or _CALL_ME.match(rest.strip())))
+
+
+def _call_me(sentence):
+    found = _CALL_ME.search(sentence)
+    if not found or _inside_task(sentence[:found.start()]):
+        return None
+    name = found.group("name")
+    if name.split()[0].lower() in _WHEN_WORDS:
+        return None
+    return Candidate(f"Call me {name}", pin=True)
+
+
+def _preference(sentence):
+    found = _PREFERENCE.search(sentence)
+    if not found or _inside_task(sentence[:found.start()]) or _PAST.match(found.group("next")):
+        return None
+    return Candidate(_tidy(found.group("rest")), pin=True)
+
+
+def _my_is(sentence):
+    found = _MY_IS.search(sentence)
+    if not found or _inside_task(sentence[:found.start()]):
+        return None
+    thing, verb, value = found.group("thing"), found.group("verb").lower(), found.group("value").strip()
+    words = [re.sub(r"['’]s$", "", word) for word in thing.lower().split()]
+    lasting = any(word in ("favorite", "favourite", "usual", "go-to", "regular", "preferred") for word in words)
+    if not lasting and any(word in _PASSING for word in words):
+        return None
+    if _PASSING_VALUE.match(value) or _PRONOUN_VALUE.match(value) or len(value) < 2:
+        return None
+    return Candidate(_tidy(f"My {thing} {verb} {value}"))
+
+
+def _from_sentence(sentence):
+    """At most one candidate per sentence: the clearest of the patterns."""
+    remembered = _remember(sentence)
+    if remembered is not None or _REMEMBER.match(sentence):
+        return remembered
+    if sentence.rstrip().endswith("?") or _QUESTION_START.match(sentence):
+        return None
+    return _call_me(sentence) or _preference(sentence) or _my_is(sentence)
+
+
+def candidates(text):
+    """Up to two suggestions from ``text`` (a goal, or a message sent while a task worked); never a secret."""
+    if not isinstance(text, str) or not text.strip():
+        return []
+    out, seen = [], set()
+    for sentence in _sentences(text[:4000]):
+        candidate = _from_sentence(sentence)
+        if candidate is None:
+            continue
+        if not MIN_CHARS <= len(candidate.text) <= 300 or is_secret(candidate.text):
+            continue
+        key = candidate.text.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(candidate)
+        if len(out) >= MAX_PER_TEXT:
+            break
+    return out
+
+
+_JOINED_TASK = re.compile(r"(?:,|;|\band\b|\bthen\b|\balso\b)\s+(?:please\s+)?" + _COMMAND.pattern[1:], re.I)
+
+
+def remember_only(text):
+    """What to remember when ``text`` asks only that, else None; a secret comes back too, so the caller refuses it."""
+    if not isinstance(text, str) or not text.strip() or len(text) > 600:
+        return None
+    found = []
+    for sentence in _sentences(text):
+        candidate = _remember(sentence)
+        if candidate is None or _JOINED_TASK.search(sentence[_REMEMBER.match(sentence).start("rest"):]):
+            return None
+        if len(candidate.text) < 3:
+            return None
+        found.append(candidate)
+    return found or None

mobile_agent/memory/listener.py

@@ -0,0 +1,100 @@
+"""Suggestions to remember something, from what a person types (SPEC §3.3 F3).
+
+A run listener (harness_api.register_run_listener): it reads the goal of a task a person started (the Mac app, the
+terminal UI, `mobster chat`) and the messages they send while it works, never text from another agent (MCP, the HTTP
+API) or from a screen. Each suggestion goes to the conversation as a ``memory_proposal`` item when the task belongs
+to one, and on the bus topic ``memory``. Nothing is remembered until the person accepts it.
+
+It runs on the listeners' own thread, so the store's file never holds up a task.
+"""
+
+import logging
+from collections.abc import Mapping
+
+from .. import harness_api
+from . import extract
+from .model import SCOPE_ID, WORKFLOW_ID
+from .store import default_store
+
+log = logging.getLogger("mobster.memory")
+ITEM_KIND = "memory_proposal"
+MESSAGE_SOURCES = ("app", "tui", "cli", "voice")
+
+
+def proposal_scope(run):
+    """Only immutable, admitted identities may narrow an automatic suggestion."""
+    account = getattr(run, "account_context", None)
+    if account is not None and not isinstance(account, Mapping):
+        return None
+    profile = account.get("id") if account is not None else None
+    workflow = getattr(run, "workflow_id", None)
+    if account is not None and (not isinstance(profile, str) or not SCOPE_ID.fullmatch(profile)):
+        return None
+    if workflow is not None and (not isinstance(workflow, str) or not WORKFLOW_ID.fullmatch(workflow)):
+        return None
+    if account is None and (getattr(run, "extras", None) or {}).get("account") is not None:
+        return None
+    if profile and workflow:
+        return f"profile:{profile}/workflow:{workflow}"
+    if profile:
+        return f"profile:{profile}"
+    if workflow:
+        return f"workflow:{workflow}"
+    return "global"
+
+
+def publish(event):
+    """On the bus topic ``memory`` (never raises)."""
+    try:
+        from .. import eventbus
+        eventbus.bus.publish("memory", event)
+    except Exception:  # noqa: BLE001
+        log.exception("memory: bus publish failed")
+
+
+def thread_item(proposal):
+    """The thread item that shows a suggestion (conversations' sink; chat-ui renders it through the registry)."""
+    return {"kind": ITEM_KIND, "proposalId": proposal["id"], "text": proposal["text"], "scope": proposal["scope"],
+            "app": proposal["app"], "pin": proposal["pin"], "status": proposal["status"], "runId": proposal["runId"],
+            "expiresAt": proposal["expiresAt"]}
+
+
+class ProposalListener:
+    """on_created reads the goal; on_event reads ``user_message`` (a message sent while the task works)."""
+
+    def __init__(self, store_factory=default_store):
+        self.store_factory = store_factory
+
+    def on_created(self, run, runtime):
+        self.consider(run, getattr(run, "goal", None))
+
+    def on_event(self, run, event):
+        if event.get("event") == "user_message" and event.get("source") in MESSAGE_SOURCES:
+            self.consider(run, event.get("text"))
+
+    def consider(self, run, text):
+        if getattr(run, "origin", "api") not in harness_api.INTERACTIVE:
+            return []
+        found = extract.candidates(text)
+        if not found:
+            return []
+        scope = proposal_scope(run)
+        if scope is None:
+            return []
+        store = self.store_factory()
+        if store is None or not store.settings().get("suggest", True):
+            return []
+        thread_id = (getattr(run, "extras", None) or {}).get("threadId")
+        made = []
+        for candidate in found:
+            proposal = store.propose(candidate.text, scope, thread_id=thread_id, run_id=getattr(run, "id", None),
+                                     pin=candidate.pin)
+            if proposal is None:
+                continue
+            made.append(proposal)
+            if thread_id:
+                item = harness_api.post_thread_item(thread_id, thread_item(proposal))
+                if isinstance(item, dict) and item.get("id"):
+                    store.set_proposal_item(proposal["id"], item["id"])
+            publish({"event": "proposal_created", "proposal": proposal})
+        return made

mobile_agent/memory/model.py

@@ -0,0 +1,144 @@
+"""What a fact is: its kinds, sources and scopes, the checks every text passes, and the fingerprint that dedupes.
+
+A fact has a kind (what it is), a source (who or what it came from) and a scope (where it applies). Scopes are
+"global" (everywhere), "app:<bundle id>", "phone:<id>", "project:<id>", "profile:<id>", "workflow:<id>"
+and "profile:<id>/workflow:<id>"; the API also takes "everywhere" for "global". A stored unknown scope
+never matches a task.
+
+Never secrets (``secret_filter.is_secret``), whoever writes. Text read from a screen is data, never an instruction, so
+only the person (source ``user``) or their profile (``profile``) may write a rule or a preference; what a run, a
+check, a lesson or the repo teaches is a path, a place, a structure, a value, an outcome, a device or a cost.
+"""
+
+import hashlib
+import re
+
+from ..api_errors import APIError
+from ..secret_filter import is_secret
+
+FACT_CHARS = 300
+
+KINDS = ("path", "place", "value", "preference", "rule", "structure", "device", "outcome", "cost", "repo")
+SOURCES = ("run", "lesson", "check", "repo", "profile", "user")
+LEARNED_SOURCES = tuple(source for source in SOURCES if source != "user")
+INSTRUCTIONAL = frozenset({"preference", "rule"})        # these read as orders, so only a person's words may be them
+INSTRUCTORS = frozenset({"user", "profile"})
+SCOPE_KINDS = ("profile_workflow", "profile", "workflow", "project", "app", "phone")
+UNPROMPTED = frozenset({"cost"})                         # kept for estimates and the page; never sent to the model
+
+BUNDLE = re.compile(r"[A-Za-z0-9][A-Za-z0-9-]*(?:\.[A-Za-z0-9-]+)+")
+SCOPE_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._@~+-]{0,119}")
+WORKFLOW_ID = re.compile(r"[0-9a-f]{12}")
+REF_KEYS = ("runId", "lessonId", "checkPath", "repo", "profileId")
+
+SECRET = "Mobster doesn't remember passwords, codes or card numbers."
+
+
+def normalize(text):
+    """For spotting the same thing said twice: lowercase, single spaces, no closing punctuation."""
+    return " ".join(str(text).lower().split()).rstrip(" .!;,")
+
+
+def digest(text):
+    return hashlib.sha256(normalize(text).encode()).hexdigest()[:32]
+
+
+def clean_text(text):
+    """Whitespace collapsed; APIError 400 for an empty, long or secret text."""
+    if not isinstance(text, str):
+        raise APIError("Write what Mobster should remember.", 400, "invalid_text")
+    text = " ".join(text.split())
+    if not text:
+        raise APIError("Write what Mobster should remember.", 400, "invalid_text")
+    if len(text) > FACT_CHARS:
+        raise APIError(f"Keep it to {FACT_CHARS} characters.", 400, "too_long", limit=FACT_CHARS)
+    if is_secret(text):
+        raise APIError(SECRET, 400, "secret")
+    return text
+
+
+def clean_scope(scope):
+    """Canonical scope for a fact. Profile/workflow conjunctions require both admitted identities."""
+    if scope in (None, "", "global", "everywhere"):
+        return "global"
+    if isinstance(scope, str) and len(scope) <= 160 and ":" in scope:
+        kind, _, ident = scope.partition(":")
+        if kind == "profile" and "/workflow:" in ident:
+            profile, _, workflow = ident.partition("/workflow:")
+            if SCOPE_ID.fullmatch(profile) and WORKFLOW_ID.fullmatch(workflow):
+                return scope
+        elif kind == "workflow" and WORKFLOW_ID.fullmatch(ident):
+            return scope
+        elif kind in ("profile", "project", "app", "phone"):
+            if (BUNDLE if kind == "app" else SCOPE_ID).fullmatch(ident):
+                return scope
+    raise APIError("Choose everywhere, or one app, phone, project, profile or workflow.", 400, "invalid_scope")
+
+
+def scope_parts(scope):
+    """Parse stored scopes without turning unknown or damaged data into global memory."""
+    if scope == "global":
+        return "global", None
+    if not isinstance(scope, str) or not scope:
+        return "unknown", str(scope) if scope is not None else None
+    try:
+        if clean_scope(scope) != scope:
+            return "unknown", scope
+    except APIError:
+        return "unknown", str(scope) if scope is not None else None
+    kind, _, ident = scope.partition(":")
+    return ("profile_workflow" if kind == "profile" and "/workflow:" in ident else kind), ident
+
+
+def clean_kind(kind, default="value"):
+    kind = default if kind in (None, "") else kind
+    if kind not in KINDS:
+        raise APIError(f"A fact's kind is one of: {', '.join(KINDS)}.", 400, "invalid_kind")
+    return kind
+
+
+def clean_source(source, kind):
+    """The source, checked against the kind: only a person's words may be a rule or a preference."""
+    if source not in SOURCES:
+        raise APIError(f"A fact's source is one of: {', '.join(SOURCES)}.", 400, "invalid_source")
+    if kind in INSTRUCTIONAL and source not in INSTRUCTORS:
+        raise APIError("Only you or a profile can set a rule or a preference. Mobster doesn't take orders from what it "
+                       "reads on a screen.", 400, "not_an_instruction")
+    return source
+
+
+def clean_ref(ref):
+    """Where a fact came from: runId, lessonId, checkPath, repo or profileId, each a short string."""
+    if not ref:
+        return {}
+    if not isinstance(ref, dict):
+        raise APIError("A source reference is an object.", 400, "invalid_ref")
+    out = {}
+    for key, value in ref.items():
+        if key not in REF_KEYS or not isinstance(value, str) or not value or len(value) > 300:
+            raise APIError(f"A source reference is one of: {', '.join(REF_KEYS)}.", 400, "invalid_ref")
+        out[key] = value
+    return out
+
+
+def clean_confidence(value, default=1.0):
+    if value is None:
+        return default
+    if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 <= value <= 1:
+        raise APIError("Confidence is a number from 0 to 1.", 400, "invalid_confidence")
+    return float(value)
+
+
+def clean_key(key):
+    if key is None:
+        return None
+    if not isinstance(key, str) or not key.strip() or len(key) > 200:
+        raise APIError("A fact's key is a short string.", 400, "invalid_key")
+    return " ".join(key.split()).lower()
+
+
+def fingerprint(scope, text, key=None):
+    """The same fact told twice is one fact. With a ``key`` (the thing a fact is about: "cost:com.x:read reels") a
+    changed value updates the fact in place, and forgetting it stops the new value too."""
+    basis = f"key\0{key}" if key else f"text\0{normalize(text)}"
+    return hashlib.sha256(f"{scope}\0{basis}".encode()).hexdigest()[:32]