mannerism 0.1.0__py3-none-any.whl

mannerism/__init__.py

@@ -0,0 +1,3 @@
+"""mannerism: your agents write in your voice."""
+
+__version__ = "0.1.0"

mannerism/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

mannerism/adapters/__init__.py

@@ -0,0 +1,86 @@
+"""Source adapters. One module per source; each defines a class implementing `mannerism.core.Source`.
+
+Adapters keep only text the user wrote. Other people's text goes only into Unit.context, masked
+with `core.context_line`, at most `core.CONTEXT_LINES` lines. Adapters use the standard library only.
+"""
+
+from __future__ import annotations
+
+from importlib import import_module
+
+from ..core import Source
+
+# name -> "module:Class". Order is the order `init` shows sources in.
+REGISTRY = {
+    "claude_code": "claude_code:ClaudeCode",
+    "codex": "codex:Codex",
+    "git": "git:Git",
+    "gh": "gh:GitHub",
+    "bluesky": "bluesky:Bluesky",
+    "mastodon": "mastodon:Mastodon",
+    "x": "x_api:XApi",
+    "slack": "slack:Slack",
+    "markdown": "markdown:Markdown",
+    "files": "files:Files",
+    "chatgpt": "chatgpt:ChatGPT",
+    "telegram": "telegram:Telegram",
+    "whatsapp": "whatsapp:WhatsApp",
+    "whatsapp_db": "whatsapp_db:WhatsAppDB",
+    "hermes": "hermes:Hermes",
+    "x_archive": "x_archive:XArchive",
+    "mbox": "mbox:Mbox",
+    "imessage": "imessage:IMessage",
+    "wispr": "wispr:Wispr",
+    "opencode": "opencode:OpenCode",
+    "qwen_code": "qwen_code:QwenCode",
+    "goose": "goose:Goose",
+    "aider": "aider:Aider",
+    "crush": "crush:Crush",
+    "pi": "pi:Pi",
+    "continue": "continue_dev:Continue",
+    "cline": "cline:Cline",
+    "kilo": "kilo:Kilo",
+}
+
+
+# Built-in adapters that can be configured as several named instances: `[sources.<instance>]` with
+# `adapter = "<key>"`. The instance name becomes the source name (units, cursors and `forget --source`
+# all use it), so an instance name must not equal a built-in source or an installed plugin.
+INSTANCE_ADAPTERS = {"files": "files:Files"}
+
+
+def load(name: str) -> Source:
+    mod, cls = REGISTRY[name].split(":")
+    return getattr(import_module(f"{__name__}.{mod}"), cls)()
+
+
+def load_for(name: str, entry: dict | None = None, *, warnings: list[str] | None = None) -> Source:
+    """The source for config entry `[sources.<name>]`: a built-in, an instance of one, or an enabled plugin.
+
+    Plain built-ins go through `load(name)`. A plugin is imported here, so callers that must never run plugin
+    code (`sync --quick`) check `entry.get("plugin")` before calling."""
+    entry = entry or {}
+    if entry.get("plugin"):
+        from .. import plugins
+
+        try:
+            return plugins.load(name, entry, warnings=warnings)
+        except plugins.PluginError as e:
+            raise ImportError(str(e)) from None
+    kind = entry.get("adapter")
+    if kind:
+        if kind not in INSTANCE_ADAPTERS:
+            raise KeyError(f"adapter {kind!r} cannot be configured as an instance")
+        mod, cls = INSTANCE_ADAPTERS[kind].split(":")
+        return getattr(import_module(f"{__name__}.{mod}"), cls)(name)
+    return load(name)
+
+
+def all_sources() -> list[Source]:
+    out = []
+    for name in REGISTRY:
+        try:
+            out.append(load(name))
+        except (ImportError, AttributeError):
+            continue  # not implemented yet
+    return out

mannerism/adapters/_http.py

@@ -0,0 +1,134 @@
+"""Minimal, strict HTTPS GET for the network sources (standard library only).
+
+Rules every network source gets from here: https only, a finite timeout, a `mannerism/<version>` User-Agent, a cap on
+the JSON body, redirects followed only inside the original origin (never to another host: credentials stay put), and
+errors reduced to a `NetError` that carries a kind, a status and a retry hint. Error text never contains headers,
+bodies or query strings, so a credential cannot leak into a log or a stored cursor.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Mapping
+from datetime import UTC, datetime
+from email.utils import parsedate_to_datetime
+
+from .. import __version__
+
+TIMEOUT_S = 10.0
+MAX_JSON_BYTES = 8 * 1024 * 1024
+MAX_REDIRECTS = 3
+DEFAULT_RETRY_S = 900  # a 429 without any reset header
+MAX_RETRY_S = 24 * 3600
+USER_AGENT = f"mannerism/{__version__}"
+
+
+class NetError(Exception):
+    """A failed request. `kind`: timeout | network | http | rate_limit | redirect | too_large | bad_json | url."""
+
+    def __init__(self, kind: str, message: str, status: int | None = None, retry_after_s: float | None = None):
+        super().__init__(message)
+        self.kind = kind
+        self.status = status
+        self.retry_after_s = retry_after_s
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, *args, **kwargs):  # noqa: ARG002 - a 3xx surfaces as an HTTPError; get_json decides
+        return None
+
+
+_OPENER = urllib.request.build_opener(_NoRedirect())
+
+
+def urlopen(req: urllib.request.Request, timeout: float):
+    """The single place a socket is opened (tests replace this function)."""
+    return _OPENER.open(req, timeout=timeout)
+
+
+def retry_after_s(headers: Mapping | None, now: datetime | None = None) -> float:
+    """Seconds to wait, from Retry-After, x-rate-limit-reset (X, epoch), ratelimit-reset (Bluesky, epoch) or
+    X-RateLimit-Reset (Mastodon, ISO-8601). Falls back to DEFAULT_RETRY_S; always within [1, MAX_RETRY_S]."""
+    now = now or datetime.now(UTC)
+    get = (lambda k: headers.get(k)) if headers is not None else (lambda k: None)
+    for key in ("retry-after", "x-rate-limit-reset", "ratelimit-reset", "x-ratelimit-reset"):
+        raw = get(key)
+        if raw is None:
+            continue
+        raw = str(raw).strip()
+        try:
+            x = float(raw)
+            wait = x - now.timestamp() if x > 1e9 else x
+        except ValueError:
+            try:
+                dt = parsedate_to_datetime(raw) if not raw[:4].isdigit() else datetime.fromisoformat(raw)
+            except (TypeError, ValueError):
+                continue
+            wait = (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).timestamp() - now.timestamp()
+        return min(max(wait, 1.0), float(MAX_RETRY_S))
+    return float(DEFAULT_RETRY_S)
+
+
+def _origin(url: str) -> tuple[str, str, int | None]:
+    p = urllib.parse.urlsplit(url)
+    return p.scheme, (p.hostname or "").lower(), p.port
+
+
+def build_url(host: str, path: str, params: Mapping[str, str | int | list] | None = None) -> str:
+    """https://host/path?query. A list value repeats the parameter (`uris=a&uris=b`); None values are left out."""
+    q = urllib.parse.urlencode(
+        {k: ([str(x) for x in v] if isinstance(v, list) else str(v)) for k, v in (params or {}).items() if v is not None},
+        safe=",", doseq=True)
+    return f"https://{host}{path}" + (f"?{q}" if q else "")
+
+
+def get_json(url: str, headers: Mapping[str, str] | None = None, *, timeout: float = TIMEOUT_S,
+             max_bytes: int = MAX_JSON_BYTES) -> tuple[object, Mapping]:
+    """GET `url` (https only) and return (parsed JSON, response headers). Raises NetError."""
+    start = _origin(url)
+    if start[0] != "https" or not start[1]:
+        raise NetError("url", "refusing a non-https URL")
+    hdrs = {"User-Agent": USER_AGENT, "Accept": "application/json", **(headers or {})}
+    cur = url
+    for _hop in range(MAX_REDIRECTS + 1):
+        host = _origin(cur)[1]
+        try:
+            with urlopen(urllib.request.Request(cur, headers=hdrs, method="GET"), timeout) as resp:
+                body = resp.read(max_bytes + 1)
+                status = getattr(resp, "status", 200)
+                resp_headers = resp.headers
+        except urllib.error.HTTPError as e:
+            if e.code in (301, 302, 303, 307, 308):
+                target = urllib.parse.urljoin(cur, (e.headers.get("Location") if e.headers else None) or "")
+                if _origin(target) != start:
+                    raise NetError("redirect", f"{host}: refused a redirect to another origin", e.code) from None
+                cur = target
+                continue
+            if e.code == 429:
+                raise NetError("rate_limit", f"{host}: HTTP 429 (rate limited)", 429,
+                               retry_after_s(e.headers)) from None
+            raise NetError("http", f"{host}: HTTP {e.code}", e.code) from None
+        except TimeoutError:
+            raise NetError("timeout", f"{host}: timed out") from None
+        except (urllib.error.URLError, OSError, ValueError) as e:
+            raise NetError("network", f"{host}: {type(e).__name__}") from None
+        if status in (301, 302, 303, 307, 308):  # an opener that did not raise for a redirect
+            target = urllib.parse.urljoin(cur, resp_headers.get("Location") or "")
+            if _origin(target) != start:
+                raise NetError("redirect", f"{host}: refused a redirect to another origin", status)
+            cur = target
+            continue
+        if status == 429:
+            raise NetError("rate_limit", f"{host}: HTTP 429 (rate limited)", 429, retry_after_s(resp_headers))
+        if status >= 400:
+            raise NetError("http", f"{host}: HTTP {status}", status)
+        if len(body) > max_bytes:
+            raise NetError("too_large", f"{host}: response over {max_bytes} bytes")
+        try:
+            return json.loads(body.decode("utf-8")), resp_headers
+        except (ValueError, UnicodeDecodeError):
+            raise NetError("bad_json", f"{host}: response is not JSON") from None
+    raise NetError("redirect", "too many redirects")

mannerism/adapters/_social.py

@@ -0,0 +1,229 @@
+"""Shared machinery for the continuous social-post sources (Bluesky, Mastodon, X): config gate, status in the cursor,
+text hygiene and the paged newest-first scan.
+
+The scan keeps two marks apart. `high_water` is the newest item a COMPLETED scan has seen. While a scan runs, its
+state lives in `scan` = {token, candidate, fetched}: the continuation token for the next page, the newest item seen
+so far (it becomes the high-water mark only when the scan finishes) and how many items were fetched. A timeout or
+error in the middle returns the units gathered so far together with the `scan` continuation, so the next read resumes
+at the same page: nothing is lost (the units and the cursor are stored in one transaction) and nothing is read twice.
+A first run (no high-water mark yet) stops after `initial` items instead of reading a whole history.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+
+from ..core import Unit
+from ._http import NetError
+
+CURSOR_VERSION = 1
+MAX_PAGES_PER_READ = 5  # a read never makes more page requests than this; the rest continues next time
+DEFAULT_INITIAL_POSTS = 200  # free sources: how much history the first run reads
+DEFAULT_RECONCILE_INTERVAL_S = 7 * 24 * 3600  # free sources: how often deletes and edits are looked for
+RECONCILE_ITEMS = 200  # the recent window a reconcile re-reads
+RECONCILE_CHECKS = 50  # at most this many vanished-looking posts are confirmed one by one per reconcile
+MAX_SEEN = 400  # refs remembered in the cursor (newest first): enough to cover the reconcile window
+REJECTED_TOKEN_STATUSES = (400, 404, 410)  # a stored continuation token the service no longer accepts
+
+
+def utcnow() -> datetime:
+    return datetime.now(UTC)
+
+
+def network_enabled(config: dict | None) -> bool:
+    """`allow_network` must be exactly true; anything else (missing, "yes", 1) keeps the source off."""
+    return isinstance(config, dict) and config.get("allow_network") is True
+
+
+def int_option(config: dict | None, key: str, default: int, low: int = 1) -> int:
+    try:
+        v = int((config or {}).get(key, default))
+    except (TypeError, ValueError):
+        return default
+    return v if v >= low else default
+
+
+# --- status in the cursor -----------------------------------------------------------------------------------------
+
+
+def with_status(cursor: dict, message: str, retry_after_s: float | None = None) -> dict:
+    """`cursor` plus a `status` entry; everything else in the cursor is untouched. The engine stores it as is, so the
+    scheduler can read `status.error` / `status.retry_after` to back off."""
+    now = utcnow()
+    status = {"error": message[:300], "at": now.isoformat(timespec="seconds")}
+    if retry_after_s is not None:
+        status["retry_after"] = (now + timedelta(seconds=retry_after_s)).isoformat(timespec="seconds")
+    return {**cursor, "status": status}
+
+
+def backoff_active(cursor: dict) -> bool:
+    """True while a stored `status.retry_after` is still in the future (no request should be made)."""
+    st = cursor.get("status")
+    if not isinstance(st, dict) or not st.get("retry_after"):
+        return False
+    try:
+        return datetime.fromisoformat(st["retry_after"]) > utcnow()
+    except ValueError:
+        return False
+
+
+def clean_cursor(cursor: dict | None) -> dict:
+    """The cursor if it has this version's shape, else a fresh one (an unknown shape is re-read from scratch; units
+    dedupe by id). Always carries `v`."""
+    if isinstance(cursor, dict) and cursor.get("v") == CURSOR_VERSION:
+        return dict(cursor)
+    return {"v": CURSOR_VERSION}
+
+
+# --- text hygiene -------------------------------------------------------------------------------------------------
+
+_HANDLE = r"@\w+(?:[.-]\w+)*(?:@\w+(?:[.-]\w+)*)?"
+_LEAD_MENTIONS = re.compile(rf"^(?:\s*{_HANDLE})+\s*")
+_MENTION = re.compile(rf"(?<![\w/@]){_HANDLE}")
+MENTION_MARK = ""  # an adapter-inserted placeholder for a structured mention (facet / mention link)
+_LEAD_MARKS = re.compile(rf"^(?:\s*(?:{MENTION_MARK}|{_HANDLE}))+\s*")
+
+
+def finish_text(text: str, reply: bool) -> str:
+    """Reply: drop the leading @mentions. Everywhere else a mention of another person becomes `@[user]`, so a third
+    party's handle is not stored. Collapses stray spacing."""
+    if reply:
+        text = _LEAD_MARKS.sub("", text)
+    text = text.replace(MENTION_MARK, "@[user]")
+    text = _MENTION.sub("@[user]", text)
+    return re.sub(r"[ \t]+\n", "\n", re.sub(r"[ \t]{2,}", " ", text)).strip()
+
+
+def context_text(text: str) -> str:
+    """Another person's text on its way into one masked context line: handles masked, spacing collapsed."""
+    return _MENTION.sub("@[user]", " ".join(str(text).replace(MENTION_MARK, "@[user]").split()))
+
+
+def text_hash(text: str) -> str:
+    """Short fingerprint of the cleaned text (a reconcile compares it to see an edit)."""
+    return hashlib.sha1(text.encode("utf-8", "replace")).hexdigest()[:10]
+
+
+def reconcile_due(cursor: dict, interval_s: int, now: datetime | None = None) -> bool:
+    """True when the last reconcile is at least `interval_s` old. A cursor that never had one is NOT due: the first
+    completed scan only stamps `reconciled_at` (it has just read the recent window anyway)."""
+    raw = cursor.get("reconciled_at")
+    try:
+        last = datetime.fromisoformat(raw) if isinstance(raw, str) else None
+    except ValueError:
+        last = None
+    return last is not None and ((now or utcnow()) - last).total_seconds() >= interval_s
+
+
+def month_key(now: datetime) -> str:
+    return now.strftime("%Y-%m")
+
+
+def next_month_start(now: datetime) -> datetime:
+    y, m = (now.year + 1, 1) if now.month == 12 else (now.year, now.month + 1)
+    return datetime(y, m, 1, tzinfo=UTC)
+
+
+# --- the scan -----------------------------------------------------------------------------------------------------
+
+
+@dataclass
+class ScanResult:
+    units: list[Unit]
+    cursor: dict  # the state fields (high_water / scan) advanced; no status set
+    error: NetError | None = None
+    completed: bool = False
+    fetched: int = 0
+    dropped: bool = False  # a stored continuation token was rejected: the scan state was dropped
+
+
+def scan(
+    cursor: dict,
+    *,
+    fetch: Callable[[str | None, int], tuple[list, str | None]],
+    marker: Callable[[object], object | None],
+    at_high_water: Callable[[object, object], bool],
+    convert: Callable[[object], list[Unit]],
+    page_size: int,
+    initial: int,
+    max_pages: int | None = None,
+    promote_on_drop: bool = False,
+) -> ScanResult:
+    """Walk a newest-first feed page by page.
+
+    `fetch(token, limit)` -> (items, next_token) or raises NetError. `marker(item)` -> the item's ordering mark, or
+    None for items that carry none (a pinned post). `at_high_water(item, hw)` -> True when the item is not newer than the
+    stored high-water mark (the scan is complete). `convert(item)` -> units. With no stored high-water mark the scan
+    stops after `initial` items.
+
+    A fetch that fails with 400/404/410 while it carried a continuation token means the service no longer accepts that
+    token: the `scan` state is dropped (the `high_water` mark is kept) so the next read starts a fresh scan instead of
+    retrying the same token forever. `promote_on_drop` (the paid source): on a first run that already read something,
+    the newest item seen becomes the high-water mark, so a restart does not read, and pay for, the same posts again."""
+    hw = cursor.get("high_water")
+    state = cursor.get("scan") if isinstance(cursor.get("scan"), dict) else {}
+    token = state.get("token")
+    candidate = state.get("candidate")
+    fetched = int(state.get("fetched") or 0)
+    max_pages = MAX_PAGES_PER_READ if max_pages is None else max_pages
+    units: list[Unit] = []
+    error: NetError | None = None
+    complete = False
+    dropped = False
+    pages = 0
+    while not complete:
+        if pages >= max_pages:
+            break
+        limit = page_size
+        if hw is None:
+            if fetched >= initial:
+                complete = True
+                break
+            limit = min(page_size, initial - fetched)
+        try:
+            items, nxt = fetch(token, limit)
+        except NetError as e:
+            error = e
+            dropped = token is not None and e.kind == "http" and e.status in REJECTED_TOKEN_STATUSES
+            break
+        pages += 1
+        for item in items:
+            m = marker(item)
+            if m is not None and hw is not None and at_high_water(item, hw):
+                complete = True
+                break
+            if hw is None and fetched >= initial:
+                complete = True
+                break
+            fetched += 1
+            if m is not None and candidate is None:
+                candidate = m
+            units.extend(convert(item))
+        if complete:
+            break
+        if not items or nxt is None:
+            complete = True
+            break
+        token = nxt
+    new = {k: v for k, v in cursor.items() if k not in ("scan", "status")}
+    if complete:
+        if candidate is not None:
+            new["high_water"] = candidate
+    elif dropped:
+        if promote_on_drop and hw is None and candidate is not None:
+            new["high_water"] = candidate
+    elif token is not None or candidate is not None or fetched:
+        new["scan"] = {"token": token, "candidate": candidate, "fetched": fetched}
+    return ScanResult(units, new, error, complete, fetched, dropped)
+
+
+def cap_seen(seen: dict, order_key: Callable[[str, object], object], limit: int = MAX_SEEN) -> dict:
+    """The `limit` newest entries of a {ref: value} map, newest by `order_key(ref, value)`."""
+    if len(seen) <= limit:
+        return seen
+    keep = sorted(seen.items(), key=lambda kv: order_key(*kv), reverse=True)[:limit]
+    return dict(keep)