boardmail 0.4.0__py3-none-any.whl

boardmail/__init__.py

@@ -0,0 +1,2 @@
+"""Public-board mail collected locally, without launching agents."""
+__version__ = "0.4.0"

boardmail/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

boardmail/adapter_clawdchat.py

@@ -0,0 +1,259 @@
+"""ClawdChat notifications, confirmed through anonymous public originals."""
+from datetime import datetime
+from http.client import HTTPException
+import json
+from pathlib import Path
+import time
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode, urlsplit
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+
+from boardmail.adapters import Batch
+from boardmail.config import MailError, uuid
+
+API_VERSION = 1
+ORIGIN = "https://clawdchat.cn"
+PAGE_SIZE = 8
+MAX_PENDING = 256
+MAX_REQUESTS = 40
+MAX_RESPONSE_BYTES = 1024 * 1024
+SOURCE_SECONDS = 45
+KINDS = {"comment": "reply_to_post", "reply": "reply_to_comment",
+         "mention_post": "mention", "mention_comment": "mention"}
+
+
+class NoRedirect(HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        fp.close()
+        raise MailError("redirect_refused")
+
+
+class Client:
+    def __init__(self, settings):
+        try:
+            self.owner = uuid(settings["account_id"])
+        except (KeyError, ValueError, TypeError, AttributeError):
+            raise MailError("invalid_config") from None
+        try:
+            with Path(settings.get("api_key_file")).open() as stream:
+                self.key = stream.read(4097).strip()
+            if not self.key or len(self.key) > 4096 or any(ord(c) < 33 or ord(c) > 126 for c in self.key):
+                raise ValueError()
+        except (OSError, UnicodeError, ValueError, TypeError):
+            raise MailError("credentials_unavailable") from None
+        self.end = time.monotonic() + SOURCE_SECONDS
+        self.deadline = self.end
+        self.requests = 0
+        self.opener = build_opener(NoRedirect())
+
+    def phase(self, seconds):
+        self.deadline = min(self.end, time.monotonic() + seconds)
+
+    def get(self, path, params=None, *, authenticated=False):
+        headers = {"Accept": "application/json", "User-Agent": "boardmail/0.2"}
+        if authenticated:
+            headers["Authorization"] = "Bearer " + self.key
+        for attempt in range(3):  # At most two retries, all inside this phase's budget.
+            remaining = self.deadline - time.monotonic()
+            if remaining <= 0 or self.requests >= MAX_REQUESTS:
+                raise MailError("budget_exhausted")
+            self.requests += 1
+            request = Request(ORIGIN + "/api/v1" + path + ("?" + urlencode(params) if params else ""), headers=headers)
+            try:
+                with self.opener.open(request, timeout=min(4, remaining)) as response:
+                    chunks, size = [], 0
+                    while True:
+                        if time.monotonic() >= self.deadline:
+                            raise MailError("budget_exhausted")
+                        chunk = response.read1(65536)
+                        if not chunk:
+                            result = json.loads(b"".join(chunks))
+                            if not isinstance(result, dict) or result.get("success") is False:
+                                raise MailError("invalid_response")
+                            return result
+                        size += len(chunk)
+                        if size > MAX_RESPONSE_BYTES:
+                            raise MailError("response_too_large")
+                        chunks.append(chunk)
+            except HTTPError as exc:
+                code = exc.code
+                exc.close()
+                if code not in (408, 500, 502, 503, 504) or attempt == 2:
+                    raise MailError("http_" + str(code)) from None
+            except (URLError, OSError, HTTPException):
+                if attempt == 2:
+                    raise MailError("network_error") from None
+        raise MailError("network_error")
+
+
+def _text(value):
+    if not isinstance(value, str):
+        raise ValueError()
+    value.encode("utf-8")
+    return value
+
+
+def _timestamp(value):
+    parsed = datetime.fromisoformat(_text(value).replace("Z", "+00:00"))
+    if parsed.tzinfo is None:
+        raise ValueError()
+    return int(parsed.timestamp())
+
+
+def _url(value, fallback):
+    if isinstance(value, str) and len(value) <= 2048 and "\\" not in value and not any(ord(c) < 33 or ord(c) == 127 for c in value):
+        try:
+            url = urlsplit(value)
+            if (url.scheme == "https" and url.hostname == "clawdchat.cn" and url.port in (None, 443)
+                    and url.username is None and url.password is None):
+                return value
+        except ValueError:
+            pass
+    return ORIGIN + "/api/v1" + fallback
+
+
+def _reference(item):
+    kind = KINDS.get(item["type"])
+    if kind is None:
+        return None
+    post = uuid(item["post_id"]) if item.get("post_id") else None
+    mid = uuid(item["post_id"] if item["type"] == "mention_post" else item["comment_id"])
+    return {"id": mid, "post": post, "kind": kind, "is_post": item["type"] == "mention_post"}
+
+
+def _original(client, entry):
+    path = ("/posts/" if entry["is_post"] else "/comments/") + entry["id"]
+    original = client.get(path)  # Never attach credentials to public-original requests.
+    if uuid(original["id"]) != entry["id"]:
+        raise ValueError()
+    post_id = entry["id"] if entry["is_post"] else uuid(original["post_id"])
+    if entry["post"] is not None and entry["post"] != post_id:
+        raise ValueError()
+    context = original if entry["is_post"] else original.get("post") or {}
+    if context.get("id") and uuid(context["id"]) != post_id:
+        raise ValueError()
+    for obj in (original, context):
+        if obj.get("is_deleted") or obj.get("is_hidden") or obj.get("visibility", "public") != "public":
+            raise MailError("original_unavailable")
+    author = original["author"]
+    if uuid(author["id"]) == client.owner:
+        return None
+    body = original.get("content")
+    if body is None and entry["is_post"]:
+        body = ""  # Link posts may have no text body.
+    return {"id": entry["id"], "thread_id": post_id, "kind": entry["kind"],
+            "parent_id": uuid(original["parent_id"]) if original.get("parent_id") else None,
+            "author": _text(author["name"]), "title": _text(context.get("title", "Public reply")),
+            "body": _text(body), "url": _url(original.get("web_url"), path),
+            "created_at": _timestamp(original["created_at"])}
+
+
+def _error(batch, exc):
+    code = str(exc) if isinstance(exc, MailError) else "invalid_response"
+    if code != "budget_exhausted" and (batch.error is None or code == "http_429"):
+        batch.error = code
+    batch.complete = False
+    return code
+
+
+FAILURES = (MailError, ValueError, KeyError, TypeError, AttributeError, OverflowError)
+
+
+def collect(settings, state, known):
+    """Rotate retries, read fresh and backfill pages, then confirm new references.
+
+    State retains references only. Overflow evicts the oldest reference with an
+    explicit error; cyclic notification scans may rediscover it while retained.
+    """
+    batch = Batch(state={"offset": 0, "pending": []})
+    pending = {}
+    try:
+        offset = state.get("offset", 0)
+        if type(offset) is not int or not 0 <= offset < 2**63:
+            raise ValueError()
+        batch.state["offset"] = offset
+        entries = state.get("pending", [])
+        if not isinstance(entries, list) or len(entries) > MAX_PENDING:
+            raise ValueError()
+        for entry in entries:
+            mid = uuid(entry["id"])
+            post = uuid(entry["post"]) if entry["post"] is not None else None
+            if entry["kind"] not in KINDS.values() or type(entry["is_post"]) is not bool:
+                raise ValueError()
+            if mid not in known:
+                pending[mid] = {"id": mid, "post": post, "kind": entry["kind"], "is_post": entry["is_post"]}
+        client = Client(settings)
+        client.phase(5)
+        if uuid(client.get("/agents/me", authenticated=True)["id"]) != client.owner:
+            raise MailError("account_mismatch")
+    except FAILURES as exc:
+        _error(batch, exc)
+        batch.state["pending"] = list(pending.values())
+        return batch
+
+    seen = set(known)
+
+    def resolve(ids, seconds):
+        client.phase(seconds)
+        for mid in ids:
+            if mid not in pending:
+                continue
+            entry = pending.pop(mid)
+            pending[mid] = entry  # A failed original must not monopolize retries.
+            try:
+                message = _original(client, entry)
+                if message is not None:
+                    batch.messages.append(message)
+                    seen.add(mid)
+                del pending[mid]
+            except FAILURES as exc:
+                if isinstance(exc, MailError) and str(exc) in ("http_403", "http_404", "http_410", "original_unavailable"):
+                    batch.unavailable += 1
+                else:
+                    code = _error(batch, exc)
+                    if code == "http_429":
+                        return False
+                    if code == "budget_exhausted":
+                        break
+        return True
+
+    if resolve(list(pending)[:8], 15):
+        fresh = []
+        # Always inspect the head. The second request resumes an independent sweep.
+        for page_offset in dict.fromkeys((0, batch.state["offset"])):
+            client.phase(5)
+            try:
+                raw = client.get("/notifications", {"limit": PAGE_SIZE, "offset": page_offset}, authenticated=True)
+                items, total = raw["items"], raw["total"]
+                if not isinstance(items, list) or len(items) > PAGE_SIZE or type(total) is not int or not 0 <= total < 2**63:
+                    raise ValueError()
+                if not items and page_offset < total:
+                    raise MailError("pagination_no_progress")
+                overflow = False
+                for item in items:
+                    try:
+                        entry = _reference(item)
+                        if entry is None or entry["id"] in seen or entry["id"] in pending:
+                            continue
+                        if len(pending) == MAX_PENDING:
+                            del pending[next(iter(pending))]
+                            overflow = True
+                            _error(batch, MailError("pending_overflow"))
+                        pending[entry["id"]] = entry
+                        fresh.append(entry["id"])
+                    except FAILURES as exc:
+                        _error(batch, exc)
+                if page_offset == batch.state["offset"]:
+                    following = page_offset + len(items)
+                    batch.state["offset"] = page_offset if overflow else following if following < total else 0
+            except FAILURES as exc:
+                code = _error(batch, exc)
+                if page_offset and code in ("http_400", "http_422", "pagination_no_progress"):
+                    batch.state["offset"] = 0
+                if code == "http_429":
+                    break
+        if batch.error != "http_429":
+            resolve(fresh[:8], 15)
+    batch.state["pending"] = list(pending.values())
+    batch.complete = batch.complete and not pending and batch.state["offset"] == 0
+    return batch

boardmail/adapter_fourclaw.py

@@ -0,0 +1,156 @@
+"""Collect personal text from explicitly watched anonymous 4claw pages."""
+from datetime import datetime
+from html.parser import HTMLParser
+from http.client import HTTPException
+import json
+import re
+import time
+from urllib.error import HTTPError, URLError
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+from uuid import UUID
+
+from .adapters import Batch
+
+API_VERSION = 1
+HOST = "https://www.4claw.org"
+MAX_BYTES = 2_000_000
+
+
+class _NoRedirect(HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None
+
+
+class _Page(HTMLParser):
+    """Read visible post fields only; never evaluate scripts or embedded data."""
+    def __init__(self):
+        super().__init__(convert_charrefs=True)
+        self.stack = []
+        self.posts = []
+        self.post = None
+        self.title = ""
+
+    def handle_starttag(self, tag, attrs):
+        attrs = dict(attrs)
+        classes = (attrs.get("class") or "").split()
+        inherited = self.stack[-1][1] if self.stack else None
+        field = inherited
+        if tag in ("script", "style", "svg", "form"):
+            field = "ignore"
+        elif inherited != "ignore":
+            if "claw-post" in classes:
+                self.post = {"op": "op" in classes, "author": "", "body": "", "date": ""}
+                self.posts.append(self.post)
+            if "claw-section-title" in classes: field = "title"
+            if "claw-post-name" in classes: field = "author"
+            if "claw-post-body" in classes: field = "body"
+            if tag == "time" and self.post is not None:
+                self.post["date"] = attrs.get("datetime", "")
+            if tag == "br": self.handle_data("\n")
+        if tag not in ("br", "img", "input", "meta", "link", "hr", "wbr", "source"):
+            self.stack.append((tag, field))
+
+    def handle_endtag(self, tag):
+        for i in range(len(self.stack) - 1, -1, -1):
+            if self.stack[i][0] == tag:
+                del self.stack[i:]
+                break
+
+    def handle_data(self, data):
+        field = self.stack[-1][1] if self.stack else None
+        if field == "title": self.title += data
+        elif field in ("author", "body") and self.post is not None:
+            self.post[field] += data
+
+
+def _fetch(thread):
+    request = Request(f"{HOST}/t/{thread}", headers={"Accept": "text/html", "User-Agent": "boardmail/1"})
+    deadline = time.monotonic() + 10
+    with build_opener(_NoRedirect).open(request, timeout=5) as response:
+        if response.headers.get_content_type() != "text/html":
+            raise ValueError("format")
+        content = bytearray()
+        while True:
+            if time.monotonic() >= deadline: raise TimeoutError()
+            chunk = response.read1(min(65536, MAX_BYTES + 1 - len(content)))
+            if time.monotonic() >= deadline: raise TimeoutError()
+            if not chunk: break
+            content.extend(chunk)
+            if len(content) > MAX_BYTES: raise ValueError("size")
+        return content.decode("utf-8")
+
+
+def _messages(html, thread, account, aliases):
+    page = _Page()
+    page.feed(html)
+    if not page.title or not page.posts or not page.posts[0]["op"]:
+        raise ValueError("format")
+    # Next.js emits public reply UUIDs as React node keys, absent from the DOM.
+    chunks = []
+    for raw in re.findall(r'<script>self\.__next_f\.push\((.*?)\)</script>', html, re.S):
+        value = json.loads(raw)
+        if isinstance(value, list) and len(value) == 2 and value[0] == 1 and isinstance(value[1], str):
+            chunks.append(value[1])
+    ids = re.findall(r'\["\$","div","([0-9a-f-]{36})",\{"className":"claw-post reply"', ''.join(chunks))
+    if len(ids) != len(page.posts) - 1 or len(set(ids)) != len(ids):
+        raise ValueError("reply_ids")
+    if any(str(UUID(reply)) != reply for reply in ids): raise ValueError("reply_ids")
+    result = []
+    own_thread = page.posts[0]["author"].strip().casefold() == account.casefold()
+    for position, post in enumerate(page.posts):
+        author = post["author"].strip()
+        if not author or not post["date"]: raise ValueError("format")
+        date = datetime.fromisoformat(post["date"].replace("Z", "+00:00"))
+        if date.tzinfo is None: raise ValueError("date")
+        created = int(date.timestamp())
+        if not -(2**63) <= created < 2**63: raise ValueError("date")
+        if author.casefold() == account.casefold(): continue
+        mention = any(re.search(r"(?<![\w@])@" + re.escape(alias) + r"(?!\w)", post["body"], re.I) for alias in aliases)
+        if not mention and not (own_thread and not post["op"]): continue
+        result.append({"id": thread if post["op"] else ids[position - 1],
+                       "thread_id": thread, "kind": "reply_to_post" if own_thread and not post["op"] else "mention",
+                       "parent_id": thread if own_thread and not post["op"] else None,
+                       "author": author, "title": page.title, "body": post["body"],
+                       "url": f"{HOST}/t/{thread}", "created_at": created})
+    return result
+
+
+def collect(settings, state, known):
+    """One round-robin window, including failures, with constant-size progress."""
+    try:
+        account = settings["account_id"]
+        threads = settings["watched_threads"]
+        aliases = settings.get("mention_aliases", [account])
+        if not isinstance(account, str) or not re.fullmatch(r"[A-Za-z0-9_]{2,64}", account): raise ValueError()
+        if not isinstance(threads, list) or not 1 <= len(threads) <= 100: raise ValueError()
+        if not all(isinstance(t, str) and str(UUID(t)) == t for t in threads): raise ValueError()
+        threads = list(dict.fromkeys(threads))
+        if not isinstance(aliases, list) or len(aliases) > 20: raise ValueError()
+        if not all(isinstance(a, str) and re.fullmatch(r"[A-Za-z0-9_]{2,64}", a) for a in aliases): raise ValueError()
+    except (KeyError, ValueError, TypeError, AttributeError):
+        return Batch(state=state, complete=False, error="invalid_config")
+    offset = state.get("next_thread", 0)
+    if type(offset) is not int or not 0 <= offset < len(threads): offset = 0
+    batch = Batch(complete=False)
+    started = time.monotonic()
+    checked = 0
+    for step in range(min(4, len(threads))):
+        if step and time.monotonic() - started >= 15: break
+        thread = threads[(offset + step) % len(threads)]
+        try:
+            messages = _messages(_fetch(thread), thread, account, aliases)
+            batch.messages.extend(m for m in messages if m["id"] not in known)
+        except HTTPError as exc:
+            exc.close()
+            batch.error = f"http_{exc.code}" if exc.code in (401, 403, 404, 429, 500, 502, 503, 504) else "fourclaw_http_error"
+            batch.unavailable += 1
+        except (URLError, TimeoutError, OSError, HTTPException):
+            batch.error = "fourclaw_network_error"
+            batch.unavailable += 1
+        except (ValueError, OverflowError, UnicodeError):
+            batch.error = "fourclaw_invalid_public_page"
+            batch.unavailable += 1
+        checked += 1
+    batch.state = {"next_thread": (offset + checked) % len(threads)}
+    batch.complete = offset + checked >= len(threads) and batch.error is None
+    return batch

boardmail/adapter_fruitflies.py

@@ -0,0 +1,152 @@
+"""Public Fruitflies mentions and replies within a documented parent window."""
+from datetime import datetime
+from http.client import HTTPException
+import json
+import re
+from time import monotonic
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import HTTPRedirectHandler, Request, build_opener
+from uuid import UUID
+
+from boardmail.adapters import Batch
+
+API_VERSION = 1
+BASE = 'https://api.fruitflies.ai/v1/feed'
+PAGE = 100
+MAX_OFFSET = 100000
+MAX_BYTES = 2 * 1024 * 1024
+
+
+class FetchError(Exception):
+    pass
+
+
+class NoRedirect(HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None
+
+
+def _fetch(params):
+    # Public requests never read configured credentials or follow redirects.
+    request = Request(BASE + '?' + urlencode(params), headers={'Accept': 'application/json',
+                      'User-Agent': 'boardmail/fruitflies'})
+    try:
+        deadline = monotonic() + 8
+        with build_opener(NoRedirect()).open(request, timeout=8) as response:
+            chunks = []
+            size = 0
+            while size <= MAX_BYTES:
+                if monotonic() >= deadline:
+                    raise FetchError('network_timeout')
+                chunk = response.read1(min(65536, MAX_BYTES + 1 - size))
+                if not chunk:
+                    break
+                chunks.append(chunk)
+                size += len(chunk)
+            raw = b''.join(chunks)
+        if len(raw) > MAX_BYTES:
+            raise FetchError('response_too_large')
+        data = json.loads(raw)
+        if not isinstance(data, dict) or not isinstance(data.get('posts'), list):
+            raise FetchError('invalid_response')
+        if len(data['posts']) > PAGE:
+            raise FetchError('invalid_response')
+        return data['posts']
+    except HTTPError as exc:
+        exc.close()
+        raise FetchError('http_' + str(exc.code)) from None
+    except (URLError, OSError, TimeoutError, HTTPException):
+        raise FetchError('network_error') from None
+    except (ValueError, UnicodeError):
+        raise FetchError('invalid_response') from None
+
+
+def _uuid(value):
+    if not isinstance(value, str) or str(UUID(value)) != value:
+        raise ValueError()
+    return value
+
+
+def _post(raw):
+    if not isinstance(raw, dict):
+        raise ValueError()
+    ident = _uuid(raw['id'])
+    parent = raw.get('parent_id')
+    if parent is not None:
+        parent = _uuid(parent)
+    kind = raw['post_type']
+    if kind not in ('post', 'question', 'answer'):
+        raise ValueError()
+    body = raw['content']
+    author = raw['agents']['handle']
+    if not isinstance(body, str) or not isinstance(author, str):
+        raise ValueError()
+    body.encode('utf-8'); author.encode('utf-8')
+    date = datetime.fromisoformat(raw['created_at'].replace('Z', '+00:00'))
+    if date.tzinfo is None:
+        raise ValueError()
+    return dict(id=ident, parent_id=parent, post_type=kind, body=body,
+                author=author, created_at=int(date.timestamp()))
+
+
+def collect(settings, state, known):
+    """Read newest and one historical page; retry failed history next call."""
+    handle = settings.get('account_id')
+    if not isinstance(handle, str) or not re.fullmatch(r'[A-Za-z0-9_-]{1,64}', handle):
+        return Batch(state={}, complete=False, error='invalid_config')
+    handle = handle.lower()
+    offset = state.get('offset', PAGE)
+    if type(offset) is not int or not PAGE <= offset <= MAX_OFFSET or offset % PAGE:
+        offset = PAGE
+    result = Batch(state={'offset': offset}, complete=False)
+    mention = re.compile(r'(?<![\w@])@' + re.escape(handle) + r'(?![\w-])', re.IGNORECASE)
+    own = {}
+    own_ok = True
+    try:
+        for raw in _fetch({'agent': handle, 'limit': PAGE, 'offset': 0}):
+            try:
+                post = _post(raw)
+                if post['author'].casefold() == handle.casefold():
+                    own[post['id']] = post['post_type']
+            except (KeyError, TypeError, ValueError, AttributeError, OverflowError):
+                own_ok = False
+                result.error = 'invalid_response'
+    except FetchError as exc:
+        own_ok = False
+        result.error = str(exc)
+
+    emitted = set(known)
+    for position in (0, offset):
+        try:
+            rows = _fetch({'limit': PAGE, 'offset': position})
+        except FetchError as exc:
+            result.error = str(exc)
+            continue
+        for raw in rows:
+            try:
+                post = _post(raw)
+            except (KeyError, TypeError, ValueError, AttributeError, OverflowError):
+                result.unavailable += 1
+                result.error = 'invalid_response'
+                continue
+            if post['id'] in emitted or post['author'].casefold() == handle.casefold():
+                continue
+            parent_kind = own.get(post['parent_id'])
+            if parent_kind:
+                kind = 'reply_to_comment' if parent_kind == 'answer' else 'reply_to_post'
+            elif mention.search(post['body']):
+                kind = 'mention'
+            else:
+                continue
+            result.messages.append(dict(id=post['id'], parent_id=post['parent_id'],
+                thread_id=post['parent_id'] or post['id'], kind=kind, author=post['author'],
+                title='', body=post['body'], url='https://fruitflies.ai/feed',
+                created_at=post['created_at']))
+            emitted.add(post['id'])
+        if position == offset and own_ok:
+            # Cycle the finite scan window. Re-visits also retry malformed originals.
+            end = len(rows) < PAGE or offset >= MAX_OFFSET
+            result.state['offset'] = PAGE if end else offset + PAGE
+            result.complete = end and result.error is None
+    return result