mailrows 0.1.0__py3-none-any.whl

mailrows/hooks/csv_append.py

@@ -0,0 +1,44 @@
+"""Append a line to a CSV file after each email (a simple local record for another tool).
+
+    [csv_append]
+    path = sent.csv
+    columns = id, email, _sent_at
+"""
+
+from __future__ import annotations
+
+import csv
+
+from mailrows.config import ConfigError
+from mailrows.hooks import EXTRA_VALUES, Hook, HookRow
+
+
+class CsvAppender(Hook):
+    name = "csv_append"
+
+    def __init__(self, settings):
+        super().__init__(settings)
+        self.path = settings.resolve(settings.get("csv_append", "path"))
+        self.columns = settings.get_list("csv_append", "columns") or ["_row_id", "_recipient", "_sent_at"]
+
+    def describe(self):
+        return f"csv_append: add a line to {self.path.name}"
+
+    def check_columns(self, columns):
+        unknown = [c for c in self.columns if c not in columns and c not in EXTRA_VALUES]
+        if unknown:
+            raise ConfigError(f"[csv_append] columns lists {', '.join(unknown)}, not in the spreadsheet.")
+
+    def run(self, row: HookRow) -> None:
+        merged = row.merged()
+        new = not self.path.exists() or self.path.stat().st_size == 0
+        with open(self.path, "a", encoding="utf-8", newline="") as f:
+            w = csv.writer(f)
+            if new:
+                w.writerow(self.columns)
+            w.writerow([_safe(merged.get(c, "")) for c in self.columns])
+
+
+def _safe(value: str) -> str:
+    """Stop a spreadsheet program from running a value as a formula."""
+    return "'" + value if value[:1] in ("=", "+", "-", "@") else value

mailrows/hooks/http.py

@@ -0,0 +1,103 @@
+"""A tiny JSON-over-HTTP client with a rate limit and retries, on the standard library."""
+
+from __future__ import annotations
+
+import http.client
+import json
+import random
+import time
+import urllib.error
+import urllib.request
+
+RETRY_CODES = {429, 500, 502, 503, 504}
+
+
+class _NoRedirects(urllib.request.HTTPRedirectHandler):
+    """urllib would follow a redirect and resend our Authorization and secret headers to
+    whatever host it points at. An API call never needs a redirect, so refuse them all."""
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        raise urllib.error.HTTPError(req.full_url, code,
+                                     f"refused to follow a redirect to {newurl.split('?')[0]}", headers, fp)
+
+
+_OPENER = urllib.request.build_opener(_NoRedirects)
+
+
+class HttpError(Exception):
+    def __init__(self, message: str, status: int | None = None, body: str = ""):
+        super().__init__(message)
+        self.status = status
+        self.body = body
+
+
+class RateLimiter:
+    """Never more than `per_second` calls in any one second (spaced evenly)."""
+
+    def __init__(self, per_second: float, clock=time.monotonic, sleep=time.sleep):
+        self.gap = 1.0 / per_second if per_second > 0 else 0.0
+        self.clock = clock
+        self.sleep = sleep
+        self.last = None
+
+    def wait(self) -> None:
+        if self.last is not None:
+            left = self.last + self.gap - self.clock()
+            if left > 0:
+                self.sleep(left)
+        self.last = self.clock()
+
+
+class JsonClient:
+    def __init__(self, headers: dict[str, str], per_second: float = 4.5, retries: int = 5,
+                 backoff: float = 1.0, max_wait: float = 30.0, timeout: float = 30.0, sleep=time.sleep):
+        self.headers = headers
+        self.limiter = RateLimiter(per_second, sleep=sleep)
+        self.retries = retries
+        self.backoff = backoff
+        self.max_wait = max_wait
+        self.timeout = timeout
+        self.sleep = sleep
+
+    def call(self, method: str, url: str, body=None):
+        data = json.dumps(body).encode("utf-8") if body is not None else None
+        headers = dict(self.headers)
+        if data is not None:
+            headers["Content-Type"] = "application/json"
+        last: HttpError | None = None
+        for attempt in range(self.retries + 1):
+            self.limiter.wait()
+            req = urllib.request.Request(url, data=data, method=method, headers=headers)
+            try:
+                with _OPENER.open(req, timeout=self.timeout) as resp:
+                    raw = resp.read()
+                    return json.loads(raw.decode("utf-8")) if raw.strip() else {}
+            except urllib.error.HTTPError as exc:
+                if 300 <= exc.code < 400:
+                    raise HttpError(f"HTTP {exc.code}: {exc.reason}", exc.code) from None
+                text = exc.read().decode("utf-8", "replace")[:500] if exc.fp else ""
+                last = HttpError(f"HTTP {exc.code}", exc.code, text)
+                if exc.code not in RETRY_CODES:
+                    raise last from None
+                wait = _retry_after(exc.headers.get("Retry-After") if exc.headers else None)
+                if wait is not None:
+                    wait = min(wait, self.max_wait * 2)
+            except (urllib.error.URLError, http.client.HTTPException, OSError) as exc:
+                reason = getattr(exc, "reason", exc)
+                last = HttpError(f"could not reach the server ({reason})")
+                wait = None
+            except ValueError:
+                raise HttpError("the server sent back a reply that isn't JSON") from None
+            if attempt == self.retries:
+                break
+            if wait is None:
+                wait = min(self.max_wait, self.backoff * (2 ** attempt)) * random.uniform(0.8, 1.2)
+            self.sleep(wait)
+        raise last
+
+
+def _retry_after(value):
+    try:
+        return max(0.0, float(value)) if value else None
+    except ValueError:
+        return None

mailrows/hooks/webhook.py

@@ -0,0 +1,66 @@
+"""POST a small JSON document to a URL after each email.
+
+    [webhook]
+    url = https://hooks.example.com/mailrows
+    ; optional header, e.g. for a shared secret. The value can come from MAILROWS_WEBHOOK_HEADER_VALUE
+    header_name = X-Mailrows-Token
+    header_value = CHANGE_ME
+    ; which columns to include (default: all)
+    columns = id, email
+
+The body is {"row_id", "recipient", "subject", "sent_at", "row": {column: value}}.
+A receiver should treat row_id as the idempotency key: a retry can deliver the same row twice.
+"""
+
+from __future__ import annotations
+
+import urllib.parse
+
+from mailrows.config import ConfigError, looks_unset
+from mailrows.hooks import Hook, HookError, HookRow
+from mailrows.hooks.http import HttpError, JsonClient
+
+def _is_local(url: str) -> bool:
+    return urllib.parse.urlsplit(url).hostname in ("localhost", "127.0.0.1", "::1")
+
+
+class PostWebhook(Hook):
+    name = "webhook"
+
+    def __init__(self, settings, sleep=None):
+        super().__init__(settings)
+        s = settings
+        self.url = s.get("webhook", "url")
+        if not self.url.lower().startswith(("https://", "http://")):
+            raise ConfigError("[webhook] url must start with https:// (or http:// for a local test).")
+        headers = {"User-Agent": "mailrows"}
+        name = s.get("webhook", "header_name", "")
+        if name:
+            value = s.get("webhook", "header_value", "")
+            if looks_unset(value):
+                raise ConfigError("[webhook] header_value is not set (or set MAILROWS_WEBHOOK_HEADER_VALUE).")
+            headers[name] = value
+            if self.url.lower().startswith("http://") and not _is_local(self.url):
+                raise ConfigError("[webhook] url uses http://, which would send header_value unencrypted. "
+                                  "Use https:// (http:// is only allowed for localhost).")
+        self.columns = s.get_list("webhook", "columns")
+        extra = {"sleep": sleep} if sleep else {}
+        self.client = JsonClient(headers, per_second=s.get_float("webhook", "per_second", 5),
+                                 retries=s.get_int("webhook", "retries", 4), **extra)
+
+    def describe(self):
+        return f"webhook: POST to {self.url.split('?')[0]}"
+
+    def check_columns(self, columns):
+        unknown = [c for c in self.columns if c not in columns]
+        if unknown:
+            raise ConfigError(f"[webhook] columns lists {', '.join(unknown)}, not in the spreadsheet.")
+
+    def run(self, row: HookRow) -> None:
+        values = {c: row.values.get(c, "") for c in self.columns} if self.columns else dict(row.values)
+        body = {"row_id": row.row_id, "recipient": row.recipient, "subject": row.subject,
+                "sent_at": row.sent_at, "row": values}
+        try:
+            self.client.call("POST", self.url, body)
+        except HttpError as exc:
+            raise HookError(f"webhook: {exc}") from None

mailrows/ledger.py

@@ -0,0 +1,210 @@
+"""The send log: a small SQLite file that is the final word on what has been sent.
+
+The order of events for one email is what makes reruns safe:
+
+1. claim():     a 'sending' record is committed BEFORE the email is handed to the server.
+2. the email is sent.
+3. mark_sent(): the record becomes 'sent'.
+
+If the process dies between 1 and 3, the record stays 'sending'. A rerun sees it and does
+not send again; a person checks their Sent folder and runs 'mailrows resolve'. If the
+server clearly refused the email, release() removes the claim so a rerun can try again.
+
+Each sent email also remembers which post-send hooks it still owes (an Airtable update,
+for example). Those are retried on later runs without sending the email again.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS sends (
+    row_id      TEXT PRIMARY KEY,
+    fingerprint TEXT NOT NULL,
+    recipient   TEXT NOT NULL,
+    state       TEXT NOT NULL CHECK (state IN ('sending', 'sent')),
+    hooks       TEXT NOT NULL DEFAULT '',
+    claimed_at  TEXT NOT NULL,
+    sent_at     TEXT
+);
+CREATE INDEX IF NOT EXISTS sends_by_fingerprint ON sends (fingerprint);
+CREATE TABLE IF NOT EXISTS hooks_done (
+    row_id  TEXT NOT NULL,
+    hook    TEXT NOT NULL,
+    done_at TEXT NOT NULL,
+    PRIMARY KEY (row_id, hook)
+);
+CREATE TABLE IF NOT EXISTS events (
+    at      TEXT NOT NULL,
+    row_id  TEXT NOT NULL,
+    event   TEXT NOT NULL,
+    detail  TEXT NOT NULL DEFAULT ''
+);
+"""
+
+
+def now() -> str:
+    return dt.datetime.now().astimezone().isoformat(timespec="seconds")
+
+
+@dataclass(frozen=True)
+class Entry:
+    row_id: str
+    fingerprint: str
+    recipient: str
+    state: str
+    hooks: tuple[str, ...]
+    done: frozenset[str]
+    sent_at: str | None
+
+    @property
+    def pending_hooks(self) -> list[str]:
+        return [h for h in self.hooks if h not in self.done]
+
+
+class ClaimTaken(Exception):
+    pass
+
+
+class Ledger:
+    def __init__(self, path: Path):
+        self.path = path
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self.db = sqlite3.connect(str(path), isolation_level=None, timeout=30)
+        self.db.execute("PRAGMA journal_mode=DELETE")
+        self.db.execute("PRAGMA synchronous=FULL")
+        self.db.executescript(SCHEMA)
+
+    @classmethod
+    def read_only(cls, path: Path) -> "ReadOnlyLedger":
+        return ReadOnlyLedger(path)
+
+    def close(self) -> None:
+        self.db.close()
+
+    # ---- reading
+
+    def entry(self, row_id: str) -> Entry | None:
+        r = self.db.execute(
+            "SELECT row_id, fingerprint, recipient, state, hooks, sent_at FROM sends WHERE row_id = ?", (row_id,)
+        ).fetchone()
+        if r is None:
+            return None
+        done = {h for (h,) in self.db.execute("SELECT hook FROM hooks_done WHERE row_id = ?", (row_id,))}
+        hooks = tuple(h for h in r[4].split(",") if h)
+        return Entry(r[0], r[1], r[2], r[3], hooks, frozenset(done), r[5])
+
+    def all_entries(self) -> dict[str, Entry]:
+        ids = [r for (r,) in self.db.execute("SELECT row_id FROM sends")]
+        return {i: self.entry(i) for i in ids}
+
+    def by_fingerprint(self, fingerprint: str) -> str | None:
+        r = self.db.execute("SELECT row_id FROM sends WHERE fingerprint = ? LIMIT 1", (fingerprint,)).fetchone()
+        return r[0] if r else None
+
+    # ---- writing
+
+    def claim(self, row_id: str, fingerprint: str, recipient: str, hooks: list[str]) -> None:
+        """Record the intent to send, atomically. Raises ClaimTaken if any run got there first."""
+        try:
+            self.db.execute("BEGIN IMMEDIATE")
+            exists = self.db.execute(
+                "SELECT 1 FROM sends WHERE row_id = ? OR fingerprint = ?", (row_id, fingerprint)
+            ).fetchone()
+            if exists:
+                self.db.execute("ROLLBACK")
+                raise ClaimTaken(row_id)
+            self.db.execute(
+                "INSERT INTO sends (row_id, fingerprint, recipient, state, hooks, claimed_at) VALUES (?,?,?,?,?,?)",
+                (row_id, fingerprint, recipient, "sending", ",".join(hooks), now()),
+            )
+            self._event(row_id, "claimed")
+            self.db.execute("COMMIT")
+        except sqlite3.Error:
+            if self.db.in_transaction:
+                self.db.execute("ROLLBACK")
+            raise
+
+    def mark_sent(self, row_id: str) -> None:
+        with self._tx():
+            self.db.execute("UPDATE sends SET state = 'sent', sent_at = ? WHERE row_id = ?", (now(), row_id))
+            self._event(row_id, "sent")
+
+    def release(self, row_id: str, why: str) -> None:
+        """The server refused the email, so it did not go: allow a later retry."""
+        with self._tx():
+            self.db.execute("DELETE FROM sends WHERE row_id = ? AND state = 'sending'", (row_id,))
+            self._event(row_id, "refused", why)
+
+    def mark_hook_done(self, row_id: str, hook: str) -> None:
+        with self._tx():
+            self.db.execute(
+                "INSERT OR REPLACE INTO hooks_done (row_id, hook, done_at) VALUES (?,?,?)", (row_id, hook, now())
+            )
+            self._event(row_id, f"hook {hook} done")
+
+    def note(self, row_id: str, event: str, detail: str = "") -> None:
+        with self._tx():
+            self._event(row_id, event, detail)
+
+    def resolve(self, row_id: str, was_sent: bool) -> Entry | None:
+        """A person checked their Sent folder for an email whose outcome wasn't known."""
+        e = self.entry(row_id)
+        if e is None or e.state != "sending":
+            return e
+        with self._tx():
+            if was_sent:
+                self.db.execute("UPDATE sends SET state = 'sent', sent_at = ? WHERE row_id = ?", (now(), row_id))
+                self._event(row_id, "resolved as sent")
+            else:
+                self.db.execute("DELETE FROM sends WHERE row_id = ?", (row_id,))
+                self._event(row_id, "resolved as not sent")
+        return self.entry(row_id)
+
+    # ---- helpers
+
+    def _event(self, row_id: str, event: str, detail: str = "") -> None:
+        self.db.execute("INSERT INTO events (at, row_id, event, detail) VALUES (?,?,?,?)",
+                        (now(), row_id, event, detail[:500]))
+
+    def _tx(self):
+        db = self.db
+
+        class _T:
+            def __enter__(self):
+                db.execute("BEGIN IMMEDIATE")
+
+            def __exit__(self, kind, *_):
+                db.execute("COMMIT" if kind is None else "ROLLBACK")
+                return False
+
+        return _T()
+
+
+class ReadOnlyLedger:
+    """For dry runs and test sends: reads the log if it exists, never creates or changes it."""
+
+    def __init__(self, path: Path):
+        self._inner = None
+        if path.exists():
+            uri = path.resolve().as_uri() + "?mode=ro"
+            self._inner = Ledger.__new__(Ledger)
+            self._inner.path = path
+            self._inner.db = sqlite3.connect(uri, uri=True, isolation_level=None, timeout=30)
+
+    def entry(self, row_id):
+        return self._inner.entry(row_id) if self._inner else None
+
+    def all_entries(self):
+        return self._inner.all_entries() if self._inner else {}
+
+    def by_fingerprint(self, fingerprint):
+        return self._inner.by_fingerprint(fingerprint) if self._inner else None
+
+    def close(self):
+        if self._inner:
+            self._inner.db.close()

mailrows/mailer.py

@@ -0,0 +1,205 @@
+"""Sending over SMTP, with a clear answer to "did that email go or not?".
+
+The SMTP conversation is driven step by step (MAIL, RCPT, DATA) instead of through
+smtplib's sendmail(), so it is known whether the server could already have the message
+when something goes wrong:
+
+* a refusal or a dropped connection before DATA: the email certainly did not go (Refused);
+* a dropped connection or timeout after DATA started: it may have gone (Uncertain).
+"""
+
+from __future__ import annotations
+
+import smtplib
+import ssl
+import time
+from dataclasses import dataclass
+from email.message import EmailMessage
+from email.policy import SMTP as SMTP_POLICY
+from email.utils import formataddr, formatdate, make_msgid
+
+from mailrows.config import ConfigError, Settings
+from mailrows.template import Rendered
+
+LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1"}
+
+
+class Refused(Exception):
+    """The email was not accepted. Safe to try again later."""
+
+    def __init__(self, message: str, unreachable: bool = False):
+        super().__init__(message)
+        self.unreachable = unreachable  # True when the server could not be reached at all
+
+
+class StoppedBeforeSending(KeyboardInterrupt):
+    """Ctrl+C arrived before any of the message reached the server: it certainly did not go."""
+
+
+class Uncertain(Exception):
+    """The connection broke while the server was receiving the email. It may have been delivered."""
+
+
+@dataclass
+class SmtpSettings:
+    host: str
+    port: int
+    security: str  # "ssl", "starttls" or "none"
+    username: str
+    password: str
+    timeout: float = 30
+
+    @classmethod
+    def from_settings(cls, s: Settings, need_password: bool = True) -> "SmtpSettings":
+        security = s.get("smtp", "security", "starttls").lower()
+        if security not in ("ssl", "starttls", "none"):
+            raise ConfigError("[smtp] security must be ssl, starttls or none.")
+        username = s.get("smtp", "username", "")
+        password = s.secret("smtp", "password") if (username and need_password) else s.get("smtp", "password", "")
+        host = s.get("smtp", "host")
+        if security == "none" and username and host.lower() not in LOCAL_HOSTS:
+            raise ConfigError(
+                "[smtp] security = none would send your password unencrypted. Use ssl or starttls "
+                "(run 'mailrows smtp-check' to find the right setting)."
+            )
+        return cls(host=host, port=s.get_int("smtp", "port", 587), security=security,
+                   username=username, password=password, timeout=s.get_float("smtp", "timeout", 30))
+
+
+@dataclass
+class Sender:
+    from_email: str
+    from_name: str = ""
+    reply_to: str = ""
+    bcc: str = ""
+
+    @classmethod
+    def from_settings(cls, s: Settings) -> "Sender":
+        return cls(from_email=s.get("sender", "from_email"), from_name=s.get("sender", "from_name", ""),
+                   reply_to=s.get("sender", "reply_to", ""), bcc=s.get("sender", "bcc", ""))
+
+
+def connect(cfg: SmtpSettings) -> smtplib.SMTP:
+    context = ssl.create_default_context()
+    if cfg.security == "ssl":
+        conn = smtplib.SMTP_SSL(cfg.host, cfg.port, timeout=cfg.timeout, context=context)
+    else:
+        conn = smtplib.SMTP(cfg.host, cfg.port, timeout=cfg.timeout)
+        conn.ehlo()
+        if cfg.security == "starttls":
+            conn.starttls(context=context)
+            conn.ehlo()
+    if cfg.username:
+        conn.login(cfg.username, cfg.password)
+    return conn
+
+
+def build_message(sender: Sender, msg: Rendered, to: str | None = None) -> EmailMessage:
+    m = EmailMessage()
+    m["From"] = formataddr((sender.from_name, sender.from_email)) if sender.from_name else sender.from_email
+    m["To"] = to or msg.to
+    m["Subject"] = msg.subject
+    m["Date"] = formatdate(localtime=True)
+    m["Message-ID"] = make_msgid(domain=sender.from_email.rpartition("@")[2] or None)
+    if sender.reply_to:
+        m["Reply-To"] = sender.reply_to
+    m.set_content(msg.text, cte=_cte(msg.text))
+    if msg.html is not None:
+        m.add_alternative(msg.html, subtype="html", cte=_cte(msg.html))
+    return m
+
+
+def _cte(text: str) -> str:
+    """7bit when that is safe, else quoted-printable, so no server needs 8BITMIME support."""
+    if text.isascii() and all(len(line) <= 900 for line in text.splitlines()):
+        return "7bit"
+    return "quoted-printable"
+
+
+class Mailer:
+    """One SMTP connection, reopened if the server drops it between emails."""
+
+    def __init__(self, cfg: SmtpSettings, sender: Sender):
+        self.cfg = cfg
+        self.sender = sender
+        self.conn: smtplib.SMTP | None = None
+
+    def __enter__(self):
+        self.conn = connect(self.cfg)
+        return self
+
+    def __exit__(self, kind, value, tb):
+        if self.conn is not None:
+            try:
+                self.conn.quit()
+            except (smtplib.SMTPException, OSError):
+                pass
+        self.conn = None
+
+    def _ready(self) -> smtplib.SMTP:
+        if self.conn is not None:
+            try:
+                code, _ = self.conn.noop()
+                if code == 250:
+                    return self.conn
+            except (smtplib.SMTPException, OSError):
+                pass
+            try:
+                self.conn.close()
+            except (smtplib.SMTPException, OSError):
+                pass
+        self.conn = connect(self.cfg)
+        return self.conn
+
+    def send(self, message: EmailMessage, envelope_to: list[str]) -> None:
+        """Send or raise Refused / Uncertain. BCC addresses go in envelope_to only, never in a header.
+
+        Ctrl+C before the message started going raises StoppedBeforeSending (it did not go);
+        Ctrl+C after that is re-raised as a plain KeyboardInterrupt (it may have gone)."""
+        data_started = False
+        try:
+            try:
+                conn = self._ready()
+            except (smtplib.SMTPException, OSError) as exc:
+                raise Refused(f"could not connect to the mail server: {exc}", unreachable=True) from exc
+            code, resp = conn.mail(self.sender.from_email)
+            if code != 250:
+                raise Refused(f"server refused the sender ({code} {resp!r})")
+            accepted = 0
+            for addr in envelope_to:
+                code, resp = conn.rcpt(addr)
+                if code in (250, 251):
+                    accepted += 1
+                elif addr == envelope_to[0]:
+                    conn.rset()
+                    raise Refused(f"server refused {addr} ({code} {_text(resp)})")
+            if not accepted:
+                raise Refused("server refused every recipient")
+            data_started = True
+            code, resp = conn.data(message.as_bytes(policy=SMTP_POLICY))
+            if code != 250:
+                raise Refused(f"server refused the message ({code} {_text(resp)})")
+        except Refused:
+            raise
+        except (KeyboardInterrupt, SystemExit):
+            self.conn = None  # the conversation was cut off mid-way; never reuse it
+            if data_started:
+                raise
+            raise StoppedBeforeSending() from None
+        except smtplib.SMTPResponseException as exc:
+            # a numbered reply from the server: it answered, and the answer was no
+            raise Refused(f"server said {exc.smtp_code} {_text(exc.smtp_error)}") from exc
+        except (smtplib.SMTPException, OSError) as exc:
+            self.conn = None
+            if data_started:
+                raise Uncertain(f"the connection broke while the server was receiving the email ({exc})") from exc
+            raise Refused(f"the connection broke before the email was handed over ({exc})") from exc
+
+
+def _text(resp) -> str:
+    return resp.decode("utf-8", "replace") if isinstance(resp, bytes) else str(resp)
+
+
+def pause(seconds: float) -> None:
+    if seconds > 0:
+        time.sleep(seconds)