docketry 0.1.0__py3-none-any.whl

docketry/config.py

@@ -0,0 +1,77 @@
+"""Home-directory config for a Docketry installation.
+
+A Docketry "home" is one directory holding config.toml, the guardrail
+manifest, the SQLite store, and attachments — the whole installation is one
+folder on the firm's own disk. The IMAP password is read from the
+DOCKETRY_IMAP_PASSWORD environment variable first; storing it in config.toml
+is supported for single-machine setups (the file is chmod 0600 on write).
+"""
+from __future__ import annotations
+
+import os
+import stat
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+from .mailbox import MailboxConfig
+
+CONFIG_NAME = "config.toml"
+MANIFEST_NAME = "guardrails.toml"
+STORE_DIR = "store"
+
+
+@dataclass
+class HomeConfig:
+    home: Path
+    mailbox: MailboxConfig | None
+    manifest_path: Path
+    store_path: Path
+
+
+def write_config(
+    home: Path,
+    *,
+    host: str,
+    user: str,
+    folder: str = "INBOX",
+    password: str | None = None,
+) -> Path:
+    home.mkdir(parents=True, exist_ok=True)
+    cfg = home / CONFIG_NAME
+    lines = [
+        "[mailbox]",
+        f'host = "{host}"',
+        f'user = "{user}"',
+        f'folder = "{folder}"',
+    ]
+    if password:
+        lines.append(f'password = "{password}"')
+    lines.append("")
+    cfg.write_text("\n".join(lines))
+    os.chmod(cfg, stat.S_IRUSR | stat.S_IWUSR)
+    return cfg
+
+
+def load_home(home: str | Path) -> HomeConfig:
+    home = Path(home)
+    cfg_path = home / CONFIG_NAME
+    mailbox = None
+    if cfg_path.exists():
+        data = tomllib.loads(cfg_path.read_text())
+        mb = data.get("mailbox", {})
+        password = os.environ.get("DOCKETRY_IMAP_PASSWORD") or mb.get("password", "")
+        if mb.get("host") and mb.get("user"):
+            mailbox = MailboxConfig(
+                host=mb["host"],
+                user=mb["user"],
+                password=password,
+                folder=mb.get("folder", "INBOX"),
+                port=int(mb.get("port", 993)),
+            )
+    return HomeConfig(
+        home=home,
+        mailbox=mailbox,
+        manifest_path=home / MANIFEST_NAME,
+        store_path=home / STORE_DIR,
+    )

docketry/envelope.py

@@ -0,0 +1,166 @@
+"""MIME message -> normalized Envelope.
+
+The port's one job: whatever arrives, reduce it to the same provenance-stamped
+shape before anything downstream sees it. Parsing is stdlib-only and read-only;
+the raw message is never modified, and the raw hash travels with the envelope
+so provenance survives every later stage.
+"""
+from __future__ import annotations
+
+import email
+import email.policy
+import hashlib
+import re
+from dataclasses import asdict, dataclass, field
+from email.message import EmailMessage
+from email.utils import getaddresses, parsedate_to_datetime
+from html.parser import HTMLParser
+
+_FILENAME_SAFE = re.compile(r"[^A-Za-z0-9._ ()\[\]-]")
+
+
+class _HTMLText(HTMLParser):
+    _SKIP = {"script", "style", "head"}
+    _BREAK = {"p", "br", "div", "tr", "li", "table"}
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._chunks: list[str] = []
+        self._skip_depth = 0
+
+    def handle_starttag(self, tag: str, attrs) -> None:
+        if tag in self._SKIP:
+            self._skip_depth += 1
+        elif tag in self._BREAK:
+            self._chunks.append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in self._SKIP and self._skip_depth:
+            self._skip_depth -= 1
+
+    def handle_data(self, data: str) -> None:
+        if not self._skip_depth:
+            self._chunks.append(data)
+
+    def text(self) -> str:
+        raw = "".join(self._chunks)
+        lines = [ln.strip() for ln in raw.splitlines()]
+        return "\n".join(ln for ln in lines if ln)
+
+
+def html_to_text(html: str) -> str:
+    parser = _HTMLText()
+    try:
+        parser.feed(html)
+        parser.close()
+    except Exception:
+        return html
+    return parser.text()
+
+
+def sanitize_filename(name: str) -> str:
+    """Keep only the basename and drop anything shell- or path-hostile."""
+    name = name.replace("\\", "/").split("/")[-1].strip()
+    name = _FILENAME_SAFE.sub("_", name)
+    return name or "attachment.bin"
+
+
+@dataclass
+class Attachment:
+    filename: str
+    content_type: str
+    sha256: str
+    size: int
+    content: bytes = field(repr=False, compare=False)
+
+
+@dataclass
+class Envelope:
+    message_id: str
+    from_addr: str
+    to: list[str]
+    cc: list[str]
+    date: str            # ISO 8601, "" when unparseable
+    subject: str
+    body_text: str
+    attachments: list[Attachment]
+    raw_sha256: str
+    source: str          # intake mailbox this arrived through
+    fetched_at: str      # ISO 8601, stamped by the port
+
+    def to_record(self) -> dict:
+        """JSON-safe form; attachment bytes are stored on disk, not in the row."""
+        d = asdict(self)
+        for a in d["attachments"]:
+            a.pop("content", None)
+        return d
+
+
+def _addresses(msg: EmailMessage, header: str) -> list[str]:
+    return [addr for _, addr in getaddresses(msg.get_all(header, [])) if addr]
+
+
+def _body_text(msg: EmailMessage) -> str:
+    body = msg.get_body(preferencelist=("plain", "html"))
+    if body is None:
+        return ""
+    try:
+        content = body.get_content()
+    except Exception:
+        payload = body.get_payload(decode=True) or b""
+        content = payload.decode("utf-8", "replace")
+    if body.get_content_type() == "text/html":
+        return html_to_text(content)
+    return content.strip()
+
+
+def _attachments(msg: EmailMessage) -> list[Attachment]:
+    out: list[Attachment] = []
+    for part in msg.iter_attachments():
+        content = part.get_payload(decode=True)
+        if content is None:
+            payload = part.get_payload()
+            content = payload.encode("utf-8", "replace") if isinstance(payload, str) else b""
+        out.append(
+            Attachment(
+                filename=sanitize_filename(part.get_filename() or "attachment.bin"),
+                content_type=part.get_content_type(),
+                sha256=hashlib.sha256(content).hexdigest(),
+                size=len(content),
+                content=content,
+            )
+        )
+    return out
+
+
+def parse_message(raw: bytes, *, source: str, fetched_at: str) -> Envelope:
+    msg = email.message_from_bytes(raw, policy=email.policy.default)
+    raw_sha = hashlib.sha256(raw).hexdigest()
+
+    message_id = (msg.get("Message-ID") or "").strip().strip("<>")
+    if not message_id:
+        message_id = f"docketry-{raw_sha[:32]}"
+
+    date_iso = ""
+    if msg.get("Date"):
+        try:
+            date_iso = parsedate_to_datetime(msg["Date"]).isoformat()
+        except Exception:
+            date_iso = ""
+
+    from_pairs = getaddresses(msg.get_all("From", []))
+    from_addr = from_pairs[0][1] if from_pairs else ""
+
+    return Envelope(
+        message_id=message_id,
+        from_addr=from_addr,
+        to=_addresses(msg, "To"),
+        cc=_addresses(msg, "Cc"),
+        date=date_iso,
+        subject=str(msg.get("Subject") or ""),
+        body_text=_body_text(msg),
+        attachments=_attachments(msg),
+        raw_sha256=raw_sha,
+        source=source,
+        fetched_at=fetched_at,
+    )

docketry/extract.py

@@ -0,0 +1,195 @@
+"""Text extraction layer: attachment in, text with a page map out.
+
+One interface for every downstream tool (classifier, citation verifier,
+linter). Provenance and confidence travel with the text:
+
+- PDF: native text per page via pypdf (extra: pdf). Pages with no native
+  text are reported as warnings; when OCR is requested/available the whole
+  scanned document is OCR'd page by page and every OCR page carries a mean
+  word confidence (0-100).
+- OCR: requires the Tesseract binary + poppler's pdftoppm on the system, and
+  the pytesseract/Pillow packages (extra: ocr). Missing pieces raise
+  ExtractionError naming exactly what to install — low-confidence text is
+  flagged, garbage is never passed downstream silently.
+- DOCX: python-docx paragraphs + table text (extra: docx). Word documents
+  have no fixed pages, so the result is one logical page and a warning says
+  pin-citing to page numbers is not possible from this format.
+- TXT: stdlib.
+"""
+from __future__ import annotations
+
+import shutil
+import subprocess
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+
+LOW_CONFIDENCE = 60.0
+
+
+class ExtractionError(RuntimeError):
+    pass
+
+
+@dataclass
+class Page:
+    number: int
+    text: str
+    method: str                      # native | ocr | docx | text
+    confidence: float | None = None  # OCR mean word confidence, None otherwise
+
+
+@dataclass
+class Extraction:
+    pages: list[Page]
+    method: str
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def full_text(self) -> str:
+        return "\n".join(p.text for p in self.pages)
+
+    def page_for_offset(self, offset: int) -> int | None:
+        """Map a character offset in full_text back to a page number."""
+        pos = 0
+        for p in self.pages:
+            end = pos + len(p.text)
+            if offset <= end:
+                return p.number
+            pos = end + 1  # the joining newline
+        return self.pages[-1].number if self.pages else None
+
+
+def _require(module: str, extra: str):
+    try:
+        return __import__(module)
+    except ImportError:
+        raise ExtractionError(
+            f"extracting this file type needs the '{extra}' extra:"
+            f" pip install 'docketry[{extra}]'"
+        ) from None
+
+
+def _extract_pdf(path: Path, *, ocr: str) -> Extraction:
+    pypdf = _require("pypdf", "pdf")
+    reader = pypdf.PdfReader(str(path))
+    pages: list[Page] = []
+    warnings: list[str] = []
+    empty = 0
+    for i, page in enumerate(reader.pages, start=1):
+        text = (page.extract_text() or "").strip()
+        if not text:
+            empty += 1
+            warnings.append(f"page {i}: no extractable text (likely scanned)")
+        pages.append(Page(number=i, text=text, method="native"))
+
+    mostly_empty = empty > len(pages) / 2 if pages else False
+    if ocr == "always" or (ocr == "auto" and mostly_empty):
+        try:
+            return _ocr_pdf(path)
+        except ExtractionError:
+            if ocr == "always":
+                raise
+            warnings.append(
+                f"{empty}/{len(pages)} pages have no native text and OCR is"
+                " unavailable — install the 'ocr' extra plus tesseract and"
+                " poppler-utils for scanned documents"
+            )
+    return Extraction(pages=pages, method="native", warnings=warnings)
+
+
+def _ocr_pdf(path: Path) -> Extraction:
+    pytesseract = _require("pytesseract", "ocr")
+    _require("PIL", "ocr")
+    from PIL import Image
+
+    for binary, package in (("tesseract", "tesseract-ocr"), ("pdftoppm", "poppler-utils")):
+        if shutil.which(binary) is None:
+            raise ExtractionError(
+                f"OCR needs the '{binary}' system binary (install {package})"
+            )
+
+    pages: list[Page] = []
+    warnings: list[str] = []
+    with tempfile.TemporaryDirectory() as tmp:
+        subprocess.run(
+            ["pdftoppm", "-r", "300", "-png", str(path), f"{tmp}/page"],
+            check=True, capture_output=True,
+        )
+        images = sorted(Path(tmp).glob("page*.png"))
+        if not images:
+            raise ExtractionError("pdftoppm produced no page images")
+        for i, img_path in enumerate(images, start=1):
+            with Image.open(img_path) as img:
+                data = pytesseract.image_to_data(
+                    img, output_type=pytesseract.Output.DICT
+                )
+            words, confs = [], []
+            for word, conf in zip(data["text"], data["conf"]):
+                if word.strip() and float(conf) >= 0:
+                    words.append(word)
+                    confs.append(float(conf))
+            confidence = sum(confs) / len(confs) if confs else 0.0
+            if confidence < LOW_CONFIDENCE:
+                warnings.append(
+                    f"page {i}: OCR confidence {confidence:.0f} is below"
+                    f" {LOW_CONFIDENCE:.0f} — text may be unreliable"
+                )
+            pages.append(
+                Page(number=i, text=" ".join(words), method="ocr", confidence=confidence)
+            )
+    return Extraction(pages=pages, method="ocr", warnings=warnings)
+
+
+def _extract_docx(path: Path) -> Extraction:
+    docx = _require("docx", "docx")
+    document = docx.Document(str(path))
+    parts = [p.text for p in document.paragraphs if p.text.strip()]
+    for table in document.tables:
+        for row in table.rows:
+            cells = [c.text.strip() for c in row.cells if c.text.strip()]
+            if cells:
+                parts.append(" | ".join(cells))
+    return Extraction(
+        pages=[Page(number=1, text="\n".join(parts), method="docx")],
+        method="docx",
+        warnings=["DOCX has no fixed pages; page-level pin cites are not"
+                  " derivable from this format"],
+    )
+
+
+def _extract_txt(path: Path) -> Extraction:
+    text = path.read_bytes().decode("utf-8", "replace")
+    return Extraction(pages=[Page(number=1, text=text, method="text")], method="text")
+
+
+_DISPATCH = {
+    ".pdf": _extract_pdf,
+    ".docx": _extract_docx,
+    ".txt": _extract_txt,
+    ".md": _extract_txt,
+}
+
+
+def extract_path(path: str | Path, *, ocr: str = "auto") -> Extraction:
+    """Extract text from a file. ocr: "auto" | "always" | "never"."""
+    if ocr not in ("auto", "always", "never"):
+        raise ValueError(f"ocr must be auto/always/never, not {ocr!r}")
+    path = Path(path)
+    if not path.exists():
+        raise ExtractionError(f"no such file: {path}")
+    handler = _DISPATCH.get(path.suffix.lower())
+    if handler is None:
+        raise ExtractionError(
+            f"unsupported file type '{path.suffix}' (supported:"
+            f" {', '.join(sorted(_DISPATCH))})"
+        )
+    if handler is _extract_pdf:
+        if ocr == "never":
+            return _extract_pdf_no_ocr(path)
+        return handler(path, ocr=ocr)
+    return handler(path)
+
+
+def _extract_pdf_no_ocr(path: Path) -> Extraction:
+    return _extract_pdf(path, ocr="off")

docketry/gates/__init__.py

@@ -0,0 +1,28 @@
+"""Gate registry: manifests reference gates by id; plugins register here."""
+from __future__ import annotations
+
+_REGISTRY: dict[str, type] = {}
+
+
+def register(cls: type) -> type:
+    gate_id = getattr(cls, "id", None)
+    if not gate_id:
+        raise ValueError(f"{cls.__name__} has no id")
+    _REGISTRY[gate_id] = cls
+    return cls
+
+
+def get(gate_id: str) -> type:
+    if gate_id not in _REGISTRY:
+        raise KeyError(
+            f"unknown gate '{gate_id}' (registered: {', '.join(sorted(_REGISTRY)) or 'none'})"
+        )
+    return _REGISTRY[gate_id]
+
+
+def all_ids() -> list[str]:
+    return sorted(_REGISTRY)
+
+
+# Built-ins register on import.
+from . import builtin, classifier, notice  # noqa: E402,F401

docketry/gates/builtin.py

@@ -0,0 +1,164 @@
+"""Starter gates. Deterministic pipeline-hygiene checks only.
+
+None of these is a security control. Docketry makes no malware, phishing, or
+other cybersecurity claims anywhere — these gates decide what the *pipeline*
+will accept, nothing more. Firms should get actual security from their mail
+provider and endpoint tooling.
+"""
+from __future__ import annotations
+
+from ..envelope import Envelope
+from ..pipeline import Finding, SEVERITY_FAIL, SEVERITY_INFO
+from . import register
+
+
+@register
+class AttachmentPolicy:
+    """What file types and sizes this pipeline accepts. Hygiene, not AV."""
+
+    id = "attachment-policy"
+    allowed_stages = {"ingest"}
+
+    _DEFAULT_DENY = (
+        ".exe .js .vbs .scr .bat .cmd .com .ps1 .jar .msi .hta .lnk".split()
+    )
+
+    def validate_options(self, options: dict) -> list[str]:
+        problems = []
+        if "max_size_mb" in options:
+            try:
+                float(options["max_size_mb"])
+            except (TypeError, ValueError):
+                problems.append("max_size_mb must be a number")
+        deny = options.get("deny_extensions")
+        if deny is not None and (not isinstance(deny, list)
+                                 or not all(isinstance(e, str) for e in deny)):
+            problems.append("deny_extensions must be a list of strings")
+        return problems
+
+    def check(self, env: Envelope, options: dict) -> list[Finding]:
+        deny = {e.lower() for e in options.get("deny_extensions", self._DEFAULT_DENY)}
+        max_mb = float(options.get("max_size_mb", 25))
+        findings: list[Finding] = []
+        for a in env.attachments:
+            ext = ("." + a.filename.rsplit(".", 1)[-1].lower()) if "." in a.filename else ""
+            if ext in deny:
+                findings.append(
+                    Finding(self.id, SEVERITY_FAIL, f"attachment type not accepted: {a.filename}")
+                )
+            if a.size > max_mb * 1024 * 1024:
+                findings.append(
+                    Finding(
+                        self.id,
+                        SEVERITY_FAIL,
+                        f"attachment over {max_mb:g} MB: {a.filename} ({a.size} bytes)",
+                    )
+                )
+        return findings
+
+
+@register
+class SenderScope:
+    """Optionally hold mail from senders outside the expected set.
+
+    An intake mailbox fed by forwarding rules mostly hears from known portals
+    and staff; anything else bounces to a human instead of flowing onward.
+    """
+
+    id = "sender-scope"
+    allowed_stages = {"ingest"}
+
+    def validate_options(self, options: dict) -> list[str]:
+        problems = []
+        for key in ("allow", "deny"):
+            val = options.get(key)
+            if val is not None and (not isinstance(val, list)
+                                    or not all(isinstance(s, str) for s in val)):
+                problems.append(f"{key} must be a list of strings")
+        return problems
+
+    @staticmethod
+    def _matches(sender: str, domain: str, entry: str) -> bool:
+        if sender == entry:
+            return True
+        if entry.startswith("@"):
+            entry_domain = entry[1:]
+            # "@uscourts.gov" covers the domain and its subdomains —
+            # federal NEFs arrive from per-district hosts like
+            # flsd.uscourts.gov.
+            return domain == entry_domain or domain.endswith("." + entry_domain)
+        return False
+
+    def check(self, env: Envelope, options: dict) -> list[Finding]:
+        sender = env.from_addr.lower()
+        domain = sender.rsplit("@", 1)[-1] if "@" in sender else ""
+        for entry in (s.lower() for s in options.get("deny", [])):
+            if self._matches(sender, domain, entry):
+                return [Finding(self.id, SEVERITY_FAIL,
+                                f"sender on the deny list: {env.from_addr}")]
+        allowed = [s.lower() for s in options.get("allow", [])]
+        if not allowed:
+            return []
+        for entry in allowed:
+            if self._matches(sender, domain, entry):
+                return []
+        return [
+            Finding(self.id, SEVERITY_FAIL, f"sender outside intake scope: {env.from_addr}")
+        ]
+
+
+@register
+class ProvenanceStamp:
+    """Records where each message came from; informational, never holds."""
+
+    id = "provenance-stamp"
+    allowed_stages = None
+
+    def check(self, env: Envelope, options: dict) -> list[Finding]:
+        return [
+            Finding(
+                self.id,
+                SEVERITY_INFO,
+                f"source={env.source} raw_sha256={env.raw_sha256[:16]} fetched_at={env.fetched_at}",
+            )
+        ]
+
+
+@register
+class NameScreen:
+    """Hold any message whose content mentions a screened name.
+
+    The legal use is an ethical wall / conflict screen: a firm lists the
+    parties or matters a reviewer must not see flowing through the pipeline
+    unreviewed, and anything mentioning them parks for the declared
+    authority. Terms match case-insensitively on word boundaries across
+    subject, body, and attachment filenames. The screen list lives in the
+    firm's local guardrails.toml and never leaves the machine.
+    """
+
+    id = "name-screen"
+    allowed_stages = None
+
+    def validate_options(self, options: dict) -> list[str]:
+        terms = options.get("terms")
+        if not isinstance(terms, list) or not terms or not all(
+            isinstance(t, str) and t.strip() for t in terms
+        ):
+            return ["terms must be a non-empty list of strings"]
+        return []
+
+    def check(self, env: Envelope, options: dict) -> list[Finding]:
+        import re as _re
+
+        haystack = "\n".join(
+            [env.subject, env.body_text, *[a.filename for a in env.attachments]]
+        )
+        note = options.get("note", "screened name")
+        findings = []
+        for term in options.get("terms", []):
+            if _re.search(rf"\b{_re.escape(term)}\b", haystack, _re.IGNORECASE):
+                findings.append(Finding(
+                    self.id, SEVERITY_FAIL,
+                    f"message mentions '{term}' ({note}) — held for the declared authority",
+                ))
+        return findings

docketry/gates/classifier.py

@@ -0,0 +1,28 @@
+"""Doc-classifier gate: proposes a type for each attachment, never applies.
+
+Informational only — the write path is the staged queue (class-apply),
+fill-only and role-recorded. Classification is free and deterministic first;
+model tiers are somebody else's optional add-on, never a default.
+"""
+from __future__ import annotations
+
+from ..classify import classify
+from ..envelope import Envelope
+from ..pipeline import Finding, SEVERITY_INFO
+from . import register
+
+
+@register
+class DocClassifier:
+    id = "doc-classifier"
+    allowed_stages = None
+
+    def check(self, env: Envelope, options: dict) -> list[Finding]:
+        findings = []
+        for a in env.attachments:
+            label, tier = classify(a.filename)
+            if tier != "low":
+                findings.append(
+                    Finding(self.id, SEVERITY_INFO, f"{a.filename}: proposed {label} ({tier})")
+                )
+        return findings