graphmail 1.0.0__py3-none-any.whl

graphmail/cli.py

@@ -0,0 +1,461 @@
+"""Command-line entry point for graphmail.
+
+Commands:
+    fetch    pull messages from Microsoft Graph folders into the archive
+    import   copy local .eml files into the archive and index them
+    reindex  rebuild index.db from the archive's own .eml files
+    export   rebuild the Excel index from index.db
+    login    run the OAuth flow (or refresh) without fetching
+
+Credentials come from environment variables (never flags; keep secrets out
+of shell history), optionally loaded from an ``--env-file``:
+
+    GRAPHMAIL_CLIENT_ID, GRAPHMAIL_CLIENT_SECRET, GRAPHMAIL_TENANT_ID
+
+Mutating commands (fetch/import/reindex) take a single-job lock and write a
+timestamped run log under ``archive/``. ``index.db``, ``token.json`` and the
+workbook live in the working directory; ``cd`` there and everything resolves
+relative to it (there is no archive-root flag).
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import signal
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from graphmail import __version__, auth, export, graph, importer, pipeline
+from graphmail.auth import OAuthConfig
+from graphmail.envfile import EnvFileError, load_env_file
+from graphmail.index import ArchiveIndex
+from graphmail.lock import ArchiveLock, LockHeldError
+from graphmail.runlog import RunLog
+
+logger = logging.getLogger("graphmail")
+
+ENV_CLIENT_ID = "GRAPHMAIL_CLIENT_ID"
+ENV_CLIENT_SECRET = "GRAPHMAIL_CLIENT_SECRET"
+ENV_TENANT_ID = "GRAPHMAIL_TENANT_ID"
+
+EXIT_LOCKED = 2  # distinct from 1 (failures) so schedulers can tell "still
+# running" from "ran and some mail failed".
+
+
+def _config(args: argparse.Namespace) -> OAuthConfig:
+    """Build the OAuth config from env vars, honoring --token-path."""
+    missing = [
+        name
+        for name in (ENV_CLIENT_ID, ENV_TENANT_ID)
+        if not os.environ.get(name)
+    ]
+    if missing:
+        raise SystemExit(
+            "Missing required environment variable(s): "
+            + ", ".join(missing)
+            + "\nSet them with your Azure app credentials, e.g.:\n"
+            f"  export {ENV_CLIENT_ID}=...\n"
+            f"  export {ENV_TENANT_ID}=common"
+        )
+    return OAuthConfig(
+        client_id=os.environ[ENV_CLIENT_ID],
+        client_secret=os.environ.get(ENV_CLIENT_SECRET, ""),
+        tenant_id=os.environ[ENV_TENANT_ID],
+        token_path=Path(args.token_path),
+    )
+
+
+def _authenticate(args: argparse.Namespace) -> auth.AuthSession | None:
+    """Run auth.authenticate with flow flags from the parsed args."""
+    return auth.authenticate(
+        _config(args),
+        flow="manual" if args.manual else "loopback",
+        auth_port=args.auth_port,
+        timeout=args.auth_timeout,
+    )
+
+
+def resolve_folders(
+    selectors: list[str], folders: list[dict[str, str]]
+) -> list[str]:
+    """Map user folder selectors to Graph folder ids.
+
+    A selector matches a display name case-insensitively; anything else is
+    passed through as a raw folder id (Graph rejects unknown ids with a clear
+    error). The special selector ``all`` selects every top-level folder.
+    """
+    by_name = {f["displayName"].casefold(): f["id"] for f in folders}
+    ids: list[str] = []
+    for selector in selectors:
+        if selector.casefold() == "all":
+            ids.extend(f["id"] for f in folders)
+            continue
+        folder_id = by_name.get(selector.casefold(), selector)
+        if folder_id not in ids:
+            ids.append(folder_id)
+    return ids
+
+
+def _expand_subfolders(
+    access_token: str, folder_ids: list[str]
+) -> list[str]:
+    """Add every descendant of the given folders (breadth-first, deduped).
+
+    Uses a dict as an ordered set: Python string hashing is randomized, so
+    iteration order of a ``set`` is not insertion order.
+    """
+    seen = dict.fromkeys(folder_ids)
+    queue = list(folder_ids)
+    while queue:
+        for child in graph.get_subfolders(access_token, queue.pop(0)):
+            if child["id"] not in seen:
+                seen[child["id"]] = None
+                queue.append(child["id"])
+    return list(seen)
+
+
+def _install_stop_handler() -> graph.StopFlag:
+    """First Ctrl-C: finish the current message, then stop. Second: exit."""
+    flag = graph.StopFlag()
+
+    def _handler(*_args: object) -> None:
+        if flag.stopped:
+            raise KeyboardInterrupt
+        print("\n⏹️ Finishing current message (Ctrl-C again to abort now)")
+        flag.trigger()
+
+    signal.signal(signal.SIGINT, _handler)
+    return flag
+
+
+def _run_fetch(args: argparse.Namespace, archive_root: Path) -> tuple[int, dict[str, int]]:
+    """Authenticate, resolve folders, and archive; returns exit code + counters."""
+    session = _authenticate(args)
+    if session is None:
+        print("Authentication failed.", file=sys.stderr)
+        return 1, {"saved": 0, "duplicates": 0, "failed": 0}
+
+    folders = graph.list_mail_folders(session.access_token)
+    if not folders:
+        print("No mail folders found (or listing failed).", file=sys.stderr)
+        return 1, {"saved": 0, "duplicates": 0, "failed": 0}
+    folder_ids = resolve_folders(args.folders, folders)
+    if not folder_ids:
+        print("No folders selected.", file=sys.stderr)
+        return 1, {"saved": 0, "duplicates": 0, "failed": 0}
+    if args.recursive:
+        folder_ids = _expand_subfolders(session.access_token, folder_ids)
+
+    stop = _install_stop_handler()
+    totals = {"saved": 0, "duplicates": 0, "failed": 0}
+    with ArchiveIndex(Path(args.index)) as index:
+        for folder_id in folder_ids:
+            if stop():
+                break
+            summary = pipeline.process_folder(
+                session.access_token,
+                folder_id,
+                sender=args.sender or "",
+                archive_index=index,
+                output_root=archive_root,
+                should_stop=stop,
+            )
+            totals["saved"] += summary.saved
+            totals["duplicates"] += summary.duplicates
+            totals["failed"] += summary.failed
+    if totals["failed"]:
+        print(
+            f"⚠️ {totals['failed']} message(s) failed and will be retried on the next run."
+        )
+        return 1, totals
+    print("📥 Fetch complete.")
+    return 0, totals
+
+
+def cmd_fetch(args: argparse.Namespace) -> int:
+    archive_root = pipeline.OUTPUT_ROOT
+    archive_root.mkdir(parents=True, exist_ok=True)
+    with (
+        ArchiveLock(archive_root, "fetch"),
+        RunLog(archive_root, "fetch", quiet=args.quiet) as runlog,
+    ):
+        code, totals = _run_fetch(args, archive_root)
+        runlog.finish(code, **totals)
+        return code
+
+
+def cmd_export(args: argparse.Namespace) -> int:
+    index_path = Path(args.index)
+    if not index_path.exists():
+        print(
+            f"Index not found: {index_path} (run `graphmail fetch` first)",
+            file=sys.stderr,
+        )
+        return 1
+    with ArchiveIndex(index_path) as index:
+        out = export.export_to_excel(
+            index,
+            Path(args.output),
+            group_by=args.group_by,
+        )
+        count = index.count()
+    print(f"📊 Wrote {out} ({count} messages)")
+    return 0
+
+
+def cmd_login(args: argparse.Namespace) -> int:
+    session = _authenticate(args)
+    if session is None:
+        print("Authentication failed.", file=sys.stderr)
+        return 1
+    print("✅ Logged in. Token cached.")
+    return 0
+
+
+def _print_stats(verb: str, stats: importer.ImportStats) -> None:
+    print(
+        f"{verb} {stats.imported} file(s), "
+        f"{stats.duplicates} duplicate(s), {stats.errors} error(s)"
+    )
+
+
+def _default_import_folder() -> str:
+    """One folder per import batch, stamped in UTC so batches never merge.
+
+    The stamp keeps the name out of the way of a real Outlook folder that
+    happens to be called "Imported" (fetch would archive that under
+    ``archive/Imported`` too).
+    """
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+    return f"Imported_{stamp}"
+
+
+def cmd_import(args: argparse.Namespace) -> int:
+    source = Path(args.source)
+    if not source.is_dir():
+        print(f"Source directory not found: {source}", file=sys.stderr)
+        return 1
+    archive_root = pipeline.OUTPUT_ROOT
+    archive_root.mkdir(parents=True, exist_ok=True)
+    folder = args.folder or _default_import_folder()
+    with (
+        ArchiveLock(archive_root, "import"),
+        RunLog(archive_root, "import", quiet=args.quiet) as runlog,
+        ArchiveIndex(Path(args.index)) as index,
+    ):
+        stats = importer.import_directory(
+            source,
+            archive_root=archive_root,
+            folder_name=folder,
+            index=index,
+        )
+        _print_stats("📥 Imported", stats)
+        code = 1 if stats.errors and not stats.imported else 0
+        runlog.finish(
+            code,
+            folder=archive_root / folder,
+            imported=stats.imported,
+            duplicates=stats.duplicates,
+            errors=stats.errors,
+        )
+        return code
+
+
+def cmd_reindex(args: argparse.Namespace) -> int:
+    root = pipeline.OUTPUT_ROOT
+    if not root.is_dir():
+        print(f"Archive root not found: {root}", file=sys.stderr)
+        return 1
+    with (
+        ArchiveLock(root, "reindex"),
+        RunLog(root, "reindex", quiet=args.quiet) as runlog,
+        ArchiveIndex(Path(args.index)) as index,
+    ):
+        stats = importer.reindex_archive(root, index=index)
+        _print_stats("🗂️ Reindexed", stats)
+        code = 1 if stats.errors and not stats.imported else 0
+        runlog.finish(
+            code,
+            imported=stats.imported,
+            duplicates=stats.duplicates,
+            errors=stats.errors,
+        )
+        return code
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="graphmail",
+        description="Archive Microsoft Graph mail into .eml files + an Excel index.",
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"graphmail {__version__}"
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="count",
+        default=0,
+        help="-v for INFO, -vv for DEBUG",
+    )
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        help="suppress progress on stdout (run log still written; "
+        "the final === done === line and errors remain)",
+    )
+    parser.add_argument(
+        "--env-file",
+        metavar="PATH",
+        help="dotenv-style KEY=VALUE file (real env vars win over the file)",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    fetch = subparsers.add_parser(
+        "fetch", help="download messages from Graph folders"
+    )
+    fetch.add_argument(
+        "--folders",
+        nargs="+",
+        required=True,
+        metavar="ID_OR_NAME",
+        help="folder display names, raw Graph folder ids, or 'all'",
+    )
+    fetch.add_argument(
+        "--recursive",
+        action="store_true",
+        help="also fetch all subfolders of the selected folders",
+    )
+    fetch.add_argument(
+        "--sender", help="only messages from this sender address"
+    )
+    fetch.add_argument(
+        "--token-path",
+        default="token.json",
+        help="OAuth token cache file (default: token.json)",
+    )
+    fetch.add_argument(
+        "--index",
+        default="index.db",
+        help="archive index path (default: index.db)",
+    )
+    _add_auth_flow_args(fetch)
+    fetch.set_defaults(func=cmd_fetch)
+
+    export_parser = subparsers.add_parser(
+        "export", help="build the Excel index from index.db"
+    )
+    export_parser.add_argument(
+        "-o",
+        "--output",
+        default=str(export.DEFAULT_EXPORT_FILE),
+        help=f"workbook path (default: {export.DEFAULT_EXPORT_FILE})",
+    )
+    export_parser.add_argument(
+        "--index",
+        default="index.db",
+        help="archive index path (default: index.db)",
+    )
+    export_parser.add_argument(
+        "--group-by",
+        choices=["folder", "year", "none"],
+        default="folder",
+        help="sheet grouping (default: folder)",
+    )
+    export_parser.set_defaults(func=cmd_export)
+
+    login = subparsers.add_parser(
+        "login", help="authenticate (or refresh the cached token)"
+    )
+    login.add_argument(
+        "--token-path",
+        default="token.json",
+        help="OAuth token cache file (default: token.json)",
+    )
+    _add_auth_flow_args(login)
+    login.set_defaults(func=cmd_login)
+
+    import_parser = subparsers.add_parser(
+        "import", help="copy local .eml files into the archive and index them"
+    )
+    import_parser.add_argument("source", help="directory containing .eml files")
+    import_parser.add_argument(
+        "--folder",
+        default=None,
+        help="archive folder to copy into "
+        "(default: Imported_<UTC timestamp>, one folder per batch)",
+    )
+    import_parser.add_argument(
+        "--index", default="index.db", help="archive index path (default: index.db)"
+    )
+    import_parser.set_defaults(func=cmd_import)
+
+    reindex = subparsers.add_parser(
+        "reindex", help="rebuild index.db from the archive's own .eml files"
+    )
+    reindex.add_argument(
+        "--index", default="index.db", help="archive index path (default: index.db)"
+    )
+    reindex.set_defaults(func=cmd_reindex)
+
+    return parser
+
+
+def _add_auth_flow_args(parser: argparse.ArgumentParser) -> None:
+    """Shared interactive-auth flags for login/fetch."""
+    parser.add_argument(
+        "--manual",
+        action="store_true",
+        help="paste the authorization code by hand (legacy flow; "
+        "default is loopback on localhost)",
+    )
+    parser.add_argument(
+        "--auth-port",
+        type=int,
+        default=auth.DEFAULT_AUTH_PORT,
+        metavar="PORT",
+        help=f"loopback redirect port (default: {auth.DEFAULT_AUTH_PORT})",
+    )
+    parser.add_argument(
+        "--auth-timeout",
+        type=float,
+        default=auth.LOOPBACK_TIMEOUT,
+        metavar="SECONDS",
+        help=f"seconds to wait for the browser redirect "
+        f"(default: {auth.LOOPBACK_TIMEOUT:.0f})",
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    level = logging.WARNING
+    if args.verbose == 1:
+        level = logging.INFO
+    elif args.verbose >= 2:
+        level = logging.DEBUG
+    logging.basicConfig(level=level, format="%(levelname)s %(name)s: %(message)s")
+
+    if args.env_file:
+        try:
+            for key, value in load_env_file(Path(args.env_file)).items():
+                os.environ.setdefault(key, value)
+        except EnvFileError as exc:
+            print(str(exc), file=sys.stderr)
+            return 1
+
+    try:
+        return int(args.func(args))
+    except LockHeldError as exc:
+        print(str(exc), file=sys.stderr)
+        return EXIT_LOCKED
+    except KeyboardInterrupt:
+        print("\nAborted.", file=sys.stderr)
+        return 130
+
+
+if __name__ == "__main__":  # pragma: no cover - script entry point
+    sys.exit(main())

graphmail/envfile.py

@@ -0,0 +1,46 @@
+"""Dotenv-style credentials file (``KEY=VALUE``) for unattended jobs.
+
+Full-line ``#`` comments, optional ``export `` prefix, quotes stripped.
+No variable expansion. Real environment variables always win.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+class EnvFileError(Exception):
+    """The env file is missing, unreadable, or malformed."""
+
+
+def load_env_file(path: Path) -> dict[str, str]:
+    """Parse *path* into a ``{KEY: VALUE}`` mapping.
+
+    Raises :class:`EnvFileError` if the file cannot be read or a line is
+    malformed.
+    """
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        raise EnvFileError(f"Cannot read env file {path}: {exc}") from exc
+
+    result: dict[str, str] = {}
+    for lineno, raw_line in enumerate(text.splitlines(), start=1):
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if line.startswith("export "):
+            line = line[len("export ") :].lstrip()
+        if "=" not in line:
+            raise EnvFileError(
+                f"{path}:{lineno}: expected KEY=VALUE, got {raw_line!r}"
+            )
+        key, _, value = line.partition("=")
+        key = key.strip()
+        value = value.strip()
+        if not key:
+            raise EnvFileError(f"{path}:{lineno}: empty key in {raw_line!r}")
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+            value = value[1:-1]
+        result[key] = value
+    return result

graphmail/export.py

@@ -0,0 +1,193 @@
+"""Excel export: a rebuildable view over the archive index.
+
+The workbook is generated on demand, never written during fetch, and every
+email row carries a *relative* ``Open`` hyperlink to its ``.eml`` file, so the
+archive folder can be moved or shared and the links keep working.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import re
+from pathlib import Path
+from typing import Literal
+
+import openpyxl
+from openpyxl.styles import Font
+from openpyxl.worksheet.worksheet import Worksheet
+
+from graphmail.index import ArchiveIndex, MessageRecord
+from graphmail.text import clean_sheetname
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_EXPORT_FILE = Path("email_index.xlsx")
+COLUMNS = [
+    "Subject",
+    "From",
+    "To",
+    "CC",
+    "Received Date",
+    "Body Content",
+    "Open",
+]
+_GROUPING = ("folder", "year", "none")
+
+# Excel cells max 32,767 chars; leave a margin.
+_EXCEL_CELL_LIMIT = 32_000
+# Control chars Excel refuses (\n and \t survive; Excel allows them).
+_EXCEL_ILLEGAL = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]")
+
+
+def clean_for_excel(value: str) -> str:
+    """Strip characters Excel rejects and cap the length."""
+    cleaned = _EXCEL_ILLEGAL.sub("", value)
+    if len(cleaned) > _EXCEL_CELL_LIMIT:
+        return cleaned[:_EXCEL_CELL_LIMIT] + "…"
+    return cleaned
+
+
+def relative_link(eml_path: str, out_dir: Path, base_dir: Path) -> str:
+    """Path of *eml_path* relative to *out_dir*, always with forward slashes.
+
+    Excel on Windows accepts ``/``; backslash paths break macOS, LibreOffice,
+    and mixed network shares. Falls back to the absolute path when the
+    archive and workbook are on different drives.
+    """
+    absolute_eml = (base_dir / eml_path).resolve()
+    try:
+        rel = os.path.relpath(absolute_eml, out_dir.resolve())
+    except ValueError:
+        logger.warning(
+            "Archive and workbook are on different drives; "
+            "using absolute link for %s",
+            eml_path,
+        )
+        return absolute_eml.as_posix()
+    # Forward slashes work everywhere; backslashes only work on Windows.
+    return rel.replace(os.sep, "/")
+
+
+def _sheet_title(name: str) -> str:
+    # Excel sheet names: max 31 chars, illegal chars already sanitized.
+    return clean_sheetname(name)[:31]
+
+
+def _unique_title(base: str, used: set[str]) -> str:
+    """Deduplicate *base* against *used* (Excel sheet names are unique,
+    case-insensitively, and max 31 chars)."""
+    candidate = base
+    counter = 2
+    while candidate.casefold() in used:
+        suffix = f" ({counter})"
+        candidate = f"{base[: 31 - len(suffix)]}{suffix}"
+        counter += 1
+    used.add(candidate.casefold())
+    return candidate
+
+
+def _write_row(
+    sheet: Worksheet, record: MessageRecord, link_target: str
+) -> None:
+    sheet.append(
+        [
+            clean_for_excel(record.subject),
+            clean_for_excel(record.sender),
+            clean_for_excel(record.recipients_to),
+            clean_for_excel(record.recipients_cc),
+            record.received_at[:10],
+            clean_for_excel(record.body_text),
+            "Open",
+        ]
+    )
+    cell = sheet.cell(row=sheet.max_row, column=len(COLUMNS))
+    cell.hyperlink = link_target
+    cell.style = "Hyperlink"
+    # Some viewers ignore the named style; pin the font explicitly.
+    # NOT =HYPERLINK(): a formula cell breaks simple parsers.
+    cell.font = Font(color="0563C1", underline="single")
+
+
+def _new_workbook() -> openpyxl.Workbook:
+    workbook = openpyxl.Workbook()
+    # Drop the empty default sheet.
+    default_sheet = workbook.active
+    if default_sheet is not None:
+        workbook.remove(default_sheet)
+    return workbook
+
+
+def export_to_excel(
+    index: ArchiveIndex,
+    out_path: Path = DEFAULT_EXPORT_FILE,
+    *,
+    group_by: Literal["folder", "year", "none"] = "folder",
+    base_dir: Path | None = None,
+) -> Path:
+    """Build an Excel index from *index* and return the written path.
+
+    ``group_by``:
+        * ``"folder"``: one sheet per mail folder (default)
+        * ``"year"``  : one sheet per year of the received date
+        * ``"none"``  : a single "Emails" sheet
+
+    ``base_dir`` is the directory ``eml_path`` values are relative to: the
+    working directory (``index.db`` is cwd-relative too, so the two move
+    together). Defaults to the current working directory.
+    """
+    if group_by not in _GROUPING:
+        raise ValueError(f"group_by must be one of {_GROUPING}, got {group_by!r}")
+
+    root = base_dir or Path.cwd()
+    out_path = Path(out_path)
+    records = index.messages()
+
+    groups: dict[str, list[MessageRecord]] = {}
+    for record in records:
+        if group_by == "folder":
+            key = record.folder_name or "Unknown Folder"
+        elif group_by == "year":
+            key = record.received_at[:4] if record.received_at else "Unknown"
+        else:
+            key = "Emails"
+        groups.setdefault(key, []).append(record)
+
+    workbook = _new_workbook()
+    out_dir = out_path.parent
+
+    titles = sorted(groups) or ["Emails"]
+    used_titles: set[str] = set()
+    missing_files = 0
+
+    for title in titles:
+        sheet = workbook.create_sheet(
+            title=_unique_title(_sheet_title(title), used_titles)
+        )
+        sheet.append(COLUMNS)
+        for record in groups.get(title, []):
+            target = relative_link(record.eml_path, out_dir, root)
+            if not (root / record.eml_path).exists():
+                missing_files += 1
+                if missing_files <= 5:
+                    logger.warning(
+                        "Missing .eml file (link will not open): %s",
+                        record.eml_path,
+                    )
+                else:
+                    logger.debug(
+                        "Missing .eml file (link will not open): %s",
+                        record.eml_path,
+                    )
+            _write_row(sheet, record, target)
+
+    if missing_files:
+        logger.warning(
+            "%d .eml file(s) missing from the archive; their hyperlinks "
+            "will not open",
+            missing_files,
+        )
+
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    workbook.save(out_path)
+    return out_path