gmail-streamer 0.2.2__py3-none-any.whl

gmail_streamer/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"

gmail_streamer/_build_info.py

@@ -0,0 +1 @@
+GIT_HASH = "unknown"

gmail_streamer/auth.py

@@ -0,0 +1,33 @@
+from pathlib import Path
+
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+
+SCOPES = ["https://www.googleapis.com/auth/gmail.readonly"]
+
+
+def get_gmail_service(profile_dir: Path):
+    creds_path = profile_dir / "credentials.json"
+    token_path = profile_dir / "token.json"
+
+    creds = None
+    if token_path.exists():
+        creds = Credentials.from_authorized_user_file(str(token_path), SCOPES)
+
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            if not creds_path.exists():
+                raise FileNotFoundError(
+                    f"OAuth credentials not found: {creds_path}\n"
+                    "Download from Google Cloud Console and place in profile directory."
+                )
+            flow = InstalledAppFlow.from_client_secrets_file(str(creds_path), SCOPES)
+            creds = flow.run_local_server(port=0)
+        with open(token_path, "w") as f:
+            f.write(creds.to_json())
+
+    return build("gmail", "v1", credentials=creds)

gmail_streamer/cli.py

@@ -0,0 +1,145 @@
+from pathlib import Path
+
+import click
+import yaml
+
+from gmail_streamer.auth import get_gmail_service
+from gmail_streamer.config import load_config
+from gmail_streamer.gmail_client import (
+    fetch_attachments,
+    fetch_message_metadata,
+    fetch_raw_message,
+    search_messages,
+)
+from gmail_streamer.paths import get_profiles_dir, list_profiles, resolve_profile
+from gmail_streamer.storage import save_attachments, save_eml, save_metadata, scan_downloaded_metadata
+
+
+@click.group()
+@click.option(
+    "--profile-dir",
+    envvar="GMAIL_STREAMER_PROFILE_DIR",
+    default=None,
+    type=click.Path(file_okay=False),
+    help="Override profiles directory.",
+)
+@click.pass_context
+def main(ctx, profile_dir):
+    """Download Gmail messages matching configurable filters."""
+    ctx.ensure_object(dict)
+    ctx.obj["profiles_dir"] = get_profiles_dir(profile_dir)
+
+
+@main.command()
+@click.argument("profile")
+@click.pass_context
+def run(ctx, profile):
+    """Download messages for a profile."""
+    profiles_dir = ctx.obj["profiles_dir"]
+    profile_path = resolve_profile(profile, profiles_dir)
+
+    if not profile_path.is_dir():
+        raise click.ClickException(f"Profile directory not found: {profile_path}")
+
+    config = load_config(profile_path)
+    target = Path(config.target_directory)
+    target.mkdir(parents=True, exist_ok=True)
+
+    click.echo(f"Authenticating profile '{profile_path.name}'...")
+    service = get_gmail_service(profile_path)
+
+    downloaded_ids, most_recent_date = scan_downloaded_metadata(target)
+    if most_recent_date:
+        click.echo(f"Resuming from {most_recent_date} ({len(downloaded_ids)} already downloaded)")
+
+    click.echo(f"Searching: {config.filter}")
+    msg_ids = search_messages(service, config.filter, after_date=most_recent_date)
+    new_ids = [mid for mid in msg_ids if mid not in downloaded_ids]
+    click.echo(f"Found {len(msg_ids)} messages, {len(new_ids)} new.")
+
+    for i, msg_id in enumerate(new_ids, 1):
+        click.echo(f"[{i}/{len(new_ids)}] Downloading {msg_id}...")
+
+        metadata = fetch_message_metadata(service, msg_id)
+        date = metadata["date"]
+
+        if config.mode == "full":
+            raw = fetch_raw_message(service, msg_id)
+            save_eml(target, msg_id, date, raw)
+            attachments = fetch_attachments(service, msg_id)
+            if attachments:
+                save_attachments(target, msg_id, date, attachments)
+
+        elif config.mode == "attachments_only":
+            attachments = fetch_attachments(service, msg_id)
+            if attachments:
+                save_attachments(target, msg_id, date, attachments)
+            else:
+                click.echo(f"  No attachments for {msg_id}")
+
+        save_metadata(target, msg_id, date, metadata)
+
+    click.echo("Done.")
+
+
+@main.group("profiles")
+def profiles_group():
+    """Manage profiles."""
+
+
+@profiles_group.command("list")
+@click.pass_context
+def profiles_list(ctx):
+    """List available profiles."""
+    profiles_dir = ctx.obj["profiles_dir"]
+    names = list_profiles(profiles_dir)
+    if not names:
+        click.echo(f"No profiles found in {profiles_dir}")
+        return
+    click.echo(f"Profiles in {profiles_dir}:\n")
+    for name in names:
+        click.echo(f"  {name}")
+
+
+@profiles_group.command("init")
+@click.argument("name")
+@click.pass_context
+def profiles_init(ctx, name):
+    """Scaffold a new profile directory with a template config."""
+    profiles_dir = ctx.obj["profiles_dir"]
+    profile_dir = profiles_dir / name
+
+    if profile_dir.exists():
+        raise click.ClickException(f"Profile already exists: {profile_dir}")
+
+    profile_dir.mkdir(parents=True)
+    config = {
+        "filter": 'from:example@gmail.com has:attachment',
+        "target_directory": "./downloads",
+        "mode": "full",
+    }
+    (profile_dir / "config.yaml").write_text(yaml.dump(config, default_flow_style=False))
+    click.echo(f"Created profile at {profile_dir}")
+    click.echo("Next steps:")
+    click.echo(f"  1. Edit {profile_dir / 'config.yaml'} with your filter")
+    click.echo(f"  2. Copy your credentials.json into {profile_dir}")
+    click.echo(f"  3. Run: gmail-streamer run {name}")
+
+
+@profiles_group.command("show")
+@click.argument("name")
+@click.pass_context
+def profiles_show(ctx, name):
+    """Show a profile's configuration."""
+    profiles_dir = ctx.obj["profiles_dir"]
+    profile_path = resolve_profile(name, profiles_dir)
+
+    if not profile_path.is_dir():
+        raise click.ClickException(f"Profile not found: {profile_path}")
+
+    config_file = profile_path / "config.yaml"
+    if not config_file.exists():
+        raise click.ClickException(f"No config.yaml in {profile_path}")
+
+    click.echo(f"Profile: {profile_path.name} ({profile_path})\n")
+    click.echo(config_file.read_text())

gmail_streamer/config.py

@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+
+@dataclass
+class ProfileConfig:
+    filter: str
+    target_directory: str
+    mode: str = "full"  # "full" or "attachments_only"
+
+    def __post_init__(self):
+        if self.mode not in ("full", "attachments_only"):
+            raise ValueError(f"Invalid mode: {self.mode!r}. Must be 'full' or 'attachments_only'.")
+
+
+def load_config(profile_dir: Path) -> ProfileConfig:
+    config_path = profile_dir / "config.yaml"
+    if not config_path.exists():
+        raise FileNotFoundError(f"Config not found: {config_path}")
+    with open(config_path) as f:
+        data = yaml.safe_load(f)
+    config = ProfileConfig(**data)
+    target = Path(config.target_directory)
+    if not target.is_absolute():
+        config.target_directory = str((profile_dir / target).resolve())
+    return config

gmail_streamer/gmail_client.py

@@ -0,0 +1,68 @@
+import base64
+from datetime import datetime, timezone
+
+
+def search_messages(service, query: str, after_date: str | None = None) -> list[str]:
+    """Return all message IDs matching the query.
+
+    If after_date (YYYY-MM-DD) is provided, appends 'after:{date}' to narrow results.
+    """
+    if after_date:
+        query = f"{query} after:{after_date}"
+    ids = []
+    request = service.users().messages().list(userId="me", q=query)
+    while request:
+        response = request.execute()
+        for msg in response.get("messages", []):
+            ids.append(msg["id"])
+        request = service.users().messages().list_next(request, response)
+    return ids
+
+
+def fetch_raw_message(service, msg_id: str) -> bytes:
+    """Fetch the full RFC 2822 message as bytes."""
+    msg = service.users().messages().get(userId="me", id=msg_id, format="raw").execute()
+    return base64.urlsafe_b64decode(msg["raw"])
+
+
+def fetch_message_metadata(service, msg_id: str) -> dict:
+    """Fetch message metadata and return a dict with key fields."""
+    msg = service.users().messages().get(
+        userId="me", id=msg_id, format="metadata",
+        metadataHeaders=["From", "To", "Subject", "Date"],
+    ).execute()
+
+    headers = {h["name"]: h["value"] for h in msg.get("payload", {}).get("headers", [])}
+    internal_ts = int(msg.get("internalDate", "0")) / 1000
+    internal_date = datetime.fromtimestamp(internal_ts, tz=timezone.utc).strftime("%Y-%m-%d")
+
+    return {
+        "id": msg_id,
+        "date": internal_date,
+        "subject": headers.get("Subject", ""),
+        "from": headers.get("From", ""),
+        "to": headers.get("To", ""),
+        "snippet": msg.get("snippet", ""),
+        "label_ids": msg.get("labelIds", []),
+    }
+
+
+def fetch_attachments(service, msg_id: str) -> list[dict]:
+    """Return list of {filename, data} for each attachment."""
+    msg = service.users().messages().get(userId="me", id=msg_id).execute()
+    attachments = []
+    for part in msg.get("payload", {}).get("parts", []):
+        filename = part.get("filename")
+        body = part.get("body", {})
+        attachment_id = body.get("attachmentId")
+        if filename and attachment_id:
+            att = (
+                service.users()
+                .messages()
+                .attachments()
+                .get(userId="me", messageId=msg_id, id=attachment_id)
+                .execute()
+            )
+            data = base64.urlsafe_b64decode(att["data"])
+            attachments.append({"filename": filename, "data": data})
+    return attachments

gmail_streamer/paths.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+DEFAULT_BASE_DIR = Path.home() / ".gmail-streamer"
+DEFAULT_PROFILES_DIR = DEFAULT_BASE_DIR / "profiles"
+
+
+def get_profiles_dir(override: str | None = None) -> Path:
+    """Return the active profiles directory.
+
+    Priority:
+    1. Explicit override (--profile-dir flag)
+    2. ./profiles/ in CWD (if it exists)
+    3. ~/.gmail-streamer/profiles/ (fallback)
+    """
+    if override:
+        return Path(override).resolve()
+    cwd_profiles = Path.cwd() / "profiles"
+    if cwd_profiles.is_dir():
+        return cwd_profiles.resolve()
+    return DEFAULT_PROFILES_DIR
+
+
+def resolve_profile(name: str, profiles_dir: Path) -> Path:
+    """Resolve a profile name or path to a directory.
+
+    If `name` is an existing directory, use it directly (backward compat).
+    Otherwise look it up as {profiles_dir}/{name}/.
+    """
+    candidate = Path(name)
+    if candidate.is_dir():
+        return candidate.resolve()
+    return profiles_dir / name
+
+
+def list_profiles(profiles_dir: Path) -> list[str]:
+    """Return profile names (subdirectories containing config.yaml)."""
+    if not profiles_dir.is_dir():
+        return []
+    return sorted(
+        d.name for d in profiles_dir.iterdir()
+        if d.is_dir() and (d / "config.yaml").exists()
+    )

gmail_streamer/storage.py

@@ -0,0 +1,50 @@
+import json
+from pathlib import Path
+
+
+def _short_id(msg_id: str) -> str:
+    return msg_id[:8]
+
+
+def save_eml(target_dir: Path, msg_id: str, date: str, raw: bytes):
+    """Save .eml file with date and short ID."""
+    target_dir.mkdir(parents=True, exist_ok=True)
+    (target_dir / f"{date} - {_short_id(msg_id)}.eml").write_bytes(raw)
+
+
+def save_metadata(target_dir: Path, msg_id: str, date: str, metadata: dict):
+    """Save metadata JSON with date and short ID."""
+    target_dir.mkdir(parents=True, exist_ok=True)
+    path = target_dir / f"{date} - {_short_id(msg_id)}.json"
+    path.write_text(json.dumps(metadata, indent=2, ensure_ascii=False))
+
+
+def save_attachments(target_dir: Path, msg_id: str, date: str, attachments: list[dict]):
+    """Save attachments flat in target_dir with date and short ID."""
+    target_dir.mkdir(parents=True, exist_ok=True)
+    sid = _short_id(msg_id)
+    for att in attachments:
+        (target_dir / f"{date} - {sid} - {att['filename']}").write_bytes(att["data"])
+
+
+def scan_downloaded_metadata(target_dir: Path) -> tuple[set[str], str | None]:
+    """Scan metadata JSON files to derive downloaded IDs and most recent date.
+
+    Returns (downloaded_ids, most_recent_date_or_none).
+    """
+    downloaded_ids: set[str] = set()
+    most_recent_date: str | None = None
+
+    for meta_path in target_dir.glob("* - *.json"):
+        try:
+            meta = json.loads(meta_path.read_text())
+        except (json.JSONDecodeError, OSError):
+            continue
+        msg_id = meta.get("id")
+        date = meta.get("date")
+        if msg_id:
+            downloaded_ids.add(msg_id)
+        if date and (most_recent_date is None or date > most_recent_date):
+            most_recent_date = date
+
+    return downloaded_ids, most_recent_date

gmail_streamer-0.2.2.dist-info/METADATA

@@ -0,0 +1,132 @@
+Metadata-Version: 2.4
+Name: gmail-streamer
+Version: 0.2.2
+Summary: CLI tool to download Gmail messages matching configurable filters via OAuth2
+Project-URL: Homepage, https://github.com/tsilva/gmail-streamer
+Project-URL: Repository, https://github.com/tsilva/gmail-streamer
+Project-URL: Issues, https://github.com/tsilva/gmail-streamer/issues
+Author: Tiago Silva
+License: MIT
+License-File: LICENSE
+Keywords: attachments,backup,cli,download,email,gmail,google,oauth2
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Communications :: Email
+Requires-Python: >=3.12
+Requires-Dist: click
+Requires-Dist: google-api-python-client
+Requires-Dist: google-auth-httplib2
+Requires-Dist: google-auth-oauthlib
+Requires-Dist: pyyaml
+Description-Content-Type: text/markdown
+
+<div align="center">
+  <img src="logo.png" alt="gmail-streamer" width="512"/>
+
+  [![Python](https://img.shields.io/badge/Python-3.12+-blue.svg)](https://python.org)
+  [![License](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+
+  **📧 Download Gmail messages matching your filters to local files 📥**
+
+</div>
+
+## ✨ Features
+
+- **🗂️ Profile-based configuration** — run multiple independent download profiles, each with its own filters, credentials, and output directory
+- **🔐 OAuth2 authentication** — secure Google sign-in with automatic token caching
+- **📧 Full message download** — save complete `.eml` files for archival
+- **📎 Attachments-only mode** — grab just the attachments, skip the rest
+- **🧠 Incremental downloads** — remembers what's already been downloaded, no duplicates across runs
+- **🔍 Gmail search filters** — use any Gmail search query (`from:`, `has:attachment`, `after:`, label filters, etc.)
+- **🏠 Works from anywhere** — install globally with `uv` and run from any directory
+
+## 🚀 Quick Start
+
+### 1. Install
+
+```bash
+git clone https://github.com/tsilva/gmail-streamer.git
+cd gmail-streamer
+uv tool install . --force --no-cache
+```
+
+### 2. Create a profile
+
+```bash
+gmail-streamer profiles init my-profile
+```
+
+This creates `~/.gmail-streamer/profiles/my-profile/` with a template `config.yaml`.
+
+### 3. Add your Gmail OAuth credentials
+
+Place your `credentials.json` (from [Google Cloud Console](https://console.cloud.google.com/apis/credentials)) into the profile directory.
+
+### 4. Configure the profile
+
+Edit `~/.gmail-streamer/profiles/my-profile/config.yaml`:
+
+```yaml
+filter: "from:example@gmail.com has:attachment"
+target_directory: "./downloads"
+mode: "full"  # or "attachments_only"
+```
+
+### 5. Run
+
+```bash
+gmail-streamer run my-profile
+```
+
+On first run, a browser window opens for OAuth authorization. Subsequent runs reuse the cached token.
+
+## 📁 Profile Resolution
+
+The profiles directory is resolved in this order:
+
+1. `--profile-dir` flag or `GMAIL_STREAMER_PROFILE_DIR` env var
+2. `./profiles/` in the current working directory (if it exists)
+3. `~/.gmail-streamer/profiles/` (default)
+
+The `profile` argument can be a **name** (looked up in the profiles directory) or a **path** to an existing directory (backward compatible).
+
+## 🛠️ CLI Reference
+
+```bash
+gmail-streamer run <profile>                     # Download messages
+gmail-streamer --profile-dir /path run <profile>  # Custom profiles directory
+gmail-streamer profiles list                      # List available profiles
+gmail-streamer profiles init <name>               # Scaffold a new profile
+gmail-streamer profiles show <name>               # Show profile config
+```
+
+## ⚙️ Profile Structure
+
+Each profile lives in its own directory with:
+
+| File | Purpose |
+|------|---------|
+| `config.yaml` | Filter query, target directory, download mode |
+| `credentials.json` | OAuth client credentials (you provide this) |
+| `token.json` | Auto-generated after first OAuth flow |
+
+## 🏗️ Architecture
+
+| Module | Responsibility |
+|--------|---------------|
+| `cli.py` | Click CLI entry point (group with `run` and `profiles` subcommands) |
+| `paths.py` | Profile directory resolution and discovery |
+| `config.py` | Loads and validates `config.yaml` into a `ProfileConfig` dataclass |
+| `auth.py` | OAuth2 flow with token caching |
+| `gmail_client.py` | Gmail API wrapper: search, fetch messages, fetch attachments |
+| `storage.py` | Saves `.eml` files and attachments to disk |
+
+## 📄 License
+
+[MIT](LICENSE)

gmail_streamer-0.2.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+gmail_streamer/__init__.py,sha256=HfjVOrpTnmZ-xVFCYSVmX50EXaBQeJteUHG-PD6iQs8,22
+gmail_streamer/auth.py,sha256=Mt3UJA4NbCrIgiKbbgODHh9HmuxVhsZPkU7tBzfr9OY,1220
+gmail_streamer/cli.py,sha256=m-0OgJja9Fjj820yZJH80CS04bPsQ8L5lf4ExpnBuD4,4791
+gmail_streamer/config.py,sha256=GRgsuRMQ7uNM6H-iwAbtoDE2aHIV9UY4HvJKYqMuF_w,860
+gmail_streamer/gmail_client.py,sha256=hc0gahvw2rP3qpRlvSX3AB-2Y_2kEX_tQza9jw0j4qU,2544
+gmail_streamer/paths.py,sha256=TYErMHc1Nh5EUImzEh2k1tw7omAnldvmMEnW71t0T3Q,1269
+gmail_streamer/storage.py,sha256=uzUDIXBgQmXd111c-zE7usxZ7Vqfe0KFlpLcDaXMqBI,1773
+gmail_streamer/_build_info.py,sha256=u78KhLyvgyWjUoCs30k7OxpPsuL-xdvmXoUoRn3hk4Q,21
+gmail_streamer-0.2.2.dist-info/METADATA,sha256=3i-EMPwL1_5PYpC8RH74G3AZ3W1vuHoGfA19Gk_UhAg,4689
+gmail_streamer-0.2.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+gmail_streamer-0.2.2.dist-info/entry_points.txt,sha256=hfb9lqqUjanoVn2vwdhNefsoZn8c5tYMdNT6Jt4fQFo,59
+gmail_streamer-0.2.2.dist-info/licenses/LICENSE,sha256=gTrdDdqFDu7VtWezebCYIHmebmc-XTVxqWTZGKvDux0,1068
+gmail_streamer-0.2.2.dist-info/RECORD,,

gmail_streamer-0.2.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

gmail_streamer-0.2.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+gmail-streamer = gmail_streamer.cli:main

gmail_streamer-0.2.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Tiago Silva
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.