git-ew 0.1.0__py3-none-any.whl

git_ew/_internal/mailing_lists/zsh_workers/sync_archives.py

@@ -0,0 +1,300 @@
+# SPDX-License-Identifier: ISC
+
+# Copyright (c) 2021, Timothée Mazzucotelli and contributors
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# Zsh-workers mailing list utilities.
+
+import argparse
+import logging
+import sys
+from datetime import date
+from html.parser import HTMLParser
+from pathlib import Path
+from time import strptime
+from urllib.error import URLError
+from urllib.request import urlopen, urlretrieve
+
+BASE_URL = "https://www.zsh.org/mla/zsh-workers/"
+"""Base URL for zsh-workers mailing list archives."""
+_logger = logging.getLogger(__name__)
+
+
+class LinkExtractor(HTMLParser):
+    """Extract .tgz archive links and dates from HTML."""
+
+    def __init__(self):
+        super().__init__()
+        self.archives: dict[str, date | None] = {}
+        """Archive filenames and their dates found in the page."""
+        self._in_pre = False
+        self._current_line = ""
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        """Track pre tags and extract href attributes."""
+        if tag == "pre":
+            self._in_pre = True
+        elif tag == "a" and self._in_pre:
+            for attr, value in attrs:
+                if attr == "href" and value and value.endswith(".tgz"):
+                    filename = value.split("/")[-1]
+                    # Date will be extracted from text content after the link
+                    self.archives[filename] = None
+
+    def handle_endtag(self, tag: str) -> None:
+        """Track end of pre tag."""
+        if tag == "pre":
+            self._in_pre = False
+
+    def handle_data(self, data: str) -> None:
+        """Extract date information from pre-formatted text."""
+        if not self._in_pre:
+            return
+
+        self._current_line += data
+
+        # Look for date patterns in the line (DD-MMM-YYYY format)
+        # Example: "12-Jun-1995"
+        if len(self._current_line) > 50:  # Approximate line length  # noqa: PLR2004
+            parts = self._current_line.split()
+            for i, part in enumerate(parts):
+                # Try to parse date from parts
+                if "-" in part and len(parts) > i + 1:
+                    try:
+                        # Try common date formats
+                        date_str = part
+                        # Convert French month names to English
+                        date_str = date_str.replace("janv.", "Jan")
+                        date_str = date_str.replace("févr.", "Feb")
+                        date_str = date_str.replace("mars", "Mar")
+                        date_str = date_str.replace("avril", "Apr")
+                        date_str = date_str.replace("mai", "May")
+                        date_str = date_str.replace("juin", "Jun")
+                        date_str = date_str.replace("juil.", "Jul")
+                        date_str = date_str.replace("août", "Aug")
+                        date_str = date_str.replace("sept.", "Sep")
+                        date_str = date_str.replace("oct.", "Oct")
+                        date_str = date_str.replace("nov.", "Nov")
+                        date_str = date_str.replace("déc.", "Dec")
+
+                        parsed_time = strptime(date_str, "%d-%b-%Y")
+                        parsed = date(parsed_time.tm_year, parsed_time.tm_mon, parsed_time.tm_mday)
+                        # Associate this date with the last archived filename found
+                        if self.archives:
+                            last_filename = list(self.archives.keys())[-1]
+                            if self.archives[last_filename] is None:
+                                self.archives[last_filename] = parsed
+                    except (ValueError, IndexError):
+                        _logger.debug("Could not parse archive date from %r", part, exc_info=True)
+
+            self._current_line = ""
+
+
+def fetch_archive_list() -> dict[str, date | None]:
+    """Fetch the list of available archives from zsh.org with dates.
+
+    Returns:
+        Dict mapping archive filenames to their dates (or None if date couldn't be parsed)
+
+    Raises:
+        URLError: If the page cannot be fetched.
+    """
+    try:
+        with urlopen(BASE_URL) as response:
+            html = response.read().decode("utf-8")
+    except URLError as e:
+        raise URLError(f"Failed to fetch {BASE_URL}: {e}") from e
+
+    parser = LinkExtractor()
+    parser.feed(html)
+    # Return sorted by filename
+    return dict(sorted(parser.archives.items()))
+
+
+def _get_matching_archives(
+    available_archives: dict[str, date | None],
+    since: date | None = None,
+    until: date | None = None,
+) -> list[str]:
+    """Return archive filenames within the requested date range.
+
+    Args:
+        available_archives: Dict of available archive filenames to dates.
+        since: Only include archives from this date onwards.
+        until: Only include archives up to this date.
+
+    Returns:
+        Archive filenames within the requested date range.
+    """
+    matching = []
+
+    for filename, file_date in available_archives.items():
+        if since is not None and file_date is not None and file_date < since:
+            continue
+        if until is not None and file_date is not None and file_date > until:
+            continue
+
+        matching.append(filename)
+
+    return matching
+
+
+def get_missing_archives(
+    archive_dir: Path,
+    available_archives: dict[str, date | None],
+    since: date | None = None,
+    until: date | None = None,
+) -> list[str]:
+    """Determine which matching archives need to be downloaded.
+
+    Args:
+        archive_dir: Directory where archives are stored.
+        available_archives: Dict of available archive filenames to dates.
+        since: Only include archives from this date onwards.
+        until: Only include archives up to this date.
+
+    Returns:
+        Matching archive filenames that do not exist locally.
+    """
+    existing = {file.name for file in archive_dir.glob("*.tgz")}
+
+    return [
+        filename for filename in _get_matching_archives(available_archives, since, until) if filename not in existing
+    ]
+
+
+def download_archive(filename: str, archive_dir: Path) -> bool:
+    """Download a single archive.
+
+    Args:
+        filename: Archive filename to download.
+        archive_dir: Directory to save the archive to.
+
+    Returns:
+        True if download succeeded, False otherwise.
+    """
+    url = BASE_URL + filename
+    output_path = archive_dir / filename
+
+    try:
+        _logger.info("Downloading %s", filename)
+        urlretrieve(url, output_path)  # noqa: S310
+        _logger.info("Downloaded %s", filename)
+    except URLError as error:
+        _logger.error("Failed to download %s: %s", filename, error)  # noqa: TRY400
+        # Clean up partially downloaded file
+        if output_path.exists():
+            output_path.unlink()
+        return False
+    else:
+        return True
+
+
+def _main() -> int:
+    """Main entry point."""
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    parser = argparse.ArgumentParser(
+        description="Sync zsh-workers mailing list archives from zsh.org",
+    )
+    parser.add_argument(
+        "-d",
+        "--directory",
+        type=Path,
+        default=Path(".archives"),
+        help="Directory to store archives (default: .archives)",
+    )
+    parser.add_argument(
+        "-s",
+        "--since",
+        type=str,
+        help="Only download archives from this date onwards. "
+        "Format: YYYY (year) or YYYY-MM-DD (specific date). "
+        "Example: --since 2020 or --since 2020-01-15",
+    )
+    parser.add_argument(
+        "-n",
+        "--dry-run",
+        action="store_true",
+        help="Show what would be downloaded without actually downloading",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="Show more detailed output",
+    )
+
+    args = parser.parse_args()
+    archive_dir = args.directory
+
+    # Parse --since argument
+    since_date: date | None = None
+    if args.since:
+        try:
+            since_date = (
+                date(int(args.since), 1, 1)
+                if len(args.since) == 4  # noqa: PLR2004
+                else date.fromisoformat(args.since)
+            )
+        except ValueError:
+            _logger.error("Invalid date format %r. Use YYYY or YYYY-MM-DD", args.since)  # noqa: TRY400
+            return 1
+
+    # Create directory if it doesn't exist
+    archive_dir.mkdir(parents=True, exist_ok=True)
+
+    # Fetch available archives
+    _logger.info("Fetching archive list from zsh.org")
+    try:
+        available = fetch_archive_list()
+    except URLError as error:
+        _logger.error("Failed to fetch archive list: %s", error)  # noqa: TRY400
+        return 1
+
+    _logger.info("Found %d archives available", len(available))
+
+    # Determine missing archives
+    missing = get_missing_archives(archive_dir, available, since_date)
+
+    if not missing:
+        _logger.info("All requested archives are already downloaded")
+        return 0
+
+    _logger.info("Found %d new archives to download", len(missing))
+
+    if args.verbose:
+        _logger.info("Missing archives:")
+        for name in missing:
+            _logger.info("  - %s", name)
+
+    if args.dry_run:
+        _logger.info("Dry-run: no archives downloaded")
+        return 0
+
+    success_count = 0
+    for filename in missing:
+        if download_archive(filename, archive_dir):
+            success_count += 1
+
+    _logger.info("Downloaded %d/%d archives", success_count, len(missing))
+
+    if success_count < len(missing):
+        _logger.warning("%d archive downloads failed", len(missing) - success_count)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(_main())

git_ew/_internal/models.py

@@ -0,0 +1,149 @@
+# SPDX-License-Identifier: ISC
+
+# Copyright (c) 2021, Timothée Mazzucotelli and contributors
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+from datetime import datetime  # noqa: TC003
+from typing import TYPE_CHECKING
+
+from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text, create_engine
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship, sessionmaker
+
+if TYPE_CHECKING:
+    from sqlalchemy.engine import Engine
+
+
+class Base(DeclarativeBase):
+    """Base class for all models."""
+
+
+class Thread(Base):
+    """Represents an email thread (like a GitHub issue or PR)."""
+
+    __tablename__ = "threads"
+    """Database table name."""
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    """Thread unique identifier."""
+    subject: Mapped[str] = mapped_column(String(500), nullable=False)
+    """Thread subject line."""
+    first_message_id: Mapped[str] = mapped_column(String(500), unique=True, nullable=False)
+    """ID of the first message in the thread."""
+    created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
+    """When the thread was created."""
+    updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False)
+    """When the thread was last updated."""
+    is_patch: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    """Whether this thread contains patches."""
+    status: Mapped[str] = mapped_column(String(50), default="open", nullable=False)  # open, closed
+    """Thread status (open or closed)."""
+
+    # Relationships
+    messages: Mapped[list[Message]] = relationship("Message", back_populates="thread", cascade="all, delete-orphan")
+    """Messages in this thread."""
+
+
+class Message(Base):
+    """Represents an individual email message in a thread."""
+
+    __tablename__ = "messages"
+    """Database table name."""
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    """Message unique identifier."""
+    message_id: Mapped[str] = mapped_column(String(500), unique=True, nullable=False, index=True)
+    """Email message ID header."""
+    in_reply_to: Mapped[str | None] = mapped_column(String(500), nullable=True, index=True)
+    """Message ID this reply refers to."""
+    thread_id: Mapped[int] = mapped_column(Integer, ForeignKey("threads.id"), nullable=False, index=True)
+    """Thread this message belongs to."""
+
+    # Email metadata
+    from_email: Mapped[str] = mapped_column(String(255), nullable=False)
+    """Sender email address."""
+    from_name: Mapped[str] = mapped_column(String(255), nullable=False)
+    """Sender name."""
+    subject: Mapped[str] = mapped_column(String(500), nullable=False)
+    """Email subject."""
+    date: Mapped[datetime] = mapped_column(DateTime, nullable=False, index=True)
+    """Email date."""
+
+    # Content
+    body: Mapped[str] = mapped_column(Text, nullable=False)
+    """Message body text."""
+    raw_email: Mapped[str] = mapped_column(Text, nullable=True)
+    """Raw email content."""
+
+    # Patch information
+    is_patch: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    """Whether this message contains a patch."""
+    patch_content: Mapped[str | None] = mapped_column(Text, nullable=True)
+    """Extracted patch content."""
+
+    # Relationships
+    thread: Mapped[Thread] = relationship("Thread", back_populates="messages")
+    """Thread this message belongs to."""
+
+
+class EmailSource(Base):
+    """Represents an email archive source to fetch from."""
+
+    __tablename__ = "email_sources"
+    """Database table name."""
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    """Source unique identifier."""
+    name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
+    """Source name."""
+    source_type: Mapped[str] = mapped_column(String(50), nullable=False)  # maildir, mbox, imap, archive_url
+    """Type of email source (maildir, mbox, imap, etc)."""
+    config: Mapped[str] = mapped_column(Text, nullable=False)  # JSON config for the source
+    """JSON configuration for the source."""
+    last_synced: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
+    """When this source was last synced."""
+    enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    """Whether this source is enabled."""
+
+
+class Configuration(Base):
+    """Stores application configuration."""
+
+    __tablename__ = "configuration"
+    """Database table name."""
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    """Configuration unique identifier."""
+    key: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
+    """Configuration key."""
+    value: Mapped[str] = mapped_column(Text, nullable=False)
+    """Configuration value."""
+
+
+def get_engine(database_url: str = "sqlite+aiosqlite:///./git_ew.db") -> Engine:
+    """Create database engine."""
+    return create_engine(database_url, echo=False)
+
+
+def init_db(database_url: str = "sqlite:///./git_ew.db") -> Engine:
+    """Initialize the database."""
+    engine = create_engine(database_url, echo=False)
+    Base.metadata.create_all(engine)
+    return engine
+
+
+def get_session_maker(engine: Engine) -> sessionmaker:
+    """Get a session maker."""
+    return sessionmaker(bind=engine)

git_ew/_internal/secrets.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: ISC
+
+# Copyright (c) 2021, Timothée Mazzucotelli and contributors
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# Runtime secret resolution helpers.
+
+from __future__ import annotations
+
+import shlex
+import subprocess
+from typing import Any, Literal, overload
+
+
+@overload
+def resolve_password(config: dict[str, Any], *, required: Literal[True] = True) -> str: ...
+
+
+@overload
+def resolve_password(config: dict[str, Any], *, required: Literal[False]) -> str | None: ...
+
+
+def resolve_password(config: dict[str, Any], *, required: bool = True) -> str | None:
+    """Resolve a configured password.
+
+    Return `None` when no password is configured and `required` is false.
+    """
+    command = config.get("password_command")
+    if command:
+        result = subprocess.run(  # noqa: S603
+            shlex.split(command),
+            check=True,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            timeout=10,
+        )
+        password = result.stdout.strip()
+        if not password:
+            raise RuntimeError("password command returned no password")
+        return password
+
+    password = config.get("password")
+    if password:
+        return password
+    if not required:
+        return None
+    raise RuntimeError("neither password nor password_command is configured")

git_ew/_internal/sync.py

@@ -0,0 +1,156 @@
+# SPDX-License-Identifier: ISC
+
+# Copyright (c) 2021, Timothée Mazzucotelli and contributors
+
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# Email synchronization utilities for git-ew.
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from typing import TypedDict
+
+from git_ew._internal.database import Database
+from git_ew._internal.email_fetcher import get_fetcher
+
+_logger = logging.getLogger(__name__)
+
+
+class _SyncStats(TypedDict):
+    """Store email synchronization counters and errors."""
+
+    total_sources: int
+    processed_sources: int
+    total_messages: int
+    new_messages: int
+    new_threads: int
+    errors: list[str]
+
+
+async def sync_all_sources(db: Database | None = None) -> _SyncStats:
+    """Sync emails from all configured sources.
+
+    Args:
+        db: Database instance. If None, creates a new one.
+
+    Returns:
+        Dictionary with sync statistics.
+    """
+    if db is None:
+        db = Database()
+        await db.init_db()
+
+    sources = await db.get_email_sources()
+    stats: _SyncStats = {
+        "total_sources": len(sources),
+        "processed_sources": 0,
+        "total_messages": 0,
+        "new_messages": 0,
+        "new_threads": 0,
+        "errors": [],
+    }
+
+    for source in sources:
+        if not source.enabled:
+            continue
+
+        try:
+            config = json.loads(source.config)
+            fetcher = get_fetcher(source.source_type, config)
+
+            source_messages = 0
+            async for parsed_email in fetcher.fetch_emails():
+                # Check if message already exists
+                existing = await db.get_message_by_id(parsed_email.message_id)
+                if existing:
+                    if parsed_email.patch_content and not existing.patch_content:
+                        await db.update_message_patch(parsed_email.message_id, parsed_email.patch_content)
+                    continue
+
+                # Find or create thread
+                thread_id_str = parsed_email.get_thread_id()
+                thread = await db.get_thread_by_message_id(thread_id_str)
+
+                if not thread:
+                    # Create new thread
+                    thread = await db.create_thread(
+                        subject=parsed_email.clean_subject,
+                        first_message_id=thread_id_str,
+                        is_patch=parsed_email.is_patch,
+                    )
+                    stats["new_threads"] += 1
+
+                # Create message
+                await db.create_message(
+                    message_id=parsed_email.message_id,
+                    thread_id=thread.id,
+                    from_email=parsed_email.from_email,
+                    from_name=parsed_email.from_name,
+                    subject=parsed_email.subject,
+                    date=parsed_email.date,
+                    body=parsed_email.body,
+                    in_reply_to=parsed_email.in_reply_to,
+                    is_patch=parsed_email.is_patch,
+                    patch_content=parsed_email.patch_content,
+                    raw_email=parsed_email.raw,
+                )
+
+                source_messages += 1
+                stats["new_messages"] += 1
+
+            stats["processed_sources"] += 1
+            stats["total_messages"] += source_messages
+
+            _logger.info(f"✓ Synced {source_messages} messages from '{source.name}'")
+
+        except Exception as e:
+            error_msg = f"Error syncing source '{source.name}': {e}"
+            stats["errors"].append(error_msg)
+            _logger.exception("✗ %s", error_msg)
+
+    return stats
+
+
+async def sync_command() -> int:
+    """Run email sync as a standalone command."""
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+
+    _logger.info("Starting email sync...")
+    _logger.info("-" * 50)
+
+    db = Database()
+    await db.init_db()
+
+    stats = await sync_all_sources(db)
+
+    _logger.info("-" * 50)
+    _logger.info("\nSync Summary:")
+    _logger.info(f"  Sources processed: {stats['processed_sources']}/{stats['total_sources']}")
+    _logger.info(f"  New threads: {stats['new_threads']}")
+    _logger.info(f"  New messages: {stats['new_messages']}")
+
+    if stats["errors"]:
+        _logger.info(f"\n  Errors: {len(stats['errors'])}")
+        for error in stats["errors"]:
+            _logger.info(f"    - {error}")
+
+    return 0 if not stats["errors"] else 1
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(asyncio.run(sync_command()))