gdown2 0.2.0__py3-none-any.whl

gdown2/__init__.py

@@ -0,0 +1,27 @@
+"""gdown2 — Fast, resumable Google Drive file and folder downloader."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from gdown2.domain.models import DownloadTask, DriveItem, FolderNode, TaskState
+from gdown2.sdk import download, download_sync, get_info, get_tree, resume_directory
+
+try:
+    __version__ = version("gdown2")
+except PackageNotFoundError:  # pragma: no cover - source checkout without installation
+    __version__ = "0.2.0"
+
+__all__ = [
+    "DownloadTask",
+    "DriveItem",
+    "FolderNode",
+    "TaskState",
+    "__version__",
+    "download",
+    "download_sync",
+    "get_info",
+    "get_tree",
+    "resume_directory",
+]
+

gdown2/__main__.py

@@ -0,0 +1,9 @@
+"""Allow running ``python -m gdown2``."""
+
+from __future__ import annotations
+
+from gdown2.interfaces.cli.app import main
+
+if __name__ == "__main__":
+    main()
+

gdown2/application/__init__.py

@@ -0,0 +1,31 @@
+"""Application orchestration services for gdown2."""
+
+from gdown2.application.crawler import FolderCrawler
+from gdown2.application.exporter import ExportService
+from gdown2.application.planner import DownloadPlan, Planner
+from gdown2.application.ports import DriveSource, MediaStream, MetadataCache, ResumeRepository
+from gdown2.application.reporter import EventType, Observer, Reporter, TransferEvent
+from gdown2.application.resolver import Resolver
+from gdown2.application.scheduler import Scheduler
+from gdown2.application.verifier import Verifier
+from gdown2.application.worker import DownloadWorker
+
+__all__ = [
+    "DownloadPlan",
+    "DownloadWorker",
+    "DriveSource",
+    "EventType",
+    "ExportService",
+    "FolderCrawler",
+    "MediaStream",
+    "MetadataCache",
+    "Observer",
+    "Planner",
+    "Reporter",
+    "Resolver",
+    "ResumeRepository",
+    "Scheduler",
+    "TransferEvent",
+    "Verifier",
+]
+

gdown2/application/crawler.py

@@ -0,0 +1,128 @@
+"""Recursive folder crawler with shortcut resolution and cycle prevention."""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import PurePosixPath
+
+from gdown2.application.ports import DriveSource, MetadataCache
+from gdown2.domain.errors import PermissionDeniedError, ResourceNotFoundError
+from gdown2.domain.models import DriveItem, FileEntry, FolderNode
+from gdown2.domain.path_policy import PathPolicy
+from gdown2.domain.url_parser import ParsedTarget, TargetKind
+
+
+class FolderCrawler:
+    """Builds a complete :class:`FolderNode` tree from a root Drive folder."""
+
+    def __init__(
+        self,
+        source: DriveSource,
+        path_policy: PathPolicy | None = None,
+        cache: MetadataCache | None = None,
+        max_depth: int = 32,
+        follow_shortcuts: bool = True,
+        preferred_export_format: str | None = None,
+        concurrency: int = 6,
+    ) -> None:
+        self._source = source
+        self._path_policy = path_policy or PathPolicy()
+        self._cache = cache
+        self._max_depth = max_depth
+        self._follow_shortcuts = follow_shortcuts
+        self._preferred_export_format = preferred_export_format
+        self._concurrency = max(1, concurrency)
+
+    async def crawl(self, root_item: DriveItem) -> FolderNode:
+        """Traverse ``root_item`` in breadth-first order and return its :class:`FolderNode`."""
+        root_name = self._path_policy.sanitize_segment(root_item.name, fallback=root_item.id)
+        root_node = FolderNode(
+            item=root_item,
+            relative_path=PurePosixPath(root_name),
+            depth=0,
+        )
+
+        visited_folders: set[str] = {root_item.id}
+        current_level: list[FolderNode] = [root_node]
+        sem = asyncio.Semaphore(self._concurrency)
+
+        async def _fetch_children(node: FolderNode) -> tuple[FolderNode, list[DriveItem]]:
+            async with sem:
+                try:
+                    children = await self._source.list_folder_children(
+                        node.item.id,
+                        resource_key=node.item.resource_key,
+                    )
+                    return node, children
+                except (ResourceNotFoundError, PermissionDeniedError):
+                    if node.depth == 0:
+                        raise
+                    return node, []
+
+        while current_level:
+            results = await asyncio.gather(*(_fetch_children(node) for node in current_level))
+            next_level: list[FolderNode] = []
+
+            for current, children in results:
+                for child in children:
+                    resolved = await self._resolve_if_shortcut(child)
+                    if resolved is None:
+                        continue
+
+                    if self._cache is not None:
+                        self._cache.put(resolved, persist=False)
+
+                    if resolved.is_folder:
+                        if resolved.id in visited_folders:
+                            # Skip shortcut cycles pointing back to an ancestor or queued folder.
+                            continue
+                        visited_folders.add(resolved.id)
+                        folder_rel = self._path_policy.allocate_child_path(
+                            current.relative_path,
+                            resolved.name,
+                        )
+                        sub_node = FolderNode(
+                            item=resolved,
+                            relative_path=folder_rel,
+                            depth=current.depth + 1,
+                        )
+                        current.subfolders.append(sub_node)
+                        if sub_node.depth < self._max_depth:
+                            next_level.append(sub_node)
+                    else:
+                        forced_ext: str | None = None
+                        if resolved.is_google_doc:
+                            _, forced_ext = resolved.resolve_export(self._preferred_export_format)
+                        file_rel = self._path_policy.allocate_child_path(
+                            current.relative_path,
+                            resolved.name,
+                            forced_extension=forced_ext,
+                        )
+                        current.files.append(
+                            FileEntry(
+                                item=resolved,
+                                relative_path=file_rel,
+                                depth=current.depth + 1,
+                            )
+                        )
+
+            if self._cache is not None:
+                self._cache.flush()
+            current_level = next_level
+
+        return root_node
+
+    async def _resolve_if_shortcut(self, item: DriveItem) -> DriveItem | None:
+        if not item.is_shortcut:
+            return item
+        if not self._follow_shortcuts or not item.shortcut_target_id:
+            return None
+        try:
+            target = await self._source.fetch_metadata(
+                ParsedTarget(id=item.shortcut_target_id, kind=TargetKind.UNKNOWN)
+            )
+        except (ResourceNotFoundError, PermissionDeniedError):
+            return None
+        if item.name:
+            target.name = item.name
+        return target

gdown2/application/exporter.py

@@ -0,0 +1,40 @@
+"""Export service for Google Workspace documents (Docs, Sheets, Slides, Drawings)."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from gdown2.application.ports import DriveSource
+from gdown2.domain.models import DownloadTask
+from gdown2.infrastructure.storage.fs_sink import FileSystemSink
+
+
+class ExportService:
+    """Streams Google Workspace documents in their target export MIME format."""
+
+    def __init__(self, source: DriveSource, sink: FileSystemSink) -> None:
+        self._source = source
+        self._sink = sink
+
+    async def export_task(
+        self,
+        task: DownloadTask,
+        *,
+        on_bytes: Callable[[int], None] | None = None,
+    ) -> int:
+        """Stream the exported document into ``task.part_path`` and return total bytes written."""
+        export_mime, _ = task.item.resolve_export(task.export_format)
+        written = 0
+
+        async with self._source.open_export_stream(task.item, export_mime) as media:
+            with self._sink.open_part(task.part_path, append=False) as handle:
+                async for chunk in media.chunks:
+                    if not chunk:
+                        continue
+                    handle.write(chunk)
+                    written += len(chunk)
+                    if on_bytes is not None:
+                        on_bytes(len(chunk))
+
+        return written
+

gdown2/application/planner.py

@@ -0,0 +1,193 @@
+"""Execution planner turning resolved Drive items or folder trees into DownloadTasks."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from pathlib import Path, PurePosixPath
+
+from gdown2.domain.models import (
+    DownloadTask,
+    DriveItem,
+    FolderNode,
+    TaskState,
+    TransferState,
+)
+from gdown2.domain.path_policy import PathPolicy
+from gdown2.domain.policies import ExistingFilePolicy
+from gdown2.domain.tree import walk_files, walk_folders_bfs
+
+
+@dataclass(slots=True)
+class DownloadPlan:
+    """Ordered directories to create and download tasks to execute."""
+
+    root_dir: Path
+    directories: list[Path] = field(default_factory=list)
+    tasks: list[DownloadTask] = field(default_factory=list)
+
+    @property
+    def total_bytes(self) -> int:
+        return sum(t.bytes_total or 0 for t in self.tasks)
+
+
+class Planner:
+    """Produces a deterministic :class:`DownloadPlan` from domain objects."""
+
+    def __init__(
+        self,
+        path_policy: PathPolicy | None = None,
+        existing_policy: ExistingFilePolicy = ExistingFilePolicy.RESUME,
+        flat: bool = False,
+        export_format: str | None = None,
+    ) -> None:
+        self._path_policy = path_policy or PathPolicy()
+        self._existing_policy = existing_policy
+        self._flat = flat
+        self._export_format = export_format
+
+    def plan_single(
+        self,
+        item: DriveItem,
+        output: Path,
+        *,
+        saved_states: dict[str, TransferState] | None = None,
+    ) -> DownloadPlan:
+        """Create a plan for a single file or Google Workspace document."""
+        forced_ext: str | None = None
+        if item.is_google_doc:
+            _, forced_ext = item.resolve_export(self._export_format)
+
+        # If output is an existing directory (or ends with a separator), place the file inside it.
+        is_directory_output = output.is_dir() or str(output).endswith(("/", "\\"))
+        if is_directory_output:
+            root_dir = output.resolve()
+            rel = self._path_policy.allocate_child_path(
+                PurePosixPath("."),
+                item.name,
+                forced_extension=forced_ext,
+            )
+            local_path = self._path_policy.resolve_local_path(root_dir, rel)
+        else:
+            root_dir = output.parent.resolve()
+            rel = PurePosixPath(self._path_policy.sanitize_segment(output.name, fallback=item.name))
+            if forced_ext and not rel.name.lower().endswith(forced_ext.lower()):
+                rel = PurePosixPath(f"{rel.name}{forced_ext}")
+            local_path = self._path_policy.resolve_local_path(root_dir, rel)
+
+        task = self._build_task(item, local_path, rel, saved_states=saved_states)
+        return DownloadPlan(
+            root_dir=root_dir,
+            directories=[root_dir],
+            tasks=[task],
+        )
+
+    def plan_tree(
+        self,
+        tree: FolderNode,
+        output_dir: Path,
+        *,
+        saved_states: dict[str, TransferState] | None = None,
+    ) -> DownloadPlan:
+        """Create a plan for an entire :class:`FolderNode` hierarchy."""
+        root_dir = output_dir.resolve()
+        directories: list[Path] = [root_dir]
+        tasks: list[DownloadTask] = []
+
+        if not self._flat:
+            for folder in walk_folders_bfs(tree):
+                folder_path = self._path_policy.resolve_local_path(root_dir, folder.relative_path)
+                if folder_path not in directories:
+                    directories.append(folder_path)
+
+        flat_policy = PathPolicy()
+        for entry in walk_files(tree):
+            if self._flat:
+                forced_ext: str | None = None
+                if entry.item.is_google_doc:
+                    _, forced_ext = entry.item.resolve_export(self._export_format)
+                rel = flat_policy.allocate_child_path(
+                    PurePosixPath("."),
+                    entry.item.name,
+                    forced_extension=forced_ext,
+                )
+            else:
+                rel = entry.relative_path
+
+            local_path = self._path_policy.resolve_local_path(root_dir, rel)
+            task = self._build_task(entry.item, local_path, rel, saved_states=saved_states)
+            tasks.append(task)
+
+        return DownloadPlan(
+            root_dir=root_dir,
+            directories=directories,
+            tasks=tasks,
+        )
+
+    def _build_task(
+        self,
+        item: DriveItem,
+        local_path: Path,
+        relative_path: PurePosixPath,
+        *,
+        saved_states: dict[str, TransferState] | None = None,
+    ) -> DownloadTask:
+        task = DownloadTask(
+            item=item,
+            local_path=local_path,
+            relative_path=relative_path,
+            bytes_total=item.size,
+            etag=item.etag,
+            export_format=self._export_format if item.is_google_doc else None,
+        )
+
+        # Check if the final file already exists on disk.
+        if local_path.exists() and local_path.is_file():
+            existing_size = local_path.stat().st_size
+            if self._existing_policy == ExistingFilePolicy.SKIP:
+                task.state = TaskState.SKIPPED
+                task.bytes_done = existing_size
+                return task
+            if self._existing_policy == ExistingFilePolicy.RESUME and self._is_complete(
+                item, local_path, existing_size
+            ):
+                task.state = TaskState.SKIPPED
+                task.bytes_done = existing_size
+                return task
+
+        # Overwriting must never resume stale bytes from a previous transfer.
+        if self._existing_policy == ExistingFilePolicy.OVERWRITE and task.part_path.exists():
+            task.stale_part = True
+            task.resume_offset = 0
+            task.bytes_done = 0
+            return task
+
+        # Inspect .part file and saved TransferState for mid-stream resume.
+        if self._existing_policy == ExistingFilePolicy.RESUME and task.part_path.exists():
+            part_size = task.part_path.stat().st_size
+            saved = (saved_states or {}).get(task.task_id)
+            if saved and saved.etag and item.etag and saved.etag != item.etag:
+                # Remote file changed since last session; discard stale partial data.
+                task.stale_part = True
+                task.resume_offset = 0
+                task.bytes_done = 0
+            else:
+                task.resume_offset = part_size
+                task.bytes_done = part_size
+                if saved and saved.etag and not task.etag:
+                    task.etag = saved.etag
+
+        return task
+
+    @staticmethod
+    def _is_complete(item: DriveItem, local_path: Path, existing_size: int) -> bool:
+        if item.size is None or existing_size != item.size:
+            return False
+        if not item.md5:
+            return True
+        hasher = hashlib.md5()
+        with local_path.open("rb") as handle:
+            while chunk := handle.read(256 * 1024):
+                hasher.update(chunk)
+        return hasher.hexdigest().lower() == item.md5.lower()
+

gdown2/application/ports.py

@@ -0,0 +1,85 @@
+"""Ports (interfaces) used by the application layer to talk to infrastructure."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from contextlib import AbstractAsyncContextManager
+from dataclasses import dataclass
+from typing import Protocol
+
+from gdown2.domain.models import DriveItem, TransferState
+from gdown2.domain.url_parser import ParsedTarget
+
+
+@dataclass(slots=True)
+class MediaStream:
+    """Opened byte stream along with HTTP transfer headers."""
+
+    chunks: AsyncIterator[bytes]
+    status_code: int = 200
+    content_length: int | None = None
+    etag: str | None = None
+    accept_ranges: bool = False
+    is_range_satisfied: bool = True
+
+
+class DriveSource(Protocol):
+    """Port for querying Google Drive metadata and streaming file contents."""
+
+    async def fetch_metadata(self, target: ParsedTarget) -> DriveItem:
+        """Fetch metadata for a single file, folder, or native doc."""
+        ...
+
+    async def list_folder_children(
+        self,
+        folder_id: str,
+        *,
+        resource_key: str | None = None,
+    ) -> list[DriveItem]:
+        """List immediate children of a Drive folder (handling pagination internally)."""
+        ...
+
+    def open_download_stream(
+        self,
+        item: DriveItem,
+        *,
+        start_byte: int = 0,
+        end_byte: int | None = None,
+    ) -> AbstractAsyncContextManager[MediaStream]:
+        """Open a binary download stream, optionally starting at ``start_byte``."""
+        ...
+
+    def open_export_stream(
+        self,
+        item: DriveItem,
+        export_mime: str,
+    ) -> AbstractAsyncContextManager[MediaStream]:
+        """Open an export stream for a Google Workspace native document."""
+        ...
+
+
+class MetadataCache(Protocol):
+    """Port for caching resolved DriveItem records."""
+
+    def get(self, item_id: str) -> DriveItem | None:
+        ...
+
+    def put(self, item: DriveItem, *, persist: bool = True) -> None:
+        ...
+
+    def flush(self) -> None:
+        ...
+
+
+class ResumeRepository(Protocol):
+    """Port for persisting and loading transfer state across runs."""
+
+    def load_all(self) -> dict[str, TransferState]:
+        ...
+
+    def save_task(self, state: TransferState) -> None:
+        ...
+
+    def remove_task(self, task_key: str) -> None:
+        ...
+