git-ftp 2.0.0.dev0__py3-none-any.whl

gitftp/session.py

@@ -0,0 +1,212 @@
+"""Everything an action needs: repository, config, URL, credentials, connections."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from gitftp import auth
+from gitftp.changeset import remote_path
+from gitftp.config import DEFAULT_DEPLOYED_SHA1_FILE, DEFAULT_JOBS, Config, validate_scope
+from gitftp.errors import GitError, MissingArgumentError
+from gitftp.gitrepo import GitRepo, GitRunner
+from gitftp.options import CliOptions
+from gitftp.output import Output
+from gitftp.transport import registry
+from gitftp.transport.base import Transport
+from gitftp.url import RemoteURL, parse
+
+
+class Session:
+    def __init__(
+        self,
+        *,
+        out: Output,
+        git: GitRunner,
+        repo: GitRepo | None,
+        scope: str | None,
+        cfg: Config,
+        url: RemoteURL,
+        creds: auth.Credentials,
+        topts: registry.TransportOptions,
+        jobs: int,
+        deployed_sha1_file: str,
+        syncroot: str,
+        worktree: bool = False,
+    ) -> None:
+        self.out = out
+        self.git = git
+        self.repo = repo
+        self.scope = scope
+        self.cfg = cfg
+        self.url = url
+        self.creds = creds
+        self.topts = topts
+        self.jobs = jobs
+        self.deployed_sha1_file = deployed_sha1_file
+        self.syncroot = syncroot
+        self.worktree = worktree
+        self._connector = registry.connector(url, creds, topts, out)
+        self._primary: Transport | None = None
+
+    def log_settings(self) -> None:
+        """Upstream's ``set_remotes`` diagnostics (printed with ``-v``)."""
+        out = self.out
+        out.debug(f"Host is '{self.url.host}'.")
+        out.debug(f"User is '{self.creds.user}'.")
+        out.debug("Password is set." if self.creds.password is not None else "No password is set.")
+        if self.syncroot:
+            out.debug(f"Syncroot is '{self.syncroot}'.")
+        out.debug(f"Insecure is '{int(self.topts.insecure)}'.")
+        if self.topts.disable_epsv:
+            out.debug("Disable EPSV is '1'.")
+        if self.jobs != DEFAULT_JOBS:
+            out.debug(f"Jobs is '{self.jobs}'.")
+
+    def connect(self) -> Transport:
+        return self._connector()
+
+    @property
+    def primary(self) -> Transport:
+        if self._primary is None:
+            self._primary = self.connect()
+        return self._primary
+
+    def close(self) -> None:
+        if self._primary is not None:
+            try:
+                self._primary.close()
+            finally:
+                self._primary = None
+
+    def require_repo(self) -> GitRepo:
+        if self.repo is None:
+            raise GitError("Not a Git project? Exiting...")
+        return self.repo
+
+    def for_submodule(self, sub_path: str) -> Session:
+        """A session for a submodule at git path ``sub_path`` (inside the syncroot)."""
+        repo = self.require_repo()
+        sub_root = repo.root / sub_path
+        sub_repo = GitRepo(sub_root)
+        cfg = Config.load(sub_repo, sub_root, self.scope)
+        session = Session(
+            out=self.out,
+            git=sub_repo,
+            repo=sub_repo,
+            scope=self.scope,
+            cfg=cfg,
+            url=self.url.child(remote_path(sub_path, self.syncroot)),
+            creds=self.creds,
+            topts=self.topts,
+            jobs=self.jobs,
+            deployed_sha1_file=cfg.get("deployedsha1file", DEFAULT_DEPLOYED_SHA1_FILE),
+            syncroot="",
+            worktree=self.worktree,
+        )
+        session.log_settings()
+        return session
+
+
+def resolve_syncroot(repo: GitRepo | None, value: str) -> str:
+    value = value.replace("\\", "/")
+    while value.startswith("./"):
+        value = value[2:]
+    value = value.strip("/")
+    if not value or value == ".":
+        return ""
+    if repo is not None and not (repo.root / value).is_dir():
+        raise GitError(f"'{value}' is not a directory! Exiting...")
+    return value + "/"
+
+
+def open_session(
+    opts: CliOptions,
+    url_arg: str | None,
+    out: Output,
+    *,
+    cwd: Path | None = None,
+    need_repo: bool = True,
+    need_url: bool = True,
+) -> Session:
+    cwd = cwd or Path.cwd()
+    git = GitRunner(cwd)
+    git.check_version()
+
+    repo: GitRepo | None
+    if need_repo:
+        repo = GitRepo.discover(cwd)
+    else:
+        try:
+            repo = GitRepo.discover(cwd)
+        except GitError:
+            repo = None
+
+    scope: str | None = None
+    if opts.scope is not None:
+        if opts.scope:
+            scope = opts.scope
+        elif repo is not None:
+            scope = repo.current_branch()
+        else:
+            raise MissingArgumentError("Missing scope argument.")
+        validate_scope(scope, from_option=True)
+
+    cfg = Config.load(git, repo.root if repo else None, scope)
+
+    raw_url = url_arg or os.environ.get("GIT_FTP_URL") or cfg.get("url")
+    if not raw_url:
+        if need_url:
+            raise MissingArgumentError("Remote host not set.")
+        raw_url = "ftp://localhost"
+    url = parse(raw_url)
+    remote_root = opts.remote_root if opts.remote_root is not None else cfg.get("remote-root")
+    if remote_root:
+        url.set_path(remote_root)
+
+    creds = auth.resolve(cfg, url, opts.auth_flags(), out)
+    # Messages show the resolved login as upstream does (``user:***@host``); the
+    # password itself is never placed in the URL.
+    url.user = creds.user or None
+    url.password = None
+
+    insecure = opts.insecure or cfg.get_bool("insecure")
+    cacert = None
+    for candidate in (opts.cacert, cfg.get("cacert")):
+        if candidate and os.access(candidate, os.R_OK):
+            cacert = candidate
+            break
+    disable_epsv = (not opts.active) and (opts.disable_epsv or cfg.get_bool("disable-epsv"))
+    proxy = opts.proxy or cfg.get("proxy") or cfg.git_option("http.proxy") or None
+    topts = registry.TransportOptions(
+        insecure=insecure,
+        cacert=cacert,
+        active=opts.active,
+        disable_epsv=disable_epsv,
+        proxy=proxy,
+        trace=out.trace if out.tracing else None,
+    )
+
+    jobs = opts.jobs if opts.jobs is not None else cfg.get_int("jobs", DEFAULT_JOBS)
+    syncroot = resolve_syncroot(
+        repo, opts.syncroot if opts.syncroot is not None else cfg.get("syncroot")
+    )
+    deployed_sha1_file = cfg.get("deployedsha1file", DEFAULT_DEPLOYED_SHA1_FILE)
+    worktree = opts.worktree or cfg.get_bool("worktree")
+
+    session = Session(
+        out=out,
+        git=git,
+        repo=repo,
+        scope=scope,
+        cfg=cfg,
+        url=url,
+        creds=creds,
+        topts=topts,
+        jobs=max(1, jobs),
+        deployed_sha1_file=deployed_sha1_file,
+        syncroot=syncroot,
+        worktree=worktree,
+    )
+    session.log_settings()
+    return session

gitftp/transfer.py

@@ -0,0 +1,269 @@
+"""Parallel transfer engine.
+
+A pool of worker threads, each with its own connection. Uploads and downloads
+are fail-fast (the first failure cancels the rest); deletes collect their
+failures, which upstream treats as warnings.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import threading
+from collections.abc import Callable, Sequence
+from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Generic, TypeVar
+
+from gitftp.output import Output
+from gitftp.transport.base import Connector, TransferCancelled, Transport
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+@dataclass(frozen=True)
+class UploadTask:
+    local: Path
+    remote: str
+    size: int
+    label: str
+
+
+@dataclass(frozen=True)
+class DeleteTask:
+    remote: str
+    label: str
+
+
+@dataclass(frozen=True)
+class DownloadTask:
+    remote: str
+    local: Path
+    size: int | None
+    mtime: int | None
+    label: str
+
+
+class TransferError(Exception):
+    def __init__(self, label: str, cause: BaseException) -> None:
+        super().__init__(f"{label}: {cause}")
+        self.label = label
+        self.cause = cause
+
+
+class _Skipped:
+    def __repr__(self) -> str:
+        return "<skipped>"
+
+
+SKIPPED = _Skipped()
+
+
+class _Result(Generic[R]):
+    __slots__ = ("value",)
+
+    def __init__(self, value: R | TransferError | _Skipped) -> None:
+        self.value = value
+
+
+class TransferPool:
+    def __init__(
+        self,
+        connect: Connector,
+        jobs: int,
+        out: Output,
+        *,
+        primary: Transport | None = None,
+    ) -> None:
+        self.connect = connect
+        self.jobs = max(1, jobs)
+        self.out = out
+        self.primary = primary
+        self._cancel = threading.Event()
+        self._local = threading.local()
+        self._opened: list[Transport] = []
+        self._opened_lock = threading.Lock()
+        self._own_primary = False
+
+    # -- lifecycle ---------------------------------------------------------
+    def __enter__(self) -> TransferPool:
+        if self.primary is not None:
+            self.primary.cancel = self._cancel
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def close(self) -> None:
+        if self.primary is not None and not self._own_primary:
+            # The session keeps using this connection after the pool; a cancelled
+            # pool must not poison it.
+            self.primary.cancel = threading.Event()
+        with self._opened_lock:
+            opened, self._opened = self._opened, []
+        for t in opened:
+            with contextlib.suppress(Exception):
+                t.close()
+        if self._own_primary and self.primary is not None:
+            with contextlib.suppress(Exception):
+                self.primary.close()
+            self.primary = None
+            self._own_primary = False
+
+    def cancel(self) -> None:
+        self._cancel.set()
+
+    @property
+    def cancelled(self) -> bool:
+        return self._cancel.is_set()
+
+    def _primary(self) -> Transport:
+        if self.primary is None:
+            self.primary = self.connect()
+            self.primary.cancel = self._cancel
+            self._own_primary = True
+        return self.primary
+
+    def _worker_transport(self) -> Transport:
+        t: Transport | None = getattr(self._local, "transport", None)
+        if t is not None:
+            return t
+        failure: BaseException | None = getattr(self._local, "failure", None)
+        if failure is not None:
+            raise failure
+        try:
+            t = self.connect()
+        except BaseException as e:
+            self._local.failure = e
+            raise
+        t.cancel = self._cancel
+        with self._opened_lock:
+            self._opened.append(t)
+        self._local.transport = t
+        return t
+
+    # -- the primitive -----------------------------------------------------
+    def map(
+        self,
+        fn: Callable[[Transport, T], R],
+        items: Sequence[T],
+        *,
+        fail_fast: bool = True,
+        label: Callable[[T], str] = str,
+    ) -> list[R | TransferError | _Skipped]:
+        if not items:
+            return []
+        if self.jobs == 1 or len(items) == 1:
+            return self._map_serial(fn, items, fail_fast=fail_fast, label=label)
+        return self._map_parallel(fn, items, fail_fast=fail_fast, label=label)
+
+    def _map_serial(
+        self,
+        fn: Callable[[Transport, T], R],
+        items: Sequence[T],
+        *,
+        fail_fast: bool,
+        label: Callable[[T], str],
+    ) -> list[R | TransferError | _Skipped]:
+        results: list[R | TransferError | _Skipped] = []
+        t = self._primary()
+        for item in items:
+            if self._cancel.is_set():
+                results.append(SKIPPED)
+                continue
+            try:
+                results.append(fn(t, item))
+            except TransferCancelled:
+                results.append(SKIPPED)
+            except Exception as e:
+                err = TransferError(label(item), e)
+                if fail_fast:
+                    raise err from e
+                results.append(err)
+        return results
+
+    def _run_one(self, fn: Callable[[Transport, T], R], item: T) -> R | _Skipped:
+        if self._cancel.is_set():
+            return SKIPPED
+        t = self._worker_transport()
+        try:
+            return fn(t, item)
+        except TransferCancelled:
+            return SKIPPED
+        except Exception:
+            if self._cancel.is_set():
+                return SKIPPED
+            raise
+
+    def _map_parallel(
+        self,
+        fn: Callable[[Transport, T], R],
+        items: Sequence[T],
+        *,
+        fail_fast: bool,
+        label: Callable[[T], str],
+    ) -> list[R | TransferError | _Skipped]:
+        workers = min(self.jobs, len(items))
+        executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="git-ftp")
+        futures: list[Future[R | _Skipped]] = []
+        first_error: TransferError | None = None
+        results: dict[Future[R | _Skipped], R | TransferError | _Skipped] = {}
+        try:
+            for item in items:
+                futures.append(executor.submit(self._run_one, fn, item))
+            pending = set(futures)
+            while pending:
+                done, pending = wait(pending, timeout=0.5, return_when=FIRST_COMPLETED)
+                for fut in done:
+                    idx = futures.index(fut)
+                    try:
+                        results[fut] = fut.result()
+                    except Exception as e:
+                        err = TransferError(label(items[idx]), e)
+                        results[fut] = err
+                        if fail_fast and first_error is None:
+                            first_error = err
+                            self._cancel.set()
+        except BaseException:
+            self._cancel.set()
+            executor.shutdown(wait=False, cancel_futures=True)
+            raise
+        executor.shutdown(wait=True)
+        if first_error is not None:
+            raise first_error
+        return [results[f] for f in futures]
+
+    # -- typed helpers -----------------------------------------------------
+    def upload(self, tasks: Sequence[UploadTask]) -> None:
+        def do(t: Transport, task: UploadTask) -> None:
+            t.put(task.local, task.remote, task.size)
+            self.out.debug(f"Uploaded '{task.label}'.")
+
+        self.map(do, tasks, fail_fast=True, label=lambda task: task.label)
+
+    def delete(self, tasks: Sequence[DeleteTask]) -> list[TransferError]:
+        def do(t: Transport, task: DeleteTask) -> None:
+            t.delete(task.remote)
+            self.out.debug(f"Deleted '{task.label}'.")
+
+        results = self.map(do, tasks, fail_fast=False, label=lambda task: task.label)
+        return [r for r in results if isinstance(r, TransferError)]
+
+    def download(self, tasks: Sequence[DownloadTask]) -> None:
+        def do(t: Transport, task: DownloadTask) -> None:
+            part = task.local.with_name(f".{task.local.name}.git-ftp-part")
+            task.local.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                t.get_file(task.remote, part)
+                if task.mtime is not None:
+                    os.utime(part, (task.mtime, task.mtime))
+                os.replace(part, task.local)
+            except BaseException:
+                with contextlib.suppress(OSError):
+                    part.unlink()
+                raise
+            self.out.debug(f"Downloaded '{task.label}'.")
+
+        self.map(do, tasks, fail_fast=True, label=lambda task: task.label)

gitftp/transport/__init__.py

@@ -0,0 +1 @@
+"""Remote transports (FTP family via libcurl, SFTP via paramiko)."""

gitftp/transport/base.py

@@ -0,0 +1,93 @@
+"""The transport interface every backend implements."""
+
+from __future__ import annotations
+
+import threading
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+
+class RemoteNotFound(Exception):
+    """The remote file or directory does not exist (not an access or network error)."""
+
+
+class TransferCancelled(Exception):
+    """Raised inside a worker when the pool's cancel event was set mid-transfer."""
+
+
+@dataclass(frozen=True)
+class Entry:
+    name: str
+    is_dir: bool
+    size: int | None = None
+    mtime: int | None = None  # unix seconds, UTC
+    mtime_exact: bool = True  # False for LIST-derived times (minute resolution)
+    is_link: bool = False
+
+
+ProgressFn = Callable[[int, int], None]
+
+
+class Transport(ABC):
+    """One open connection.
+
+    Paths are relative to the URL's remote directory and never start with '/'.
+    Instances are not thread-safe; the transfer pool gives each worker its own.
+    """
+
+    def __init__(self) -> None:
+        self.cancel = threading.Event()
+
+    @abstractmethod
+    def open(self) -> None: ...
+
+    @abstractmethod
+    def close(self) -> None: ...
+
+    @abstractmethod
+    def get(self, path: str) -> bytes:
+        """Read a small file. Raises RemoteNotFound, else DownloadError."""
+
+    @abstractmethod
+    def get_file(self, path: str, local: Path, *, progress: ProgressFn | None = None) -> None:
+        """Stream a remote file into ``local``."""
+
+    @abstractmethod
+    def put(
+        self, local: Path, remote: str, size: int, *, progress: ProgressFn | None = None
+    ) -> None:
+        """Upload a file, creating missing parent directories. Raises UploadError."""
+
+    @abstractmethod
+    def put_bytes(self, data: bytes, remote: str) -> None: ...
+
+    @abstractmethod
+    def delete(self, path: str) -> None:
+        """Remove a file; an already absent file is success. Raises UploadError."""
+
+    @abstractmethod
+    def mkdir_p(self, directory: str) -> None:
+        """Create ``directory`` and parents; '' is the remote root itself."""
+
+    @abstractmethod
+    def exists(self, path: str) -> bool: ...
+
+    @abstractmethod
+    def stat(self, path: str) -> Entry | None: ...
+
+    @abstractmethod
+    def list_dir(self, path: str) -> list[Entry]:
+        """Non-recursive listing without '.' and '..'. Raises RemoteNotFound."""
+
+    def __enter__(self) -> Transport:
+        self.open()
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+
+Connector = Callable[[], Transport]
+"""Returns a fresh, *opened* connection; the pool calls it once per worker."""