nonebot-plugin-git-poller 0.1.5__py3-none-any.whl

nonebot_plugin_git_poller/mirror.py

@@ -0,0 +1,485 @@
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime, timezone
+import shutil
+from threading import RLock
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+from nonebot import logger
+
+from .archive import ArchiveBuilder, ArchiveFile
+from .config import Config
+from .git import GitRepositoryCache
+from .models import PollResult, RepositoryIdentity, Subscription, UpdatePayload
+from .repository import (
+    build_compare_url,
+    build_identity,
+    build_identity_from_normalized_url,
+    normalize_branch,
+    normalize_repo_url,
+)
+from .state import StateStore
+
+
+@dataclass(frozen=True)
+class FollowResult:
+    identity: RepositoryIdentity
+    subscription: Subscription
+    already_following: bool
+
+
+@dataclass(frozen=True)
+class PullResult:
+    identity: RepositoryIdentity
+    subscription: Subscription
+    previous_sha: str | None
+    target_sha: str
+    payload: UpdatePayload
+    archive: ArchiveFile
+
+
+@dataclass(frozen=True)
+class SummaryResult:
+    result: PollResult
+    behind_count: int | None
+
+
+@dataclass(frozen=True)
+class DeliveryResult:
+    result: PollResult
+    archive: ArchiveFile
+
+
+class GitPollerService:
+    def __init__(
+        self,
+        config: Config,
+        *,
+        state: StateStore | None = None,
+        git_cache: GitRepositoryCache | None = None,
+        archive_builder: ArchiveBuilder | None = None,
+    ) -> None:
+        self.config = config
+        self.state = state or StateStore()
+        self.git_cache = git_cache or GitRepositoryCache(config)
+        self.archive_builder = archive_builder or ArchiveBuilder(
+            config.git_poller_archive_password
+        )
+        self._lock = asyncio.Lock()
+        # Protect StateStore's read-modify-write JSON operations only.
+        self._state_lock = RLock()
+
+    async def follow_repo(self, group_id: int, url: str, branch: str | None = None) -> FollowResult:
+        async with self._lock:
+            return await asyncio.to_thread(self._follow_repo_sync, group_id, url, branch)
+
+    async def pull_repo(self, group_id: int, url: str, branch: str | None = None) -> PullResult:
+        async with self._lock:
+            return await asyncio.to_thread(self._pull_repo_sync, group_id, url, branch)
+
+    async def summarize_repo(self, group_id: int, url: str, branch: str | None = None) -> SummaryResult:
+        async with self._lock:
+            return await asyncio.to_thread(self._summarize_repo_sync, group_id, url, branch)
+
+    async def poll_schedule(self, schedule: str):
+        subscriptions = await asyncio.to_thread(self._subscriptions_for_schedule_sync, schedule)
+        for group_id, repo_key, subscription in subscriptions:
+            try:
+                async with self._lock:
+                    result = await asyncio.to_thread(
+                        self._poll_scheduled_subscription_sync,
+                        group_id,
+                        repo_key,
+                        subscription,
+                    )
+            except Exception:
+                logger.exception(
+                    f"git poller scheduled subscription failed: "
+                    f"group={group_id}, repo={repo_key}"
+                )
+                continue
+            if result is not None:
+                yield result
+
+    def update_repo_schedule(
+        self,
+        group_id: int,
+        url: str,
+        branch: str | None,
+        schedule: str,
+    ) -> tuple[RepositoryIdentity, Subscription]:
+        with self._state_lock:
+            identity, _ = self._get_subscription(group_id, url, branch)
+            subscription = self.state.update_schedule(
+                group_id,
+                identity.key,
+                schedule,
+                _now_iso(self.config.git_poller_timezone),
+            )
+            return identity, subscription
+
+    def update_repo_archive_password(
+        self,
+        group_id: int,
+        url: str,
+        branch: str | None,
+        password: str | None,
+    ) -> tuple[RepositoryIdentity, Subscription]:
+        with self._state_lock:
+            identity, _ = self._get_subscription(group_id, url, branch)
+            subscription = self.state.update_archive_password(
+                group_id,
+                identity.key,
+                password,
+                _now_iso(self.config.git_poller_timezone),
+            )
+            return identity, subscription
+
+    def unfollow_repo(
+        self,
+        group_id: int,
+        url: str,
+        branch: str | None = None,
+    ) -> tuple[RepositoryIdentity, bool]:
+        archive_path: str | None = None
+        with self._state_lock:
+            identity, existing = self._find_subscription(group_id, url, branch)
+            if existing is None:
+                return identity, False
+            removed = self.state.remove_subscription(group_id, identity.key)
+            if removed:
+                archive_path = existing.last_archive_path
+        if archive_path:
+            self.archive_builder.remove_archive(archive_path)
+        return identity, removed
+
+    def get_repo_subscription(
+        self,
+        group_id: int,
+        url: str,
+        branch: str | None = None,
+    ) -> tuple[RepositoryIdentity, Subscription]:
+        with self._state_lock:
+            return self._get_subscription(group_id, url, branch)
+
+    def list_group_subscriptions(self, group_id: int) -> dict[str, Subscription]:
+        with self._state_lock:
+            return self.state.list_group_subscriptions(group_id)
+
+    def scheduled_rules(self) -> set[str]:
+        with self._state_lock:
+            rules = {self.config.git_poller_default_schedule.strip()}
+            for subscriptions in self.state.list_all_subscriptions().values():
+                for subscription in subscriptions.values():
+                    if subscription.schedule.strip():
+                        rules.add(subscription.schedule.strip())
+            return {rule for rule in rules if rule}
+
+    def mark_success(self, result: PollResult) -> None:
+        with self._state_lock:
+            self.state.update_last_success(
+                result.group_id,
+                result.repo_key,
+                result.payload.target_sha,
+                _now_iso(self.config.git_poller_timezone),
+            )
+
+    def mark_pull_success(self, group_id: int, repo_key: str, target_sha: str) -> None:
+        with self._state_lock:
+            self.state.update_last_success(group_id, repo_key, target_sha, _now_iso(self.config.git_poller_timezone))
+
+    def cleanup_unsubscribed_repo(self, repo_key: str) -> bool:
+        with self._state_lock:
+            is_subscribed = self.state.is_repo_key_subscribed(repo_key)
+        if is_subscribed:
+            logger.info(f"git poller cleanup skipped for subscribed repo: {repo_key}")
+            return False
+        removed_cache = self.git_cache.remove_cache(repo_key)
+        removed_archives = self.archive_builder.remove_archives_for_repo(repo_key)
+        logger.info(
+            f"git poller cleanup finished: repo={repo_key}, "
+            f"cache={removed_cache}, archives={removed_archives}"
+        )
+        return removed_cache or bool(removed_archives)
+
+    def cleanup_orphaned_storage(self) -> tuple[int, int]:
+        with self._state_lock:
+            active_repo_keys = {
+                repo_key
+                for subscriptions in self.state.list_all_subscriptions().values()
+                for repo_key in subscriptions
+            }
+        removed_caches = 0
+        for repo_key in sorted(self.git_cache.cached_repo_keys() - active_repo_keys):
+            if self.git_cache.remove_cache(repo_key):
+                removed_caches += 1
+        removed_archives = self.archive_builder.remove_archives_except(active_repo_keys)
+        logger.info(
+            f"git poller orphan cleanup finished: "
+            f"cache={removed_caches}, archives={removed_archives}"
+        )
+        return removed_caches, removed_archives
+
+    def _follow_repo_sync(self, group_id: int, url: str, branch: str | None) -> FollowResult:
+        normalized_url = normalize_repo_url(url)
+        remote_head = self.git_cache.resolve_remote_head(
+            normalized_url,
+            self._explicit_branch(branch),
+        )
+        target_branch = remote_head.branch
+        identity = build_identity_from_normalized_url(normalized_url, target_branch)
+        with self._state_lock:
+            existing = self.state.get_subscription(group_id, identity.key)
+            if existing is not None:
+                return FollowResult(
+                    identity=identity,
+                    subscription=existing,
+                    already_following=True,
+                )
+
+            now = _now_iso(self.config.git_poller_timezone)
+            subscription = Subscription(
+                url=identity.url,
+                branch=target_branch,
+                schedule=self.config.git_poller_default_schedule,
+                last_success_sha=None,
+                enabled=True,
+                created_at=now,
+                updated_at=now,
+            )
+
+            subscription.last_success_sha = remote_head.sha
+            self.state.upsert_subscription(group_id, identity.key, subscription)
+            return FollowResult(
+                identity=identity,
+                subscription=subscription,
+                already_following=False,
+            )
+
+    def _pull_repo_sync(self, group_id: int, url: str, branch: str | None) -> PullResult:
+        with self._state_lock:
+            identity, subscription = self._get_subscription(group_id, url, branch)
+        fetched = self.git_cache.fetch(identity.key, subscription.url, subscription.branch)
+        try:
+            previous_sha = subscription.last_success_sha
+            payload = self._build_payload(identity, subscription, fetched, previous_sha)
+            archive = self._build_archive(
+                payload,
+                subscription,
+                fetched,
+                group_id=group_id,
+            )
+            return PullResult(
+                identity=identity,
+                subscription=subscription,
+                previous_sha=previous_sha,
+                target_sha=fetched.head_sha,
+                payload=payload,
+                archive=archive,
+            )
+        finally:
+            fetched.close()
+
+    def _summarize_repo_sync(self, group_id: int, url: str, branch: str | None) -> SummaryResult:
+        with self._state_lock:
+            identity, subscription = self._get_subscription(group_id, url, branch)
+        return self._poll_subscription(group_id, identity.key, identity, subscription)
+
+    def _get_subscription(
+        self,
+        group_id: int,
+        url: str,
+        branch: str | None,
+    ) -> tuple[RepositoryIdentity, Subscription]:
+        identity, subscription = self._find_subscription(group_id, url, branch)
+        if subscription is None:
+            raise KeyError("本群尚未关注这个仓库。")
+        return identity, subscription
+
+    def _poll_subscription(
+        self,
+        group_id: int,
+        repo_key: str,
+        identity: RepositoryIdentity,
+        subscription: Subscription,
+    ) -> SummaryResult:
+        fetched = self.git_cache.fetch(repo_key, subscription.url, subscription.branch)
+        try:
+            behind_count = fetched.count_commits_since(subscription.last_success_sha)
+            payload = self._build_payload(
+                identity,
+                subscription,
+                fetched,
+                subscription.last_success_sha,
+            )
+            return SummaryResult(
+                result=PollResult(
+                    group_id=group_id,
+                    repo_key=repo_key,
+                    subscription=subscription,
+                    payload=payload,
+                ),
+                behind_count=behind_count,
+            )
+        finally:
+            fetched.close()
+
+    def _subscriptions_for_schedule_sync(
+        self,
+        schedule: str,
+    ) -> list[tuple[int, str, Subscription]]:
+        with self._state_lock:
+            return self.state.subscriptions_for_schedule(schedule)
+
+    def _poll_scheduled_subscription_sync(
+        self,
+        group_id: int,
+        repo_key: str,
+        subscription: Subscription,
+    ) -> DeliveryResult | None:
+        with self._state_lock:
+            fresh = self.state.get_subscription(group_id, repo_key)
+            if fresh is None:
+                return None
+            subscription = fresh
+        identity = build_identity(subscription.url, subscription.branch)
+        fetched = self.git_cache.fetch(repo_key, subscription.url, subscription.branch)
+        try:
+            if not subscription.last_success_sha:
+                with self._state_lock:
+                    self.state.update_last_success(
+                        group_id,
+                        repo_key,
+                        fetched.head_sha,
+                        _now_iso(self.config.git_poller_timezone),
+                    )
+                return None
+
+            previous_sha = subscription.last_success_sha
+            if previous_sha == fetched.head_sha:
+                logger.debug(
+                    f"git poller no update: group={group_id}, repo={repo_key}, "
+                    f"head={fetched.head_sha[:8]}"
+                )
+                return None
+
+            payload = self._build_payload(
+                identity,
+                subscription,
+                fetched,
+                previous_sha,
+            )
+            result = PollResult(
+                group_id=group_id,
+                repo_key=repo_key,
+                subscription=subscription,
+                payload=payload,
+            )
+            return DeliveryResult(
+                result=result,
+                archive=self._build_archive(
+                    payload,
+                    subscription,
+                    fetched,
+                    group_id=group_id,
+                ),
+            )
+        finally:
+            fetched.close()
+
+    def _build_payload(
+        self,
+        identity: RepositoryIdentity,
+        subscription: Subscription,
+        fetched,
+        previous_sha: str | None,
+    ) -> UpdatePayload:
+        if previous_sha == fetched.head_sha:
+            commits = []
+        else:
+            commits = fetched.commits_since(previous_sha)
+        return UpdatePayload(
+            repo_key=identity.key,
+            repo_url=subscription.url,
+            repo_name=identity.display_name,
+            branch=subscription.branch,
+            previous_sha=previous_sha,
+            target_sha=fetched.head_sha,
+            target_short_sha=fetched.head_sha[:8],
+            generated_at=_now_iso(self.config.git_poller_timezone),
+            commits=commits,
+            compare_url=build_compare_url(subscription.url, previous_sha, fetched.head_sha),
+        )
+
+    def _build_archive(
+        self,
+        payload: UpdatePayload,
+        subscription: Subscription,
+        fetched,
+        *,
+        group_id: int,
+    ) -> ArchiveFile:
+        if subscription.last_archive_path:
+            self.archive_builder.remove_archive(subscription.last_archive_path)
+            with self._state_lock:
+                self.state.update_last_archive_path(
+                    group_id,
+                    payload.repo_key,
+                    None,
+                    _now_iso(self.config.git_poller_timezone),
+                )
+        source_dir = self.archive_builder.source_root(payload)
+        try:
+            fetched.export_head_tree(source_dir)
+            archive = self.archive_builder.build(payload, subscription, source_dir)
+            with self._state_lock:
+                self.state.update_last_archive_path(
+                    group_id,
+                    payload.repo_key,
+                    str(archive.path),
+                    _now_iso(self.config.git_poller_timezone),
+                )
+            return archive
+        finally:
+            shutil.rmtree(source_dir.parent, ignore_errors=True)
+
+    def _find_subscription(
+        self,
+        group_id: int,
+        url: str,
+        branch: str | None,
+    ) -> tuple[RepositoryIdentity, Subscription | None]:
+        explicit = self._explicit_branch(branch)
+        if explicit is not None:
+            identity = build_identity(url, explicit)
+            return identity, self.state.get_subscription(group_id, identity.key)
+
+        requested = build_identity(url)
+        matches = [
+            (repo_key, subscription)
+            for repo_key, subscription in self.state.list_group_subscriptions(group_id).items()
+            if subscription.url == requested.url
+        ]
+        if not matches:
+            return requested, None
+        if len(matches) > 1:
+            raise ValueError("本群关注了这个仓库的多个分支，请使用 --分支名 指定。")
+        _, subscription = matches[0]
+        return build_identity(subscription.url, subscription.branch), subscription
+
+    @staticmethod
+    def _explicit_branch(branch: str | None) -> str | None:
+        if branch is None:
+            return None
+        return normalize_branch(branch)
+
+
+def _now_iso(timezone_name: str | None = None) -> str:
+    if timezone_name:
+        try:
+            return datetime.now(ZoneInfo(timezone_name)).isoformat(timespec="seconds")
+        except ZoneInfoNotFoundError:
+            logger.warning(f"git poller fell back to local timezone for timestamps: {timezone_name!r}")
+    return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")

nonebot_plugin_git_poller/models.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class RepositoryIdentity:
+    key: str
+    url: str
+    display_name: str
+    web_url: str | None
+
+
+@dataclass
+class Subscription:
+    url: str
+    branch: str
+    schedule: str
+    last_success_sha: str | None = None
+    last_archive_path: str | None = None
+    archive_password: str | None = None
+    enabled: bool = True
+    created_at: str = ""
+    updated_at: str = ""
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "url": self.url,
+            "branch": self.branch,
+            "schedule": self.schedule,
+            "last_success_sha": self.last_success_sha,
+            "last_archive_path": self.last_archive_path,
+            "archive_password": self.archive_password,
+            "enabled": self.enabled,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+        }
+
+    @classmethod
+    def from_json(cls, data: dict[str, Any]) -> "Subscription":
+        return cls(
+            url=str(data["url"]),
+            branch=str(data.get("branch") or "main"),
+            schedule=str(data.get("schedule") or ""),
+            last_success_sha=(
+                str(data["last_success_sha"])
+                if data.get("last_success_sha")
+                else None
+            ),
+            last_archive_path=(
+                str(data["last_archive_path"])
+                if data.get("last_archive_path")
+                else None
+            ),
+            archive_password=(
+                str(data["archive_password"])
+                if data.get("archive_password")
+                else None
+            ),
+            enabled=bool(data.get("enabled", True)),
+            created_at=str(data.get("created_at") or ""),
+            updated_at=str(data.get("updated_at") or ""),
+        )
+
+
+@dataclass(frozen=True)
+class CommitInfo:
+    sha: str
+    short_sha: str
+    title: str
+    committed_at: str
+    author: str
+    url: str | None = None
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "sha": self.sha,
+            "short_sha": self.short_sha,
+            "title": self.title,
+            "committed_at": self.committed_at,
+            "author": self.author,
+            "url": self.url,
+        }
+
+    @classmethod
+    def from_json(cls, data: dict[str, Any]) -> "CommitInfo":
+        sha = str(data["sha"])
+        return cls(
+            sha=sha,
+            short_sha=str(data.get("short_sha") or sha[:8]),
+            title=str(data.get("title") or sha[:8]).strip(),
+            committed_at=str(data.get("committed_at") or ""),
+            author=str(data.get("author") or ""),
+            url=str(data["url"]) if data.get("url") else None,
+        )
+
+
+@dataclass(frozen=True)
+class UpdatePayload:
+    repo_key: str
+    repo_url: str
+    repo_name: str
+    branch: str
+    previous_sha: str | None
+    target_sha: str
+    target_short_sha: str
+    generated_at: str
+    commits: list[CommitInfo]
+    compare_url: str | None = None
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "repo_key": self.repo_key,
+            "repo_url": self.repo_url,
+            "repo_name": self.repo_name,
+            "branch": self.branch,
+            "previous_sha": self.previous_sha,
+            "target_sha": self.target_sha,
+            "target_short_sha": self.target_short_sha,
+            "generated_at": self.generated_at,
+            "commits": [commit.to_json() for commit in self.commits],
+            "compare_url": self.compare_url,
+        }
+
+    @classmethod
+    def from_json(cls, data: dict[str, Any]) -> "UpdatePayload":
+        return cls(
+            repo_key=str(data["repo_key"]),
+            repo_url=str(data["repo_url"]),
+            repo_name=str(data["repo_name"]),
+            branch=str(data.get("branch") or "main"),
+            previous_sha=(
+                str(data["previous_sha"]) if data.get("previous_sha") else None
+            ),
+            target_sha=str(data["target_sha"]),
+            target_short_sha=str(data.get("target_short_sha") or str(data["target_sha"])[:8]),
+            generated_at=str(data.get("generated_at") or ""),
+            commits=[
+                CommitInfo.from_json(item)
+                for item in data.get("commits", [])
+                if isinstance(item, dict)
+            ],
+            compare_url=str(data["compare_url"]) if data.get("compare_url") else None,
+        )
+
+
+@dataclass(frozen=True)
+class PollResult:
+    group_id: int
+    repo_key: str
+    subscription: Subscription
+    payload: UpdatePayload