invenio-github 3.0.0__tar.gz → 3.0.2__tar.gz

{invenio_github-3.0.0 → invenio_github-3.0.2}/AUTHORS.rst

@@ -1,6 +1,6 @@
 ..
     This file is part of Invenio.
-    Copyright (C) 2016 CERN.
+    Copyright (C) 2016-2025 CERN.
 
     Invenio is free software; you can redistribute it
     and/or modify it under the terms of the GNU General Public License as
@@ -35,3 +35,4 @@ Invenio module that adds GitHub integration to the platform.
 - Jiri Kuncar <jiri.kuncar@cern.ch>
 - Lars Holm Nielsen <lars.holm.nielsen@cern.ch>
 - Tibor Simko <tibor.simko@cern.ch>
+- Till Korten <t.korten@hzdr.de>

{invenio_github-3.0.0 → invenio_github-3.0.2}/CHANGES.rst

@@ -26,6 +26,17 @@
 Changes
 =======
 
+Version v3.0.2 (released 2025-11-26)
+
+- api: optimize sync process with batch task execution
+    * prevent GitHub page timeout with 504 after sync
+    * optimize sync process with batch task execution
+    * add tests for sync() and sync_repo_hooks() methods
+
+Version v3.0.1 (released 2025-07-30)
+
+- api: fix set alternate zipball URL when tag and branch having same name
+
 Version 3.0.0 (release 2025-06-03)
 
 - setup: bump major dependencies

{invenio_github-3.0.0 → invenio_github-3.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: invenio-github
-Version: 3.0.0
+Version: 3.0.2
 Summary: "Invenio module that adds GitHub integration to the platform."
 Home-page: https://github.com/inveniosoftware/invenio-github
 Author: CERN
@@ -39,7 +39,6 @@ Requires-Dist: invenio-db[mysql,postgresql]<3.0.0,>=2.0.0; extra == "tests"
 Requires-Dist: invenio-files-rest<4.0.0,>=3.0.0; extra == "tests"
 Requires-Dist: isort>=4.2.2; extra == "tests"
 Requires-Dist: mock>=2.0.0; extra == "tests"
-Requires-Dist: pluggy<1.0,>=0.12; extra == "tests"
 Requires-Dist: pytest-black-ng>=0.4.0; extra == "tests"
 Requires-Dist: pytest-invenio<4.0.0,>=3.0.0; extra == "tests"
 Requires-Dist: pytest-mock>=2.0.0; extra == "tests"
@@ -130,6 +129,17 @@ Invenio module that adds GitHub integration to the platform.
 Changes
 =======
 
+Version v3.0.2 (released 2025-11-26)
+
+- api: optimize sync process with batch task execution
+    * prevent GitHub page timeout with 504 after sync
+    * optimize sync process with batch task execution
+    * add tests for sync() and sync_repo_hooks() methods
+
+Version v3.0.1 (released 2025-07-30)
+
+- api: fix set alternate zipball URL when tag and branch having same name
+
 Version 3.0.0 (release 2025-06-03)
 
 - setup: bump major dependencies

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2023-2024 CERN.
+# Copyright (C) 2023-2025 CERN.
 # Copyright (C) 2024-2025 Graz University of Technology.
 #
 # Invenio is free software; you can redistribute it
@@ -27,6 +27,6 @@
 
 from .ext import InvenioGitHub
 
-__version__ = "3.0.0"
+__version__ = "3.0.2"
 
 __all__ = ("__version__", "InvenioGitHub")

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github/api.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2023 CERN.
+# Copyright (C) 2023-2025 CERN.
 # Copyright (C) 2024 KTH Royal Institute of Technology.
 #
 # Invenio is free software; you can redistribute it
@@ -32,7 +32,6 @@ from copy import deepcopy
 from urllib.parse import urlparse
 
 import github3
-import humanize
 import requests
 from flask import current_app
 from invenio_access.permissions import authenticated_user
@@ -53,6 +52,7 @@ from invenio_github.tasks import sync_hooks as sync_hooks_task
 from invenio_github.utils import iso_utcnow, parse_timestamp, utcnow
 
 from .errors import (
+    ReleaseZipballFetchError,
     RemoteAccountDataNotSet,
     RemoteAccountNotFound,
     RepositoryAccessError,
@@ -219,7 +219,7 @@ class GitHubAPI(object):
         Repository.query.filter(
             Repository.user_id == self.user_id,
             ~Repository.github_id.in_(github_repos.keys()),
-        ).update(dict(user_id=None, hook=None), synchronize_session=False)
+        ).update({"user_id": None, "hook": None}, synchronize_session=False)
 
         # Update repos and last sync
         self.account.extra_data.update(
@@ -246,7 +246,11 @@ class GitHubAPI(object):
         else:
             # If hooks will run asynchronously, we need to commit any changes done so far
             db.session.commit()
-            sync_hooks_task.delay(self.user_id, repos)
+            # Create a task for several batches of repositories to speed up the process
+            # and ensure that each task finishes within the time limit on Zenodo
+            batch_size = current_app.config["GITHUB_WEBHOOK_SYNC_BATCH_SIZE"]
+            for i in range(0, len(repos), batch_size):
+                sync_hooks_task.delay(self.user_id, repos[i : i + batch_size])
 
     def _valid_webhook(self, url):
         """Check if webhook url is valid.
@@ -482,6 +486,7 @@ class GitHubRelease(object):
     def __init__(self, release):
         """Constructor."""
         self.release_object = release
+        self._resolved_zipball_url = None
 
     @cached_property
     def record(self):
@@ -534,7 +539,7 @@ class GitHubRelease(object):
 
     @cached_property
     def release_zipball_url(self):
-        """Returns the release zipball url."""
+        """Returns the release zipball URL."""
         return self.release_payload["zipball_url"]
 
     @cached_property
@@ -599,32 +604,58 @@ class GitHubRelease(object):
         return True if not latest_release else False
 
     def test_zipball(self):
-        """Extract files to download from GitHub payload."""
-        zipball_url = self.release_payload["zipball_url"]
+        """Test if the zipball URL is accessible and return the resolved URL."""
+        return self.resolve_zipball_url()
 
-        # Execute a HEAD request to the zipball url to test the url.
-        response = self.gh.api.session.head(zipball_url, allow_redirects=True)
+    def resolve_zipball_url(self, cache=True):
+        """Resolve the zipball URL.
 
-        # In case where there is a tag and branch with the same name, we might
-        # get back a "300 Mutliple Choices" response, which requires fetching
-        # an "alternate" link.
+        This method will try to resolve the zipball URL by making a HEAD request,
+        handling the following edge cases:
+
+        - In the case of a 300 Multiple Choices response, which can happen when a tag
+          and branch have the same name, it will try to fetch an "alternate" link.
+        - If the access token does not have the required scopes/permissions to access
+          public links, it will fallback to a non-authenticated request.
+        """
+        if self._resolved_zipball_url and cache:
+            return self._resolved_zipball_url
+
+        url = self.release_zipball_url
+
+        # Execute a HEAD request to the zipball url to test if it is accessible.
+        response = self.gh.api.session.head(url, allow_redirects=True)
+
+        # In case where there is a tag and branch with the same name, we might get back
+        # a "300 Multiple Choices" response, which requires fetching an "alternate"
+        # link.
         if response.status_code == 300:
-            zipball_url = response.links.get("alternate", {}).get("url")
-            if zipball_url:
-                response = self.gh.api.session.head(zipball_url, allow_redirects=True)
-                # Another edge-case, is when the access token we have does not
-                # have the scopes/permissions to access public links. In that
-                # rare case we fallback to a non-authenticated request.
-                if response.status_code == 404:
-                    response = requests.head(zipball_url, allow_redirects=True)
-                    # If this response is successful we want to use the finally
-                    # resolved URL to fetch the ZIP from.
-                    if response.status_code == 200:
-                        zipball_url = response.url
-
-        assert (
-            response.status_code == 200
-        ), f"Could not retrieve archive from GitHub: {zipball_url}"
+            alternate_url = response.links.get("alternate", {}).get("url")
+            if alternate_url:
+                url = alternate_url  # Use the alternate URL
+                response = self.gh.api.session.head(url, allow_redirects=True)
+
+        # Another edge-case, is when the access token we have does not have the
+        # scopes/permissions to access public links. In that rare case we fallback to a
+        # non-authenticated request.
+        if response.status_code == 404:
+            current_app.logger.warning(
+                "GitHub zipball URL {url} not found, trying unauthenticated request.",
+                extra={"url": response.url},
+            )
+            response = requests.head(url, allow_redirects=True)
+            # If this response is successful we want to use the finally resolved URL to
+            # fetch the ZIP from.
+            if response.status_code == 200:
+                return response.url
+
+        if response.status_code != 200:
+            raise ReleaseZipballFetchError()
+
+        if cache:
+            self._resolved_zipball_url = response.url
+
+        return response.url
 
     # High level API
 
@@ -663,8 +694,9 @@ class GitHubRelease(object):
         """Fetch release zipball file using the current github session."""
         session = self.gh.api.session
         timeout = current_app.config.get("GITHUB_ZIPBALL_TIMEOUT", 300)
-        with session.get(self.release_zipball_url, stream=True, timeout=timeout) as s:
-            yield s.raw
+        zipball_url = self.resolve_zipball_url()
+        with session.get(zipball_url, stream=True, timeout=timeout) as resp:
+            yield resp.raw
 
     def publish(self):
         """Publish a GitHub release."""

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github/assets/semantic-ui/js/invenio_github/index.js

@@ -1,5 +1,5 @@
 // This file is part of InvenioGithub
-// Copyright (C) 2023 CERN.
+// Copyright (C) 2023-2025 CERN.
 //
 // Invenio Github is free software; you can redistribute it and/or modify it
 // under the terms of the MIT License; see LICENSE file for more details.
@@ -75,7 +75,7 @@ if (sync_button) {
             resultMessage,
             "positive",
             "checkmark",
-            "Repositories synced successfully. Please reload the page."
+            "Repositories synced successfully. Some items may still be updating, please reload the page in about a minute."
           );
           sync_button.classList.remove("disabled");
           setTimeout(function () {
@@ -95,7 +95,7 @@ if (sync_button) {
         }
       } catch (error) {
         loaderIcon.classList.remove("loading");
-        if(error.message === "timeout"){
+        if (error.message === "timeout") {
           addResultMessage(
             resultMessage,
             "warning",
@@ -110,7 +110,7 @@ if (sync_button) {
             "cancel",
             `There has been a problem: ${error}`
           );
-           setTimeout(function () {
+          setTimeout(function () {
             resultMessage.classList.add("hidden");
           }, 7000);
         }

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github/config.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2023 CERN.
+# Copyright (C) 2023-2025 CERN.
 #
 # Invenio is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -41,6 +41,16 @@ This configuration variable must be set explicitly. Example::
     context, doesn't work as expected.
 """
 
+GITHUB_WEBHOOK_SYNC_BATCH_SIZE = 20
+"""Number of repositories to be processed in a single batch when syncing hooks. 
+
+If the user has more than 20 repositories, multiple tasks will be created, 
+syncing them in parallel. Thereby the sync process should finish in a timely 
+manner and we avoid timeouts on platforms like Zenodo.
+
+Decrease this value if you experience task timeouts.
+"""
+
 GITHUB_SHARED_SECRET = "CHANGEME"
 """Shared secret between you and GitHub.
 

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github/errors.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # This file is part of Invenio.
-# Copyright (C) 2023 CERN.
+# Copyright (C) 2023-2025 CERN.
 # Copyright (C) 2024 KTH Royal Institute of Technology.
 #
 # Invenio is free software; you can redistribute it
@@ -153,3 +153,13 @@ class UnexpectedGithubResponse(GitHubError):
     def __init__(self, message=None):
         """Constructor."""
         super().__init__(message or self.message)
+
+
+class ReleaseZipballFetchError(GitHubError):
+    """Error fetching release zipball file."""
+
+    message = _("Error fetching release zipball file.")
+
+    def __init__(self, message=None):
+        """Constructor."""
+        super().__init__(message or self.message)

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github/views/github.py

@@ -155,7 +155,7 @@ def register_api_routes(blueprint):
         """
         try:
             github = GitHubAPI(user_id=current_user.id)
-            github.sync(async_hooks=False)
+            github.sync()
             db.session.commit()
         except Exception as exc:
             current_app.logger.exception(str(exc))
@@ -171,7 +171,7 @@ def register_api_routes(blueprint):
         try:
             github = GitHubAPI(user_id=current_user.id)
             github.init_account()
-            github.sync(async_hooks=False)
+            github.sync()
             db.session.commit()
         except Exception as exc:
             current_app.logger.exception(str(exc))

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: invenio-github
-Version: 3.0.0
+Version: 3.0.2
 Summary: "Invenio module that adds GitHub integration to the platform."
 Home-page: https://github.com/inveniosoftware/invenio-github
 Author: CERN
@@ -39,7 +39,6 @@ Requires-Dist: invenio-db[mysql,postgresql]<3.0.0,>=2.0.0; extra == "tests"
 Requires-Dist: invenio-files-rest<4.0.0,>=3.0.0; extra == "tests"
 Requires-Dist: isort>=4.2.2; extra == "tests"
 Requires-Dist: mock>=2.0.0; extra == "tests"
-Requires-Dist: pluggy<1.0,>=0.12; extra == "tests"
 Requires-Dist: pytest-black-ng>=0.4.0; extra == "tests"
 Requires-Dist: pytest-invenio<4.0.0,>=3.0.0; extra == "tests"
 Requires-Dist: pytest-mock>=2.0.0; extra == "tests"
@@ -130,6 +129,17 @@ Invenio module that adds GitHub integration to the platform.
 Changes
 =======
 
+Version v3.0.2 (released 2025-11-26)
+
+- api: optimize sync process with batch task execution
+    * prevent GitHub page timeout with 504 after sync
+    * optimize sync process with batch task execution
+    * add tests for sync() and sync_repo_hooks() methods
+
+Version v3.0.1 (released 2025-07-30)
+
+- api: fix set alternate zipball URL when tag and branch having same name
+
 Version 3.0.0 (release 2025-06-03)
 
 - setup: bump major dependencies

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github.egg-info/SOURCES.txt

@@ -18,22 +18,6 @@ run-i18n-tests.sh
 run-tests.sh
 setup.cfg
 setup.py
-.github/workflows/i18n-pull.yml
-.github/workflows/i18n-push.yml
-.github/workflows/pypi-publish.yml
-.github/workflows/tests.yml
-docs/Makefile
-docs/api.rst
-docs/authors.rst
-docs/changes.rst
-docs/conf.py
-docs/contributing.rst
-docs/index.rst
-docs/installation.rst
-docs/license.rst
-docs/make.bat
-docs/requirements.txt
-docs/usage.rst
 invenio_github/__init__.py
 invenio_github/api.py
 invenio_github/config.py

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github.egg-info/requires.txt

@@ -35,7 +35,6 @@ invenio-db[mysql,postgresql]<3.0.0,>=2.0.0
 invenio-files-rest<4.0.0,>=3.0.0
 isort>=4.2.2
 mock>=2.0.0
-pluggy<1.0,>=0.12
 pytest-black-ng>=0.4.0
 pytest-invenio<4.0.0,>=3.0.0
 pytest-mock>=2.0.0

{invenio_github-3.0.0 → invenio_github-3.0.2}/invenio_github.egg-info/top_level.txt

@@ -1,2 +1 @@
 invenio_github
-tests

{invenio_github-3.0.0 → invenio_github-3.0.2}/setup.cfg

@@ -47,7 +47,6 @@ tests =
 	invenio-files-rest>=3.0.0,<4.0.0
 	isort>=4.2.2
 	mock>=2.0.0
-	pluggy>=0.12,<1.0
 	pytest-black-ng>=0.4.0
 	pytest-invenio>=3.0.0,<4.0.0
 	pytest-mock>=2.0.0

invenio_github-3.0.0/.github/workflows/i18n-pull.yml

@@ -1,14 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2023 Graz University of Technology.
-#
-# Invenio is free software; you can redistribute it and/or modify
-# it under the terms of the MIT License; see LICENSE file for more details.
-
-name: i18n:pull translations
-on: workflow_dispatch # manually trigger
-
-jobs:
-  i18n-pull:
-    uses: inveniosoftware/invenio-i18n/.github/workflows/i18n-pull-base.yml@master
-    secrets: inherit

invenio_github-3.0.0/.github/workflows/i18n-push.yml

@@ -1,16 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2023 Graz University of Technology.
-#
-# Invenio is free software; you can redistribute it and/or modify
-# it under the terms of the MIT License; see LICENSE file for more details.
-
-name: i18n:push translations
-on: workflow_dispatch # manually trigger
-
-jobs:
-  i18n-extract:
-    uses: inveniosoftware/invenio-i18n/.github/workflows/i18n-push-base.yml@master
-    with:
-      extract-backend: true
-    secrets: inherit

invenio_github-3.0.0/.github/workflows/pypi-publish.yml

@@ -1,22 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of Invenio.
-# Copyright (C) 2020-21 CERN.
-# Copyright (C) 2024 Graz University of Technology.
-#
-# Invenio is free software; you can redistribute it and/or modify it
-# under the terms of the MIT License; see LICENSE file for more details
-
-name: Publish
-
-on:
-  push:
-    tags:
-      - v*
-
-jobs:
-  Publish:
-    uses: inveniosoftware/workflows/.github/workflows/pypi-publish.yml@master
-    secrets: inherit
-    with:
-      babel-compile-catalog: true

invenio_github-3.0.0/.github/workflows/tests.yml

@@ -1,31 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of Invenio.
-# Copyright (C) 2023-2024 Graz University of Technology.
-# Copyright (C) 2024 KTH Royal Institute of Technology.
-#
-# Invenio is free software; you can redistribute it and/or modify it
-# under the terms of the MIT License; see LICENSE file for more details.
-
-name: CI
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    - cron: "0 3 * * 6"
-  workflow_dispatch:
-    inputs:
-      reason:
-        description: "Reason"
-        required: false
-        default: "Manual trigger"
-
-jobs:
-  Tests:
-    uses: inveniosoftware/workflows/.github/workflows/tests-python.yml@master