github-backup 0.63.0__tar.gz → 0.64.1__tar.gz

{github_backup-0.63.0 → github_backup-0.64.1}/CHANGES.rst

@@ -1,9 +1,94 @@
 Changelog
 =========
 
-0.63.0 (2026-06-17)
+0.64.1 (2026-07-17)
 -------------------
 ------------------------
+- Fix exception when DMCA'd/empty gists were backed up. [Duncan Ogilvie]
+
+
+0.64.0 (2026-07-07)
+-------------------
+
+Fix
+~~~
+- Drain subprocess pipes with reader threads to prevent Windows deadlock
+  (#519) [Rodos]
+
+  On Windows the child's stdout/stderr pipes were never drained, so any
+  git operation producing more output than the pipe buffer (~8KB) hung
+  the backup at the first sizeable clone or fetch. Reader threads now
+  drain both pipes on all platforms, replacing the select() loop.
+
+  - Windows no longer hangs and now gets subprocess logging
+  - Log lines no longer carry a stray \r on Windows, and a final line
+    without a trailing newline is no longer truncated
+  - CPU no longer pinned at 100% during git operations on Windows
+
+  Adds tests covering the deadlock, log levels, and line handling.
+
+  Fixes #519
+
+Other
+~~~~~
+- Applied diff issuecomment-4900281663. [Peter Gabaldon]
+- Fix template URL for watched repositories backup. [Peter Gabaldon]
+- Chore(deps): bump setuptools in the python-packages group.
+  [dependabot[bot]]
+
+  Bumps the python-packages group with 1 update: [setuptools](https://github.com/pypa/setuptools).
+
+
+  Updates `setuptools` from 82.0.1 to 83.0.0
+  - [Release notes](https://github.com/pypa/setuptools/releases)
+  - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst)
+  - [Commits](https://github.com/pypa/setuptools/compare/v82.0.1...v83.0.0)
+
+  ---
+  updated-dependencies:
+  - dependency-name: setuptools
+    dependency-version: 83.0.0
+    dependency-type: direct:production
+    update-type: version-update:semver-major
+    dependency-group: python-packages
+  ...
+- Chore(deps): bump actions/checkout from 6 to 7. [dependabot[bot]]
+
+  Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7.
+  - [Release notes](https://github.com/actions/checkout/releases)
+  - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
+  - [Commits](https://github.com/actions/checkout/compare/v6...v7)
+
+  ---
+  updated-dependencies:
+  - dependency-name: actions/checkout
+    dependency-version: '7'
+    dependency-type: direct:production
+    update-type: version-update:semver-major
+  ...
+- Chore(deps): bump pytest in the python-packages group.
+  [dependabot[bot]]
+
+  Bumps the python-packages group with 1 update: [pytest](https://github.com/pytest-dev/pytest).
+
+
+  Updates `pytest` from 9.1.0 to 9.1.1
+  - [Release notes](https://github.com/pytest-dev/pytest/releases)
+  - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
+  - [Commits](https://github.com/pytest-dev/pytest/compare/9.1.0...9.1.1)
+
+  ---
+  updated-dependencies:
+  - dependency-name: pytest
+    dependency-version: 9.1.1
+    dependency-type: direct:production
+    update-type: version-update:semver-patch
+    dependency-group: python-packages
+  ...
+
+
+0.63.0 (2026-06-17)
+-------------------
 - Skip repositories with pull requests disabled (HTTP 404) instead of
   crashing (#511) [Rodos]
 

{github_backup-0.63.0/github_backup.egg-info → github_backup-0.64.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: github-backup
-Version: 0.63.0
+Version: 0.64.1
 Summary: backup a github user or organization
 Home-page: http://github.com/josegonzalez/python-github-backup
 Author: Jose Diaz-Gonzalez

github_backup-0.64.1/github_backup/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.64.1"

{github_backup-0.63.0 → github_backup-0.64.1}/github_backup/cli.py

@@ -81,7 +81,7 @@ def main():
     repositories = retrieve_repositories(args, authenticated_user)
     repositories = filter_repositories(args, repositories)
     backup_repositories(args, output_directory, repositories)
-    backup_account(args, output_directory)
+    backup_account(args, output_directory, authenticated_user)
 
 
 if __name__ == "__main__":

{github_backup-0.63.0 → github_backup-0.64.1}/github_backup/github_backup.py

@@ -12,11 +12,11 @@ import os
 import platform
 import random
 import re
-import select
 import socket
 import ssl
 import subprocess
 import sys
+import threading
 import time
 from collections.abc import Generator
 from datetime import datetime
@@ -91,32 +91,37 @@ def logging_subprocess(
     child = subprocess.Popen(
         popenargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs
     )
-    if sys.platform == "win32":
-        logger.info(
-            "Windows operating system detected - no subprocess logging will be returned"
-        )
-
-    log_level = {child.stdout: stdout_log_level, child.stderr: stderr_log_level}
-
-    def check_io():
-        if sys.platform == "win32":
-            return
-        ready_to_read = select.select([child.stdout, child.stderr], [], [], 1000)[0]
-        for io in ready_to_read:
-            line = io.readline()
-            if not logger:
-                continue
-            if not (io == child.stderr and not line):
-                logger.log(log_level[io], line[:-1])
 
-    # keep checking stdout/stderr until the child exits
-    while child.poll() is None:
-        check_io()
-
-    check_io()  # check again to catch anything after the process exits
+    def log_output(pipe, log_level):
+        # Drain the pipe from a thread so the child never blocks on a full
+        # pipe buffer (issue #519), logging each line as it arrives.
+        with pipe:
+            for line in iter(pipe.readline, b""):
+                try:
+                    logger.log(log_level, line.rstrip(b"\r\n"))
+                except Exception:
+                    # Keep draining even if logging fails, or the child
+                    # blocks on a full pipe buffer again
+                    pass
+
+    threads = [
+        threading.Thread(
+            target=log_output, args=(child.stdout, stdout_log_level), daemon=True
+        ),
+        threading.Thread(
+            target=log_output, args=(child.stderr, stderr_log_level), daemon=True
+        ),
+    ]
+    for thread in threads:
+        thread.start()
 
     rc = child.wait()
 
+    # Timeout in case a grandchild inherited the pipe handles and keeps
+    # them open past the child's exit, which would delay EOF indefinitely
+    for thread in threads:
+        thread.join(timeout=60)
+
     if rc != 0:
         print("{} returned {}:".format(popenargs[0], rc), file=sys.stderr)
         print("\t", " ".join(popenargs), file=sys.stderr)
@@ -2104,7 +2109,10 @@ def backup_repositories(args, output_directory, repositories):
             )
 
             if repository.get("is_gist"):
-                # dump gist information to a file as well
+                # dump gist information to a file as well; the clone may have
+                # been skipped (e.g. DMCA-blocked or empty gist), so make sure
+                # the directory exists
+                mkdir_p(repo_cwd)
                 output_file = "{0}/gist.json".format(repo_cwd)
                 with codecs.open(output_file, "w", encoding="utf-8") as f:
                     json_dump(repository, f)
@@ -3063,8 +3071,10 @@ def fetch_repository(
                 logging_subprocess(git_command, cwd=local_dir)
 
 
-def backup_account(args, output_directory):
+def backup_account(args, output_directory, authenticated_user=None):
     account_cwd = os.path.join(output_directory, "account")
+    if authenticated_user is None:
+        authenticated_user = {"login": None}
 
     if args.include_starred or args.include_everything:
         output_file = "{0}/starred.json".format(account_cwd)
@@ -3074,11 +3084,25 @@ def backup_account(args, output_directory):
         _backup_data(args, "starred repositories", template, output_file, account_cwd)
 
     if args.include_watched or args.include_everything:
-        output_file = "{0}/watched.json".format(account_cwd)
-        template = "https://{0}/users/{1}/subscriptions".format(
-            get_github_api_host(args), args.user
-        )
-        _backup_data(args, "watched repositories", template, output_file, account_cwd)
+        # GitHub deprecated /users/{username}/subscriptions; watched
+        # repositories are only available for the authenticated user via
+        # /user/subscriptions.
+        if (
+            not authenticated_user.get("login")
+            or args.user.lower() != authenticated_user["login"].lower()
+        ):
+            logger.warning(
+                "Cannot retrieve watched repositories for '%s'. GitHub only allows access to the authenticated user's watched repositories.",
+                args.user,
+            )
+        else:
+            output_file = "{0}/watched.json".format(account_cwd)
+            template = "https://{0}/user/subscriptions".format(
+                get_github_api_host(args)
+            )
+            _backup_data(
+                args, "watched repositories", template, output_file, account_cwd
+            )
 
     if args.include_followers or args.include_everything:
         output_file = "{0}/followers.json".format(account_cwd)

{github_backup-0.63.0 → github_backup-0.64.1/github_backup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: github-backup
-Version: 0.63.0
+Version: 0.64.1
 Summary: backup a github user or organization
 Home-page: http://github.com/josegonzalez/python-github-backup
 Author: Jose Diaz-Gonzalez

{github_backup-0.63.0 → github_backup-0.64.1}/github_backup.egg-info/SOURCES.txt

@@ -26,6 +26,7 @@ tests/test_discussions.py
 tests/test_http_451.py
 tests/test_incremental_per_repository.py
 tests/test_json_dump_if_changed.py
+tests/test_logging_subprocess.py
 tests/test_pagination.py
 tests/test_pull_incremental_pagination.py
 tests/test_pull_reviews.py

{github_backup-0.63.0 → github_backup-0.64.1}/release-requirements.txt

@@ -4,12 +4,12 @@ black==26.5.1
 flake8==7.3.0
 
 # Testing
-pytest==9.1.0
+pytest==9.1.1
 
 # Release & Publishing
 twine==6.2.0
 gitchangelog==3.0.4
-setuptools==82.0.1
+setuptools==83.0.0
 
 # Documentation
 restructuredtext-lint==2.0.2

github_backup-0.64.1/tests/test_logging_subprocess.py

@@ -0,0 +1,113 @@
+"""Tests for logging_subprocess pipe handling (issue #519)."""
+
+import logging
+import sys
+import threading
+
+import pytest
+
+from github_backup import github_backup
+
+
+class TestLoggingSubprocess:
+    """Test suite for logging_subprocess deadlock and logging behavior."""
+
+    def test_large_stderr_output_does_not_deadlock(self):
+        """Child output larger than the OS pipe buffer must not hang.
+
+        Regression test for issue #519: on Windows the pipes were never
+        drained, so the child blocked once its output exceeded the pipe
+        buffer (~8KB) and the parent spun forever on poll().
+        """
+        # Write 256KB to stderr, far past any platform's pipe buffer
+        child_code = (
+            "import sys\n"
+            "for _ in range(3200):\n"
+            "    sys.stderr.write('x' * 79 + '\\n')\n"
+        )
+        result = {}
+
+        def run():
+            result["rc"] = github_backup.logging_subprocess(
+                [sys.executable, "-c", child_code]
+            )
+
+        thread = threading.Thread(target=run, daemon=True)
+        thread.start()
+        thread.join(timeout=30)
+
+        assert not thread.is_alive(), "logging_subprocess deadlocked on large output"
+        assert result["rc"] == 0
+
+    def test_stdout_logged_at_debug_stderr_at_error(self, caplog):
+        """stdout lines log at DEBUG and stderr lines at ERROR."""
+        child_code = (
+            "import sys\n"
+            "print('to stdout')\n"
+            "print('to stderr', file=sys.stderr)\n"
+        )
+
+        with caplog.at_level(logging.DEBUG, logger="github_backup.github_backup"):
+            rc = github_backup.logging_subprocess([sys.executable, "-c", child_code])
+
+        assert rc == 0
+        records = [
+            (r.levelno, r.getMessage())
+            for r in caplog.records
+            if r.name == "github_backup.github_backup"
+        ]
+        assert (logging.DEBUG, str(b"to stdout")) in records
+        assert (logging.ERROR, str(b"to stderr")) in records
+
+    def test_trailing_newlines_stripped(self, caplog):
+        """Logged lines have trailing \\r\\n stripped, including Windows CRLF."""
+        child_code = (
+            "import sys\n"
+            "sys.stdout.buffer.write(b'crlf line\\r\\n')\n"
+            "sys.stdout.buffer.write(b'lf line\\n')\n"
+        )
+
+        with caplog.at_level(logging.DEBUG, logger="github_backup.github_backup"):
+            rc = github_backup.logging_subprocess([sys.executable, "-c", child_code])
+
+        assert rc == 0
+        messages = [
+            r.getMessage()
+            for r in caplog.records
+            if r.name == "github_backup.github_backup"
+        ]
+        assert str(b"crlf line") in messages
+        assert str(b"lf line") in messages
+
+    def test_final_line_without_newline_not_truncated(self, caplog):
+        """A final line with no trailing newline keeps its last character.
+
+        The old code stripped the newline with line[:-1], which chopped the
+        last character off any line that did not end with a newline.
+        """
+        child_code = "import sys\nsys.stdout.write('no newline')\n"
+
+        with caplog.at_level(logging.DEBUG, logger="github_backup.github_backup"):
+            rc = github_backup.logging_subprocess([sys.executable, "-c", child_code])
+
+        assert rc == 0
+        messages = [
+            r.getMessage()
+            for r in caplog.records
+            if r.name == "github_backup.github_backup"
+        ]
+        assert str(b"no newline") in messages
+
+    def test_returns_child_exit_code(self, capsys):
+        """Non-zero child exit codes are returned and summarized on stderr."""
+        rc = github_backup.logging_subprocess(
+            [sys.executable, "-c", "import sys; sys.exit(3)"]
+        )
+
+        assert rc == 3
+        captured = capsys.readouterr()
+        assert "returned 3" in captured.err
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

github_backup-0.63.0/github_backup/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.63.0"