github-backup 0.62.1__tar.gz → 0.64.0__tar.gz

{github_backup-0.62.1 → github_backup-0.64.0}/CHANGES.rst

@@ -1,9 +1,135 @@
 Changelog
 =========
 
-0.62.1 (2026-04-30)
+0.64.0 (2026-07-07)
 -------------------
 ------------------------
+
+Fix
+~~~
+- Drain subprocess pipes with reader threads to prevent Windows deadlock
+  (#519) [Rodos]
+
+  On Windows the child's stdout/stderr pipes were never drained, so any
+  git operation producing more output than the pipe buffer (~8KB) hung
+  the backup at the first sizeable clone or fetch. Reader threads now
+  drain both pipes on all platforms, replacing the select() loop.
+
+  - Windows no longer hangs and now gets subprocess logging
+  - Log lines no longer carry a stray \r on Windows, and a final line
+    without a trailing newline is no longer truncated
+  - CPU no longer pinned at 100% during git operations on Windows
+
+  Adds tests covering the deadlock, log levels, and line handling.
+
+  Fixes #519
+
+Other
+~~~~~
+- Applied diff issuecomment-4900281663. [Peter Gabaldon]
+- Fix template URL for watched repositories backup. [Peter Gabaldon]
+- Chore(deps): bump setuptools in the python-packages group.
+  [dependabot[bot]]
+
+  Bumps the python-packages group with 1 update: [setuptools](https://github.com/pypa/setuptools).
+
+
+  Updates `setuptools` from 82.0.1 to 83.0.0
+  - [Release notes](https://github.com/pypa/setuptools/releases)
+  - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst)
+  - [Commits](https://github.com/pypa/setuptools/compare/v82.0.1...v83.0.0)
+
+  ---
+  updated-dependencies:
+  - dependency-name: setuptools
+    dependency-version: 83.0.0
+    dependency-type: direct:production
+    update-type: version-update:semver-major
+    dependency-group: python-packages
+  ...
+- Chore(deps): bump actions/checkout from 6 to 7. [dependabot[bot]]
+
+  Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7.
+  - [Release notes](https://github.com/actions/checkout/releases)
+  - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
+  - [Commits](https://github.com/actions/checkout/compare/v6...v7)
+
+  ---
+  updated-dependencies:
+  - dependency-name: actions/checkout
+    dependency-version: '7'
+    dependency-type: direct:production
+    update-type: version-update:semver-major
+  ...
+- Chore(deps): bump pytest in the python-packages group.
+  [dependabot[bot]]
+
+  Bumps the python-packages group with 1 update: [pytest](https://github.com/pytest-dev/pytest).
+
+
+  Updates `pytest` from 9.1.0 to 9.1.1
+  - [Release notes](https://github.com/pytest-dev/pytest/releases)
+  - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
+  - [Commits](https://github.com/pytest-dev/pytest/compare/9.1.0...9.1.1)
+
+  ---
+  updated-dependencies:
+  - dependency-name: pytest
+    dependency-version: 9.1.1
+    dependency-type: direct:production
+    update-type: version-update:semver-patch
+    dependency-group: python-packages
+  ...
+
+
+0.63.0 (2026-06-17)
+-------------------
+- Skip repositories with pull requests disabled (HTTP 404) instead of
+  crashing (#511) [Rodos]
+
+  Also harden the advisories/hooks 404 checks to match HTTPError.code instead of a substring.
+- Chore(deps): bump pytest in the python-packages group.
+  [dependabot[bot]]
+
+  Bumps the python-packages group with 1 update: [pytest](https://github.com/pytest-dev/pytest).
+
+
+  Updates `pytest` from 9.0.3 to 9.1.0
+  - [Release notes](https://github.com/pytest-dev/pytest/releases)
+  - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
+  - [Commits](https://github.com/pytest-dev/pytest/compare/9.0.3...9.1.0)
+
+  ---
+  updated-dependencies:
+  - dependency-name: pytest
+    dependency-version: 9.1.0
+    dependency-type: direct:production
+    update-type: version-update:semver-minor
+    dependency-group: python-packages
+  ...
+- Chore(deps): bump black in the python-packages group.
+  [dependabot[bot]]
+
+  Bumps the python-packages group with 1 update: [black](https://github.com/psf/black).
+
+
+  Updates `black` from 26.3.1 to 26.5.1
+  - [Release notes](https://github.com/psf/black/releases)
+  - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
+  - [Commits](https://github.com/psf/black/compare/26.3.1...26.5.1)
+
+  ---
+  updated-dependencies:
+  - dependency-name: black
+    dependency-version: 26.5.1
+    dependency-type: direct:production
+    update-type: version-update:semver-minor
+    dependency-group: python-packages
+  ...
+
+
+0.62.1 (2026-04-30)
+-------------------
 - Document that nothing is saved by default. [Changaco]
 - Eliminate trailing spaces. [Changaco]
 - Remove pointless and unsafe `export`s in examples. [Changaco]

{github_backup-0.62.1/github_backup.egg-info → github_backup-0.64.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: github-backup
-Version: 0.62.1
+Version: 0.64.0
 Summary: backup a github user or organization
 Home-page: http://github.com/josegonzalez/python-github-backup
 Author: Jose Diaz-Gonzalez

github_backup-0.64.0/github_backup/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.64.0"

{github_backup-0.62.1 → github_backup-0.64.0}/github_backup/cli.py

@@ -81,7 +81,7 @@ def main():
     repositories = retrieve_repositories(args, authenticated_user)
     repositories = filter_repositories(args, repositories)
     backup_repositories(args, output_directory, repositories)
-    backup_account(args, output_directory)
+    backup_account(args, output_directory, authenticated_user)
 
 
 if __name__ == "__main__":

{github_backup-0.62.1 → github_backup-0.64.0}/github_backup/github_backup.py

@@ -12,11 +12,11 @@ import os
 import platform
 import random
 import re
-import select
 import socket
 import ssl
 import subprocess
 import sys
+import threading
 import time
 from collections.abc import Generator
 from datetime import datetime
@@ -91,32 +91,37 @@ def logging_subprocess(
     child = subprocess.Popen(
         popenargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs
     )
-    if sys.platform == "win32":
-        logger.info(
-            "Windows operating system detected - no subprocess logging will be returned"
-        )
-
-    log_level = {child.stdout: stdout_log_level, child.stderr: stderr_log_level}
-
-    def check_io():
-        if sys.platform == "win32":
-            return
-        ready_to_read = select.select([child.stdout, child.stderr], [], [], 1000)[0]
-        for io in ready_to_read:
-            line = io.readline()
-            if not logger:
-                continue
-            if not (io == child.stderr and not line):
-                logger.log(log_level[io], line[:-1])
-
-    # keep checking stdout/stderr until the child exits
-    while child.poll() is None:
-        check_io()
 
-    check_io()  # check again to catch anything after the process exits
+    def log_output(pipe, log_level):
+        # Drain the pipe from a thread so the child never blocks on a full
+        # pipe buffer (issue #519), logging each line as it arrives.
+        with pipe:
+            for line in iter(pipe.readline, b""):
+                try:
+                    logger.log(log_level, line.rstrip(b"\r\n"))
+                except Exception:
+                    # Keep draining even if logging fails, or the child
+                    # blocks on a full pipe buffer again
+                    pass
+
+    threads = [
+        threading.Thread(
+            target=log_output, args=(child.stdout, stdout_log_level), daemon=True
+        ),
+        threading.Thread(
+            target=log_output, args=(child.stderr, stderr_log_level), daemon=True
+        ),
+    ]
+    for thread in threads:
+        thread.start()
 
     rc = child.wait()
 
+    # Timeout in case a grandchild inherited the pipe handles and keeps
+    # them open past the child's exit, which would delay EOF indefinitely
+    for thread in threads:
+        thread.join(timeout=60)
+
     if rc != 0:
         print("{} returned {}:".format(popenargs[0], rc), file=sys.stderr)
         print("\t", " ".join(popenargs), file=sys.stderr)
@@ -2657,10 +2662,20 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     def pull_is_due_for_repository_checkpoint(pull):
         return not repository_since or pull["updated_at"] > repository_since
 
-    if not args.include_pull_details:
-        pull_states = ["open", "closed"]
-        for pull_state in pull_states:
-            query_args["state"] = pull_state
+    try:
+        if not args.include_pull_details:
+            pull_states = ["open", "closed"]
+            for pull_state in pull_states:
+                query_args["state"] = pull_state
+                for pull in retrieve_data(
+                    args, _pulls_template, query_args=query_args, lazy=True
+                ):
+                    track_newest_pull_update(pull)
+                    if pulls_since and pull["updated_at"] <= pulls_since:
+                        break
+                    if not pulls_since or pull["updated_at"] > pulls_since:
+                        pulls[pull["number"]] = pull
+        else:
             for pull in retrieve_data(
                 args, _pulls_template, query_args=query_args, lazy=True
             ):
@@ -2669,22 +2684,29 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
                     break
                 if not pulls_since or pull["updated_at"] > pulls_since:
                     pulls[pull["number"]] = pull
-    else:
-        for pull in retrieve_data(
-            args, _pulls_template, query_args=query_args, lazy=True
-        ):
-            track_newest_pull_update(pull)
-            if pulls_since and pull["updated_at"] <= pulls_since:
-                break
-            if not pulls_since or pull["updated_at"] > pulls_since:
-                if pull_is_due_for_repository_checkpoint(pull):
-                    pulls[pull["number"]] = retrieve_data(
-                        args,
-                        _pulls_template + "/{}".format(pull["number"]),
-                        paginated=False,
-                    )[0]
-                else:
-                    pulls[pull["number"]] = pull
+    except HTTPError as e:
+        # Repositories with pull requests disabled return HTTP 404 for the
+        # listing endpoint; treat that as "no pull requests" rather than a
+        # fatal error. retrieve_data is lazy here, so the 404 surfaces during
+        # iteration. Only the listing call is guarded — a 404 on an individual
+        # pull's detail fetch below is a different condition and must propagate.
+        if e.code == 404:
+            logger.info("Pull requests are disabled for this repository, skipping")
+            return
+        raise
+
+    if args.include_pull_details:
+        # Replace the summary payload with the full pull detail. Done outside
+        # the disabled-feature guard above so a 404 on a single pull (e.g. one
+        # deleted between listing and fetch) is not mistaken for the whole
+        # feature being disabled, which would discard every collected pull.
+        for number in list(pulls.keys()):
+            if pull_is_due_for_repository_checkpoint(pulls[number]):
+                pulls[number] = retrieve_data(
+                    args,
+                    _pulls_template + "/{}".format(number),
+                    paginated=False,
+                )[0]
 
     logger.info("Saving {0} pull requests to disk".format(len(list(pulls.keys()))))
     # Comments from pulls API are only _review_ comments
@@ -2813,8 +2835,8 @@ def backup_security_advisories(args, repo_cwd, repository, repos_template):
 
     try:
         _advisories = retrieve_data(args, template)
-    except Exception as e:
-        if "404" in str(e):
+    except HTTPError as e:
+        if e.code == 404:
             logger.info("Security advisories are not available for this repository, skipping")
             return
         raise
@@ -2861,8 +2883,8 @@ def backup_hooks(args, repo_cwd, repository, repos_template):
     template = "{0}/{1}/hooks".format(repos_template, repository["full_name"])
     try:
         _backup_data(args, "hooks", template, output_file, hook_cwd)
-    except Exception as e:
-        if "404" in str(e):
+    except HTTPError as e:
+        if e.code == 404:
             logger.info("Unable to read hooks, skipping")
         else:
             raise e
@@ -3046,8 +3068,10 @@ def fetch_repository(
                 logging_subprocess(git_command, cwd=local_dir)
 
 
-def backup_account(args, output_directory):
+def backup_account(args, output_directory, authenticated_user=None):
     account_cwd = os.path.join(output_directory, "account")
+    if authenticated_user is None:
+        authenticated_user = {"login": None}
 
     if args.include_starred or args.include_everything:
         output_file = "{0}/starred.json".format(account_cwd)
@@ -3057,11 +3081,25 @@ def backup_account(args, output_directory):
         _backup_data(args, "starred repositories", template, output_file, account_cwd)
 
     if args.include_watched or args.include_everything:
-        output_file = "{0}/watched.json".format(account_cwd)
-        template = "https://{0}/users/{1}/subscriptions".format(
-            get_github_api_host(args), args.user
-        )
-        _backup_data(args, "watched repositories", template, output_file, account_cwd)
+        # GitHub deprecated /users/{username}/subscriptions; watched
+        # repositories are only available for the authenticated user via
+        # /user/subscriptions.
+        if (
+            not authenticated_user.get("login")
+            or args.user.lower() != authenticated_user["login"].lower()
+        ):
+            logger.warning(
+                "Cannot retrieve watched repositories for '%s'. GitHub only allows access to the authenticated user's watched repositories.",
+                args.user,
+            )
+        else:
+            output_file = "{0}/watched.json".format(account_cwd)
+            template = "https://{0}/user/subscriptions".format(
+                get_github_api_host(args)
+            )
+            _backup_data(
+                args, "watched repositories", template, output_file, account_cwd
+            )
 
     if args.include_followers or args.include_everything:
         output_file = "{0}/followers.json".format(account_cwd)

{github_backup-0.62.1 → github_backup-0.64.0/github_backup.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: github-backup
-Version: 0.62.1
+Version: 0.64.0
 Summary: backup a github user or organization
 Home-page: http://github.com/josegonzalez/python-github-backup
 Author: Jose Diaz-Gonzalez

{github_backup-0.62.1 → github_backup-0.64.0}/github_backup.egg-info/SOURCES.txt

@@ -21,10 +21,12 @@ tests/test_all_starred.py
 tests/test_attachments.py
 tests/test_auth.py
 tests/test_case_sensitivity.py
+tests/test_disabled_issues_pulls.py
 tests/test_discussions.py
 tests/test_http_451.py
 tests/test_incremental_per_repository.py
 tests/test_json_dump_if_changed.py
+tests/test_logging_subprocess.py
 tests/test_pagination.py
 tests/test_pull_incremental_pagination.py
 tests/test_pull_reviews.py

{github_backup-0.62.1 → github_backup-0.64.0}/release-requirements.txt

@@ -1,15 +1,15 @@
 # Linting & Formatting
 autopep8==2.3.2
-black==26.3.1
+black==26.5.1
 flake8==7.3.0
 
 # Testing
-pytest==9.0.3
+pytest==9.1.1
 
 # Release & Publishing
 twine==6.2.0
 gitchangelog==3.0.4
-setuptools==82.0.1
+setuptools==83.0.0
 
 # Documentation
 restructuredtext-lint==2.0.2

github_backup-0.64.0/tests/test_disabled_issues_pulls.py

@@ -0,0 +1,105 @@
+"""Tests for repositories with pull requests disabled (GitHub issue #511).
+
+When pull requests are disabled on a repository, the ``/pulls`` endpoint
+responds with HTTP 404. The backup should treat that as "no pull requests"
+and continue, rather than aborting the whole run. (The ``/issues`` endpoint
+returns 200 with an empty list when issues are disabled, so it needs no such
+handling.)
+"""
+
+import logging
+from urllib.error import HTTPError
+
+from github_backup import github_backup
+
+
+def _http_error(template, code, reason):
+    return HTTPError(template, code, reason, hdrs=None, fp=None)
+
+
+def test_backup_pulls_skips_on_404(create_args, tmp_path, monkeypatch, caplog):
+    args = create_args(include_pulls=True, since=None)
+    repository = {"full_name": "owner/repo"}
+
+    def fake_retrieve_data(passed_args, template, query_args=None, lazy=False, **kwargs):
+        # retrieve_data is called lazily for the pull listing; the 404 must
+        # surface while iterating the returned generator.
+        def gen():
+            raise _http_error(template, 404, "Not Found")
+            yield  # pragma: no cover - generator marker
+
+        if lazy:
+            return gen()
+        raise _http_error(template, 404, "Not Found")
+
+    monkeypatch.setattr(github_backup, "retrieve_data", fake_retrieve_data)
+
+    # Should not raise, and should log that it skipped.
+    with caplog.at_level(logging.INFO, logger=github_backup.logger.name):
+        github_backup.backup_pulls(
+            args, tmp_path, repository, "https://api.github.com/repos"
+        )
+
+    assert "Pull requests are disabled for this repository, skipping" in caplog.text
+
+    pulls_dir = tmp_path / "pulls"
+    if pulls_dir.exists():
+        assert not [p for p in pulls_dir.iterdir() if p.suffix == ".json"]
+
+
+def test_backup_pulls_reraises_non_404(create_args, tmp_path, monkeypatch):
+    args = create_args(include_pulls=True, since=None)
+    repository = {"full_name": "owner/repo"}
+
+    def fake_retrieve_data(passed_args, template, query_args=None, lazy=False, **kwargs):
+        def gen():
+            raise _http_error(template, 500, "Server Error")
+            yield  # pragma: no cover - generator marker
+
+        if lazy:
+            return gen()
+        raise _http_error(template, 500, "Server Error")
+
+    monkeypatch.setattr(github_backup, "retrieve_data", fake_retrieve_data)
+
+    try:
+        github_backup.backup_pulls(
+            args, tmp_path, repository, "https://api.github.com/repos"
+        )
+    except HTTPError as exc:
+        assert exc.code == 500
+    else:
+        raise AssertionError("Expected non-404 error to propagate")
+
+
+def test_backup_pulls_details_individual_404_propagates(
+    create_args, tmp_path, monkeypatch
+):
+    # With --pull-details the listing succeeds (feature is enabled), but an
+    # individual pull's detail fetch 404s (e.g. deleted mid-backup). That must
+    # propagate, NOT be swallowed as "pull requests are disabled".
+    args = create_args(include_pulls=True, include_pull_details=True, since=None)
+    repository = {"full_name": "owner/repo"}
+
+    listing = "https://api.github.com/repos/owner/repo/pulls"
+
+    def fake_retrieve_data(passed_args, template, query_args=None, lazy=False, **kwargs):
+        if template == listing:
+            # Listing works fine -> pull requests are clearly not disabled.
+            def gen():
+                yield {"number": 1, "updated_at": "2026-02-01T00:00:00Z"}
+
+            return gen()
+        # Individual pull detail fetch (.../pulls/1) is missing.
+        raise _http_error(template, 404, "Not Found")
+
+    monkeypatch.setattr(github_backup, "retrieve_data", fake_retrieve_data)
+
+    try:
+        github_backup.backup_pulls(
+            args, tmp_path, repository, "https://api.github.com/repos"
+        )
+    except HTTPError as exc:
+        assert exc.code == 404
+    else:
+        raise AssertionError("Expected individual-pull 404 to propagate")

github_backup-0.64.0/tests/test_logging_subprocess.py

@@ -0,0 +1,113 @@
+"""Tests for logging_subprocess pipe handling (issue #519)."""
+
+import logging
+import sys
+import threading
+
+import pytest
+
+from github_backup import github_backup
+
+
+class TestLoggingSubprocess:
+    """Test suite for logging_subprocess deadlock and logging behavior."""
+
+    def test_large_stderr_output_does_not_deadlock(self):
+        """Child output larger than the OS pipe buffer must not hang.
+
+        Regression test for issue #519: on Windows the pipes were never
+        drained, so the child blocked once its output exceeded the pipe
+        buffer (~8KB) and the parent spun forever on poll().
+        """
+        # Write 256KB to stderr, far past any platform's pipe buffer
+        child_code = (
+            "import sys\n"
+            "for _ in range(3200):\n"
+            "    sys.stderr.write('x' * 79 + '\\n')\n"
+        )
+        result = {}
+
+        def run():
+            result["rc"] = github_backup.logging_subprocess(
+                [sys.executable, "-c", child_code]
+            )
+
+        thread = threading.Thread(target=run, daemon=True)
+        thread.start()
+        thread.join(timeout=30)
+
+        assert not thread.is_alive(), "logging_subprocess deadlocked on large output"
+        assert result["rc"] == 0
+
+    def test_stdout_logged_at_debug_stderr_at_error(self, caplog):
+        """stdout lines log at DEBUG and stderr lines at ERROR."""
+        child_code = (
+            "import sys\n"
+            "print('to stdout')\n"
+            "print('to stderr', file=sys.stderr)\n"
+        )
+
+        with caplog.at_level(logging.DEBUG, logger="github_backup.github_backup"):
+            rc = github_backup.logging_subprocess([sys.executable, "-c", child_code])
+
+        assert rc == 0
+        records = [
+            (r.levelno, r.getMessage())
+            for r in caplog.records
+            if r.name == "github_backup.github_backup"
+        ]
+        assert (logging.DEBUG, str(b"to stdout")) in records
+        assert (logging.ERROR, str(b"to stderr")) in records
+
+    def test_trailing_newlines_stripped(self, caplog):
+        """Logged lines have trailing \\r\\n stripped, including Windows CRLF."""
+        child_code = (
+            "import sys\n"
+            "sys.stdout.buffer.write(b'crlf line\\r\\n')\n"
+            "sys.stdout.buffer.write(b'lf line\\n')\n"
+        )
+
+        with caplog.at_level(logging.DEBUG, logger="github_backup.github_backup"):
+            rc = github_backup.logging_subprocess([sys.executable, "-c", child_code])
+
+        assert rc == 0
+        messages = [
+            r.getMessage()
+            for r in caplog.records
+            if r.name == "github_backup.github_backup"
+        ]
+        assert str(b"crlf line") in messages
+        assert str(b"lf line") in messages
+
+    def test_final_line_without_newline_not_truncated(self, caplog):
+        """A final line with no trailing newline keeps its last character.
+
+        The old code stripped the newline with line[:-1], which chopped the
+        last character off any line that did not end with a newline.
+        """
+        child_code = "import sys\nsys.stdout.write('no newline')\n"
+
+        with caplog.at_level(logging.DEBUG, logger="github_backup.github_backup"):
+            rc = github_backup.logging_subprocess([sys.executable, "-c", child_code])
+
+        assert rc == 0
+        messages = [
+            r.getMessage()
+            for r in caplog.records
+            if r.name == "github_backup.github_backup"
+        ]
+        assert str(b"no newline") in messages
+
+    def test_returns_child_exit_code(self, capsys):
+        """Non-zero child exit codes are returned and summarized on stderr."""
+        rc = github_backup.logging_subprocess(
+            [sys.executable, "-c", "import sys; sys.exit(3)"]
+        )
+
+        assert rc == 3
+        captured = capsys.readouterr()
+        assert "returned 3" in captured.err
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

github_backup-0.62.1/github_backup/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.62.1"