github-backup 0.47.0__py3-none-any.whl → 0.49.0__py3-none-any.whl

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = "0.47.0"
+__version__ = "0.49.0"

github_backup/github_backup.py

@@ -19,12 +19,12 @@ import ssl
 import subprocess
 import sys
 import time
+from datetime import datetime
 from http.client import IncompleteRead
 from urllib.error import HTTPError, URLError
 from urllib.parse import quote as urlquote
 from urllib.parse import urlencode, urlparse
 from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen
-from datetime import datetime
 
 try:
     from . import __version__
@@ -40,16 +40,21 @@ logger = logging.getLogger(__name__)
 https_ctx = ssl.create_default_context()
 if not https_ctx.get_ca_certs():
     import warnings
-    warnings.warn('\n\nYOUR DEFAULT CA CERTS ARE EMPTY.\n' +
-                  'PLEASE POPULATE ANY OF:' +
-                  ''.join([
-                    '\n - ' + x
-                    for x in ssl.get_default_verify_paths()
-                    if type(x) is str
-                  ]) + '\n', stacklevel=2)
+
+    warnings.warn(
+        "\n\nYOUR DEFAULT CA CERTS ARE EMPTY.\n"
+        + "PLEASE POPULATE ANY OF:"
+        + "".join(
+            ["\n - " + x for x in ssl.get_default_verify_paths() if type(x) is str]
+        )
+        + "\n",
+        stacklevel=2,
+    )
     import certifi
+
     https_ctx = ssl.create_default_context(cafile=certifi.where())
 
+
 def logging_subprocess(
     popenargs, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs
 ):
@@ -176,6 +181,12 @@ def parse_args(args=None):
         dest="incremental",
         help="incremental backup",
     )
+    parser.add_argument(
+        "--incremental-by-files",
+        action="store_true",
+        dest="incremental_by_files",
+        help="incremental backup based on modification date of files",
+    )
     parser.add_argument(
         "--starred",
         action="store_true",
@@ -524,7 +535,7 @@ def get_github_host(args):
 
 
 def read_file_contents(file_uri):
-    return open(file_uri[len(FILE_URI_PREFIX):], "rt").readline().strip()
+    return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip()
 
 
 def get_github_repo_url(args, repository):
@@ -795,13 +806,15 @@ def download_file(url, path, auth, as_app=False, fine=False):
     if os.path.exists(path):
         return
 
-    request = _construct_request(per_page=100,
-                                 page=1,
-                                 query_args={},
-                                 template=url,
-                                 auth=auth,
-                                 as_app=as_app,
-                                 fine=fine)
+    request = _construct_request(
+        per_page=100,
+        page=1,
+        query_args={},
+        template=url,
+        auth=auth,
+        as_app=as_app,
+        fine=fine,
+    )
     request.add_header("Accept", "application/octet-stream")
     opener = build_opener(S3HTTPRedirectHandler)
 
@@ -944,11 +957,15 @@ def filter_repositories(args, unfiltered_repositories):
             if r.get("language") and r.get("language").lower() in languages
         ]  # noqa
     if name_regex:
-        repositories = [r for r in repositories if "name" not in r or name_regex.match(r["name"])]
+        repositories = [
+            r for r in repositories if "name" not in r or name_regex.match(r["name"])
+        ]
     if args.skip_archived:
         repositories = [r for r in repositories if not r.get("archived")]
     if args.exclude:
-        repositories = [r for r in repositories if "name" not in r or r["name"] not in args.exclude]
+        repositories = [
+            r for r in repositories if "name" not in r or r["name"] not in args.exclude
+        ]
 
     return repositories
 
@@ -1103,6 +1120,14 @@ def backup_issues(args, repo_cwd, repository, repos_template):
     comments_template = _issue_template + "/{0}/comments"
     events_template = _issue_template + "/{0}/events"
     for number, issue in list(issues.items()):
+        issue_file = "{0}/{1}.json".format(issue_cwd, number)
+        if args.incremental_by_files and os.path.isfile(issue_file):
+            modified = os.path.getmtime(issue_file)
+            modified = datetime.fromtimestamp(modified).strftime("%Y-%m-%dT%H:%M:%SZ")
+            if modified > issue["updated_at"]:
+                logger.info("Skipping issue {0} because it wasn't modified since last backup".format(number))
+                continue
+
         if args.include_issue_comments or args.include_everything:
             template = comments_template.format(number)
             issues[number]["comment_data"] = retrieve_data(args, template)
@@ -1110,9 +1135,9 @@ def backup_issues(args, repo_cwd, repository, repos_template):
             template = events_template.format(number)
             issues[number]["event_data"] = retrieve_data(args, template)
 
-        issue_file = "{0}/{1}.json".format(issue_cwd, number)
-        with codecs.open(issue_file, "w", encoding="utf-8") as f:
+        with codecs.open(issue_file + ".temp", "w", encoding="utf-8") as f:
             json_dump(issue, f)
+            os.rename(issue_file + ".temp", issue_file) # Unlike json_dump, this is atomic
 
 
 def backup_pulls(args, repo_cwd, repository, repos_template):
@@ -1165,6 +1190,13 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     comments_template = _pulls_template + "/{0}/comments"
     commits_template = _pulls_template + "/{0}/commits"
     for number, pull in list(pulls.items()):
+        pull_file = "{0}/{1}.json".format(pulls_cwd, number)
+        if args.incremental_by_files and os.path.isfile(pull_file):
+            modified = os.path.getmtime(pull_file)
+            modified = datetime.fromtimestamp(modified).strftime("%Y-%m-%dT%H:%M:%SZ")
+            if modified > pull["updated_at"]:
+                logger.info("Skipping pull request {0} because it wasn't modified since last backup".format(number))
+                continue
         if args.include_pull_comments or args.include_everything:
             template = comments_regular_template.format(number)
             pulls[number]["comment_regular_data"] = retrieve_data(args, template)
@@ -1174,9 +1206,9 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
             template = commits_template.format(number)
             pulls[number]["commit_data"] = retrieve_data(args, template)
 
-        pull_file = "{0}/{1}.json".format(pulls_cwd, number)
-        with codecs.open(pull_file, "w", encoding="utf-8") as f:
+        with codecs.open(pull_file + ".temp", "w", encoding="utf-8") as f:
             json_dump(pull, f)
+            os.rename(pull_file + ".temp", pull_file) # Unlike json_dump, this is atomic
 
 
 def backup_milestones(args, repo_cwd, repository, repos_template):
@@ -1244,10 +1276,16 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
     if args.skip_prerelease:
         releases = [r for r in releases if not r["prerelease"] and not r["draft"]]
 
-    if args.number_of_latest_releases and args.number_of_latest_releases < len(releases):
-        releases.sort(key=lambda item: datetime.strptime(item["created_at"], "%Y-%m-%dT%H:%M:%SZ"),
-                      reverse=True)
-        releases = releases[:args.number_of_latest_releases]
+    if args.number_of_latest_releases and args.number_of_latest_releases < len(
+        releases
+    ):
+        releases.sort(
+            key=lambda item: datetime.strptime(
+                item["created_at"], "%Y-%m-%dT%H:%M:%SZ"
+            ),
+            reverse=True,
+        )
+        releases = releases[: args.number_of_latest_releases]
         logger.info("Saving the latest {0} releases to disk".format(len(releases)))
     else:
         logger.info("Saving {0} releases to disk".format(len(releases)))
@@ -1274,7 +1312,7 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
                         os.path.join(release_assets_cwd, asset["name"]),
                         get_auth(args, encode=not args.as_app),
                         as_app=args.as_app,
-                        fine=True if args.token_fine is not None else False
+                        fine=True if args.token_fine is not None else False,
                     )
 
 

{github_backup-0.47.0.dist-info → github_backup-0.49.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: github-backup
-Version: 0.47.0
+Version: 0.49.0
 Summary: backup a github user or organization
 Home-page: http://github.com/josegonzalez/python-github-backup
 Author: Jose Diaz-Gonzalez
@@ -16,6 +16,14 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Description-Content-Type: text/x-rst
 License-File: LICENSE.txt
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: summary
 
 =============
 github-backup
@@ -99,6 +107,7 @@ CLI Help output::
                             log level to use (default: info, possible levels:
                             debug, info, warning, error, critical)
       -i, --incremental     incremental backup
+      --incremental-by-files incremental backup using modified time of files
       --starred             include JSON output of starred repositories in backup
       --all-starred         include starred repositories in backup [*]
       --watched             include JSON output of watched repositories in backup
@@ -186,7 +195,7 @@ Customise the permissions for your use case, but for a personal account full bac
 
 **User permissions**: Read access to followers, starring, and watching.
 
-**Repository permissions**: Read access to code, commit statuses, issues, metadata, pages, pull requests, and repository hooks.
+**Repository permissions**: Read access to contents, issues, metadata, pull requests, and webhooks.
 
 
 Prefer SSH
@@ -244,7 +253,7 @@ Gotchas / Known-issues
 All is not everything
 ---------------------
 
-The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--starred-gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
+The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
 
 Cloning all starred size
 ------------------------
@@ -258,6 +267,12 @@ Using (``-i, --incremental``) will only request new data from the API **since th
 
 This means any blocking errors on previous runs can cause a large amount of missing data in backups.
 
+Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
+
+Still saver than the previous version.
+
+Specifically, issues and pull requests are handled like this.
+
 Known blocking errors
 ---------------------
 
@@ -273,12 +288,6 @@ It's therefore recommended to only use the incremental argument if the output/re
 
    This is due to needing the correct permission for ``--hooks`` on public repos.
 
-2. **Releases blocking**
-
-   A known ``--releases`` (required for ``--assets``) error will sometimes block the backup. 
-
-   If you're backing up a lot of repositories with releases e.g. an organisation or ``--all-starred``. You may need to remove ``--releases`` (and therefore ``--assets``) to complete a backup. Documented in `issue 209 <https://github.com/josegonzalez/python-github-backup/issues/209>`_.
-
 
 "bare" is actually "mirror"
 ---------------------------

github_backup-0.49.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+github_backup/__init__.py,sha256=09FNUq8XbPsGZin0IRhNT7qwi97ZXQafNn7GixZiqAw,23
+github_backup/github_backup.py,sha256=TI4cb_oHwI6P58upMF1jTbC-1nquog7C4q2sCM9PqMA,49844
+github_backup-0.49.0.data/scripts/github-backup,sha256=I79TNA67phytRmI9nm_LlD9h2zauXFOsOn9tYyAy9pU,1546
+github_backup-0.49.0.dist-info/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
+github_backup-0.49.0.dist-info/METADATA,sha256=YfkUDfbWpZGTFN3wlThxbblaXVRMNFBLQD1jwpo9RSc,19380
+github_backup-0.49.0.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+github_backup-0.49.0.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
+github_backup-0.49.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+github_backup-0.49.0.dist-info/RECORD,,

{github_backup-0.47.0.dist-info → github_backup-0.49.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

github_backup-0.47.0.dist-info/RECORD

@@ -1,9 +0,0 @@
-github_backup/__init__.py,sha256=aqLqdmGXCjcE1UvvJF8OHznHKbGMeoRrk-C4_mjzdPs,23
-github_backup/github_backup.py,sha256=h6lu2QBhjx5yHrSA0ocQ3xga0GdJuj4yG92zJZYtz-0,48732
-github_backup-0.47.0.data/scripts/github-backup,sha256=I79TNA67phytRmI9nm_LlD9h2zauXFOsOn9tYyAy9pU,1546
-github_backup-0.47.0.dist-info/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
-github_backup-0.47.0.dist-info/METADATA,sha256=Yokav5VhHSroGqxzvkvXZSSPRrD4K4t9vGsCSt1H0Mg,19291
-github_backup-0.47.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-github_backup-0.47.0.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
-github_backup-0.47.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-github_backup-0.47.0.dist-info/RECORD,,