github-backup 0.64.1__py3-none-any.whl → 0.65.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- github_backup/__init__.py +1 -1
- github_backup/github_backup.py +268 -13
- github_backup/graphql_queries.py +60 -0
- {github_backup-0.64.1.dist-info → github_backup-0.65.0.dist-info}/METADATA +28 -4
- github_backup-0.65.0.dist-info/RECORD +12 -0
- github_backup-0.64.1.dist-info/RECORD +0 -12
- {github_backup-0.64.1.dist-info → github_backup-0.65.0.dist-info}/WHEEL +0 -0
- {github_backup-0.64.1.dist-info → github_backup-0.65.0.dist-info}/entry_points.txt +0 -0
- {github_backup-0.64.1.dist-info → github_backup-0.65.0.dist-info}/licenses/LICENSE.txt +0 -0
- {github_backup-0.64.1.dist-info → github_backup-0.65.0.dist-info}/top_level.txt +0 -0
- {github_backup-0.64.1.dist-info → github_backup-0.65.0.dist-info}/zip-safe +0 -0
github_backup/__init__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.
|
|
1
|
+
__version__ = "0.65.0"
|
github_backup/github_backup.py
CHANGED
|
@@ -18,6 +18,7 @@ import subprocess
|
|
|
18
18
|
import sys
|
|
19
19
|
import threading
|
|
20
20
|
import time
|
|
21
|
+
import traceback
|
|
21
22
|
from collections.abc import Generator
|
|
22
23
|
from datetime import datetime
|
|
23
24
|
from http.client import IncompleteRead
|
|
@@ -33,10 +34,14 @@ except ImportError:
|
|
|
33
34
|
VERSION = "unknown"
|
|
34
35
|
|
|
35
36
|
from .graphql_queries import (
|
|
37
|
+
CROSS_REFERENCE_NODE_LIMIT,
|
|
38
|
+
CROSS_REFERENCE_PAGE_SIZE,
|
|
36
39
|
DISCUSSION_DETAIL_QUERY,
|
|
37
40
|
DISCUSSION_LIST_QUERY,
|
|
38
41
|
DISCUSSION_PAGE_SIZE,
|
|
39
42
|
DISCUSSION_REPLIES_QUERY,
|
|
43
|
+
ISSUE_CROSS_REFERENCE_COUNT_QUERY,
|
|
44
|
+
PULL_CROSS_REFERENCE_COUNT_QUERY,
|
|
40
45
|
)
|
|
41
46
|
|
|
42
47
|
FILE_URI_PREFIX = "file://"
|
|
@@ -278,6 +283,12 @@ def parse_args(args=None):
|
|
|
278
283
|
dest="include_issue_events",
|
|
279
284
|
help="include issue events in backup",
|
|
280
285
|
)
|
|
286
|
+
parser.add_argument(
|
|
287
|
+
"--issue-timeline",
|
|
288
|
+
action="store_true",
|
|
289
|
+
dest="include_issue_timeline",
|
|
290
|
+
help="include issue timeline in backup (cross-references, commits and reviews; complements --issue-events)",
|
|
291
|
+
)
|
|
281
292
|
parser.add_argument(
|
|
282
293
|
"--pulls",
|
|
283
294
|
action="store_true",
|
|
@@ -1789,7 +1800,39 @@ def check_git_lfs_install():
|
|
|
1789
1800
|
)
|
|
1790
1801
|
|
|
1791
1802
|
|
|
1803
|
+
# Resources fetched from their own endpoints, not from the repository listing
|
|
1804
|
+
# (gists and account-level data). Every other include_* flag needs the listing.
|
|
1805
|
+
NON_REPOSITORY_RESOURCES = frozenset(
|
|
1806
|
+
{
|
|
1807
|
+
"include_gists",
|
|
1808
|
+
"include_starred_gists",
|
|
1809
|
+
"include_starred",
|
|
1810
|
+
"include_watched",
|
|
1811
|
+
"include_followers",
|
|
1812
|
+
"include_following",
|
|
1813
|
+
}
|
|
1814
|
+
)
|
|
1815
|
+
|
|
1816
|
+
|
|
1817
|
+
def repository_list_needed(args):
|
|
1818
|
+
"""The base repository listing is only needed when a per-repository
|
|
1819
|
+
resource was requested; e.g. a gists-only backup can skip it entirely.
|
|
1820
|
+
Flags not listed in NON_REPOSITORY_RESOURCES default to needing it, so a
|
|
1821
|
+
newly added resource can never be silently skipped."""
|
|
1822
|
+
if args.repository:
|
|
1823
|
+
return True
|
|
1824
|
+
return any(
|
|
1825
|
+
value
|
|
1826
|
+
for name, value in vars(args).items()
|
|
1827
|
+
if name.startswith("include_") and name not in NON_REPOSITORY_RESOURCES
|
|
1828
|
+
)
|
|
1829
|
+
|
|
1830
|
+
|
|
1792
1831
|
def retrieve_repositories(args, authenticated_user):
|
|
1832
|
+
if not repository_list_needed(args):
|
|
1833
|
+
repos = []
|
|
1834
|
+
return retrieve_additional_repositories(args, authenticated_user, repos)
|
|
1835
|
+
|
|
1793
1836
|
logger.info("Retrieving repositories")
|
|
1794
1837
|
paginated = True
|
|
1795
1838
|
if args.user == authenticated_user["login"]:
|
|
@@ -1825,6 +1868,12 @@ def retrieve_repositories(args, authenticated_user):
|
|
|
1825
1868
|
logger.warning(f"Legal notice: {e.legal_url}")
|
|
1826
1869
|
return []
|
|
1827
1870
|
|
|
1871
|
+
return retrieve_additional_repositories(args, authenticated_user, repos)
|
|
1872
|
+
|
|
1873
|
+
|
|
1874
|
+
def retrieve_additional_repositories(args, authenticated_user, repos):
|
|
1875
|
+
"""Append starred repositories, gists and starred gists to the base
|
|
1876
|
+
repository list when requested."""
|
|
1828
1877
|
if args.all_starred:
|
|
1829
1878
|
starred_template = "https://{0}/users/{1}/starred".format(
|
|
1830
1879
|
get_github_api_host(args), args.user
|
|
@@ -2058,6 +2107,24 @@ def remove_legacy_last_update_if_migrated(
|
|
|
2058
2107
|
)
|
|
2059
2108
|
|
|
2060
2109
|
|
|
2110
|
+
def gist_backup_is_current(repository, repo_cwd, repo_dir):
|
|
2111
|
+
"""Whether fetching a gist can be skipped because a clone exists and the
|
|
2112
|
+
stored gist.json matches the listing's updated_at. Any edit to a gist —
|
|
2113
|
+
including a git push — bumps updated_at, so this can never miss a content
|
|
2114
|
+
change; at worst a comment bumps it and causes a harmless refetch."""
|
|
2115
|
+
if not os.path.exists(repo_dir):
|
|
2116
|
+
return False
|
|
2117
|
+
updated_at = repository.get("updated_at")
|
|
2118
|
+
if not updated_at:
|
|
2119
|
+
return False
|
|
2120
|
+
try:
|
|
2121
|
+
with codecs.open(os.path.join(repo_cwd, "gist.json"), encoding="utf-8") as f:
|
|
2122
|
+
stored = json.load(f)
|
|
2123
|
+
except (OSError, ValueError):
|
|
2124
|
+
return False
|
|
2125
|
+
return stored.get("updated_at") == updated_at
|
|
2126
|
+
|
|
2127
|
+
|
|
2061
2128
|
def backup_repositories(args, output_directory, repositories):
|
|
2062
2129
|
logger.info("Backing up repositories")
|
|
2063
2130
|
repos_template = "https://{0}/repos".format(get_github_api_host(args))
|
|
@@ -2065,6 +2132,7 @@ def backup_repositories(args, output_directory, repositories):
|
|
|
2065
2132
|
args, output_directory
|
|
2066
2133
|
)
|
|
2067
2134
|
incremental_resource_work_attempted = False
|
|
2135
|
+
skipped_unchanged_gists = 0
|
|
2068
2136
|
|
|
2069
2137
|
for repository in repositories:
|
|
2070
2138
|
if repository.get("is_gist"):
|
|
@@ -2098,20 +2166,29 @@ def backup_repositories(args, output_directory, repositories):
|
|
|
2098
2166
|
if not repository.get("is_gist")
|
|
2099
2167
|
else repository.get("id")
|
|
2100
2168
|
)
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2169
|
+
if repository.get("is_gist") and gist_backup_is_current(
|
|
2170
|
+
repository, repo_cwd, repo_dir
|
|
2171
|
+
):
|
|
2172
|
+
logger.debug(
|
|
2173
|
+
"Skipping gist {0} (unchanged since last backup)".format(repo_name)
|
|
2174
|
+
)
|
|
2175
|
+
skipped_unchanged_gists += 1
|
|
2176
|
+
else:
|
|
2177
|
+
fetch_repository(
|
|
2178
|
+
repo_name,
|
|
2179
|
+
repo_url,
|
|
2180
|
+
repo_dir,
|
|
2181
|
+
skip_existing=args.skip_existing,
|
|
2182
|
+
bare_clone=args.bare_clone,
|
|
2183
|
+
lfs_clone=args.lfs_clone,
|
|
2184
|
+
no_prune=args.no_prune,
|
|
2185
|
+
)
|
|
2110
2186
|
|
|
2111
2187
|
if repository.get("is_gist"):
|
|
2112
|
-
# dump gist information to a file as well
|
|
2113
|
-
#
|
|
2114
|
-
# the
|
|
2188
|
+
# dump gist information to a file as well, even when the fetch
|
|
2189
|
+
# was skipped, so listing metadata (fork counts etc.) stays
|
|
2190
|
+
# fresh; the clone may also have been skipped entirely (e.g.
|
|
2191
|
+
# DMCA-blocked or empty gist), so make sure the directory exists
|
|
2115
2192
|
mkdir_p(repo_cwd)
|
|
2116
2193
|
output_file = "{0}/gist.json".format(repo_cwd)
|
|
2117
2194
|
with codecs.open(output_file, "w", encoding="utf-8") as f:
|
|
@@ -2181,6 +2258,9 @@ def backup_repositories(args, output_directory, repositories):
|
|
|
2181
2258
|
logger.info(f"Skipping remaining resources for {repository['full_name']}")
|
|
2182
2259
|
continue
|
|
2183
2260
|
|
|
2261
|
+
if skipped_unchanged_gists:
|
|
2262
|
+
logger.info("Skipped {0} unchanged gists".format(skipped_unchanged_gists))
|
|
2263
|
+
|
|
2184
2264
|
if incremental_resource_work_attempted:
|
|
2185
2265
|
remove_legacy_last_update_if_migrated(
|
|
2186
2266
|
args, output_directory, legacy_last_update_path
|
|
@@ -2500,6 +2580,141 @@ def backup_discussions(args, repo_cwd, repository):
|
|
|
2500
2580
|
)
|
|
2501
2581
|
|
|
2502
2582
|
|
|
2583
|
+
def cross_reference_state(timeline_data):
|
|
2584
|
+
"""Return (count, newest timestamp) of the cross-references in a timeline."""
|
|
2585
|
+
created = [
|
|
2586
|
+
item.get("created_at")
|
|
2587
|
+
for item in timeline_data or []
|
|
2588
|
+
if item.get("event") == "cross-referenced"
|
|
2589
|
+
]
|
|
2590
|
+
return len(created), max((c for c in created if c), default=None)
|
|
2591
|
+
|
|
2592
|
+
|
|
2593
|
+
def _cross_reference_page(args, repository, query, connection, use_total_count):
|
|
2594
|
+
"""Page one GraphQL connection, yielding {number: (count, newest)}.
|
|
2595
|
+
|
|
2596
|
+
totalCount honours the itemTypes filter on issues but not on pull
|
|
2597
|
+
requests, so pull request cross-references are counted from the nodes.
|
|
2598
|
+
"""
|
|
2599
|
+
owner, name = _repository_owner_name(repository)
|
|
2600
|
+
state = {}
|
|
2601
|
+
after = None
|
|
2602
|
+
page = 1
|
|
2603
|
+
|
|
2604
|
+
while True:
|
|
2605
|
+
variables = {
|
|
2606
|
+
"owner": owner,
|
|
2607
|
+
"name": name,
|
|
2608
|
+
"after": after,
|
|
2609
|
+
"pageSize": CROSS_REFERENCE_PAGE_SIZE,
|
|
2610
|
+
}
|
|
2611
|
+
if not use_total_count:
|
|
2612
|
+
# Only the pull request query declares this.
|
|
2613
|
+
variables["nodeLimit"] = CROSS_REFERENCE_NODE_LIMIT
|
|
2614
|
+
|
|
2615
|
+
data = retrieve_graphql_data(
|
|
2616
|
+
args,
|
|
2617
|
+
query,
|
|
2618
|
+
variables,
|
|
2619
|
+
log_context="cross-references {0} {1} page {2}".format(
|
|
2620
|
+
repository["full_name"], connection, page
|
|
2621
|
+
),
|
|
2622
|
+
)
|
|
2623
|
+
repository_data = data.get("repository")
|
|
2624
|
+
if repository_data is None:
|
|
2625
|
+
raise Exception(
|
|
2626
|
+
"Repository {0} not found in GraphQL response".format(
|
|
2627
|
+
repository["full_name"]
|
|
2628
|
+
)
|
|
2629
|
+
)
|
|
2630
|
+
|
|
2631
|
+
items = repository_data.get(connection) or {}
|
|
2632
|
+
for node in _connection_nodes(items):
|
|
2633
|
+
timeline = node.get("timelineItems") or {}
|
|
2634
|
+
created = [n.get("createdAt") for n in timeline.get("nodes") or [] if n]
|
|
2635
|
+
if use_total_count:
|
|
2636
|
+
count = timeline.get("totalCount", 0)
|
|
2637
|
+
elif len(created) >= CROSS_REFERENCE_NODE_LIMIT:
|
|
2638
|
+
# Counted from the nodes, so it may be truncated. Compare on
|
|
2639
|
+
# the timestamp alone rather than on a count we cannot trust.
|
|
2640
|
+
count = None
|
|
2641
|
+
else:
|
|
2642
|
+
count = len(created)
|
|
2643
|
+
state[node["number"]] = (count, created[-1] if created else None)
|
|
2644
|
+
|
|
2645
|
+
page_info = items.get("pageInfo") or {}
|
|
2646
|
+
if not page_info.get("hasNextPage"):
|
|
2647
|
+
break
|
|
2648
|
+
|
|
2649
|
+
after = page_info.get("endCursor")
|
|
2650
|
+
page += 1
|
|
2651
|
+
|
|
2652
|
+
return state
|
|
2653
|
+
|
|
2654
|
+
|
|
2655
|
+
def retrieve_cross_reference_state(args, repository, include_pulls=False):
|
|
2656
|
+
"""Ask GitHub for each item's cross-reference count and newest timestamp."""
|
|
2657
|
+
state = _cross_reference_page(
|
|
2658
|
+
args,
|
|
2659
|
+
repository,
|
|
2660
|
+
ISSUE_CROSS_REFERENCE_COUNT_QUERY,
|
|
2661
|
+
"issues",
|
|
2662
|
+
use_total_count=True,
|
|
2663
|
+
)
|
|
2664
|
+
if include_pulls:
|
|
2665
|
+
state.update(
|
|
2666
|
+
_cross_reference_page(
|
|
2667
|
+
args,
|
|
2668
|
+
repository,
|
|
2669
|
+
PULL_CROSS_REFERENCE_COUNT_QUERY,
|
|
2670
|
+
"pullRequests",
|
|
2671
|
+
use_total_count=False,
|
|
2672
|
+
)
|
|
2673
|
+
)
|
|
2674
|
+
return state
|
|
2675
|
+
|
|
2676
|
+
|
|
2677
|
+
def find_stale_timeline_issues(args, issue_cwd, repository, include_pulls=False):
|
|
2678
|
+
"""Issue numbers whose stored cross-references no longer match GitHub's.
|
|
2679
|
+
|
|
2680
|
+
Incremental runs select issues by updated_at, which a cross-reference does
|
|
2681
|
+
not change, so those issues would otherwise never be revisited.
|
|
2682
|
+
"""
|
|
2683
|
+
if not get_graphql_auth(args):
|
|
2684
|
+
# The check needs the GraphQL API, which always requires a token.
|
|
2685
|
+
logger.debug("Skipping cross-reference check, no authentication")
|
|
2686
|
+
return set()
|
|
2687
|
+
|
|
2688
|
+
try:
|
|
2689
|
+
live = retrieve_cross_reference_state(args, repository, include_pulls)
|
|
2690
|
+
except Exception as e:
|
|
2691
|
+
logger.warning(
|
|
2692
|
+
"Unable to check {0} for new cross-references, "
|
|
2693
|
+
"issues referenced since the last backup may be missed: {1}".format(
|
|
2694
|
+
repository["full_name"], e
|
|
2695
|
+
)
|
|
2696
|
+
)
|
|
2697
|
+
logger.debug(traceback.format_exc())
|
|
2698
|
+
return set()
|
|
2699
|
+
|
|
2700
|
+
stale = set()
|
|
2701
|
+
for number, (count, newest) in live.items():
|
|
2702
|
+
existing = read_json_file_if_exists("{0}/{1}.json".format(issue_cwd, number))
|
|
2703
|
+
if existing is None:
|
|
2704
|
+
continue
|
|
2705
|
+
stored = cross_reference_state(existing.get("timeline_data"))
|
|
2706
|
+
changed = stored[1] != newest if count is None else stored != (count, newest)
|
|
2707
|
+
if changed:
|
|
2708
|
+
stale.add(number)
|
|
2709
|
+
|
|
2710
|
+
if stale:
|
|
2711
|
+
logger.info(
|
|
2712
|
+
"Refreshing {0} issues with new cross-references".format(len(stale))
|
|
2713
|
+
)
|
|
2714
|
+
logger.debug("Stale timelines: {0}".format(sorted(stale)))
|
|
2715
|
+
return stale
|
|
2716
|
+
|
|
2717
|
+
|
|
2503
2718
|
def backup_issues(args, repo_cwd, repository, repos_template):
|
|
2504
2719
|
has_issues_dir = os.path.isdir("{0}/issues/.git".format(repo_cwd))
|
|
2505
2720
|
if args.skip_existing and has_issues_dir:
|
|
@@ -2515,6 +2730,7 @@ def backup_issues(args, repo_cwd, repository, repos_template):
|
|
|
2515
2730
|
_issue_template = "{0}/{1}/issues".format(repos_template, repository["full_name"])
|
|
2516
2731
|
|
|
2517
2732
|
should_include_pulls = args.include_pulls or args.include_everything
|
|
2733
|
+
include_timeline = args.include_issue_timeline or args.include_everything
|
|
2518
2734
|
issue_states = ["open", "closed"]
|
|
2519
2735
|
for issue_state in issue_states:
|
|
2520
2736
|
query_args = {"filter": "all", "state": issue_state}
|
|
@@ -2531,6 +2747,30 @@ def backup_issues(args, repo_cwd, repository, repos_template):
|
|
|
2531
2747
|
|
|
2532
2748
|
issues[issue["number"]] = issue
|
|
2533
2749
|
|
|
2750
|
+
stale_timelines = set()
|
|
2751
|
+
incremental_run = args.since or args.incremental_by_files
|
|
2752
|
+
if (
|
|
2753
|
+
include_timeline
|
|
2754
|
+
and incremental_run
|
|
2755
|
+
and any(f.endswith(".json") for f in os.listdir(issue_cwd))
|
|
2756
|
+
):
|
|
2757
|
+
stale_timelines = find_stale_timeline_issues(
|
|
2758
|
+
args, issue_cwd, repository, include_pulls=not should_include_pulls
|
|
2759
|
+
)
|
|
2760
|
+
for number in sorted(stale_timelines - set(issues)):
|
|
2761
|
+
try:
|
|
2762
|
+
issues[number] = retrieve_data(
|
|
2763
|
+
args, "{0}/{1}".format(_issue_template, number), paginated=False
|
|
2764
|
+
)[0]
|
|
2765
|
+
except (HTTPError, RepositoryUnavailableError, IndexError) as e:
|
|
2766
|
+
# Deleted or transferred between the sweep and this fetch. The
|
|
2767
|
+
# refresh is opportunistic, so skip rather than fail the run.
|
|
2768
|
+
logger.warning(
|
|
2769
|
+
"Unable to refresh cross-references for issue {0}: {1}".format(
|
|
2770
|
+
number, e
|
|
2771
|
+
)
|
|
2772
|
+
)
|
|
2773
|
+
|
|
2534
2774
|
if issues_skipped:
|
|
2535
2775
|
issues_skipped_message = " (skipped {0} pull requests)".format(issues_skipped)
|
|
2536
2776
|
|
|
@@ -2541,9 +2781,14 @@ def backup_issues(args, repo_cwd, repository, repos_template):
|
|
|
2541
2781
|
)
|
|
2542
2782
|
comments_template = _issue_template + "/{0}/comments"
|
|
2543
2783
|
events_template = _issue_template + "/{0}/events"
|
|
2784
|
+
timeline_template = _issue_template + "/{0}/timeline"
|
|
2544
2785
|
for number, issue in list(issues.items()):
|
|
2545
2786
|
issue_file = "{0}/{1}.json".format(issue_cwd, number)
|
|
2546
|
-
if
|
|
2787
|
+
if (
|
|
2788
|
+
args.incremental_by_files
|
|
2789
|
+
and os.path.isfile(issue_file)
|
|
2790
|
+
and number not in stale_timelines
|
|
2791
|
+
):
|
|
2547
2792
|
modified = os.path.getmtime(issue_file)
|
|
2548
2793
|
modified = datetime.fromtimestamp(modified).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
2549
2794
|
if modified > issue["updated_at"]:
|
|
@@ -2560,6 +2805,16 @@ def backup_issues(args, repo_cwd, repository, repos_template):
|
|
|
2560
2805
|
if args.include_issue_events or args.include_everything:
|
|
2561
2806
|
template = events_template.format(number)
|
|
2562
2807
|
issues[number]["event_data"] = retrieve_data(args, template)
|
|
2808
|
+
if include_timeline:
|
|
2809
|
+
template = timeline_template.format(number)
|
|
2810
|
+
# "commented" timeline entries embed the full comment body, which
|
|
2811
|
+
# --issue-comments already backs up; storing them twice invites the
|
|
2812
|
+
# two copies to diverge.
|
|
2813
|
+
issues[number]["timeline_data"] = [
|
|
2814
|
+
item
|
|
2815
|
+
for item in retrieve_data(args, template)
|
|
2816
|
+
if item.get("event") != "commented"
|
|
2817
|
+
]
|
|
2563
2818
|
if args.include_attachments:
|
|
2564
2819
|
download_attachments(
|
|
2565
2820
|
args, issue_cwd, issues[number], number, repository, item_type="issue"
|
github_backup/graphql_queries.py
CHANGED
|
@@ -2,6 +2,66 @@
|
|
|
2
2
|
|
|
3
3
|
DISCUSSION_PAGE_SIZE = 100
|
|
4
4
|
|
|
5
|
+
CROSS_REFERENCE_PAGE_SIZE = 100
|
|
6
|
+
|
|
7
|
+
# Cross-references do not change an issue's updatedAt, so the REST listing
|
|
8
|
+
# cannot report them as changed. Count and newest timestamp are compared
|
|
9
|
+
# against what is on disk: the count alone would miss an add paired with a
|
|
10
|
+
# delete, the timestamp alone would miss a delete.
|
|
11
|
+
ISSUE_CROSS_REFERENCE_COUNT_QUERY = """
|
|
12
|
+
query($owner: String!, $name: String!, $after: String, $pageSize: Int!) {
|
|
13
|
+
repository(owner: $owner, name: $name) {
|
|
14
|
+
issues(first: $pageSize, after: $after) {
|
|
15
|
+
nodes {
|
|
16
|
+
number
|
|
17
|
+
timelineItems(last: 1, itemTypes: [CROSS_REFERENCED_EVENT]) {
|
|
18
|
+
totalCount
|
|
19
|
+
nodes {
|
|
20
|
+
... on CrossReferencedEvent {
|
|
21
|
+
createdAt
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
pageInfo {
|
|
27
|
+
hasNextPage
|
|
28
|
+
endCursor
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
# Pull requests are stored as issues when --pulls is not used, and the issues
|
|
36
|
+
# connection above excludes them. This query is shaped differently because
|
|
37
|
+
# totalCount ignores itemTypes on a pull request, unlike on an issue, so the
|
|
38
|
+
# cross-references have to be counted from the nodes themselves.
|
|
39
|
+
CROSS_REFERENCE_NODE_LIMIT = 100
|
|
40
|
+
|
|
41
|
+
PULL_CROSS_REFERENCE_COUNT_QUERY = """
|
|
42
|
+
query($owner: String!, $name: String!, $after: String, $pageSize: Int!,
|
|
43
|
+
$nodeLimit: Int!) {
|
|
44
|
+
repository(owner: $owner, name: $name) {
|
|
45
|
+
pullRequests(first: $pageSize, after: $after) {
|
|
46
|
+
nodes {
|
|
47
|
+
number
|
|
48
|
+
timelineItems(last: $nodeLimit, itemTypes: [CROSS_REFERENCED_EVENT]) {
|
|
49
|
+
nodes {
|
|
50
|
+
... on CrossReferencedEvent {
|
|
51
|
+
createdAt
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
pageInfo {
|
|
57
|
+
hasNextPage
|
|
58
|
+
endCursor
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
"""
|
|
64
|
+
|
|
5
65
|
DISCUSSION_LIST_QUERY = """
|
|
6
66
|
query($owner: String!, $name: String!, $after: String, $pageSize: Int!) {
|
|
7
67
|
repository(owner: $owner, name: $name) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: github-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.65.0
|
|
4
4
|
Summary: backup a github user or organization
|
|
5
5
|
Home-page: http://github.com/josegonzalez/python-github-backup
|
|
6
6
|
Author: Jose Diaz-Gonzalez
|
|
@@ -71,7 +71,8 @@ CLI Help output::
|
|
|
71
71
|
[--incremental-by-files]
|
|
72
72
|
[--starred] [--all-starred] [--starred-skip-size-over MB]
|
|
73
73
|
[--watched] [--followers] [--following] [--all]
|
|
74
|
-
[--issues] [--issue-comments] [--issue-events]
|
|
74
|
+
[--issues] [--issue-comments] [--issue-events]
|
|
75
|
+
[--issue-timeline] [--pulls]
|
|
75
76
|
[--pull-comments] [--pull-reviews] [--pull-commits]
|
|
76
77
|
[--pull-details]
|
|
77
78
|
[--labels] [--hooks] [--milestones] [--security-advisories]
|
|
@@ -126,6 +127,8 @@ CLI Help output::
|
|
|
126
127
|
--issues include issues in backup
|
|
127
128
|
--issue-comments include issue comments in backup
|
|
128
129
|
--issue-events include issue events in backup
|
|
130
|
+
--issue-timeline include issue timeline in backup (cross-references,
|
|
131
|
+
commits and reviews; complements --issue-events)
|
|
129
132
|
--pulls include pull requests in backup
|
|
130
133
|
--pull-comments include pull request review comments in backup
|
|
131
134
|
--pull-reviews include pull request reviews in backup
|
|
@@ -377,6 +380,27 @@ For finer control, avoid using ``--assets`` with starred repos, or use ``--skip-
|
|
|
377
380
|
|
|
378
381
|
Alternatively, consider just storing links to starred repos in JSON format with ``--starred``.
|
|
379
382
|
|
|
383
|
+
|
|
384
|
+
About issue events and the issue timeline
|
|
385
|
+
-----------------------------------------
|
|
386
|
+
|
|
387
|
+
Use ``--issue-events`` and ``--issue-timeline`` with ``--issues``; on their own they back up nothing.
|
|
388
|
+
|
|
389
|
+
``--issue-events`` backs up GitHub's issue events endpoint under each issue's ``event_data`` key. That endpoint returns a restricted set of event types and **never** includes cross-references, so an issue that was mentioned from another issue or pull request shows no sign of it. GitHub only exposes those through the timeline.
|
|
390
|
+
|
|
391
|
+
``--issue-timeline`` backs up the timeline endpoint under a separate ``timeline_data`` key. The two flags are independent: ``event_data`` keeps its existing contents and shape whether or not the timeline is enabled, so nothing that already reads it is affected.
|
|
392
|
+
|
|
393
|
+
Use both flags together for a complete picture. Neither endpoint is a superset of the other, so they are complementary rather than alternatives. Only the timeline reports ``cross-referenced``, ``committed`` and ``reviewed`` entries. Only the events endpoint reliably reports ``referenced`` entries, where a commit elsewhere in the repository mentions the issue; the timeline omits many of these, substituting the issue's own ``committed`` entries instead.
|
|
394
|
+
|
|
395
|
+
Note that timeline entries are not all the same shape as events. ``cross-referenced`` and ``committed`` entries have no ``id`` field, so anything consuming ``timeline_data`` should not assume one is present. ``commented`` entries are excluded from the backup because they embed the full comment body, which ``--issue-comments`` already saves.
|
|
396
|
+
|
|
397
|
+
``--issue-timeline`` is included in ``--all``. On issues with many cross-references it can be noticeably larger and slower than ``--issue-events`` alone, because each ``cross-referenced`` entry embeds the referencing issue in full. On a heavily cross-referenced issue this can mean several times the API requests and many times the JSON on disk. Ordinary repositories see little difference.
|
|
398
|
+
|
|
399
|
+
Incremental backups need extra help here, because being referenced from elsewhere does not change an issue's ``updated_at``. An issue can be mentioned from another repository years after its last activity, and the usual ``since`` listing will never report it as changed. On an incremental run with ``--issue-timeline``, github-backup therefore asks GitHub's GraphQL API for each item's cross-reference count and newest timestamp, compares those against what is already saved, and re-fetches the timeline of anything that differs. Pull requests are included in that check when they are being stored as issues, which is the case unless ``--pulls`` is used. The check needs no extra state on disk and also notices references that were later deleted. It costs one rate-limit point per 100 items checked, against GitHub's GraphQL quota, which is separate from the REST one. That is cheap in quota terms even on very large repositories, but it is not instant: a repository with 18,000 issues and pull requests took around 200 points and five minutes for the check alone.
|
|
400
|
+
|
|
401
|
+
If you enable ``--issue-timeline`` on an existing incremental backup, cross-references on older issues are picked up automatically by that check. Timeline entries of other kinds, such as commits and reviews, are only fetched when the issue itself is updated, so run once without ``--incremental`` if you want those filled in for historical issues too.
|
|
402
|
+
|
|
403
|
+
|
|
380
404
|
About pull request reviews
|
|
381
405
|
--------------------------
|
|
382
406
|
|
|
@@ -480,14 +504,14 @@ Quietly and incrementally backup useful Github user data (public and private rep
|
|
|
480
504
|
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
481
505
|
GH_USER=YOUR-GITHUB-USER
|
|
482
506
|
|
|
483
|
-
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --security-advisories --discussions --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
507
|
+
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --issue-timeline --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --security-advisories --discussions --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
|
484
508
|
|
|
485
509
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
|
486
510
|
|
|
487
511
|
FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
|
488
512
|
GH_USER=YOUR-GITHUB-USER
|
|
489
513
|
|
|
490
|
-
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --discussions --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
514
|
+
github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --issue-timeline --pulls --pull-comments --pull-reviews --pull-commits --labels --milestones --discussions --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
|
|
491
515
|
|
|
492
516
|
Pipe a token from stdin to avoid storing it in environment variables or command history (Unix-like systems only)::
|
|
493
517
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
github_backup/__init__.py,sha256=eauVKKmGRrPjcW6OLz78zXL8jRoYaQPOL0tbet7E8Vo,23
|
|
2
|
+
github_backup/__main__.py,sha256=3yorCiC63maaxbjdtaeAs-G-yAAxKSO3mprhGfa1Fho,271
|
|
3
|
+
github_backup/cli.py,sha256=JTiXPOvB8id8LopREOEd1W_FLvuf9SOZU5buMRPFd9U,2922
|
|
4
|
+
github_backup/github_backup.py,sha256=jDGgAnhT_vn7QUmEJ95hwe8jfuVc3dETyLblGSQTZ0k,126285
|
|
5
|
+
github_backup/graphql_queries.py,sha256=XcAQyp9nF7Ip9UzxsrXMkl2pO-W7SK8hhO2oE9HVcmA,6658
|
|
6
|
+
github_backup-0.65.0.dist-info/licenses/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
|
|
7
|
+
github_backup-0.65.0.dist-info/METADATA,sha256=lWXqgFoH_QhZEHy0fsp8cs_Iu8McBJZ4bUDrOL2VDp8,36048
|
|
8
|
+
github_backup-0.65.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
9
|
+
github_backup-0.65.0.dist-info/entry_points.txt,sha256=cLPKK7MqC77QVb6HnT2mnyL_f-ciavJG_Y9vVHlCF68,57
|
|
10
|
+
github_backup-0.65.0.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
|
|
11
|
+
github_backup-0.65.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
12
|
+
github_backup-0.65.0.dist-info/RECORD,,
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
github_backup/__init__.py,sha256=T4-vW_ByO6apa0V-uPVXwtgvQiE-ODq0knBtcK_morc,23
|
|
2
|
-
github_backup/__main__.py,sha256=3yorCiC63maaxbjdtaeAs-G-yAAxKSO3mprhGfa1Fho,271
|
|
3
|
-
github_backup/cli.py,sha256=JTiXPOvB8id8LopREOEd1W_FLvuf9SOZU5buMRPFd9U,2922
|
|
4
|
-
github_backup/github_backup.py,sha256=nVScdDkx-UVu-c1kU2Sx5z6YlapO_2rFFQYi_3leJ3U,116809
|
|
5
|
-
github_backup/graphql_queries.py,sha256=QBIXcGzGunilMpHfL32cTO_AAzxuQD27QII1V9K_Yw4,4883
|
|
6
|
-
github_backup-0.64.1.dist-info/licenses/LICENSE.txt,sha256=ML-pLyn1qtAKo_DHMcQ8-p7k3f8cp-IR-mf5B49R2rk,1085
|
|
7
|
-
github_backup-0.64.1.dist-info/METADATA,sha256=dO_Y2fyGPk7gi12MfanNPjW2P3E7rlO1JmsNjs402ts,32426
|
|
8
|
-
github_backup-0.64.1.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
9
|
-
github_backup-0.64.1.dist-info/entry_points.txt,sha256=cLPKK7MqC77QVb6HnT2mnyL_f-ciavJG_Y9vVHlCF68,57
|
|
10
|
-
github_backup-0.64.1.dist-info/top_level.txt,sha256=Za6fKRAlxZ0b3ZBH_osPqklLe14LetdNAkvAQANT2AA,14
|
|
11
|
-
github_backup-0.64.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
12
|
-
github_backup-0.64.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|