pywaybackup 1.4.2__tar.gz → 1.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pywaybackup-1.5.0/.github/ISSUE_TEMPLATE/feature_request.md +17 -0
- pywaybackup-1.5.0/.github/workflows/stale.yml +20 -0
- {pywaybackup-1.4.2/pywaybackup.egg-info → pywaybackup-1.5.0}/PKG-INFO +11 -7
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/README.md +10 -6
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/Arguments.py +3 -1
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/Exception.py +26 -29
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/SnapshotCollection.py +16 -3
- pywaybackup-1.5.0/pywaybackup/__version__.py +1 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/archive.py +69 -29
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/helper.py +0 -2
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/main.py +2 -3
- {pywaybackup-1.4.2 → pywaybackup-1.5.0/pywaybackup.egg-info}/PKG-INFO +11 -7
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/SOURCES.txt +4 -1
- pywaybackup-1.5.0/test/test_links.js +63 -0
- pywaybackup-1.4.2/pywaybackup/__version__.py +0 -1
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/.github/ISSUE_TEMPLATE/bug.md +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/.gitignore +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/LICENSE +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/dev/pip_build.sh +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/dev/venv_create.sh +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/Converter.py +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/Verbosity.py +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/__init__.py +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/dependency_links.txt +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/entry_points.txt +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/requires.txt +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/top_level.txt +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/requirements.txt +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/setup.cfg +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/setup.py +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/test/cdx.example.com.200 +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/test/cdx.example.com.404 +0 -0
- {pywaybackup-1.4.2 → pywaybackup-1.5.0}/test/cdx.example.com.small.200 +0 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Feature request
|
|
3
|
+
about: Suggest an idea for this project
|
|
4
|
+
title: "[FEATURE]"
|
|
5
|
+
labels: enhancement
|
|
6
|
+
assignees: ''
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
**Is your feature request related to a problem? Please describe.**
|
|
11
|
+
Describe the problem (not a bug - more of a usage issue).
|
|
12
|
+
|
|
13
|
+
**Describe the solution you'd like**
|
|
14
|
+
How should the feature improve your usage?
|
|
15
|
+
|
|
16
|
+
**Additional context**
|
|
17
|
+
Any other context or screenshots about the feature.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
name: 'Close by inactivity'
|
|
2
|
+
on:
|
|
3
|
+
schedule:
|
|
4
|
+
- cron: "00 00 * * *"
|
|
5
|
+
workflow_dispatch:
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
close-issues:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
permissions:
|
|
11
|
+
issues: write
|
|
12
|
+
pull-requests: write
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/stale@v9.0.0
|
|
15
|
+
with:
|
|
16
|
+
days-before-issue-stale: 30
|
|
17
|
+
days-before-issue-close: 14
|
|
18
|
+
stale-issue-label: "stale"
|
|
19
|
+
stale-issue-message: "This issue is marked as stale because there was no activity for 30 days."
|
|
20
|
+
close-issue-message: "This issue has been closed because there has been no activity for 14 days while it was marked as stale."
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pywaybackup
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.5.0
|
|
4
4
|
Summary: Download snapshots from the Wayback Machine
|
|
5
5
|
Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
|
|
6
6
|
Author: bitdruid
|
|
@@ -44,10 +44,11 @@ This tool allows you to download content from the Wayback Machine (archive.org).
|
|
|
44
44
|
```pip install .```
|
|
45
45
|
- in a virtual env or use `--break-system-package`
|
|
46
46
|
|
|
47
|
-
## Usage infos
|
|
47
|
+
## Usage infos - important notes
|
|
48
48
|
|
|
49
49
|
- Linux recommended: On Windows machines, the path length is limited. This can only be overcome by editing the registry. Files that exceed the path length will not be downloaded.
|
|
50
50
|
- If you query an explicit file (e.g. a query-string `?query=this` or `login.html`), the `--explicit`-argument is recommended as a wildcard query may lead to an empty result.
|
|
51
|
+
- The tool will inform you if your query has an immense amount of snapshots which could consume your system memory and lead to a crash. Consider splitting your query into smaller jobs by specifying a range e.g. `--start 2023 --end 2024` or `--range 1`.
|
|
51
52
|
|
|
52
53
|
## Arguments
|
|
53
54
|
|
|
@@ -113,10 +114,13 @@ Specifies number of retry attempts for failed downloads.
|
|
|
113
114
|
- **`--delay`** `<seconds>`:<br>
|
|
114
115
|
Specifies delay between download requests in seconds. Default is no delay (0).
|
|
115
116
|
|
|
117
|
+
- **`--limit`** `<count>`:<br>
|
|
118
|
+
Limits the amount of snapshots to query from the CDX server. If an existing CDX file is injected (with `--cdxinject` or `--auto`), the limit will have no effect.
|
|
119
|
+
|
|
116
120
|
<!-- - **`--convert-links`**:<br>
|
|
117
121
|
If set, all links in the downloaded files will be converted to local links. This is useful for offline browsing. The links are converted to the local path structure. Show output with `--verbosity trace`. -->
|
|
118
122
|
|
|
119
|
-
**CDX Query Handling:**
|
|
123
|
+
**CDX Query Result Handling:**
|
|
120
124
|
- **`--cdxbackup`** `<path>`:<br>
|
|
121
125
|
Path defaults to output-dir. Saves the result of CDX query as a file. Useful for later downloading snapshots and overcoming refused connections by CDX server due to too many queries. Named as `waybackup_<sanitized_url>.cdx`.
|
|
122
126
|
|
|
@@ -127,10 +131,6 @@ Injects a CDX query file to download snapshots. Ensure the query matches the pre
|
|
|
127
131
|
- **`--auto`**:<br>
|
|
128
132
|
If set, csv, skip and cdxbackup/cdxinject are handled automatically. Keep the files and folders as they are. Otherwise they will not be recognized when restarting a download.
|
|
129
133
|
|
|
130
|
-
### Debug
|
|
131
|
-
|
|
132
|
-
- `--debug`: If set, full traceback will be printed in case of an error. The full exception will be written into `waybackup_error.log`.
|
|
133
|
-
|
|
134
134
|
### Examples
|
|
135
135
|
|
|
136
136
|
Download latest snapshot of all files:<br>
|
|
@@ -232,6 +232,10 @@ For list queries:
|
|
|
232
232
|
|
|
233
233
|
The csv contains the json response in a table format.
|
|
234
234
|
|
|
235
|
+
### Debugging
|
|
236
|
+
|
|
237
|
+
Exceptions will be written into `waybackup_error.log` (each run overwrites the file).
|
|
238
|
+
|
|
235
239
|
## Contributing
|
|
236
240
|
|
|
237
241
|
I'm always happy for some feature requests to improve the usability of this tool.
|
|
@@ -28,10 +28,11 @@ This tool allows you to download content from the Wayback Machine (archive.org).
|
|
|
28
28
|
```pip install .```
|
|
29
29
|
- in a virtual env or use `--break-system-package`
|
|
30
30
|
|
|
31
|
-
## Usage infos
|
|
31
|
+
## Usage infos - important notes
|
|
32
32
|
|
|
33
33
|
- Linux recommended: On Windows machines, the path length is limited. This can only be overcome by editing the registry. Files that exceed the path length will not be downloaded.
|
|
34
34
|
- If you query an explicit file (e.g. a query-string `?query=this` or `login.html`), the `--explicit`-argument is recommended as a wildcard query may lead to an empty result.
|
|
35
|
+
- The tool will inform you if your query has an immense amount of snapshots which could consume your system memory and lead to a crash. Consider splitting your query into smaller jobs by specifying a range e.g. `--start 2023 --end 2024` or `--range 1`.
|
|
35
36
|
|
|
36
37
|
## Arguments
|
|
37
38
|
|
|
@@ -97,10 +98,13 @@ Specifies number of retry attempts for failed downloads.
|
|
|
97
98
|
- **`--delay`** `<seconds>`:<br>
|
|
98
99
|
Specifies delay between download requests in seconds. Default is no delay (0).
|
|
99
100
|
|
|
101
|
+
- **`--limit`** `<count>`:<br>
|
|
102
|
+
Limits the amount of snapshots to query from the CDX server. If an existing CDX file is injected (with `--cdxinject` or `--auto`), the limit will have no effect.
|
|
103
|
+
|
|
100
104
|
<!-- - **`--convert-links`**:<br>
|
|
101
105
|
If set, all links in the downloaded files will be converted to local links. This is useful for offline browsing. The links are converted to the local path structure. Show output with `--verbosity trace`. -->
|
|
102
106
|
|
|
103
|
-
**CDX Query Handling:**
|
|
107
|
+
**CDX Query Result Handling:**
|
|
104
108
|
- **`--cdxbackup`** `<path>`:<br>
|
|
105
109
|
Path defaults to output-dir. Saves the result of CDX query as a file. Useful for later downloading snapshots and overcoming refused connections by CDX server due to too many queries. Named as `waybackup_<sanitized_url>.cdx`.
|
|
106
110
|
|
|
@@ -111,10 +115,6 @@ Injects a CDX query file to download snapshots. Ensure the query matches the pre
|
|
|
111
115
|
- **`--auto`**:<br>
|
|
112
116
|
If set, csv, skip and cdxbackup/cdxinject are handled automatically. Keep the files and folders as they are. Otherwise they will not be recognized when restarting a download.
|
|
113
117
|
|
|
114
|
-
### Debug
|
|
115
|
-
|
|
116
|
-
- `--debug`: If set, full traceback will be printed in case of an error. The full exception will be written into `waybackup_error.log`.
|
|
117
|
-
|
|
118
118
|
### Examples
|
|
119
119
|
|
|
120
120
|
Download latest snapshot of all files:<br>
|
|
@@ -216,6 +216,10 @@ For list queries:
|
|
|
216
216
|
|
|
217
217
|
The csv contains the json response in a table format.
|
|
218
218
|
|
|
219
|
+
### Debugging
|
|
220
|
+
|
|
221
|
+
Exceptions will be written into `waybackup_error.log` (each run overwrites the file).
|
|
222
|
+
|
|
219
223
|
## Contributing
|
|
220
224
|
|
|
221
225
|
I'm always happy for some feature requests to improve the usability of this tool.
|
|
@@ -13,7 +13,6 @@ class Arguments:
|
|
|
13
13
|
|
|
14
14
|
parser = argparse.ArgumentParser(description='Download from wayback machine (archive.org)')
|
|
15
15
|
parser.add_argument('-a', '--about', action='version', version='%(prog)s ' + __version__ + ' by @bitdruid -> https://github.com/bitdruid')
|
|
16
|
-
parser.add_argument('-d', '--debug', action='store_true', help='Debug mode (Always full traceback and creates an error.log')
|
|
17
16
|
|
|
18
17
|
required = parser.add_argument_group('required (one exclusive)')
|
|
19
18
|
required.add_argument('-u', '--url', type=str, metavar="", help='url (with subdir/subdomain) to download')
|
|
@@ -40,6 +39,7 @@ class Arguments:
|
|
|
40
39
|
special.add_argument('--workers', type=int, default=1, metavar="", help='number of workers (simultaneous downloads)')
|
|
41
40
|
# special.add_argument('--convert-links', action='store_true', help='Convert all links in the files to local paths. Requires -c/--current')
|
|
42
41
|
special.add_argument('--delay', type=int, default=0, metavar="", help='delay between each download in seconds')
|
|
42
|
+
special.add_argument('--limit', type=int, nargs='?', const=True, metavar='int', help='limit the number of snapshots to download')
|
|
43
43
|
|
|
44
44
|
cdx = parser.add_argument_group('cdx (one exclusive)')
|
|
45
45
|
exclusive_cdx = cdx.add_mutually_exclusive_group()
|
|
@@ -84,6 +84,8 @@ class Configuration:
|
|
|
84
84
|
if cls.current:
|
|
85
85
|
cls.mode = "current"
|
|
86
86
|
|
|
87
|
+
cls.cdxbackup = cls.output if cls.cdxbackup is None else cls.cdxbackup
|
|
88
|
+
|
|
87
89
|
if cls.auto:
|
|
88
90
|
cls.skip = cls.output
|
|
89
91
|
cls.csv = cls.output
|
|
@@ -12,16 +12,14 @@ from pywaybackup.__version__ import __version__
|
|
|
12
12
|
class Exception:
|
|
13
13
|
|
|
14
14
|
new_debug = True
|
|
15
|
-
debug = False
|
|
16
15
|
output = None
|
|
17
16
|
command = None
|
|
18
17
|
|
|
19
18
|
@classmethod
|
|
20
|
-
def init(cls,
|
|
19
|
+
def init(cls, output=None, command=None):
|
|
21
20
|
sys.excepthook = cls.exception_handler # set custom exception handler (uncaught exceptions)
|
|
22
21
|
cls.output = output
|
|
23
22
|
cls.command = command
|
|
24
|
-
cls.debug = True if debug else False
|
|
25
23
|
|
|
26
24
|
@classmethod
|
|
27
25
|
def exception(cls, message: str, e: Exception, tb=None):
|
|
@@ -53,33 +51,32 @@ class Exception:
|
|
|
53
51
|
"-------------------------"
|
|
54
52
|
)
|
|
55
53
|
print(exception_message)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
f = open(debug_file, "w")
|
|
64
|
-
f.write("-------------------------\n")
|
|
65
|
-
f.write(f"Version: {__version__}\n")
|
|
66
|
-
f.write("-------------------------\n")
|
|
67
|
-
f.write(f"Command: {cls.command}\n")
|
|
68
|
-
f.write("-------------------------\n\n")
|
|
69
|
-
else: # current run, append to file
|
|
70
|
-
f = open(debug_file, "a")
|
|
71
|
-
f.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n")
|
|
72
|
-
f.write(exception_message + "\n")
|
|
73
|
-
f.write("!-- Local Variables:\n")
|
|
74
|
-
for var_name, value in local_vars.items():
|
|
75
|
-
if var_name in ["status_message", "headers"]:
|
|
76
|
-
continue
|
|
77
|
-
value = cls.relativate_path(str(value))
|
|
78
|
-
value = value[:666] + " ... " if len(value) > 666 else value
|
|
79
|
-
f.write(f" -- {var_name} = {value}\n")
|
|
54
|
+
debug_file = os.path.join(cls.output, "waybackup_error.log")
|
|
55
|
+
print(f"Exception log: {debug_file}")
|
|
56
|
+
# print("-------------------------")
|
|
57
|
+
# print(f"Full traceback:\n{original_tb}")
|
|
58
|
+
if cls.new_debug: # new run, overwrite file
|
|
59
|
+
cls.new_debug = False
|
|
60
|
+
f = open(debug_file, "w")
|
|
80
61
|
f.write("-------------------------\n")
|
|
81
|
-
f.write(
|
|
82
|
-
f.
|
|
62
|
+
f.write(f"Version: {__version__}\n")
|
|
63
|
+
f.write("-------------------------\n")
|
|
64
|
+
f.write(f"Command: {cls.command}\n")
|
|
65
|
+
f.write("-------------------------\n\n")
|
|
66
|
+
else: # current run, append to file
|
|
67
|
+
f = open(debug_file, "a")
|
|
68
|
+
f.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n")
|
|
69
|
+
f.write(exception_message + "\n")
|
|
70
|
+
f.write("!-- Local Variables:\n")
|
|
71
|
+
for var_name, value in local_vars.items():
|
|
72
|
+
if var_name in ["status_message", "headers"]:
|
|
73
|
+
continue
|
|
74
|
+
value = cls.relativate_path(str(value))
|
|
75
|
+
value = value[:666] + " ... " if len(value) > 666 else value
|
|
76
|
+
f.write(f" -- {var_name} = {value}\n")
|
|
77
|
+
f.write("-------------------------\n")
|
|
78
|
+
f.write(original_tb + "\n")
|
|
79
|
+
f.close()
|
|
83
80
|
|
|
84
81
|
@classmethod
|
|
85
82
|
def relativate_path(cls, input: str) -> str:
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from pywaybackup.helper import url_split
|
|
2
|
+
import json
|
|
2
3
|
import os
|
|
3
4
|
|
|
4
5
|
class SnapshotCollection:
|
|
@@ -7,15 +8,27 @@ class SnapshotCollection:
|
|
|
7
8
|
MODE_CURRENT = 0
|
|
8
9
|
|
|
9
10
|
@classmethod
|
|
10
|
-
def create_list(cls,
|
|
11
|
+
def create_list(cls, cdxfile, mode):
|
|
11
12
|
"""
|
|
12
13
|
Create the snapshot collection list from a cdx result.
|
|
13
14
|
|
|
14
15
|
- mode `full`: All snapshots are included.
|
|
15
16
|
- mode `current`: Only the latest snapshot of each file is included.
|
|
16
17
|
"""
|
|
17
|
-
|
|
18
|
-
|
|
18
|
+
with open(cdxfile, "r") as f:
|
|
19
|
+
first_line = True
|
|
20
|
+
for line in f:
|
|
21
|
+
if first_line:
|
|
22
|
+
first_line = False
|
|
23
|
+
continue
|
|
24
|
+
line = line.strip()
|
|
25
|
+
if line.endswith("]]"): line = line.rsplit("]", 1)[0]
|
|
26
|
+
if line.endswith(","): line = line.rsplit(",", 1)[0]
|
|
27
|
+
else: continue # drop incomplete line, maybe cdx response was cut off
|
|
28
|
+
line = json.loads(line)
|
|
29
|
+
line = {"timestamp": line[0], "digest": line[1], "mimetype": line[2], "status": line[3], "url": line[4]}
|
|
30
|
+
cls.SNAPSHOT_COLLECTION.append(line)
|
|
31
|
+
cls.SNAPSHOT_COLLECTION = sorted(cls.SNAPSHOT_COLLECTION, key=lambda k: k['timestamp'], reverse=True)
|
|
19
32
|
if mode == "current":
|
|
20
33
|
cls.MODE_CURRENT = 1
|
|
21
34
|
cdxResult_list_filtered = []
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "1.5.0"
|
|
@@ -5,12 +5,13 @@ import csv
|
|
|
5
5
|
import threading
|
|
6
6
|
import queue
|
|
7
7
|
import time
|
|
8
|
-
import json
|
|
9
8
|
import urllib.parse
|
|
10
9
|
import http.client
|
|
11
10
|
from urllib.parse import urljoin
|
|
12
11
|
from datetime import datetime, timezone
|
|
13
12
|
|
|
13
|
+
from tqdm import tqdm
|
|
14
|
+
|
|
14
15
|
from socket import timeout
|
|
15
16
|
|
|
16
17
|
from pywaybackup.helper import url_get_timestamp, move_index, sanitize_filename, check_nt
|
|
@@ -20,10 +21,10 @@ from pywaybackup.Arguments import Configuration as config
|
|
|
20
21
|
|
|
21
22
|
from pywaybackup.__version__ import __version__
|
|
22
23
|
|
|
23
|
-
from pywaybackup.Converter import Converter as convert
|
|
24
24
|
from pywaybackup.Verbosity import Message
|
|
25
25
|
from pywaybackup.Verbosity import Verbosity as vb
|
|
26
26
|
from pywaybackup.Exception import Exception as ex
|
|
27
|
+
import threading
|
|
27
28
|
|
|
28
29
|
|
|
29
30
|
|
|
@@ -99,28 +100,41 @@ def print_list():
|
|
|
99
100
|
|
|
100
101
|
# create filelist
|
|
101
102
|
# timestamp format yyyyMMddhhmmss
|
|
102
|
-
def query_list(
|
|
103
|
+
def query_list(queryrange: int, limit: int, start: int, end: int, explicit: bool, mode: str, cdxbackup: str, cdxinject: str):
|
|
104
|
+
|
|
105
|
+
def input_countdown():
|
|
106
|
+
for i in range(10, -1, -1):
|
|
107
|
+
vb.write(message=f"{i}")
|
|
108
|
+
print("\033[F", end="")
|
|
109
|
+
print("\033[K", end="")
|
|
110
|
+
time.sleep(1)
|
|
111
|
+
|
|
112
|
+
def input_detection():
|
|
113
|
+
input()
|
|
114
|
+
vb.write(message="\nExiting...")
|
|
115
|
+
os._exit(1)
|
|
116
|
+
|
|
117
|
+
def count_cdxfile(cdxfile):
|
|
118
|
+
with open(cdxfile, "r") as file:
|
|
119
|
+
return file.read().count("\n") - 1
|
|
103
120
|
|
|
104
121
|
def inject(cdxinject):
|
|
105
122
|
if os.path.isfile(cdxinject):
|
|
106
123
|
vb.write(message="\nInjecting CDX data...")
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
linecount = cdxResult.count("\n") - 1
|
|
110
|
-
vb.write(message=f"\n-----> {linecount} snapshots injected")
|
|
111
|
-
return cdxResult
|
|
124
|
+
vb.write(message=f"\n-----> {count_cdxfile(cdxinject):,} lines injected")
|
|
125
|
+
return cdxinject
|
|
112
126
|
else:
|
|
113
127
|
vb.write(message="\nNo CDX file found to inject - querying snapshots...")
|
|
114
128
|
return False
|
|
115
129
|
|
|
116
|
-
def query(
|
|
130
|
+
def query(queryrange, limit, start, end, explicit):
|
|
117
131
|
vb.write(message="\nQuerying snapshots...")
|
|
118
132
|
query_range = ""
|
|
119
|
-
if not
|
|
133
|
+
if not queryrange:
|
|
120
134
|
if start: query_range = query_range + f"&from={start}"
|
|
121
135
|
if end: query_range = query_range + f"&to={end}"
|
|
122
136
|
else:
|
|
123
|
-
query_range = "&from=" + str(datetime.now().year -
|
|
137
|
+
query_range = "&from=" + str(datetime.now().year - queryrange)
|
|
124
138
|
|
|
125
139
|
if config.domain and not config.subdir and not config.filename:
|
|
126
140
|
cdx_url = f"{config.domain}"
|
|
@@ -133,32 +147,58 @@ def query_list(range: int, start: int, end: int, explicit: bool, mode: str, cdxb
|
|
|
133
147
|
if not explicit:
|
|
134
148
|
cdx_url = f"{cdx_url}/*"
|
|
135
149
|
|
|
136
|
-
|
|
137
|
-
|
|
150
|
+
limit = f"limit={limit}&" if limit else ""
|
|
151
|
+
|
|
152
|
+
vb.write(message=f"-----> {cdx_url}")
|
|
153
|
+
cdxQuery = f"https://web.archive.org/cdx/search/cdx?output=json&url={cdx_url}{query_range}&fl=timestamp,digest,mimetype,statuscode,original&{limit}filter!=statuscode:200"
|
|
154
|
+
|
|
155
|
+
cdxfile = os.path.join(cdxbackup, f"waybackup_{sanitize_filename(config.url)}.cdx")
|
|
138
156
|
|
|
139
157
|
try:
|
|
140
|
-
|
|
141
|
-
|
|
158
|
+
cdxfile_IO = open(cdxfile, "w")
|
|
159
|
+
with requests.get(cdxQuery, stream=True) as r:
|
|
160
|
+
r.raise_for_status()
|
|
161
|
+
with tqdm(unit='B', unit_scale=True, desc='-----> Downloading CDX result') as pbar:
|
|
162
|
+
for chunk in r.iter_content(chunk_size=8192):
|
|
163
|
+
if chunk:
|
|
164
|
+
pbar.update(len(chunk))
|
|
165
|
+
chunk = chunk.decode("utf-8")
|
|
166
|
+
cdxfile_IO.write(chunk)
|
|
167
|
+
cdxfile_IO.close()
|
|
168
|
+
except requests.exceptions.ConnectionError:
|
|
142
169
|
vb.write(message="\nCONNECTION REFUSED -> could not query cdx server (max retries exceeded)\n")
|
|
143
170
|
os._exit(1)
|
|
144
|
-
|
|
145
|
-
if cdxbackup:
|
|
146
|
-
os.makedirs(cdxbackup, exist_ok=True)
|
|
147
|
-
with open(os.path.join(cdxbackup, f"waybackup_{sanitize_filename(config.url)}.cdx"), "w") as file:
|
|
148
|
-
file.write(cdxResult)
|
|
149
|
-
vb.write(message="\n-----> CDX backup generated")
|
|
150
171
|
|
|
151
|
-
return
|
|
172
|
+
return cdxfile
|
|
152
173
|
|
|
153
|
-
|
|
174
|
+
cdxfile = None
|
|
154
175
|
if cdxinject:
|
|
155
|
-
|
|
156
|
-
if not
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
176
|
+
cdxfile = inject(cdxinject)
|
|
177
|
+
if not cdxfile:
|
|
178
|
+
cdxfile = query(queryrange, limit, start, end, explicit)
|
|
179
|
+
|
|
180
|
+
snapshot_count = count_cdxfile(cdxfile) - 1
|
|
181
|
+
if snapshot_count > 1000000:
|
|
182
|
+
vb.write(message="\n!!!!! WARNING")
|
|
183
|
+
vb.write(message="Excessive amount of snapshots detected. System may run out of memory!")
|
|
184
|
+
vb.write(message="---> Consider splitting the query into smaller jobs (range start/end).")
|
|
185
|
+
vb.write(message="\nPress ANY key to abort...")
|
|
186
|
+
|
|
187
|
+
abort_listener = threading.Thread(target=input_detection)
|
|
188
|
+
abort_listener.start()
|
|
189
|
+
|
|
190
|
+
input_countdown()
|
|
191
|
+
|
|
192
|
+
abort_listener.join(timeout=1)
|
|
193
|
+
|
|
194
|
+
sc.create_list(cdxfile, mode)
|
|
195
|
+
if not cdxbackup and not cdxinject:
|
|
196
|
+
os.remove(cdxfile)
|
|
197
|
+
else:
|
|
198
|
+
vb.write(message="\n-----> CDX backup generated")
|
|
161
199
|
|
|
200
|
+
snapshot_count = sc.count(collection=True)
|
|
201
|
+
vb.write(message=f"\n-----> {snapshot_count:,} snapshots to utilize")
|
|
162
202
|
|
|
163
203
|
|
|
164
204
|
|
|
@@ -7,19 +7,18 @@ import pywaybackup.archive as archive
|
|
|
7
7
|
from pywaybackup.Arguments import Configuration as config
|
|
8
8
|
from pywaybackup.Verbosity import Verbosity as vb
|
|
9
9
|
from pywaybackup.Exception import Exception as ex
|
|
10
|
-
from pywaybackup.Converter import Converter as convert
|
|
11
10
|
|
|
12
11
|
def main():
|
|
13
12
|
|
|
14
13
|
config.init()
|
|
15
|
-
ex.init(config.
|
|
14
|
+
ex.init(config.output, config.command)
|
|
16
15
|
vb.init(config.verbosity, config.log)
|
|
17
16
|
if config.save:
|
|
18
17
|
archive.save_page(config.url)
|
|
19
18
|
else:
|
|
20
19
|
try:
|
|
21
20
|
skipset = archive.skip_open(config.skip, config.url) if config.skip else None
|
|
22
|
-
archive.query_list(config.range, config.start, config.end, config.explicit, config.mode, config.cdxbackup, config.cdxinject)
|
|
21
|
+
archive.query_list(config.range, config.limit, config.start, config.end, config.explicit, config.mode, config.cdxbackup, config.cdxinject)
|
|
23
22
|
if config.list:
|
|
24
23
|
archive.print_list()
|
|
25
24
|
else:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: pywaybackup
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.5.0
|
|
4
4
|
Summary: Download snapshots from the Wayback Machine
|
|
5
5
|
Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
|
|
6
6
|
Author: bitdruid
|
|
@@ -44,10 +44,11 @@ This tool allows you to download content from the Wayback Machine (archive.org).
|
|
|
44
44
|
```pip install .```
|
|
45
45
|
- in a virtual env or use `--break-system-package`
|
|
46
46
|
|
|
47
|
-
## Usage infos
|
|
47
|
+
## Usage infos - important notes
|
|
48
48
|
|
|
49
49
|
- Linux recommended: On Windows machines, the path length is limited. This can only be overcome by editing the registry. Files that exceed the path length will not be downloaded.
|
|
50
50
|
- If you query an explicit file (e.g. a query-string `?query=this` or `login.html`), the `--explicit`-argument is recommended as a wildcard query may lead to an empty result.
|
|
51
|
+
- The tool will inform you if your query has an immense amount of snapshots which could consume your system memory and lead to a crash. Consider splitting your query into smaller jobs by specifying a range e.g. `--start 2023 --end 2024` or `--range 1`.
|
|
51
52
|
|
|
52
53
|
## Arguments
|
|
53
54
|
|
|
@@ -113,10 +114,13 @@ Specifies number of retry attempts for failed downloads.
|
|
|
113
114
|
- **`--delay`** `<seconds>`:<br>
|
|
114
115
|
Specifies delay between download requests in seconds. Default is no delay (0).
|
|
115
116
|
|
|
117
|
+
- **`--limit`** `<count>`:<br>
|
|
118
|
+
Limits the amount of snapshots to query from the CDX server. If an existing CDX file is injected (with `--cdxinject` or `--auto`), the limit will have no effect.
|
|
119
|
+
|
|
116
120
|
<!-- - **`--convert-links`**:<br>
|
|
117
121
|
If set, all links in the downloaded files will be converted to local links. This is useful for offline browsing. The links are converted to the local path structure. Show output with `--verbosity trace`. -->
|
|
118
122
|
|
|
119
|
-
**CDX Query Handling:**
|
|
123
|
+
**CDX Query Result Handling:**
|
|
120
124
|
- **`--cdxbackup`** `<path>`:<br>
|
|
121
125
|
Path defaults to output-dir. Saves the result of CDX query as a file. Useful for later downloading snapshots and overcoming refused connections by CDX server due to too many queries. Named as `waybackup_<sanitized_url>.cdx`.
|
|
122
126
|
|
|
@@ -127,10 +131,6 @@ Injects a CDX query file to download snapshots. Ensure the query matches the pre
|
|
|
127
131
|
- **`--auto`**:<br>
|
|
128
132
|
If set, csv, skip and cdxbackup/cdxinject are handled automatically. Keep the files and folders as they are. Otherwise they will not be recognized when restarting a download.
|
|
129
133
|
|
|
130
|
-
### Debug
|
|
131
|
-
|
|
132
|
-
- `--debug`: If set, full traceback will be printed in case of an error. The full exception will be written into `waybackup_error.log`.
|
|
133
|
-
|
|
134
134
|
### Examples
|
|
135
135
|
|
|
136
136
|
Download latest snapshot of all files:<br>
|
|
@@ -232,6 +232,10 @@ For list queries:
|
|
|
232
232
|
|
|
233
233
|
The csv contains the json response in a table format.
|
|
234
234
|
|
|
235
|
+
### Debugging
|
|
236
|
+
|
|
237
|
+
Exceptions will be written into `waybackup_error.log` (each run overwrites the file).
|
|
238
|
+
|
|
235
239
|
## Contributing
|
|
236
240
|
|
|
237
241
|
I'm always happy for some feature requests to improve the usability of this tool.
|
|
@@ -4,6 +4,8 @@ README.md
|
|
|
4
4
|
requirements.txt
|
|
5
5
|
setup.py
|
|
6
6
|
.github/ISSUE_TEMPLATE/bug.md
|
|
7
|
+
.github/ISSUE_TEMPLATE/feature_request.md
|
|
8
|
+
.github/workflows/stale.yml
|
|
7
9
|
dev/pip_build.sh
|
|
8
10
|
dev/venv_create.sh
|
|
9
11
|
pywaybackup/Arguments.py
|
|
@@ -24,4 +26,5 @@ pywaybackup.egg-info/requires.txt
|
|
|
24
26
|
pywaybackup.egg-info/top_level.txt
|
|
25
27
|
test/cdx.example.com.200
|
|
26
28
|
test/cdx.example.com.404
|
|
27
|
-
test/cdx.example.com.small.200
|
|
29
|
+
test/cdx.example.com.small.200
|
|
30
|
+
test/test_links.js
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// Example JavaScript File: example.js
|
|
2
|
+
|
|
3
|
+
// External script with absolute URL
|
|
4
|
+
var externalScript = document.createElement('script');
|
|
5
|
+
externalScript.src = 'http://example.com/js/external-script.js';
|
|
6
|
+
document.head.appendChild(externalScript);
|
|
7
|
+
|
|
8
|
+
// External script with relative URL
|
|
9
|
+
var localScript = document.createElement('script');
|
|
10
|
+
localScript.src = '/js/local-script.js';
|
|
11
|
+
document.head.appendChild(localScript);
|
|
12
|
+
|
|
13
|
+
// Inline style with absolute URL in background image
|
|
14
|
+
var element = document.createElement('div');
|
|
15
|
+
element.style.backgroundImage = "url('http://example.com/images/bg.png')";
|
|
16
|
+
document.body.appendChild(element);
|
|
17
|
+
|
|
18
|
+
// Inline style with relative URL in background image
|
|
19
|
+
element.style.backgroundImage = "url('/images/bg.png')";
|
|
20
|
+
|
|
21
|
+
// CSS in JavaScript with absolute URL
|
|
22
|
+
var css = "body { background-image: url('http://example.com/images/body-bg.png'); }";
|
|
23
|
+
var style = document.createElement('style');
|
|
24
|
+
style.type = 'text/css';
|
|
25
|
+
style.appendChild(document.createTextNode(css));
|
|
26
|
+
document.head.appendChild(style);
|
|
27
|
+
|
|
28
|
+
// CSS in JavaScript with relative URL
|
|
29
|
+
var cssLocal = "body { background-image: url('/images/body-bg.png'); }";
|
|
30
|
+
var styleLocal = document.createElement('style');
|
|
31
|
+
styleLocal.type = 'text/css';
|
|
32
|
+
styleLocal.appendChild(document.createTextNode(cssLocal));
|
|
33
|
+
document.head.appendChild(styleLocal);
|
|
34
|
+
|
|
35
|
+
// Image element with absolute URL
|
|
36
|
+
var imgElement = document.createElement('img');
|
|
37
|
+
imgElement.src = 'http://example.com/images/logo.png';
|
|
38
|
+
document.body.appendChild(imgElement);
|
|
39
|
+
|
|
40
|
+
// Image element with relative URL
|
|
41
|
+
var imgLocalElement = document.createElement('img');
|
|
42
|
+
imgLocalElement.src = '/images/logo.png';
|
|
43
|
+
document.body.appendChild(imgLocalElement);
|
|
44
|
+
|
|
45
|
+
// Fetch API call with absolute URL
|
|
46
|
+
fetch('http://example.com/api/data')
|
|
47
|
+
.then(response => response.json())
|
|
48
|
+
.then(data => console.log(data));
|
|
49
|
+
|
|
50
|
+
// Fetch API call with relative URL
|
|
51
|
+
fetch('/api/data')
|
|
52
|
+
.then(response => response.json())
|
|
53
|
+
.then(data => console.log(data));
|
|
54
|
+
|
|
55
|
+
// XMLHttpRequest with absolute URL
|
|
56
|
+
var xhr = new XMLHttpRequest();
|
|
57
|
+
xhr.open('GET', 'http://example.com/api/data', true);
|
|
58
|
+
xhr.send();
|
|
59
|
+
|
|
60
|
+
// XMLHttpRequest with relative URL
|
|
61
|
+
var xhrLocal = new XMLHttpRequest();
|
|
62
|
+
xhrLocal.open('GET', '/api/data', true);
|
|
63
|
+
xhrLocal.send();
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "1.4.2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|