pywaybackup 1.4.2__tar.gz → 1.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. pywaybackup-1.5.0/.github/ISSUE_TEMPLATE/feature_request.md +17 -0
  2. pywaybackup-1.5.0/.github/workflows/stale.yml +20 -0
  3. {pywaybackup-1.4.2/pywaybackup.egg-info → pywaybackup-1.5.0}/PKG-INFO +11 -7
  4. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/README.md +10 -6
  5. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/Arguments.py +3 -1
  6. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/Exception.py +26 -29
  7. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/SnapshotCollection.py +16 -3
  8. pywaybackup-1.5.0/pywaybackup/__version__.py +1 -0
  9. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/archive.py +69 -29
  10. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/helper.py +0 -2
  11. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/main.py +2 -3
  12. {pywaybackup-1.4.2 → pywaybackup-1.5.0/pywaybackup.egg-info}/PKG-INFO +11 -7
  13. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/SOURCES.txt +4 -1
  14. pywaybackup-1.5.0/test/test_links.js +63 -0
  15. pywaybackup-1.4.2/pywaybackup/__version__.py +0 -1
  16. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/.github/ISSUE_TEMPLATE/bug.md +0 -0
  17. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/.gitignore +0 -0
  18. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/LICENSE +0 -0
  19. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/dev/pip_build.sh +0 -0
  20. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/dev/venv_create.sh +0 -0
  21. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/Converter.py +0 -0
  22. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/Verbosity.py +0 -0
  23. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup/__init__.py +0 -0
  24. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/dependency_links.txt +0 -0
  25. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/entry_points.txt +0 -0
  26. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/requires.txt +0 -0
  27. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/pywaybackup.egg-info/top_level.txt +0 -0
  28. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/requirements.txt +0 -0
  29. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/setup.cfg +0 -0
  30. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/setup.py +0 -0
  31. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/test/cdx.example.com.200 +0 -0
  32. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/test/cdx.example.com.404 +0 -0
  33. {pywaybackup-1.4.2 → pywaybackup-1.5.0}/test/cdx.example.com.small.200 +0 -0
@@ -0,0 +1,17 @@
1
+ ---
2
+ name: Feature request
3
+ about: Suggest an idea for this project
4
+ title: "[FEATURE]"
5
+ labels: enhancement
6
+ assignees: ''
7
+
8
+ ---
9
+
10
+ **Is your feature request related to a problem? Please describe.**
11
+ Describe the problem (not a bug - more of a usage issue).
12
+
13
+ **Describe the solution you'd like**
14
+ How should the feature improve your usage?
15
+
16
+ **Additional context**
17
+ Any other context or screenshots about the feature.
@@ -0,0 +1,20 @@
1
+ name: 'Close by inactivity'
2
+ on:
3
+ schedule:
4
+ - cron: "00 00 * * *"
5
+ workflow_dispatch:
6
+
7
+ jobs:
8
+ close-issues:
9
+ runs-on: ubuntu-latest
10
+ permissions:
11
+ issues: write
12
+ pull-requests: write
13
+ steps:
14
+ - uses: actions/stale@v9.0.0
15
+ with:
16
+ days-before-issue-stale: 30
17
+ days-before-issue-close: 14
18
+ stale-issue-label: "stale"
19
+ stale-issue-message: "This issue is marked as stale because there was no activity for 30 days."
20
+ close-issue-message: "This issue has been closed because there has been no activity for 14 days while it was marked as stale."
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pywaybackup
3
- Version: 1.4.2
3
+ Version: 1.5.0
4
4
  Summary: Download snapshots from the Wayback Machine
5
5
  Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
6
6
  Author: bitdruid
@@ -44,10 +44,11 @@ This tool allows you to download content from the Wayback Machine (archive.org).
44
44
  ```pip install .```
45
45
  - in a virtual env or use `--break-system-package`
46
46
 
47
- ## Usage infos
47
+ ## Usage infos - important notes
48
48
 
49
49
  - Linux recommended: On Windows machines, the path length is limited. This can only be overcome by editing the registry. Files that exceed the path length will not be downloaded.
50
50
  - If you query an explicit file (e.g. a query-string `?query=this` or `login.html`), the `--explicit`-argument is recommended as a wildcard query may lead to an empty result.
51
+ - The tool will inform you if your query has an immense amount of snapshots which could consume your system memory and lead to a crash. Consider splitting your query into smaller jobs by specifying a range e.g. `--start 2023 --end 2024` or `--range 1`.
51
52
 
52
53
  ## Arguments
53
54
 
@@ -113,10 +114,13 @@ Specifies number of retry attempts for failed downloads.
113
114
  - **`--delay`** `<seconds>`:<br>
114
115
  Specifies delay between download requests in seconds. Default is no delay (0).
115
116
 
117
+ - **`--limit`** `<count>`:<br>
118
+ Limits the amount of snapshots to query from the CDX server. If an existing CDX file is injected (with `--cdxinject` or `--auto`), the limit will have no effect.
119
+
116
120
  <!-- - **`--convert-links`**:<br>
117
121
  If set, all links in the downloaded files will be converted to local links. This is useful for offline browsing. The links are converted to the local path structure. Show output with `--verbosity trace`. -->
118
122
 
119
- **CDX Query Handling:**
123
+ **CDX Query Result Handling:**
120
124
  - **`--cdxbackup`** `<path>`:<br>
121
125
  Path defaults to output-dir. Saves the result of CDX query as a file. Useful for later downloading snapshots and overcoming refused connections by CDX server due to too many queries. Named as `waybackup_<sanitized_url>.cdx`.
122
126
 
@@ -127,10 +131,6 @@ Injects a CDX query file to download snapshots. Ensure the query matches the pre
127
131
  - **`--auto`**:<br>
128
132
  If set, csv, skip and cdxbackup/cdxinject are handled automatically. Keep the files and folders as they are. Otherwise they will not be recognized when restarting a download.
129
133
 
130
- ### Debug
131
-
132
- - `--debug`: If set, full traceback will be printed in case of an error. The full exception will be written into `waybackup_error.log`.
133
-
134
134
  ### Examples
135
135
 
136
136
  Download latest snapshot of all files:<br>
@@ -232,6 +232,10 @@ For list queries:
232
232
 
233
233
  The csv contains the json response in a table format.
234
234
 
235
+ ### Debugging
236
+
237
+ Exceptions will be written into `waybackup_error.log` (each run overwrites the file).
238
+
235
239
  ## Contributing
236
240
 
237
241
  I'm always happy for some feature requests to improve the usability of this tool.
@@ -28,10 +28,11 @@ This tool allows you to download content from the Wayback Machine (archive.org).
28
28
  ```pip install .```
29
29
  - in a virtual env or use `--break-system-package`
30
30
 
31
- ## Usage infos
31
+ ## Usage infos - important notes
32
32
 
33
33
  - Linux recommended: On Windows machines, the path length is limited. This can only be overcome by editing the registry. Files that exceed the path length will not be downloaded.
34
34
  - If you query an explicit file (e.g. a query-string `?query=this` or `login.html`), the `--explicit`-argument is recommended as a wildcard query may lead to an empty result.
35
+ - The tool will inform you if your query has an immense amount of snapshots which could consume your system memory and lead to a crash. Consider splitting your query into smaller jobs by specifying a range e.g. `--start 2023 --end 2024` or `--range 1`.
35
36
 
36
37
  ## Arguments
37
38
 
@@ -97,10 +98,13 @@ Specifies number of retry attempts for failed downloads.
97
98
  - **`--delay`** `<seconds>`:<br>
98
99
  Specifies delay between download requests in seconds. Default is no delay (0).
99
100
 
101
+ - **`--limit`** `<count>`:<br>
102
+ Limits the amount of snapshots to query from the CDX server. If an existing CDX file is injected (with `--cdxinject` or `--auto`), the limit will have no effect.
103
+
100
104
  <!-- - **`--convert-links`**:<br>
101
105
  If set, all links in the downloaded files will be converted to local links. This is useful for offline browsing. The links are converted to the local path structure. Show output with `--verbosity trace`. -->
102
106
 
103
- **CDX Query Handling:**
107
+ **CDX Query Result Handling:**
104
108
  - **`--cdxbackup`** `<path>`:<br>
105
109
  Path defaults to output-dir. Saves the result of CDX query as a file. Useful for later downloading snapshots and overcoming refused connections by CDX server due to too many queries. Named as `waybackup_<sanitized_url>.cdx`.
106
110
 
@@ -111,10 +115,6 @@ Injects a CDX query file to download snapshots. Ensure the query matches the pre
111
115
  - **`--auto`**:<br>
112
116
  If set, csv, skip and cdxbackup/cdxinject are handled automatically. Keep the files and folders as they are. Otherwise they will not be recognized when restarting a download.
113
117
 
114
- ### Debug
115
-
116
- - `--debug`: If set, full traceback will be printed in case of an error. The full exception will be written into `waybackup_error.log`.
117
-
118
118
  ### Examples
119
119
 
120
120
  Download latest snapshot of all files:<br>
@@ -216,6 +216,10 @@ For list queries:
216
216
 
217
217
  The csv contains the json response in a table format.
218
218
 
219
+ ### Debugging
220
+
221
+ Exceptions will be written into `waybackup_error.log` (each run overwrites the file).
222
+
219
223
  ## Contributing
220
224
 
221
225
  I'm always happy for some feature requests to improve the usability of this tool.
@@ -13,7 +13,6 @@ class Arguments:
13
13
 
14
14
  parser = argparse.ArgumentParser(description='Download from wayback machine (archive.org)')
15
15
  parser.add_argument('-a', '--about', action='version', version='%(prog)s ' + __version__ + ' by @bitdruid -> https://github.com/bitdruid')
16
- parser.add_argument('-d', '--debug', action='store_true', help='Debug mode (Always full traceback and creates an error.log')
17
16
 
18
17
  required = parser.add_argument_group('required (one exclusive)')
19
18
  required.add_argument('-u', '--url', type=str, metavar="", help='url (with subdir/subdomain) to download')
@@ -40,6 +39,7 @@ class Arguments:
40
39
  special.add_argument('--workers', type=int, default=1, metavar="", help='number of workers (simultaneous downloads)')
41
40
  # special.add_argument('--convert-links', action='store_true', help='Convert all links in the files to local paths. Requires -c/--current')
42
41
  special.add_argument('--delay', type=int, default=0, metavar="", help='delay between each download in seconds')
42
+ special.add_argument('--limit', type=int, nargs='?', const=True, metavar='int', help='limit the number of snapshots to download')
43
43
 
44
44
  cdx = parser.add_argument_group('cdx (one exclusive)')
45
45
  exclusive_cdx = cdx.add_mutually_exclusive_group()
@@ -84,6 +84,8 @@ class Configuration:
84
84
  if cls.current:
85
85
  cls.mode = "current"
86
86
 
87
+ cls.cdxbackup = cls.output if cls.cdxbackup is None else cls.cdxbackup
88
+
87
89
  if cls.auto:
88
90
  cls.skip = cls.output
89
91
  cls.csv = cls.output
@@ -12,16 +12,14 @@ from pywaybackup.__version__ import __version__
12
12
  class Exception:
13
13
 
14
14
  new_debug = True
15
- debug = False
16
15
  output = None
17
16
  command = None
18
17
 
19
18
  @classmethod
20
- def init(cls, debug=False, output=None, command=None):
19
+ def init(cls, output=None, command=None):
21
20
  sys.excepthook = cls.exception_handler # set custom exception handler (uncaught exceptions)
22
21
  cls.output = output
23
22
  cls.command = command
24
- cls.debug = True if debug else False
25
23
 
26
24
  @classmethod
27
25
  def exception(cls, message: str, e: Exception, tb=None):
@@ -53,33 +51,32 @@ class Exception:
53
51
  "-------------------------"
54
52
  )
55
53
  print(exception_message)
56
- if cls.debug:
57
- debug_file = os.path.join(cls.output, "waybackup_error.log")
58
- print(f"Exception log: {debug_file}")
59
- print("-------------------------")
60
- print(f"Full traceback:\n{original_tb}")
61
- if cls.new_debug: # new run, overwrite file
62
- cls.new_debug = False
63
- f = open(debug_file, "w")
64
- f.write("-------------------------\n")
65
- f.write(f"Version: {__version__}\n")
66
- f.write("-------------------------\n")
67
- f.write(f"Command: {cls.command}\n")
68
- f.write("-------------------------\n\n")
69
- else: # current run, append to file
70
- f = open(debug_file, "a")
71
- f.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n")
72
- f.write(exception_message + "\n")
73
- f.write("!-- Local Variables:\n")
74
- for var_name, value in local_vars.items():
75
- if var_name in ["status_message", "headers"]:
76
- continue
77
- value = cls.relativate_path(str(value))
78
- value = value[:666] + " ... " if len(value) > 666 else value
79
- f.write(f" -- {var_name} = {value}\n")
54
+ debug_file = os.path.join(cls.output, "waybackup_error.log")
55
+ print(f"Exception log: {debug_file}")
56
+ # print("-------------------------")
57
+ # print(f"Full traceback:\n{original_tb}")
58
+ if cls.new_debug: # new run, overwrite file
59
+ cls.new_debug = False
60
+ f = open(debug_file, "w")
80
61
  f.write("-------------------------\n")
81
- f.write(original_tb + "\n")
82
- f.close()
62
+ f.write(f"Version: {__version__}\n")
63
+ f.write("-------------------------\n")
64
+ f.write(f"Command: {cls.command}\n")
65
+ f.write("-------------------------\n\n")
66
+ else: # current run, append to file
67
+ f = open(debug_file, "a")
68
+ f.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n")
69
+ f.write(exception_message + "\n")
70
+ f.write("!-- Local Variables:\n")
71
+ for var_name, value in local_vars.items():
72
+ if var_name in ["status_message", "headers"]:
73
+ continue
74
+ value = cls.relativate_path(str(value))
75
+ value = value[:666] + " ... " if len(value) > 666 else value
76
+ f.write(f" -- {var_name} = {value}\n")
77
+ f.write("-------------------------\n")
78
+ f.write(original_tb + "\n")
79
+ f.close()
83
80
 
84
81
  @classmethod
85
82
  def relativate_path(cls, input: str) -> str:
@@ -1,4 +1,5 @@
1
1
  from pywaybackup.helper import url_split
2
+ import json
2
3
  import os
3
4
 
4
5
  class SnapshotCollection:
@@ -7,15 +8,27 @@ class SnapshotCollection:
7
8
  MODE_CURRENT = 0
8
9
 
9
10
  @classmethod
10
- def create_list(cls, cdxResult, mode):
11
+ def create_list(cls, cdxfile, mode):
11
12
  """
12
13
  Create the snapshot collection list from a cdx result.
13
14
 
14
15
  - mode `full`: All snapshots are included.
15
16
  - mode `current`: Only the latest snapshot of each file is included.
16
17
  """
17
- # creates a list of dictionaries for each snapshot entry
18
- cls.SNAPSHOT_COLLECTION = sorted([{"timestamp": snapshot[0], "digest": snapshot[1], "mimetype": snapshot[2], "status": snapshot[3], "url": snapshot[4]} for snapshot in cdxResult[1:]], key=lambda k: k['timestamp'], reverse=True)
18
+ with open(cdxfile, "r") as f:
19
+ first_line = True
20
+ for line in f:
21
+ if first_line:
22
+ first_line = False
23
+ continue
24
+ line = line.strip()
25
+ if line.endswith("]]"): line = line.rsplit("]", 1)[0]
26
+ if line.endswith(","): line = line.rsplit(",", 1)[0]
27
+ else: continue # drop incomplete line, maybe cdx response was cut off
28
+ line = json.loads(line)
29
+ line = {"timestamp": line[0], "digest": line[1], "mimetype": line[2], "status": line[3], "url": line[4]}
30
+ cls.SNAPSHOT_COLLECTION.append(line)
31
+ cls.SNAPSHOT_COLLECTION = sorted(cls.SNAPSHOT_COLLECTION, key=lambda k: k['timestamp'], reverse=True)
19
32
  if mode == "current":
20
33
  cls.MODE_CURRENT = 1
21
34
  cdxResult_list_filtered = []
@@ -0,0 +1 @@
1
+ __version__ = "1.5.0"
@@ -5,12 +5,13 @@ import csv
5
5
  import threading
6
6
  import queue
7
7
  import time
8
- import json
9
8
  import urllib.parse
10
9
  import http.client
11
10
  from urllib.parse import urljoin
12
11
  from datetime import datetime, timezone
13
12
 
13
+ from tqdm import tqdm
14
+
14
15
  from socket import timeout
15
16
 
16
17
  from pywaybackup.helper import url_get_timestamp, move_index, sanitize_filename, check_nt
@@ -20,10 +21,10 @@ from pywaybackup.Arguments import Configuration as config
20
21
 
21
22
  from pywaybackup.__version__ import __version__
22
23
 
23
- from pywaybackup.Converter import Converter as convert
24
24
  from pywaybackup.Verbosity import Message
25
25
  from pywaybackup.Verbosity import Verbosity as vb
26
26
  from pywaybackup.Exception import Exception as ex
27
+ import threading
27
28
 
28
29
 
29
30
 
@@ -99,28 +100,41 @@ def print_list():
99
100
 
100
101
  # create filelist
101
102
  # timestamp format yyyyMMddhhmmss
102
- def query_list(range: int, start: int, end: int, explicit: bool, mode: str, cdxbackup: str, cdxinject: str):
103
+ def query_list(queryrange: int, limit: int, start: int, end: int, explicit: bool, mode: str, cdxbackup: str, cdxinject: str):
104
+
105
+ def input_countdown():
106
+ for i in range(10, -1, -1):
107
+ vb.write(message=f"{i}")
108
+ print("\033[F", end="")
109
+ print("\033[K", end="")
110
+ time.sleep(1)
111
+
112
+ def input_detection():
113
+ input()
114
+ vb.write(message="\nExiting...")
115
+ os._exit(1)
116
+
117
+ def count_cdxfile(cdxfile):
118
+ with open(cdxfile, "r") as file:
119
+ return file.read().count("\n") - 1
103
120
 
104
121
  def inject(cdxinject):
105
122
  if os.path.isfile(cdxinject):
106
123
  vb.write(message="\nInjecting CDX data...")
107
- cdxResult = open(cdxinject, "r")
108
- cdxResult = cdxResult.read()
109
- linecount = cdxResult.count("\n") - 1
110
- vb.write(message=f"\n-----> {linecount} snapshots injected")
111
- return cdxResult
124
+ vb.write(message=f"\n-----> {count_cdxfile(cdxinject):,} lines injected")
125
+ return cdxinject
112
126
  else:
113
127
  vb.write(message="\nNo CDX file found to inject - querying snapshots...")
114
128
  return False
115
129
 
116
- def query(range, start, end, explicit):
130
+ def query(queryrange, limit, start, end, explicit):
117
131
  vb.write(message="\nQuerying snapshots...")
118
132
  query_range = ""
119
- if not range:
133
+ if not queryrange:
120
134
  if start: query_range = query_range + f"&from={start}"
121
135
  if end: query_range = query_range + f"&to={end}"
122
136
  else:
123
- query_range = "&from=" + str(datetime.now().year - range)
137
+ query_range = "&from=" + str(datetime.now().year - queryrange)
124
138
 
125
139
  if config.domain and not config.subdir and not config.filename:
126
140
  cdx_url = f"{config.domain}"
@@ -133,32 +147,58 @@ def query_list(range: int, start: int, end: int, explicit: bool, mode: str, cdxb
133
147
  if not explicit:
134
148
  cdx_url = f"{cdx_url}/*"
135
149
 
136
- vb.write(message=f"---> {cdx_url}")
137
- cdxQuery = f"https://web.archive.org/cdx/search/cdx?output=json&url={cdx_url}{query_range}&fl=timestamp,digest,mimetype,statuscode,original&filter!=statuscode:200"
150
+ limit = f"limit={limit}&" if limit else ""
151
+
152
+ vb.write(message=f"-----> {cdx_url}")
153
+ cdxQuery = f"https://web.archive.org/cdx/search/cdx?output=json&url={cdx_url}{query_range}&fl=timestamp,digest,mimetype,statuscode,original&{limit}filter!=statuscode:200"
154
+
155
+ cdxfile = os.path.join(cdxbackup, f"waybackup_{sanitize_filename(config.url)}.cdx")
138
156
 
139
157
  try:
140
- cdxResult = requests.get(cdxQuery).text
141
- except requests.exceptions.ConnectionError as e:
158
+ cdxfile_IO = open(cdxfile, "w")
159
+ with requests.get(cdxQuery, stream=True) as r:
160
+ r.raise_for_status()
161
+ with tqdm(unit='B', unit_scale=True, desc='-----> Downloading CDX result') as pbar:
162
+ for chunk in r.iter_content(chunk_size=8192):
163
+ if chunk:
164
+ pbar.update(len(chunk))
165
+ chunk = chunk.decode("utf-8")
166
+ cdxfile_IO.write(chunk)
167
+ cdxfile_IO.close()
168
+ except requests.exceptions.ConnectionError:
142
169
  vb.write(message="\nCONNECTION REFUSED -> could not query cdx server (max retries exceeded)\n")
143
170
  os._exit(1)
144
-
145
- if cdxbackup:
146
- os.makedirs(cdxbackup, exist_ok=True)
147
- with open(os.path.join(cdxbackup, f"waybackup_{sanitize_filename(config.url)}.cdx"), "w") as file:
148
- file.write(cdxResult)
149
- vb.write(message="\n-----> CDX backup generated")
150
171
 
151
- return cdxResult
172
+ return cdxfile
152
173
 
153
- cdxResult = None
174
+ cdxfile = None
154
175
  if cdxinject:
155
- cdxResult = inject(cdxinject)
156
- if not cdxResult:
157
- cdxResult = query(range, start, end, explicit)
158
- cdxResult = json.loads(cdxResult)
159
- sc.create_list(cdxResult, mode)
160
- vb.write(message=f"\n-----> {sc.count(collection=True)} snapshots to utilize")
176
+ cdxfile = inject(cdxinject)
177
+ if not cdxfile:
178
+ cdxfile = query(queryrange, limit, start, end, explicit)
179
+
180
+ snapshot_count = count_cdxfile(cdxfile) - 1
181
+ if snapshot_count > 1000000:
182
+ vb.write(message="\n!!!!! WARNING")
183
+ vb.write(message="Excessive amount of snapshots detected. System may run out of memory!")
184
+ vb.write(message="---> Consider splitting the query into smaller jobs (range start/end).")
185
+ vb.write(message="\nPress ANY key to abort...")
186
+
187
+ abort_listener = threading.Thread(target=input_detection)
188
+ abort_listener.start()
189
+
190
+ input_countdown()
191
+
192
+ abort_listener.join(timeout=1)
193
+
194
+ sc.create_list(cdxfile, mode)
195
+ if not cdxbackup and not cdxinject:
196
+ os.remove(cdxfile)
197
+ else:
198
+ vb.write(message="\n-----> CDX backup generated")
161
199
 
200
+ snapshot_count = sc.count(collection=True)
201
+ vb.write(message=f"\n-----> {snapshot_count:,} snapshots to utilize")
162
202
 
163
203
 
164
204
 
@@ -1,8 +1,6 @@
1
1
 
2
2
  import os
3
- import re
4
3
  import shutil
5
- from urllib.parse import urlparse, urljoin
6
4
  import magic
7
5
 
8
6
 
@@ -7,19 +7,18 @@ import pywaybackup.archive as archive
7
7
  from pywaybackup.Arguments import Configuration as config
8
8
  from pywaybackup.Verbosity import Verbosity as vb
9
9
  from pywaybackup.Exception import Exception as ex
10
- from pywaybackup.Converter import Converter as convert
11
10
 
12
11
  def main():
13
12
 
14
13
  config.init()
15
- ex.init(config.debug, config.output, config.command)
14
+ ex.init(config.output, config.command)
16
15
  vb.init(config.verbosity, config.log)
17
16
  if config.save:
18
17
  archive.save_page(config.url)
19
18
  else:
20
19
  try:
21
20
  skipset = archive.skip_open(config.skip, config.url) if config.skip else None
22
- archive.query_list(config.range, config.start, config.end, config.explicit, config.mode, config.cdxbackup, config.cdxinject)
21
+ archive.query_list(config.range, config.limit, config.start, config.end, config.explicit, config.mode, config.cdxbackup, config.cdxinject)
23
22
  if config.list:
24
23
  archive.print_list()
25
24
  else:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: pywaybackup
3
- Version: 1.4.2
3
+ Version: 1.5.0
4
4
  Summary: Download snapshots from the Wayback Machine
5
5
  Home-page: https://github.com/bitdruid/python-wayback-machine-downloader
6
6
  Author: bitdruid
@@ -44,10 +44,11 @@ This tool allows you to download content from the Wayback Machine (archive.org).
44
44
  ```pip install .```
45
45
  - in a virtual env or use `--break-system-package`
46
46
 
47
- ## Usage infos
47
+ ## Usage infos - important notes
48
48
 
49
49
  - Linux recommended: On Windows machines, the path length is limited. This can only be overcome by editing the registry. Files that exceed the path length will not be downloaded.
50
50
  - If you query an explicit file (e.g. a query-string `?query=this` or `login.html`), the `--explicit`-argument is recommended as a wildcard query may lead to an empty result.
51
+ - The tool will inform you if your query has an immense amount of snapshots which could consume your system memory and lead to a crash. Consider splitting your query into smaller jobs by specifying a range e.g. `--start 2023 --end 2024` or `--range 1`.
51
52
 
52
53
  ## Arguments
53
54
 
@@ -113,10 +114,13 @@ Specifies number of retry attempts for failed downloads.
113
114
  - **`--delay`** `<seconds>`:<br>
114
115
  Specifies delay between download requests in seconds. Default is no delay (0).
115
116
 
117
+ - **`--limit`** `<count>`:<br>
118
+ Limits the amount of snapshots to query from the CDX server. If an existing CDX file is injected (with `--cdxinject` or `--auto`), the limit will have no effect.
119
+
116
120
  <!-- - **`--convert-links`**:<br>
117
121
  If set, all links in the downloaded files will be converted to local links. This is useful for offline browsing. The links are converted to the local path structure. Show output with `--verbosity trace`. -->
118
122
 
119
- **CDX Query Handling:**
123
+ **CDX Query Result Handling:**
120
124
  - **`--cdxbackup`** `<path>`:<br>
121
125
  Path defaults to output-dir. Saves the result of CDX query as a file. Useful for later downloading snapshots and overcoming refused connections by CDX server due to too many queries. Named as `waybackup_<sanitized_url>.cdx`.
122
126
 
@@ -127,10 +131,6 @@ Injects a CDX query file to download snapshots. Ensure the query matches the pre
127
131
  - **`--auto`**:<br>
128
132
  If set, csv, skip and cdxbackup/cdxinject are handled automatically. Keep the files and folders as they are. Otherwise they will not be recognized when restarting a download.
129
133
 
130
- ### Debug
131
-
132
- - `--debug`: If set, full traceback will be printed in case of an error. The full exception will be written into `waybackup_error.log`.
133
-
134
134
  ### Examples
135
135
 
136
136
  Download latest snapshot of all files:<br>
@@ -232,6 +232,10 @@ For list queries:
232
232
 
233
233
  The csv contains the json response in a table format.
234
234
 
235
+ ### Debugging
236
+
237
+ Exceptions will be written into `waybackup_error.log` (each run overwrites the file).
238
+
235
239
  ## Contributing
236
240
 
237
241
  I'm always happy for some feature requests to improve the usability of this tool.
@@ -4,6 +4,8 @@ README.md
4
4
  requirements.txt
5
5
  setup.py
6
6
  .github/ISSUE_TEMPLATE/bug.md
7
+ .github/ISSUE_TEMPLATE/feature_request.md
8
+ .github/workflows/stale.yml
7
9
  dev/pip_build.sh
8
10
  dev/venv_create.sh
9
11
  pywaybackup/Arguments.py
@@ -24,4 +26,5 @@ pywaybackup.egg-info/requires.txt
24
26
  pywaybackup.egg-info/top_level.txt
25
27
  test/cdx.example.com.200
26
28
  test/cdx.example.com.404
27
- test/cdx.example.com.small.200
29
+ test/cdx.example.com.small.200
30
+ test/test_links.js
@@ -0,0 +1,63 @@
1
+ // Example JavaScript File: example.js
2
+
3
+ // External script with absolute URL
4
+ var externalScript = document.createElement('script');
5
+ externalScript.src = 'http://example.com/js/external-script.js';
6
+ document.head.appendChild(externalScript);
7
+
8
+ // External script with relative URL
9
+ var localScript = document.createElement('script');
10
+ localScript.src = '/js/local-script.js';
11
+ document.head.appendChild(localScript);
12
+
13
+ // Inline style with absolute URL in background image
14
+ var element = document.createElement('div');
15
+ element.style.backgroundImage = "url('http://example.com/images/bg.png')";
16
+ document.body.appendChild(element);
17
+
18
+ // Inline style with relative URL in background image
19
+ element.style.backgroundImage = "url('/images/bg.png')";
20
+
21
+ // CSS in JavaScript with absolute URL
22
+ var css = "body { background-image: url('http://example.com/images/body-bg.png'); }";
23
+ var style = document.createElement('style');
24
+ style.type = 'text/css';
25
+ style.appendChild(document.createTextNode(css));
26
+ document.head.appendChild(style);
27
+
28
+ // CSS in JavaScript with relative URL
29
+ var cssLocal = "body { background-image: url('/images/body-bg.png'); }";
30
+ var styleLocal = document.createElement('style');
31
+ styleLocal.type = 'text/css';
32
+ styleLocal.appendChild(document.createTextNode(cssLocal));
33
+ document.head.appendChild(styleLocal);
34
+
35
+ // Image element with absolute URL
36
+ var imgElement = document.createElement('img');
37
+ imgElement.src = 'http://example.com/images/logo.png';
38
+ document.body.appendChild(imgElement);
39
+
40
+ // Image element with relative URL
41
+ var imgLocalElement = document.createElement('img');
42
+ imgLocalElement.src = '/images/logo.png';
43
+ document.body.appendChild(imgLocalElement);
44
+
45
+ // Fetch API call with absolute URL
46
+ fetch('http://example.com/api/data')
47
+ .then(response => response.json())
48
+ .then(data => console.log(data));
49
+
50
+ // Fetch API call with relative URL
51
+ fetch('/api/data')
52
+ .then(response => response.json())
53
+ .then(data => console.log(data));
54
+
55
+ // XMLHttpRequest with absolute URL
56
+ var xhr = new XMLHttpRequest();
57
+ xhr.open('GET', 'http://example.com/api/data', true);
58
+ xhr.send();
59
+
60
+ // XMLHttpRequest with relative URL
61
+ var xhrLocal = new XMLHttpRequest();
62
+ xhrLocal.open('GET', '/api/data', true);
63
+ xhrLocal.send();
@@ -1 +0,0 @@
1
- __version__ = "1.4.2"
File without changes
File without changes
File without changes
File without changes