datamaestro 1.2.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

datamaestro/download/wayback.py

@@ -0,0 +1,163 @@
+import logging
+import json
+from datamaestro.download import Resource
+from typing import Callable, Iterator
+from pathlib import Path
+import requests
+import random
+import re
+from requests.exceptions import HTTPError
+from tqdm.auto import tqdm
+import time
+import urllib.parse
+import uuid
+
+
+wayback_prefix = re.compile(r"^https:\/\/web\.archive\.org\/web")
+replace_pattern = re.compile(r"(web\.archive\.org\/web\/\d+)")
+
+
+def download_with_retry(url: str, max_retries: int = 10) -> requests.Response:
+    """Download a URL with exponential backoff, until max_retries is reached."""
+    retry_num = 0
+    while True:
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+            return response
+        except HTTPError as e:
+            status_code = e.response.status_code
+            if not (status_code == 429 or status_code >= 500):
+                # This is not an error we should retry on
+                raise e
+
+            if retry_num > max_retries:
+                logging.error(
+                    f"Failed to perform GET request on {url}"
+                    f"after {max_retries} retries."
+                )
+                raise e
+
+            if status_code == 429:
+                time.sleep(5 + 2**retry_num + random.randint(0, 1000) / 1000)
+            else:
+                time.sleep(2**retry_num + random.randint(0, 1000) / 1000)
+            retry_num += 1
+
+
+def download_link(link: str, timestamp: str):
+    page_id = str(uuid.uuid4())
+    url_no_header = None
+
+    try:
+        # Find the Wayback Machine link
+        if not wayback_prefix.match(link):
+            link_encoded = urllib.parse.quote(link)
+
+            available, availability_attempt = False, 0
+            # Sometimes the API returns HTTP success code 200, but archived
+            # snapshots shows page is unavailable when it actually is. Give it a
+            # total of three tries.
+            while not available and availability_attempt < 3:
+                response = download_with_retry(
+                    "http://archive.org/wayback/available?"
+                    f"url={link_encoded}&timestamp={timestamp}"
+                )
+                json_response = response.json()
+                available = "closest" in json_response["archived_snapshots"]
+                availability_attempt += 1
+
+            if not available:
+                logging.warning(
+                    f"Not available on Wayback Machine: {link}, "
+                    f"HTTP code {response.status_code}, {json_response}"
+                )
+                return {"link": link, "page_id": page_id, "available": False}
+
+            url = json_response["archived_snapshots"]["closest"]["url"]
+        else:
+            url = link
+
+        match = replace_pattern.search(url)
+        assert match
+        url_no_header = replace_pattern.sub(f"{match.group(1)}id_", url)
+
+        response = download_with_retry(url_no_header)
+        html_page = response.text
+
+        return {
+            "link": link,
+            "id": url_no_header,
+            "contents": html_page,
+        }
+
+    except HTTPError as http_err:
+        logging.warning(f"HTTP error occurred: {http_err} for {link}")
+        return {
+            "link": link,
+            "page_id": page_id,
+            "available": False,
+            "status_code": http_err.response.status_code if http_err.response else None,
+            "wayback_url": url_no_header,
+        }
+    except UnicodeDecodeError as e:
+        logging.warning(f"Unicode decode error occurred: {e} for {link}")
+        return {
+            "link": link,
+            "page_id": page_id,
+            "available": False,
+            "status_code": response.status_code,
+            "wayback_url": url_no_header,
+        }
+    except Exception as e:
+        logging.warning(f"Exception occurred: {e} for {link}")
+        return {
+            "link": link,
+            "page_id": page_id,
+            "available": False,
+            "status_code": None,
+            "wayback_url": url_no_header,
+        }
+
+
+class wayback_documents(Resource):
+    """Collect documents from wayback"""
+
+    def __init__(self, timestamp: str, urls_fn: Callable[[], Iterator[str]], name=None):
+        super().__init__(name)
+        self.timestamp = timestamp
+        self.urls_fn = urls_fn
+
+    def prepare(self):
+        return self.definition.datapath / self.varname
+
+    def download(self, force=False):
+        # Creates directory if needed
+        destination: Path = self.definition.datapath / self.varname
+        self.definition.datapath.mkdir(exist_ok=True)
+
+        # Early exit
+        done_path = destination.with_suffix(".done")
+        if done_path.is_file() and not force:
+            return True
+
+        # Reads the URLs
+        logging.info("Retrieving URLs from wayback")
+        pos = 0
+        urls = set()
+        with destination.open("at+") as fp:
+            fp.seek(0)
+            try:
+                for line in fp:
+                    pos = fp.tell()
+                    urls.add(json.loads(line)["url"])
+            except json.JSONDecodeError:
+                logging.warning(f"JSON decoding error: getting back to position {pos}")
+                fp.seek(pos)
+
+            # Get the remaining ones
+            for url in tqdm(self.urls_fn()):
+                fp.write(json.dumps(download_link(url, self.timestamp)))
+
+        # Everything is fine
+        done_path.touch()

datamaestro/version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.2.0'
-__version_tuple__ = version_tuple = (1, 2, 0)
+__version__ = version = '1.2.1'
+__version_tuple__ = version_tuple = (1, 2, 1)

{datamaestro-1.2.0.dist-info → datamaestro-1.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: datamaestro
-Version: 1.2.0
+Version: 1.2.1
 Summary: "Dataset management command line and API"
 Home-page: https://github.com/experimaestro/datamaestro
 Author: Benjamin Piwowarski

{datamaestro-1.2.0.dist-info → datamaestro-1.2.1.dist-info}/RECORD

@@ -8,7 +8,7 @@ datamaestro/search.py,sha256=PMceNp5hcp0dlzs4cLb6LJT7XHrdXo58oO7oTucawbE,2887
 datamaestro/settings.py,sha256=HYSElTUYZ6DZocBb9o3ifm6WW9knRO64XJUwxGIpvwQ,1304
 datamaestro/sphinx.py,sha256=bp7x_2BFoTSwTqcVZDM8R8cWa7G2pz0Zb8GS054lLYM,6996
 datamaestro/utils.py,sha256=Y3_aqeOHW8vuifwggGWJfgONyDG1FLX7ONAnX85jENI,6511
-datamaestro/version.py,sha256=zMnMemknXglcJs59xkicNzeEJTVgYd1omSfLWj76yWw,411
+datamaestro/version.py,sha256=2U0Gn26fYI3Vgj5hgkLM8I3wI6YEVdffJGllaVW-sSc,411
 datamaestro/annotations/__init__.py,sha256=jLprrxSBa5QIqc--vqycEcxU4CR9WjVNRaqR5lH0EuE,39
 datamaestro/annotations/agreement.py,sha256=xEH0ddZxdJ_oG_150PoOa-WjY_OaeQja3FzMzY5IB6k,955
 datamaestro/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -28,6 +28,7 @@ datamaestro/download/multiple.py,sha256=Mrr0ObHM5cE1CPSHE9PKIrox3qZVgxwRyxLzNXp0
 datamaestro/download/single.py,sha256=QSEviTP9lHLh3ZGyo_KoW3ro8UvWCGNPHeZiNj-9rLA,4134
 datamaestro/download/sync.py,sha256=Z_LsXj4kbZWIYKTVJZEhfdpYiv6wXOOIyw8LahmEcqs,836
 datamaestro/download/todo.py,sha256=y3YnmWC_i-u23ce-vreIwIXZcoO-uA0HXErgJPThnco,256
+datamaestro/download/wayback.py,sha256=B9X1P9jElvd_qnUs9aX0TAO-NrNyvuHLYDAcpNq354w,5430
 datamaestro/stream/__init__.py,sha256=Angu_Yg9rNKXb8s4at-DXYcnE-OTgSMLfUEfrL6APD8,896
 datamaestro/stream/compress.py,sha256=0ViFGpJc6pdvZGUNERE-3XV8jAOTSvhJurb2t0NW2eU,260
 datamaestro/stream/lines.py,sha256=UNGcyZlZxN0Q7kw717jbhZFdDVmtfJfkJZCgK7xzF9A,1996
@@ -38,9 +39,9 @@ datamaestro/test/conftest.py,sha256=it4S5Qq1CA_U8qM0pr4m7v-1dhLj5Y49WjVg5Ee3mpM,
 datamaestro/test/test_annotations.py,sha256=kRPUmS_UAN6JSSVPUwV4OM_LEuEUHF1OcLSiYXjsKjw,246
 datamaestro/test/test_download_handlers.py,sha256=Qqm-fML1KVp6dPwAUcH6xzi_dpQIshvROzviSYCUzc0,603
 datamaestro/test/test_record.py,sha256=hNZ3uo2i5FZ0VsOHRwvLO1Z6Zce92PdipAF65UptPB8,1156
-datamaestro-1.2.0.dist-info/LICENSE,sha256=WJ7YI-moTFb-uVrFjnzzhGJrnL9P2iqQe8NuED3hutI,35141
-datamaestro-1.2.0.dist-info/METADATA,sha256=RM87g0kAI517bjmcEFlvRiGC6Dxy1xmiWt0gYG1eZEs,8999
-datamaestro-1.2.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-datamaestro-1.2.0.dist-info/entry_points.txt,sha256=8qMhwSRvFG2iBqtJYVD22Zd4s4c3YkODtcp0Ajw1knw,133
-datamaestro-1.2.0.dist-info/top_level.txt,sha256=XSznaMNAA8jELV7-TOqaAgDsjLzUf9G9MxL7C4helT0,12
-datamaestro-1.2.0.dist-info/RECORD,,
+datamaestro-1.2.1.dist-info/LICENSE,sha256=WJ7YI-moTFb-uVrFjnzzhGJrnL9P2iqQe8NuED3hutI,35141
+datamaestro-1.2.1.dist-info/METADATA,sha256=2_TL_ysMtfV2a84_0Uu3UQloCHCvetGZWo5tcjdhNCA,8999
+datamaestro-1.2.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+datamaestro-1.2.1.dist-info/entry_points.txt,sha256=8qMhwSRvFG2iBqtJYVD22Zd4s4c3YkODtcp0Ajw1knw,133
+datamaestro-1.2.1.dist-info/top_level.txt,sha256=XSznaMNAA8jELV7-TOqaAgDsjLzUf9G9MxL7C4helT0,12
+datamaestro-1.2.1.dist-info/RECORD,,