warn-scraper 1.2.155.dev0__py3-none-any.whl → 1.2.156.dev0__py3-none-any.whl

warn/utils.py

@@ -59,6 +59,8 @@ def fetch_if_not_cached(filename, url, **kwargs):
         url: The URL from which the file may be downloaded.
     Notes: Should this even be in utils vs. cache? Should it exist?
     """
+    if "timeout" not in kwargs:
+        kwargs["timeout"] = 120
     create_directory(Path(filename), is_file=True)
     if not os.path.exists(filename):
         logger.debug(f"Fetching {filename} from {url}")
@@ -81,6 +83,8 @@ def save_if_good_url(filename, url, **kwargs):
     Notes: Should this even be in utils vs. cache? Should it exist?
     """
     create_directory(Path(filename), is_file=True)
+    if "timeout" not in kwargs:
+        kwargs["timeout"] = 120
     response = requests.get(url, **kwargs)
     if not response.ok:
         logger.error(f"URL {url} fetch failed with {response.status_code}")
@@ -96,7 +100,7 @@ def save_if_good_url(filename, url, **kwargs):
     return success_flag, content
 
 
-def get_with_zyte(url, json_extras=None):
+def get_with_zyte(url, json_extras=None, **kwargs):
     """Use Zyte as a proxy server to retrieve data not available without it.
 
     Args:
@@ -119,6 +123,9 @@ def get_with_zyte(url, json_extras=None):
         )
         return (None, None)
 
+    if "timeout" not in kwargs:
+        kwargs["timeout"] = 120
+
     myjson = {
         "url": url,
         "httpResponseBody": True,
@@ -129,7 +136,10 @@ def get_with_zyte(url, json_extras=None):
             myjson[item] = json_extras[item]
 
     api_response = requests.post(
-        "https://api.zyte.com/v1/extract", auth=(zyte_api_key, ""), json=myjson
+        "https://api.zyte.com/v1/extract",
+        auth=(zyte_api_key, ""),
+        json=myjson,
+        **kwargs,
     )
 
     if not api_response.ok:
@@ -150,7 +160,7 @@ def get_with_zyte(url, json_extras=None):
     return (returnbin, returntext)
 
 
-def post_with_zyte(url, payload):
+def post_with_zyte(url, payload, **kwargs):
     """Use Zyte as a proxy server to retrieve data not available without it.
 
     Args:
@@ -175,6 +185,9 @@ def post_with_zyte(url, payload):
         )
         return (None, None)
 
+    if "timeout" not in kwargs:
+        kwargs["timeout"] = 120
+
     if isinstance(payload, dict):
         payload = json.dumps(payload)
 
@@ -191,6 +204,7 @@ def post_with_zyte(url, payload):
             "httpResponseBody": True,
             "followRedirect": True,
         },
+        **kwargs,
     )
 
     if not api_response.ok:
@@ -326,6 +340,9 @@ def get_url(
         kwargs["headers"] = {}
     kwargs["headers"]["User-Agent"] = user_agent
 
+    if "timeout" not in kwargs:
+        kwargs["timeout"] = 120
+
     # Go get it
     if session is not None:
         logger.debug(f"Requesting with session {session}")

{warn_scraper-1.2.155.dev0.dist-info → warn_scraper-1.2.156.dev0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.155.dev0
+Version: 1.2.156.dev0
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Author-email: Big Local News <biglocalnews@stanford.edu>
 License-Expression: Apache-2.0

{warn_scraper-1.2.155.dev0.dist-info → warn_scraper-1.2.156.dev0.dist-info}/RECORD

@@ -2,7 +2,7 @@ warn/__init__.py,sha256=A07JFY1TyaPtVIndBa7IvTk13DETqIkLgRdk0A-MCoE,85
 warn/cache.py,sha256=QBSHycchvRTkOQfHptOtZeTYiPgLP383jS8MTiGln_c,5969
 warn/cli.py,sha256=ZqyJwICdHFkn2hEgbArj_upbElR9-TSDlYDqyEGeexE,2019
 warn/runner.py,sha256=oeGRybGwpnkQKlPzRMlKxhsDt1GN4PZoX-vUwrsPgos,1894
-warn/utils.py,sha256=h1V5IISj9c2REtNnGctp0JqVjCi_Mi0r_FlvzNlrKb0,13093
+warn/utils.py,sha256=MTqbzF4h_TlrtBWqH4pEVA-X8yfw6b6ObafSGkqxKb4,13489
 warn/pdfrodent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warn/pdfrodent/pdfrodent.py,sha256=S2LXkKOdP0Jqzcg10hzGijTu83TMS72DzZqQceslJ70,14914
 warn/platforms/__init__.py,sha256=wIZRDf4tbTuC8oKM4ZrTAtwNgbtMQGzPXMwDYCFyrog,81
@@ -54,9 +54,9 @@ warn/scrapers/va.py,sha256=7Nle7qL0VNPiE653XyaP9HQqSfuJFDRr2kEkjOqLvFM,11269
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.155.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
-warn_scraper-1.2.155.dev0.dist-info/METADATA,sha256=Vz4NRjIMPAx1mNm6337Va3PAEhD4Zd4VTNeiGBumCG0,1780
-warn_scraper-1.2.155.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-warn_scraper-1.2.155.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.155.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
-warn_scraper-1.2.155.dev0.dist-info/RECORD,,
+warn_scraper-1.2.156.dev0.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
+warn_scraper-1.2.156.dev0.dist-info/METADATA,sha256=3ZYa-9I2ULmcbnva6Uyjqw5XNgxv4Lc5LetTtWT0w_s,1780
+warn_scraper-1.2.156.dev0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+warn_scraper-1.2.156.dev0.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.156.dev0.dist-info/top_level.txt,sha256=dZfms6N3kqVXufiPOo7YqOrAcUtYfNH_oyGvYUk9FB4,5
+warn_scraper-1.2.156.dev0.dist-info/RECORD,,