warn-scraper 1.2.55__tar.gz → 1.2.57__tar.gz

{warn-scraper-1.2.55/warn_scraper.egg-info → warn-scraper-1.2.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.55
+Version: 1.2.57
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn-scraper-1.2.55 → warn-scraper-1.2.57}/warn/scrapers/ct.py

@@ -3,6 +3,7 @@ from datetime import datetime
 from pathlib import Path
 
 from bs4 import BeautifulSoup
+import requests
 
 from .. import utils
 from ..cache import Cache
@@ -40,8 +41,20 @@ def scrape(
     # We start in 2015
     current_year = datetime.now().year
 
-    # Get the full range of years
-    year_range = range(2015, current_year + 1)
+    if cache.exists(f"ct/{current_year}.html"):
+        # Get the full range of years
+        year_range = range(2015, current_year + 1)
+    else:
+        url = f"https://www.ctdol.state.ct.us/progsupt/bussrvce/warnreports/warn{current_year}.htm"
+        r = requests.head(url)
+        if r.ok:
+            logger.debug(f"Found first entry for {current_year}")
+            year_range = range(2015, current_year + 1)
+        else:
+            logger.debug(
+                f"No data for {current_year} found at {url}. Dropping back a year."
+            )
+            year_range = range(2015, current_year + 0)
 
     output_rows = []
     for year in year_range:
@@ -100,7 +113,6 @@ def _scrape_table(table) -> list:
     row_list = []
     # loop over table to process each row, skipping the header
     for table_row in table[0].find_all("tr")[1:]:
-
         # Get all the cells
         table_cells = table_row.find_all("td")
 

{warn-scraper-1.2.55 → warn-scraper-1.2.57}/warn/scrapers/dc.py

@@ -5,6 +5,7 @@ from datetime import datetime
 from pathlib import Path
 
 from bs4 import BeautifulSoup
+import requests
 
 from .. import utils
 from ..cache import Cache
@@ -38,13 +39,22 @@ def scrape(
     # Get the root page
     today = datetime.today()
     current_year = today.year
-    url = f"https://does.dc.gov/page/industry-closings-and-layoffs-warn-notifications-{current_year}"
+    targetfile = f"dc/{current_year}.html"
+    if not cache.exists(targetfile):  # Check if we have an entry for the latest year
+        url = f"https://does.dc.gov/page/industry-closings-and-layoffs-warn-notifications-{current_year}"
+        r = requests.head(url)
+        if not r.ok:
+            logger.debug(f"Still no data found for {current_year}. Falling back.")
+            current_year = today.year - 1
+            targetfile = f"dc/{current_year}.html"
+            url = f"https://does.dc.gov/page/industry-closings-and-layoffs-warn-notifications-{current_year}"
+
     r = utils.get_url(url)
     r.encoding = "utf-8"
     root_html = r.text
 
     # Save it to the cache
-    cache.write(f"dc/{current_year}.html", root_html)
+    cache.write(targetfile, root_html)
 
     # Parse the list of links
     soup = BeautifulSoup(root_html, "html5lib")
@@ -70,7 +80,6 @@ def scrape(
         root_html,
     ]
     for href in link_lookup.values():
-
         # Request the HTML
         r = utils.get_url(href)
         r.encoding = "utf-8"

{warn-scraper-1.2.55 → warn-scraper-1.2.57}/warn/scrapers/hi.py

@@ -1,13 +1,14 @@
 import datetime
 import logging
 from pathlib import Path
+from urllib.parse import quote
 
 from bs4 import BeautifulSoup
 
 from .. import utils
 
 __authors__ = ["Ash1R", "stucka"]
-__tags__ = ["html"]
+__tags__ = ["html", "pdf"]
 __source__ = {
     "name": "Workforce Development Hawaii",
     "url": "https://labor.hawaii.gov/wdc/real-time-warn-updates/",
@@ -28,15 +29,17 @@ def scrape(
     cache_dir -- the Path where results can be cached (default WARN_CACHE_DIR)
     Returns: the Path where the file is written
     """
-    firstpage = utils.get_url("https://labor.hawaii.gov/wdc/real-time-warn-updates/")
+    cacheprefix = "https://webcache.googleusercontent.com/search?q=cache%3A"    # Use Google Cache, per #600
+
+    firstpage = utils.get_url(cacheprefix + quote("https://labor.hawaii.gov/wdc/real-time-warn-updates/"))
     soup = BeautifulSoup(firstpage.text, features="html5lib")
     pagesection = soup.select("div.primary-content")[0]
     subpageurls = []
     for atag in pagesection.find_all("a"):
         href = atag["href"]
         if href.endswith("/"):
-            href = href[:-1]
-        subpageurls.append(href)
+            href = href         # [:-1]
+        subpageurls.append(cacheprefix + quote(href))
 
     headers = ["Company", "Date", "PDF url", "location", "jobs"]
     data = [headers]
@@ -85,8 +88,8 @@ def scrape(
             row.append(dates[i])
 
             row.append(url)
-            row.append(None)  # location
-            row.append(None)  # jobs
+            row.append(None)     # location
+            row.append(None)     # jobs
             data.append(row)
 
     output_csv = data_dir / "hi.csv"

{warn-scraper-1.2.55 → warn-scraper-1.2.57/warn_scraper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.55
+Version: 1.2.57
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News