warn-scraper 1.2.57__py3-none-any.whl → 1.2.59__py3-none-any.whl

warn/scrapers/hi.py

@@ -1,6 +1,7 @@
 import datetime
 import logging
 from pathlib import Path
+from time import sleep
 from urllib.parse import quote
 
 from bs4 import BeautifulSoup
@@ -29,23 +30,34 @@ def scrape(
     cache_dir -- the Path where results can be cached (default WARN_CACHE_DIR)
     Returns: the Path where the file is written
     """
-    cacheprefix = "https://webcache.googleusercontent.com/search?q=cache%3A"    # Use Google Cache, per #600
+    # Google Cache is a backup if the state re-implements its JS-enabled browser equivalent
+    usegooglecache = False
+    cacheprefix = "https://webcache.googleusercontent.com/search?q=cache%3A"
 
-    firstpage = utils.get_url(cacheprefix + quote("https://labor.hawaii.gov/wdc/real-time-warn-updates/"))
+    firstpageurl = "https://labor.hawaii.gov/wdc/real-time-warn-updates/"
+    if usegooglecache:
+        firstpageurl = cacheprefix + quote(firstpageurl)
+
+    firstpage = utils.get_url(firstpageurl)
     soup = BeautifulSoup(firstpage.text, features="html5lib")
     pagesection = soup.select("div.primary-content")[0]
     subpageurls = []
     for atag in pagesection.find_all("a"):
         href = atag["href"]
         if href.endswith("/"):
-            href = href         # [:-1]
-        subpageurls.append(cacheprefix + quote(href))
+            href = href  # [:-1]
+        subpageurl = href
+        if usegooglecache:
+            subpageurl = cacheprefix + quote(subpageurl)
+        subpageurls.append(subpageurl)
 
+    masterlist = []
     headers = ["Company", "Date", "PDF url", "location", "jobs"]
-    data = [headers]
+    #    data = [headers]
     # lastdateseen = "2099-12-31"
 
     for subpageurl in reversed(subpageurls):
+        sleep(2)
         # Conditionally here, we want to check and see if we have the old cached files, or if the year is current or previous.
         # Only need to download if it's current or previous year.
         # But do we care enough to implement right now?
@@ -53,47 +65,67 @@ def scrape(
         logger.debug(f"Parsing page {subpageurl}")
         page = utils.get_url(subpageurl)
         soup = BeautifulSoup(page.text, features="html5lib")
+        if subpageurl.endswith("/"):
+            subpageurl = subpageurl[:-1]  # Trim off the final slash, if there is one
         pageyear = subpageurl.split("/")[-1][:4]
-        tags = soup.select("p a[href*=pdf]")
-        p_tags = [i.parent.get_text().replace("\xa0", " ").split("\n") for i in tags]
-        clean_p_tags = [j for i in p_tags for j in i]
 
-        dates = [k.split("–")[0].strip() for k in clean_p_tags]
-        for i in range(len(dates)):
+        # There are at least two formats for Hawaii. In some years, each individual layoff is in a paragraph tag.
+        # In others, all the layoffs are grouped under a single paragraph tag, separated by <br>
+        # BeautifulSoup converts that to a <br/>.
+        # But the call to parent also repeats a bunch of entries, so we need to ensure they're not.
+        # So in more recent years, finding the parent of the "p a" there find essentially the row of data.
+        # In the older years, the parent is ... all the rows of data, which gets repeated.
+        # So take each chunk of data, find the parent, do some quality checks, clean up the text,
+        # don't engage with duplicates.
+
+        selection = soup.select("p a[href*=pdf]")
+        rows = []
+        for child in selection:
+            parent = child.parent
+            for subitem in parent.prettify().split("<br/>"):
+                if len(subitem.strip()) > 5 and ".pdf" in subitem:
+                    subitem = subitem.replace("\xa0", " ").replace("\n", "").strip()
+                    row = BeautifulSoup(subitem, features="html5lib")
+                    if row not in rows:
+                        rows.append(row)
+
+        for row in rows:
+            line: dict = {}
+            for item in headers:
+                line[item] = None
+            graftext = row.get_text().strip()
+            tempdate = graftext
+
+            # Check to see if it's not an amendment, doesn't have 3/17/2022 date format
+            # Most dates should be like "March 17, 2022"
+            if pageyear in tempdate and f"/{pageyear}" not in tempdate:
+                try:
+                    tempdate = (
+                        graftext.strip().split(pageyear)[0].strip() + f" {pageyear}"
+                    )
+                except ValueError:
+                    print(f"Date conversion failed on row: {row}")
+
+            line["Date"] = tempdate
+
             try:
-                tempdate = dates[i].split(pageyear)[0].strip() + f" {pageyear}"
                 parsed_date = datetime.datetime.strptime(
                     tempdate, "%B %d, %Y"
                 ).strftime("%Y-%m-%d")
-                dates[i] = parsed_date
-            #    lastdateseen = parsed_date
-
-            # Disabling amendment automation to shift fixes into warn-transformer instead.
-            # If this needs to come back, uncomment the lastseendate references
-            # then rebuild the below section as an else
+                line["Date"] = parsed_date
             except ValueError:
-                logger.debug(f"Date error: {dates[i]}, leaving intact")
-        #                if "*" in dates[i]:
-        #                    logger.debug(
-        #                        f"Date error: {dates[i]} as apparent amendment; saving as {lastdateseen}"
-        #                    )
-        #                    dates[i] = lastdateseen
-        #                else:
-
-        for i in range(len(tags)):
-            row = []
-            url = tags[i].get("href")
-            row.append(tags[i].get_text())
-
-            row.append(dates[i])
-
-            row.append(url)
-            row.append(None)     # location
-            row.append(None)     # jobs
-            data.append(row)
+                logger.debug(f"Date error: '{tempdate}',  leaving intact")
+
+            line["PDF url"] = row.select("a")[0].get("href")
+            line["Company"] = row.select("a")[0].get_text().strip()
+            masterlist.append(line)
 
+    if len(masterlist) == 0:
+        logger.error(
+            "No data scraped -- anti-scraping mechanism may be back in play -- try Google Cache?"
+        )
     output_csv = data_dir / "hi.csv"
-    utils.write_rows_to_csv(output_csv, data)
+    utils.write_dict_rows_to_csv(output_csv, headers, masterlist)
     return output_csv
 
 

{warn_scraper-1.2.57.dist-info → warn_scraper-1.2.59.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.57
+Version: 1.2.59
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.57.dist-info → warn_scraper-1.2.59.dist-info}/RECORD

@@ -36,7 +36,7 @@ warn/scrapers/dc.py,sha256=kAWmERzEIOtGHla9tn8hK0NfP4B-aP4jknNGUiPw4C4,4493
 warn/scrapers/de.py,sha256=yMpCFAAlIO2f4DVUQpPKKLzm52_Zpn9IuPPFBNX1pjQ,1386
 warn/scrapers/fl.py,sha256=mHymxjwNGtYXRxAwjUSIG1qeSp4Y_zzr4XDxPz9LAfY,9560
 warn/scrapers/ga.py,sha256=EuqBrMlBojH6eXOHisNqJAQcsnb8FPHDwWomNopw9Ys,7285
-warn/scrapers/hi.py,sha256=IrwgUMNPqsHExiLZ8dFM25am7KTtVRrLDjIesNtJIsk,3736
+warn/scrapers/hi.py,sha256=vpX-F82e_CYhxY7EAPLTMXG9cLCmPAprdDfsx4GziNo,5226
 warn/scrapers/ia.py,sha256=zOncaA9M0d6paT4pB7UU_4D_yxUgeUiGRcnpKi9DsRA,1999
 warn/scrapers/id.py,sha256=rRkE9g9ZOL7JgTdIz46kyGOeetzSla3e1Xr6gJ1v_74,5443
 warn/scrapers/il.py,sha256=sygdvsNuB_Gvu3o_HidtpSP4FLz0szKb1zEHqGxVtlI,1563
@@ -66,9 +66,9 @@ warn/scrapers/va.py,sha256=DDuR4_2Jpaxg9nVmuM6PAR8v8xz3VgxTBG5sWJgz2q0,1582
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=Il3RmJpKr7SbwUBxHxlhEFLoxy7zSiduyo8F2EddB2Y,4021
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.57.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warn_scraper-1.2.57.dist-info/METADATA,sha256=Z__kggmqalQaYaWaynUtNFXqNiBW9X8u7SDh0Y3qjkI,2025
-warn_scraper-1.2.57.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-warn_scraper-1.2.57.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.57.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
-warn_scraper-1.2.57.dist-info/RECORD,,
+warn_scraper-1.2.59.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warn_scraper-1.2.59.dist-info/METADATA,sha256=WspYx_YFC0ojeHKwqV2SXwIAVzNGay3_ChH2j35sCA8,2025
+warn_scraper-1.2.59.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+warn_scraper-1.2.59.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.59.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
+warn_scraper-1.2.59.dist-info/RECORD,,