warn-scraper 1.2.73__py3-none-any.whl → 1.2.75__py3-none-any.whl

warn/scrapers/ca.py

@@ -224,20 +224,25 @@ def _extract_pdf_data(pdf_path):
             if "summary" in first_cell:
                 continue
             for row in rows:
-                data_row = {}
-                for i, value in enumerate(row):
-                    this_raw_header = raw_header[i]
-                    this_clean_header = header_crosswalk[this_raw_header]
-                    data_row[this_clean_header] = value
-                # Data clean-ups
-                data_row.update(
-                    {
-                        "effective_date": data_row["effective_date"].replace(" ", ""),
-                        "received_date": data_row["received_date"].replace(" ", ""),
-                        "source_file": str(pdf_path).split("/")[-1],
-                    }
-                )
-                data.append(data_row)
+                # Summary rows have an extra field, and the above code does not
+                # block the summary table from being parsed if it jumps onto another page.
+                if len(row) != len(raw_header) + 1:
+                    data_row = {}
+                    for i, value in enumerate(row):
+                        this_raw_header = raw_header[i]
+                        this_clean_header = header_crosswalk[this_raw_header]
+                        data_row[this_clean_header] = value
+                    # Data clean-ups
+                    data_row.update(
+                        {
+                            "effective_date": data_row["effective_date"].replace(
+                                " ", ""
+                            ),
+                            "received_date": data_row["received_date"].replace(" ", ""),
+                            "source_file": str(pdf_path).split("/")[-1],
+                        }
+                    )
+                    data.append(data_row)
     return data
 
 

warn/scrapers/id.py

@@ -12,7 +12,7 @@ __authors__ = ["chriszs", "stucka"]
 __tags__ = ["pdf"]
 __source__ = {
     "name": "Idaho Department of Labor",
-    "url": "https://www.labor.idaho.gov/dnn/Businesses/Layoff-Assistance#2",
+    "url": "https://www.labor.idaho.gov/warnnotice/",
 }
 
 logger = logging.getLogger(__name__)
@@ -32,7 +32,7 @@ def scrape(
     Returns: the Path where the file is written
     """
     # Create the URL of the source PDF
-    base_url = "https://www.labor.idaho.gov/dnn/Portals/0/Publications/"
+    base_url = "https://www.labor.idaho.gov/warnnotice/"
     file_name = "WARNNotice.pdf"
     # There's a numeric parameter called v on this PDF URL that updates
     # from time to time. Suspect this is a cache-buster. We're using a
@@ -40,10 +40,8 @@ def scrape(
     min_cache_buster = 0
     max_cache_buster = 10000000000
     cache_buster = random.randrange(min_cache_buster, max_cache_buster)
-    url = f"{base_url}{file_name}?v={cache_buster}"
+    url = f"{base_url}?v={cache_buster}"
 
-    # Download the PDF with verify=False because
-    # there's a persistent cert error we're working around.
     cache = Cache(cache_dir)
     state_code = "id"
     cache_key = f"{state_code}/{file_name}"
@@ -126,9 +124,9 @@ def filter_garbage_rows(incoming: list):
             badrows += 1
     if badrows == 0:
         logger.debug("No bad rows found.")
-    logger.debug(
-        f"!!!!! {badrows:,} bad rows dropped from the data set with insufficient number of fields."
-    )
+        logger.debug(
+            f"!!!!! {badrows:,} bad rows dropped from the data set with insufficient number of fields."
+        )
     return outgoing
 
 

{warn_scraper-1.2.73.dist-info → warn_scraper-1.2.75.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.73
+Version: 1.2.75
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.73.dist-info → warn_scraper-1.2.75.dist-info}/RECORD

@@ -28,7 +28,7 @@ warn/scrapers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warn/scrapers/ak.py,sha256=h7BYMTV0whwWAPhbzVDVKMMoVCFphKly70aiTHabPq4,1847
 warn/scrapers/al.py,sha256=D0rT9GQ0vwfkRuveVAt-Po-T6b2TI1EPGeLOBy2m3_M,2240
 warn/scrapers/az.py,sha256=elGbue01Gjf_DQ66Wy9qqGIOJsiY-KIKJOVeft8pCXg,1447
-warn/scrapers/ca.py,sha256=_LvkIci1nTUKBt5KC-wEcazWG7zoeUeadxj4D0XD97k,8170
+warn/scrapers/ca.py,sha256=rBTB-6LmNIlbGCqrCtI3O-w2e_0kcVSFxvjvh4EHBlk,8511
 warn/scrapers/co.py,sha256=g076Zqe8XA8tbW03HP6-03mJV8fft1niHfa5Sy6me9A,7388
 warn/scrapers/ct.py,sha256=HLMmBSFhT5Y3vZQUwRyCTxiG5BMQXTfG3SEj5rkQEL4,4771
 warn/scrapers/dc.py,sha256=_sHLnVqK_W90QqJb_W88yDlgPjoMl63LYZP3CJfdN9g,4484
@@ -37,7 +37,7 @@ warn/scrapers/fl.py,sha256=YJ6Qt-jJZ7_iUKlHDaQuaV2gRmae8AJKS5dwwChadBE,9563
 warn/scrapers/ga.py,sha256=iuvXoP5nPEnjm8wiEWhtBwLnkdmElhJILin3No5HIyo,7412
 warn/scrapers/hi.py,sha256=pSplAP15_ZBfQtcywyErmvNcrk7u55TjZj_F0Nqw9L8,5660
 warn/scrapers/ia.py,sha256=zOncaA9M0d6paT4pB7UU_4D_yxUgeUiGRcnpKi9DsRA,1999
-warn/scrapers/id.py,sha256=rRkE9g9ZOL7JgTdIz46kyGOeetzSla3e1Xr6gJ1v_74,5443
+warn/scrapers/id.py,sha256=Snqk-FXUq6_ovlpj-t9YKIOJEPF_ixJb3kfZRPJ02yU,5296
 warn/scrapers/il.py,sha256=sygdvsNuB_Gvu3o_HidtpSP4FLz0szKb1zEHqGxVtlI,1563
 warn/scrapers/in.py,sha256=dAT40ROhhKiwLcwa_YJ6EyhsYBLe0IX2rOWXmNa6JMs,2026
 warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
@@ -65,9 +65,9 @@ warn/scrapers/va.py,sha256=13lhkQrSkPGHEiWUuf1qiS890PWYE5gV-TgISpoiQnc,1711
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.73.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warn_scraper-1.2.73.dist-info/METADATA,sha256=5FQltUNKR1LZmPu4Yqz8aqogBoQNNhSyISPC0SQ1sdg,2025
-warn_scraper-1.2.73.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-warn_scraper-1.2.73.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.73.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
-warn_scraper-1.2.73.dist-info/RECORD,,
+warn_scraper-1.2.75.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warn_scraper-1.2.75.dist-info/METADATA,sha256=qh4oJpc2dj53TxBiItdDRgzlWmOX5ilDx2XgOUG4KmM,2025
+warn_scraper-1.2.75.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+warn_scraper-1.2.75.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.75.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
+warn_scraper-1.2.75.dist-info/RECORD,,