warn-scraper 1.2.104__py3-none-any.whl → 1.2.106__py3-none-any.whl

warn/scrapers/dc.py

@@ -85,6 +85,11 @@ def scrape(
         r.encoding = "utf-8"
         html = r.text
 
+        # A June 2025 entry includes a weird table inside a table cell.
+        # This is an ugly patch.
+        weirdtable = r"(\s+<table>\s+<tbody>\s+<tr>\s+<td>)(.*?)(</td>\s+</tr>\s+</tbody>\s+</table>\s+)"
+        html = re.sub(weirdtable, r"\2", html)
+
         # Save it to the cache
         cache_key = uuid.uuid5(uuid.NAMESPACE_URL, href)
         cache.write(f"dc/{cache_key}.html", html)

warn/scrapers/id.py

@@ -69,6 +69,11 @@ def scrape(
     with pdfplumber.open(pdf_file) as pdf:
         for index, page in enumerate(pdf.pages):
             rows = page.extract_table()
+            if rows[0][0] in ["Date of\nLetter", "Date of Letter"] and index > 1:
+                rows = rows[
+                    1:
+                ]  # Drop inside header rows that _clean_table will mangle if merged cells span pages
+            # logger.debug(f"\n\nRows for page {page}: {rows}")
             output_rows += _clean_table(rows, index)
 
     # Write out the data to a CSV
@@ -118,8 +123,9 @@ def _clean_table(rows, page_index) -> list:
     #            logger.debug(f"Dropping faulty row with {len(output_row)} elements: {output_row}")
 
     # Only include the header on the first page
-    if page_index != 0:
-        return output_rows[1:]
+    # No, this needed to be filtered earlier
+    # if page_index != 0:
+    #    return output_rows[1:]
 
     return output_rows
 
@@ -132,18 +138,26 @@ def filter_garbage_rows(incoming: list):
 
     Returns: List of lists that have a minimum number of elements.
     """
-    badrows: int = 0
+    shortrows: int = 0
+    mixedrows: int = 0
     outgoing: list = []
-    for row in incoming:
-        if len(row) >= 4:
+    for rowindex, row in enumerate(incoming):
+        error = False
+        if len(row) < 5:
+            error = True
+            logger.debug(f"Dropping short row: {row}")
+            shortrows += 1
+        if row[0] == "Date of Letter" and rowindex != 0:  # Keep the header row
+            error = True
+            logger.debug(f"Dropping partial row: {row}")
+            mixedrows += 1
+        if not error:
             outgoing.append(row)
-        else:
-            badrows += 1
-    if badrows == 0:
+    if shortrows == 0 and mixedrows == 0:
         logger.debug("No bad rows found.")
     else:
         logger.debug(
-            f"!!!!! {badrows:,} bad rows dropped from the data set with insufficient number of fields."
+            f"!!!! Dropped {shortrows} rows with insufficient number of fields, and dropped {mixedrows} rows scrambled with header info"
         )
     return outgoing
 

{warn_scraper-1.2.104.dist-info → warn_scraper-1.2.106.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.104
+Version: 1.2.106
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.104.dist-info → warn_scraper-1.2.106.dist-info}/RECORD

@@ -31,13 +31,13 @@ warn/scrapers/az.py,sha256=elGbue01Gjf_DQ66Wy9qqGIOJsiY-KIKJOVeft8pCXg,1447
 warn/scrapers/ca.py,sha256=VQOfjHXPCc-jYwh-EPGVVfnzvXB7pdmCt2uJ6QnMPRM,8600
 warn/scrapers/co.py,sha256=hUfqrzlhXQBkP4vxewVRrMZrgInoLer5S2MZlyYIQE4,7878
 warn/scrapers/ct.py,sha256=PKeZtlB0-z2wCmYmGl_WYoVo2gzwKV36upZcJVaJxjM,4852
-warn/scrapers/dc.py,sha256=_sHLnVqK_W90QqJb_W88yDlgPjoMl63LYZP3CJfdN9g,4484
+warn/scrapers/dc.py,sha256=NNTdjgUzC3jR-8F64U7dA77MzKo0dTTo64XjjpBu4cc,4743
 warn/scrapers/de.py,sha256=GyM92A-lFwZAfRxgbO-sIWhRfmBEKirzchaPIv-u0o4,1364
 warn/scrapers/fl.py,sha256=YJ6Qt-jJZ7_iUKlHDaQuaV2gRmae8AJKS5dwwChadBE,9563
 warn/scrapers/ga.py,sha256=o_OF4zPQ3vJM8USQPD7l_ThyRWAzUZkwrwLHCvWmHMI,7429
 warn/scrapers/hi.py,sha256=pSplAP15_ZBfQtcywyErmvNcrk7u55TjZj_F0Nqw9L8,5660
 warn/scrapers/ia.py,sha256=zOncaA9M0d6paT4pB7UU_4D_yxUgeUiGRcnpKi9DsRA,1999
-warn/scrapers/id.py,sha256=vl9r7ItpLUnUaugMihf74DKQ321FAY7A7LGpzYAusZ8,5926
+warn/scrapers/id.py,sha256=qJLcLgCgAfKzLpuwW32JqNwXn9NxZRZvQ50nZZKUhmE,6674
 warn/scrapers/il.py,sha256=sygdvsNuB_Gvu3o_HidtpSP4FLz0szKb1zEHqGxVtlI,1563
 warn/scrapers/in.py,sha256=dAT40ROhhKiwLcwa_YJ6EyhsYBLe0IX2rOWXmNa6JMs,2026
 warn/scrapers/ks.py,sha256=F_3biEMF7zgCX2XVuUACR74Vyzapta4SaM9SY3EuZCU,1266
@@ -65,9 +65,9 @@ warn/scrapers/va.py,sha256=hOPuiAjnTmtXCOdnBM_jAJuz9_u6oCxtbm2F-9m3ot0,10732
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.104.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
-warn_scraper-1.2.104.dist-info/METADATA,sha256=j9eq6IQ5GXReevRBLWI0Cv-_D7EtKIRCCRXI6I-U_4g,2385
-warn_scraper-1.2.104.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-warn_scraper-1.2.104.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.104.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
-warn_scraper-1.2.104.dist-info/RECORD,,
+warn_scraper-1.2.106.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
+warn_scraper-1.2.106.dist-info/METADATA,sha256=eGgAOPnHhNXmKPGssgfGwKdtkZfn5QPz2qa2dP0kBQU,2385
+warn_scraper-1.2.106.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+warn_scraper-1.2.106.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.106.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
+warn_scraper-1.2.106.dist-info/RECORD,,