warn-scraper 1.2.73__tar.gz → 1.2.75__tar.gz

{warn-scraper-1.2.73/warn_scraper.egg-info → warn-scraper-1.2.75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.73
+Version: 1.2.75
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn-scraper-1.2.73 → warn-scraper-1.2.75}/warn/scrapers/ca.py

@@ -224,20 +224,25 @@ def _extract_pdf_data(pdf_path):
             if "summary" in first_cell:
                 continue
             for row in rows:
-                data_row = {}
-                for i, value in enumerate(row):
-                    this_raw_header = raw_header[i]
-                    this_clean_header = header_crosswalk[this_raw_header]
-                    data_row[this_clean_header] = value
-                # Data clean-ups
-                data_row.update(
-                    {
-                        "effective_date": data_row["effective_date"].replace(" ", ""),
-                        "received_date": data_row["received_date"].replace(" ", ""),
-                        "source_file": str(pdf_path).split("/")[-1],
-                    }
-                )
-                data.append(data_row)
+                # Summary rows have an extra field, and the above code does not
+                # block the summary table from being parsed if it jumps onto another page.
+                if len(row) != len(raw_header) + 1:
+                    data_row = {}
+                    for i, value in enumerate(row):
+                        this_raw_header = raw_header[i]
+                        this_clean_header = header_crosswalk[this_raw_header]
+                        data_row[this_clean_header] = value
+                    # Data clean-ups
+                    data_row.update(
+                        {
+                            "effective_date": data_row["effective_date"].replace(
+                                " ", ""
+                            ),
+                            "received_date": data_row["received_date"].replace(" ", ""),
+                            "source_file": str(pdf_path).split("/")[-1],
+                        }
+                    )
+                    data.append(data_row)
     return data
 
 

{warn-scraper-1.2.73 → warn-scraper-1.2.75}/warn/scrapers/id.py

@@ -12,7 +12,7 @@ __authors__ = ["chriszs", "stucka"]
 __tags__ = ["pdf"]
 __source__ = {
     "name": "Idaho Department of Labor",
-    "url": "https://www.labor.idaho.gov/dnn/Businesses/Layoff-Assistance#2",
+    "url": "https://www.labor.idaho.gov/warnnotice/",
 }
 
 logger = logging.getLogger(__name__)
@@ -32,7 +32,7 @@ def scrape(
     Returns: the Path where the file is written
     """
     # Create the URL of the source PDF
-    base_url = "https://www.labor.idaho.gov/dnn/Portals/0/Publications/"
+    base_url = "https://www.labor.idaho.gov/warnnotice/"
     file_name = "WARNNotice.pdf"
     # There's a numeric parameter called v on this PDF URL that updates
     # from time to time. Suspect this is a cache-buster. We're using a
@@ -40,10 +40,8 @@ def scrape(
     min_cache_buster = 0
     max_cache_buster = 10000000000
     cache_buster = random.randrange(min_cache_buster, max_cache_buster)
-    url = f"{base_url}{file_name}?v={cache_buster}"
+    url = f"{base_url}?v={cache_buster}"
 
-    # Download the PDF with verify=False because
-    # there's a persistent cert error we're working around.
     cache = Cache(cache_dir)
     state_code = "id"
     cache_key = f"{state_code}/{file_name}"
@@ -126,9 +124,9 @@ def filter_garbage_rows(incoming: list):
             badrows += 1
     if badrows == 0:
         logger.debug("No bad rows found.")
-    logger.debug(
-        f"!!!!! {badrows:,} bad rows dropped from the data set with insufficient number of fields."
-    )
+        logger.debug(
+            f"!!!!! {badrows:,} bad rows dropped from the data set with insufficient number of fields."
+        )
     return outgoing
 
 

{warn-scraper-1.2.73 → warn-scraper-1.2.75/warn_scraper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: warn-scraper
-Version: 1.2.73
+Version: 1.2.75
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News