warn-scraper 1.2.112__tar.gz → 1.2.114__tar.gz

{warn_scraper-1.2.112 → warn_scraper-1.2.114}/.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         additional_dependencies: [black]
 
 -   repo: https://github.com/timothycrosley/isort
-    rev: 5.13.2
+    rev: 6.0.1
     hooks:
     -   id: isort
         args: ["--profile", "black"]

{warn_scraper-1.2.112/warn_scraper.egg-info → warn_scraper-1.2.114}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.112
+Version: 1.2.114
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.112 → warn_scraper-1.2.114}/Pipfile

@@ -32,6 +32,7 @@ sphinxcontrib-serializinghtml = "1.1.5"
 setuptools = "*"
 jinja2 = "*"
 pytest = "==8.3.5"
+isort = "==6.0.1"
 
 [packages]
 beautifulsoup4 = "*"

{warn_scraper-1.2.112 → warn_scraper-1.2.114}/Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "aac81610731435f6fe712525f4e5c72e3d7598da79d5a3a2f7580d7c013008a2"
+            "sha256": "b1cc4814822457ec04972c1a07991d1f983c946e8cdcfc315cbe1b6d9d84b2cd"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -1067,6 +1067,15 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.1.0"
         },
+        "isort": {
+            "hashes": [
+                "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450",
+                "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615"
+            ],
+            "index": "pypi",
+            "markers": "python_full_version >= '3.9.0'",
+            "version": "==6.0.1"
+        },
         "jaraco.classes": {
             "hashes": [
                 "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd",

{warn_scraper-1.2.112 → warn_scraper-1.2.114}/warn/platforms/job_center/site.py

@@ -112,7 +112,7 @@ class Site:
             logger.debug("Fetching from cache")
             return self.cache.fetch(url, params)
         else:
-            logger.debug("Pulling from the web")
+            logger.debug(f"Pulling from the web: {url} with params {params}")
             response = requests.get(url, params=params, verify=self.verify)
             logger.debug(f"Response code: {response.status_code}")
             html = response.text

{warn_scraper-1.2.112 → warn_scraper-1.2.114}/warn/scrapers/la.py

@@ -1,13 +1,10 @@
 import logging
 import os
 import re
-import sys
-from base64 import b64decode
 from datetime import datetime
 from pathlib import Path
 
 import pdfplumber
-import requests
 from bs4 import BeautifulSoup
 
 from .. import utils
@@ -36,14 +33,6 @@ def scrape(
 
     Returns: the Path where the file is written
     """
-    try:
-        zyte_api_key = os.environ["ZYTE_API_KEY"]
-    except KeyError:
-        logger.error(
-            "No ZYTE_API_KEY variable found in environment. Please get an API key from Zyte and export it."
-        )
-        sys.exit(1)
-
     # Fire up the cache
     cache = Cache(cache_dir)
 
@@ -54,18 +43,7 @@ def scrape(
 
     # Download the root page
     url = f"{base_url}Downloads/{file_base}.asp"
-    api_response = requests.post(
-        "https://api.zyte.com/v1/extract",
-        auth=(zyte_api_key, ""),
-        json={
-            "url": url,
-            "httpResponseBody": True,
-            "followRedirect": True,
-        },
-    )
-    html_bytes: bytes = b64decode(api_response.json()["httpResponseBody"])
-    # html = utils.get_url(url).text
-    html = html_bytes.decode("utf-8", errors="backslashreplace")
+    htmlbin, html = utils.get_with_zyte(url)
 
     # Save it to the cache
     cache_key = cache_dir / f"{state_code}/{file_base}.html"
@@ -82,25 +60,11 @@ def scrape(
         if "WARN Notices" in link.text:
             # Download the PDF
             pdf_url = f"{base_url}{link['href']}"
-            logger.debug(pdf_url)
-            api_response = requests.post(
-                "https://api.zyte.com/v1/extract",
-                auth=(zyte_api_key, ""),
-                json={
-                    "url": pdf_url,
-                    "httpResponseBody": True,
-                    "followRedirect": True,
-                },
-            )
-            http_response_body: bytes = b64decode(
-                api_response.json()["httpResponseBody"]
-            )
+            rawbin, rawtext = utils.get_with_zyte(pdf_url)
             pdf_path = cache_dir / f"{state_code}/{os.path.basename(pdf_url)}"
 
             with open(pdf_path, "wb") as fp:
-                fp.write(http_response_body)
-
-            # pdf_path = _read_or_download(cache, state_code, pdf_url)
+                fp.write(rawbin)
 
             # Process the PDF
             logger.debug(f"Attempting to parse {pdf_path}")

warn_scraper-1.2.114/warn/scrapers/ok.py

@@ -0,0 +1,117 @@
+import logging
+from pathlib import Path
+
+import requests
+
+from .. import utils
+
+__authors__ = ["zstumgoren", "Dilcia19", "stucka"]
+__tags__ = [""]
+__source__ = {
+    "name": "Oklahoma Office of Workforces Development",
+    "url": "https://www.employoklahoma.gov/Participants/s/warnnotices",
+}
+
+logger = logging.getLogger(__name__)
+
+
+def scrape(
+    data_dir: Path = utils.WARN_DATA_DIR,
+    cache_dir: Path = utils.WARN_CACHE_DIR,
+    use_cache: bool = True,
+) -> Path:
+    """
+    Scrape data from Oklahoma.
+
+    Keyword arguments:
+    data_dir -- the Path were the result will be saved (default WARN_DATA_DIR)
+    cache_dir -- the Path where results can be cached (default WARN_CACHE_DIR)
+    use_cache -- a Boolean indicating whether the cache should be used (default True)
+
+    Returns: the Path where the file is written
+    """
+    output_csv = data_dir / "ok.csv"
+    # search_url = "https://okjobmatch.com/search/warn_lookups"
+    # search_url = "https://www.employoklahoma.gov/Participants/s/warnnotices"
+    posturl = "https://www.employoklahoma.gov/Participants/s/sfsites/aura?r=2&aura.ApexAction.execute=6"
+
+    # There are a bunch of hard-coded values in here that seem to work for at least a day.
+    # Undetermined:
+    # -- Will this continue working in the short- or medium-term?
+    # -- What is the signficance of each variable?
+    # -- How do we refresh these?
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/142.0",
+        "Accept": "*/*",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Accept-Encoding": "gzip, deflate, br, zstd",
+        "Referer": "https://www.employoklahoma.gov/Participants/s/warnnotices",
+        "X-SFDC-LDS-Endpoints": "ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.hasDocument, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.checkJobExpiry, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.checkResumeExpiry, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.checkUIRegistered, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.getLoginMaintenanceMessage, ApexActionController.execute:OESC_JS_getWARNLayoffNotices.getListofLayoffAccService",
+        "X-SFDC-Page-Scope-Id": "9c659a19-8020-41b0-a81c-36335e22801a",
+        "X-SFDC-Request-Id": "16140000007a08bd2f",
+        "X-SFDC-Page-Cache": "9439898463d86806",
+        "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
+        "X-B3-TraceId": "856a2236ba7d283e",
+        "X-B3-SpanId": "b79b2da3a7dc4544",
+        "X-B3-Sampled": "0",
+        "Origin": "https://www.employoklahoma.gov",
+        "Connection": "keep-alive",
+        "Cookie": "renderCtx=%7B%22pageId%22%3A%223823bba2-3b00-4db7-aca6-5ca0eb67fc63%22%2C%22schema%22%3A%22Published%22%2C%22viewType%22%3A%22Published%22%2C%22brandingSetId%22%3A%22fa0b6362-0214-44b9-947d-2543eaab22c7%22%2C%22audienceIds%22%3A%22%22%7D; CookieConsentPolicy=0:1; LSKey-c$CookieConsentPolicy=0:1; pctrk=f3070d0c-7078-4062-96bb-de9e82cbb1db",
+        "Sec-Fetch-Dest": "empty",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Site": "same-origin",
+    }
+
+    payload = "message=%7B%22actions%22%3A%5B%7B%22id%22%3A%22156%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22hasDocument%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22157%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22checkJobExpiry%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22158%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22checkResumeExpiry%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22159%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22checkUIRegistered%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22160%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22getLoginMaintenanceMessage%22%2C%22params%22%3A%7B%22displayTo%22%3A%22Job%20Seekers%22%2C%22messageType%22%3A%22Portal%20Login%20Messages%22%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22161%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22OESC_JS_getWARNLayoffNotices%22%2C%22method%22%3A%22getListofLayoffAccService%22%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%5D%7D&aura.context=%7B%22mode%22%3A%22PROD%22%2C%22fwuid%22%3A%22eE5UbjZPdVlRT3M0d0xtOXc5MzVOQWg5TGxiTHU3MEQ5RnBMM0VzVXc1cmcxMi42MjkxNDU2LjE2Nzc3MjE2%22%2C%22app%22%3A%22siteforce%3AcommunityApp%22%2C%22loaded%22%3A%7B%22APPLICATION%40markup%3A%2F%2Fsiteforce%3AcommunityApp%22%3A%221305_7pTC6grCTP7M16KdvDQ-Xw%22%7D%2C%22dn%22%3A%5B%5D%2C%22globals%22%3A%7B%7D%2C%22uad%22%3Atrue%7D&aura.pageURI=%2FParticipants%2Fs%2Fwarnnotices&aura.token=null"
+
+    logger.debug(f"Attempting to send hard-coded data to {posturl}")
+    r = requests.post(posturl, headers=headers, data=payload)
+    rawdata = r.json()
+
+    for entry in rawdata["actions"]:
+        if (
+            entry["id"] == "161;a"
+        ):  # What is this value? Will this change? Also no idea.
+            cleanerdata = entry["returnValue"]["returnValue"]
+    """
+    fields = set()
+    for entry in cleanerdata:
+        for field in entry:
+            fields.add(field)
+    {'Id',
+     'Launchpad__Layoff_Closure_Type__c',
+     'Launchpad__Notice_Date__c',
+     'OESC_Employer_City__c',
+     'OESC_Employer_Name__c',
+     'OESC_Employer_Zip_Code__c',
+     'RecordTypeId',
+     'Select_Local_Workforce_Board__c'}
+    """
+    fields = {
+        "Id": "id",
+        "Launchpad__Layoff_Closure_Type__c": "closure_type",
+        "Launchpad__Notice_Date__c": "notice_date",
+        "OESC_Employer_City__c": "city",
+        "OESC_Employer_Name__c": "company name",
+        "OESC_Employer_Zip_Code__c": "zip_code",
+        "RecordTypeId": "record_type_id",
+        "Select_Local_Workforce_Board__c": "workforce_board",
+    }
+
+    masterlist = []
+    for entry in cleanerdata:
+        line = {}
+        for item in fields:
+            if item in entry:
+                line[fields[item]] = entry[item]
+            else:
+                line[fields[item]] = None
+        masterlist.append(line)
+
+    utils.write_dict_rows_to_csv(output_csv, list(fields.values()), masterlist)
+    return output_csv
+
+
+if __name__ == "__main__":
+    scrape()

{warn_scraper-1.2.112 → warn_scraper-1.2.114}/warn/utils.py

@@ -1,7 +1,9 @@
 import csv
+import json
 import logging
 import os
 import typing
+from base64 import b64decode, b64encode
 from pathlib import Path
 from time import sleep
 
@@ -94,6 +96,103 @@ def save_if_good_url(filename, url, **kwargs):
     return success_flag, content
 
 
+def get_with_zyte(url):
+    """Use Zyte as a proxy server to retrieve data not available without it.
+
+    Args:
+        url (str): URL to retrieve
+    Returns:
+        returnbin (bin): raw binary representation of returned data object
+        returntext (str): utf-8 conversion of returned data object, e.g., HTML
+    Failures:
+        Returns (None, None) if it encounters a problem and logs an error.
+    Requires:
+        ZYTE_API_KEY to be set in environment
+    """
+    logger.debug(f"Seeking to fetch {url} with Zyte")
+    try:
+        zyte_api_key = os.environ["ZYTE_API_KEY"]
+    except KeyError:
+        logger.error(
+            "No ZYTE_API_KEY variable found in environment. Please get an API key from Zyte and export it."
+        )
+        return (None, None)
+
+    api_response = requests.post(
+        "https://api.zyte.com/v1/extract",
+        auth=(zyte_api_key, ""),
+        json={
+            "url": url,
+            "httpResponseBody": True,
+            "followRedirect": True,
+        },
+    )
+
+    if not api_response.ok:
+        logger.error(
+            f"Error downloading {url} with get_with_zyte. Repsonse code: {api_response.status_code}"
+        )
+        return (None, None)
+    returnbin: bytes = b64decode(api_response.json()["httpResponseBody"])
+    returntext: str = returnbin.decode("utf-8", errors="backslashreplace")
+    logger.debug(f"Fetched {url}")
+    return (returnbin, returntext)
+
+
+def post_with_zyte(url, payload):
+    """Use Zyte as a proxy server to retrieve data not available without it.
+
+    Args:
+        url (str): URL to retrieve
+        payload: (dict, str or binary): POST body.
+            If type dict: Convert to utf-8 text then:
+            If type str: Convert to b64encoded
+    Returns:
+        returnbin (bin): raw binary representation of returned data object
+        returntext (str): utf-8 conversion of returned data object, e.g., HTML
+    Failures:
+        Returns (None, None) if it encounters a problem and logs an error.
+    Requires:
+        ZYTE_API_KEY to be set in environment
+    """
+    logger.debug(f"Seeking to fetch {url} with Zyte")
+    try:
+        zyte_api_key = os.environ["ZYTE_API_KEY"]
+    except KeyError:
+        logger.error(
+            "No ZYTE_API_KEY variable found in environment. Please get an API key from Zyte and export it."
+        )
+        return (None, None)
+
+    if isinstance(payload, dict):
+        payload = json.dumps(payload)
+
+    if isinstance(payload, str):
+        payload = b64encode(payload.encode("utf-8"))
+
+    api_response = requests.post(
+        "https://api.zyte.com/v1/extract",
+        auth=(zyte_api_key, ""),
+        json={
+            "url": url,
+            "httpRequestMethod": "POST",
+            "httpRequestBody": payload,
+            "httpResponseBody": True,
+            "followRedirect": True,
+        },
+    )
+
+    if not api_response.ok:
+        logger.error(
+            f"Error downloading {url} with post_with_zyte. Repsonse code: {api_response.status_code}. Reponse: {api_response.json()}"
+        )
+        return (None, None)
+    returnbin: bytes = b64decode(api_response.json()["httpResponseBody"])
+    returntext: str = returnbin.decode("utf-8", errors="backslashreplace")
+    logger.debug(f"Fetched {url}")
+    return (returnbin, returntext)
+
+
 def write_rows_to_csv(output_path: Path, rows: list, mode="w"):
     """Write the provided list to the provided path as comma-separated values.
 
@@ -109,21 +208,19 @@ def write_rows_to_csv(output_path: Path, rows: list, mode="w"):
         writer.writerows(rows)
 
 
-def write_dict_rows_to_csv(
-    output_path, headers, rows, mode="w", extrasaction="raise", encoding="utf-8"
-):
-    """Write the provided dictionary to the provided path as comma-separated values.
+def write_dict_rows_to_csv(output_path, headers, rows, mode="w", extrasaction="raise"):
+    """Write the provided list of dictionaries to the provided path as comma-separated values.
 
     Args:
         output_path (Path): the Path were the result will be saved
         headers (list): a list of the headers for the output file
-        rows (list): the dict to be saved
+        rows (list): the list of dictionaries to be saved
         mode (str): the mode to be used when opening the file (default 'w')
         extrasaction (str): what to do if the if a field isn't in the headers (default 'raise')
     """
     create_directory(output_path, is_file=True)
     logger.debug(f"Writing {len(rows)} rows to {output_path}")
-    with open(output_path, mode, newline="", encoding=encoding) as f:
+    with open(output_path, mode, newline="") as f:
         # Create the writer object
         writer = csv.DictWriter(f, fieldnames=headers, extrasaction=extrasaction)
         # If we are writing a new row ...

{warn_scraper-1.2.112 → warn_scraper-1.2.114/warn_scraper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.112
+Version: 1.2.114
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

warn_scraper-1.2.112/warn/scrapers/ok.py

@@ -1,42 +0,0 @@
-from pathlib import Path
-
-from warn.platforms.job_center.utils import scrape_state
-
-from .. import utils
-
-__authors__ = ["zstumgoren", "Dilcia19"]
-__tags__ = ["jobcenter"]
-__source__ = {
-    "name": "Oklahoma Office of Workforces Development",
-    "url": "https://okjobmatch.com/search/warn_lookups/new",
-}
-
-
-def scrape(
-    data_dir: Path = utils.WARN_DATA_DIR,
-    cache_dir: Path = utils.WARN_CACHE_DIR,
-    use_cache: bool = True,
-) -> Path:
-    """
-    Scrape data from Oklahoma.
-
-    Keyword arguments:
-    data_dir -- the Path were the result will be saved (default WARN_DATA_DIR)
-    cache_dir -- the Path where results can be cached (default WARN_CACHE_DIR)
-    use_cache -- a Boolean indicating whether the cache should be used (default True)
-
-    Returns: the Path where the file is written
-    """
-    output_csv = data_dir / "ok.csv"
-    search_url = "https://okjobmatch.com/search/warn_lookups"
-    # Date chosen based on manual research
-    stop_year = 1999
-    # Use cache for years before current and prior year
-    scrape_state(
-        "OK", search_url, output_csv, stop_year, cache_dir, use_cache=use_cache
-    )
-    return output_csv
-
-
-if __name__ == "__main__":
-    scrape()