warn-scraper 1.2.113__py3-none-any.whl → 1.2.114__py3-none-any.whl

warn/platforms/job_center/site.py

@@ -112,7 +112,7 @@ class Site:
             logger.debug("Fetching from cache")
             return self.cache.fetch(url, params)
         else:
-            logger.debug("Pulling from the web")
+            logger.debug(f"Pulling from the web: {url} with params {params}")
             response = requests.get(url, params=params, verify=self.verify)
             logger.debug(f"Response code: {response.status_code}")
             html = response.text

warn/scrapers/ok.py

@@ -1,16 +1,19 @@
+import logging
 from pathlib import Path
 
-from warn.platforms.job_center.utils import scrape_state
+import requests
 
 from .. import utils
 
-__authors__ = ["zstumgoren", "Dilcia19"]
-__tags__ = ["jobcenter"]
+__authors__ = ["zstumgoren", "Dilcia19", "stucka"]
+__tags__ = [""]
 __source__ = {
     "name": "Oklahoma Office of Workforces Development",
-    "url": "https://okjobmatch.com/search/warn_lookups/new",
+    "url": "https://www.employoklahoma.gov/Participants/s/warnnotices",
 }
 
+logger = logging.getLogger(__name__)
+
 
 def scrape(
     data_dir: Path = utils.WARN_DATA_DIR,
@@ -28,13 +31,85 @@ def scrape(
     Returns: the Path where the file is written
     """
     output_csv = data_dir / "ok.csv"
-    search_url = "https://okjobmatch.com/search/warn_lookups"
-    # Date chosen based on manual research
-    stop_year = 1999
-    # Use cache for years before current and prior year
-    scrape_state(
-        "OK", search_url, output_csv, stop_year, cache_dir, use_cache=use_cache
-    )
+    # search_url = "https://okjobmatch.com/search/warn_lookups"
+    # search_url = "https://www.employoklahoma.gov/Participants/s/warnnotices"
+    posturl = "https://www.employoklahoma.gov/Participants/s/sfsites/aura?r=2&aura.ApexAction.execute=6"
+
+    # There are a bunch of hard-coded values in here that seem to work for at least a day.
+    # Undetermined:
+    # -- Will this continue working in the short- or medium-term?
+    # -- What is the signficance of each variable?
+    # -- How do we refresh these?
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/142.0",
+        "Accept": "*/*",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Accept-Encoding": "gzip, deflate, br, zstd",
+        "Referer": "https://www.employoklahoma.gov/Participants/s/warnnotices",
+        "X-SFDC-LDS-Endpoints": "ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.hasDocument, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.checkJobExpiry, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.checkResumeExpiry, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.checkUIRegistered, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.getLoginMaintenanceMessage, ApexActionController.execute:OESC_JS_getWARNLayoffNotices.getListofLayoffAccService",
+        "X-SFDC-Page-Scope-Id": "9c659a19-8020-41b0-a81c-36335e22801a",
+        "X-SFDC-Request-Id": "16140000007a08bd2f",
+        "X-SFDC-Page-Cache": "9439898463d86806",
+        "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
+        "X-B3-TraceId": "856a2236ba7d283e",
+        "X-B3-SpanId": "b79b2da3a7dc4544",
+        "X-B3-Sampled": "0",
+        "Origin": "https://www.employoklahoma.gov",
+        "Connection": "keep-alive",
+        "Cookie": "renderCtx=%7B%22pageId%22%3A%223823bba2-3b00-4db7-aca6-5ca0eb67fc63%22%2C%22schema%22%3A%22Published%22%2C%22viewType%22%3A%22Published%22%2C%22brandingSetId%22%3A%22fa0b6362-0214-44b9-947d-2543eaab22c7%22%2C%22audienceIds%22%3A%22%22%7D; CookieConsentPolicy=0:1; LSKey-c$CookieConsentPolicy=0:1; pctrk=f3070d0c-7078-4062-96bb-de9e82cbb1db",
+        "Sec-Fetch-Dest": "empty",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Site": "same-origin",
+    }
+
+    payload = "message=%7B%22actions%22%3A%5B%7B%22id%22%3A%22156%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22hasDocument%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22157%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22checkJobExpiry%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22158%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22checkResumeExpiry%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22159%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22checkUIRegistered%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22160%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22getLoginMaintenanceMessage%22%2C%22params%22%3A%7B%22displayTo%22%3A%22Job%20Seekers%22%2C%22messageType%22%3A%22Portal%20Login%20Messages%22%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22161%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22OESC_JS_getWARNLayoffNotices%22%2C%22method%22%3A%22getListofLayoffAccService%22%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%5D%7D&aura.context=%7B%22mode%22%3A%22PROD%22%2C%22fwuid%22%3A%22eE5UbjZPdVlRT3M0d0xtOXc5MzVOQWg5TGxiTHU3MEQ5RnBMM0VzVXc1cmcxMi42MjkxNDU2LjE2Nzc3MjE2%22%2C%22app%22%3A%22siteforce%3AcommunityApp%22%2C%22loaded%22%3A%7B%22APPLICATION%40markup%3A%2F%2Fsiteforce%3AcommunityApp%22%3A%221305_7pTC6grCTP7M16KdvDQ-Xw%22%7D%2C%22dn%22%3A%5B%5D%2C%22globals%22%3A%7B%7D%2C%22uad%22%3Atrue%7D&aura.pageURI=%2FParticipants%2Fs%2Fwarnnotices&aura.token=null"
+
+    logger.debug(f"Attempting to send hard-coded data to {posturl}")
+    r = requests.post(posturl, headers=headers, data=payload)
+    rawdata = r.json()
+
+    for entry in rawdata["actions"]:
+        if (
+            entry["id"] == "161;a"
+        ):  # What is this value? Will this change? Also no idea.
+            cleanerdata = entry["returnValue"]["returnValue"]
+    """
+    fields = set()
+    for entry in cleanerdata:
+        for field in entry:
+            fields.add(field)
+    {'Id',
+     'Launchpad__Layoff_Closure_Type__c',
+     'Launchpad__Notice_Date__c',
+     'OESC_Employer_City__c',
+     'OESC_Employer_Name__c',
+     'OESC_Employer_Zip_Code__c',
+     'RecordTypeId',
+     'Select_Local_Workforce_Board__c'}
+    """
+    fields = {
+        "Id": "id",
+        "Launchpad__Layoff_Closure_Type__c": "closure_type",
+        "Launchpad__Notice_Date__c": "notice_date",
+        "OESC_Employer_City__c": "city",
+        "OESC_Employer_Name__c": "company name",
+        "OESC_Employer_Zip_Code__c": "zip_code",
+        "RecordTypeId": "record_type_id",
+        "Select_Local_Workforce_Board__c": "workforce_board",
+    }
+
+    masterlist = []
+    for entry in cleanerdata:
+        line = {}
+        for item in fields:
+            if item in entry:
+                line[fields[item]] = entry[item]
+            else:
+                line[fields[item]] = None
+        masterlist.append(line)
+
+    utils.write_dict_rows_to_csv(output_csv, list(fields.values()), masterlist)
     return output_csv
 
 

warn/utils.py

@@ -209,12 +209,12 @@ def write_rows_to_csv(output_path: Path, rows: list, mode="w"):
 
 
 def write_dict_rows_to_csv(output_path, headers, rows, mode="w", extrasaction="raise"):
-    """Write the provided dictionary to the provided path as comma-separated values.
+    """Write the provided list of dictionaries to the provided path as comma-separated values.
 
     Args:
         output_path (Path): the Path were the result will be saved
         headers (list): a list of the headers for the output file
-        rows (list): the dict to be saved
+        rows (list): the list of dictionaries to be saved
         mode (str): the mode to be used when opening the file (default 'w')
         extrasaction (str): what to do if the if a field isn't in the headers (default 'raise')
     """

{warn_scraper-1.2.113.dist-info → warn_scraper-1.2.114.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.113
+Version: 1.2.114
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.113.dist-info → warn_scraper-1.2.114.dist-info}/RECORD

@@ -17,11 +17,11 @@ warn/__init__.py,sha256=A07JFY1TyaPtVIndBa7IvTk13DETqIkLgRdk0A-MCoE,85
 warn/cache.py,sha256=hyta04_G-ALGwcKl4xNc7EgHS_xklyVD5d8SXNrJekY,5520
 warn/cli.py,sha256=ZqyJwICdHFkn2hEgbArj_upbElR9-TSDlYDqyEGeexE,2019
 warn/runner.py,sha256=oeGRybGwpnkQKlPzRMlKxhsDt1GN4PZoX-vUwrsPgos,1894
-warn/utils.py,sha256=GOZE2koct8JlZkHFahPuTt1w8DLRyqeh58DRqGq6TTg,10382
+warn/utils.py,sha256=Jd1pIVtfUXxDweKa_6vHTNX13E47Ms7FHSw110unDHk,10408
 warn/platforms/__init__.py,sha256=wIZRDf4tbTuC8oKM4ZrTAtwNgbtMQGzPXMwDYCFyrog,81
 warn/platforms/job_center/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 warn/platforms/job_center/cache.py,sha256=yhA3sE46lNFg8vEewSoRYVByi0YSlkBiKm7qoSUiTdM,1868
-warn/platforms/job_center/site.py,sha256=Voo2PG2YC_kbAa6RUoBiqVBEoOCzIZ8imGko45KBWuY,10207
+warn/platforms/job_center/site.py,sha256=J_J6WYrfrdP9AyOuRJ8Myg_Gh5aB3-lxxp_-PBuB_A4,10236
 warn/platforms/job_center/urls.py,sha256=IWhpuzN_xcNdHh23GbZPGvuHCsMcmb03qx3pRn1Gy-k,414
 warn/platforms/job_center/utils.py,sha256=HdUKgKirmpPP7e4Cu_ZyB3zPVS_p-_ylo-lXFhxK2QM,5696
 warn/scrapers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -53,7 +53,7 @@ warn/scrapers/nj.py,sha256=nwbMbeQuUJbYRVoyUyKZBmNqvqsXu3Habt-10r8DvZE,2230
 warn/scrapers/nm.py,sha256=HZpfLzn0LvLeRztYvqJ9n6FR5PYpyMndo8tzI8h9S2o,3581
 warn/scrapers/ny.py,sha256=hXbxPhiK-Eyc9h_05wkAsfdVIT0vayKX4EE5aiJVdBc,2291
 warn/scrapers/oh.py,sha256=2MEB_0AT37dsAsrhdl_Y0LUNHu0xGy4B1F7aSMhuUu0,3151
-warn/scrapers/ok.py,sha256=qJE49VY6dMhbokFB9IAOL2XyuYSJpEKKxITPO9sUHS4,1197
+warn/scrapers/ok.py,sha256=mn9NrhVzNo9ZFtYwBtSgBhPqevHWrz1Hs37gRRjwoSo,7640
 warn/scrapers/or.py,sha256=0PjyrW3CHdxtHhqEo3Ob-9B6YckACoBD3K0c4FPQUcg,5208
 warn/scrapers/ri.py,sha256=EUyLy59eNiYHqiJR8C0YcJrZtp09KyVc45AFD0_Uc0U,4497
 warn/scrapers/sc.py,sha256=p3kscSNSW9C8C5QaSUbCAo6XibgB7G2iH6zaMH7Mnsc,4819
@@ -65,9 +65,9 @@ warn/scrapers/va.py,sha256=7Nle7qL0VNPiE653XyaP9HQqSfuJFDRr2kEkjOqLvFM,11269
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.113.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
-warn_scraper-1.2.113.dist-info/METADATA,sha256=EZuXeaW5PRvvefKsYRGscdPoLjr61_5Bz8WKVTmwtm4,2385
-warn_scraper-1.2.113.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
-warn_scraper-1.2.113.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.113.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
-warn_scraper-1.2.113.dist-info/RECORD,,
+warn_scraper-1.2.114.dist-info/licenses/LICENSE,sha256=ZV-QHyqPwyMuwuj0lI05JeSjV1NyzVEk8Yeu7FPtYS0,585
+warn_scraper-1.2.114.dist-info/METADATA,sha256=wsOFbBrbNx2mb-YWlWVggvalyIbPnOreJVgTZ222jQY,2385
+warn_scraper-1.2.114.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+warn_scraper-1.2.114.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.114.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
+warn_scraper-1.2.114.dist-info/RECORD,,