warn-scraper 1.2.113__tar.gz → 1.2.115__tar.gz

{warn_scraper-1.2.113 → warn_scraper-1.2.115}/.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
         additional_dependencies: [black]
 
 -   repo: https://github.com/timothycrosley/isort
-    rev: 5.13.2
+    rev: 6.0.1
     hooks:
     -   id: isort
         args: ["--profile", "black"]

{warn_scraper-1.2.113/warn_scraper.egg-info → warn_scraper-1.2.115}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.113
+Version: 1.2.115
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

{warn_scraper-1.2.113 → warn_scraper-1.2.115}/Pipfile

@@ -32,6 +32,7 @@ sphinxcontrib-serializinghtml = "1.1.5"
 setuptools = "*"
 jinja2 = "*"
 pytest = "==8.3.5"
+isort = "==6.0.1"
 
 [packages]
 beautifulsoup4 = "*"

{warn_scraper-1.2.113 → warn_scraper-1.2.115}/Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "aac81610731435f6fe712525f4e5c72e3d7598da79d5a3a2f7580d7c013008a2"
+            "sha256": "b1cc4814822457ec04972c1a07991d1f983c946e8cdcfc315cbe1b6d9d84b2cd"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -1067,6 +1067,15 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.1.0"
         },
+        "isort": {
+            "hashes": [
+                "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450",
+                "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615"
+            ],
+            "index": "pypi",
+            "markers": "python_full_version >= '3.9.0'",
+            "version": "==6.0.1"
+        },
         "jaraco.classes": {
             "hashes": [
                 "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd",

{warn_scraper-1.2.113 → warn_scraper-1.2.115}/warn/platforms/job_center/site.py

@@ -112,7 +112,7 @@ class Site:
             logger.debug("Fetching from cache")
             return self.cache.fetch(url, params)
         else:
-            logger.debug("Pulling from the web")
+            logger.debug(f"Pulling from the web: {url} with params {params}")
             response = requests.get(url, params=params, verify=self.verify)
             logger.debug(f"Response code: {response.status_code}")
             html = response.text

warn_scraper-1.2.115/warn/scrapers/ok.py

@@ -0,0 +1,118 @@
+import logging
+from pathlib import Path
+
+import requests
+
+from .. import utils
+
+__authors__ = ["zstumgoren", "Dilcia19", "stucka"]
+__tags__ = [""]
+__source__ = {
+    "name": "Oklahoma Office of Workforces Development",
+    "url": "https://www.employoklahoma.gov/Participants/s/warnnotices",
+}
+
+logger = logging.getLogger(__name__)
+
+
+def scrape(
+    data_dir: Path = utils.WARN_DATA_DIR,
+    cache_dir: Path = utils.WARN_CACHE_DIR,
+    use_cache: bool = True,
+) -> Path:
+    """
+    Scrape data from Oklahoma.
+
+    Keyword arguments:
+    data_dir -- the Path were the result will be saved (default WARN_DATA_DIR)
+    cache_dir -- the Path where results can be cached (default WARN_CACHE_DIR)
+    use_cache -- a Boolean indicating whether the cache should be used (default True)
+
+    Returns: the Path where the file is written
+    """
+    output_csv = data_dir / "ok.csv"
+    # search_url = "https://okjobmatch.com/search/warn_lookups"
+    # search_url = "https://www.employoklahoma.gov/Participants/s/warnnotices"
+    posturl = "https://www.employoklahoma.gov/Participants/s/sfsites/aura?r=2&aura.ApexAction.execute=6"
+
+    # There are a bunch of hard-coded values in here that seem to work for at least a day.
+    # Undetermined:
+    # -- Will this continue working in the short- or medium-term?
+    # -- What is the signficance of each variable?
+    # -- How do we refresh these?
+
+    headers = {
+        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:142.0) Gecko/20100101 Firefox/142.0",
+        "Accept": "*/*",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Accept-Encoding": "gzip, deflate, br, zstd",
+        "Referer": "https://www.employoklahoma.gov/Participants/s/warnnotices",
+        "X-SFDC-LDS-Endpoints": "ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.hasDocument, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.checkJobExpiry, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.checkResumeExpiry, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.checkUIRegistered, ApexActionController.execute:ConfigurableLoginAndMaintenanceMessages.getLoginMaintenanceMessage, ApexActionController.execute:OESC_JS_getWARNLayoffNotices.getListofLayoffAccService",
+        "X-SFDC-Page-Scope-Id": "9c659a19-8020-41b0-a81c-36335e22801a",
+        "X-SFDC-Request-Id": "16140000007a08bd2f",
+        "X-SFDC-Page-Cache": "9439898463d86806",
+        "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8",
+        "X-B3-TraceId": "856a2236ba7d283e",
+        "X-B3-SpanId": "b79b2da3a7dc4544",
+        "X-B3-Sampled": "0",
+        "Origin": "https://www.employoklahoma.gov",
+        "Connection": "keep-alive",
+        "Cookie": "renderCtx=%7B%22pageId%22%3A%223823bba2-3b00-4db7-aca6-5ca0eb67fc63%22%2C%22schema%22%3A%22Published%22%2C%22viewType%22%3A%22Published%22%2C%22brandingSetId%22%3A%22fa0b6362-0214-44b9-947d-2543eaab22c7%22%2C%22audienceIds%22%3A%22%22%7D; CookieConsentPolicy=0:1; LSKey-c$CookieConsentPolicy=0:1; pctrk=f3070d0c-7078-4062-96bb-de9e82cbb1db",
+        "Sec-Fetch-Dest": "empty",
+        "Sec-Fetch-Mode": "cors",
+        "Sec-Fetch-Site": "same-origin",
+    }
+
+    payload = "message=%7B%22actions%22%3A%5B%7B%22id%22%3A%22156%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22hasDocument%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22157%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22checkJobExpiry%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22158%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22checkResumeExpiry%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22159%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22checkUIRegistered%22%2C%22params%22%3A%7B%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22160%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22ConfigurableLoginAndMaintenanceMessages%22%2C%22method%22%3A%22getLoginMaintenanceMessage%22%2C%22params%22%3A%7B%22displayTo%22%3A%22Job%20Seekers%22%2C%22messageType%22%3A%22Portal%20Login%20Messages%22%7D%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%2C%7B%22id%22%3A%22161%3Ba%22%2C%22descriptor%22%3A%22aura%3A%2F%2FApexActionController%2FACTION%24execute%22%2C%22callingDescriptor%22%3A%22UNKNOWN%22%2C%22params%22%3A%7B%22namespace%22%3A%22%22%2C%22classname%22%3A%22OESC_JS_getWARNLayoffNotices%22%2C%22method%22%3A%22getListofLayoffAccService%22%2C%22cacheable%22%3Afalse%2C%22isContinuation%22%3Afalse%7D%7D%5D%7D&aura.context=%7B%22mode%22%3A%22PROD%22%2C%22fwuid%22%3A%22eE5UbjZPdVlRT3M0d0xtOXc5MzVOQWg5TGxiTHU3MEQ5RnBMM0VzVXc1cmcxMi42MjkxNDU2LjE2Nzc3MjE2%22%2C%22app%22%3A%22siteforce%3AcommunityApp%22%2C%22loaded%22%3A%7B%22APPLICATION%40markup%3A%2F%2Fsiteforce%3AcommunityApp%22%3A%221305_7pTC6grCTP7M16KdvDQ-Xw%22%7D%2C%22dn%22%3A%5B%5D%2C%22globals%22%3A%7B%7D%2C%22uad%22%3Atrue%7D&aura.pageURI=%2FParticipants%2Fs%2Fwarnnotices&aura.token=null"
+
+    logger.debug(f"Attempting to send hard-coded data to {posturl}")
+    r = requests.post(posturl, headers=headers, data=payload)
+    rawdata = r.json()
+
+    for entry in rawdata["actions"]:
+        if (
+            entry["id"] == "161;a"
+        ):  # What is this value? Will this change? Also no idea.
+            cleanerdata = entry["returnValue"]["returnValue"]
+    """
+    fields = set()
+    for entry in cleanerdata:
+        for field in entry:
+            fields.add(field)
+    {'Id',
+     'Launchpad__Layoff_Closure_Type__c',
+     'Launchpad__Notice_Date__c',
+     'OESC_Employer_City__c',
+     'OESC_Employer_Name__c',
+     'OESC_Employer_Zip_Code__c',
+     'RecordTypeId',
+     'Select_Local_Workforce_Board__c'}
+    """
+    fields = {
+        "Id": "id",
+        "Launchpad__Layoff_Closure_Type__c": "closure_type",
+        "Launchpad__Notice_Date__c": "notice_date",
+        "OESC_Employer_City__c": "city",
+        "OESC_Employer_Name__c": "company_name",
+        "OESC_Employer_Zip_Code__c": "zip_code",
+        "RecordTypeId": "record_type_id",
+        "Select_Local_Workforce_Board__c": "workforce_board",
+        "jobs-not-in-this": "jobs",
+    }
+
+    masterlist = []
+    for entry in cleanerdata:
+        line = {}
+        for item in fields:
+            if item in entry:
+                line[fields[item]] = entry[item]
+            else:
+                line[fields[item]] = None
+        masterlist.append(line)
+
+    utils.write_dict_rows_to_csv(output_csv, list(fields.values()), masterlist)
+    return output_csv
+
+
+if __name__ == "__main__":
+    scrape()

{warn_scraper-1.2.113 → warn_scraper-1.2.115}/warn/utils.py

@@ -209,12 +209,12 @@ def write_rows_to_csv(output_path: Path, rows: list, mode="w"):
 
 
 def write_dict_rows_to_csv(output_path, headers, rows, mode="w", extrasaction="raise"):
-    """Write the provided dictionary to the provided path as comma-separated values.
+    """Write the provided list of dictionaries to the provided path as comma-separated values.
 
     Args:
         output_path (Path): the Path were the result will be saved
         headers (list): a list of the headers for the output file
-        rows (list): the dict to be saved
+        rows (list): the list of dictionaries to be saved
         mode (str): the mode to be used when opening the file (default 'w')
         extrasaction (str): what to do if the if a field isn't in the headers (default 'raise')
     """

{warn_scraper-1.2.113 → warn_scraper-1.2.115/warn_scraper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.113
+Version: 1.2.115
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News

warn_scraper-1.2.113/warn/scrapers/ok.py

@@ -1,42 +0,0 @@
-from pathlib import Path
-
-from warn.platforms.job_center.utils import scrape_state
-
-from .. import utils
-
-__authors__ = ["zstumgoren", "Dilcia19"]
-__tags__ = ["jobcenter"]
-__source__ = {
-    "name": "Oklahoma Office of Workforces Development",
-    "url": "https://okjobmatch.com/search/warn_lookups/new",
-}
-
-
-def scrape(
-    data_dir: Path = utils.WARN_DATA_DIR,
-    cache_dir: Path = utils.WARN_CACHE_DIR,
-    use_cache: bool = True,
-) -> Path:
-    """
-    Scrape data from Oklahoma.
-
-    Keyword arguments:
-    data_dir -- the Path were the result will be saved (default WARN_DATA_DIR)
-    cache_dir -- the Path where results can be cached (default WARN_CACHE_DIR)
-    use_cache -- a Boolean indicating whether the cache should be used (default True)
-
-    Returns: the Path where the file is written
-    """
-    output_csv = data_dir / "ok.csv"
-    search_url = "https://okjobmatch.com/search/warn_lookups"
-    # Date chosen based on manual research
-    stop_year = 1999
-    # Use cache for years before current and prior year
-    scrape_state(
-        "OK", search_url, output_csv, stop_year, cache_dir, use_cache=use_cache
-    )
-    return output_csv
-
-
-if __name__ == "__main__":
-    scrape()