warn-scraper 1.2.99__py3-none-any.whl → 1.2.101__py3-none-any.whl

warn/scrapers/va.py

@@ -2,14 +2,19 @@ import datetime
 import logging
 import os
 import platform
+
+# import subprocess
 from glob import glob
 from pathlib import Path
+from random import random
 from shutil import copyfile
 from time import sleep
 
 from selenium import webdriver
 from selenium.webdriver.chrome.options import Options as ChromeOptions
 from selenium.webdriver.chrome.service import Service as ChromeService
+from selenium.webdriver.common.by import By
+from stealthenium import stealth
 from webdriver_manager.chrome import ChromeDriverManager
 
 from .. import utils
@@ -50,6 +55,7 @@ def scrape(
     """
     cache = Cache(cache_dir)
     # csv_url = "https://vec.virginia.gov/warn-notices-csv.csv"
+    start_page = "https://www.virginiaworks.gov/warn-notices/"
     csv_url = "https://vec.virginia.gov/warn_notices.csv"
 
     """
@@ -118,6 +124,10 @@ def scrape(
     So, yes, this is a weird implementation. It's a terrible model. It's
     even got a hard-coded wait. At least as of late December 2024, however,
     it does work. ... in late December 2024.
+
+    And then it broke in early January 2025! But it's not an IP block.
+    They may have started blocking direct calls to the CSV. Code patched
+    in late January 2025 to use the Download button.
     """
 
     # driver = webdriver.Chrome(options=chromeoptionsholder, service=Service(ChromeDriverManager().install()))
@@ -127,6 +137,12 @@ def scrape(
     chromeoptionsholder.add_argument("--disable-dev-shm-usage")
     chromeoptionsholder.add_argument("--remote-debugging-pipe")
     chromeoptionsholder.add_argument("--verbose")
+    chromeoptionsholder.add_argument("start-maximized")
+    chromeoptionsholder.add_experimental_option(
+        "excludeSwitches", ["enable-automation"]
+    )
+    chromeoptionsholder.add_experimental_option("useAutomationExtension", False)
+    chromeoptionsholder.add_argument("--disable-blink-features=AutomationControlled")
 
     if "CHROMEWEBDRIVER" in os.environ:
         chrome_install = os.environ["CHROMEWEBDRIVER"] + "/chromedriver"
@@ -140,15 +156,49 @@ def scrape(
             )
     logger.debug(f"Chrome install variable is {chrome_install}")
 
+    # Hack on chromedriver itself, to try to be sneakier
+    # So many bad ideas coming together here
+    # perlstr = f"perl -pi -e 's/cdc_/ugh_/g' {chrome_install}"
+    # logger.debug(perlstr)
+    # process = subprocess.run(perlstr.split(), capture_output=True, text=True)
+    # logger.debug(f"process stdout: {process.stdout}")
+    # logger.debug(f"process stderr: {process.stderr}")
+
     # Launch X Windows emulator, then launch Chrome to run with it
     with Xvfb() as xvfb:  # noqa: F841
         service = ChromeService(chrome_install, service_args=["--verbose"])
+        # driver = webdriver.Chrome(options=chromeoptionsholder, service=service)
+        # driver = webdriver.Remote(options=chromeoptionsholder, service=service)
+        # capabilities = DesiredCapabilities.CHROME.copy()
+        # driver = webdriver.Remote(options=chromeoptionsholder, desired_capapabilities=capabilities, command_executor="http://localhost:4444/wd/hub")
+        # driver = webdriver.Chrome(options=chromeoptionsholder, service=service)
+        service = ChromeService(chrome_install, service_args=["--verbose"], port=5600)
         driver = webdriver.Chrome(options=chromeoptionsholder, service=service)
+        driver.command_executor._url = "http://localhost:5600"
+        # driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
+
+        stealth(
+            driver,
+            languages=["en-US", "en"],
+            vendor="Google Inc.",
+            platform="Win32",
+            webgl_vendor="Intel Inc.",
+            renderer="Intel Iris OpenGL Engine",
+            fix_hairline=True,
+        )
+
+        logger.debug(f"Attempting to fetch {start_page}")
+        driver.get(start_page)
+        sleep((4 * random()) + 3)
+        driver.find_element(By.ID, "warn-notice-well").find_element(
+            By.PARTIAL_LINK_TEXT, "Download"
+        ).click()
+
         logger.debug(f"Attempting to fetch {csv_url}")
-        driver.get(csv_url)
+        # driver.get(csv_url)
         sleep(45)  # Give it plenty of time to evaluate Javascript
-        driver.get(csv_url)
-        sleep(10)
+        # driver.get(csv_url)
+        # sleep(10)
         driver.quit()
 
     download_dir = os.path.expanduser("~") + "/Downloads"

{warn_scraper-1.2.99.dist-info → warn_scraper-1.2.101.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: warn-scraper
-Version: 1.2.99
+Version: 1.2.101
 Summary: Command-line interface for downloading WARN Act notices of qualified plant closings and mass layoffs from state government websites
 Home-page: https://github.com/biglocalnews/warn-scraper
 Author: Big Local News
@@ -29,12 +29,23 @@ Requires-Dist: pdfplumber
 Requires-Dist: requests
 Requires-Dist: openpyxl
 Requires-Dist: pyopenssl
-Requires-Dist: retry
+Requires-Dist: retry2
 Requires-Dist: selenium
+Requires-Dist: stealthenium
 Requires-Dist: tenacity
 Requires-Dist: xlrd
 Requires-Dist: xvfbwrapper
 Requires-Dist: webdriver-manager
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: summary
 
 ## Links
 

{warn_scraper-1.2.99.dist-info → warn_scraper-1.2.101.dist-info}/RECORD

@@ -61,13 +61,13 @@ warn/scrapers/sd.py,sha256=_4R19Ybzsyx1PvcWV3_laJmJ3etrwVGfhNEQm6njwoA,1904
 warn/scrapers/tn.py,sha256=i1H7c09Ea3CDrTXqqRMLBMPT_34QtGA0-x7T8rm_j5Q,2945
 warn/scrapers/tx.py,sha256=watfR1gyN9w7nluiAOnnIghEmoq3eShNUzYSZ8SkZy4,4438
 warn/scrapers/ut.py,sha256=iUh38YIjbvv5MyyKacsiZNe8KjfdBeDaOf-qMQEF_kc,2245
-warn/scrapers/va.py,sha256=AXcj3VpNfprhwVqVyc7hYzLamWtsf1_yOvpXSitpZeM,8389
+warn/scrapers/va.py,sha256=hOPuiAjnTmtXCOdnBM_jAJuz9_u6oCxtbm2F-9m3ot0,10732
 warn/scrapers/vt.py,sha256=d-bo4WK2hkrk4BhCCmLpEovcoZltlvdIUB6O0uaMx5A,1186
 warn/scrapers/wa.py,sha256=UXdVtHZo_a-XfoiyOooTRfTb9W3PErSZdKca6SRORgs,4282
 warn/scrapers/wi.py,sha256=ClEzXkwZbop0W4fkQgsb5oHAPUrb4luUPGV-jOKwkcg,4855
-warn_scraper-1.2.99.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-warn_scraper-1.2.99.dist-info/METADATA,sha256=UQ6xOoUHiV0l3bZONrApFK92uoWhZnoVbSvTb4QXVlg,2145
-warn_scraper-1.2.99.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-warn_scraper-1.2.99.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
-warn_scraper-1.2.99.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
-warn_scraper-1.2.99.dist-info/RECORD,,
+warn_scraper-1.2.101.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+warn_scraper-1.2.101.dist-info/METADATA,sha256=sgTsL26CgS1htJ6S-r0m62ljnet7Ilg4-zWjBSiJmdA,2385
+warn_scraper-1.2.101.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
+warn_scraper-1.2.101.dist-info/entry_points.txt,sha256=poh_oSweObGlBSs1_2qZmnTodlOYD0KfO7-h7W2UQIw,47
+warn_scraper-1.2.101.dist-info/top_level.txt,sha256=gOhHgNEkrUvajlzoKkVOo-TlQht9MoXnKOErjzqLGHo,11
+warn_scraper-1.2.101.dist-info/RECORD,,

{warn_scraper-1.2.99.dist-info → warn_scraper-1.2.101.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.1.0)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any