datamarket 0.9.49__py3-none-any.whl → 0.9.51__py3-none-any.whl

datamarket/utils/playwright/async_api.py

@@ -2,22 +2,151 @@
 # IMPORTS
 
 import asyncio
+import logging
+from datetime import timedelta
 from random import randint
+from types import TracebackType
+from typing import Optional, Self
 
-from playwright.async_api import Page
+# 'BdbQuit' import is removed as it's no longer used
+from camoufox.async_api import AsyncCamoufox as Camoufox
+from playwright.async_api import (
+    Browser,
+    BrowserContext,
+    Error as PlaywrightError,
+    Page,
+    TimeoutError as PlaywrightTimeoutError,
+)
+from tenacity import (
+    before_sleep_log,
+    retry,
+    retry_if_exception_type,
+    stop_after_delay,
+    wait_exponential,
+)
+
+from datamarket.interfaces.proxy import ProxyInterface
 
 
 ########################################################################################################################
-# FUNCTIONS
+# SETUP LOGGER
+
+logger = logging.getLogger(__name__)
+
+########################################################################################################################
+# ASYNC HELPER FUNCTIONS
 
 
 async def human_type(page: Page, text: str, delay: int = 100):
     for char in text:
-        await page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+        await page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))
 
 
-async def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, sleep=True):
+async def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, add_sleep: bool = True) -> None:
+    """Asynchronously presses a key with a random delay, optionally sleeping between presses."""
     for _ in range(count):
-        await page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
-        if sleep:
-            await asyncio.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)  # noqa: S311
+        await page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))
+        if add_sleep:
+            await asyncio.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)
+
+
+########################################################################################################################
+# ASYNC CRAWLER CLASS
+
+
+class PlaywrightCrawler:
+    """An robust, proxy-enabled asynchronous Playwright crawler with captcha bypass and retry logic."""
+
+    def __init__(self, proxy_interface: ProxyInterface):
+        """
+        Initializes the async crawler with a proxy interface.
+
+        Args:
+            proxy_interface (ProxyInterface): An async-compatible object to fetch proxy credentials.
+        """
+        self.proxy_interface = proxy_interface
+        self.pw: Optional[Camoufox] = None
+        self.browser: Optional[Browser] = None
+        self.context: Optional[BrowserContext] = None
+        self.page: Optional[Page] = None
+
+    async def __aenter__(self) -> Self:
+        """Initializes the browser context when entering the `async with` statement."""
+        await self.init_context()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        """Safely closes the browser context upon exit."""
+        if self.pw:
+            await self.pw.__aexit__(exc_type, exc_val, exc_tb)
+
+    @retry(
+        wait=wait_exponential(exp_base=2, multiplier=3, max=90),
+        stop=stop_after_delay(timedelta(minutes=10)),
+        before_sleep=before_sleep_log(logger, logging.INFO),
+        reraise=True,
+    )
+    async def init_context(self) -> Self:
+        """Initializes a new async browser instance and context with a fresh proxy."""
+        try:
+            # Correctly wrap the blocking I/O call
+            host, port, user, pwd = await asyncio.to_thread(self.proxy_interface.get_proxies, raw=True, use_auth=True)
+            proxy_url = f"http://{host}:{port}"
+            proxy_cfg = {"server": proxy_url}
+
+            if user and pwd:
+                proxy_cfg.update({"username": user, "password": pwd})
+
+            logger.info(f"Starting browser with proxy: {proxy_url}")
+            self.pw = Camoufox(headless=True, geoip=True, humanize=True, proxy=proxy_cfg)
+            self.browser = await self.pw.__aenter__()
+            self.context = await self.browser.new_context()
+            self.page = await self.context.new_page()
+        except Exception as e:
+            logger.error(f"Failed to initialize browser context: {e}")
+            if self.pw:
+                await self.pw.__aexit__(type(e), e, e.__traceback__)
+            raise
+        return self
+
+    async def restart_context(self) -> None:
+        """Closes the current browser instance and initializes a new one."""
+        logger.info("Restarting browser context...")
+        if self.pw:
+            await self.pw.__aexit__(None, None, None)
+        await self.init_context()
+
+    @retry(
+        retry=retry_if_exception_type((PlaywrightTimeoutError, PlaywrightError)),
+        wait=wait_exponential(exp_base=2, multiplier=3, max=90),
+        stop=stop_after_delay(timedelta(minutes=10)),
+        before_sleep=before_sleep_log(logger, logging.INFO),
+        reraise=True,
+    )
+    async def _goto_with_retry(self, url: str) -> Page:
+        """
+        Asynchronously navigates to a URL with retries for common Playwright errors.
+        Restarts the browser context on repeated failures.
+        """
+        if not (self.page and not self.page.is_closed()):
+            logger.warning("Page is not available or closed. Restarting context.")
+            await self.restart_context()
+
+        assert self.page is not None
+        await self.page.goto(url, timeout=30000, wait_until="domcontentloaded")
+        return self.page
+
+    async def goto(self, url: str) -> Page:
+        """
+        Ensures the browser is initialized and navigates to the given URL.
+        Public wrapper for the internal retry-enabled navigation method.
+        """
+        if not self.page:
+            logger.info("Browser context not found, initializing now...")
+            await self.init_context()
+        return await self._goto_with_retry(url)

datamarket/utils/playwright/sync_api.py

@@ -1,23 +1,149 @@
 ########################################################################################################################
 # IMPORTS
 
+import logging
 import time
+from datetime import timedelta
 from random import randint
+from types import TracebackType
+from typing import Optional, Self
 
-from playwright.sync_api import Page
-
+from camoufox import Camoufox
+from playwright.sync_api import (
+    Browser,
+    BrowserContext,
+    Error as PlaywrightError,
+    Page,
+    TimeoutError as PlaywrightTimeoutError,
+)
+from tenacity import (
+    before_sleep_log,
+    retry,
+    retry_if_exception_type,
+    stop_after_delay,
+    wait_exponential,
+)
+from datamarket.interfaces.proxy import ProxyInterface
 
 ########################################################################################################################
-# FUNCTIONS
+# SETUP LOGGER
+
+logger = logging.getLogger(__name__)
 
 
+########################################################################################################################
+# HELPER FUNCTIONS
 def human_type(page: Page, text: str, delay: int = 100):
     for char in text:
-        page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
+        page.keyboard.type(char, delay=randint(int(delay * 0.5), int(delay * 1.5)))
 
 
-def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, sleep=True):
+def human_press_key(page: Page, key: str, count: int = 1, delay: int = 100, add_sleep: bool = True) -> None:
+    """Presses a key with a random delay, optionally sleeping between presses."""
     for _ in range(count):
-        page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))  # noqa: S311
-        if sleep:
-            time.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)  # noqa: S311
+        page.keyboard.press(key, delay=randint(int(delay * 0.5), int(delay * 1.5)))
+        if add_sleep:
+            time.sleep(randint(int(delay * 1.5), int(delay * 2.5)) / 1000)
+
+
+########################################################################################################################
+# CRAWLER CLASS
+
+
+class PlaywrightCrawler:
+    """A robust, proxy-enabled Playwright crawler with captcha bypass and retry logic."""
+
+    def __init__(self, proxy_interface: ProxyInterface):
+        """
+        Initializes the crawler with a proxy interface.
+
+        Args:
+            proxy_interface (ProxyInterface): An object to fetch proxy credentials.
+        """
+        self.proxy_interface = proxy_interface
+        self.pw: Optional[Camoufox] = None
+        self.browser: Optional[Browser] = None
+        self.context: Optional[BrowserContext] = None
+        self.page: Optional[Page] = None
+
+    def __enter__(self) -> Self:
+        """Initializes the browser context when entering the `with` statement."""
+        self.init_context()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        """Safely closes the browser context upon exit."""
+        if self.pw:
+            self.pw.__exit__(exc_type, exc_val, exc_tb)
+
+    @retry(
+        wait=wait_exponential(exp_base=2, multiplier=3, max=90),
+        stop=stop_after_delay(timedelta(minutes=10)),
+        before_sleep=before_sleep_log(logger, logging.INFO),
+        reraise=True,
+    )
+    def init_context(self) -> Self:
+        """Initializes a new browser instance and context with a fresh proxy."""
+        try:
+            host, port, user, pwd = self.proxy_interface.get_proxies(raw=True, use_auth=True)
+            proxy_url = f"http://{host}:{port}"
+            proxy_cfg = {"server": proxy_url}
+
+            if user and pwd:
+                proxy_cfg.update({"username": user, "password": pwd})
+
+            logger.info(f"Starting browser with proxy: {proxy_url}")
+            self.pw = Camoufox(headless=True, geoip=True, humanize=True, proxy=proxy_cfg)
+            self.browser = self.pw.__enter__()
+            self.context = self.browser.new_context()
+            self.page = self.context.new_page()
+        except Exception as e:
+            logger.error(f"Failed to initialize browser context: {e}")
+            if self.pw:
+                self.pw.__exit__(type(e), e, e.__traceback__)
+            raise
+        return self
+
+    def restart_context(self) -> None:
+        """Closes the current browser instance and initializes a new one."""
+        logger.info("Restarting browser context...")
+        if self.pw:
+            self.pw.__exit__(None, None, None)
+        self.init_context()
+
+    @retry(
+        retry=retry_if_exception_type((PlaywrightTimeoutError, PlaywrightError)),
+        wait=wait_exponential(exp_base=2, multiplier=3, max=90),
+        stop=stop_after_delay(timedelta(minutes=10)),
+        before_sleep=before_sleep_log(logger, logging.INFO),
+        before=lambda rs: rs.args[0].restart_context() if rs.attempt_number > 1 else None,
+        reraise=True,
+    )
+    def _goto_with_retry(self, url: str) -> Page:
+        """
+        Navigates to a URL with retries for common Playwright errors.
+        Restarts the browser context on repeated failures.
+        """
+        if not (self.page and not self.page.is_closed()):
+            logger.warning("Page is not available or closed. Restarting context.")
+            self.restart_context()
+
+        # self.page is guaranteed to be valid here by the logic above
+        assert self.page is not None
+        self.page.goto(url, timeout=30000, wait_until="domcontentloaded")
+        return self.page
+
+    def goto(self, url: str) -> Page:
+        """
+        Ensures the browser is initialized and navigates to the given URL.
+        Public wrapper for the internal retry-enabled navigation method.
+        """
+        if not self.page:
+            logger.info("Browser context not found, initializing now...")
+            self.init_context()
+        return self._goto_with_retry(url)

datamarket/utils/strings/normalization.py

@@ -137,8 +137,13 @@ def normalize(
     # Parameter mapping
     if isinstance(mode, str):
         mode = NormalizationMode[mode.upper()]
+    if not isinstance(mode, NormalizationMode):
+        raise TypeError("mode must be NormalizationMode or str")
+
     if isinstance(naming, str):
         naming = NamingConvention[naming.upper()]
+    if not isinstance(naming, NamingConvention):
+        raise TypeError("naming must be NamingConvention or str")
 
     _allowed_symbols_set: Set[str] = set(allowed_symbols) if allowed_symbols else set()
 
@@ -148,7 +153,11 @@ def normalize(
     elif not isinstance(s, str):
         return str(s)
     else:
-        text = prettify(strip_html(str(s), True))
+        raw_text = str(s)
+        if naming is NamingConvention.NONE:
+            text = raw_text
+        else:
+            text = prettify(strip_html(raw_text, True))
 
         if mode is NormalizationMode.NONE:
             normalized = text
@@ -170,9 +179,7 @@ def normalize(
 
             for c in intermediate_text:
                 cat = unicodedata.category(c)
-                if c in _allowed_symbols_set:  # Allowed symbols are part of tokens
-                    current_token_chars.append(c)
-                elif c.isalnum():
+                if c in _allowed_symbols_set or c.isalnum():  # Allowed symbols are part of tokens
                     current_token_chars.append(c)
                 elif mode is NormalizationMode.FULL and cat.startswith("S"):
                     # Transliterate S* category symbols not in allowed_symbols

{datamarket-0.9.49.dist-info → datamarket-0.9.51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: datamarket
-Version: 0.9.49
+Version: 0.9.51
 Summary: Utilities that integrate advanced scraping knowledge into just one library.
 License: GPL-3.0-or-later
 Author: DataMarket

{datamarket-0.9.49.dist-info → datamarket-0.9.51.dist-info}/RECORD

@@ -17,16 +17,16 @@ datamarket/utils/airflow.py,sha256=al0vc0YUikNu3Oy51VSn52I7pMU40akFBOl_UlHa2E4,7
 datamarket/utils/alchemy.py,sha256=SRq6kgh1aANXVShBPgAuglmNhZssPWwWEY503gKSia8,635
 datamarket/utils/main.py,sha256=DMMgkQnMS6fNziTru8FM9z2ERfYfkdR9qFPF7s6sp7U,9657
 datamarket/utils/playwright/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-datamarket/utils/playwright/async_api.py,sha256=pWfVj-ItfIeZBxG7WiUHKSeZDcHQFUQ5mrNGyIh1IdA,883
-datamarket/utils/playwright/sync_api.py,sha256=lIGm8mLxhFg04LVNdF8SO_9yjOLBnWe5pPry6ZFMnIg,846
+datamarket/utils/playwright/async_api.py,sha256=UbA2D4ScBtYeMfrRjly4RO-s8wXIub9c05J1eoOCpsQ,5782
+datamarket/utils/playwright/sync_api.py,sha256=Tw_-KLB3vipFuEQwcX8iCbj7giCzcwXB-bhl_ncR-2Q,5542
 datamarket/utils/selenium.py,sha256=IMKlbLzXABFhACnWzhHmB0l2hhVzNwHGZwbo14nEewQ,2499
 datamarket/utils/soda.py,sha256=eZTXFbI1P3WoMd1MM-YjoVTpdjTcDSWuvBb7ViBMhSQ,941
 datamarket/utils/strings/__init__.py,sha256=b6TYOT9v7y9ID-lDyZk4E8BH2uIPbsF2ZSLGjCQ1MCQ,43
-datamarket/utils/strings/normalization.py,sha256=QLZ-THzjGOK9eWPPR1PrsffwQkSOx_Mgha4IYaJPrR0,8713
+datamarket/utils/strings/normalization.py,sha256=rj0wfJSjqcCRp-ruHqc5pylO3_TOmY5_V1lKzkyWoAA,8991
 datamarket/utils/strings/obfuscation.py,sha256=8gMepfjPq0N4_IpKR6i2dy_9VJugQ3qJiRiRvKavB3s,5246
 datamarket/utils/typer.py,sha256=FDF3l6gh3UlAFPsHCtesnekvct2rKz0oFn3uKARBQvE,814
 datamarket/utils/types.py,sha256=vxdQZdwdXrfPR4Es52gBgol-tMRIOD6oK9cBo3rB0JQ,74
-datamarket-0.9.49.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-datamarket-0.9.49.dist-info/METADATA,sha256=p0SoJ9ST9H6bBvoyEATxbOau5b-DDviXBFTf2-WlDM8,7326
-datamarket-0.9.49.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-datamarket-0.9.49.dist-info/RECORD,,
+datamarket-0.9.51.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+datamarket-0.9.51.dist-info/METADATA,sha256=UPV2cxDlddvKHiKY5tSt-dDkA7reLhyIX1KAIfcjxag,7326
+datamarket-0.9.51.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+datamarket-0.9.51.dist-info/RECORD,,