pydoll-python 1.2.0__py3-none-any.whl

pydoll/browser/managers.py

@@ -0,0 +1,154 @@
+import os
+import shutil
+import subprocess
+from contextlib import suppress
+from tempfile import TemporaryDirectory
+
+from pydoll.browser.options import Options
+
+
+class ProxyManager:
+    def __init__(self, options):
+        self.options = options
+
+    def get_proxy_credentials(self) -> tuple[bool, tuple[str, str]]:
+        """
+        Configura as configurações de proxy e extrai credenciais se presentes.
+
+        Returns:
+            tuple[bool, tuple[str, str]]: (private_proxy, (username, password))
+        """
+        private_proxy = False
+        credentials = (None, None)
+
+        proxy_arg = self._find_proxy_argument()
+
+        if proxy_arg is not None:
+            index, proxy_value = proxy_arg
+            has_credentials, username, password, clean_proxy = (
+                self._parse_proxy(proxy_value)
+            )
+
+            if has_credentials:
+                self._update_proxy_argument(index, clean_proxy)
+                private_proxy = True
+                credentials = (username, password)
+
+        return private_proxy, credentials
+
+    def _find_proxy_argument(self) -> tuple[int, str] | None:
+        """Encontra o primeiro argumento --proxy-server válido"""
+        for index, arg in enumerate(self.options.arguments):
+            if arg.startswith('--proxy-server='):
+                return index, arg.split('=', 1)[1]
+        return None
+
+    @staticmethod
+    def _parse_proxy(proxy_value: str) -> tuple[bool, str, str, str]:
+        """Extrai credenciais e limpa o valor do proxy"""
+        if '@' not in proxy_value:
+            return False, None, None, proxy_value
+
+        try:
+            creds_part, server_part = proxy_value.split('@', 1)
+            username, password = creds_part.split(':', 1)
+            return True, username, password, server_part
+        except ValueError:
+            return False, None, None, proxy_value
+
+    def _update_proxy_argument(self, index: int, clean_proxy: str) -> None:
+        """Atualiza a lista de argumentos com proxy limpo"""
+        self.options.arguments[index] = f'--proxy-server={clean_proxy}'
+
+
+class BrowserProcessManager:
+    def __init__(self, process_creator=None):
+        self._process_creator = (
+            process_creator or self._default_process_creator
+        )
+        self._process = None
+
+    def start_browser_process(
+        self, binary_location: str, port: int, arguments: list
+    ) -> None:
+        """Inicia o processo do navegador"""
+        self._process = self._process_creator([
+            binary_location,
+            f'--remote-debugging-port={port}',
+            *arguments,
+        ])
+        return self._process
+
+    @staticmethod
+    def _default_process_creator(command: list[str]):
+        return subprocess.Popen(
+            command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+
+    def stop_process(self):
+        """Para o processo do navegador se estiver em execução"""
+        if self._process:
+            self._process.terminate()
+
+
+class TempDirectoryManager:
+    def __init__(self, temp_dir_factory=TemporaryDirectory):
+        self._temp_dir_factory = temp_dir_factory
+        self._temp_dirs = []
+
+    def create_temp_dir(self):
+        """
+        Cria um diretório temporário para a instância do navegador.
+
+        Returns:
+            TemporaryDirectory: O diretório temporário.
+        """
+        temp_dir = self._temp_dir_factory()
+        self._temp_dirs.append(temp_dir)
+        return temp_dir
+
+    def cleanup(self):
+        """Limpa todos os diretórios temporários"""
+        for temp_dir in self._temp_dirs:
+            with suppress(OSError):
+                shutil.rmtree(temp_dir.name)
+
+
+class BrowserOptionsManager:
+    @staticmethod
+    def initialize_options(options: Options | None) -> Options:
+        """
+        Inicializa as opções para o navegador.
+
+        Args:
+            options (Options | None): Uma instância da classe Options ou None.
+
+        Returns:
+            Options: A instância de opções inicializada.
+        """
+        if options is None:
+            return Options()
+        if not isinstance(options, Options):
+            raise ValueError('Invalid options')
+        return options
+
+    @staticmethod
+    def add_default_arguments(options: Options):
+        """Adiciona argumentos padrão aos argumentos fornecidos"""
+        options.arguments.append('--no-first-run')
+        options.arguments.append('--no-default-browser-check')
+
+    @staticmethod
+    def validate_browser_path(path: str) -> str:
+        """
+        Valida o caminho fornecido do navegador.
+
+        Args:
+            path (str): O caminho do arquivo executável do navegador.
+
+        Returns:
+            str: O caminho do navegador validado.
+        """
+        if not os.path.exists(path):
+            raise ValueError(f'Browser not found: {path}')
+        return path

pydoll/browser/options.py

@@ -0,0 +1,62 @@
+class Options:
+    """
+    A class to manage command-line options for a browser instance.
+
+    This class allows the user to specify command-line arguments and
+    the binary location of the browser executable.
+    """
+
+    def __init__(self):
+        """
+        Initializes the Options instance.
+
+        Sets up an empty list for command-line arguments and a string
+        for the binary location of the browser.
+        """
+        self._arguments = []
+        self._binary_location = ''
+
+    @property
+    def arguments(self) -> list:
+        """
+        Gets the list of command-line arguments.
+
+        Returns:
+            list: A list of command-line arguments added to the options.
+        """
+        return self._arguments
+
+    @property
+    def binary_location(self) -> str:
+        """
+        Gets the location of the browser binary.
+
+        Returns:
+            str: The file path to the browser executable.
+        """
+        return self._binary_location
+
+    @binary_location.setter
+    def binary_location(self, location: str):
+        """
+        Sets the location of the browser binary.
+
+        Args:
+            location (str): The file path to the browser executable.
+        """
+        self._binary_location = location
+
+    def add_argument(self, argument: str):
+        """
+        Adds a command-line argument to the options.
+
+        Args:
+            argument (str): The command-line argument to be added.
+
+        Raises:
+            ValueError: If the argument is already in the list of arguments.
+        """
+        if argument not in self.arguments:
+            self.arguments.append(argument)
+        else:
+            raise ValueError(f'Argument already exists: {argument}')

pydoll/browser/page.py

@@ -0,0 +1,433 @@
+import asyncio
+import json
+
+import aiofiles
+
+from pydoll.commands.dom import DomCommands
+from pydoll.commands.fetch import FetchCommands
+from pydoll.commands.network import NetworkCommands
+from pydoll.commands.page import PageCommands
+from pydoll.commands.runtime import RuntimeCommands
+from pydoll.commands.storage import StorageCommands
+from pydoll.connection.connection import ConnectionHandler
+from pydoll.element import WebElement
+from pydoll.mixins.find_elements import FindElementsMixin
+from pydoll.utils import decode_image_to_bytes
+
+
+class Page(FindElementsMixin):  # noqa: PLR0904
+    def __init__(self, connection_port: int, page_id: str):
+        """
+        Initializes the Page instance.
+
+        Args:
+            connection_handler (ConnectionHandler): The connection handler
+                instance.
+            page_id (str): The ID of the page, obtained via the DevTools
+                Protocol.
+        """
+        self._connection_handler = ConnectionHandler(connection_port, page_id)
+        self._page_events_enabled = False
+        self._network_events_enabled = False
+        self._fetch_events_enabled = False
+        self._dom_events_enabled = False
+
+    @property
+    def page_events_enabled(self) -> bool:
+        """
+        Returns whether page events are enabled or not.
+
+        Returns:
+            bool: True if page events are enabled, False otherwise.
+        """
+        return self._page_events_enabled
+
+    @property
+    def network_events_enabled(self) -> bool:
+        """
+        Returns whether network events are enabled or not.
+
+        Returns:
+            bool: True if network events are enabled, False otherwise.
+        """
+        return self._network_events_enabled
+
+    @property
+    def fetch_events_enabled(self) -> bool:
+        """
+        Returns whether fetch events are enabled or not.
+
+        Returns:
+            bool: True if fetch events are enabled, False otherwise.
+        """
+        return self._fetch_events_enabled
+
+    @property
+    def dom_events_enabled(self) -> bool:
+        """
+        Returns whether DOM events are enabled or not.
+
+        Returns:
+            bool: True if DOM events are enabled, False otherwise.
+        """
+        return self._dom_events_enabled
+
+    @property
+    async def current_url(self) -> str:
+        """
+        Retrieves the current URL of the page.
+
+        Returns:
+            str: The current URL of the page.
+        """
+        response = await self._execute_command(DomCommands.get_current_url())
+        return response['result']['result']['value']
+
+    @property
+    async def page_source(self) -> str:
+        """
+        Retrieves the source code of the page.
+
+        Returns:
+            str: The source code of the page.
+
+        """
+        response = await self._execute_command(
+            RuntimeCommands.evaluate_script(
+                'document.documentElement.outerHTML'
+            )
+        )
+        return response['result']['result']['value']
+
+    async def close(self):
+        """
+        Closes the page.
+        """
+        await self._execute_command(PageCommands.close())
+
+    async def get_cookies(self) -> list[dict]:
+        """
+        Retrieves the cookies of the page.
+
+        Returns:
+            list: A list of cookies.
+        """
+        response = await self._execute_command(
+            NetworkCommands.get_all_cookies()
+        )
+        return response['result']['cookies']
+
+    async def set_cookies(self, cookies: list[dict]):
+        """
+        Sets cookies for the page.
+
+        Args:
+            cookies (list): A list of cookies to set.
+        """
+        await self._execute_command(NetworkCommands.set_cookies(cookies))
+
+    async def delete_all_cookies(self):
+        """
+        Deletes all cookies from the browser.
+        """
+        await self._execute_command(StorageCommands.clear_cookies())
+        await self._execute_command(NetworkCommands.clear_browser_cookies())
+
+    async def has_dialog(self) -> bool:
+        """
+        Checks if a dialog is present on the page.
+
+        Returns:
+            bool: True if a dialog is present, False otherwise.
+        """
+        if self._connection_handler.dialog:
+            return True
+        return False
+
+    async def get_dialog_message(self) -> str:
+        """
+        Retrieves the message of the dialog on the page.
+
+        Returns:
+            str: The message of the dialog.
+        """
+        if not await self.has_dialog():
+            raise LookupError('No dialog present on the page')
+        return self._connection_handler.dialog['params']['message']
+
+    async def accept_dialog(self):
+        """
+        Accepts the dialog on the page.
+
+        Raises:
+            LookupError: If no dialog is present on the page.
+        """
+        if not await self.has_dialog():
+            raise LookupError('No dialog present on the page')
+        await self._execute_command(PageCommands.handle_dialog(True))
+
+    async def go_to(self, url: str, timeout=300):
+        """
+        Navigates to a URL in the page.
+
+        Args:
+            url (str): The URL to navigate to.
+        """
+        if await self._refresh_if_url_not_changed(url):
+            return
+
+        await self._execute_command(PageCommands.go_to(url))
+
+        try:
+            await self._wait_page_load(timeout=timeout)
+        except asyncio.TimeoutError:
+            raise TimeoutError('Page load timed out')
+
+    async def refresh(self):
+        """
+        Refreshes the page.
+        """
+        await self._execute_command(PageCommands.refresh())
+        try:
+            await self._wait_page_load()
+        except asyncio.TimeoutError:
+            raise TimeoutError('Page load timed out')
+
+    async def get_screenshot(self, path: str):
+        """
+        Captures a screenshot of the page.
+
+        Args:
+            path (str): The file path to save the screenshot to.
+        """
+        response = await self._execute_command(PageCommands.screenshot())
+        screenshot_b64 = response['result']['data'].encode('utf-8')
+        screenshot_bytes = decode_image_to_bytes(screenshot_b64)
+        async with aiofiles.open(path, 'wb') as file:
+            await file.write(screenshot_bytes)
+
+    async def set_download_path(self, path: str):
+        """
+        Sets the download path for the page.
+
+        Args:
+            path (str): The path where the downloaded files should be saved.
+        """
+        await self._execute_command(PageCommands.set_download_path(path))
+
+    async def get_pdf_base64(self):
+        """
+        Retrieves the PDF data of the page.
+
+        Returns:
+            str: The PDF data of the page.
+        """
+        response = await self._execute_command(PageCommands.print_to_pdf())
+        return response['result']['data']
+
+    async def print_to_pdf(self, path: str):
+        """
+        Prints the page to a PDF file.
+
+        Args:
+            path (str): The file path to save the PDF file to.
+        """
+        response = await self._execute_command(PageCommands.print_to_pdf(path))
+        pdf_b64 = response['result']['data'].encode('utf-8')
+        pdf_bytes = decode_image_to_bytes(pdf_b64)
+        async with aiofiles.open(path, 'wb') as file:
+            await file.write(pdf_bytes)
+
+    async def get_network_logs(self, matches: list[str] = []):
+        """
+        Retrieves network logs from the page.
+
+        Args:
+            matches (str): The URL pattern to match network logs against.
+
+        Returns:
+            list: A list of network logs that match the specified pattern.
+        """
+        network_logs = self._connection_handler.network_logs
+        logs_matched = []
+        for log in network_logs:
+            if not log.get('params', {}).get('request', {}).get('url'):
+                continue
+            for match in matches:
+                if match in log['params']['request']['url']:
+                    logs_matched.append(log)
+                    break
+
+        if not logs_matched:
+            raise LookupError('No network logs matched the specified pattern')
+
+        return logs_matched
+
+    async def get_network_response_bodies(self, matches: list[str] = []):
+        """
+        Retrieves the response bodies of network requests that match the
+        specified pattern.
+
+        Args:
+            matches (list): The URL patterns to match network requests against.
+
+        Returns:
+            list: A list of response bodies from network requests that match
+                the specified patterns.
+        """
+        logs_matched = await self.get_network_logs(matches)
+        responses = []
+        for log in logs_matched:
+            try:
+                body, base64encoded = await self.get_network_response_body(
+                    log['params']['requestId']
+                )
+            except KeyError:
+                continue
+            response = json.loads(body) if not base64encoded else body
+            responses.append(response)
+        return responses
+
+    async def get_network_response_body(self, request_id: str):
+        """
+        Retrieves the response body of a network request.
+
+        Args:
+            request_id (str): The ID of the network request.
+
+        Returns:
+            str: The response body of the network request.
+        """
+        response = await self._execute_command(
+            NetworkCommands.get_response_body(request_id)
+        )
+        return (
+            response['result']['body'],
+            response['result']['base64Encoded'],
+        )
+
+    async def enable_page_events(self):
+        """
+        Enables page events for the page.
+        """
+        await self._execute_command(PageCommands.enable_page())
+        self._page_events_enabled = True
+
+    async def enable_network_events(self):
+        """
+        Enables network events for the page.
+        """
+        await self._execute_command(NetworkCommands.enable_network_events())
+        self._network_events_enabled = True
+
+    async def enable_fetch_events(
+        self, handle_auth: bool = False, resource_type: str = 'Document'
+    ):
+        """
+        Enables fetch events for the page.
+        """
+        await self._execute_command(
+            FetchCommands.enable_fetch_events(handle_auth, resource_type)
+        )
+        self._fetch_events_enabled = True
+
+    async def enable_dom_events(self):
+        """
+        Enables DOM events for the page.
+        """
+        await self._execute_command(DomCommands.enable_dom_events())
+        self._dom_events_enabled = True
+
+    async def disable_fetch_events(self):
+        """
+        Disables fetch events for the page.
+        """
+        await self._execute_command(FetchCommands.disable_fetch_events())
+        self._fetch_events_enabled = False
+
+    async def disable_page_events(self):
+        """
+        Disables page events for the page.
+        """
+        await self._execute_command(PageCommands.disable_page())
+        self._page_events_enabled = False
+
+    async def on(
+        self, event_name: str, callback: callable, temporary: bool = False
+    ):
+        """
+        Registers an event listener for the page.
+
+        Args:
+            event (str): The event to listen for.
+            callback (callable): The callback function to execute when the
+                event is triggered.
+            temporary (bool): Whether the event listener is temporary or not.
+        """
+
+        async def callback_wrapper(event):
+            asyncio.create_task(callback(event))
+
+        if asyncio.iscoroutinefunction(callback):
+            function_to_register = callback_wrapper
+        else:
+            function_to_register = callback
+
+        return await self._connection_handler.register_callback(
+            event_name, function_to_register, temporary
+        )
+
+    async def execute_script(self, script: str, element: WebElement = None):
+        """
+        Executes a JavaScript script in the page.
+        If an element is provided, the script will be executed in the context
+        of that element. To provide the element context, use the 'argument'
+        keyword in the script.
+
+        Examples:
+        ```python
+        await page.execute_script('argument.click()', element)
+        await page.execute_script('argument.value = "Hello, World!"', element)
+        ```
+
+        Args:
+            script (str): The JavaScript script to execute.
+        """
+        if element:
+            script = script.replace('argument', 'this')
+            script = f'function(){{ {script} }}'
+            object_id = element._object_id
+            command = RuntimeCommands.call_function_on(
+                object_id, script, return_by_value=True
+            )
+        else:
+            command = RuntimeCommands.evaluate_script(script)
+        return await self._execute_command(command)
+
+    async def _refresh_if_url_not_changed(self, url: str):
+        """
+        Refreshes the page if the URL has not changed.
+
+        Args:
+            url (str): The URL to compare against.
+        """
+        current_url = await self.current_url
+        if current_url == url:
+            await self.refresh()
+            return True
+        return False
+
+    async def _wait_page_load(self, timeout: int = 300):
+        """
+        Waits for the page to finish loading.
+        """
+        start_time = asyncio.get_event_loop().time()
+        while True:
+            response = await self._execute_command(
+                RuntimeCommands.evaluate_script('document.readyState')
+            )
+            if response['result']['result']['value'] == 'complete':
+                break
+            if asyncio.get_event_loop().time() - start_time > timeout:
+                raise asyncio.TimeoutError('Page load timed out')
+            await asyncio.sleep(0.5)

pydoll/commands/__init__.py

@@ -0,0 +1,18 @@
+# global imports
+from pydoll.commands.dom import DomCommands
+from pydoll.commands.fetch import FetchCommands
+from pydoll.commands.input import InputCommands
+from pydoll.commands.network import NetworkCommands
+from pydoll.commands.page import PageCommands
+from pydoll.commands.runtime import RuntimeCommands
+from pydoll.commands.storage import StorageCommands
+
+__all__ = [
+    'DomCommands',
+    'FetchCommands',
+    'InputCommands',
+    'NetworkCommands',
+    'PageCommands',
+    'RuntimeCommands',
+    'StorageCommands',
+]