atomicshop 2.11.47__py3-none-any.whl → 3.10.5__py3-none-any.whl

atomicshop/web.py

@@ -1,20 +1,29 @@
 import os
 import urllib.request
+import urllib.error
+import ssl
+from typing import Any
+import http.client
+
+# noinspection PyPackageRequirements
+import certifi
+from dkarchiver.arch_wrappers import zips
 
-from .print_api import print_api
-from .archiver import zips
 from .urls import url_parser
 from .file_io import file_io
 from .wrappers.playwrightw import scenarios
-from . import filesystem
+from . import filesystem, print_api
 
 
 # https://www.useragents.me/
 # https://user-agents.net/
 USER_AGENTS = {
-    'Chrome_111.0.0_Windows_10-11_x64':
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
-        'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36'
+    'Chrome 111.0.0 Windows_10/11 x64':
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
+    'Chrome 132.0.0 Windows 10/11 x64':
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
+    'Chrome 142.0.0 Windows 10/11 x64':
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36'
 }
 
 
@@ -26,10 +35,10 @@ def is_status_ok(status_code: int, **kwargs) -> bool:
     """
 
     if status_code != 200:
-        print_api(f'URL Error, status code: {str(status_code)}', error_type=True, **kwargs)
+        print_api.print_api(f'URL Error, status code: {str(status_code)}', error_type=True, **kwargs)
         return False
     else:
-        print_api('URL Status: 200 OK', color="green", **kwargs)
+        print_api.print_api('URL Status: 200 OK', color="green", **kwargs)
         return True
 
 
@@ -68,7 +77,7 @@ def get_page_bytes(
         raise ValueError('ERROR: [user_agent] specified and [chrome_user_agent] usage is [True]. Choose one.')
 
     if chrome_user_agent:
-        user_agent = USER_AGENTS['Chrome_111.0.0_Windows_10-11_x64']
+        user_agent = USER_AGENTS['Chrome 142.0.0 Windows 10/11 x64']
 
     if user_agent:
         # Create a 'Request' object with the URL and user agent.
@@ -88,10 +97,13 @@ def get_page_bytes(
 
 
 def get_page_content(
-        url: str, get_method: str = 'urllib', path: str = None,
+        url: str,
+        get_method: str = 'urllib',
+        path: str = None,
         playwright_pdf_format: str = 'A4',
         playwright_html_txt_convert_to_bytes: bool = True,
-        print_kwargs: dict = None) -> any:
+        print_kwargs: dict = None
+) -> Any:
     """
     Function returns the page content from the given URL.
 
@@ -144,23 +156,41 @@ def get_page_content(
     return result
 
 
-def download(file_url: str, target_directory: str, file_name: str = None, **kwargs) -> str:
+def download(
+        file_url: str,
+        target_directory: str = None,
+        file_name: str = None,
+        headers: dict = None,
+        overwrite: bool = False,
+        # use_certifi_ca_repository: bool = False,
+        **kwargs
+) -> str | None:
     """
     The function receives url and target filesystem directory to download the file.
 
+    Note: Install 'pip-system-certs' package if you want to use system's CA store for SSL context
+    in an environment where 'certifi' package is installed.
+
     :param file_url: full URL to download the file.
     :param target_directory: The directory on the filesystem to save the file to.
+        If not specified, temporary directory will be used.
     :param file_name: string, file name (example: file.zip) that you want the downloaded file to be saved as.
         If not specified, the default filename from 'file_url' will be used.
+    :param headers: dictionary, HTTP headers to use when downloading the file.
+    :param overwrite: boolean, if True, the file will be overwritten if it already exists.
+        If False, the file will not be overwritten and the function will return None if the file already exists.
+        Default is False.
+    :param use_certifi_ca_repository: boolean, if True, the certifi CA store will be used for SSL context
+        instead of the system's default CA store.
     :return: string, full file path of downloaded file. If download failed, 'None' will be returned.
     """
 
     def print_to_console(print_end=None):
         if file_size_bytes_int:
-            print_api(
+            print_api.print_api(
                 f'Downloaded bytes: {aggregated_bytes_int} / {file_size_bytes_int}', print_end=print_end, **kwargs)
         else:
-            print_api(f'Downloaded bytes: {aggregated_bytes_int}', print_end=print_end, **kwargs)
+            print_api.print_api(f'Downloaded bytes: {aggregated_bytes_int}', print_end=print_end, **kwargs)
 
     # Size of the buffer to read each time from url.
     buffer_size: int = 4096
@@ -170,20 +200,66 @@ def download(file_url: str, target_directory: str, file_name: str = None, **kwar
         # Get only the filename from URL.
         file_name = get_filename_from_url(file_url=file_url)
 
+    # If 'target_directory' wasn't specified, we will use the temporary directory.
+    if not target_directory:
+        target_directory = filesystem.get_temp_directory()
+
     # Build full path to file.
     file_path: str = f'{target_directory}{os.sep}{file_name}'
 
-    print_api(f'Downloading: {file_url}', **kwargs)
+    if os.path.exists(file_path):
+        if overwrite:
+            print_api.print_api(f'File already exists: {file_path}. Overwriting...', **kwargs)
+        else:
+            print_api.print_api(f'File already exists: {file_path}. Skipping download.', **kwargs)
+            return file_path
+
+    print_api.print_api(f'Downloading: {file_url}', **kwargs)
+    print_api.print_api(f'To: {file_path}', **kwargs)
+
+    # Open the URL for data gathering with SSL context.
+    # if not use_certifi_ca_repository:
+    #     # Create a default SSL context using the system's CA store.
+    #     ssl_context = ssl.create_default_context()
+    # else:
+
+    # Create a default SSL context using the certifi CA store.
+    # This is useful for environments where the system's CA store is not available or not trusted.
+    # 'certifi.where()' returns the path to the certifi CA bundle.
+    ssl_context: ssl.SSLContext = ssl.create_default_context(cafile=certifi.where())
 
     # In order to use 'urllib.request', it is not enough to 'import urllib', you need to 'import urllib.request'.
-    # Open the URL for data gathering.
-    file_to_download = urllib.request.urlopen(file_url)
+    # Build a Request object with headers if provided.
+    req = urllib.request.Request(file_url, headers=headers or {})
+
+    def do_urlopen(ssl_context_internal: ssl.SSLContext) -> http.client.HTTPResponse | None:
+        try:
+            response: http.client.HTTPResponse = urllib.request.urlopen(req, context=ssl_context_internal)
+            return response
+        except urllib.error.URLError as e:
+            if getattr(e, 'reason', None) and isinstance(e.reason, ssl.SSLCertVerificationError):
+                if getattr(e.reason, 'reason', None) and e.reason.reason == 'CERTIFICATE_VERIFY_FAILED':
+                    if getattr(e.reason, 'verify_message', None) and e.reason.verify_message == 'unable to get local issuer certificate':
+                        return None
+
+            raise e
+
+    # Try to open the URL with the created SSL context with certifi.
+    file_to_download = do_urlopen(ssl_context_internal=ssl_context)
+    if not file_to_download:
+        # If failed, try to open the URL with the system's default SSL context.
+        ssl_context = ssl.create_default_context()
+        file_to_download = do_urlopen(ssl_context_internal=ssl_context)
+        if not file_to_download:
+            print_api.print_api(
+                'ERROR: URL open failed with both certifi and system\'s default SSL context.', error_type=True, **kwargs)
+            return None
 
     # Check status of url.
     if not is_status_ok(status_code=file_to_download.status, **kwargs):
         return None
 
-    file_size_bytes_int: int = None
+    file_size_bytes_int: int | None = None
     # Get file size. For some reason doesn't show for GitHub branch downloads.
     if file_to_download.headers['Content-Length']:
         file_size_bytes_int = int(file_to_download.headers['Content-Length'])
@@ -208,19 +284,27 @@ def download(file_url: str, target_directory: str, file_name: str = None, **kwar
             else:
                 print_to_console()
                 break
+
     if aggregated_bytes_int == file_size_bytes_int:
-        print_api(f'Successfully Downloaded to: {file_path}', color="green", **kwargs)
+        print_api.print_api(f'Successfully Downloaded to: {file_path}', color="green", **kwargs)
+    elif file_size_bytes_int is None:
+        pass
     else:
         message = f'Download failed: {aggregated_bytes_int} / {file_size_bytes_int}. File: {file_path}'
-        print_api(
+        print_api.print_api(
             message, error_type=True, color="red", **kwargs)
 
     return file_path
 
 
 def download_and_extract_file(
-        file_url: str, target_directory: str, file_name: str = str(), archive_remove_first_directory: bool = False,
-        **kwargs):
+        file_url: str,
+        target_directory: str,
+        file_name: str = str(),
+        archive_remove_first_directory: bool = False,
+        headers: dict = None,
+        **kwargs
+):
     """
     This function will download the branch file from GitHub, extract the file and remove the file, leaving
     only the extracted folder.
@@ -230,18 +314,19 @@ def download_and_extract_file(
         Default is empty. If it is empty, then the filename will be extracted from 'file_url'.
     :param target_directory: string, target directory where to save the file.
     :param archive_remove_first_directory: boolean, sets if archive extract function will extract the archive without
-        first directory in the archive. Check reference in the 'extract_archive_with_zipfile' function.
+        first directory in the archive. Check reference in the 'dkarchiver.arch_wrappers.zips.extract_archive_with_zipfile' function.
+    :param headers: dictionary, HTTP headers to use when downloading the file.
     :return:
     """
 
     # Download the repo to current working directory and return full file path of downloaded file.
     file_path = download(
-        file_url=file_url, target_directory=target_directory, file_name=file_name, **kwargs)
+        file_url=file_url, target_directory=target_directory, file_name=file_name, headers=headers, **kwargs)
 
     # Extract the archive and remove the first directory.
     zips.extract_archive_with_zipfile(
         archive_path=f'{file_path}', extract_directory=target_directory,
-        remove_first_directory=archive_remove_first_directory, **kwargs)
+        remove_first_directory=archive_remove_first_directory)
 
     # Remove the archive file.
     filesystem.remove_file(file_path=f'{file_path}', **kwargs)

atomicshop/web_apis/google_custom_search.py

@@ -0,0 +1,44 @@
+from typing import Union
+
+from googleapiclient.discovery import build
+import googleapiclient.errors
+
+
+def search_google(
+        query: str,
+        api_key: str,
+        search_engine_id: str
+) -> tuple[
+        Union[list[str], None],
+        str]:
+    """
+    Function to search Google using Google Custom Search API for links related to a query.
+    :param query: string, the search query to search on Google Custom Search.
+    :param api_key: string, the API key for the Google Custom Search API.
+    :param search_engine_id: string, the search engine ID for the Google Custom Search API.
+
+    :return: tuple(list of strings - the links related to the query, string - the error message if any)
+    """
+
+    # noinspection PyTypeChecker
+    error: str = None
+
+    try:
+        service = build("customsearch", "v1", developerKey=api_key)
+        result = service.cse().list(
+            q=query,
+            cx=search_engine_id,
+            # gl="us",  # Country code
+            # lr="lang_en",  # Language restriction
+            # safe="off",  # Safe search off
+            # dateRestrict="m1"  # Restrict results to the last month
+        ).execute()
+        items = result.get('items', [])
+        links = [item['link'] for item in items if 'link' in item]
+        return links, error
+    except googleapiclient.errors.HttpError as e:
+        # In case of rate limit error, return the error message.
+        if e.status_code == 429:
+            return None, str(e.reason)
+        else:
+            raise e

atomicshop/web_apis/google_llm.py

@@ -0,0 +1,188 @@
+from typing import Literal
+
+from google import genai
+from google.genai.types import GenerateContentConfig
+
+from . import google_custom_search
+from ..wrappers.playwrightw import scenarios
+from .. import urls
+
+
+class GoogleCustomSearchError(Exception):
+    pass
+
+class GoogleLLMNoContentError(Exception):
+    pass
+
+
+"""
+Rate Limits and Quotas: https://ai.google.dev/gemini-api/docs/rate-limits
+"""
+
+
+class GoogleLLM:
+    def __init__(
+            self,
+            llm_api_key: str,
+            search_api_key: str,
+            search_engine_id: str
+    ) -> None:
+        """
+        Constructor for the GoogleLLM class.
+        :param llm_api_key: str, the API key for the Gemini API.
+        :param search_api_key: str, the API key for the Google Custom Search API.
+        :param search_engine_id: str, the search engine ID for the Google Custom Search API.
+        """
+
+        self.client = genai.Client(api_key=llm_api_key)
+        self.search_api_key: str = search_api_key
+        self.search_engine_id: str = search_engine_id
+
+    def get_current_models(
+            self,
+            full_info: bool = False,
+            model_type: str = None,
+            verbose: bool = False
+    ) -> list:
+        """
+        Function to get the current models available in the Gemini API
+
+        :param full_info: bool, if True, returns the full information about the models, otherwise only the names for API usage.
+        :param model_type: str, the type of models to filter by. None, for all models.
+            Examples of known types: 'gemini', 'veo', 'imagen', 'deep-research', 'nano-banana'.
+        :param verbose: bool, if True, prints the models information to the console.
+        """
+        result_list: list = []
+        for model in self.client.models.list():
+            if model_type and model_type not in model.name:
+                continue
+
+            if full_info:
+                result_list.append(model)
+            else:
+                result_list.append(model.name)
+
+        if verbose:
+            for model in result_list:
+                print(model)
+
+        return result_list
+
+    def get_answer_online(
+            self,
+            search_query_or_url: str,
+            text_fetch_method: Literal[
+                'playwright_text',
+                'js_text',
+                'playwright_html',
+                'js_html',
+                'playwright_copypaste'
+            ],
+            llm_query: str,
+            llm_post_instructions: str,
+            number_of_top_links: int = 2,
+            number_of_characters_per_link: int = 15000,
+            temperature: float = 0,
+            # max_output_tokens: int = 4096,
+            # model_name: str = 'gemini-2.0-flash-thinking-exp-01-21'
+            model_name: str = 'gemini-2.5-pro'
+    ) -> str:
+        """
+        Function to get the answer to a question by searching Google Custom Console API and processing the content using Gemini API.
+
+        :param search_query_or_url: string, is checked if it is a URL or a search query.
+            Search query: the search query to search on Google Custom Search.
+            URL: the URL to fetch content from without using Google Custom Search.
+        :param text_fetch_method: string, the method to fetch text from the URL.
+            playwright_text: uses native Playwright to fetch text from the URL.
+            js_text: uses Playwright and JavaScript evaluation to fetch text from the URL.
+            playwright_html: uses native Playwright to fetch HTML from the URL and then parse it to text using beautiful soup.
+            js_html: uses Playwright and JavaScript evaluation to fetch HTML from the URL and then parse it to text using beautiful soup.
+            playwright_copypaste: uses native Playwright to fetch text from the URL by copying and pasting the text from rendered page using clipboard.
+        :param llm_query: string, the question to ask the LLM about the text content that is returned from the search query or the URL.
+        :param llm_post_instructions: string, additional instructions to provide to the LLM on the answer it provided after the llm_query.
+        :param number_of_top_links: integer, the number of top links to fetch content from.
+        :param number_of_characters_per_link: integer, the number of characters to fetch from each link.
+        :param temperature: float, the temperature parameter for the LLM.
+        :param max_output_tokens: integer, the maximum number of tokens to generate in the LLM response.
+        :param model_name: string, the name of the model to use for the LLM.
+
+        :return: string, the answer by LLM to the question.
+        """
+
+        # Check if the search query is a URL.
+        if urls.is_valid_url(search_query_or_url):
+            # Fetch content from the URL
+            contents = scenarios.fetch_urls_content_in_threads(
+                urls=[search_query_or_url], number_of_characters_per_link=number_of_characters_per_link,
+                text_fetch_method=text_fetch_method)
+        # If not a URL, Search Google for links related to the query
+        else:
+            links, search_error = google_custom_search.search_google(
+                query=search_query_or_url, api_key=self.search_api_key, search_engine_id=self.search_engine_id)
+
+            if search_error:
+                raise GoogleCustomSearchError(f"Error occurred when searching Google: {search_error}")
+
+            # Get only the first X links to not overload the LLM.
+            contents = scenarios.fetch_urls_content_in_threads(
+                urls=links[:number_of_top_links], number_of_characters_per_link=number_of_characters_per_link,
+                text_fetch_method=text_fetch_method)
+
+        if not contents:
+            raise GoogleLLMNoContentError("No content was fetched from the provided URL(s).")
+
+        combined_content = ""
+        for content in contents:
+            combined_content += f'{content}\n\n\n\n================================================================'
+
+        final_question = (f'Answer this question: {llm_query}\n\n'
+                          f'Follow these instructions: {llm_post_instructions}\n\n'
+                          f'Based on these data contents:\n\n'
+                          f'{combined_content}')
+
+        # Ask Gemini to process the combined content
+        # gemini_response = self.ask_gemini(final_question, temperature, max_output_tokens, model_name)
+        gemini_response = self.ask_gemini(final_question, temperature, model_name)
+        return gemini_response
+
+    def ask_gemini(
+            self,
+            question: str,
+            temperature: float,
+            # max_output_tokens: int,
+            model_name: str = 'gemini-2.5-pro'
+    ) -> str:
+        r"""
+        Function to ask the Gemini API a question and get the response.
+        :param question: str, the question to ask the Gemini API.
+        :param temperature: float, the temperature parameter for the LLM.
+            While 0 is deterministic, higher values can lead to more creative responses.
+        :param model_name: str, the name of the model to use for the LLM.
+
+        max_output_tokens: int, the maximum number of tokens to generate in the LLM response.
+            UPDATE: Disabled this feature since it gave exceptions in some situations.
+            Example:
+                  File ".\Lib\site-packages\google\generativeai\types\generation_types.py", line 464, in text
+                    parts = self.parts
+                            ^^^^^^^^^^
+                  File ".\Lib\site-packages\google\generativeai\types\generation_types.py", line 447, in parts
+                    raise ValueError(msg)
+                ValueError: Invalid operation: The `response.parts` quick accessor requires a single candidate, but but `response.candidates` is empty.
+
+
+        :return: str, the response from the Gemini API.
+        """
+        # Model Configuration
+        model_config = {
+            "temperature": temperature,
+            "top_p": 0.99,
+            "top_k": 0,
+            # "max_output_tokens": max_output_tokens,
+        }
+
+        response = self.client.models.generate_content(
+            model=model_name,
+            contents=question,
+            config=GenerateContentConfig(**model_config))
+        return response.text