atomicshop 3.3.8__py3-none-any.whl → 3.10.0__py3-none-any.whl

atomicshop/web.py

@@ -1,10 +1,14 @@
 import os
 import urllib.request
+import urllib.error
 import ssl
+from typing import Any
+import http.client
+
 # noinspection PyPackageRequirements
 import certifi
+from dkarchiver.arch_wrappers import zips
 
-from .archiver import zips
 from .urls import url_parser
 from .file_io import file_io
 from .wrappers.playwrightw import scenarios
@@ -14,12 +18,12 @@ from . import filesystem, print_api
 # https://www.useragents.me/
 # https://user-agents.net/
 USER_AGENTS = {
-    'Windows_Chrome_Latest':
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
-    'Chrome_111.0.0_Windows_10-11_x64':
+    'Chrome 111.0.0 Windows_10/11 x64':
         'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36',
-    'Chrome 132.0.0, Windows 10/11':
-        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36'
+    'Chrome 132.0.0 Windows 10/11 x64':
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
+    'Chrome 142.0.0 Windows 10/11 x64':
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36'
 }
 
 
@@ -73,7 +77,7 @@ def get_page_bytes(
         raise ValueError('ERROR: [user_agent] specified and [chrome_user_agent] usage is [True]. Choose one.')
 
     if chrome_user_agent:
-        user_agent = USER_AGENTS['Chrome_111.0.0_Windows_10-11_x64']
+        user_agent = USER_AGENTS['Chrome 142.0.0 Windows 10/11 x64']
 
     if user_agent:
         # Create a 'Request' object with the URL and user agent.
@@ -99,7 +103,7 @@ def get_page_content(
         playwright_pdf_format: str = 'A4',
         playwright_html_txt_convert_to_bytes: bool = True,
         print_kwargs: dict = None
-) -> any:
+) -> Any:
     """
     Function returns the page content from the given URL.
 
@@ -157,18 +161,25 @@ def download(
         target_directory: str = None,
         file_name: str = None,
         headers: dict = None,
-        use_certifi_ca_repository: bool = False,
+        overwrite: bool = False,
+        # use_certifi_ca_repository: bool = False,
         **kwargs
-) -> str:
+) -> str | None:
     """
     The function receives url and target filesystem directory to download the file.
 
+    Note: Install 'pip-system-certs' package if you want to use system's CA store for SSL context
+    in an environment where 'certifi' package is installed.
+
     :param file_url: full URL to download the file.
     :param target_directory: The directory on the filesystem to save the file to.
         If not specified, temporary directory will be used.
     :param file_name: string, file name (example: file.zip) that you want the downloaded file to be saved as.
         If not specified, the default filename from 'file_url' will be used.
     :param headers: dictionary, HTTP headers to use when downloading the file.
+    :param overwrite: boolean, if True, the file will be overwritten if it already exists.
+        If False, the file will not be overwritten and the function will return None if the file already exists.
+        Default is False.
     :param use_certifi_ca_repository: boolean, if True, the certifi CA store will be used for SSL context
         instead of the system's default CA store.
     :return: string, full file path of downloaded file. If download failed, 'None' will be returned.
@@ -196,29 +207,59 @@ def download(
     # Build full path to file.
     file_path: str = f'{target_directory}{os.sep}{file_name}'
 
+    if os.path.exists(file_path):
+        if overwrite:
+            print_api.print_api(f'File already exists: {file_path}. Overwriting...', **kwargs)
+        else:
+            print_api.print_api(f'File already exists: {file_path}. Skipping download.', **kwargs)
+            return file_path
+
     print_api.print_api(f'Downloading: {file_url}', **kwargs)
     print_api.print_api(f'To: {file_path}', **kwargs)
 
     # Open the URL for data gathering with SSL context.
-    if not use_certifi_ca_repository:
-        # Create a default SSL context using the system's CA store.
-        ssl_context = ssl.create_default_context()
-    else:
-        # Create a default SSL context using the certifi CA store.
-        # This is useful for environments where the system's CA store is not available or not trusted.
-        # 'certifi.where()' returns the path to the certifi CA bundle.
-        ssl_context = ssl.create_default_context(cafile=certifi.where())
+    # if not use_certifi_ca_repository:
+    #     # Create a default SSL context using the system's CA store.
+    #     ssl_context = ssl.create_default_context()
+    # else:
+
+    # Create a default SSL context using the certifi CA store.
+    # This is useful for environments where the system's CA store is not available or not trusted.
+    # 'certifi.where()' returns the path to the certifi CA bundle.
+    ssl_context: ssl.SSLContext = ssl.create_default_context(cafile=certifi.where())
 
     # In order to use 'urllib.request', it is not enough to 'import urllib', you need to 'import urllib.request'.
     # Build a Request object with headers if provided.
     req = urllib.request.Request(file_url, headers=headers or {})
-    file_to_download = urllib.request.urlopen(req, context=ssl_context)
+
+    def do_urlopen(ssl_context_internal: ssl.SSLContext) -> http.client.HTTPResponse | None:
+        try:
+            response: http.client.HTTPResponse = urllib.request.urlopen(req, context=ssl_context_internal)
+            return response
+        except urllib.error.URLError as e:
+            if getattr(e, 'reason', None) and isinstance(e.reason, ssl.SSLCertVerificationError):
+                if getattr(e.reason, 'reason', None) and e.reason.reason == 'CERTIFICATE_VERIFY_FAILED':
+                    if getattr(e.reason, 'verify_message', None) and e.reason.verify_message == 'unable to get local issuer certificate':
+                        return None
+
+            raise e
+
+    # Try to open the URL with the created SSL context with certifi.
+    file_to_download = do_urlopen(ssl_context_internal=ssl_context)
+    if not file_to_download:
+        # If failed, try to open the URL with the system's default SSL context.
+        ssl_context = ssl.create_default_context()
+        file_to_download = do_urlopen(ssl_context_internal=ssl_context)
+        if not file_to_download:
+            print_api.print_api(
+                'ERROR: URL open failed with both certifi and system\'s default SSL context.', error_type=True, **kwargs)
+            return None
 
     # Check status of url.
     if not is_status_ok(status_code=file_to_download.status, **kwargs):
         return None
 
-    file_size_bytes_int: int = None
+    file_size_bytes_int: int | None = None
     # Get file size. For some reason doesn't show for GitHub branch downloads.
     if file_to_download.headers['Content-Length']:
         file_size_bytes_int = int(file_to_download.headers['Content-Length'])
@@ -273,7 +314,7 @@ def download_and_extract_file(
         Default is empty. If it is empty, then the filename will be extracted from 'file_url'.
     :param target_directory: string, target directory where to save the file.
     :param archive_remove_first_directory: boolean, sets if archive extract function will extract the archive without
-        first directory in the archive. Check reference in the 'extract_archive_with_zipfile' function.
+        first directory in the archive. Check reference in the 'dkarchiver.arch_wrappers.zips.extract_archive_with_zipfile' function.
     :param headers: dictionary, HTTP headers to use when downloading the file.
     :return:
     """
@@ -285,7 +326,7 @@ def download_and_extract_file(
     # Extract the archive and remove the first directory.
     zips.extract_archive_with_zipfile(
         archive_path=f'{file_path}', extract_directory=target_directory,
-        remove_first_directory=archive_remove_first_directory, **kwargs)
+        remove_first_directory=archive_remove_first_directory)
 
     # Remove the archive file.
     filesystem.remove_file(file_path=f'{file_path}', **kwargs)

atomicshop/web_apis/google_llm.py

@@ -1,7 +1,7 @@
-import os
 from typing import Literal
 
-import google.generativeai as genai
+from google import genai
+from google.genai.types import GenerateContentConfig
 
 from . import google_custom_search
 from ..wrappers.playwrightw import scenarios
@@ -11,6 +11,14 @@ from .. import urls
 class GoogleCustomSearchError(Exception):
     pass
 
+class GoogleLLMNoContentError(Exception):
+    pass
+
+
+"""
+Rate Limits and Quotas: https://ai.google.dev/gemini-api/docs/rate-limits
+"""
+
 
 class GoogleLLM:
     def __init__(
@@ -26,13 +34,10 @@ class GoogleLLM:
         :param search_engine_id: str, the search engine ID for the Google Custom Search API.
         """
 
-        self.genai = genai
+        self.client = genai.Client(api_key=llm_api_key)
         self.search_api_key: str = search_api_key
         self.search_engine_id: str = search_engine_id
 
-        os.environ["API_KEY"] = llm_api_key
-        genai.configure(api_key=os.environ["API_KEY"])
-
     def get_current_models(
             self,
             full_info: bool = False
@@ -43,7 +48,7 @@ class GoogleLLM:
         :param full_info: bool, if True, returns the full information about the models, otherwise only the names for API usage.
         """
         result_list: list = []
-        for model in self.genai.list_models():
+        for model in self.client.models.list():
             if full_info:
                 result_list.append(model)
             else:
@@ -68,7 +73,7 @@ class GoogleLLM:
             temperature: float = 0,
             # max_output_tokens: int = 4096,
             # model_name: str = 'gemini-2.0-flash-thinking-exp-01-21'
-            model_name: str = 'models/gemini-2.5-pro-preview-03-25'
+            model_name: str = 'gemini-2.5-pro'
     ) -> str:
         """
         Function to get the answer to a question by searching Google Custom Console API and processing the content using Gemini API.
@@ -112,6 +117,9 @@ class GoogleLLM:
                 urls=links[:number_of_top_links], number_of_characters_per_link=number_of_characters_per_link,
                 text_fetch_method=text_fetch_method)
 
+        if not contents:
+            raise GoogleLLMNoContentError("No content was fetched from the provided URL(s).")
+
         combined_content = ""
         for content in contents:
             combined_content += f'{content}\n\n\n\n================================================================'
@@ -126,12 +134,12 @@ class GoogleLLM:
         gemini_response = self.ask_gemini(final_question, temperature, model_name)
         return gemini_response
 
-    @staticmethod
     def ask_gemini(
+            self,
             question: str,
             temperature: float,
             # max_output_tokens: int,
-            model_name: str = 'gemini-2.0-flash-thinking-exp-01-21'
+            model_name: str = 'gemini-2.5-pro'
     ) -> str:
         """
         Function to ask the Gemini API a question and get the response.
@@ -161,8 +169,8 @@ class GoogleLLM:
             # "max_output_tokens": max_output_tokens,
         }
 
-        # model = genai.GenerativeModel('gemini-1.5-pro-latest',
-        # noinspection PyTypeChecker
-        model = genai.GenerativeModel(model_name, generation_config=model_config)
-        response = model.generate_content(question)
+        response = self.client.models.generate_content(
+            model=model_name,
+            contents=question,
+            config=GenerateContentConfig(**model_config))
         return response.text

atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py

@@ -1,8 +1,9 @@
 import os
 from pathlib import Path
 
+from dkarchiver.arch_wrappers import sevenz_app_w
+
 from . import tables
-from ....archiver import sevenz_app_w
 
 
 def resolve_directory_path(directory_info, directory_key):

atomicshop/wrappers/ctyping/msi_windows_installer/extract_msi_main.py

@@ -1,11 +1,12 @@
 import os
 import argparse
 
+from dkarchiver.arch_wrappers import sevenz_app_w
+
 from .base import msi
 from . import base, tables, cabs
 from ... import olefilew
 from ....print_api import print_api
-from ....archiver import sevenz_app_w
 
 
 # Directory names.

atomicshop/wrappers/dockerw/dockerw.py

@@ -68,8 +68,8 @@ def change_image_content(
         dockerw.change_image_content(
             image_id_or_name="your_docker_image_id_or_name",
             list_of_commands=[
-                "apt-get update",
-                "apt-get install -y python3"
+                "apt update",
+                "apt install -y python3"
             ]
         )
     ----------------------

atomicshop/wrappers/elasticsearchw/config_basic.py

@@ -7,17 +7,5 @@ DEFAULT_KIBANA_PORT: str = '5601'
 DEFAULT_KIBANA_HOST: str = 'localhost'
 DEFAULT_KIBANA_URL: str = f"http://{DEFAULT_KIBANA_HOST}:{DEFAULT_KIBANA_PORT}"
 
-ELASTIC_SEARCH_CONFIG_DIRECTORY: str = "/etc/elasticsearch"
-
-ELASTIC_CONFIG_FILE: str = f"{ELASTIC_SEARCH_CONFIG_DIRECTORY}/elasticsearch.yml"
-XPACK_SECURITY_SETTING_NAME: str = "xpack.security.enabled"
-
-ELASTIC_JVM_OPTIONS_DIRECTORY: str = f"{ELASTIC_SEARCH_CONFIG_DIRECTORY}/jvm.options.d"
-ELASTIC_JVM_OPTIONS_4GB_CUSTOM_FILE: str = f"{ELASTIC_JVM_OPTIONS_DIRECTORY}/4gb_memory_heap.options"
-ELASTIC_JVM_OPTIONS_4GB_MEMORY_USAGE: list[str] = ['-Xms4g', '-Xmx4g']
-
-UBUNTU_DEPENDENCY_PACKAGES: list[str] = ['apt-transport-https', 'openjdk-11-jdk', 'wget']
-UBUNTU_ELASTIC_PACKAGE_NAME: str = 'elasticsearch'
 UBUNTU_ELASTIC_SERVICE_NAME: str = 'elasticsearch'
-UBUNTU_KIBANA_PACKAGE_NAME: str = 'kibana'
 UBUNTU_KIBANA_SERVICE_NAME: str = 'kibana'

atomicshop/wrappers/elasticsearchw/elastic_infra.py

@@ -32,174 +32,6 @@ def start_kibana_service():
     ubuntu_terminal.start_service(config_basic.UBUNTU_KIBANA_SERVICE_NAME, sudo=True)
 
 
-def start_elastic_and_check_service_availability(wait_time_seconds: float = 30, exit_on_error: bool = True):
-    """
-    Function starts the Elasticsearch service and checks its availability.
-    :param wait_time_seconds: float, the time to wait after starting the service before checking
-        the service availability.
-    :param exit_on_error: bool, if True, the function will exit the program if the service is not available.
-    :return:
-    """
-
-    # Start, enable and check the Elasticsearch service.
-    ubuntu_terminal.start_enable_service_check_availability(
-        service_name=config_basic.UBUNTU_ELASTIC_SERVICE_NAME,
-        wait_time_seconds=wait_time_seconds,
-        exit_on_error=exit_on_error
-    )
-
-    # Check if Elasticsearch is running.
-    if not is_server_available():
-        if exit_on_error:
-            sys.exit(1)
-
-
-def start_kibana_and_check_service_availability(wait_time_seconds: float = 30, exit_on_error: bool = True):
-    """
-    Function starts the Kibana service and checks its availability.
-    :param wait_time_seconds: float, the time to wait after starting the service before checking
-        the service availability.
-    :param exit_on_error: bool, if True, the function will exit the program if the service is not available.
-    :return:
-    """
-
-    # Start, enable and check the Elasticsearch service.
-    ubuntu_terminal.start_enable_service_check_availability(
-        service_name=config_basic.UBUNTU_KIBANA_SERVICE_NAME,
-        wait_time_seconds=wait_time_seconds,
-        exit_on_error=exit_on_error
-    )
-
-
-def is_elastic_config_file_exists(
-        config_file_path: str = None,
-        exit_on_error: bool = False,
-        output_message: bool = False
-) -> bool:
-    """
-    The function checks if the Elasticsearch configuration file exists.
-
-    :param config_file_path: str, the path to the configuration file.
-    :param exit_on_error: bool, if True, the function will exit the program if the file does not exist.
-    :param output_message: bool, if True, the function will print a message if the file does not exist.
-    :return:
-    """
-
-    if not config_file_path:
-        config_file_path = config_basic.ELASTIC_CONFIG_FILE
-
-    # if not ubuntu_terminal.is_sudo_file_exists(config_file_path):
-    if not filesystem.is_file_exists(config_file_path):
-        if output_message:
-            message = f"Configuration file does not exist at {config_file_path}."
-            print_api(message, color='red', error_type=True)
-        if exit_on_error:
-            sys.exit(1)
-        return False
-    else:
-        return True
-
-
-def check_xpack_security_setting(config_file_path: str = None):
-    """
-    The function checks if the 'xpack.security.enabled' setting is set to 'false' in the Elasticsearch
-    configuration file.
-
-    :param config_file_path:
-    :return:
-    """
-
-    if not config_file_path:
-        config_file_path = config_basic.ELASTIC_CONFIG_FILE
-
-    with open(config_file_path, 'r') as file:
-        # Read the file contents
-        contents = file.read()
-        # Check if the specific setting exists
-        if f"{config_basic.XPACK_SECURITY_SETTING_NAME}: false" in contents:
-            return False
-        elif f"{config_basic.XPACK_SECURITY_SETTING_NAME}: true" in contents:
-            return True
-        # If the setting doesn't exist, return None.
-        else:
-            return None
-
-
-def modify_xpack_security_setting(
-        config_file_path: str = None,
-        setting: bool = False,
-        output_message: bool = True
-):
-    """
-    The function modifies the 'xpack.security.enabled' setting in the Elasticsearch configuration file.
-    :param config_file_path: str, the path to the configuration file.
-    :param setting: bool, the setting to change to. Will be added, if doesn't exist.
-    :param output_message: bool, if True, the function will print a message.
-    :return:
-    """
-
-    if not config_file_path:
-        config_file_path = config_basic.ELASTIC_CONFIG_FILE
-
-    # The setting to set in the configuration file.
-    xpack_setting_to_set: str = f'{config_basic.XPACK_SECURITY_SETTING_NAME}: {str(setting).lower()}'
-
-    # Check if the setting exists in the configuration file and get its value.
-    current_xpack_security_setting = check_xpack_security_setting(config_file_path)
-
-    # If the setting doesn't exist, add it to the configuration file.
-    if current_xpack_security_setting is None:
-        with open(config_file_path, 'a') as file:
-            file.write(f'{xpack_setting_to_set}\n')
-        if output_message:
-            print_api(f"Added [{xpack_setting_to_set}] to the configuration.")
-    # If the setting exists and is different from the desired setting, change it.
-    elif current_xpack_security_setting != setting:
-        with open(config_file_path, 'r') as file:
-            lines = file.readlines()
-        with open(config_file_path, 'w') as file:
-            for line in lines:
-                if f"{config_basic.XPACK_SECURITY_SETTING_NAME}:" in line:
-                    file.write(f'{xpack_setting_to_set}\n')
-                else:
-                    file.write(line)
-        if output_message:
-            print_api(f"Changed [{config_basic.XPACK_SECURITY_SETTING_NAME}] to [{setting}].")
-    # If the setting is already set to the desired value, print a message.
-    elif current_xpack_security_setting == setting:
-        if output_message:
-            print_api(f"The setting is already set to [{setting}].")
-
-
-def create_jvm_options_custom_file(file_path: str, options: list):
-    """
-    The function creates a custom JVM options file for Elasticsearch.
-    You can use the default directory path as 'config_basic.ELASTIC_JVM_OPTIONS_DIRECTORY'.
-    :param file_path: str, the path to the custom JVM options file.
-    :param options: list, the list of JVM options.
-    :return:
-    """
-
-    # Write the options to the file.
-    with open(file_path, 'w') as file:
-        for option in options:
-            file.write(f"{option}\n")
-
-
-def create_jvm_options_custom_4gb_memory_heap_file(file_path: str = None):
-    """
-    The function creates a custom JVM options file with 4GB memory heap usage.
-    The 4GB memory usage options are needed for the Elasticsearch to work properly and not to crash.
-    :param file_path: str, the path to the custom JVM options file.
-    :return:
-    """
-
-    if not file_path:
-        file_path = config_basic.ELASTIC_JVM_OPTIONS_4GB_CUSTOM_FILE
-
-    create_jvm_options_custom_file(file_path, config_basic.ELASTIC_JVM_OPTIONS_4GB_MEMORY_USAGE)
-
-
 def is_server_available(
         max_attempts: int = 5,
         wait_between_attempts_seconds: float = 10,
@@ -241,25 +73,3 @@ def is_server_available(
 
     print_api("Elasticsearch did not start within the expected time.", color='red', **print_kwargs)
     return False
-
-
-def is_4gb_memory_heap_options_applied_on_server() -> bool:
-    """
-    The function checks if the 4GB memory heap options are applied on the Elasticsearch server.
-    :return: bool.
-    """
-
-    # Send a GET request
-    response = requests.get(config_basic.DEFAULT_ELASTIC_URL_JVM_OPTIONS)
-    response.raise_for_status()  # Raise an exception for HTTP errors
-
-    # Load JSON data from the response
-    jvm_data = response.json()
-
-    # Check if memory heap options are applied in 'input_arguments' key.
-    for node in jvm_data['nodes'].values():
-        # Get the JVM input arguments values.
-        input_arguments = node['jvm']['input_arguments']
-
-        # Check that the 4GB memory heap options are applied.
-        return all(options in input_arguments for options in config_basic.ELASTIC_JVM_OPTIONS_4GB_MEMORY_USAGE)

atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py

@@ -1,13 +1,13 @@
-import sys
 import subprocess
 from pathlib import Path
 
+from dkarchiver.arch_wrappers import zips
+from dkinst.installers.helpers import docker_installer
+
 from .... import filesystem
 from ....permissions import ubuntu_permissions
-from ....archiver import zips
 from ....print_api import print_api
 from ... import githubw, pipw, ubuntu_terminal
-from ...dockerw import install_docker
 from .. import config_install
 
 
@@ -87,7 +87,7 @@ def install_before_restart(
 
         # Install docker. FACT installs the docker, but there can be a problem with permissions, so we need to add
         # the user permissions to the docker group before restart.
-        if not install_docker.add_current_user_to_docker_group():
+        if not docker_installer.add_current_user_to_docker_group():
             print_api("Docker is installed, but the current user was not added to the docker group.", color='red')
             return 1
     else:
@@ -108,7 +108,7 @@ def install_before_restart(
         #     use_docker_installer=True, rootless=True, add_current_user_to_docker_group_bool=False)
 
         # Install docker in regular mode.
-        result: int = install_docker.install_docker_ubuntu(
+        result: int = docker_installer.install_docker_ubuntu(
             use_docker_installer=True, rootless=False, add_current_user_to_docker_group_bool=True)
         if result != 0:
             print_api("Docker installation failed. Please install Docker manually.", color='red')