atomicshop 2.18.4__py3-none-any.whl → 2.18.6__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '2.18.4'
+__version__ = '2.18.6'

atomicshop/permissions/ubuntu_permissions.py

@@ -99,3 +99,52 @@ def expand_user_path(user_name, path):
     pwnam = pwd.getpwnam(user_name)
     home_dir = pwnam.pw_dir
     return path.replace("~", home_dir)
+
+
+def set_folder_permissions(
+        folder_path: str,
+        username: str = None,
+        logged_in_non_sudo_user: bool = False
+):
+    """
+    Set ownership and permissions for an existing folder.
+
+    :param folder_path: Path to the folder (must already exist)
+    :param username: Username to assign ownership to (ignored if non_sudo_user=True)
+    :param logged_in_non_sudo_user: If True, use the current logged-in user unless running under sudo
+    """
+
+    if not username and not logged_in_non_sudo_user:
+        raise ValueError("A username must be provided, or 'non_sudo_user' must be set to True.")
+
+    # Handle non_sudo_user case
+    if logged_in_non_sudo_user:
+        # Get the current logged-in user
+        username = os.getlogin()
+
+    # Get the UID and GID of the specified user
+    user_info = pwd.getpwnam(username)
+    user_uid = user_info.pw_uid
+    user_gid = user_info.pw_gid
+
+    # Change ownership of the folder to the specified user
+    # print(f"Changing ownership of {folder_path} to user '{username}'...")
+    os.chown(folder_path, user_uid, user_gid)
+
+    # Set appropriate permissions (read, write, execute for the owner)
+    # print(f"Setting permissions for {folder_path}...")
+    os.chmod(folder_path, 0o755)  # Owner rwx, group r-x, others r-x
+
+    # print(f"Ownership and permissions updated for folder: '{folder_path}'")
+
+
+def is_directory_owner(directory_path: str, username: str) -> bool:
+    """
+    Function checks if the directory is owned by the specified user.
+    :param directory_path: str, path to the directory.
+    :param username: str, username of the user.
+    :return: bool, True / False.
+    """
+
+    uid = pwd.getpwnam(username).pw_uid
+    return os.stat(directory_path).st_uid == uid

atomicshop/web_apis/google_custom_search.py

@@ -0,0 +1,37 @@
+from typing import Union
+
+from googleapiclient.discovery import build
+import googleapiclient.errors
+
+
+def search_google(
+        query: str,
+        api_key: str,
+        search_engine_id: str
+) -> tuple[
+        Union[list[str], None],
+        str]:
+    """
+    Function to search Google using Google Custom Search API for links related to a query.
+    :param query: string, the search query to search on Google Custom Search.
+    :param api_key: string, the API key for the Google Custom Search API.
+    :param search_engine_id: string, the search engine ID for the Google Custom Search API.
+
+    :return: tuple(list of strings - the links related to the query, string - the error message if any)
+    """
+
+    # noinspection PyTypeChecker
+    error: str = None
+
+    try:
+        service = build("customsearch", "v1", developerKey=api_key)
+        result = service.cse().list(q=query, cx=search_engine_id).execute()
+        items = result.get('items', [])
+        links = [item['link'] for item in items if 'link' in item]
+        return links, error
+    except googleapiclient.errors.HttpError as e:
+        # In case of rate limit error, return the error message.
+        if e.status_code == 429:
+            return None, str(e.reason)
+        else:
+            raise e

atomicshop/web_apis/google_llm.py

@@ -2,13 +2,31 @@ import os
 
 import google.generativeai as genai
 
+from . import google_custom_search
+from ..wrappers.playwrightw import scenarios
+
+
+class GoogleCustomSearchError(Exception):
+    pass
+
 
 class GoogleLLM:
     def __init__(
             self,
-            llm_api_key: str
+            llm_api_key: str,
+            search_api_key: str,
+            search_engine_id: str
     ) -> None:
+        """
+        Constructor for the GoogleLLM class.
+        :param llm_api_key: str, the API key for the Gemini API.
+        :param search_api_key: str, the API key for the Google Custom Search API.
+        :param search_engine_id: str, the search engine ID for the Google Custom Search API.
+        """
+
         self.genai = genai
+        self.search_api_key: str = search_api_key
+        self.search_engine_id: str = search_engine_id
 
         os.environ["API_KEY"] = llm_api_key
         genai.configure(api_key=os.environ["API_KEY"])
@@ -28,15 +46,73 @@ class GoogleLLM:
             number_of_top_links: int = 2,
             number_of_characters_per_link: int = 15000,
             temperature: float = 0,
-            max_output_tokens: int = 4096
-    ):
+            max_output_tokens: int = 4096,
+            model_name: str = 'gemini-pro'
+    ) -> str:
         """
         Function to get the answer to a question by searching Google Custom Console API and processing the content using Gemini API.
-        :param search_query:
-        :param additional_llm_instructions:
-        :param number_of_top_links:
-        :param number_of_characters_per_link:
-        :param temperature:
-        :param max_output_tokens:
-        :return:
-        """
+
+        :param search_query: string, the search query to search on Google Custom Search.
+        :param additional_llm_instructions: string, additional instructions to provide to the LLM.
+        :param number_of_top_links: integer, the number of top links to fetch content from.
+        :param number_of_characters_per_link: integer, the number of characters to fetch from each link.
+        :param temperature: float, the temperature parameter for the LLM.
+        :param max_output_tokens: integer, the maximum number of tokens to generate in the LLM response.
+        :param model_name: string, the name of the model to use for the LLM.
+
+        :return: string, the answer by LLM to the question.
+        """
+
+        # Search Google for links related to the query
+        links, search_error = google_custom_search.search_google(
+            query=search_query, api_key=self.search_api_key, search_engine_id=self.search_engine_id)
+
+        if search_error:
+            raise GoogleCustomSearchError(f"Error occurred when searching Google: {search_error}")
+
+        # Get only the first X links to not overload the LLM.
+        contents = scenarios.fetch_urls_content_in_threads(links[:number_of_top_links], number_of_characters_per_link)
+
+        combined_content = ""
+        for content in contents:
+            combined_content += f'{content}\n\n\n\n================================================================'
+
+        final_question = (f'Answer this question: {search_query}\n\n'
+                          f'Follow these instructions: {additional_llm_instructions}\n\n'
+                          f'Based on these data contents:\n\n'
+                          f'{combined_content}')
+
+        # Ask Gemini to process the combined content
+        gemini_response = self.ask_gemini(final_question, temperature, max_output_tokens, model_name)
+        return gemini_response
+
+    @staticmethod
+    def ask_gemini(
+            question: str,
+            temperature: float,
+            max_output_tokens: int,
+            model_name: str = 'gemini-pro'
+    ) -> str:
+        """
+        Function to ask the Gemini API a question and get the response.
+        :param question: str, the question to ask the Gemini API.
+        :param temperature: float, the temperature parameter for the LLM.
+            While 0 is deterministic, higher values can lead to more creative responses.
+        :param max_output_tokens: int, the maximum number of tokens to generate in the LLM response.
+        :param model_name: str, the name of the model to use for the LLM.
+
+        :return: str, the response from the Gemini API.
+        """
+        # Model Configuration
+        model_config = {
+            "temperature": temperature,
+            "top_p": 0.99,
+            "top_k": 0,
+            "max_output_tokens": max_output_tokens,
+        }
+
+        # model = genai.GenerativeModel('gemini-1.5-pro-latest',
+        # noinspection PyTypeChecker
+        model = genai.GenerativeModel(model_name, generation_config=model_config)
+        response = model.generate_content(question)
+        return response.text

atomicshop/wrappers/playwrightw/scenarios.py

@@ -3,6 +3,11 @@ Scenarios file contains full execution scenarios of playwright wrapper.
 For example: run playwright, navigate to URL, get text from a locator.
 """
 
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from playwright.sync_api import sync_playwright
+from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
 from . import engine, base, combos
 from ...basics import threads, multiprocesses
 
@@ -139,3 +144,68 @@ def _get_page_content_in_process(
             html_txt_convert_to_bytes=html_txt_convert_to_bytes,
             print_kwargs=print_kwargs
         )
+
+
+def fetch_urls_content_in_threads(
+        urls: list[str],
+        number_of_characters_per_link: int
+) -> list[str]:
+    """ The function to fetch all URLs concurrently using threads """
+    contents = []
+
+    # Use ThreadPoolExecutor to run multiple threads
+    with ThreadPoolExecutor() as executor:
+        # Submit tasks for each URL
+        future_to_url = {executor.submit(_fetch_content, url, number_of_characters_per_link): url for url in urls}
+
+        # Collect results as they complete
+        for future in as_completed(future_to_url):
+            url = future_to_url[future]
+            try:
+                data = future.result()
+                contents.append(data)
+            except Exception as exc:
+                print(f"An error occurred when fetching {url}: {exc}")
+
+    return contents
+
+
+def _fetch_content(url, number_of_characters_per_link):
+    """ Function to fetch content from a single URL using the synchronous Playwright API """
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        page = browser.new_page()
+        page.goto(url)
+
+        # Wait for the page to load using all possible methods, since there is no specific method
+        # that will work for all websites.
+        page.wait_for_load_state("load", timeout=5000)
+        page.wait_for_load_state("domcontentloaded", timeout=5000)
+        # The above is not enough, wait for network to stop loading files.
+        response_list: list = []
+        while True:
+            try:
+                # "**/*" is the wildcard for all URLs.
+                # 'page.expect_response' will wait for the response to be received, and then return the response object.
+                # When timeout is reached, it will raise a TimeoutError, which will break the while loop.
+                with page.expect_response("**/*", timeout=2000) as response_info:
+                    response_list.append(response_info.value)
+            except PlaywrightTimeoutError:
+                break
+
+        # Use JavaScript to extract only the visible text from the page
+        text_content: str = page.evaluate("document.body.innerText")
+        # text = page.evaluate('document.body.textContent')
+        # text = page.eval_on_selector('body', 'element => element.innerText')
+        # text = page.eval_on_selector('body', 'element => element.textContent')
+        # text = page.inner_text('body')
+        # text = page.text_content('body')
+
+        # text = page.evaluate('document.documentElement.innerText')
+        # text = page.inner_text(':root')
+        # html = page.content()
+        # html = page.evaluate('document.documentElement.outerHTML')
+
+        browser.close()
+    # Return only the first X characters of the text content to not overload the LLM.
+    return text_content[:number_of_characters_per_link]

atomicshop/wrappers/socketw/creator.py

@@ -25,13 +25,21 @@ def add_reusable_address_option(socket_instance):
     socket_instance.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 
 
-def create_ssl_context_for_server():
+def create_ssl_context_for_server() -> ssl.SSLContext:
     # Creating context with SSL certificate and the private key before the socket
     # https://docs.python.org/3/library/ssl.html
     # Creating context for SSL wrapper, specifying "PROTOCOL_TLS_SERVER" will pick the best TLS version protocol for
     # the server.
-    return ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+
+    ssl_context: ssl.SSLContext = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+
+    # # Enforce the use of TLS 1.2 only (disable TLS 1.0, TLS 1.1, and TLS 1.3)
+    # ssl_context.options |= ssl.OP_NO_TLSv1           # Disable TLS 1.0
+    # ssl_context.options |= ssl.OP_NO_TLSv1_1         # Disable TLS 1.1
+    # ssl_context.options |= ssl.OP_NO_TLSv1_3         # Disable TLS 1.3
+
     # return ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    return ssl_context
 
 
 def create_ssl_context_for_client(
@@ -130,9 +138,9 @@ def load_certificate_and_key_into_server_ssl_context(
             print_api(message, error_type=True, logger_method="critical", **print_kwargs)
 
 
-def create_server_ssl_context___load_certificate_and_key(certificate_file_path: str, key_file_path):
+def create_server_ssl_context___load_certificate_and_key(certificate_file_path: str, key_file_path) -> ssl.SSLContext:
     # Create and set ssl context for server.
-    ssl_context = create_ssl_context_for_server()
+    ssl_context: ssl.SSLContext = create_ssl_context_for_server()
     # Load certificate into context.
     load_certificate_and_key_into_server_ssl_context(ssl_context, certificate_file_path, key_file_path)
     # Return ssl context only.

atomicshop/wrappers/socketw/sni.py

@@ -82,7 +82,7 @@ class SNISetup:
     ):
 
         # Create SSL Socket to wrap the raw socket with.
-        ssl_context = creator.create_ssl_context_for_server()
+        ssl_context: ssl.SSLContext = creator.create_ssl_context_for_server()
 
         self.certificator_instance = certificator.Certificator(
             ca_certificate_name=self.ca_certificate_name,

{atomicshop-2.18.4.dist-info → atomicshop-2.18.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: atomicshop
-Version: 2.18.4
+Version: 2.18.6
 Summary: Atomic functions and classes to make developer life easier
 Author: Denis Kras
 License: MIT License
@@ -38,6 +38,8 @@ Requires-Dist: cryptography
 Requires-Dist: dnslib
 Requires-Dist: dnspython
 Requires-Dist: docker
+Requires-Dist: google-api-python-client
+Requires-Dist: google-generativeai
 Requires-Dist: numpy
 Requires-Dist: olefile
 Requires-Dist: openpyxl

{atomicshop-2.18.4.dist-info → atomicshop-2.18.6.dist-info}/RECORD

@@ -1,4 +1,4 @@
-atomicshop/__init__.py,sha256=85MhliVqiJ5CTAxGKQv7PfsSM76Jo4Xtu8UHYW8DdFA,123
+atomicshop/__init__.py,sha256=MFXW2AXFnL4Ju9Vx1r8oICO19rBqtQx1mgjqa83ULSM,123
 atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
 atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
 atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
@@ -159,7 +159,7 @@ atomicshop/monitor/checks/process_running.py,sha256=x66wd6-l466r8sbRQaIli0yswyGt
 atomicshop/monitor/checks/url.py,sha256=1PvKt_d7wFg7rDMFpUejAQhj0mqWsmlmrNfjNAV2G4g,4123
 atomicshop/permissions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/permissions/permissions.py,sha256=CYTDVOI0jh9ks0ZLnnOuPzppgCszFEc9-92DTkVTYi4,522
-atomicshop/permissions/ubuntu_permissions.py,sha256=nMmr8ywwS0or8qJdg4ztMeqB2vppE_aytQTRQrg6lFc,2842
+atomicshop/permissions/ubuntu_permissions.py,sha256=bKxZ0hl6cIOJzU1AF8EaLyscQ957_eOe7lEw4UvqKhY,4634
 atomicshop/permissions/win_permissions.py,sha256=eDQm1jfK9x_hkbLqIJjFTwfqinAWQ0iSr0kW3XrF1BE,1272
 atomicshop/process_poller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/process_poller/process_pool.py,sha256=4Qs427qd7OcBxu5PMFU5PTmyuxRy0vgj2GLsRt0IoEw,9565
@@ -177,8 +177,8 @@ atomicshop/startup/win/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 atomicshop/startup/win/startup_folder.py,sha256=2RZEyF-Mf8eWPlt_-OaoGKKnMs6YhELEzJZ376EI0E0,1891
 atomicshop/startup/win/task_scheduler.py,sha256=qALe-8sfthYxsdCViH2r8OsH3x-WauDqteg5RzElPdk,4348
 atomicshop/web_apis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-atomicshop/web_apis/google_custom_search.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-atomicshop/web_apis/google_llm.py,sha256=WVLqyfZHFIGEncxdBvrHCv2FbvQw40z75uMGzq9lxB4,1291
+atomicshop/web_apis/google_custom_search.py,sha256=evixI7y8JYyGwurRZH03nAWdD-417VFaNe1mAtuKPNA,1310
+atomicshop/web_apis/google_llm.py,sha256=UzZkPtyNA03g0xcb4vJ8imdjdNvyjUvmPUtxf9J9HnY,4898
 atomicshop/wrappers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/wrappers/_process_wrapper_curl.py,sha256=XkZZXYl7D0Q6UfdWqy-18AvpU0yVp9i2BVD2qRcXlkk,841
 atomicshop/wrappers/_process_wrapper_tar.py,sha256=WUMZFKNrlG4nJP9tWZ51W7BR1j_pIjsjgyAStmWjRGs,655
@@ -275,7 +275,7 @@ atomicshop/wrappers/playwrightw/javascript.py,sha256=_bW7CAtm0Y8IHYrAalg5HpPFnk6
 atomicshop/wrappers/playwrightw/keyboard.py,sha256=zN3YddGO-qUkn6C0CRVFejP4cTuaUwXLDNFhFREjERY,422
 atomicshop/wrappers/playwrightw/locators.py,sha256=6wsLywZxDuii7mwv-zQsRbqQC8r7j96Bma5b5_7ZoVo,2411
 atomicshop/wrappers/playwrightw/mouse.py,sha256=-2FZbQtjgH7tdXWld6ZPGqlKFUdf5in0ujN0hewxa50,656
-atomicshop/wrappers/playwrightw/scenarios.py,sha256=OzI3SV0QgazRwMZ0hMEopDHUYG-aygBSxZ50w78lIP8,5310
+atomicshop/wrappers/playwrightw/scenarios.py,sha256=RY56hH7UKvDoBr5j1JwP5xRoQtaz0AnCAkA602MurPk,8396
 atomicshop/wrappers/playwrightw/waits.py,sha256=PBFdz_PoM7Fo7O8hLqMrxNPzBEYgPoXwZceFFCGGeu8,7182
 atomicshop/wrappers/psutilw/cpus.py,sha256=w6LPBMINqS-T_X8vzdYkLS2Wzuve28Ydp_GafTCngrc,236
 atomicshop/wrappers/psutilw/disks.py,sha256=3ZSVoommKH1TWo37j_83frB-NqXF4Nf5q5mBCX8G4jE,9221
@@ -305,13 +305,13 @@ atomicshop/wrappers/socketw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
 atomicshop/wrappers/socketw/accepter.py,sha256=hZZKVYlF3LOHQJsSIEKXZUf6QXXWm-AtqXZevvaYigE,1732
 atomicshop/wrappers/socketw/base.py,sha256=zYwFxiEzTcItFi1RZQCMxMTLBvECVUiKwivPYKcu44g,2713
 atomicshop/wrappers/socketw/certificator.py,sha256=mtWPJ_ew3OSwt0-1W4jaoco1VIY4NRCrMv3mDUxb_Cc,12418
-atomicshop/wrappers/socketw/creator.py,sha256=zMWLsOF07vX-xQZR720LeHQVndUT8q-ytdCrKF5tt9I,12835
+atomicshop/wrappers/socketw/creator.py,sha256=aSwfN_IwXXf4Hob35vHXUxD_OPeshZcRDZU2hMyfKs0,13243
 atomicshop/wrappers/socketw/dns_server.py,sha256=RklzINNuoMQn4PGGQEI5hiAldprbVwwvikY6u9X-jTY,49067
 atomicshop/wrappers/socketw/exception_wrapper.py,sha256=B-X5SHLSUIWToihH2MKnOB1F4A81_X0DpLLfnYKYbEc,7067
 atomicshop/wrappers/socketw/get_process.py,sha256=aJC-_qFUv3NgWCSUzDI72E4z8_-VTZE9NVZ0CwUoNlM,5698
 atomicshop/wrappers/socketw/receiver.py,sha256=LRQO-RIY0ZRjSMGVHLVJAXFTVO1zvjgIKSefEngPFfc,8186
 atomicshop/wrappers/socketw/sender.py,sha256=aX_K8l_rHjd5AWb8bi5mt8-YTkMYVRDB6DnPqK_XDUE,4754
-atomicshop/wrappers/socketw/sni.py,sha256=Nc8WMZZR21o5GXILQLVWbf7OzNPXAfE8trJY153e9Qk,17591
+atomicshop/wrappers/socketw/sni.py,sha256=T9PXROiTYYxrd_7X4Hoj9hoNPXXTQpa2HdvmBJJIoeA,17607
 atomicshop/wrappers/socketw/socket_client.py,sha256=oa3GwS4OPgokrJE5_Oc4-5_wlXHxSH9J5f2DKebms8k,22035
 atomicshop/wrappers/socketw/socket_server_tester.py,sha256=Qobmh4XV8ZxLUaw-eW4ESKAbeSLecCKn2OWFzMhadk0,6420
 atomicshop/wrappers/socketw/socket_wrapper.py,sha256=WtylpezgIIBuz-A6PfM0hO1sm9Exd4j3qhDXcFc74-E,35567
@@ -319,8 +319,8 @@ atomicshop/wrappers/socketw/ssl_base.py,sha256=kmiif84kMhBr5yjQW17p935sfjR5JKG0L
 atomicshop/wrappers/socketw/statistics_csv.py,sha256=fgMzDXI0cybwUEqAxprRmY3lqbh30KAV-jOpoFKT-m8,3395
 atomicshop/wrappers/winregw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/wrappers/winregw/winreg_network.py,sha256=zZQfps-CdODQaTUADbHAwKHr5RUg7BLafnKWBbKaLN4,8728
-atomicshop-2.18.4.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
-atomicshop-2.18.4.dist-info/METADATA,sha256=Eabxa6pWlDm-BYBMU-rJ_YCVnxKylpxEYJ2GmbskdZM,10499
-atomicshop-2.18.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-atomicshop-2.18.4.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
-atomicshop-2.18.4.dist-info/RECORD,,
+atomicshop-2.18.6.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
+atomicshop-2.18.6.dist-info/METADATA,sha256=sMVjybxEDSWZC48hrJM-Md_y3VH4v5p9UbmQEc1XdOg,10576
+atomicshop-2.18.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+atomicshop-2.18.6.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
+atomicshop-2.18.6.dist-info/RECORD,,