sunholo 0.119.16__py3-none-any.whl → 0.120.0__py3-none-any.whl

sunholo/auth/refresh.py

@@ -9,8 +9,9 @@ from ..custom_logging import log
 def get_default_email():
 
     # https://stackoverflow.com/questions/64234214/how-to-generate-a-blob-signed-url-in-google-cloud-run
-
-    gcs_credentials, project_id = refresh_credentials()
+    gcs_credentials, project_id = get_default_creds()
+    if gcs_credentials is None:
+        gcs_credentials, project_id = refresh_credentials()
 
     if gcs_credentials is None:
         log.error("Could not refresh the credentials properly.")

sunholo/discovery_engine/discovery_engine_client.py

@@ -344,6 +344,7 @@ class DiscoveryEngineClient:
         return search_response
     
     def chunk_format(self, chunk):
+
         return (
                     f"# {chunk.id}\n"
                     f"{chunk.content}\n"
@@ -351,7 +352,7 @@ class DiscoveryEngineClient:
                     f"Relevance score: {chunk.relevance_score}\n"
                     f"Document URI: {chunk.document_metadata.uri}\n"
                     f"Document Title: {chunk.document_metadata.title}\n"
-                    f"Document Metadata: {chunk.document_metadata.struct_data.dict()}\n"
+                    f"Document Metadata: {dict(chunk.document_metadata.struct_data)}\n"
                 )        
 
     def process_chunks(self, response):

sunholo/genai/file_handling.py

@@ -3,6 +3,7 @@ from ..gcs import get_bytes_from_gcs
 
 from functools import partial
 import mimetypes
+import uuid
 import asyncio
 import tempfile
 import re
@@ -11,6 +12,7 @@ import traceback
 try:
     import google.generativeai as genai
     from google import genai as genaiv2
+
 except ImportError:
     genai = None
     genaiv2 = None
@@ -89,20 +91,16 @@ def sanitize_file(filename):
 
 async def construct_file_content(gs_list, bucket:str, genai_lib=False):
     """
+    Thread-safe implementation for processing multiple files concurrently.
+    
     Args:
     - gs_list: a list of dicts representing files in a bucket
-       - contentType: The content type of the file on GCS
-       - storagePath: The path in the bucket
-       - name: The name of the file
-       - url: The URL of the file that can be used to display the contents
     - bucket: The bucket the files are in
-    - genai: whether its using the genai SDK
-
+    - genai_lib: whether its using the genai SDK
     """
     
     file_list = []
     for element in gs_list:
-
         the_mime_type = element.get('contentType')
         if the_mime_type is None:
             continue
@@ -124,34 +122,42 @@ async def construct_file_content(gs_list, bucket:str, genai_lib=False):
         img_url = f"gs://{bucket}/{file_info['storagePath']}"
         display_url = file_info.get('url')
         mime_type = file_info['contentType']
-        name = sanitize_file(file_info['name'])
+        # Generate a unique name for each file to avoid conflicts
+        original_name = sanitize_file(file_info['name'])
+        unique_name = f"{original_name}_{str(uuid.uuid4())[:8]}"
         display_name = file_info['name']
-        log.info(f"Processing {name=} {display_name=}")
+        log.info(f"Processing {unique_name=} {display_name=}")
+        
         try:
             if not genai_lib:
-                myfile = genai.get_file(name)
+                myfile = genai.get_file(unique_name)
             else:
                 client = genaiv2.Client()
-                myfile = client.files.get(name=name)
+                myfile = client.files.get(name=unique_name)
             content.append(myfile)
             content.append(f"You have been given the ability to work with file {display_name=} with {mime_type=} {display_url=}")
-            log.info(f"Found existing genai.get_file {name=}")
+            log.info(f"Found existing genai.get_file {unique_name=}")
         except Exception as e:
-            log.info(f"Not found checking genai.get_file: '{name}' {str(e)}")
+            log.info(f"Not found checking genai.get_file: '{unique_name}' {str(e)}")
             tasks.append(
                 download_gcs_upload_genai(img_url, 
-                                          mime_type=mime_type, 
-                                          name=name, 
-                                          display_url=display_url, 
-                                          display_name=display_name,
-                                          genai_lib=genai_lib)
-                                          )
-
-    # Run all tasks in parallel
-    if tasks:
-        task_content = await asyncio.gather(*tasks)
-        content.extend(task_content)
+                                         mime_type=mime_type, 
+                                         name=unique_name, 
+                                         display_url=display_url, 
+                                         display_name=display_name,
+                                         genai_lib=genai_lib)
+                                        )
 
+    # Process files in batches to avoid overwhelming the system
+    content_results = []
+    batch_size = 3  # Process 3 files at a time
+    
+    for i in range(0, len(tasks), batch_size):
+        batch = tasks[i:i+batch_size]
+        batch_results = await asyncio.gather(*batch)
+        content_results.extend(batch_results)
+    
+    content.extend(content_results)
     return content
 
 # Helper function to handle each file download with error handling
@@ -169,10 +175,9 @@ async def download_gcs_upload_genai(img_url,
                                     display_url=None, 
                                     display_name=None, 
                                     retries=3, delay=2, genai_lib=False):
-    import aiofiles
-    from google.generativeai.types import file_types
     """
     Downloads and uploads a file with retries in case of failure.
+    Thread-safe implementation using unique file paths.
     
     Args:
       - img_url: str The URL of the file to download.
@@ -184,9 +189,10 @@ async def download_gcs_upload_genai(img_url,
     Returns:
       - downloaded_content: The result of the file upload if successful.
     """
+    import aiofiles
     for attempt in range(retries):
         try:
-            log.info(f"Upload {attempt} for {img_url=}")
+            log.info(f"Upload attempt [{attempt}] for {img_url=}")
             # Download the file bytes asynchronously
             file_bytes = await asyncio.to_thread(get_bytes_from_gcs, img_url)
             if not file_bytes:
@@ -200,50 +206,72 @@ async def download_gcs_upload_genai(img_url,
 
             if file_size > 19434343:
                 log.warning(f"File size for {img_url}: {file_size} is too big.")
-                msg =  f"The file for {img_url} is too large ({file_size} bytes) to be used directly.  Use RAG instead or {display_url=}"
+                msg = f"The file for {img_url} is too large ({file_size} bytes) to be used directly. Use RAG instead or {display_url=}"
                 return {"role": "user", "parts": [{"text": msg}]}
             
             extension = mimetypes.guess_extension(mime_type)
             
-            # Use aiofiles for asynchronous file operations
-            temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=extension)
-            downloaded_file = temp_file.name
-
-            sanitized_file = sanitize_file(downloaded_file)
-
-            log.info(f"Writing file {sanitized_file}")
-            async with aiofiles.open(sanitized_file, 'wb') as f:
+            # Create a unique directory for this upload task
+            unique_id = str(uuid.uuid4())
+            temp_dir = os.path.join(tempfile.gettempdir(), f"upload_{unique_id}")
+            os.makedirs(temp_dir, exist_ok=True)
+            
+            # Create a file with unique path
+            file_path = os.path.join(temp_dir, f"file_{unique_id}{extension}")
+            
+            log.info(f"Writing file {file_path}")
+            async with aiofiles.open(file_path, 'wb') as f:
                 await f.write(file_bytes)
 
-            # Upload the file and get its content reference
-            if not genai_lib:
-                try:
-                    downloaded_content: file_types.File = await asyncio.to_thread(
+            try:
+                if not genai_lib:
+                    downloaded_content = await asyncio.to_thread(
                         partial(genai.upload_file, name=name, mime_type=mime_type, display_name=display_name), 
-                        sanitized_file
-                        )
+                        file_path
+                    )
+                    
+                    # Clean up after successful upload
+                    try:
+                        os.remove(file_path)
+                        os.rmdir(temp_dir)
+                    except OSError as e:
+                        log.warning(f"Cleanup error (non-critical): {str(e)}")
+                        
                     return {"role": "user", "parts": [{"file_data": downloaded_content}, 
-                                                    {"text": f"You have been given the ability to read and work with filename '{display_name=}' with {mime_type=} {display_url=}"}
-                                                    ]}
-                except Exception as err:
-                    msg = f"Could not upload {sanitized_file} to genai.upload_file: {str(err)} {traceback.format_exc()} {display_url=}"
-                    log.error(msg)
-                    return {"role": "user", "parts": [{"text": msg}]}
-            else:
-                try:
+                                                    {"text": f"You have been given the ability to read and work with filename '{display_name}' with {mime_type=} {display_url=}"}
+                                                   ]}
+                else:
                     client = genaiv2.Client()
-                    downloaded_content = await asyncio.to_thread(
-                        client.files.upload, 
-                        file=sanitized_file,  
-                        config=dict(mime_type=mime_type, display_name=display_name)
-                    )
+                    
+                    # Use semaphore to limit concurrent uploads
+                    async with upload_semaphore:
+                        downloaded_content = await asyncio.to_thread(
+                            client.files.upload, 
+                            file=file_path,  
+                            config=dict(mime_type=mime_type, display_name=display_name)
+                        )
+                    
+                    # Clean up after successful upload
+                    try:
+                        os.remove(file_path)
+                        os.rmdir(temp_dir)
+                    except OSError as e:
+                        log.warning(f"Cleanup error (non-critical): {str(e)}")
+                        
                     return [downloaded_content, 
-                            f"You have been given the ability to read and work with filename '{display_name=}' with {mime_type=} {display_url=}"]
+                            f"You have been given the ability to read and work with filename '{display_name}' with {mime_type=} {display_url=}"]
 
-                except Exception as err:
-                    msg = f"Could not upload {sanitized_file} to genaiv2.client.files.upload: {str(err)} {traceback.format_exc()} {display_url=}"
-                    log.error(msg)
-                    return {"role": "user", "parts": [{"text": msg}]}
+            except Exception as err:
+                # Clean up on error
+                try:
+                    os.remove(file_path)
+                    os.rmdir(temp_dir)
+                except OSError:
+                    pass
+                
+                msg = f"Could not upload {file_path} to {'genai.upload_file' if not genai_lib else 'genaiv2.client.files.upload'}: {str(err)} {traceback.format_exc()} {display_url=}"
+                log.error(msg)
+                return {"role": "user", "parts": [{"text": msg}]}
         
         except Exception as err:
             log.error(f"Error processing file {img_url} {mime_type=} on attempt {attempt + 1}/{retries}: {str(err)}")
@@ -255,3 +283,7 @@ async def download_gcs_upload_genai(img_url,
             else:
                 raise err  # Raise the error after max retries
 
+# Add this at the module level
+# Create a semaphore to limit concurrent uploads
+upload_semaphore = asyncio.Semaphore(5)  # Adjust the value based on your needs
+

{sunholo-0.119.16.dist-info → sunholo-0.120.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sunholo
-Version: 0.119.16
+Version: 0.120.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0
@@ -28,6 +28,7 @@ Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"
 Provides-Extra: all
+Requires-Dist: aiofiles; extra == "all"
 Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: anthropic[vertex]; extra == "all"
 Requires-Dist: asyncpg; extra == "all"
@@ -50,7 +51,7 @@ Requires-Dist: google-cloud-pubsub; extra == "all"
 Requires-Dist: google-cloud-discoveryengine>=0.13.4; extra == "all"
 Requires-Dist: google-cloud-texttospeech; extra == "all"
 Requires-Dist: google-generativeai>=0.7.1; extra == "all"
-Requires-Dist: google-genai; extra == "all"
+Requires-Dist: google-genai>=0.2.2; extra == "all"
 Requires-Dist: gunicorn; extra == "all"
 Requires-Dist: httpcore; extra == "all"
 Requires-Dist: httpx; extra == "all"
@@ -122,6 +123,7 @@ Requires-Dist: pytesseract; extra == "pipeline"
 Requires-Dist: tabulate; extra == "pipeline"
 Requires-Dist: unstructured[all-docs,local-inference]; extra == "pipeline"
 Provides-Extra: gcp
+Requires-Dist: aiofiles; extra == "gcp"
 Requires-Dist: anthropic[vertex]; extra == "gcp"
 Requires-Dist: google-api-python-client; extra == "gcp"
 Requires-Dist: google-auth-httplib2; extra == "gcp"
@@ -136,7 +138,7 @@ Requires-Dist: google-cloud-logging; extra == "gcp"
 Requires-Dist: google-cloud-pubsub; extra == "gcp"
 Requires-Dist: google-cloud-discoveryengine>=0.13.4; extra == "gcp"
 Requires-Dist: google-cloud-texttospeech; extra == "gcp"
-Requires-Dist: google-genai; extra == "gcp"
+Requires-Dist: google-genai>=0.2.2; extra == "gcp"
 Requires-Dist: google-generativeai>=0.8.3; extra == "gcp"
 Requires-Dist: langchain-google-genai>=2.0.0; extra == "gcp"
 Requires-Dist: langchain_google_alloydb_pg>=0.2.2; extra == "gcp"

{sunholo-0.119.16.dist-info → sunholo-0.120.0.dist-info}/RECORD

@@ -20,7 +20,7 @@ sunholo/archive/__init__.py,sha256=qNHWm5rGPVOlxZBZCpA1wTYPbalizRT7f8X4rs2t290,3
 sunholo/archive/archive.py,sha256=PxVfDtO2_2ZEEbnhXSCbXLdeoHoQVImo4y3Jr2XkCFY,1204
 sunholo/auth/__init__.py,sha256=TeP-OY0XGxYV_8AQcVGoh35bvyWhNUcMRfhuD5l44Sk,91
 sunholo/auth/gcloud.py,sha256=PdbwkuTdRi4RKBmgG9uwsReegqC4VG15_tw5uzmA7Fs,298
-sunholo/auth/refresh.py,sha256=WSlKa8TQ70GlZ2e0u83nYknhUsgTeiyyTVi-EFOa8Uc,2029
+sunholo/auth/refresh.py,sha256=cj2t337KbGfsqulrTLuhNszQcJ6PyRHzHvbD6qH-wUs,2118
 sunholo/auth/run.py,sha256=pMSp2lzL6e6ZqlltVUH92bkeUt341yMue027qrE0jQU,2821
 sunholo/azure/__init__.py,sha256=S1WQ5jndzNgzhSBh9UpX_yw7hRVm3hCzkAWNxUdK4dA,48
 sunholo/azure/auth.py,sha256=Y3fDqFLYwbsIyi5hS5L-3hYnwrLWVL96yPng5Sj5c2c,2236
@@ -75,7 +75,7 @@ sunholo/discovery_engine/__init__.py,sha256=hLgqRDJ22Aov9o2QjAEfsVgnL3kMdM-g5p8R
 sunholo/discovery_engine/chunker_handler.py,sha256=2775W5wHzdkYXqkRMlwh8MRbas20wcMnnnNngo0ljms,6160
 sunholo/discovery_engine/cli.py,sha256=KGVle5rkLL49oF9TQhrGI--8017IvvLOEoYur545Qb0,12790
 sunholo/discovery_engine/create_new.py,sha256=WUi4_xh_dFaGX3xA9jkNKZhaR6LCELjMPeRb0hyj4FU,1226
-sunholo/discovery_engine/discovery_engine_client.py,sha256=ZPe4MPHWJpq1MXKkEmyG26bIiwuRoSeomCPiNtduezM,36935
+sunholo/discovery_engine/discovery_engine_client.py,sha256=NjIcP10I2-8yj6QZKrxGzNbh3SqQ5vGYsq9OwxCpWas,36935
 sunholo/discovery_engine/get_ai_search_chunks.py,sha256=I6Dt1CznqEvE7XIZ2PkLqopmjpO96iVEWJJqL5cJjOU,5554
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=did2pKkWM2o0KkRcb0H9l2x_WjCq6OyuHDxGbITFKPM,6530
@@ -89,7 +89,7 @@ sunholo/gcs/download_url.py,sha256=Ul81n1rklr8WogPsuxWWD1Nr8RHU451LzHPMJNhAKzw,6
 sunholo/gcs/extract_and_sign.py,sha256=paRrTCvCN5vkQwCB7OSkxWi-pfOgOtZ0bwdXE08c3Ps,1546
 sunholo/gcs/metadata.py,sha256=oQLcXi4brsZ74aegWyC1JZmhlaEV270HS5_UWtAYYWE,898
 sunholo/genai/__init__.py,sha256=TV3PYHWoR4cChdmCOaYB0PtAEQ86qol9XYYEtb1JmSA,239
-sunholo/genai/file_handling.py,sha256=5wN8ynrrLLiY4JRRCCWLcqnxXVeYjgw26IH4itTrFCc,9564
+sunholo/genai/file_handling.py,sha256=qHbSs2dFt_N5obL6btB8gCOhEioLrjoB76Tn8COTBLY,10820
 sunholo/genai/genaiv2.py,sha256=uqWcfvlsPVPyQo-W_cP9h2TTzyYfzj4lyJlyqPyKTkI,20269
 sunholo/genai/images.py,sha256=EyjsDqt6XQw99pZUQamomCpMOoIah9bp3XY94WPU7Ms,1678
 sunholo/genai/init.py,sha256=yG8E67TduFCTQPELo83OJuWfjwTnGZsyACospahyEaY,687
@@ -166,9 +166,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.119.16.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.119.16.dist-info/METADATA,sha256=5ZynkhkxJECDG9oZcwhMCI3bM-_odAsKR4zzizwg6v8,9715
-sunholo-0.119.16.dist-info/WHEEL,sha256=nn6H5-ilmfVryoAQl3ZQ2l8SH5imPWFpm1A5FgEuFV4,91
-sunholo-0.119.16.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.119.16.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.119.16.dist-info/RECORD,,
+sunholo-0.120.0.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.120.0.dist-info/METADATA,sha256=hDqIBciQ8Rq9Xq2ESCQErYMFFygdKvs_yZsmzcFZ24s,9808
+sunholo-0.120.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+sunholo-0.120.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.120.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.120.0.dist-info/RECORD,,

{sunholo-0.119.16.dist-info → sunholo-0.120.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.1)
+Generator: setuptools (75.8.2)
 Root-Is-Purelib: true
 Tag: py3-none-any