sunholo 0.120.0__tar.gz → 0.120.2__tar.gz

{sunholo-0.120.0/src/sunholo.egg-info → sunholo-0.120.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sunholo
-Version: 0.120.0
+Version: 0.120.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.120.0 → sunholo-0.120.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sunholo"
-version = "0.120.0"
+version = "0.120.2"
 description = "Large Language Model DevOps - a package to help deploy LLMs to the Cloud."
 readme = "README.md"
 requires-python = ">=3.10"

{sunholo-0.120.0 → sunholo-0.120.2}/src/sunholo/genai/file_handling.py

@@ -79,24 +79,40 @@ ALLOWED_MIME_TYPES = set(AUDIO_MIMES + VIDEO_MIMES + IMAGE_MIMES + DOCUMENT_MIME
 # ]
 
 def sanitize_file(filename):
+    """
+    Sanitize filename to conform to Gemini API requirements:
+    - Only lowercase alphanumeric characters and dashes
+    - Cannot begin or end with a dash
+    """
     # Split the filename into name and extension
     name, extension = os.path.splitext(filename)
     
-    # Sanitize the name by removing invalid characters and converting to lowercase
-    sanitized_name = re.sub(r'[^a-z0-9-]', '', name.lower())
-    sanitized_name = re.sub(r'^-+|-+$', '', sanitized_name)  # Remove leading or trailing dashes
+    # Convert to lowercase
+    name = name.lower()
+    
+    # Replace any non-alphanumeric characters (including underscores) with dashes
+    sanitized_name = re.sub(r'[^a-z0-9]', '-', name)
+    
+    # Remove consecutive dashes
+    sanitized_name = re.sub(r'-+', '-', sanitized_name)
     
-    # Reattach the original extension
+    # Remove leading or trailing dashes
+    sanitized_name = sanitized_name.strip('-')
+    
+    # If the name is empty after sanitization, use a default
+    if not sanitized_name:
+        sanitized_name = 'file'
+    
+    # Limit the length
     return sanitized_name[:40]
 
-async def construct_file_content(gs_list, bucket:str, genai_lib=False):
+async def construct_file_content(gs_list, bucket:str, genai_lib=False, timeout=60):
     """
-    Thread-safe implementation for processing multiple files concurrently.
-    
     Args:
     - gs_list: a list of dicts representing files in a bucket
     - bucket: The bucket the files are in
     - genai_lib: whether its using the genai SDK
+    - timeout: timeout in seconds for the gather operation
     """
     
     file_list = []
@@ -112,7 +128,7 @@ async def construct_file_content(gs_list, bucket:str, genai_lib=False):
             log.warning(f'{the_mime_type} is not in allowed MIME types for {element.get("name")}')
     
     if not file_list:
-        return {"role": "user", "parts": [{"text": "No eligible contentTypes were found"}]}
+        return [{"role": "user", "parts": [{"text": "No eligible contentTypes were found"}]}]
 
     content = []
     
@@ -122,9 +138,10 @@ async def construct_file_content(gs_list, bucket:str, genai_lib=False):
         img_url = f"gs://{bucket}/{file_info['storagePath']}"
         display_url = file_info.get('url')
         mime_type = file_info['contentType']
-        # Generate a unique name for each file to avoid conflicts
+        # Generate a unique name for each file
         original_name = sanitize_file(file_info['name'])
-        unique_name = f"{original_name}_{str(uuid.uuid4())[:8]}"
+        unique_name = sanitize_file(f"{original_name}")
+
         display_name = file_info['name']
         log.info(f"Processing {unique_name=} {display_name=}")
         
@@ -148,16 +165,30 @@ async def construct_file_content(gs_list, bucket:str, genai_lib=False):
                                          genai_lib=genai_lib)
                                         )
 
-    # Process files in batches to avoid overwhelming the system
-    content_results = []
-    batch_size = 3  # Process 3 files at a time
-    
-    for i in range(0, len(tasks), batch_size):
-        batch = tasks[i:i+batch_size]
-        batch_results = await asyncio.gather(*batch)
-        content_results.extend(batch_results)
+    # Process files with timeout
+    if tasks:
+        try:
+            # Add timeout to prevent hanging
+            task_results = await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=timeout)
+            
+            # Process results, handle any exceptions
+            for result in task_results:
+                if isinstance(result, Exception):
+                    log.error(f"Task failed with error: {str(result)}")
+                    # Add error message to content
+                    content.append({"role": "user", "parts": [{"text": f"Error processing file: {str(result)}"}]})
+                else:
+                    # Normalize the result structure based on genai_lib
+                    if genai_lib and isinstance(result, list):
+                        content.append(result[0])  # The file object
+                        content.append({"role": "user", "parts": [{"text": result[1]}]})  # The message
+                    else:
+                        content.append(result)
+        except asyncio.TimeoutError:
+            log.error(f"Timeout occurred after {timeout} seconds while processing files")
+            content.append({"role": "user", "parts": [{"text": f"Some files could not be processed within the time limit ({timeout} seconds)"}]})
     
-    content.extend(content_results)
+    log.info(f"construct_file_content completed with {len(content)} items")
     return content
 
 # Helper function to handle each file download with error handling
@@ -218,7 +249,7 @@ async def download_gcs_upload_genai(img_url,
             
             # Create a file with unique path
             file_path = os.path.join(temp_dir, f"file_{unique_id}{extension}")
-            
+            file_path = sanitize_file(file_path)
             log.info(f"Writing file {file_path}")
             async with aiofiles.open(file_path, 'wb') as f:
                 await f.write(file_bytes)

{sunholo-0.120.0 → sunholo-0.120.2/src/sunholo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: sunholo
-Version: 0.120.0
+Version: 0.120.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0