sunholo 0.109.1__py3-none-any.whl → 0.109.4__py3-none-any.whl

sunholo/genai/file_handling.py

@@ -1,18 +1,18 @@
 from ..custom_logging import log
 from ..gcs import get_bytes_from_gcs
 
+from functools import partial
 import mimetypes
 import asyncio
 import tempfile
 import re
+import os
 import traceback
 try:
     import google.generativeai as genai
-    from google.generativeai.types import file_types
 except ImportError:
     genai = None
-    file_types = None
-    
+
 DOCUMENT_MIMES = [
     'application/pdf',
     'application/x-javascript',
@@ -72,12 +72,25 @@ ALLOWED_MIME_TYPES = set(AUDIO_MIMES + VIDEO_MIMES + IMAGE_MIMES + DOCUMENT_MIME
 #   'url': 'https://firebasestorage.googleapis.com/v0/b/multiv...', 
 #   'contentType': 'image/jpeg'}
 # ]
+
+def sanitize_file(filename):
+    # Split the filename into name and extension
+    name, extension = os.path.splitext(filename)
+    
+    # Sanitize the name by removing invalid characters and converting to lowercase
+    sanitized_name = re.sub(r'[^a-z0-9-]', '', name.lower())
+    sanitized_name = re.sub(r'^-+|-+$', '', sanitized_name)  # Remove leading or trailing dashes
+    
+    # Reattach the original extension
+    return f"{sanitized_name}"
+
 async def construct_file_content(gs_list, bucket:str):
     """
     Args:
     - gs_list: a list of dicts representing files in a bucket
        - contentType: The content type of the file on GCS
        - storagePath: The path in the bucket
+       - name: The name of the file
     - bucket: The bucket the files are in
 
     """
@@ -102,32 +115,50 @@ async def construct_file_content(gs_list, bucket:str):
     tasks = []
     for file_info in file_list:
         img_url = f"gs://{bucket}/{file_info['storagePath']}"
+        display_url = file_info.get('url')
         mime_type = file_info['contentType']
-        # Append the async download task to the task list
-        tasks.append(download_gcs_upload_genai(img_url, mime_type))
+        name = sanitize_file(file_info['name'])
+        log.info(f"Processing {name=}")
+        try:
+            myfile = genai.get_file(name)
+            content.append(
+                {"role": "user", "parts": [
+                    {"file_data": myfile}, 
+                    {"text": f"You have been given the ability to work with file {name=} with {mime_type=} {display_url=}"}
+                    ]
+                })
+            log.info(f"Found existing genai.get_file {name=}")
+            
+        except Exception as e:
+            log.info(f"Not found checking genai.get_file: '{name}' {str(e)}")
+            tasks.append(download_gcs_upload_genai(img_url, mime_type, name=name))
 
     # Run all tasks in parallel
-    content = await asyncio.gather(*tasks)
+    if tasks:
+        task_content = await asyncio.gather(*tasks)
+        content.extend(task_content)
 
     return content
 
 # Helper function to handle each file download with error handling
-async def download_file_with_error_handling(img_url, mime_type):
+async def download_file_with_error_handling(img_url, mime_type, name):
     try:
-        return await download_gcs_upload_genai(img_url, mime_type)
+        return await download_gcs_upload_genai(img_url, mime_type, name)
     except Exception as err:
         msg= f"Error processing file from {img_url}: {str(err)}"
         log.error(msg)
         return {"role": "user", "parts": [{"text": msg}]}
 
-async def download_gcs_upload_genai(img_url, mime_type, retries=3, delay=2):
+async def download_gcs_upload_genai(img_url, mime_type, name=None, retries=3, delay=2):
     import aiofiles
+    from google.generativeai.types import file_types
     """
     Downloads and uploads a file with retries in case of failure.
     
     Args:
       - img_url: str The URL of the file to download.
       - mime_type: str The MIME type of the file.
+      - name: str Optional name, else a random one will be created
       - retries: int Number of retry attempts before failing.
       - delay: int Initial delay between retries, exponentially increasing.
       
@@ -159,7 +190,7 @@ async def download_gcs_upload_genai(img_url, mime_type, retries=3, delay=2):
             temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=extension)
             downloaded_file = temp_file.name
 
-            sanitized_file = re.sub(r'[^\w\-.]', '_', downloaded_file)
+            sanitized_file = sanitize_file(downloaded_file)
 
             log.info(f"Writing file {sanitized_file}")
             async with aiofiles.open(sanitized_file, 'wb') as f:
@@ -167,8 +198,13 @@ async def download_gcs_upload_genai(img_url, mime_type, retries=3, delay=2):
 
             # Upload the file and get its content reference
             try:
-                downloaded_content: file_types.File = await asyncio.to_thread(genai.upload_file, sanitized_file )
-                return {"role": "user", "parts": [{"file_data": downloaded_content}]}
+                downloaded_content: file_types.File = await asyncio.to_thread(
+                    partial(genai.upload_file, name=name, mime_type=mime_type), 
+                    sanitized_file
+                    )
+                return {"role": "user", "parts": [{"file_data": downloaded_content}, 
+                                                  {"text": f"You have been given the ability to read and work with filename '{name}' with {mime_type=}."}
+                                                  ]}
             except Exception as err:
                 msg = f"Could not upload {sanitized_file} to genai.upload_file: {str(err)} {traceback.format_exc()}"
                 log.error(msg)

sunholo/senses/stream_voice.py

@@ -6,6 +6,8 @@ try:
     import sounddevice as sd
 except ImportError:
     sd = None
+except OSError:
+    sd = None
 
 try:
     import numpy as np

{sunholo-0.109.1.dist-info → sunholo-0.109.4.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.109.1
+Version: 0.109.4
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.109.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.109.4.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -21,8 +21,9 @@ License-File: LICENSE.txt
 Requires-Dist: google-auth
 Requires-Dist: ruamel.yaml
 Requires-Dist: langchain ==0.2.16
-Requires-Dist: langchain-experimental >=0.0.61
-Requires-Dist: langchain-community >=0.2.11
+Requires-Dist: langchain-experimental ==0.0.65
+Requires-Dist: langchain-community ==0.2.17
+Requires-Dist: langsmith ==0.1.140
 Provides-Extra: all
 Requires-Dist: anthropic[vertex] ; extra == 'all'
 Requires-Dist: asyncpg ; extra == 'all'
@@ -50,7 +51,7 @@ Requires-Dist: httpcore ; extra == 'all'
 Requires-Dist: httpx ; extra == 'all'
 Requires-Dist: jsonschema ; extra == 'all'
 Requires-Dist: lancedb ; extra == 'all'
-Requires-Dist: langchain >=0.2.12 ; extra == 'all'
+Requires-Dist: langchain >=0.2.16 ; extra == 'all'
 Requires-Dist: langchain-experimental >=0.0.61 ; extra == 'all'
 Requires-Dist: langchain-community >=0.2.11 ; extra == 'all'
 Requires-Dist: langchain-openai ==0.1.25 ; extra == 'all'

{sunholo-0.109.1.dist-info → sunholo-0.109.4.dist-info}/RECORD

@@ -86,7 +86,7 @@ sunholo/gcs/download_url.py,sha256=Ul81n1rklr8WogPsuxWWD1Nr8RHU451LzHPMJNhAKzw,6
 sunholo/gcs/extract_and_sign.py,sha256=paRrTCvCN5vkQwCB7OSkxWi-pfOgOtZ0bwdXE08c3Ps,1546
 sunholo/gcs/metadata.py,sha256=oQLcXi4brsZ74aegWyC1JZmhlaEV270HS5_UWtAYYWE,898
 sunholo/genai/__init__.py,sha256=6SWK7uV5F625J-P3xQoD6WKL59a9RSaidj-Guslyt8Q,192
-sunholo/genai/file_handling.py,sha256=b3vT_MIsinJSMqEa1MoelmABPTZH5iXNdCLvA6z7Qg8,5995
+sunholo/genai/file_handling.py,sha256=cGQeDvB93-3XgHRA020E7344yG7_EgX0KAsJLitQJBg,7552
 sunholo/genai/images.py,sha256=EyjsDqt6XQw99pZUQamomCpMOoIah9bp3XY94WPU7Ms,1678
 sunholo/genai/init.py,sha256=yG8E67TduFCTQPELo83OJuWfjwTnGZsyACospahyEaY,687
 sunholo/genai/process_funcs_cls.py,sha256=7_RQMqIAZ3nPP-GFgCHBvS39fwuWuGtvSyuJaJN_G3E,31590
@@ -117,7 +117,7 @@ sunholo/qna/__init__.py,sha256=F8q1uR_HreoSX0IfmKY1qoSwIgXhO2Q8kuDSxh9_-EE,28
 sunholo/qna/parsers.py,sha256=YpOaK5S_LxJ6FbliSYDc3AVOJ62RVduayoNnzi_p8CM,2494
 sunholo/qna/retry.py,sha256=yMw7RTkw-RXCzfENPJOt8c32mXlpvOR589EGkvK-6yI,2028
 sunholo/senses/__init__.py,sha256=fbWqVwwzkV5uRSb8lQzo4pn0ja_VYVWbUYapurSowBs,39
-sunholo/senses/stream_voice.py,sha256=tQL5TDH43OuXRlxWRLiIAceYApXyqNczN1D3jUiNMUo,18092
+sunholo/senses/stream_voice.py,sha256=VpCmooEKghBT1jPJe9mX7gKggGqY6qt-bpO7hwY4sPE,18122
 sunholo/streaming/__init__.py,sha256=MpbydI2UYo_adttPQFkxNM33b-QRyNEbrKJx0C2AGPc,241
 sunholo/streaming/content_buffer.py,sha256=0LHMwH4ctq5kjhIgMFNH0bA1RL0jMISlLVzzLcFrvv4,12766
 sunholo/streaming/langserve.py,sha256=hi7q8WY8DPKrALl9m_dOMxWOdE-iEuk7YW05SVDFIX8,6514
@@ -150,9 +150,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.109.1.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.109.1.dist-info/METADATA,sha256=U-SgUIRCDApHuOqKQPELVpyiTMqQo9TQ2E2HTNA0WuQ,8670
-sunholo-0.109.1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-sunholo-0.109.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.109.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.109.1.dist-info/RECORD,,
+sunholo-0.109.4.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.109.4.dist-info/METADATA,sha256=_nE3mKEViVRioU2NglcZCIXJ0PMqdsM7lYi__xkj8ao,8705
+sunholo-0.109.4.dist-info/WHEEL,sha256=a7TGlA-5DaHMRrarXjVbQagU3Man_dCnGIWMJr5kRWo,91
+sunholo-0.109.4.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.109.4.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.109.4.dist-info/RECORD,,

{sunholo-0.109.1.dist-info → sunholo-0.109.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (75.4.0)
 Root-Is-Purelib: true
 Tag: py3-none-any