sunholo 0.139.0__py3-none-any.whl → 0.139.1__py3-none-any.whl

sunholo/gcs/download_gcs_text.py

@@ -3,12 +3,21 @@ import json
 
 from ..custom_logging import log
 from ..utils.mime import get_mime_type_gemini
+from .metadata import check_gcs_file_size
 from .download_url import get_bytes_from_gcs
 
-def download_gcs_source_to_string(source:str) -> str:
+def download_gcs_source_to_string(source:str, max_size_bytes: int = 1024*1024) -> str:
     """
-    source: str The Google Cloud Storage URI of the file to download (e.g., 'gs://bucket_name/file_name').
+    Download a file from Google Cloud Storage and convert it to a string.
+    
+    Args:
+        source: str The Google Cloud Storage URI of the file to download (e.g., 'gs://bucket_name/file_name').
+        max_size_bytes: int Maximum file size to download, defaults to 1MB (1024*1024 bytes)
+        
+    Returns:
+        str: The contents of the file as a string, or an empty string if the file could not be downloaded.
     """
+
     mime_type = get_mime_type_gemini(source)
     if mime_type == "":
         log.warning(f"Can not download to string file source {source}")
@@ -52,6 +61,15 @@ def download_gcs_source_to_string(source:str) -> str:
 
     try:
         log.info(f"Extracting text for {source}")
+        # Check file size before downloading
+        file_size = check_gcs_file_size(source)
+        if file_size == -1:
+            log.warning(f"Could not determine file size for {source}")
+            return ""
+        elif file_size > max_size_bytes:
+            log.warning(f"File size {file_size} bytes exceeds maximum size limit of {max_size_bytes} bytes for {source}")
+            return ""
+        
         bytes = get_bytes_from_gcs(source)
         string = bytes.decode('utf-8', errors='replace')
         log.info(f"Extracted {len(string)} characters from {source}: {string[:100]}")

sunholo/gcs/metadata.py

@@ -30,4 +30,37 @@ def get_object_metadata(bucket_name, object_name):
     custom_metadata = blob.metadata
 
     log.info(f"Custom Metadata for {object_name}: {custom_metadata}")
-    return custom_metadata
+    return custom_metadata
+
+def check_gcs_file_size(source: str) -> int:
+    """
+    Check the size of a file in Google Cloud Storage without downloading the entire file.
+    
+    Args:
+        source: str The Google Cloud Storage URI of the file to check (e.g., 'gs://bucket_name/file_name').
+        
+    Returns:
+        int: The size of the file in bytes, or -1 if the size cannot be determined.
+    """
+    from google.cloud import storage
+    
+    try:
+        # Parse the GCS URI
+        if not source.startswith('gs://'):
+            log.warning(f"Invalid GCS URI format: {source}")
+            return -1
+            
+        bucket_name, blob_path = source[5:].split('/', 1)
+        
+        # Create a client and get the bucket
+        storage_client = storage.Client()
+        bucket = storage_client.bucket(bucket_name)
+        
+        # Get the blob (file) and retrieve its metadata
+        blob = bucket.blob(blob_path)
+        blob.reload()  # Fetch the latest metadata
+        
+        return blob.size
+    except Exception as err:
+        log.error(f"Error checking file size for {source}: {str(err)}")
+        return -1

{sunholo-0.139.0.dist-info → sunholo-0.139.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.139.0
+Version: 0.139.1
 Summary: AI DevOps - a package to help deploy GenAI to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.139.0.dist-info → sunholo-0.139.1.dist-info}/RECORD

@@ -85,10 +85,10 @@ sunholo/excel/plugin.py,sha256=TJJdcKWyqEIce1agCJImvqvNp2CvLhzi4wUmLYHcLc8,4032
 sunholo/gcs/__init__.py,sha256=SZvbsMFDko40sIRHTHppA37IijvJTae54vrhooEF5-4,90
 sunholo/gcs/add_file.py,sha256=Pd5Zc1a3gqbuBgSI-UDC2mQnYGLJbAh_-IUzkDN5s9k,8273
 sunholo/gcs/download_folder.py,sha256=ijJTnS595JqZhBH8iHFErQilMbkuKgL-bnTCMLGuvlA,1614
-sunholo/gcs/download_gcs_text.py,sha256=odgmuYimf1Jiy67Q06jNlb3FAKQmIq3KcakxsCnoMtY,4784
+sunholo/gcs/download_gcs_text.py,sha256=apopEBxC6OT0KBwnlFOeNOXyiY4A8BKEVixzb1wgQrk,5583
 sunholo/gcs/download_url.py,sha256=9QMEtZhrN-y1VAqvi-7Tw2GI9iRG_uuZzCg6Qhq8_yw,6421
 sunholo/gcs/extract_and_sign.py,sha256=paRrTCvCN5vkQwCB7OSkxWi-pfOgOtZ0bwdXE08c3Ps,1546
-sunholo/gcs/metadata.py,sha256=oQLcXi4brsZ74aegWyC1JZmhlaEV270HS5_UWtAYYWE,898
+sunholo/gcs/metadata.py,sha256=GEDxb_B_teNPGd6chPzQrK9df78R_kytKfthIlKPwKQ,2010
 sunholo/genai/__init__.py,sha256=TV3PYHWoR4cChdmCOaYB0PtAEQ86qol9XYYEtb1JmSA,239
 sunholo/genai/file_handling.py,sha256=JUFTlSnrxqKR3hczduyMiZ234UaSqiBdMOYpY2v4TYA,13720
 sunholo/genai/genaiv2.py,sha256=uqWcfvlsPVPyQo-W_cP9h2TTzyYfzj4lyJlyqPyKTkI,20269
@@ -169,9 +169,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.139.0.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.139.0.dist-info/METADATA,sha256=ZMNryoyWlaoHvg9c_VBaKznsSlYhyYheSNCh0c3fZm4,10067
-sunholo-0.139.0.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-sunholo-0.139.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.139.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.139.0.dist-info/RECORD,,
+sunholo-0.139.1.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.139.1.dist-info/METADATA,sha256=vOG7X6ZpBgF3og9_BNDil-Loy2tAW38orcqYo3ObTTk,10067
+sunholo-0.139.1.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+sunholo-0.139.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.139.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.139.1.dist-info/RECORD,,