sunholo 0.138.1__py3-none-any.whl → 0.139.0__py3-none-any.whl

sunholo/gcs/download_gcs_text.py

@@ -0,0 +1,132 @@
+import os
+import json
+
+from ..custom_logging import log
+from ..utils.mime import get_mime_type_gemini
+from .download_url import get_bytes_from_gcs
+
+def download_gcs_source_to_string(source:str) -> str:
+    """
+    source: str The Google Cloud Storage URI of the file to download (e.g., 'gs://bucket_name/file_name').
+    """
+    mime_type = get_mime_type_gemini(source)
+    if mime_type == "":
+        log.warning(f"Can not download to string file source {source}")
+        return ""
+    """
+        mime_types = {
+
+        # Images
+        'png': 'image/png',
+        'jpg': 'image/jpeg',
+        'jpeg': 'image/jpeg',
+        'gif': 'image/gif',
+        'webp': 'image/webp',
+
+        # Document formats
+        'pdf': 'application/pdf',
+        
+        # Programming languages
+        'js': 'text/javascript',
+        'py': 'text/x-python',
+        
+        # Web formats
+        'html': 'text/html',
+        'htm': 'text/html',
+        'css': 'text/css',
+        
+        # Text formats
+        'txt': 'text/plain',
+        'md': 'text/md',
+        'csv': 'text/csv',
+        'xml': 'text/xml',
+        'rtf': 'text/rtf',
+        
+        # Special case: JSON files are treated as plain text
+        'json': 'text/plain'
+    }
+    """
+    if mime_type.startswith("image/") or mime_type == "application/pdf":
+        log.warning(f"Can not download to string file source {source} of type {mime_type}")
+        return ""
+
+    try:
+        log.info(f"Extracting text for {source}")
+        bytes = get_bytes_from_gcs(source)
+        string = bytes.decode('utf-8', errors='replace')
+        log.info(f"Extracted {len(string)} characters from {source}: {string[:100]}")
+
+    except Exception as err:
+        log.error(f"Could not extract string text for {source}: {str(err)}")
+
+        return ""
+
+    if not string:
+        raise ValueError(f"No string text for {source}")
+
+    file_ext = os.path.splitext(source)[1].lower().lstrip('.')
+    if file_ext == "json":
+        try:
+            extracted_data = json.loads(string)
+            log.debug("Turning json text into markdown format so as not to confuse structured output", log_struct=extracted_data)
+            string = json_data_to_markdown(extracted_data)
+        except json.JSONDecodeError:
+            log.warning(f"Could not get valid json from .json file: {source}")
+    
+    return string
+
+def json_data_to_markdown(data, indent_level: int = 0) -> str:
+    """
+    Recursively converts a Python object (from parsed JSON) into a Markdown string.
+    """
+    indent = "  " * indent_level  # Use 2 spaces for indentation
+    markdown_parts = []
+
+    if isinstance(data, dict):
+        if not data:
+            return f"{indent}(empty object)"
+        for key, value in data.items():
+            # Process the value recursively
+            value_md = json_data_to_markdown(value, indent_level + 1)
+            # Determine if the rendered value is complex (multi-line or was list/dict)
+            is_complex_render = "\n" in value_md.strip() or (isinstance(value, (dict, list)) and value)
+
+            if is_complex_render:
+                markdown_parts.append(f"{indent}**{key}**:")
+                markdown_parts.append(value_md)
+            else:
+                # Simple value rendering, strip its own indent before adding key
+                markdown_parts.append(f"{indent}**{key}**: {value_md.strip()}")
+        return "\n".join(markdown_parts)
+
+    elif isinstance(data, list):
+        if not data:
+            return f"{indent}(empty list)"
+        for item in data:
+            # Process item recursively
+            item_md = json_data_to_markdown(item, indent_level + 1)
+            # Remove leading indent from the recursive call before processing lines
+            lines = item_md.lstrip(' ').split('\n')
+            # Add bullet point to the first line
+            first_line = f"{indent}- {lines[0]}"
+            # Ensure subsequent lines are indented correctly relative to the bullet
+            rest_lines = [f"{indent}  {line}" for line in lines[1:]]
+            markdown_parts.append(first_line)
+            markdown_parts.extend(rest_lines)
+        return "\n".join(markdown_parts)
+
+    elif isinstance(data, str):
+        # Handle multi-line strings: indent subsequent lines
+        lines = data.split('\n')
+        if len(lines) <= 1:
+            return f"{indent}{data}" # Single line string
+        else:
+            indented_lines = [f"{indent}{lines[0]}"] + [f"{indent}  {line}" for line in lines[1:]]
+            return "\n".join(indented_lines)
+
+    elif data is None:
+        return f"{indent}*null*" # Represent None distinctly
+    elif isinstance(data, bool):
+        return f"{indent}{str(data).lower()}" # true / false
+    else:  # Numbers (int, float)
+        return f"{indent}{str(data)}"

sunholo/utils/mime.py

@@ -66,3 +66,53 @@ def guess_mime_type(file_path: str) -> str:
 
     return mime
 
+
+def get_mime_type_gemini(file_path:str) -> str:
+    """
+    Determine the MIME type based on file extension.
+    Only returns valid Gemini formats, or None if they are not supported.
+    
+    Args:
+        file_path (str): Path to the file
+        
+    Returns:
+        str: The appropriate MIME type for the file
+    """
+    # Extract the file extension (lowercase)
+    ext = os.path.splitext(file_path)[1].lower().lstrip('.')
+    
+    # Define the mapping of extensions to MIME types
+    mime_types = {
+
+        # Images
+        'png': 'image/png',
+        'jpg': 'image/jpeg',
+        'jpeg': 'image/jpeg',
+        'gif': 'image/gif',
+        'webp': 'image/webp',
+
+        # Document formats
+        'pdf': 'application/pdf',
+        
+        # Programming languages
+        'js': 'text/javascript',
+        'py': 'text/x-python',
+        
+        # Web formats
+        'html': 'text/html',
+        'htm': 'text/html',
+        'css': 'text/css',
+        
+        # Text formats
+        'txt': 'text/plain',
+        'md': 'text/md',
+        'csv': 'text/csv',
+        'xml': 'text/xml',
+        'rtf': 'text/rtf',
+        
+        # Special case: JSON files are treated as plain text
+        'json': 'text/plain'
+    }
+    
+    # Return the appropriate MIME type, defaulting to None if unknown
+    return mime_types.get(ext, "")

{sunholo-0.138.1.dist-info → sunholo-0.139.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.138.1
+Version: 0.139.0
 Summary: AI DevOps - a package to help deploy GenAI to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.138.1.dist-info → sunholo-0.139.0.dist-info}/RECORD

@@ -85,6 +85,7 @@ sunholo/excel/plugin.py,sha256=TJJdcKWyqEIce1agCJImvqvNp2CvLhzi4wUmLYHcLc8,4032
 sunholo/gcs/__init__.py,sha256=SZvbsMFDko40sIRHTHppA37IijvJTae54vrhooEF5-4,90
 sunholo/gcs/add_file.py,sha256=Pd5Zc1a3gqbuBgSI-UDC2mQnYGLJbAh_-IUzkDN5s9k,8273
 sunholo/gcs/download_folder.py,sha256=ijJTnS595JqZhBH8iHFErQilMbkuKgL-bnTCMLGuvlA,1614
+sunholo/gcs/download_gcs_text.py,sha256=odgmuYimf1Jiy67Q06jNlb3FAKQmIq3KcakxsCnoMtY,4784
 sunholo/gcs/download_url.py,sha256=9QMEtZhrN-y1VAqvi-7Tw2GI9iRG_uuZzCg6Qhq8_yw,6421
 sunholo/gcs/extract_and_sign.py,sha256=paRrTCvCN5vkQwCB7OSkxWi-pfOgOtZ0bwdXE08c3Ps,1546
 sunholo/gcs/metadata.py,sha256=oQLcXi4brsZ74aegWyC1JZmhlaEV270HS5_UWtAYYWE,898
@@ -155,7 +156,7 @@ sunholo/utils/config_class.py,sha256=uSRiJLj8t5UgWNxaq8W4KPnzxb4SkUJ1avXecDHuP-E
 sunholo/utils/config_schema.py,sha256=Wv-ncitzljOhgbDaq9qnFqH5LCuxNv59dTGDWgd1qdk,4189
 sunholo/utils/gcp.py,sha256=lus1HH8YhFInw6QRKwfvKZq-Lz-2KQg4ips9v1I_3zE,4783
 sunholo/utils/gcp_project.py,sha256=Fa0IhCX12bZ1ctF_PKN8PNYd7hihEUfb90kilBfUDjg,1411
-sunholo/utils/mime.py,sha256=7_J1PnWOlvAPRoHWKESAncdRVVldVwRdKvuDvi9sRfE,2020
+sunholo/utils/mime.py,sha256=mELAiZcGa69PshBxV7y770E0K09YfX4Z4ZRBPL-7gXs,3352
 sunholo/utils/parsers.py,sha256=wES0fRn3GONoymRXOXt-z62HCoOiUvvFXa-MfKfjCls,6421
 sunholo/utils/timedelta.py,sha256=BbLabEx7_rbErj_YbNM0MBcaFN76DC4PTe4zD2ucezg,493
 sunholo/utils/user_ids.py,sha256=SQd5_H7FE7vcTZp9AQuQDWBXd4FEEd7TeVMQe1H4Ny8,292
@@ -168,9 +169,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.138.1.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.138.1.dist-info/METADATA,sha256=GC4bwGLlBT68d6uqc99tTFX7Y_WtWnombuk2fPrTzls,10067
-sunholo-0.138.1.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-sunholo-0.138.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.138.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.138.1.dist-info/RECORD,,
+sunholo-0.139.0.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.139.0.dist-info/METADATA,sha256=ZMNryoyWlaoHvg9c_VBaKznsSlYhyYheSNCh0c3fZm4,10067
+sunholo-0.139.0.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
+sunholo-0.139.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.139.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.139.0.dist-info/RECORD,,

{sunholo-0.138.1.dist-info → sunholo-0.139.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.3.1)
+Generator: setuptools (80.4.0)
 Root-Is-Purelib: true
 Tag: py3-none-any