sunholo 0.75.1__py3-none-any.whl → 0.76.3__py3-none-any.whl

sunholo/auth/refresh.py

@@ -0,0 +1,55 @@
+# needs to be in minimal to check gcp
+import os
+
+import google.auth 
+from google.auth.transport import requests
+from ..utils.gcp import is_running_on_gcp
+
+
+from ..logging import log
+
+def get_default_email():
+    if not refresh_credentials():
+        log.error("Could not refresh the credentials properly.")
+        return None
+    # https://stackoverflow.com/questions/64234214/how-to-generate-a-blob-signed-url-in-google-cloud-run
+
+    gcs_credentials, project_id = get_default_creds()
+
+    service_account_email = getattr(gcs_credentials, 'service_account_email', None)
+    # If you use a service account credential, you can use the embedded email
+    if not service_account_email:
+        service_account_email = os.getenv('GCS_MAIL_USER')
+        if not service_account_email:
+            log.error("Could not create the credentials for signed requests - no credentials.service_account_email or GCS_MAIL_USER with roles/iam.serviceAccountTokenCreator")
+            
+            return None
+    
+    log.info(f"Found default email: {service_account_email=} for {project_id=}")
+    return service_account_email
+
+def get_default_creds():
+    gcs_credentials = None
+    project_id = None
+    gcs_credentials, project_id = google.auth.default()
+
+    return gcs_credentials, project_id
+
+def refresh_credentials():
+    if not is_running_on_gcp():
+        log.debug("Not running on Google Cloud so no credentials available for GCS.")
+        return False
+    
+    gcs_credentials, project_id = get_default_creds()
+
+    if not gcs_credentials.token or gcs_credentials.expired or not gcs_credentials.valid:
+        try:
+            gcs_credentials.refresh(requests.Request())
+
+            return True
+        
+        except Exception as e:
+            log.error(f"Failed to refresh gcs credentials: {e}")
+
+            return False
+    

sunholo/azure/__init__.py

@@ -0,0 +1 @@
+from .event_grid import process_azure_blob_event

sunholo/azure/event_grid.py

@@ -0,0 +1,69 @@
+# process_azure_blob_event.py
+from ..logging import log
+
+def process_azure_blob_event(events: list) -> tuple:
+    """
+    Extracts message data and metadata from an Azure Blob Storage event.
+
+    Args:
+        events (list): The list of Azure Event Grid event data.
+
+    Returns:
+        tuple: A tuple containing the blob URL, attributes as metadata, and the vector name.
+        
+    Example of Event Grid schema:
+    {
+        "topic": "/subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Storage/storageAccounts/{storage-account}",
+        "subject": "/blobServices/default/containers/{container}/blobs/{blob}",
+        "eventType": "Microsoft.Storage.BlobCreated",
+        "eventTime": "2021-01-01T12:34:56.789Z",
+        "id": "event-id",
+        "data": {
+            "api": "PutBlob",
+            "clientRequestId": "client-request-id",
+            "requestId": "request-id",
+            "eTag": "etag",
+            "contentType": "application/octet-stream",
+            "contentLength": 524288,
+            "blobType": "BlockBlob",
+            "url": "https://{storage-account}.blob.core.windows.net/{container}/{blob}",
+            "sequencer": "0000000000000000000000000000000000000000000000000000000000000000",
+            "storageDiagnostics": {
+                "batchId": "batch-id"
+            }
+        },
+        "dataVersion": "",
+        "metadataVersion": "1"
+    }
+    """
+    storage_blob_created_event = "Microsoft.Storage.BlobCreated"
+    
+    for event in events:
+        event_type = event['eventType']
+        data = event['data']
+
+        if event_type == storage_blob_created_event:
+            blob_url = data['url']
+            event_time = event['eventTime']
+            event_id = event['id']
+            subject = event['subject']
+            attributes = {
+                'event_type': event_type,
+                'event_time': event_time,
+                'event_id': event_id,
+                'subject': subject,
+                'url': blob_url
+            }
+
+            vector_name = subject.split('/')[4]  # Extracting the container name
+            
+            log.info(f"Process Azure Blob Event was triggered by eventId {event_id} at {event_time}")
+            log.debug(f"Process Azure Blob Event data: {blob_url}")
+            
+            # Check for a valid Azure Blob Storage event type
+            if event_type == "Microsoft.Storage.BlobCreated":
+                log.info(f"Got valid event from Azure Blob Storage: {blob_url}")
+            
+            return blob_url, attributes, vector_name
+
+    return None, None, None

sunholo/chunker/__init__.py

@@ -1 +1,3 @@
-from .data_to_embed_pubsub import data_to_embed_pubsub, direct_file_to_embed
+from .pubsub import data_to_embed_pubsub
+from .azure import data_to_embed_azure
+from .process_chunker_data import direct_file_to_embed

sunholo/chunker/azure.py

@@ -0,0 +1,41 @@
+#   Copyright [2024] [Holosun ApS]
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from ..logging import log
+from ..azure import process_azure_blob_event
+from .process_chunker_data import process_chunker_data
+
+def data_to_embed_azure(events: list):
+    """Triggered from a message on an Azure Data Grid event.
+    Args:
+         data JSON
+    """
+    validation_event_type = "Microsoft.EventGrid.SubscriptionValidationEvent"
+    storage_blob_created_event = "Microsoft.Storage.BlobCreated"
+    
+    for event in events:
+        event_type = event['eventType']
+        data = event['data']
+
+        if event_type == validation_event_type:
+            validation_code = data['validationCode']
+            log.info(f"Got SubscriptionValidation event data, validation code: {validation_code}, topic: {event['topic']}")
+            
+            # Return the validation response
+            return {"ValidationResponse": validation_code}
+        elif event_type == storage_blob_created_event:
+
+            message_data, metadata, vector_name = process_azure_blob_event(events)
+
+            return process_chunker_data(message_data, metadata, vector_name)

sunholo/chunker/message_data.py

@@ -173,10 +173,15 @@ def handle_http_message(message_data: str, metadata: dict, vector_name:str):
 
     return chunks, metadata
 
-def handle_json_content_message(message_data: str, metadata: dict, vector_name: str):
+def handle_json_content_message(message_data: dict, metadata: dict, vector_name: str):
     log.info("No tailored message_data detected, processing message json")
     # Process message containing direct JSON content
-    the_json = json.loads(message_data)
+    try:
+        the_json = json.loads(message_data)
+    except Exception as e:
+        log.error(f"Could not load message {message_data} as JSON - {str(e)}")
+        return None, {"metadata": f"Could not load message as JSON - {str(e)}"}
+    
     the_metadata = the_json.get("metadata", {})
     metadata.update(the_metadata)
     the_content = the_json.get("page_content", None)
@@ -186,7 +191,7 @@ def handle_json_content_message(message_data: str, metadata: dict, vector_name:
 
     if the_content is None:
         log.info("No content found")
-        return {"metadata": "No content found in 'page_content' JSON field"}
+        return None, {"metadata": "No content found in 'page_content' JSON field"}
     
     docs = [Document(page_content=the_content, metadata=metadata)]
 

sunholo/chunker/{data_to_embed_pubsub.py → process_chunker_data.py}

@@ -1,58 +1,21 @@
-#   Copyright [2024] [Holosun ApS]
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
 import pathlib
 
-from ..logging import log
-from ..pubsub import process_pubsub_message
-from .message_data import handle_gcs_message, handle_google_drive_message, handle_github_message, handle_http_message, handle_json_content_message
-from .publish import process_docs_chunks_vector_name
-from .splitter import chunk_doc_to_docs
+from .message_data import (
+    handle_gcs_message, 
+    handle_google_drive_message, 
+    handle_github_message, 
+    handle_http_message, 
+    handle_json_content_message
+)
 
+from . import loaders
 from ..llamaindex.import_files import llamaindex_chunker_check
 from ..discovery_engine.chunker_handler import discovery_engine_chunker_check
+from .publish import process_docs_chunks_vector_name
+from .splitter import chunk_doc_to_docs
 
-from . import loaders
-
-def direct_file_to_embed(file_name: pathlib.Path, metadata: dict, vector_name: str):
-    """
-    Send direct files to chunking embed pipeline
-
-    
-    
-    """
-    log.info(f"Sending direct file upload {file_name} to loaders.read_file_to_documents {metadata}")
-    docs = loaders.read_file_to_documents(file_name, metadata=metadata)
-    if docs is None:
-        log.warning(f"loaders.read_file_to_documents docs2 failed to load file {metadata}")
-
-        return None
-
-    chunks = chunk_doc_to_docs(docs, file_name.suffix, vector_name=vector_name)
-    
-    return format_chunk_return(chunks, metadata, vector_name)
-
-
-
-def data_to_embed_pubsub(data: dict):
-    """Triggered from a message on a Cloud Pub/Sub topic.
-    Args:
-         data JSON
-    """
-
-    message_data, metadata, vector_name = process_pubsub_message(data)
 
-    return process_chunker_data(message_data, metadata, vector_name)
+from ..logging import log
 
 def process_chunker_data(message_data, metadata, vector_name):
 
@@ -95,7 +58,6 @@ def process_chunker_data(message_data, metadata, vector_name):
     
     return format_chunk_return(chunks, metadata, vector_name)
 
-
 def format_chunk_return(chunks, metadata, vector_name):
     # to be really sure
     if metadata:
@@ -110,8 +72,23 @@ def format_chunk_return(chunks, metadata, vector_name):
                     
             return output_list
 
+    # returns None when not on GCP
     process_docs_chunks_vector_name(chunks, vector_name, metadata)
 
     return metadata
 
 
+def direct_file_to_embed(file_name: pathlib.Path, metadata: dict, vector_name: str):
+    """
+    Send direct files to chunking embed pipeline
+    """
+    log.info(f"Sending direct file upload {file_name} to loaders.read_file_to_documents {metadata}")
+    docs = loaders.read_file_to_documents(file_name, metadata=metadata)
+    if docs is None:
+        log.warning(f"loaders.read_file_to_documents docs2 failed to load file {metadata}")
+
+        return None
+
+    chunks = chunk_doc_to_docs(docs, file_name.suffix, vector_name=vector_name)
+    
+    return format_chunk_return(chunks, metadata, vector_name)

sunholo/chunker/pubsub.py

@@ -0,0 +1,31 @@
+#   Copyright [2024] [Holosun ApS]
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from ..logging import log
+from ..pubsub import process_pubsub_message
+from .process_chunker_data import process_chunker_data
+
+def data_to_embed_pubsub(data: dict):
+    """Triggered from a message on a Cloud Pub/Sub topic.
+    Args:
+         data JSON
+    """
+
+    message_data, metadata, vector_name = process_pubsub_message(data)
+
+    return process_chunker_data(message_data, metadata, vector_name)
+
+
+
+

sunholo/cli/cli.py

@@ -9,6 +9,7 @@ from .run_proxy import setup_proxy_subparser
 from .chat_vac import setup_vac_subparser
 from .embedder import setup_embedder_subparser
 from .swagger import setup_swagger_subparser
+from .vertex import setup_vertex_subparser
 
 from ..utils import ConfigManager
 
@@ -84,6 +85,8 @@ def main(args=None):
     setup_embedder_subparser(subparsers)
     # swagger generation
     setup_swagger_subparser(subparsers)
+    # vertex
+    setup_vertex_subparser(subparsers)
 
     #TODO: add database setup commands: alloydb and supabase
 

sunholo/cli/vertex.py

@@ -0,0 +1,46 @@
+from ..vertex import VertexAIExtensions  
+
+from .sun_rich import console
+
+def deploy_extension(args):
+    vex = VertexAIExtensions(args.project)
+    console.rule(f"Creating Vertex extension '{args.display_name}' within '{args.project}'")
+
+    vex.create_extension(
+        args.display_name,
+        description=args.description,
+        tool_example_file=args.tool_example_file,
+        open_api_file=args.open_api_file,
+        service_account=args.service_account,
+        project_id=args.project,
+        bucket_name=args.bucket_name
+    )
+    extensions = vex.list_extensions(args.project)
+    console.print(extensions)
+
+def list_extensions(args):
+    vex = VertexAIExtensions()
+    extensions = vex.list_extensions(args.project)
+    console.print(extensions)
+
+def setup_vertex_subparser(subparsers):
+    """
+    Sets up an argparse subparser for the 'vertex' command.
+
+    Args:
+        subparsers: The subparsers object to add the 'vertex' subcommand to.
+    """
+    vertex_parser = subparsers.add_parser('vertex', help='Work with Google Vertex AI')
+    vertex_subparsers = vertex_parser.add_subparsers(dest='subcommand', help='Vertex AI subcommands')
+
+    create_parser = vertex_subparsers.add_parser('create-extension', help='Create a Vertex AI extension')
+    create_parser.add_argument('--display_name', required=True, help='Display name of the extension')
+    create_parser.add_argument('--description', required=True, help='Description of the extension')
+    create_parser.add_argument('--tool_example_file', required=True, help='Tool example file path')
+    create_parser.add_argument('--open_api_file', required=True, help='OpenAPI file path')
+    create_parser.add_argument('--service_account', required=True, help='Service account email')
+    create_parser.add_argument('--bucket_name', help='Bucket name to upload files to.  Uses EXTENSION_BUCKET env var if not specified')
+    create_parser.set_defaults(func=deploy_extension)
+
+    list_parser = vertex_subparsers.add_parser('list-extensions', help='List all Vertex AI extensions')
+    list_parser.set_defaults(func=list_extensions)

sunholo/discovery_engine/chunker_handler.py

@@ -102,6 +102,9 @@ def check_discovery_engine_in_memory(vector_name):
 def discovery_engine_chunker_check(message_data, metadata, vector_name):
     # discovery engine handles its own chunking/embedding
     memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")
+    if not memories:
+        return None
+    
     total_memories = len(memories)
     llama = None
     if check_discovery_engine_in_memory(vector_name):

sunholo/gcs/download_url.py

@@ -3,8 +3,6 @@ from urllib.parse import quote
 from datetime import datetime, timedelta
 
 # needs to be in minimal to check gcp
-import google.auth 
-from google.auth.transport import requests
 from google.auth.exceptions import RefreshError
 
 try:
@@ -14,6 +12,7 @@ except ImportError:
 
 from ..logging import log
 from ..utils.gcp import is_running_on_gcp
+from ..auth.refresh import refresh_credentials, get_default_creds, get_default_email
 from io import BytesIO
 try:
     from PIL import Image
@@ -77,47 +76,22 @@ def get_bytes_from_gcs(gs_uri):
 
 if is_running_on_gcp():
     # Perform a refresh request to get the access token of the current credentials (Else, it's None)
-    gcs_credentials, project_id = google.auth.default()
+    gcs_credentials, project_id = get_default_creds()
     # Prepare global variables for client reuse
     if storage:
         gcs_client = storage.Client()
 
-def refresh_credentials():
-    if not is_running_on_gcp():
-        log.debug("Not running on Google Cloud so no credentials available for GCS.")
-        return False
-    if not gcs_credentials.token or gcs_credentials.expired or not gcs_credentials.valid:
-        try:
-            gcs_credentials.refresh(requests.Request())
-        except Exception as e:
-            log.error(f"Failed to refresh gcs credentials: {e}")
-            return False
-    return True
-
-refresh_credentials()
-
 def get_bucket(bucket_name):
     if bucket_name not in gcs_bucket_cache:
         gcs_bucket_cache[bucket_name] = gcs_client.get_bucket(bucket_name)
     return gcs_bucket_cache[bucket_name]
 
 def sign_gcs_url(bucket_name:str, object_name:str, expiry_secs = 86400):
-    if not refresh_credentials():
-        log.error("Could not refresh the credentials properly.")
-        return None
-    # https://stackoverflow.com/questions/64234214/how-to-generate-a-blob-signed-url-in-google-cloud-run
+    
+    service_account_email = get_default_email()
 
     expires = datetime.now() + timedelta(seconds=expiry_secs)
 
-    service_account_email = getattr(gcs_credentials, 'service_account_email', None)
-    # If you use a service account credential, you can use the embedded email
-    if not service_account_email:
-        service_account_email = os.getenv('GCS_MAIL_USER')
-        if service_account_email is None:
-            log.error("For local testing must set a GCS_MAIL_USER to sign GCS URLs")
-        log.error("Could not create the credentials for signed requests - no credentials.service_account_email or GCS_MAIL_USER with roles/iam.serviceAccountTokenCreator")
-        return None
-
     try:
         bucket = get_bucket(bucket_name)
         blob = bucket.blob(object_name)

sunholo/llamaindex/import_files.py

@@ -131,7 +131,10 @@ def check_llamaindex_in_memory(vector_name):
 
 def llamaindex_chunker_check(message_data, metadata, vector_name):
     # llamaindex handles its own chunking/embedding
-    memories = ConfigManager(vector_name).vacConfig("memory")
+    memories = load_memories(vector_name)
+    if not memories:
+        return None
+    
     total_memories = len(memories)
     llama = None
     if check_llamaindex_in_memory(vector_name):

sunholo/vertex/extensions_class.py

@@ -50,7 +50,7 @@ class VertexAIExtensions:
                           operation_params = operation_params)
     ```
     """
-    def __init__(self):
+    def __init__(self, project_id=None):
         if extensions is None:
             raise ImportError("VertexAIExtensions needs vertexai.previewextensions to be installed. Install via `pip install sunholo[gcp]`")
         
@@ -69,9 +69,13 @@ class VertexAIExtensions:
         self.tool_use_examples = None
         self.manifest = {}
         self.created_extensions = []
+        self.bucket_name = os.getenv('EXTENSIONS_BUCKET')
+        init_vertex(location=self.location, project_id=project_id)
 
-    def list_extensions(self):
-        the_list = extensions.Extension.list()
+    def list_extensions(self, project_id:str=None):
+        project_id = project_id or get_gcp_project()
+        log.info(f"Creating extension within {project_id=}")
+        the_list = extensions.Extension.list(project=project_id)
         
         extensions_list = []
         for ext in the_list:
@@ -94,18 +98,21 @@ class VertexAIExtensions:
         validate(spec_dict)
     
     def upload_to_gcs(self, filename):
-        if not os.getenv('EXTENSIONS_BUCKET'):
-            raise ValueError('Please specify env var EXTENSIONS_BUCKET for location to upload openapi spec')
+        if not self.bucket_name:
+            raise ValueError('Please specify bucket_name or env var EXTENSIONS_BUCKET for location to upload openapi spec')
         
         from ..gcs.add_file import add_file_to_gcs
         file_base = os.path.basename(filename)
 
-        self_uri = add_file_to_gcs(file_base, bucket_filepath=file_base)
+        self_uri = add_file_to_gcs(file_base, bucket_filepath=file_base, bucket_name=self.bucket_name)
 
         return self_uri
     
     def upload_openapi_file(self, filename: str):
         self.validate_openapi(filename)
+        if not self.bucket_name:
+            raise ValueError('Please specify env var EXTENSIONS_BUCKET for location to upload openapi spec')
+        
 
         self.openapi_file_gcs = self.upload_to_gcs(filename)
     
@@ -193,15 +200,23 @@ class VertexAIExtensions:
                          open_api_file: str = None,
                          tool_example_file: str = None,
                          runtime_config: dict = None,
-                         service_account: str = None):
+                         service_account: str = None,
+                         project_id: str = None,
+                         bucket_name: str = None):
         
-        project_id = get_gcp_project()
+        project_id = project_id or get_gcp_project()
+        log.info(f"Creating extension within {project_id=}")
         extension_name = f"projects/{project_id}/locations/us-central1/extensions/{validate_extension_id(display_name)}"
 
-        listed_extensions = self.list_extensions()
+        if bucket_name:
+            log.info(f"Setting extension bucket name to {bucket_name}")
+            self.bucket_name = bucket_name
+
+        listed_extensions = self.list_extensions(project_id)
+        log.info(f"Listing extensions:\n {listed_extensions}")
         for ext in listed_extensions:
-            if ext.get('resource_name') == extension_name:
-                raise NameError(f"resouce_name {extension_name} already exists.  Delete it or rename your new extension")
+            if ext.get('display_name') == display_name:
+                raise NameError(f"display_name {display_name} already exists.  Delete it or rename your new extension")
 
         if open_api_file:
             self.upload_openapi_file(open_api_file)
@@ -233,8 +248,8 @@ class VertexAIExtensions:
 
         return extension.resource_name
 
-    def execute_extension(self, operation_id: str, operation_params: dict, extension_id: str=None):
-        init_vertex(location=self.location)
+    def execute_extension(self, operation_id: str, operation_params: dict, extension_id: str=None, project_id: str=None):
+        init_vertex(location=self.location, project_id=project_id)
 
         if not extension_id:
             extension_name = self.created_extension.resource_name
@@ -243,7 +258,7 @@ class VertexAIExtensions:
         else:  
             extension_id = str(extension_id)
             if not extension_id.startswith("projects/"):
-                project_id = get_gcp_project()
+                project_id = project_id or get_gcp_project()
                 extension_name = f"projects/{project_id}/locations/{self.location}/extensions/{extension_id}"
             else:
                 extension_name = extension_id

sunholo/vertex/init.py

@@ -1,5 +1,6 @@
 from ..logging import log
 from ..utils.gcp_project import get_gcp_project
+from ..auth.refresh import get_default_email
 import os
 
 def init_genai():
@@ -19,7 +20,7 @@ def init_genai():
 
     genai.configure(api_key=GOOGLE_API_KEY)
 
-def init_vertex(gcp_config=None, location="eu"):
+def init_vertex(gcp_config=None, location="eu", project_id=None):
     """
     Initializes the Vertex AI environment using the provided Google Cloud Platform configuration.
 
@@ -62,6 +63,10 @@ def init_vertex(gcp_config=None, location="eu"):
         project_id = gcp_config.get('project_id')
         location = gcp_config.get('location') or location
     else:
-        project_id = get_gcp_project()
+        project_id = project_id or get_gcp_project()
+
+    log.info(f"Auth with email: {get_default_email()} in {project_id}")
 
     vertexai.init(project=project_id, location=location)
+
+

{sunholo-0.75.1.dist-info → sunholo-0.76.3.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.75.1
+Version: 0.76.3
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.75.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.76.3.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -25,6 +25,7 @@ Requires-Dist: langchain-experimental >0.0.60
 Requires-Dist: langchain-community
 Provides-Extra: all
 Requires-Dist: asyncpg ; extra == 'all'
+Requires-Dist: azure-storage-blob ; extra == 'all'
 Requires-Dist: fastapi ; extra == 'all'
 Requires-Dist: flask ; extra == 'all'
 Requires-Dist: google-auth ; extra == 'all'
@@ -71,6 +72,8 @@ Requires-Dist: tiktoken ; extra == 'all'
 Requires-Dist: unstructured[local-inference] ==0.14.9 ; extra == 'all'
 Provides-Extra: anthropic
 Requires-Dist: langchain-anthropic >=0.1.13 ; extra == 'anthropic'
+Provides-Extra: azure
+Requires-Dist: azure-storage-blob ; extra == 'azure'
 Provides-Extra: cli
 Requires-Dist: jsonschema >=4.21.1 ; extra == 'cli'
 Requires-Dist: rich ; extra == 'cli'

{sunholo-0.75.1.dist-info → sunholo-0.76.3.dist-info}/RECORD

@@ -19,23 +19,28 @@ sunholo/archive/__init__.py,sha256=qNHWm5rGPVOlxZBZCpA1wTYPbalizRT7f8X4rs2t290,3
 sunholo/archive/archive.py,sha256=C-UhG5x-XtZ8VheQp92IYJqgD0V3NFQjniqlit94t18,1197
 sunholo/auth/__init__.py,sha256=Y4Wpd6m0d3R7U7Ser51drO0Eg7VrfSS2VphZxRgtih8,70
 sunholo/auth/gcloud.py,sha256=PdbwkuTdRi4RKBmgG9uwsReegqC4VG15_tw5uzmA7Fs,298
+sunholo/auth/refresh.py,sha256=uOdT7oQRVl0YsUP__NXj6PdUdLyXFSv2ylwF283esuw,1831
 sunholo/auth/run.py,sha256=SG53ToQJ8hyjdN4634osfvDEUv5gJU6dlHe4nGwMMYU,2612
+sunholo/azure/__init__.py,sha256=S1WQ5jndzNgzhSBh9UpX_yw7hRVm3hCzkAWNxUdK4dA,48
+sunholo/azure/event_grid.py,sha256=Gky7D5a-xxMzbcst_wOFfcI8AH5qOzWbKbt5iqOTr6U,2606
 sunholo/bots/__init__.py,sha256=EMFd7e2z68l6pzYOnkzHbLd2xJRvxTKFRNCTuhZ8hIw,130
 sunholo/bots/discord.py,sha256=cCFae5K1BCa6JVkWGLh_iZ9qFO1JpXb6K4eJrlDfEro,2442
 sunholo/bots/github_webhook.py,sha256=5pQPRLM_wxxcILVaIzUDV8Kt7Arcm2dL1r1kMMHA524,9629
 sunholo/bots/webapp.py,sha256=EIMxdAJ_xtufwJmvnn7N_Fb_1hZ9DjhJ0Kf_hp02vEU,1926
-sunholo/chunker/__init__.py,sha256=yWYwpejyYxDpZv1joTrFMsh2SWAkd0z7a1VKtmOfMhA,77
-sunholo/chunker/data_to_embed_pubsub.py,sha256=-bXtm3Tn6YczUWL9kSAc6OWdD6dDY2rKhgpD-90H6ms,4203
+sunholo/chunker/__init__.py,sha256=A5canS0XPgisHu0OZ7sVdILgEHGzgH9kpkDi4oBwLZk,135
+sunholo/chunker/azure.py,sha256=mfY1-g_J21d1Ubx0Pt89KGk5PBGGjpbhRUALfP2jIHw,1666
 sunholo/chunker/doc_handling.py,sha256=rIyknpzDyj5A0u_DqSQVD_CXLRNZPOU6TCL4bhCdjOI,8563
 sunholo/chunker/images.py,sha256=Xmh1vwHrVhoXm5iH2dhCc52O8YgdzE8KrDSdL-pGnp8,1861
 sunholo/chunker/loaders.py,sha256=xiToUVgPz2ZzcqpUAq7aNP3PTenb_rBUAFzu0JPycIg,10268
-sunholo/chunker/message_data.py,sha256=X6aA4yX5aGN_mEvsDPWvdYRqqn5GO1BU9QhT9w5A0ec,6789
+sunholo/chunker/message_data.py,sha256=fDiwO_0HmuIq_1tZFK98ZOff6NAYCydWF8mjJ4Rx3Kk,7000
 sunholo/chunker/pdfs.py,sha256=daCZ1xjn1YvxlifIyxskWNpLJLe-Q9D_Jq12MWx3tZo,2473
+sunholo/chunker/process_chunker_data.py,sha256=lomCmWPzFf_-zHr2-iRMhZ_cj1b2q04UsG6ynoQCJ08,3333
 sunholo/chunker/publish.py,sha256=tiO615A2uo_ZjzdFDzNH1PL_1kJeLMUQwLJ4w67rNIc,2932
+sunholo/chunker/pubsub.py,sha256=XgLAuOFNDSqKEBvzRa0TSylZdPecRVHMp0nmmQ_OVco,1005
 sunholo/chunker/splitter.py,sha256=jtGfi_ZdhVdyFhfw0e4ynEpmwIyrxQtV63OituYWy6o,6729
 sunholo/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/cli/chat_vac.py,sha256=MjwGJQUJOkHV4vLAlhyYVQ02JoI5pE7zaLSSaBfcTco,23019
-sunholo/cli/cli.py,sha256=u70fcSQzQx2iPvE23SVCVYRFabmZ-XtgEd6vHcrABi0,3725
+sunholo/cli/cli.py,sha256=3ZMcsR1VLCdrsfm0zGBQ9TKqO5qkOrtZ6-iVNmr6f_8,3820
 sunholo/cli/cli_init.py,sha256=JMZ9AX2cPDZ-_mv3adiv2ToFVNyRPtjk9Biszl1kiR0,2358
 sunholo/cli/configs.py,sha256=QUM9DvKOdZmEQRM5uI3Nh887T0YDiSMr7O240zTLqws,4546
 sunholo/cli/deploy.py,sha256=zxdwUsRTRMC8U5vyRv0JiKBLFn84Ug_Tc88-_h9hJSs,1609
@@ -44,6 +49,7 @@ sunholo/cli/merge_texts.py,sha256=U9vdMwKmcPoc6iPOWX5MKSxn49dNGbNzVLw8ui5PhEU,18
 sunholo/cli/run_proxy.py,sha256=OeR12ZfnasbJ-smBZQznmGufoDa4iNjUN9FCFo5JxSc,11520
 sunholo/cli/sun_rich.py,sha256=UpMqeJ0C8i0pkue1AHnnyyX0bFJ9zZeJ7HBR6yhuA8A,54
 sunholo/cli/swagger.py,sha256=absYKAU-7Yd2eiVNUY-g_WLl2zJfeRUNdWQ0oH8M_HM,1564
+sunholo/cli/vertex.py,sha256=yN1ezeiweV1UIBZYvDYgtyMS0dXH374rYMxVVHid9UY,2101
 sunholo/components/__init__.py,sha256=IDoylb74zFKo6NIS3RQqUl0PDFBGVxM1dfUmO7OJ44U,176
 sunholo/components/llm.py,sha256=T4we3tGmqUj4tPwxQr9M6AXv_BALqZV_dRSvINan-oU,10374
 sunholo/components/retriever.py,sha256=BFUw_6turT3CQJZWv_uXylmH5fHdb0gKfKJrQ_j6MGY,6533
@@ -62,14 +68,14 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=qUKWzuHApDRJIUoynukVdGRBEq8eC9T7l9a3bWckgI0,59
-sunholo/discovery_engine/chunker_handler.py,sha256=H1HHDqWMCkchJER1_oU9TOLxqf2PygiMO6CL3uKZP64,4563
+sunholo/discovery_engine/chunker_handler.py,sha256=fDqvXeXr58s6TB75MMIGKKEg42T21ZIeDGuDfJN-vAw,4609
 sunholo/discovery_engine/create_new.py,sha256=7oZG78T6lW0EspRzlo7-qRyXFSuFxDn2dfSAVEaqlqY,978
 sunholo/discovery_engine/discovery_engine_client.py,sha256=YYsFeaW41l8jmWCruQnYxJGKEYBZ7dduTBDhdxI63hQ,17719
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=d_dIzeNF630Q0Ar-u1hxos60s0tLIImJccAvuo_LTIw,6814
 sunholo/gcs/__init__.py,sha256=SZvbsMFDko40sIRHTHppA37IijvJTae54vrhooEF5-4,90
 sunholo/gcs/add_file.py,sha256=vWRjxuHBQkrPNrr9tRSFGT0N_nVIw120mqDEHiaHwuQ,7115
-sunholo/gcs/download_url.py,sha256=Kg9EdPnc---YSUTAZEdzJeITjDtQSLMYwb4uiU9LhIQ,6440
+sunholo/gcs/download_url.py,sha256=iCIPESi2viQ-TcCINpbJXxUt7XJFFpF0KiVgSA6zFis,5228
 sunholo/gcs/metadata.py,sha256=C9sMPsHsq1ETetdQCqB3EBs3Kws8b8QHS9L7ei_v5aw,891
 sunholo/invoke/__init__.py,sha256=Dxivd9cU92X4v2JAZet4f7L2RJ5l_30rt9t2NiD-iLA,55
 sunholo/invoke/invoke_vac_utils.py,sha256=0JkCZDBEkRImzuB-nf70dF75t0WKtgA9G4TdaQJUB08,5240
@@ -79,7 +85,7 @@ sunholo/langfuse/prompts.py,sha256=EkbzSw9Jr05ULMsRDoGOp-frbtCZpnvdYSJEYNpzfX8,1
 sunholo/llamaindex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/llamaindex/generate.py,sha256=l1Picr-hVwkmAUD7XmTCa63qY9ERliFHQXwyX3BqB2Q,686
 sunholo/llamaindex/get_files.py,sha256=6rhXCDqQ_lrIapISQ_OYQDjiSATXvS_9m3qq53-oIl0,781
-sunholo/llamaindex/import_files.py,sha256=Iy_wkZCUSyrue_tAEHgnYaKDgg3-5GVygokHn3kd134,5747
+sunholo/llamaindex/import_files.py,sha256=tzwOZdoJ6vHDQ-Rc2lSr3XNfm-ddmOxOITNzFLNzmoY,5773
 sunholo/lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/lookup/model_lookup.yaml,sha256=O7o-jP53MLA06C8pI-ILwERShO-xf6z_258wtpZBv6A,739
 sunholo/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -114,13 +120,13 @@ sunholo/utils/timedelta.py,sha256=BbLabEx7_rbErj_YbNM0MBcaFN76DC4PTe4zD2ucezg,49
 sunholo/utils/user_ids.py,sha256=SQd5_H7FE7vcTZp9AQuQDWBXd4FEEd7TeVMQe1H4Ny8,292
 sunholo/utils/version.py,sha256=P1QAJQdZfT2cMqdTSmXmcxrD2PssMPEGM-WI6083Fck,237
 sunholo/vertex/__init__.py,sha256=XH7FUKxdIgN9H2iDcWxL3sRnVHC3297G24RqEn4Ob0Y,240
-sunholo/vertex/extensions_class.py,sha256=vjYOaqU-KQsqkDsJSWBmydzMA-kY-PqZ0j_GUb8ZlNY,16884
-sunholo/vertex/init.py,sha256=-w7b9GKsyJnAJpYHYz6_zBUtmeJeLXlEkgOfwoe4DEI,2715
+sunholo/vertex/extensions_class.py,sha256=cVpr0AbbBQV9WWZ9_X7S52aXclXNxvHywHDoBjDTxl8,17802
+sunholo/vertex/init.py,sha256=aLdNjrX3bUPfnWRhKUg5KUxSu0Qnq2YvuFNsgml4QEY,2866
 sunholo/vertex/memory_tools.py,sha256=pomHrDKqvY8MZxfUqoEwhdlpCvSGP6KmFJMVKOimXjs,6842
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
-sunholo-0.75.1.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.75.1.dist-info/METADATA,sha256=GDyIzrBHNdyewLXAJ0_PcLkgMlp6scIbI5sEL8izPrY,7010
-sunholo-0.75.1.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
-sunholo-0.75.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.75.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.75.1.dist-info/RECORD,,
+sunholo-0.76.3.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.76.3.dist-info/METADATA,sha256=SHLQ84yDXYXlm3Mb1sCqMUZtwXxzsd2UzgEwrd-fU_4,7136
+sunholo-0.76.3.dist-info/WHEEL,sha256=Z4pYXqR_rTB7OWNDYFOm1qRk0RX6GFP2o8LgvP453Hk,91
+sunholo-0.76.3.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.76.3.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.76.3.dist-info/RECORD,,