sunholo 0.77.4__tar.gz → 0.78.2__tar.gz

{sunholo-0.77.4 → sunholo-0.78.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.77.4
+Version: 0.78.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.77.4.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.78.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -25,6 +25,7 @@ Requires-Dist: langchain_experimental>0.0.60
 Requires-Dist: langchain-community
 Provides-Extra: all
 Requires-Dist: asyncpg; extra == "all"
+Requires-Dist: azure-identity; extra == "all"
 Requires-Dist: azure-storage-blob; extra == "all"
 Requires-Dist: fastapi; extra == "all"
 Requires-Dist: flask; extra == "all"
@@ -72,6 +73,7 @@ Requires-Dist: tantivy; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
 Requires-Dist: unstructured[local-inference]==0.14.9; extra == "all"
 Provides-Extra: azure
+Requires-Dist: azure-identity; extra == "azure"
 Requires-Dist: azure-storage-blob; extra == "azure"
 Provides-Extra: cli
 Requires-Dist: jsonschema>=4.21.1; extra == "cli"

{sunholo-0.77.4 → sunholo-0.78.2}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.77.4'
+version = '0.78.2'
 
 setup(
     name='sunholo',
@@ -40,6 +40,7 @@ setup(
         # Define optional dependencies with feature names
         'all': [
             "asyncpg",
+            "azure-identity",
             "azure-storage-blob",
             "fastapi",
             "flask",
@@ -89,6 +90,7 @@ setup(
 
         ],
         'azure': [
+            "azure-identity",
             "azure-storage-blob"
         ],
         'cli': [

{sunholo-0.77.4 → sunholo-0.78.2}/sunholo/agents/route.py

@@ -14,6 +14,19 @@
 from ..logging import log
 from ..utils import load_config, ConfigManager
 
+def read_cloud_run_url(agent, cloud_run_urls_file='config/cloud_run_urls.json'):
+    agent_route, _ = load_config(cloud_run_urls_file)
+    log.info(f'agent_route: {agent_route}')
+
+    try:
+        agent_url = agent_route[agent]
+    except KeyError:
+        raise ValueError(f'agent_url not found for {agent}')
+    
+    log.info(f'agent_url: {agent_url}')
+
+    return agent_url
+
 def route_vac(vector_name: str=None, config=None) -> str :
     """
     Considers what VAC this vector_name belongs to
@@ -30,18 +43,8 @@ def route_vac(vector_name: str=None, config=None) -> str :
         return agent_url
 
     agent = config.vacConfig('agent')
-    log.info(f'agent_type: {agent}')
-
-    agent_route, _ = load_config('config/cloud_run_urls.json')
-    log.info(f'agent_route: {agent_route}')
 
-    try:
-        agent_url = agent_route[agent]
-    except KeyError:
-        raise ValueError(f'agent_url not found for {agent}')
-    
-    log.info(f'agent_url: {agent_url}')
-    return agent_url
+    return read_cloud_run_url(agent)
 
 def route_endpoint(vector_name=None, method = 'post', override_endpoint=None, config=None):
 

sunholo-0.78.2/sunholo/azure/auth.py

@@ -0,0 +1,61 @@
+import os
+try:
+    from azure.identity import DefaultAzureCredential, ClientSecretCredential
+except ImportError:
+    DefaultAzureCredential = None
+    ClientSecretCredential = None
+
+from ..logging import log
+
+def azure_auth():
+    """
+    Will attempt to authenticate using default credentials first (e.g. you are running within Azure Container Apps or similar)
+
+    If default credentials are not available, will attempt to authenticate via env vars - set up via:
+
+    ```bash
+    az ad sp create-for-rbac --name "myApp" --role contributor \
+        --scopes /subscriptions/{subscription-id}/resourceGroups/{resource-group} \
+        --sdk-auth
+
+    export AZURE_CLIENT_ID="your-client-id"
+    export AZURE_CLIENT_SECRET="your-client-secret"
+    export AZURE_TENANT_ID="your-tenant-id"
+    ```
+
+    """
+    if DefaultAzureCredential is None: 
+        raise ImportError("Azure identity credentials library needed - install via `pip install sunholo[azure]`")
+    
+    # Use DefaultAzureCredential to authenticate
+    try:
+        credential = DefaultAzureCredential()
+        return credential
+    
+    except Exception as e:
+        log.error(f"Failed to authenticate with default credentials: {str(e)}")
+        log.info("Attempting to authenticate using ClientSecretCredential")
+
+        # Use ClientSecretCredential to authenticate with a service principal
+        client_id = os.getenv("AZURE_CLIENT_ID")
+        client_secret = os.getenv("AZURE_CLIENT_SECRET")
+        tenant_id = os.getenv("AZURE_TENANT_ID")
+
+        if not client_id or not client_secret or not tenant_id:
+            log.error("Service principal credentials are not set in environment variables")
+            return None
+
+        if ClientSecretCredential is None:
+            raise ImportError("Azure identity credentials library needed - install via `pip install sunholo[azure]`")
+
+        try:
+            credential = ClientSecretCredential(
+                client_id=client_id,
+                client_secret=client_secret,
+                tenant_id=tenant_id
+            )
+            return credential
+        except Exception as e:
+            log.error(f"Failed to authenticate with service principal: {str(e)}")
+            return None
+        

sunholo-0.78.2/sunholo/azure/blobs.py

@@ -0,0 +1,47 @@
+import re
+from ..logging import log
+
+
+def is_azure_blob(message_data):
+    """
+    Checks if the provided URL is an Azure Blob Storage URL.
+
+    Args:
+        message_data (str): The URL to be checked.
+
+    Returns:
+        bool: True if the URL is an Azure Blob Storage URL, False otherwise.
+    """
+    blob_url_pattern = r"https://(.*).blob.core.windows.net/(.*)/(.*)"
+    match = re.match(blob_url_pattern, message_data)
+    if not match:
+        return False
+    
+    return True
+
+def extract_blob_parts(message_data):
+    """
+    Extracts the account name, container name, and blob name from an Azure Blob Storage URL.
+
+    Args:
+        message_data (str): The Azure Blob Storage URL.
+
+    Returns:
+        tuple: A tuple containing the account name, container name, and blob name.
+               Returns (None, None, None) if the URL is invalid.
+    """
+    if not is_azure_blob(message_data):
+        return None, None, None
+    
+    log.debug("Detected Azure blob storage URL")
+    # Extract the account name, container name, and blob name from the URL
+    blob_url_pattern = r"https://(.*).blob.core.windows.net/(.*)/(.*)"
+    match = re.match(blob_url_pattern, message_data)
+    if not match:
+        log.error("Invalid Azure blob URL format")
+        return None, None
+
+    account_name, container_name, blob_name = match.groups()
+
+    return account_name, container_name, blob_name
+

sunholo-0.78.2/sunholo/chunker/azure.py

@@ -0,0 +1,90 @@
+#   Copyright [2024] [Holosun ApS]
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+import json
+from datetime import datetime, timezone
+
+from ..logging import log
+from ..azure import process_azure_blob_event
+
+from ..invoke import invoke_vac
+from .process_chunker_data import process_chunker_data
+from ..chunker.encode_metadata import create_metadata, encode_data
+from ..agents.route import read_cloud_run_url
+
+def data_to_embed_azure(events: list):
+    """Triggered from a message on an Azure Data Grid event.
+    Args:
+         data JSON
+    """
+    validation_event_type = "Microsoft.EventGrid.SubscriptionValidationEvent"
+    storage_blob_created_event = "Microsoft.Storage.BlobCreated"
+    
+    all_chunks = []
+    vac_name = None
+    for event in events:
+        event_type = event['eventType']
+        data = event['data']
+
+        if event_type == validation_event_type:
+            validation_code = data['validationCode']
+            log.info(f"Got SubscriptionValidation event data, validation code: {validation_code}, topic: {event['topic']}")
+            
+            # Return the validation response
+            return {"ValidationResponse": validation_code}
+        elif event_type == storage_blob_created_event:
+
+            message_data, metadata, vac_name = process_azure_blob_event(events)
+            metadata["return_chunks"] = True
+
+            #TODO: process the azure blob URL and download it
+            
+            chunks = process_chunker_data(message_data, metadata, vac_name)
+            if chunks:
+                all_chunks.extend(chunks)
+    
+    if not all_chunks or len(chunks) == 0:
+        return {'status': 'error', 'message': f'No chunks were found in events: {events}'}
+    
+    if not vac_name:
+        return {'status': 'error', 'message': f'Could not find a valid VAC config name in payload {all_chunks}'}
+    
+    metadata = create_metadata(vac_name, metadata)
+
+    embeds = []
+
+    for chunk in chunks:
+        log.info(f"Working on chunk {chunk['metadata']}")
+
+        # do this async?
+        content = chunk.get("page_content")
+        now_utc = datetime.now(timezone.utc)
+        formatted_time = now_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+        chunk["metadata"]["eventTime"]  = formatted_time
+        if not content:
+            log.error("No content chunk found, skipping")
+
+            continue
+
+        log.info(f"Sending chunk length {len(content)} to embedder")
+        processed_chunk = encode_data(vac = vac_name, content = json.dumps(chunk))
+        
+        embed_url = read_cloud_run_url('embedder')
+
+        embed_res = invoke_vac(f"{embed_url}/embed_chunk", processed_chunk)
+        embeds.append(embed_res)
+
+    log.info("Embedding pipeline finished")
+    
+    return embed_res        
+

sunholo-0.78.2/sunholo/chunker/encode_metadata.py

@@ -0,0 +1,64 @@
+import uuid
+import base64
+import json
+from datetime import datetime, timezone
+
+from ..logging import log
+
+def create_metadata(vac, metadata):
+    now_utc = datetime.now(timezone.utc)
+    formatted_time = now_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    # Default metadata if none provided
+    default_metadata = {"vector_name": vac, "source": "sunholo-cli", "eventTime": formatted_time}
+
+    try:
+        # Merge default metadata with provided metadata
+        if metadata:
+            if not isinstance(metadata, dict):
+                metadata = json.loads(metadata)
+        else:
+            metadata = {}    
+    except Exception as err:
+        log.error(f"[bold red]ERROR: metadata not parsed: {err} for {metadata}")
+
+    # Update metadata with default values if not present
+    metadata.update(default_metadata)
+
+    return metadata
+
+def encode_data(vac, content, metadata=None, local_chunks=False):
+
+    metadata = create_metadata(vac, metadata)
+
+    # Encode the content (URL)
+    if isinstance(content, str):
+        message_data = base64.b64encode(content.encode('utf-8')).decode('utf-8')
+    else:
+        raise ValueError(f"Unsupported content type: {type(content)}")
+
+    now_utc = datetime.now(timezone.utc)
+    formatted_time = now_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    # Construct the message dictionary
+    messageId = str(uuid.uuid4())
+    message = {
+        "message": {
+            "data": message_data,
+            "messageId": messageId,
+            "publishTime": formatted_time,
+            "attributes": {
+                "namespace": vac,
+                "return_chunks": str(local_chunks).lower()
+            },
+        }
+    }
+
+    # Merge metadata with attributes
+    message["message"]["attributes"].update(metadata)
+
+    #console.print()
+    #console.print(f"Sending message: {messageId} with metadata:")
+    #console.print(f"{message['message']['attributes']}")
+
+    return message

{sunholo-0.77.4 → sunholo-0.78.2}/sunholo/chunker/message_data.py

@@ -18,11 +18,17 @@ import tempfile
 import os
 import re
 import json
+
 try:
     from google.cloud import storage
 except ImportError:
     storage = None
 
+try:
+    from azure.storage.blob import BlobServiceClient
+except ImportError:
+    BlobServiceClient = None
+
 from langchain.schema import Document
 
 
@@ -33,8 +39,8 @@ from . import loaders
 
 from ..utils.parsers import extract_urls
 from ..gcs.add_file import add_file_to_gcs, get_pdf_split_file_name
-
-
+from ..azure.blobs import extract_blob_parts
+from ..azure.auth import azure_auth
 
 def handle_gcs_message(message_data: str, metadata: dict, vector_name: str):
 
@@ -199,4 +205,95 @@ def handle_json_content_message(message_data: dict, metadata: dict, vector_name:
 
     chunks = chunk_doc_to_docs(docs, vector_name=vector_name)
 
-    return chunks, metadata
+    return chunks, metadata
+
+
+def handle_azure_blob(message_data: str, metadata: dict, vector_name: str):
+    """
+    Processes a message from Azure Blob storage, downloads the file, processes it,
+    and returns chunks and metadata.
+
+    Args:
+        message_data (str): URL of the Azure blob.
+        metadata (dict): Metadata associated with the file.
+        vector_name (str): Vector name for processing.
+
+    Returns:
+        chunks (list): List of document chunks.
+        metadata (dict): Updated metadata.
+    """
+
+    if BlobServiceClient is None:
+        raise ImportError("BlobServiceClient is not installed - install via pip install sunholo[azure]")
+
+    account_name, container_name, blob_name = extract_blob_parts(message_data)
+
+    credential = azure_auth()
+    if credential is None:
+        log.error("BlobServiceClient could not find auth credentials")
+        return None, None
+    
+    # Create a BlobServiceClient
+    blob_service_client = BlobServiceClient(
+        account_url=f"https://{account_name}.blob.core.windows.net",
+        credential=credential)
+
+    # Get the blob client
+    blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name)
+
+    file_name = pathlib.Path(blob_name)
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        tmp_file_path = os.path.join(temp_dir, file_name.name)
+        with open(tmp_file_path, "wb") as file:
+            download_stream = blob_client.download_blob()
+            file.write(download_stream.readall())
+
+        if file_name.suffix.lower() == ".pdf":
+            pages = split_pdf_to_pages(tmp_file_path, temp_dir)
+            if not metadata.get("source"):
+                metadata["source"] = str(file_name)
+            if len(pages) > 1:
+                log.info(f"Got back {len(pages)} pages for file {tmp_file_path}")
+                for pp in pages:
+                    pp_basename = os.path.basename(pp)
+                    # file_name/pdf_parts/file_name_1.pdf
+                    azure_blob_path = f"{file_name.stem}_parts/{pp_basename}"
+
+                    # Upload split pages back to Azure Blob storage
+                    container_client = blob_service_client.get_container_client(container=container_name)
+                    with open(file=pp, mode="rb") as page_file:
+                        blob_client = container_client.upload_blob(name=azure_blob_path, data=page_file, overwrite=True)
+
+                    log.info(f"{azure_blob_path} is now in container {container_name}")
+                log.info(f"Sent split pages for {file_name.name} back to Azure Blob to parallelize the imports")
+                return None, None
+        else:
+            # just original temp file
+            pages = [tmp_file_path]
+
+        the_metadata = {
+            "type": "file_load_azure_blob",
+            "container_name": container_name
+        }
+
+        if metadata.get("source") is None:
+            the_metadata["source"] = str(file_name)
+
+        metadata.update(the_metadata)
+
+        docs = []
+        for page in pages:
+            log.info(f"Sending file {page} to loaders.read_file_to_documents {metadata}")
+            docs2 = loaders.read_file_to_documents(page, metadata=metadata)
+            if docs2 is None:
+                log.warning(f"loaders.read_file_to_documents docs2 failed to load file {metadata}")
+            docs.extend(docs2)
+
+        if docs is None:
+            log.warning(f"loaders.read_file_to_documents docs failed to load file {metadata}")
+            return None, metadata
+        else:
+            chunks = chunk_doc_to_docs(docs, file_name.suffix, vector_name=vector_name)
+
+        return chunks, metadata

{sunholo-0.77.4 → sunholo-0.78.2}/sunholo/chunker/process_chunker_data.py

@@ -5,7 +5,8 @@ from .message_data import (
     handle_google_drive_message, 
     handle_github_message, 
     handle_http_message, 
-    handle_json_content_message
+    handle_json_content_message,
+    handle_azure_blob
 )
 
 from . import loaders
@@ -13,7 +14,7 @@ from ..llamaindex.import_files import llamaindex_chunker_check
 from ..discovery_engine.chunker_handler import discovery_engine_chunker_check
 from .publish import process_docs_chunks_vector_name
 from .splitter import chunk_doc_to_docs
-
+from ..azure.blobs import is_azure_blob
 
 from ..logging import log
 
@@ -43,6 +44,9 @@ def process_chunker_data(message_data, metadata, vector_name):
     if message_data.startswith("gs://"):
         chunks, metadata =  handle_gcs_message(message_data, metadata, vector_name)
 
+    elif is_azure_blob(message_data):
+        chunks, metadata = handle_azure_blob(message_data, metadata, vector_name)
+
     elif message_data.startswith("https://drive.google.com") or message_data.startswith("https://docs.google.com"):
         chunks, metadata = handle_google_drive_message(message_data, metadata, vector_name)
 

{sunholo-0.77.4 → sunholo-0.78.2}/sunholo/cli/embedder.py

@@ -11,64 +11,7 @@ from rich.progress import Progress
 from ..invoke import invoke_vac
 from .chat_vac import resolve_service_url
 from .run_proxy import stop_proxy
-
-def create_metadata(vac, metadata):
-    now_utc = datetime.now(timezone.utc)
-    formatted_time = now_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    # Default metadata if none provided
-    default_metadata = {"vector_name": vac, "source": "sunholo-cli", "eventTime": formatted_time}
-
-    try:
-        # Merge default metadata with provided metadata
-        if metadata:
-            if not isinstance(metadata, dict):
-                metadata = json.loads(metadata)
-        else:
-            metadata = {}    
-    except Exception as err:
-        console.print(f"[bold red]ERROR: metadata not parsed: {err} for {metadata}")
-
-    # Update metadata with default values if not present
-    metadata.update(default_metadata)
-
-    return metadata
-
-def encode_data(vac, content, metadata=None, local_chunks=False):
-
-    metadata = create_metadata(vac, metadata)
-
-    # Encode the content (URL)
-    if isinstance(content, str):
-        message_data = base64.b64encode(content.encode('utf-8')).decode('utf-8')
-    else:
-        raise ValueError(f"Unsupported content type: {type(content)}")
-
-    now_utc = datetime.now(timezone.utc)
-    formatted_time = now_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    # Construct the message dictionary
-    messageId = str(uuid.uuid4())
-    message = {
-        "message": {
-            "data": message_data,
-            "messageId": messageId,
-            "publishTime": formatted_time,
-            "attributes": {
-                "namespace": vac,
-                "return_chunks": str(local_chunks).lower()
-            },
-        }
-    }
-
-    # Merge metadata with attributes
-    message["message"]["attributes"].update(metadata)
-
-    #console.print()
-    #console.print(f"Sending message: {messageId} with metadata:")
-    #console.print(f"{message['message']['attributes']}")
-
-    return message
+from ..chunker.encode_metadata import create_metadata, encode_data
 
 def embed_command(args):
     chunk_args = vars(args).copy()

{sunholo-0.77.4 → sunholo-0.78.2}/sunholo/invoke/invoke_vac_utils.py

@@ -33,10 +33,10 @@ def invoke_vac(service_url, data, vector_name=None, metadata=None, is_file=False
                 else:
                     json_data = json.loads(data)
             except json.JSONDecodeError as err:
-                log.error(f"[bold red]ERROR: invalid JSON: {str(err)} [/bold red]")
+                log.error(f"ERROR: invalid JSON: {str(err)}")
                 raise err
             except Exception as err:
-                log.error(f"[bold red]ERROR: could not parse JSON: {str(err)} [/bold red]")
+                log.error(f"ERROR: could not parse JSON: {str(err)}")
                 raise err
 
             log.debug(f"Sending data: {data} or json_data: {json.dumps(json_data)}")

{sunholo-0.77.4 → sunholo-0.78.2}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.77.4
+Version: 0.78.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.77.4.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.78.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -25,6 +25,7 @@ Requires-Dist: langchain_experimental>0.0.60
 Requires-Dist: langchain-community
 Provides-Extra: all
 Requires-Dist: asyncpg; extra == "all"
+Requires-Dist: azure-identity; extra == "all"
 Requires-Dist: azure-storage-blob; extra == "all"
 Requires-Dist: fastapi; extra == "all"
 Requires-Dist: flask; extra == "all"
@@ -72,6 +73,7 @@ Requires-Dist: tantivy; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
 Requires-Dist: unstructured[local-inference]==0.14.9; extra == "all"
 Provides-Extra: azure
+Requires-Dist: azure-identity; extra == "azure"
 Requires-Dist: azure-storage-blob; extra == "azure"
 Provides-Extra: cli
 Requires-Dist: jsonschema>=4.21.1; extra == "cli"

{sunholo-0.77.4 → sunholo-0.78.2}/sunholo.egg-info/SOURCES.txt

@@ -33,6 +33,8 @@ sunholo/auth/gcloud.py
 sunholo/auth/refresh.py
 sunholo/auth/run.py
 sunholo/azure/__init__.py
+sunholo/azure/auth.py
+sunholo/azure/blobs.py
 sunholo/azure/event_grid.py
 sunholo/bots/__init__.py
 sunholo/bots/discord.py
@@ -41,6 +43,7 @@ sunholo/bots/webapp.py
 sunholo/chunker/__init__.py
 sunholo/chunker/azure.py
 sunholo/chunker/doc_handling.py
+sunholo/chunker/encode_metadata.py
 sunholo/chunker/images.py
 sunholo/chunker/loaders.py
 sunholo/chunker/message_data.py

{sunholo-0.77.4 → sunholo-0.78.2}/sunholo.egg-info/requires.txt

@@ -6,6 +6,7 @@ langchain-community
 
 [all]
 asyncpg
+azure-identity
 azure-storage-blob
 fastapi
 flask
@@ -57,6 +58,7 @@ unstructured[local-inference]==0.14.9
 langchain-anthropic>=0.1.13
 
 [azure]
+azure-identity
 azure-storage-blob
 
 [cli]

sunholo-0.77.4/sunholo/chunker/azure.py

@@ -1,41 +0,0 @@
-#   Copyright [2024] [Holosun ApS]
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-from ..logging import log
-from ..azure import process_azure_blob_event
-from .process_chunker_data import process_chunker_data
-
-def data_to_embed_azure(events: list):
-    """Triggered from a message on an Azure Data Grid event.
-    Args:
-         data JSON
-    """
-    validation_event_type = "Microsoft.EventGrid.SubscriptionValidationEvent"
-    storage_blob_created_event = "Microsoft.Storage.BlobCreated"
-    
-    for event in events:
-        event_type = event['eventType']
-        data = event['data']
-
-        if event_type == validation_event_type:
-            validation_code = data['validationCode']
-            log.info(f"Got SubscriptionValidation event data, validation code: {validation_code}, topic: {event['topic']}")
-            
-            # Return the validation response
-            return {"ValidationResponse": validation_code}
-        elif event_type == storage_blob_created_event:
-
-            message_data, metadata, vector_name = process_azure_blob_event(events)
-
-            return process_chunker_data(message_data, metadata, vector_name)