sunholo 0.75.1__tar.gz → 0.76.2__tar.gz

{sunholo-0.75.1 → sunholo-0.76.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.75.1
+Version: 0.76.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.75.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.76.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -25,6 +25,7 @@ Requires-Dist: langchain_experimental>0.0.60
 Requires-Dist: langchain-community
 Provides-Extra: all
 Requires-Dist: asyncpg; extra == "all"
+Requires-Dist: azure-storage-blob; extra == "all"
 Requires-Dist: fastapi; extra == "all"
 Requires-Dist: flask; extra == "all"
 Requires-Dist: google-auth; extra == "all"
@@ -69,6 +70,8 @@ Requires-Dist: tabulate; extra == "all"
 Requires-Dist: tantivy; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
 Requires-Dist: unstructured[local-inference]==0.14.9; extra == "all"
+Provides-Extra: azure
+Requires-Dist: azure-storage-blob; extra == "azure"
 Provides-Extra: cli
 Requires-Dist: jsonschema>=4.21.1; extra == "cli"
 Requires-Dist: rich; extra == "cli"

{sunholo-0.75.1 → sunholo-0.76.2}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.75.1'
+version = '0.76.2'
 
 setup(
     name='sunholo',
@@ -40,6 +40,7 @@ setup(
         # Define optional dependencies with feature names
         'all': [
             "asyncpg",
+            "azure-storage-blob",
             "fastapi",
             "flask",
             "google-auth",
@@ -86,6 +87,9 @@ setup(
             "unstructured[local-inference]==0.14.9",
 
         ],
+        'azure': [
+            "azure-storage-blob"
+        ],
         'cli': [
             "jsonschema>=4.21.1",
             "rich"

sunholo-0.76.2/sunholo/azure/__init__.py

@@ -0,0 +1 @@
+from .event_grid import process_azure_blob_event

sunholo-0.76.2/sunholo/azure/event_grid.py

@@ -0,0 +1,69 @@
+# process_azure_blob_event.py
+from ..logging import log
+
+def process_azure_blob_event(events: list) -> tuple:
+    """
+    Extracts message data and metadata from an Azure Blob Storage event.
+
+    Args:
+        events (list): The list of Azure Event Grid event data.
+
+    Returns:
+        tuple: A tuple containing the blob URL, attributes as metadata, and the vector name.
+        
+    Example of Event Grid schema:
+    {
+        "topic": "/subscriptions/{subscription-id}/resourceGroups/{resource-group}/providers/Microsoft.Storage/storageAccounts/{storage-account}",
+        "subject": "/blobServices/default/containers/{container}/blobs/{blob}",
+        "eventType": "Microsoft.Storage.BlobCreated",
+        "eventTime": "2021-01-01T12:34:56.789Z",
+        "id": "event-id",
+        "data": {
+            "api": "PutBlob",
+            "clientRequestId": "client-request-id",
+            "requestId": "request-id",
+            "eTag": "etag",
+            "contentType": "application/octet-stream",
+            "contentLength": 524288,
+            "blobType": "BlockBlob",
+            "url": "https://{storage-account}.blob.core.windows.net/{container}/{blob}",
+            "sequencer": "0000000000000000000000000000000000000000000000000000000000000000",
+            "storageDiagnostics": {
+                "batchId": "batch-id"
+            }
+        },
+        "dataVersion": "",
+        "metadataVersion": "1"
+    }
+    """
+    storage_blob_created_event = "Microsoft.Storage.BlobCreated"
+    
+    for event in events:
+        event_type = event['eventType']
+        data = event['data']
+
+        if event_type == storage_blob_created_event:
+            blob_url = data['url']
+            event_time = event['eventTime']
+            event_id = event['id']
+            subject = event['subject']
+            attributes = {
+                'event_type': event_type,
+                'event_time': event_time,
+                'event_id': event_id,
+                'subject': subject,
+                'url': blob_url
+            }
+
+            vector_name = subject.split('/')[4]  # Extracting the container name
+            
+            log.info(f"Process Azure Blob Event was triggered by eventId {event_id} at {event_time}")
+            log.debug(f"Process Azure Blob Event data: {blob_url}")
+            
+            # Check for a valid Azure Blob Storage event type
+            if event_type == "Microsoft.Storage.BlobCreated":
+                log.info(f"Got valid event from Azure Blob Storage: {blob_url}")
+            
+            return blob_url, attributes, vector_name
+
+    return None, None, None

sunholo-0.76.2/sunholo/chunker/__init__.py

@@ -0,0 +1,3 @@
+from .pubsub import data_to_embed_pubsub
+from .azure import data_to_embed_azure
+from .process_chunker_data import direct_file_to_embed

sunholo-0.76.2/sunholo/chunker/azure.py

@@ -0,0 +1,41 @@
+#   Copyright [2024] [Holosun ApS]
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from ..logging import log
+from ..azure import process_azure_blob_event
+from .process_chunker_data import process_chunker_data
+
+def data_to_embed_azure(events: list):
+    """Triggered from a message on an Azure Data Grid event.
+    Args:
+         data JSON
+    """
+    validation_event_type = "Microsoft.EventGrid.SubscriptionValidationEvent"
+    storage_blob_created_event = "Microsoft.Storage.BlobCreated"
+    
+    for event in events:
+        event_type = event['eventType']
+        data = event['data']
+
+        if event_type == validation_event_type:
+            validation_code = data['validationCode']
+            log.info(f"Got SubscriptionValidation event data, validation code: {validation_code}, topic: {event['topic']}")
+            
+            # Return the validation response
+            return {"ValidationResponse": validation_code}
+        elif event_type == storage_blob_created_event:
+
+            message_data, metadata, vector_name = process_azure_blob_event(events)
+
+            return process_chunker_data(message_data, metadata, vector_name)

{sunholo-0.75.1 → sunholo-0.76.2}/sunholo/chunker/message_data.py

@@ -173,10 +173,15 @@ def handle_http_message(message_data: str, metadata: dict, vector_name:str):
 
     return chunks, metadata
 
-def handle_json_content_message(message_data: str, metadata: dict, vector_name: str):
+def handle_json_content_message(message_data: dict, metadata: dict, vector_name: str):
     log.info("No tailored message_data detected, processing message json")
     # Process message containing direct JSON content
-    the_json = json.loads(message_data)
+    try:
+        the_json = json.loads(message_data)
+    except Exception as e:
+        log.error(f"Could not load message {message_data} as JSON - {str(e)}")
+        return None, {"metadata": f"Could not load message as JSON - {str(e)}"}
+    
     the_metadata = the_json.get("metadata", {})
     metadata.update(the_metadata)
     the_content = the_json.get("page_content", None)
@@ -186,7 +191,7 @@ def handle_json_content_message(message_data: str, metadata: dict, vector_name:
 
     if the_content is None:
         log.info("No content found")
-        return {"metadata": "No content found in 'page_content' JSON field"}
+        return None, {"metadata": "No content found in 'page_content' JSON field"}
     
     docs = [Document(page_content=the_content, metadata=metadata)]
 

sunholo-0.75.1/sunholo/chunker/data_to_embed_pubsub.py → sunholo-0.76.2/sunholo/chunker/process_chunker_data.py

@@ -1,58 +1,21 @@
-#   Copyright [2024] [Holosun ApS]
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
 import pathlib
 
-from ..logging import log
-from ..pubsub import process_pubsub_message
-from .message_data import handle_gcs_message, handle_google_drive_message, handle_github_message, handle_http_message, handle_json_content_message
-from .publish import process_docs_chunks_vector_name
-from .splitter import chunk_doc_to_docs
+from .message_data import (
+    handle_gcs_message, 
+    handle_google_drive_message, 
+    handle_github_message, 
+    handle_http_message, 
+    handle_json_content_message
+)
 
+from . import loaders
 from ..llamaindex.import_files import llamaindex_chunker_check
 from ..discovery_engine.chunker_handler import discovery_engine_chunker_check
+from .publish import process_docs_chunks_vector_name
+from .splitter import chunk_doc_to_docs
 
-from . import loaders
-
-def direct_file_to_embed(file_name: pathlib.Path, metadata: dict, vector_name: str):
-    """
-    Send direct files to chunking embed pipeline
-
-    
-    
-    """
-    log.info(f"Sending direct file upload {file_name} to loaders.read_file_to_documents {metadata}")
-    docs = loaders.read_file_to_documents(file_name, metadata=metadata)
-    if docs is None:
-        log.warning(f"loaders.read_file_to_documents docs2 failed to load file {metadata}")
-
-        return None
-
-    chunks = chunk_doc_to_docs(docs, file_name.suffix, vector_name=vector_name)
-    
-    return format_chunk_return(chunks, metadata, vector_name)
-
-
-
-def data_to_embed_pubsub(data: dict):
-    """Triggered from a message on a Cloud Pub/Sub topic.
-    Args:
-         data JSON
-    """
-
-    message_data, metadata, vector_name = process_pubsub_message(data)
 
-    return process_chunker_data(message_data, metadata, vector_name)
+from ..logging import log
 
 def process_chunker_data(message_data, metadata, vector_name):
 
@@ -95,7 +58,6 @@ def process_chunker_data(message_data, metadata, vector_name):
     
     return format_chunk_return(chunks, metadata, vector_name)
 
-
 def format_chunk_return(chunks, metadata, vector_name):
     # to be really sure
     if metadata:
@@ -110,8 +72,23 @@ def format_chunk_return(chunks, metadata, vector_name):
                     
             return output_list
 
+    # returns None when not on GCP
     process_docs_chunks_vector_name(chunks, vector_name, metadata)
 
     return metadata
 
 
+def direct_file_to_embed(file_name: pathlib.Path, metadata: dict, vector_name: str):
+    """
+    Send direct files to chunking embed pipeline
+    """
+    log.info(f"Sending direct file upload {file_name} to loaders.read_file_to_documents {metadata}")
+    docs = loaders.read_file_to_documents(file_name, metadata=metadata)
+    if docs is None:
+        log.warning(f"loaders.read_file_to_documents docs2 failed to load file {metadata}")
+
+        return None
+
+    chunks = chunk_doc_to_docs(docs, file_name.suffix, vector_name=vector_name)
+    
+    return format_chunk_return(chunks, metadata, vector_name)

sunholo-0.76.2/sunholo/chunker/pubsub.py

@@ -0,0 +1,31 @@
+#   Copyright [2024] [Holosun ApS]
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from ..logging import log
+from ..pubsub import process_pubsub_message
+from .process_chunker_data import process_chunker_data
+
+def data_to_embed_pubsub(data: dict):
+    """Triggered from a message on a Cloud Pub/Sub topic.
+    Args:
+         data JSON
+    """
+
+    message_data, metadata, vector_name = process_pubsub_message(data)
+
+    return process_chunker_data(message_data, metadata, vector_name)
+
+
+
+

{sunholo-0.75.1 → sunholo-0.76.2}/sunholo/cli/cli.py

@@ -9,6 +9,7 @@ from .run_proxy import setup_proxy_subparser
 from .chat_vac import setup_vac_subparser
 from .embedder import setup_embedder_subparser
 from .swagger import setup_swagger_subparser
+from .vertex import setup_vertex_subparser
 
 from ..utils import ConfigManager
 
@@ -84,6 +85,8 @@ def main(args=None):
     setup_embedder_subparser(subparsers)
     # swagger generation
     setup_swagger_subparser(subparsers)
+    # vertex
+    setup_vertex_subparser(subparsers)
 
     #TODO: add database setup commands: alloydb and supabase
 

sunholo-0.76.2/sunholo/cli/vertex.py

@@ -0,0 +1,46 @@
+from ..vertex import VertexAIExtensions  
+
+from .sun_rich import console
+
+def deploy_extension(args):
+    vex = VertexAIExtensions()
+    console.rule(f"Creating Vertex extension '{args.display_name}'")
+
+    vex.create_extension(
+        args.display_name,
+        description=args.description,
+        tool_example_file=args.tool_example_file,
+        open_api_file=args.open_api_file,
+        service_account=args.service_account,
+        project_id=args.project,
+        bucket_name=args.bucket_name
+    )
+    extensions = vex.list_extensions(args.project)
+    console.print(extensions)
+
+def list_extensions(args):
+    vex = VertexAIExtensions()
+    extensions = vex.list_extensions(args.project)
+    console.print(extensions)
+
+def setup_vertex_subparser(subparsers):
+    """
+    Sets up an argparse subparser for the 'vertex' command.
+
+    Args:
+        subparsers: The subparsers object to add the 'vertex' subcommand to.
+    """
+    vertex_parser = subparsers.add_parser('vertex', help='Work with Google Vertex AI')
+    vertex_subparsers = vertex_parser.add_subparsers(dest='subcommand', help='Vertex AI subcommands')
+
+    create_parser = vertex_subparsers.add_parser('create-extension', help='Create a Vertex AI extension')
+    create_parser.add_argument('--display_name', required=True, help='Display name of the extension')
+    create_parser.add_argument('--description', required=True, help='Description of the extension')
+    create_parser.add_argument('--tool_example_file', required=True, help='Tool example file path')
+    create_parser.add_argument('--open_api_file', required=True, help='OpenAPI file path')
+    create_parser.add_argument('--service_account', required=True, help='Service account email')
+    create_parser.add_argument('--bucket_name', help='Bucket name to upload files to.  Uses EXTENSION_BUCKET env var if not specified')
+    create_parser.set_defaults(func=deploy_extension)
+
+    list_parser = vertex_subparsers.add_parser('list-extensions', help='List all Vertex AI extensions')
+    list_parser.set_defaults(func=list_extensions)

{sunholo-0.75.1 → sunholo-0.76.2}/sunholo/discovery_engine/chunker_handler.py

@@ -102,6 +102,9 @@ def check_discovery_engine_in_memory(vector_name):
 def discovery_engine_chunker_check(message_data, metadata, vector_name):
     # discovery engine handles its own chunking/embedding
     memories = load_config_key("memory", vector_name=vector_name, kind="vacConfig")
+    if not memories:
+        return None
+    
     total_memories = len(memories)
     llama = None
     if check_discovery_engine_in_memory(vector_name):

{sunholo-0.75.1 → sunholo-0.76.2}/sunholo/llamaindex/import_files.py

@@ -131,7 +131,10 @@ def check_llamaindex_in_memory(vector_name):
 
 def llamaindex_chunker_check(message_data, metadata, vector_name):
     # llamaindex handles its own chunking/embedding
-    memories = ConfigManager(vector_name).vacConfig("memory")
+    memories = load_memories(vector_name)
+    if not memories:
+        return None
+    
     total_memories = len(memories)
     llama = None
     if check_llamaindex_in_memory(vector_name):

{sunholo-0.75.1 → sunholo-0.76.2}/sunholo/vertex/extensions_class.py

@@ -69,9 +69,12 @@ class VertexAIExtensions:
         self.tool_use_examples = None
         self.manifest = {}
         self.created_extensions = []
+        self.bucket_name = os.getenv('EXTENSIONS_BUCKET')
 
-    def list_extensions(self):
-        the_list = extensions.Extension.list()
+    def list_extensions(self, project_id:str=None):
+        project_id = project_id or get_gcp_project()
+        log.info(f"Creating extension within {project_id=}")
+        the_list = extensions.Extension.list(project=project_id)
         
         extensions_list = []
         for ext in the_list:
@@ -94,20 +97,23 @@ class VertexAIExtensions:
         validate(spec_dict)
     
     def upload_to_gcs(self, filename):
-        if not os.getenv('EXTENSIONS_BUCKET'):
-            raise ValueError('Please specify env var EXTENSIONS_BUCKET for location to upload openapi spec')
+        if not self.bucket_name:
+            raise ValueError('Please specify bucket_name or env var EXTENSIONS_BUCKET for location to upload openapi spec')
         
         from ..gcs.add_file import add_file_to_gcs
         file_base = os.path.basename(filename)
 
-        self_uri = add_file_to_gcs(file_base, bucket_filepath=file_base)
+        self_uri = add_file_to_gcs(file_base, bucket_filepath=file_base, bucket_name=self.bucket_name)
 
         return self_uri
     
     def upload_openapi_file(self, filename: str):
         self.validate_openapi(filename)
+        if not self.bucket_name:
+            raise ValueError('Please specify env var EXTENSIONS_BUCKET for location to upload openapi spec')
+        
 
-        self.openapi_file_gcs = self.upload_to_gcs(filename)
+        self.openapi_file_gcs = self.upload_to_gcs(filename, bucket_name=self.bucket_name)
     
     def load_tool_use_examples(self, filename: str):
         import yaml
@@ -193,15 +199,23 @@ class VertexAIExtensions:
                          open_api_file: str = None,
                          tool_example_file: str = None,
                          runtime_config: dict = None,
-                         service_account: str = None):
+                         service_account: str = None,
+                         project_id: str = None,
+                         bucket_name: str = None):
         
-        project_id = get_gcp_project()
+        project_id = project_id or get_gcp_project()
+        log.info(f"Creating extension within {project_id=}")
         extension_name = f"projects/{project_id}/locations/us-central1/extensions/{validate_extension_id(display_name)}"
 
+        if bucket_name:
+            log.info(f"Setting extension bucket name to {bucket_name}")
+            self.bucket_name = bucket_name
+
         listed_extensions = self.list_extensions()
+        log.info(f"Listing extensions:\n {listed_extensions}")
         for ext in listed_extensions:
-            if ext.get('resource_name') == extension_name:
-                raise NameError(f"resouce_name {extension_name} already exists.  Delete it or rename your new extension")
+            if ext.get('display_name') == display_name:
+                raise NameError(f"display_name {display_name} already exists.  Delete it or rename your new extension")
 
         if open_api_file:
             self.upload_openapi_file(open_api_file)
@@ -233,7 +247,7 @@ class VertexAIExtensions:
 
         return extension.resource_name
 
-    def execute_extension(self, operation_id: str, operation_params: dict, extension_id: str=None):
+    def execute_extension(self, operation_id: str, operation_params: dict, extension_id: str=None, project_id: str=None):
         init_vertex(location=self.location)
 
         if not extension_id:
@@ -243,7 +257,7 @@ class VertexAIExtensions:
         else:  
             extension_id = str(extension_id)
             if not extension_id.startswith("projects/"):
-                project_id = get_gcp_project()
+                project_id = project_id or get_gcp_project()
                 extension_name = f"projects/{project_id}/locations/{self.location}/extensions/{extension_id}"
             else:
                 extension_name = extension_id

{sunholo-0.75.1 → sunholo-0.76.2}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.75.1
+Version: 0.76.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.75.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.76.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -25,6 +25,7 @@ Requires-Dist: langchain_experimental>0.0.60
 Requires-Dist: langchain-community
 Provides-Extra: all
 Requires-Dist: asyncpg; extra == "all"
+Requires-Dist: azure-storage-blob; extra == "all"
 Requires-Dist: fastapi; extra == "all"
 Requires-Dist: flask; extra == "all"
 Requires-Dist: google-auth; extra == "all"
@@ -69,6 +70,8 @@ Requires-Dist: tabulate; extra == "all"
 Requires-Dist: tantivy; extra == "all"
 Requires-Dist: tiktoken; extra == "all"
 Requires-Dist: unstructured[local-inference]==0.14.9; extra == "all"
+Provides-Extra: azure
+Requires-Dist: azure-storage-blob; extra == "azure"
 Provides-Extra: cli
 Requires-Dist: jsonschema>=4.21.1; extra == "cli"
 Requires-Dist: rich; extra == "cli"

{sunholo-0.75.1 → sunholo-0.76.2}/sunholo.egg-info/SOURCES.txt

@@ -31,18 +31,22 @@ sunholo/archive/archive.py
 sunholo/auth/__init__.py
 sunholo/auth/gcloud.py
 sunholo/auth/run.py
+sunholo/azure/__init__.py
+sunholo/azure/event_grid.py
 sunholo/bots/__init__.py
 sunholo/bots/discord.py
 sunholo/bots/github_webhook.py
 sunholo/bots/webapp.py
 sunholo/chunker/__init__.py
-sunholo/chunker/data_to_embed_pubsub.py
+sunholo/chunker/azure.py
 sunholo/chunker/doc_handling.py
 sunholo/chunker/images.py
 sunholo/chunker/loaders.py
 sunholo/chunker/message_data.py
 sunholo/chunker/pdfs.py
+sunholo/chunker/process_chunker_data.py
 sunholo/chunker/publish.py
+sunholo/chunker/pubsub.py
 sunholo/chunker/splitter.py
 sunholo/cli/__init__.py
 sunholo/cli/chat_vac.py
@@ -55,6 +59,7 @@ sunholo/cli/merge_texts.py
 sunholo/cli/run_proxy.py
 sunholo/cli/sun_rich.py
 sunholo/cli/swagger.py
+sunholo/cli/vertex.py
 sunholo/components/__init__.py
 sunholo/components/llm.py
 sunholo/components/retriever.py
@@ -130,5 +135,4 @@ sunholo/vertex/init.py
 sunholo/vertex/memory_tools.py
 sunholo/vertex/safety.py
 tests/test_chat_history.py
-tests/test_chunker.py
 tests/test_config.py

{sunholo-0.75.1 → sunholo-0.76.2}/sunholo.egg-info/requires.txt

@@ -6,6 +6,7 @@ langchain-community
 
 [all]
 asyncpg
+azure-storage-blob
 fastapi
 flask
 google-auth
@@ -54,6 +55,9 @@ unstructured[local-inference]==0.14.9
 [anthropic]
 langchain-anthropic>=0.1.13
 
+[azure]
+azure-storage-blob
+
 [cli]
 jsonschema>=4.21.1
 rich

sunholo-0.75.1/sunholo/chunker/__init__.py

@@ -1 +0,0 @@
-from .data_to_embed_pubsub import data_to_embed_pubsub, direct_file_to_embed

sunholo-0.75.1/tests/test_chunker.py

@@ -1,23 +0,0 @@
-import pytest
-from unittest.mock import patch, MagicMock
-from sunholo.chunker.data_to_embed_pubsub import data_to_embed_pubsub
-
-# Mock external calls within the function
-@patch('sunholo.chunker.data_to_embed_pubsub.process_pubsub_message', return_value=({}, {}, 'test_vector'))
-@patch('sunholo.chunker.data_to_embed_pubsub.process_chunker_data', return_value='processed_data')
-def test_data_to_embed_pubsub(mock_process_chunker_data, mock_process_pubsub_message):
-    # Test the function with various inputs including edge cases
-    assert data_to_embed_pubsub({}) == 'processed_data'
-    assert data_to_embed_pubsub({'key': 'value'}) == 'processed_data'
-    mock_process_pubsub_message.assert_called()
-    mock_process_chunker_data.assert_called()
-
-    # Ensure tests are self-contained and do not require external dependencies
-    mock_process_pubsub_message = MagicMock(return_value=({}, {}, 'test_vector'))
-    mock_process_chunker_data = MagicMock(return_value='processed_data')
-    assert data_to_embed_pubsub({'key': 'value'}) == 'processed_data'
-
-    # Validate the function's output against expected results
-    expected_output = 'processed_data'
-    actual_output = data_to_embed_pubsub({'key': 'value'})
-    assert actual_output == expected_output, f"Expected {expected_output}, got {actual_output}"