sunholo 0.61.5__py3-none-any.whl → 0.61.6__py3-none-any.whl

sunholo/chunker/data_to_embed_pubsub.py

@@ -25,6 +25,10 @@ def data_to_embed_pubsub(data: dict):
 
     message_data, metadata, vector_name = process_pubsub_message(data)
 
+    return process_chunker_data(message_data, metadata, vector_name)
+
+def process_chunker_data(message_data, metadata, vector_name):
+
     if metadata:
         metadata["vector_name"] = vector_name
 

sunholo/chunker/message_data.py

@@ -186,7 +186,7 @@ def handle_json_content_message(message_data: str, metadata: dict, vector_name:
 
     if the_content is None:
         log.info("No content found")
-        return {"metadata": "No content found"}
+        return {"metadata": "No content found in 'page_content' JSON field"}
     
     docs = [Document(page_content=the_content, metadata=metadata)]
 

sunholo/cli/chat_vac.py

@@ -20,14 +20,19 @@ from rich.text import Text
 from rich.table import Table
 
 
-def get_service_url(vac_name, project, region):
-    agent_name = load_config_key("agent", vac_name, kind="vacConfig")
+def get_service_url(vac_name, project, region, no_config=False):
+
+    if no_config:
+        agent_name = vac_name
+    else:
+        agent_name = load_config_key("agent", vac_name, kind="vacConfig")
+
     proxies = clean_proxy_list()
     if agent_name in proxies:
         port = proxies[agent_name]['port']
         url = f"http://127.0.0.1:{port}"
     else:
-        print(f"No proxy found running for service: {agent_name} required for {vac_name} - attempting to connect")
+        console.print(f"No proxy found running for service: [bold orange]'{agent_name}[/bold orange] required for [bold orange]{vac_name}[/bold orange] - attempting to connect")
         url = start_proxy(agent_name, region, project)
 
     return url
@@ -144,15 +149,17 @@ def headless_mode(service_url, service_name, user_input, chat_history=None):
 
     return chat_history
 
-def resolve_service_url(args):
-
+def resolve_service_url(args, no_config=False):
+    """
+    no_config: some VACs do not have an entry in the config file e.g. chunker, embedder etc.
+    """
     if args.url_override:
 
         return args.url_override
 
     if not args.no_proxy:
         try:
-            service_url = get_service_url(args.vac_name, args.project, args.region)
+            service_url = get_service_url(args.vac_name, args.project, args.region, no_config=no_config)
         except ValueError as e:
             console.print(f"[bold red]ERROR: Could not start {args.vac_name} proxy URL: {str(e)}[/bold red]")
             sys.exit(1)
@@ -169,8 +176,6 @@ def resolve_service_url(args):
 
 def vac_command(args):
 
-    service_url = resolve_service_url(args)
-
     if args.action == 'list':
 
         list_cloud_run_services(args.project, args.region)
@@ -178,13 +183,13 @@ def vac_command(args):
         return
     
     elif args.action == 'get-url':
-
+        service_url = resolve_service_url(args)
         console.print(service_url)
 
         return
     
     elif args.action == 'chat':
-    
+        service_url = resolve_service_url(args)
         agent_name   = load_config_key("agent", args.vac_name, kind="vacConfig")
 
         if args.headless:
@@ -209,6 +214,7 @@ def vac_command(args):
         stop_proxy(agent_name, stop_local=False)
 
     elif args.action == 'invoke':
+        service_url = resolve_service_url(args, no_config=True)
         try:
             json_data = json.loads(args.data)
         except json.JSONDecodeError as err:
@@ -222,7 +228,12 @@ def invoke_vac(service_url, data):
         headers = {"Content-Type": "application/json"}
         response = requests.post(service_url, headers=headers, data=json.dumps(data))
         response.raise_for_status()
-        print(response.json())
+
+        the_data = response.json()
+        console.print(the_data)
+
+        return the_data
+    
     except requests.exceptions.RequestException as e:
         console.print(f"[bold red]ERROR: Failed to invoke VAC: {e}[/bold red]")
 

sunholo/cli/cli.py

@@ -7,6 +7,8 @@ from .cli_init import setup_init_subparser
 from .merge_texts import setup_merge_text_subparser
 from .run_proxy import setup_proxy_subparser
 from .chat_vac import setup_vac_subparser
+from .embedder import setup_embedder_subparser
+
 from ..utils.config import load_config_key
 
 from ..logging import log
@@ -64,6 +66,8 @@ def main(args=None):
     setup_proxy_subparser(subparsers)
     # vac command
     setup_vac_subparser(subparsers)
+    # embed command
+    setup_embedder_subparser(subparsers)
 
     args = parser.parse_args(args)
 

sunholo/cli/embedder.py

@@ -0,0 +1,148 @@
+import json
+import uuid
+import base64
+from datetime import datetime, timezone
+from argparse import Namespace
+
+from .sun_rich import console
+from rich.progress import Progress
+
+from .chat_vac import resolve_service_url, invoke_vac
+
+def encode_data(vac, content, metadata=None, local_chunks=False):
+    # Current time in UTC
+    now_utc = datetime.now(timezone.utc)
+    formatted_time = now_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    # Default metadata if none provided
+    default_metadata = {"vector_name": vac, "source": "sunholo-cli", "eventTime": formatted_time}
+
+    # Merge default metadata with provided metadata
+    if metadata:
+        if not isinstance(metadata, dict):
+            metadata = json.loads(metadata)
+    else:
+        metadata = {}
+
+    # Update metadata with default values if not present
+    metadata.update(default_metadata)
+
+    # Encode the content (URL)
+    if isinstance(content, str):
+        message_data = base64.b64encode(content.encode('utf-8')).decode('utf-8')
+    else:
+        raise ValueError(f"Unsupported content type: {type(content)}")
+
+    # Construct the message dictionary
+    messageId = str(uuid.uuid4())
+    message = {
+        "message": {
+            "data": message_data,
+            "messageId": messageId,
+            "publishTime": formatted_time,
+            "attributes": {
+                "namespace": vac,
+                "return_chunks": str(local_chunks).lower()
+            },
+        }
+    }
+
+    # Merge metadata with attributes
+    message["message"]["attributes"].update(metadata)
+
+    #console.print()
+    #console.print(f"Sending message: {messageId} with metadata:")
+    #console.print(f"{message['message']['attributes']}")
+
+    return message
+
+def embed_command(args):
+    chunk_args = vars(args).copy()
+    embed_args = vars(args).copy()
+
+    console.rule("Sending data for chunking")
+
+    if args.chunk_override:
+        chunk_args["url_override"] = args.chunk_override
+    else:
+        chunk_args["vac_name"] = "chunker"
+        chunk_args["url_override"] = ""
+    chunk_args = Namespace(**chunk_args)
+    chunk_url = resolve_service_url(chunk_args, no_config=True)
+
+    json_data = encode_data(args.vac_name, args.data, args.metadata, args.local_chunks)
+
+    with console.status(f"[bold orange]Sending {args.data} to chunk via {chunk_url}[/bold orange]", spinner="star"):
+        chunk_res = invoke_vac(f"{chunk_url}/pubsub_to_store", json_data)
+
+    if not args.local_chunks:
+        console.rule(f"Chunks sent for processing in cloud: {chunk_res}")
+
+        return
+    
+    console.rule("Processing chunks locally")
+
+    if args.embed_override:
+        embed_args["url_override"] = args.embed_override
+    else:
+        embed_args["vac_name"] = "embedder"
+        embed_args["url_override"] = ""
+    embed_args = Namespace(**embed_args)
+    embed_url = resolve_service_url(embed_args, no_config=True)
+    
+    if not chunk_res:
+        console.print(f"[bold red]ERROR: Did not get any chunks from {chunk_url} for {json_data}")
+
+        return
+
+    chunks = chunk_res.get('chunks')
+    if not chunks:
+        console.print(f"[bold red]ERROR: No chunks found within json data: {str(chunk_res)} [/bold red]")
+
+        return
+    
+    embeds = []
+    with Progress() as progress:
+        task = progress.add_task(f"Embedding [{len(chunks)}] chunks via {embed_url}", total=len(chunks))
+        for chunk in chunks:
+            progress.console.print(f"Working on chunk {chunk['metadata']}")
+
+            # do this async?
+            content = chunk.get("page_content")
+            now_utc = datetime.now(timezone.utc)
+            formatted_time = now_utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+            chunk["metadata"]["eventTime"]  = formatted_time
+            if not content:
+                progress.console.print("[bold red]No content chunk found, skipping.[/bold red]")
+                progress.advance(task)
+                continue
+            progress.console.print(f"Sending chunk length {len(content)} to embedder")
+            processed_chunk = encode_data(vac = args.vac_name, 
+                                          content = json.dumps(chunk))
+            
+            embed_res = invoke_vac(f"{embed_url}/embed_chunk", processed_chunk)
+            embeds.append(embed_res)
+            progress.advance(task)
+
+    console.rule("Embedding pipeline finished")
+    
+    return embed_res
+
+
+def setup_embedder_subparser(subparsers):
+    """
+    Sets up an argparse subparser for the 'embed' command.
+
+    Args:
+        subparsers: The subparsers object from argparse.ArgumentParser().
+    """
+    embed_parser = subparsers.add_parser('embed', help='Send data for embedding to a VAC vector store')
+    embed_parser.add_argument('--embed-override', help='Override the embed VAC service URL.')
+    embed_parser.add_argument('--chunk-override', help='Override the chunk VAC service URL.')
+    embed_parser.add_argument('--no-proxy', action='store_true', help='Do not use the proxy and connect directly to the VAC service.')
+    embed_parser.add_argument('-m', '--metadata', default=None, help='Metadata to send with the embedding (as JSON string).')
+    embed_parser.add_argument('--local-chunks',  action='store_true', help='Whether to process chunks to embed locally, or via the cloud.')
+    embed_parser.add_argument('vac_name', help='VAC service to embed the data for')
+    embed_parser.add_argument('data', help='String content to send for embedding')
+
+    embed_parser.set_defaults(func=embed_command)

{sunholo-0.61.5.dist-info → sunholo-0.61.6.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.61.5
+Version: 0.61.6
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.61.5.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.61.6.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.61.5.dist-info → sunholo-0.61.6.dist-info}/RECORD

@@ -22,20 +22,21 @@ sunholo/bots/discord.py,sha256=cCFae5K1BCa6JVkWGLh_iZ9qFO1JpXb6K4eJrlDfEro,2442
 sunholo/bots/github_webhook.py,sha256=5pQPRLM_wxxcILVaIzUDV8Kt7Arcm2dL1r1kMMHA524,9629
 sunholo/bots/webapp.py,sha256=EIMxdAJ_xtufwJmvnn7N_Fb_1hZ9DjhJ0Kf_hp02vEU,1926
 sunholo/chunker/__init__.py,sha256=UhQBZTKwDfBXm0TPv4LvsGc5pdUGCbYzi3qPTOkU4gw,55
-sunholo/chunker/data_to_embed_pubsub.py,sha256=t-pWNYv2mnwVAkMcIOK2CrIb3yr2aS9iAdtryk7hT8o,2931
+sunholo/chunker/data_to_embed_pubsub.py,sha256=IY9SBRA7IO77QJBEgQuO1FiSCd6Dfm-TMEf1Ey-pLoo,3065
 sunholo/chunker/doc_handling.py,sha256=rIyknpzDyj5A0u_DqSQVD_CXLRNZPOU6TCL4bhCdjOI,8563
 sunholo/chunker/images.py,sha256=Xmh1vwHrVhoXm5iH2dhCc52O8YgdzE8KrDSdL-pGnp8,1861
 sunholo/chunker/loaders.py,sha256=xiToUVgPz2ZzcqpUAq7aNP3PTenb_rBUAFzu0JPycIg,10268
-sunholo/chunker/message_data.py,sha256=iDP94dySU3Xct-gWGnB4NNRSh2luQmgJeCfQb7ktt3U,6760
+sunholo/chunker/message_data.py,sha256=X6aA4yX5aGN_mEvsDPWvdYRqqn5GO1BU9QhT9w5A0ec,6789
 sunholo/chunker/pdfs.py,sha256=daCZ1xjn1YvxlifIyxskWNpLJLe-Q9D_Jq12MWx3tZo,2473
 sunholo/chunker/publish.py,sha256=PoT8q3XJeFCg10WrLkYhuaaXIrGVkvUD3-R9IfoWoH4,2703
 sunholo/chunker/splitter.py,sha256=FLkDhkePkg_zGQpFBK13Cznw575D-Rf9pcaCpc1HUxY,6726
 sunholo/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sunholo/cli/chat_vac.py,sha256=hroQtcf12nYaHaYObE4EeNlHmcB4BZl6LBs_yhSnwYI,12713
-sunholo/cli/cli.py,sha256=cogY1F5rcIGFYpZVFtbDNlAIElpfyPSCvSLC1ZIpHXg,2666
+sunholo/cli/chat_vac.py,sha256=FEtaKQVYWenV1WKh4mfW7lTAaxOTvzmbG9MucG-UZoU,13179
+sunholo/cli/cli.py,sha256=HEuCRCxzwMPO0JtT3X1liGVyvEq20Mb1-SlKn7HPx6E,2775
 sunholo/cli/cli_init.py,sha256=JMZ9AX2cPDZ-_mv3adiv2ToFVNyRPtjk9Biszl1kiR0,2358
 sunholo/cli/configs.py,sha256=QUM9DvKOdZmEQRM5uI3Nh887T0YDiSMr7O240zTLqws,4546
 sunholo/cli/deploy.py,sha256=zxdwUsRTRMC8U5vyRv0JiKBLFn84Ug_Tc88-_h9hJSs,1609
+sunholo/cli/embedder.py,sha256=hqIfqGCeV5UI_0dllNFsjdyjVWgC0Kmnw8kAKhN4jCI,5482
 sunholo/cli/merge_texts.py,sha256=U9vdMwKmcPoc6iPOWX5MKSxn49dNGbNzVLw8ui5PhEU,1823
 sunholo/cli/run_proxy.py,sha256=9ILCxSVHPzS-cSBvjdHhfZFlwsJ4Ttmu0vLtNoPCRgo,11469
 sunholo/cli/sun_rich.py,sha256=UpMqeJ0C8i0pkue1AHnnyyX0bFJ9zZeJ7HBR6yhuA8A,54
@@ -95,9 +96,9 @@ sunholo/utils/parsers.py,sha256=OrHmASqIbI45atVOhiGodgLvnfrzkvVzyHnSvAXD89I,3841
 sunholo/utils/user_ids.py,sha256=SQd5_H7FE7vcTZp9AQuQDWBXd4FEEd7TeVMQe1H4Ny8,292
 sunholo/vertex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/vertex/init_vertex.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
-sunholo-0.61.5.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.61.5.dist-info/METADATA,sha256=tJ2thVA-C3VH57sGLxsCI1dIVZ-H9gmxX-x8bzvaeQY,8057
-sunholo-0.61.5.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-sunholo-0.61.5.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.61.5.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.61.5.dist-info/RECORD,,
+sunholo-0.61.6.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.61.6.dist-info/METADATA,sha256=Hoo-nKG0luMJ4XKms0VT9vm6LhE7ZGskG3nv3tfkpjg,8057
+sunholo-0.61.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+sunholo-0.61.6.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.61.6.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.61.6.dist-info/RECORD,,