trustgraph-cli 2.2.14__tar.gz → 2.2.16__tar.gz

{trustgraph_cli-2.2.14 → trustgraph_cli-2.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-cli
-Version: 2.2.14
+Version: 2.2.16
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_cli-2.2.14 → trustgraph_cli-2.2.16}/trustgraph/cli/dump_queues.py

@@ -8,8 +8,6 @@ message flows, diagnosing stuck services, and understanding system behavior.
 Uses TrustGraph's Subscriber abstraction for future-proof pub/sub compatibility.
 """
 
-import pulsar
-from pulsar.schema import BytesSchema
 import sys
 import json
 import asyncio
@@ -17,7 +15,7 @@ from datetime import datetime
 import argparse
 
 from trustgraph.base.subscriber import Subscriber
-from trustgraph.base.pubsub import get_pubsub
+from trustgraph.base.pubsub import get_pubsub, add_pubsub_args
 
 def decode_json_strings(obj):
     """Recursively decode JSON-encoded string values within a dict/list."""
@@ -172,15 +170,13 @@ async def log_writer(central_queue, file_handle, shutdown_event, console_output=
                 break
 
 
-async def async_main(queues, output_file, pulsar_host, listener_name, subscriber_name, append_mode):
+async def async_main(queues, output_file, subscriber_name, append_mode, **pubsub_config):
     """
     Main async function to monitor multiple queues concurrently.
 
     Args:
         queues: List of queue names to monitor
         output_file: Path to output file
-        pulsar_host: Pulsar connection URL
-        listener_name: Pulsar listener name
         subscriber_name: Base name for subscribers
         append_mode: Whether to append to existing file
     """
@@ -194,9 +190,9 @@ async def async_main(queues, output_file, pulsar_host, listener_name, subscriber
 
     # Create backend connection
     try:
-        backend = get_pubsub(pulsar_host=pulsar_host, pulsar_listener=listener_name, pubsub_backend='pulsar')
+        backend = get_pubsub(**pubsub_config)
     except Exception as e:
-        print(f"Error connecting to backend at {pulsar_host}: {e}", file=sys.stderr)
+        print(f"Error connecting to backend: {e}", file=sys.stderr)
         sys.exit(1)
 
     # Create Subscribers and central queue
@@ -291,25 +287,20 @@ def main():
         description='Monitor and dump messages from multiple Pulsar queues',
         epilog="""
 Examples:
-  # Monitor agent and prompt queues
-  tg-dump-queues non-persistent://tg/request/agent:default \\
-                 non-persistent://tg/request/prompt:default
+  # Monitor agent and prompt flow queues
+  tg-dump-queues flow:tg:agent-request:default \\
+                 flow:tg:prompt-request:default
 
   # Monitor with custom output file
-  tg-dump-queues non-persistent://tg/request/agent:default \\
+  tg-dump-queues flow:tg:agent-request:default \\
                  --output debug.log
 
   # Append to existing log file
-  tg-dump-queues non-persistent://tg/request/agent:default \\
+  tg-dump-queues flow:tg:agent-request:default \\
                  --output queue.log --append
 
-Common queue patterns:
-  - Agent requests:   non-persistent://tg/request/agent:default
-  - Agent responses:  non-persistent://tg/response/agent:default
-  - Prompt requests:  non-persistent://tg/request/prompt:default
-  - Prompt responses: non-persistent://tg/response/prompt:default
-  - LLM requests:     non-persistent://tg/request/text-completion:default
-  - LLM responses:    non-persistent://tg/response/text-completion:default
+  # Raw Pulsar URIs also accepted
+  tg-dump-queues persistent://tg/flow/agent-request:default
 
 IMPORTANT:
   This tool subscribes to queues without a schema (schema-less mode). To avoid
@@ -340,17 +331,7 @@ IMPORTANT:
         help='Append to output file instead of overwriting'
     )
 
-    parser.add_argument(
-        '--pulsar-host',
-        default='pulsar://localhost:6650',
-        help='Pulsar host URL (default: pulsar://localhost:6650)'
-    )
-
-    parser.add_argument(
-        '--listener-name',
-        default='localhost',
-        help='Pulsar listener name (default: localhost)'
-    )
+    add_pubsub_args(parser, standalone=True)
 
     parser.add_argument(
         '--subscriber',
@@ -371,10 +352,10 @@ IMPORTANT:
         asyncio.run(async_main(
             queues=queues,
             output_file=args.output,
-            pulsar_host=args.pulsar_host,
-            listener_name=args.listener_name,
             subscriber_name=args.subscriber,
-            append_mode=args.append
+            append_mode=args.append,
+            **{k: v for k, v in vars(args).items()
+               if k not in ('queues', 'output', 'subscriber', 'append')},
         ))
     except KeyboardInterrupt:
         # Already handled in async_main

{trustgraph_cli-2.2.14 → trustgraph_cli-2.2.16}/trustgraph/cli/init_trustgraph.py

@@ -1,5 +1,8 @@
 """
-Initialises Pulsar with Trustgraph tenant / namespaces & policy.
+Initialises TrustGraph pub/sub infrastructure and pushes initial config.
+
+For Pulsar: creates tenant, namespaces, and retention policies.
+For RabbitMQ: queues are auto-declared, so only config push is needed.
 """
 
 import requests
@@ -8,10 +11,11 @@ import argparse
 import json
 
 from trustgraph.clients.config_client import ConfigClient
+from trustgraph.base.pubsub import add_pubsub_args
 
 default_pulsar_admin_url = "http://pulsar:8080"
-default_pulsar_host = "pulsar://pulsar:6650"
-subscriber = "tg-init-pulsar"
+subscriber = "tg-init-pubsub"
+
 
 def get_clusters(url):
 
@@ -65,12 +69,11 @@ def ensure_namespace(url, tenant, namespace, config):
 
     print(f"Namespace {tenant}/{namespace} created.", flush=True)
 
-def ensure_config(config, pulsar_host, pulsar_api_key):
+def ensure_config(config, **pubsub_config):
 
     cli = ConfigClient(
         subscriber=subscriber,
-        pulsar_host=pulsar_host,
-        pulsar_api_key=pulsar_api_key,
+        **pubsub_config,
     )
 
     while True:
@@ -115,11 +118,9 @@ def ensure_config(config, pulsar_host, pulsar_api_key):
             time.sleep(2)
             print("Retrying...", flush=True)
             continue
-    
-def init(
-        pulsar_admin_url, pulsar_host, pulsar_api_key, tenant,
-        config, config_file,
-):
+
+def init_pulsar(pulsar_admin_url, tenant):
+    """Pulsar-specific setup: create tenant, namespaces, retention policies."""
 
     clusters = get_clusters(pulsar_admin_url)
 
@@ -137,7 +138,7 @@ def init(
         }
     })
 
-    ensure_namespace(pulsar_admin_url, tenant, "config", {
+    ensure_namespace(pulsar_admin_url, tenant, "state", {
         "retention_policies": {
             "retentionSizeInMB": 10,
             "retentionTimeInMinutes": -1,
@@ -145,17 +146,21 @@ def init(
         }
     })
 
-    if config is not None:
+
+def push_config(config_json, config_file, **pubsub_config):
+    """Push initial config if provided."""
+
+    if config_json is not None:
 
         try:
             print("Decoding config...", flush=True)
-            dec = json.loads(config)
+            dec = json.loads(config_json)
             print("Decoded.", flush=True)
         except Exception as e:
             print("Exception:", e, flush=True)
             raise e
 
-        ensure_config(dec, pulsar_host, pulsar_api_key)
+        ensure_config(dec, **pubsub_config)
 
     elif config_file is not None:
 
@@ -167,11 +172,12 @@ def init(
             print("Exception:", e, flush=True)
             raise e
 
-        ensure_config(dec, pulsar_host, pulsar_api_key)
+        ensure_config(dec, **pubsub_config)
 
     else:
         print("No config to update.", flush=True)
 
+
 def main():
 
     parser = argparse.ArgumentParser(
@@ -180,22 +186,11 @@ def main():
     )
 
     parser.add_argument(
-        '-p', '--pulsar-admin-url',
+        '--pulsar-admin-url',
         default=default_pulsar_admin_url,
         help=f'Pulsar admin URL (default: {default_pulsar_admin_url})',
     )
 
-    parser.add_argument(
-        '--pulsar-host',
-        default=default_pulsar_host,
-        help=f'Pulsar host (default: {default_pulsar_host})',
-    )
-
-    parser.add_argument(
-        '--pulsar-api-key',
-        help=f'Pulsar API key',
-    )
-
     parser.add_argument(
         '-c', '--config',
         help=f'Initial configuration to load',
@@ -212,18 +207,43 @@ def main():
         help=f'Tenant (default: tg)',
     )
 
+    add_pubsub_args(parser)
+
     args = parser.parse_args()
 
+    backend_type = args.pubsub_backend
+
+    # Extract pubsub config from args
+    pubsub_config = {
+        k: v for k, v in vars(args).items()
+        if k not in ('pulsar_admin_url', 'config', 'config_file', 'tenant')
+    }
+
     while True:
 
         try:
 
-            print(flush=True)
-            print(
-                f"Initialising with Pulsar {args.pulsar_admin_url}...",
-                flush=True
+            # Pulsar-specific setup (tenants, namespaces)
+            if backend_type == 'pulsar':
+                print(flush=True)
+                print(
+                    f"Initialising Pulsar at {args.pulsar_admin_url}...",
+                    flush=True,
+                )
+                init_pulsar(args.pulsar_admin_url, args.tenant)
+            else:
+                print(flush=True)
+                print(
+                    f"Using {backend_type} backend (no admin setup needed).",
+                    flush=True,
+                )
+
+            # Push config (works with any backend)
+            push_config(
+                args.config, args.config_file,
+                **pubsub_config,
             )
-            init(**vars(args))
+
             print("Initialisation complete.", flush=True)
             break
 
@@ -236,4 +256,4 @@ def main():
             print("Will retry...", flush=True)
 
 if __name__ == "__main__":
-    main()
+    main()

{trustgraph_cli-2.2.14 → trustgraph_cli-2.2.16}/trustgraph/cli/monitor_prompts.py

@@ -1,7 +1,7 @@
 """
 Monitor prompt request/response queues and log activity with timing.
 
-Subscribes to prompt request and response Pulsar queues, correlates
+Subscribes to prompt request and response queues, correlates
 them by message ID, and logs a summary of each request/response with
 elapsed time. Streaming responses are accumulated and shown once at
 completion.
@@ -19,8 +19,7 @@ import argparse
 from datetime import datetime
 from collections import OrderedDict
 
-import pulsar
-from pulsar.schema import BytesSchema
+from trustgraph.base.pubsub import get_pubsub, add_pubsub_args
 
 
 default_flow = "default"
@@ -85,7 +84,7 @@ def format_terms(terms, max_lines, max_width):
 
 
 def parse_raw_message(msg):
-    """Parse a raw Pulsar message into (correlation_id, body_dict)."""
+    """Parse a raw message into (correlation_id, body_dict)."""
     try:
         props = msg.properties()
         corr_id = props.get("id", "")
@@ -94,53 +93,46 @@ def parse_raw_message(msg):
 
     try:
         value = msg.value()
-        if isinstance(value, bytes):
-            value = value.decode("utf-8")
-        body = json.loads(value) if isinstance(value, str) else {}
+        if isinstance(value, dict):
+            body = value
+        elif isinstance(value, bytes):
+            body = json.loads(value.decode("utf-8"))
+        elif isinstance(value, str):
+            body = json.loads(value)
+        else:
+            body = {}
     except Exception:
         body = {}
 
     return corr_id, body
 
 
-def receive_with_timeout(consumer, timeout_ms=500):
-    """Receive a message with timeout, returning None on timeout."""
-    try:
-        return consumer.receive(timeout_millis=timeout_ms)
-    except Exception:
-        return None
-
-
-async def monitor(flow, queue_type, max_lines, max_width,
-                  pulsar_host, listener_name):
+async def monitor(flow, queue_type, max_lines, max_width, **config):
 
-    request_queue = f"non-persistent://tg/request/{queue_type}:{flow}"
-    response_queue = f"non-persistent://tg/response/{queue_type}:{flow}"
+    request_queue = f"request:tg:{queue_type}:{flow}"
+    response_queue = f"response:tg:{queue_type}:{flow}"
 
     print(f"Monitoring prompt queues:")
     print(f"  Request:  {request_queue}")
     print(f"  Response: {response_queue}")
     print(f"Press Ctrl+C to stop\n")
 
-    client = pulsar.Client(
-        pulsar_host,
-        listener_name=listener_name,
-    )
+    backend = get_pubsub(**config)
 
-    req_consumer = client.subscribe(
-        request_queue,
-        subscription_name="prompt-monitor-req",
-        consumer_type=pulsar.ConsumerType.Shared,
-        schema=BytesSchema(),
-        initial_position=pulsar.InitialPosition.Latest,
+    req_consumer = backend.create_consumer(
+        topic=request_queue,
+        subscription="prompt-monitor-req",
+        schema=None,
+        consumer_type='shared',
+        initial_position='latest',
     )
 
-    resp_consumer = client.subscribe(
-        response_queue,
-        subscription_name="prompt-monitor-resp",
-        consumer_type=pulsar.ConsumerType.Shared,
-        schema=BytesSchema(),
-        initial_position=pulsar.InitialPosition.Latest,
+    resp_consumer = backend.create_consumer(
+        topic=response_queue,
+        subscription="prompt-monitor-resp",
+        schema=None,
+        consumer_type='shared',
+        initial_position='latest',
     )
 
     # Track in-flight requests: corr_id -> (timestamp, template_id)
@@ -156,8 +148,8 @@ async def monitor(flow, queue_type, max_lines, max_width,
             got_message = False
 
             # Poll request queue
-            msg = receive_with_timeout(req_consumer, 100)
-            if msg:
+            try:
+                msg = req_consumer.receive(timeout_millis=100)
                 got_message = True
                 timestamp = datetime.now()
                 corr_id, body = parse_raw_message(msg)
@@ -182,10 +174,12 @@ async def monitor(flow, queue_type, max_lines, max_width,
                     print(format_terms(terms, max_lines, max_width))
 
                 req_consumer.acknowledge(msg)
+            except TimeoutError:
+                pass
 
             # Poll response queue
-            msg = receive_with_timeout(resp_consumer, 100)
-            if msg:
+            try:
+                msg = resp_consumer.receive(timeout_millis=100)
                 got_message = True
                 timestamp = datetime.now()
                 corr_id, body = parse_raw_message(msg)
@@ -265,6 +259,8 @@ async def monitor(flow, queue_type, max_lines, max_width,
                         print(f"  {truncated}")
 
                 resp_consumer.acknowledge(msg)
+            except TimeoutError:
+                pass
 
             if not got_message:
                 await asyncio.sleep(0.05)
@@ -274,7 +270,7 @@ async def monitor(flow, queue_type, max_lines, max_width,
     finally:
         req_consumer.close()
         resp_consumer.close()
-        client.close()
+        backend.close()
 
 
 def main():
@@ -310,17 +306,7 @@ def main():
         help=f"Max width per line (default: {default_max_width})",
     )
 
-    parser.add_argument(
-        "--pulsar-host",
-        default="pulsar://localhost:6650",
-        help="Pulsar host URL (default: pulsar://localhost:6650)",
-    )
-
-    parser.add_argument(
-        "--listener-name",
-        default="localhost",
-        help="Pulsar listener name (default: localhost)",
-    )
+    add_pubsub_args(parser, standalone=True)
 
     args = parser.parse_args()
 
@@ -330,8 +316,8 @@ def main():
             queue_type=args.queue_type,
             max_lines=args.max_lines,
             max_width=args.max_width,
-            pulsar_host=args.pulsar_host,
-            listener_name=args.listener_name,
+            **{k: v for k, v in vars(args).items()
+               if k not in ('flow', 'queue_type', 'max_lines', 'max_width')},
         ))
     except KeyboardInterrupt:
         pass

trustgraph_cli-2.2.16/trustgraph/cli_version.py

@@ -0,0 +1 @@
+__version__ = "2.2.16"

{trustgraph_cli-2.2.14 → trustgraph_cli-2.2.16}/trustgraph_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-cli
-Version: 2.2.14
+Version: 2.2.16
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

trustgraph_cli-2.2.14/trustgraph/cli_version.py

@@ -1 +0,0 @@
-__version__ = "2.2.14"