trustgraph-cli 2.8.0__tar.gz → 2.8.2__tar.gz

{trustgraph_cli-2.8.0 → trustgraph_cli-2.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-cli
-Version: 2.8.0
+Version: 2.8.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph

{trustgraph_cli-2.8.0 → trustgraph_cli-2.8.2}/trustgraph/cli/dump_queues.py

@@ -1,11 +1,11 @@
 """
-Multi-queue Pulsar message dumper for debugging TrustGraph message flows.
+Multi-queue message dumper for debugging TrustGraph message flows.
 
-This utility monitors multiple Pulsar queues simultaneously and logs all messages
+This utility monitors multiple queues simultaneously and logs all messages
 to a file with timestamps and pretty-printed formatting. Useful for debugging
 message flows, diagnosing stuck services, and understanding system behavior.
 
-Uses TrustGraph's Subscriber abstraction for future-proof pub/sub compatibility.
+Uses TrustGraph's async pub/sub backend for future-proof compatibility.
 """
 
 import sys
@@ -14,8 +14,7 @@ import asyncio
 from datetime import datetime
 import argparse
 
-from trustgraph.base.subscriber import Subscriber
-from trustgraph.base.pubsub import get_pubsub, add_pubsub_args
+from trustgraph.base.pubsub import get_async_pubsub, add_pubsub_args
 
 def decode_json_strings(obj):
     """Recursively decode JSON-encoded string values within a dict/list."""
@@ -54,7 +53,7 @@ def to_dict(value):
     if isinstance(value, (list, tuple)):
         return [to_dict(v) for v in value]
 
-    # Pulsar schema objects expose fields via __dict__
+    # Schema objects expose fields via __dict__
     if hasattr(value, '__dict__'):
         return {
             k: to_dict(v) for k, v in value.__dict__.items()
@@ -87,31 +86,27 @@ def format_message(queue_name, msg):
     return header + body + "\n"
 
 
-async def monitor_queue(subscriber, queue_name, central_queue, monitor_id, shutdown_event):
+async def monitor_queue(consumer, queue_name, central_queue, shutdown_event):
     """
-    Monitor a single queue via Subscriber and forward messages to central queue.
+    Monitor a single queue via async consumer and forward messages to central queue.
 
     Args:
-        subscriber: Subscriber instance for this queue
+        consumer: Async consumer instance for this queue
         queue_name: Name of the queue (for logging)
         central_queue: asyncio.Queue to forward messages to
-        monitor_id: Unique ID for this monitor's subscription
         shutdown_event: asyncio.Event to signal shutdown
     """
-    msg_queue = None
     try:
-        # Subscribe to all messages from this Subscriber
-        msg_queue = await subscriber.subscribe_all(monitor_id)
-
         while not shutdown_event.is_set():
             try:
-                # Read from Subscriber's internal queue with timeout
-                msg = await asyncio.wait_for(msg_queue.get(), timeout=0.5)
+                msg = await asyncio.wait_for(consumer.receive(), timeout=0.5)
                 timestamp = datetime.now()
                 formatted = format_message(queue_name, msg)
 
                 # Forward to central queue for writing
                 await central_queue.put((timestamp, queue_name, formatted))
+
+                await consumer.acknowledge(msg)
             except asyncio.TimeoutError:
                 # No message, check shutdown flag again
                 continue
@@ -120,13 +115,6 @@ async def monitor_queue(subscriber, queue_name, central_queue, monitor_id, shutd
         if not shutdown_event.is_set():
             error_msg = f"\n{'='*80}\n[{datetime.now().isoformat()}] ERROR in monitor for {queue_name}\n{'='*80}\n{e}\n"
             await central_queue.put((datetime.now(), queue_name, error_msg))
-    finally:
-        # Clean unsubscribe
-        if msg_queue is not None:
-            try:
-                await subscriber.unsubscribe_all(monitor_id)
-            except Exception:
-                pass
 
 
 async def log_writer(central_queue, file_handle, shutdown_event, console_output=True):
@@ -190,33 +178,31 @@ async def async_main(queues, output_file, subscriber_name, append_mode, **pubsub
 
     # Create backend connection
     try:
-        backend = get_pubsub(**pubsub_config)
+        backend = get_async_pubsub(**pubsub_config)
     except Exception as e:
         print(f"Error connecting to backend: {e}", file=sys.stderr)
         sys.exit(1)
 
-    # Create Subscribers and central queue
+    # Create consumers and central queue
     central_queue = asyncio.Queue()
-    subscribers = []
+    consumers = []
 
     for queue_name in queues:
         try:
-            sub = Subscriber(
-                backend=backend,
+            consumer = await backend.create_consumer(
                 topic=queue_name,
                 subscription=subscriber_name,
-                consumer_name=f"{subscriber_name}-{queue_name}",
-                schema=None,  # No schema - accept any message type
+                schema=None,
+                initial_position='latest',
             )
-            await sub.start()
-            subscribers.append((queue_name, sub))
-            print(f"✓ Subscribed to: {queue_name}")
+            consumers.append((queue_name, consumer))
+            print(f"  Subscribed to: {queue_name}")
         except Exception as e:
-            print(f"✗ Error subscribing to {queue_name}: {e}", file=sys.stderr)
+            print(f"  Error subscribing to {queue_name}: {e}", file=sys.stderr)
 
-    if not subscribers:
-        print("\nNo subscribers created. Exiting.", file=sys.stderr)
-        backend.close()
+    if not consumers:
+        print("\nNo consumers created. Exiting.", file=sys.stderr)
+        await backend.close()
         sys.exit(1)
 
     print(f"\nListening for messages...\n")
@@ -237,10 +223,10 @@ async def async_main(queues, output_file, subscriber_name, append_mode, **pubsub
             # Start monitoring tasks
             tasks = []
             try:
-                # Create one monitor task per subscriber
-                for queue_name, sub in subscribers:
+                # Create one monitor task per consumer
+                for queue_name, consumer in consumers:
                     task = asyncio.create_task(
-                        monitor_queue(sub, queue_name, central_queue, "logger", shutdown_event)
+                        monitor_queue(consumer, queue_name, central_queue, shutdown_event)
                     )
                     tasks.append(task)
 
@@ -274,17 +260,20 @@ async def async_main(queues, output_file, subscriber_name, append_mode, **pubsub
         print(f"Error writing to {output_file}: {e}", file=sys.stderr)
         sys.exit(1)
     finally:
-        # Clean shutdown of Subscribers
-        for _, sub in subscribers:
-            await sub.stop()
-        backend.close()
+        # Clean shutdown of consumers
+        for _, consumer in consumers:
+            try:
+                await consumer.close()
+            except Exception:
+                pass
+        await backend.close()
 
     print(f"\nMessages logged to: {output_file}")
 
 def main():
     parser = argparse.ArgumentParser(
         prog='tg-dump-queues',
-        description='Monitor and dump messages from multiple Pulsar queues',
+        description='Monitor and dump messages from multiple queues',
         epilog="""
 Examples:
   # Monitor agent and prompt flow queues
@@ -308,7 +297,7 @@ IMPORTANT:
   before running this tool. If this tool subscribes first, the real services may
   encounter schema mismatch errors when they try to connect.
 
-  Best practice: Start services → Set up flows → Run tg-dump-queues
+  Best practice: Start services -> Set up flows -> Run tg-dump-queues
         """,
         formatter_class=argparse.RawDescriptionHelpFormatter
     )
@@ -316,7 +305,7 @@ IMPORTANT:
     parser.add_argument(
         'queues',
         nargs='+',
-        help='Pulsar queue names to monitor'
+        help='Queue names to monitor'
     )
 
     parser.add_argument(

trustgraph_cli-2.8.2/trustgraph/cli/monitor_prompts.py

@@ -0,0 +1,330 @@
+"""
+Monitor prompt request/response queues and log activity with timing.
+
+Subscribes to prompt request and response queues, correlates
+them by message ID, and logs a summary of each request/response with
+elapsed time. Streaming responses are accumulated and shown once at
+completion.
+
+Examples:
+  tg-monitor-prompts
+  tg-monitor-prompts --flow default --max-lines 5
+  tg-monitor-prompts --queue-type prompt-rag
+"""
+
+import json
+import asyncio
+import sys
+import argparse
+from datetime import datetime
+from collections import OrderedDict
+
+from trustgraph.base.pubsub import get_async_pubsub, add_pubsub_args
+
+
+default_flow = "default"
+default_queue_type = "prompt-rag"
+default_max_lines = 3
+default_max_width = 80
+
+
+def truncate_text(text, max_lines, max_width):
+    """Truncate text to max_lines lines, each at most max_width chars."""
+    if not text:
+        return "(empty)"
+
+    lines = text.splitlines()
+    result = []
+    for line in lines[:max_lines]:
+        if len(line) > max_width:
+            result.append(line[:max_width - 3] + "...")
+        else:
+            result.append(line)
+
+    if len(lines) > max_lines:
+        result.append(f"  ... ({len(lines) - max_lines} more lines)")
+
+    return "\n".join(result)
+
+
+def summarise_value(value, max_width):
+    """Summarise a term value -- show type and size for large values."""
+    # Try to parse JSON
+    try:
+        parsed = json.loads(value)
+    except (json.JSONDecodeError, TypeError):
+        parsed = value
+
+    if isinstance(parsed, list):
+        return f"[{len(parsed)} items]"
+    elif isinstance(parsed, dict):
+        return f"{{{len(parsed)} keys}}"
+    elif isinstance(parsed, str):
+        if len(parsed) > max_width:
+            return parsed[:max_width - 3] + "..."
+        return parsed
+    else:
+        s = str(parsed)
+        if len(s) > max_width:
+            return s[:max_width - 3] + "..."
+        return s
+
+
+def format_terms(terms, max_lines, max_width):
+    """Format prompt terms for display -- concise summary."""
+    if not terms:
+        return ""
+
+    parts = []
+    for key, value in terms.items():
+        summary = summarise_value(value, max_width - len(key) - 4)
+        parts.append(f"  {key}: {summary}")
+
+    return "\n".join(parts)
+
+
+def parse_raw_message(msg):
+    """Parse a raw message into (correlation_id, body_dict)."""
+    try:
+        props = msg.properties()
+        corr_id = props.get("id", "")
+    except Exception:
+        corr_id = ""
+
+    try:
+        value = msg.value()
+        if isinstance(value, dict):
+            body = value
+        elif isinstance(value, bytes):
+            body = json.loads(value.decode("utf-8"))
+        elif isinstance(value, str):
+            body = json.loads(value)
+        else:
+            body = {}
+    except Exception:
+        body = {}
+
+    return corr_id, body
+
+
+async def poll_consumer(consumer, queue_label, handler, shutdown_event):
+    """Poll a single consumer and call handler for each message."""
+    while not shutdown_event.is_set():
+        try:
+            msg = await asyncio.wait_for(consumer.receive(), timeout=0.1)
+            await handler(msg)
+            await consumer.acknowledge(msg)
+        except asyncio.TimeoutError:
+            continue
+
+
+async def monitor(flow, queue_type, max_lines, max_width, **config):
+
+    request_queue = f"request:tg:{queue_type}:{flow}"
+    response_queue = f"response:tg:{queue_type}:{flow}"
+
+    print(f"Monitoring prompt queues:")
+    print(f"  Request:  {request_queue}")
+    print(f"  Response: {response_queue}")
+    print(f"Press Ctrl+C to stop\n")
+
+    backend = get_async_pubsub(**config)
+
+    req_consumer = await backend.create_consumer(
+        topic=request_queue,
+        subscription="prompt-monitor-req",
+        schema=None,
+        initial_position='latest',
+    )
+
+    resp_consumer = await backend.create_consumer(
+        topic=response_queue,
+        subscription="prompt-monitor-resp",
+        schema=None,
+        initial_position='latest',
+    )
+
+    # Track in-flight requests: corr_id -> (timestamp, template_id)
+    in_flight = OrderedDict()
+
+    # Accumulate streaming responses: corr_id -> list of text chunks
+    streaming_chunks = {}
+
+    print("Listening...\n")
+
+    shutdown_event = asyncio.Event()
+
+    async def handle_request(msg):
+        timestamp = datetime.now()
+        corr_id, body = parse_raw_message(msg)
+        time_str = timestamp.strftime("%H:%M:%S.%f")[:-3]
+
+        template_id = body.get("id", "(unknown)")
+        terms = body.get("terms", {})
+        streaming = body.get("streaming", False)
+
+        in_flight[corr_id] = (timestamp, template_id)
+
+        # Limit size
+        while len(in_flight) > 1000:
+            in_flight.popitem(last=False)
+
+        stream_flag = " [streaming]" if streaming else ""
+        id_display = corr_id[:8] if corr_id else "--------"
+        print(f"[{time_str}] REQ  {id_display}  "
+              f"template={template_id}{stream_flag}")
+
+        if terms:
+            print(format_terms(terms, max_lines, max_width))
+
+    async def handle_response(msg):
+        timestamp = datetime.now()
+        corr_id, body = parse_raw_message(msg)
+        time_str = timestamp.strftime("%H:%M:%S.%f")[:-3]
+        id_display = corr_id[:8] if corr_id else "--------"
+
+        error = body.get("error")
+        text = body.get("text", "")
+        obj = body.get("object", "")
+        eos = body.get("end_of_stream", False)
+
+        if error:
+            # Error -- show immediately
+            elapsed_str = ""
+            if corr_id in in_flight:
+                req_timestamp, _ = in_flight.pop(corr_id)
+                elapsed = (timestamp - req_timestamp).total_seconds()
+                elapsed_str = f"  ({elapsed:.3f}s)"
+            streaming_chunks.pop(corr_id, None)
+
+            err_msg = error
+            if isinstance(error, dict):
+                err_msg = error.get("message", str(error))
+            print(f"[{time_str}] ERR  {id_display}  "
+                  f"{err_msg}{elapsed_str}")
+
+        elif eos:
+            # End of stream -- show accumulated text + timing
+            elapsed_str = ""
+            if corr_id in in_flight:
+                req_timestamp, _ = in_flight.pop(corr_id)
+                elapsed = (timestamp - req_timestamp).total_seconds()
+                elapsed_str = f"  ({elapsed:.3f}s)"
+
+            accumulated = streaming_chunks.pop(corr_id, [])
+            if text:
+                accumulated.append(text)
+
+            full_text = "".join(accumulated)
+            if full_text:
+                truncated = truncate_text(
+                    full_text, max_lines, max_width
+                )
+                print(f"[{time_str}] RESP {id_display}"
+                      f"{elapsed_str}")
+                print(f"  {truncated}")
+            else:
+                print(f"[{time_str}] RESP {id_display}"
+                      f"{elapsed_str}  (empty)")
+
+        elif text or obj:
+            # Streaming chunk or non-streaming response
+            if corr_id in streaming_chunks or (
+                corr_id in in_flight
+            ):
+                # Accumulate streaming chunk
+                if corr_id not in streaming_chunks:
+                    streaming_chunks[corr_id] = []
+                streaming_chunks[corr_id].append(text or obj)
+            else:
+                # Non-streaming single response
+                elapsed_str = ""
+                if corr_id in in_flight:
+                    req_timestamp, _ = in_flight.pop(corr_id)
+                    elapsed = (
+                        timestamp - req_timestamp
+                    ).total_seconds()
+                    elapsed_str = f"  ({elapsed:.3f}s)"
+
+                content = text or obj
+                label = "" if text else "  (object)"
+                truncated = truncate_text(
+                    content, max_lines, max_width
+                )
+                print(f"[{time_str}] RESP {id_display}"
+                      f"{label}{elapsed_str}")
+                print(f"  {truncated}")
+
+    try:
+        await asyncio.gather(
+            poll_consumer(
+                req_consumer, "request", handle_request, shutdown_event,
+            ),
+            poll_consumer(
+                resp_consumer, "response", handle_response, shutdown_event,
+            ),
+        )
+    except KeyboardInterrupt:
+        print("\nStopping...")
+    finally:
+        shutdown_event.set()
+        await req_consumer.close()
+        await resp_consumer.close()
+        await backend.close()
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="tg-monitor-prompts",
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        "-f", "--flow",
+        default=default_flow,
+        help=f"Flow ID (default: {default_flow})",
+    )
+
+    parser.add_argument(
+        "-q", "--queue-type",
+        default=default_queue_type,
+        help=f"Queue type: prompt or prompt-rag (default: {default_queue_type})",
+    )
+
+    parser.add_argument(
+        "-l", "--max-lines",
+        type=int,
+        default=default_max_lines,
+        help=f"Max lines of text per term/response (default: {default_max_lines})",
+    )
+
+    parser.add_argument(
+        "-w", "--max-width",
+        type=int,
+        default=default_max_width,
+        help=f"Max width per line (default: {default_max_width})",
+    )
+
+    add_pubsub_args(parser, standalone=True)
+
+    args = parser.parse_args()
+
+    try:
+        asyncio.run(monitor(
+            flow=args.flow,
+            queue_type=args.queue_type,
+            max_lines=args.max_lines,
+            max_width=args.max_width,
+            **{k: v for k, v in vars(args).items()
+               if k not in ('flow', 'queue_type', 'max_lines', 'max_width')},
+        ))
+    except KeyboardInterrupt:
+        pass
+    except Exception as e:
+        print(f"Fatal error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

trustgraph_cli-2.8.2/trustgraph/cli_version.py

@@ -0,0 +1 @@
+__version__ = "2.8.2"

{trustgraph_cli-2.8.0 → trustgraph_cli-2.8.2}/trustgraph_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: trustgraph-cli
-Version: 2.8.0
+Version: 2.8.2
 Summary: TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.
 Author-email: "trustgraph.ai" <security@trustgraph.ai>
 Project-URL: Homepage, https://github.com/trustgraph-ai/trustgraph