netra-zen 1.2.0__py3-none-any.whl → 1.2.1__py3-none-any.whl

scripts/verify_log_transmission.py

@@ -1,140 +1,140 @@
-#!/usr/bin/env python3
-"""
-Verification script to prove JSONL logs are bundled in payload
-"""
-
-import json
-import sys
-from pathlib import Path
-
-# Add parent directory to path
-sys.path.insert(0, str(Path(__file__).parent.parent))
-
-from scripts.agent_logs import collect_recent_logs
-
-
-def verify_log_bundling(log_path: str):
-    """
-    Verify that logs are properly collected and bundled
-
-    Args:
-        log_path: Path to JSONL file or directory
-    """
-    print("=" * 70)
-    print("JSONL LOG TRANSMISSION VERIFICATION")
-    print("=" * 70)
-    print()
-
-    # Step 1: Collect logs
-    print("Step 1: Collecting logs from file...")
-    result = collect_recent_logs(limit=1, base_path=log_path)
-
-    if not result:
-        print("❌ FAILED: No logs collected")
-        return False
-
-    logs, files_read, file_info = result
-    print(f"✓ Successfully collected {len(logs)} log entries from {files_read} file(s)")
-    print()
-
-    # Step 2: Show file details
-    print("Step 2: File details...")
-    for info in file_info:
-        print(f"  File: {info['name']}")
-        print(f"  Hash: {info['hash']}")
-        print(f"  Entries: {info['entries']}")
-    print()
-
-    # Step 3: Simulate payload creation
-    print("Step 3: Simulating WebSocket payload creation...")
-    payload = {
-        "type": "message_create",
-        "run_id": "test-run-id",
-        "payload": {
-            "message": "Test message with logs",
-            "jsonl_logs": logs  # This is where logs are added
-        }
-    }
-
-    print(f"✓ Payload created with 'jsonl_logs' key")
-    print(f"  Payload keys: {list(payload['payload'].keys())}")
-    print()
-
-    # Step 4: Verify payload size
-    print("Step 4: Calculating payload size...")
-    payload_json = json.dumps(payload)
-    payload_size_bytes = len(payload_json.encode('utf-8'))
-    payload_size_kb = payload_size_bytes / 1024
-    payload_size_mb = payload_size_kb / 1024
-
-    if payload_size_mb >= 1:
-        size_str = f"{payload_size_mb:.2f} MB"
-    elif payload_size_kb >= 1:
-        size_str = f"{payload_size_kb:.2f} KB"
-    else:
-        size_str = f"{payload_size_bytes} bytes"
-
-    print(f"✓ Total payload size: {size_str}")
-    print()
-
-    # Step 5: Show sample log entries
-    print("Step 5: Sample log entries in payload...")
-    if logs:
-        print(f"  First entry keys: {list(logs[0].keys())}")
-        print(f"  First entry timestamp: {logs[0].get('timestamp', 'N/A')}")
-        print(f"  Last entry timestamp: {logs[-1].get('timestamp', 'N/A')}")
-    print()
-
-    # Step 6: Verify transmission-ready
-    print("Step 6: Transmission verification...")
-    print(f"✓ Payload is valid JSON: {payload_json is not None}")
-    print(f"✓ Payload contains 'jsonl_logs': {'jsonl_logs' in payload['payload']}")
-    print(f"✓ Log count in payload: {len(payload['payload']['jsonl_logs'])}")
-    print()
-
-    print("=" * 70)
-    print("✅ VERIFICATION COMPLETE")
-    print("=" * 70)
-    print()
-    print("PROOF OF TRANSMISSION:")
-    print(f"  • {len(logs)} JSONL log entries are bundled in the payload")
-    print(f"  • Payload size: {size_str}")
-    print(f"  • Ready for WebSocket transmission to backend")
-    print()
-
-    # Optional: Save proof file
-    proof_file = Path("/tmp/zen_transmission_proof.json")
-    proof_payload = {
-        "verification_timestamp": "verification_run",
-        "log_count": len(logs),
-        "files_read": files_read,
-        "file_info": file_info,
-        "payload_size": size_str,
-        "sample_first_entry": logs[0] if logs else None,
-        "sample_last_entry": logs[-1] if logs else None,
-        "payload_structure": {
-            "type": payload["type"],
-            "run_id": payload["run_id"],
-            "payload_keys": list(payload["payload"].keys()),
-            "jsonl_logs_present": "jsonl_logs" in payload["payload"],
-            "jsonl_logs_count": len(payload["payload"]["jsonl_logs"])
-        }
-    }
-
-    with open(proof_file, 'w') as f:
-        json.dump(proof_payload, f, indent=2)
-
-    print(f"📝 Detailed proof saved to: {proof_file}")
-    print()
-
-    return True
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: python verify_log_transmission.py <path-to-jsonl-file>")
-        sys.exit(1)
-
-    log_path = sys.argv[1]
-    success = verify_log_bundling(log_path)
-    sys.exit(0 if success else 1)
+#!/usr/bin/env python3
+"""
+Verification script to prove JSONL logs are bundled in payload
+"""
+
+import json
+import sys
+from pathlib import Path
+
+# Add parent directory to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from scripts.agent_logs import collect_recent_logs
+
+
+def verify_log_bundling(log_path: str):
+    """
+    Verify that logs are properly collected and bundled
+
+    Args:
+        log_path: Path to JSONL file or directory
+    """
+    print("=" * 70)
+    print("JSONL LOG TRANSMISSION VERIFICATION")
+    print("=" * 70)
+    print()
+
+    # Step 1: Collect logs
+    print("Step 1: Collecting logs from file...")
+    result = collect_recent_logs(limit=1, base_path=log_path)
+
+    if not result:
+        print("❌ FAILED: No logs collected")
+        return False
+
+    logs, files_read, file_info = result
+    print(f"✓ Successfully collected {len(logs)} log entries from {files_read} file(s)")
+    print()
+
+    # Step 2: Show file details
+    print("Step 2: File details...")
+    for info in file_info:
+        print(f"  File: {info['name']}")
+        print(f"  Hash: {info['hash']}")
+        print(f"  Entries: {info['entries']}")
+    print()
+
+    # Step 3: Simulate payload creation
+    print("Step 3: Simulating WebSocket payload creation...")
+    payload = {
+        "type": "message_create",
+        "run_id": "test-run-id",
+        "payload": {
+            "message": "Test message with logs",
+            "jsonl_logs": logs  # This is where logs are added
+        }
+    }
+
+    print(f"✓ Payload created with 'jsonl_logs' key")
+    print(f"  Payload keys: {list(payload['payload'].keys())}")
+    print()
+
+    # Step 4: Verify payload size
+    print("Step 4: Calculating payload size...")
+    payload_json = json.dumps(payload)
+    payload_size_bytes = len(payload_json.encode('utf-8'))
+    payload_size_kb = payload_size_bytes / 1024
+    payload_size_mb = payload_size_kb / 1024
+
+    if payload_size_mb >= 1:
+        size_str = f"{payload_size_mb:.2f} MB"
+    elif payload_size_kb >= 1:
+        size_str = f"{payload_size_kb:.2f} KB"
+    else:
+        size_str = f"{payload_size_bytes} bytes"
+
+    print(f"✓ Total payload size: {size_str}")
+    print()
+
+    # Step 5: Show sample log entries
+    print("Step 5: Sample log entries in payload...")
+    if logs:
+        print(f"  First entry keys: {list(logs[0].keys())}")
+        print(f"  First entry timestamp: {logs[0].get('timestamp', 'N/A')}")
+        print(f"  Last entry timestamp: {logs[-1].get('timestamp', 'N/A')}")
+    print()
+
+    # Step 6: Verify transmission-ready
+    print("Step 6: Transmission verification...")
+    print(f"✓ Payload is valid JSON: {payload_json is not None}")
+    print(f"✓ Payload contains 'jsonl_logs': {'jsonl_logs' in payload['payload']}")
+    print(f"✓ Log count in payload: {len(payload['payload']['jsonl_logs'])}")
+    print()
+
+    print("=" * 70)
+    print("✅ VERIFICATION COMPLETE")
+    print("=" * 70)
+    print()
+    print("PROOF OF TRANSMISSION:")
+    print(f"  • {len(logs)} JSONL log entries are bundled in the payload")
+    print(f"  • Payload size: {size_str}")
+    print(f"  • Ready for WebSocket transmission to backend")
+    print()
+
+    # Optional: Save proof file
+    proof_file = Path("/tmp/zen_transmission_proof.json")
+    proof_payload = {
+        "verification_timestamp": "verification_run",
+        "log_count": len(logs),
+        "files_read": files_read,
+        "file_info": file_info,
+        "payload_size": size_str,
+        "sample_first_entry": logs[0] if logs else None,
+        "sample_last_entry": logs[-1] if logs else None,
+        "payload_structure": {
+            "type": payload["type"],
+            "run_id": payload["run_id"],
+            "payload_keys": list(payload["payload"].keys()),
+            "jsonl_logs_present": "jsonl_logs" in payload["payload"],
+            "jsonl_logs_count": len(payload["payload"]["jsonl_logs"])
+        }
+    }
+
+    with open(proof_file, 'w') as f:
+        json.dump(proof_payload, f, indent=2)
+
+    print(f"📝 Detailed proof saved to: {proof_file}")
+    print()
+
+    return True
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python verify_log_transmission.py <path-to-jsonl-file>")
+        sys.exit(1)
+
+    log_path = sys.argv[1]
+    success = verify_log_bundling(log_path)
+    sys.exit(0 if success else 1)

shared/README.md

@@ -0,0 +1,47 @@
+# Shared Utilities (Vendored Subset)
+
+This directory contains a **minimal vendored subset** of the Apex `shared/` package, containing only the files required by `zen --apex` CLI functionality.
+
+## Included Files
+
+### 1. `__init__.py`
+Package initialization stub with basic documentation.
+
+### 2. `windows_encoding.py`
+Windows UTF-8 console encoding fixes. The `setup_windows_encoding()` function is called early in `agent_cli.py` startup to ensure proper Unicode handling on Windows platforms.
+
+### 3. `types/__init__.py`
+Type definitions package that re-exports WebSocket closure code utilities.
+
+### 4. `types/websocket_closure_codes.py`
+WebSocket closure code validation utilities:
+- `WebSocketClosureCode`: Enum of standard RFC 6455 closure codes
+- `WebSocketClosureCategory`: Categories for classifying closure types
+- `categorize_closure_code()`: Categorize a code into normal/client/server/infrastructure
+- `get_closure_description()`: Human-readable description of closure codes
+- `is_infrastructure_error()`: Check if a code represents infrastructure failure
+
+## Maintenance
+
+⚠️ **Important**: These files are vendored from the Apex repository. If Apex updates its closure-code definitions or Windows encoding logic, these files must be manually synchronized.
+
+### What's NOT Included
+
+Everything else under Apex's `shared/` directory is intentionally excluded because it's not referenced by `agent_cli.py`. This keeps the vendored code minimal and avoids pulling in backend logic, secrets, or unnecessary dependencies.
+
+## Usage
+
+These modules are imported by `scripts/agent_cli.py`:
+
+```python
+from shared.windows_encoding import setup_windows_encoding
+from shared.types.websocket_closure_codes import (
+    WebSocketClosureCode,
+    WebSocketClosureCategory,
+    categorize_closure_code,
+    get_closure_description,
+    is_infrastructure_error
+)
+```
+
+The `setup_windows_encoding()` function is called immediately at startup, before any console I/O operations.

shared/TIMING_FIX_COMPLETE.md

@@ -0,0 +1,80 @@
+# WebSocket Timing Fix - Complete Implementation
+
+## Problem Statement
+The CLI was sending messages too early, before the server completed its lifecycle phases and entered the message processing loop (Phase 5). Messages sent before Phase 5 were lost because the server's message handler wasn't active yet.
+
+## Root Cause
+The CLI code had backward compatibility logic that allowed connections to proceed even when the handshake failed, causing messages to be sent before the server was ready.
+
+## Complete Fix Implementation
+
+### 1. Connection Method (lines 2879-2928)
+**REMOVED:** Backward compatibility code that returned `True` even when handshake failed
+**ADDED:**
+- Proper failure when handshake doesn't complete
+- Retry mechanism with 3-second delay
+- Second handshake attempt after delay
+- WebSocket closure on failure
+
+### 2. Handshake Response Processing (lines 3307-3363)
+**CHANGED:** Message type from `session_acknowledged` to `handshake_acknowledged`
+**ADDED:**
+- Wait for `handshake_complete` confirmation
+- 500ms delay after handshake_complete for server to enter Phase 5
+- Fallback delay if no handshake_complete received
+
+### 3. Message Sending Validation (lines 3703-3722)
+**ADDED:** Check for `self.connected` flag before sending messages
+- Ensures handshake is fully complete
+- Prevents messages being sent during server initialization
+- Clear error messages when connection not ready
+
+## Order of Operations (Enforced)
+
+1. **Connect** → WebSocket connection established
+2. **Wait** for `connection_established` event
+3. **Handshake** → Wait for `handshake_response`
+4. **Acknowledge** → Send `handshake_acknowledged` with thread_id
+5. **Confirm** → Wait for `handshake_complete` message
+6. **Delay** → Add 500ms for server to enter Phase 5 (Processing)
+7. **Ready** → NOW messages can be safely sent
+
+## Server Phases Reference
+
+```
+Phase 1: INITIALIZING   → Accept connection, assign ID
+Phase 2: AUTHENTICATING → Validate user credentials
+Phase 3: HANDSHAKING    → Exchange thread IDs
+Phase 4: READY          → Initialize services, register with router
+Phase 5: PROCESSING     → Message loop active ✓ (Messages accepted here!)
+Phase 6: CLEANING_UP    → Coordinated cleanup
+Phase 7: CLOSED         → Terminal state
+```
+
+## Key Benefits
+
+✓ **No Message Loss** - Messages only sent when server is ready to process them
+✓ **Proper Sequencing** - Enforces documented WebSocket lifecycle
+✓ **Retry Logic** - Handles transient delays during server startup
+✓ **Clear Errors** - Descriptive messages when connection fails
+✓ **Fail Fast** - Connection fails quickly if server isn't ready
+
+## Testing Verification
+
+The fix ensures:
+- `self.connected` is only set to `True` after full handshake completion
+- `send_message()` validates both `self.connected` and `self.current_thread_id`
+- Messages cannot be sent until server reaches Phase 5 (Processing)
+- Connection properly fails if server doesn't complete handshake
+
+## Files Modified
+
+- `scripts/agent_cli.py`:
+  - `connect()` method - lines 2879-2928
+  - `_perform_handshake()` method - lines 3057-3058
+  - `_process_handshake_response()` method - lines 3307-3363
+  - `send_message()` method - lines 3703-3722
+
+## Implementation Complete
+
+The timing issue is now fully resolved. The CLI will properly wait for the server to complete all initialization phases before sending any messages, preventing message loss and ensuring reliable communication.

shared/__init__.py

@@ -0,0 +1,12 @@
+"""
+Shared utilities package (minimal vendored subset for zen --apex CLI).
+
+This package contains only the minimal files required by agent_cli.py:
+- windows_encoding.py: Windows UTF-8 console fixes
+- types/websocket_closure_codes.py: WebSocket closure code validation
+
+TODO: Manually sync these files if Apex updates its closure-code definitions
+or Windows handling logic.
+"""
+
+__version__ = "1.0.0"

shared/types/__init__.py

@@ -0,0 +1,21 @@
+"""
+Shared type definitions (minimal vendored subset for zen --apex CLI).
+
+This package provides WebSocket closure code utilities required by agent_cli.py.
+"""
+
+from shared.types.websocket_closure_codes import (
+    WebSocketClosureCode,
+    WebSocketClosureCategory,
+    categorize_closure_code,
+    get_closure_description,
+    is_infrastructure_error
+)
+
+__all__ = [
+    'WebSocketClosureCode',
+    'WebSocketClosureCategory',
+    'categorize_closure_code',
+    'get_closure_description',
+    'is_infrastructure_error'
+]

shared/types/websocket_closure_codes.py

@@ -0,0 +1,124 @@
+"""
+WebSocket closure code definitions and categorization.
+
+Provides enums and helper functions for validating and categorizing WebSocket
+closure codes according to RFC 6455 and common extensions.
+"""
+
+from enum import IntEnum
+from typing import Optional
+
+
+class WebSocketClosureCode(IntEnum):
+    """
+    Standard WebSocket closure codes from RFC 6455 and extensions.
+
+    See: https://datatracker.ietf.org/doc/html/rfc6455#section-7.4.1
+    """
+    # Standard codes (1000-1015)
+    NORMAL_CLOSURE = 1000
+    GOING_AWAY = 1001
+    PROTOCOL_ERROR = 1002
+    UNSUPPORTED_DATA = 1003
+    # 1004 is reserved
+    NO_STATUS_RECEIVED = 1005
+    ABNORMAL_CLOSURE = 1006
+    INVALID_FRAME_PAYLOAD_DATA = 1007
+    POLICY_VIOLATION = 1008
+    MESSAGE_TOO_BIG = 1009
+    MANDATORY_EXTENSION = 1010
+    INTERNAL_SERVER_ERROR = 1011
+    SERVICE_RESTART = 1012
+    TRY_AGAIN_LATER = 1013
+    BAD_GATEWAY = 1014
+    TLS_HANDSHAKE_FAILURE = 1015
+
+    # Custom application codes (4000-4999)
+    # These are application-specific and can be defined as needed
+
+
+class WebSocketClosureCategory(IntEnum):
+    """
+    Categories for WebSocket closure codes to classify failure types.
+    """
+    NORMAL = 0           # Expected closures (1000, 1001)
+    CLIENT_ERROR = 1     # Client-side errors (1002-1003, 1007-1010)
+    SERVER_ERROR = 2     # Server-side errors (1011-1014)
+    INFRASTRUCTURE = 3   # Infrastructure/network errors (1006, 1015)
+    UNKNOWN = 4          # Unrecognized codes
+
+
+def categorize_closure_code(code: int) -> WebSocketClosureCategory:
+    """
+    Categorize a WebSocket closure code into a failure category.
+
+    Args:
+        code: The WebSocket closure code
+
+    Returns:
+        The category of the closure code
+    """
+    # Normal closures
+    if code in (1000, 1001):
+        return WebSocketClosureCategory.NORMAL
+
+    # Infrastructure/network errors
+    if code in (1006, 1015):
+        return WebSocketClosureCategory.INFRASTRUCTURE
+
+    # Server errors
+    if code in (1011, 1012, 1013, 1014):
+        return WebSocketClosureCategory.SERVER_ERROR
+
+    # Client errors
+    if code in (1002, 1003, 1007, 1008, 1009, 1010):
+        return WebSocketClosureCategory.CLIENT_ERROR
+
+    # Unknown/unrecognized codes
+    return WebSocketClosureCategory.UNKNOWN
+
+
+def is_infrastructure_error(code: int) -> bool:
+    """
+    Check if a closure code represents an infrastructure/network error.
+
+    Infrastructure errors are typically transient and may be retryable.
+
+    Args:
+        code: The WebSocket closure code
+
+    Returns:
+        True if the code represents an infrastructure error
+    """
+    return categorize_closure_code(code) == WebSocketClosureCategory.INFRASTRUCTURE
+
+
+def get_closure_description(code: int) -> str:
+    """
+    Get a human-readable description of a WebSocket closure code.
+
+    Args:
+        code: The WebSocket closure code
+
+    Returns:
+        A description of what the closure code means
+    """
+    descriptions = {
+        1000: "Normal closure - connection completed successfully",
+        1001: "Going away - endpoint is going away (e.g., server shutdown, browser navigation)",
+        1002: "Protocol error - endpoint received a malformed message",
+        1003: "Unsupported data - endpoint received data of unsupported type",
+        1005: "No status received - no status code was provided (internal use only)",
+        1006: "Abnormal closure - connection closed without close frame (network/infrastructure issue)",
+        1007: "Invalid frame payload data - message contains invalid UTF-8 or violates payload requirements",
+        1008: "Policy violation - endpoint received message that violates its policy",
+        1009: "Message too big - endpoint received message that is too large to process",
+        1010: "Mandatory extension - client expected server to negotiate an extension",
+        1011: "Internal server error - server encountered unexpected condition",
+        1012: "Service restart - server is restarting",
+        1013: "Try again later - server is temporarily overloaded or under maintenance",
+        1014: "Bad gateway - server acting as gateway received invalid response",
+        1015: "TLS handshake failure - TLS handshake failed (internal use only)",
+    }
+
+    return descriptions.get(code, f"Unknown closure code: {code}")

shared/windows_encoding.py

@@ -0,0 +1,45 @@
+"""
+Windows UTF-8 console encoding fixes.
+
+Ensures proper UTF-8 handling on Windows consoles by setting appropriate
+encoding for stdin, stdout, and stderr streams.
+"""
+
+import sys
+import io
+
+
+def setup_windows_encoding():
+    """
+    Configure Windows console to use UTF-8 encoding.
+
+    This function reconfigures sys.stdin, sys.stdout, and sys.stderr to use
+    UTF-8 encoding with error handling on Windows platforms. This prevents
+    Unicode encoding errors when displaying special characters or emoji.
+
+    Must be called early in the startup sequence before any console I/O.
+    """
+    if sys.platform == "win32":
+        # Reconfigure stdout and stderr to use UTF-8 with error handling
+        if sys.stdout is not None:
+            sys.stdout = io.TextIOWrapper(
+                sys.stdout.buffer,
+                encoding='utf-8',
+                errors='replace',
+                line_buffering=True
+            )
+
+        if sys.stderr is not None:
+            sys.stderr = io.TextIOWrapper(
+                sys.stderr.buffer,
+                encoding='utf-8',
+                errors='replace',
+                line_buffering=True
+            )
+
+        if sys.stdin is not None:
+            sys.stdin = io.TextIOWrapper(
+                sys.stdin.buffer,
+                encoding='utf-8',
+                errors='replace'
+            )

zen/__init__.py

@@ -1,7 +1,7 @@
-"""Zen namespace package placeholder.
-
-This lightweight module exists so repository scripts can import
-`zen.telemetry` directly without requiring the full orchestrator package.
-"""
-
-__all__: list[str] = []
+"""Zen namespace package placeholder.
+
+This lightweight module exists so repository scripts can import
+`zen.telemetry` directly without requiring the full orchestrator package.
+"""
+
+__all__: list[str] = []

zen/__main__.py

@@ -1,11 +1,11 @@
-"""Module entry point to support `python -m zen` invocation."""
-
-from zen_orchestrator import run
-
-
-def main() -> None:
-    run()
-
-
-if __name__ == "__main__":
-    main()
+"""Module entry point to support `python -m zen` invocation."""
+
+from zen_orchestrator import run
+
+
+def main() -> None:
+    run()
+
+
+if __name__ == "__main__":
+    main()