opengradient 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

opengradient/__init__.py

@@ -17,9 +17,11 @@ from .types import (
     InferenceResult,
     LlmInferenceMode,
     TextGenerationOutput,
+    TextGenerationStream,
     ModelOutput,
     ModelRepository,
     FileUploadResult,
+    x402SettlementMode,
 )
 
 from . import llm, alphasense
@@ -47,10 +49,25 @@ def new_client(
         contract_address: Optional inference contract address
     """
 
-    return Client(email=email, password=password, private_key=private_key, rpc_url=rpc_url, api_url=api_url, contract_address=contract_address, **kwargs)
+    return Client(
+        email=email,
+        password=password,
+        private_key=private_key,
+        rpc_url=rpc_url,
+        api_url=api_url,
+        contract_address=contract_address,
+        **kwargs,
+    )
 
 
-def init(email: str, password: str, private_key: str, rpc_url=DEFAULT_RPC_URL, api_url=DEFAULT_API_URL, contract_address=DEFAULT_INFERENCE_CONTRACT_ADDRESS):
+def init(
+    email: str,
+    password: str,
+    private_key: str,
+    rpc_url=DEFAULT_RPC_URL,
+    api_url=DEFAULT_API_URL,
+    contract_address=DEFAULT_INFERENCE_CONTRACT_ADDRESS,
+):
     """Initialize the OpenGradient SDK with authentication and network settings.
 
     Args:
@@ -62,8 +79,10 @@ def init(email: str, password: str, private_key: str, rpc_url=DEFAULT_RPC_URL, a
         contract_address: Optional inference contract address
     """
     global _client
-    
-    _client = Client(private_key=private_key, rpc_url=rpc_url, api_url=api_url, email=email, password=password, contract_address=contract_address)
+
+    _client = Client(
+        private_key=private_key, rpc_url=rpc_url, api_url=api_url, email=email, password=password, contract_address=contract_address
+    )
     return _client
 
 
@@ -162,6 +181,7 @@ def llm_completion(
     stop_sequence: Optional[List[str]] = None,
     temperature: float = 0.0,
     max_retries: Optional[int] = None,
+    x402_settlement_mode: Optional[x402SettlementMode] = x402SettlementMode.SETTLE_BATCH,
 ) -> TextGenerationOutput:
     """Generate text completion using an LLM.
 
@@ -173,6 +193,7 @@ def llm_completion(
         stop_sequence: Optional list of sequences where generation should stop
         temperature: Sampling temperature (0.0 = deterministic, 1.0 = creative)
         max_retries: Maximum number of retries for failed transactions
+        x402_settlement_mode: Settlement modes for x402 payment protocol transactions (enum x402SettlementMode)
 
     Returns:
         TextGenerationOutput: Transaction hash and generated text
@@ -190,6 +211,7 @@ def llm_completion(
         stop_sequence=stop_sequence,
         temperature=temperature,
         max_retries=max_retries,
+        x402_settlement_mode=x402_settlement_mode
     )
 
 
@@ -203,7 +225,9 @@ def llm_chat(
     tools: Optional[List[Dict]] = None,
     tool_choice: Optional[str] = None,
     max_retries: Optional[int] = None,
-) -> TextGenerationOutput:
+    x402_settlement_mode: Optional[x402SettlementMode] = x402SettlementMode.SETTLE_BATCH,
+    stream: Optional[bool] = False,
+) -> Union[TextGenerationOutput, TextGenerationStream]:
     """Have a chat conversation with an LLM.
 
     Args:
@@ -216,9 +240,11 @@ def llm_chat(
         tools: Optional list of tools the model can use
         tool_choice: Optional specific tool to use
         max_retries: Maximum number of retries for failed transactions
+        x402_settlement_mode: Settlement modes for x402 payment protocol transactions (enum x402SettlementMode)
+        stream: Optional boolean to enable streaming
 
     Returns:
-        TextGenerationOutput
+        TextGenerationOutput or TextGenerationStream
 
     Raises:
         RuntimeError: If SDK is not initialized
@@ -235,6 +261,8 @@ def llm_chat(
         tools=tools,
         tool_choice=tool_choice,
         max_retries=max_retries,
+        x402_settlement_mode=x402_settlement_mode,
+        stream=stream,
     )
 
 

opengradient/cli.py

@@ -6,6 +6,7 @@ import logging
 import webbrowser
 from pathlib import Path
 from typing import Dict, List, Optional
+import sys
 
 import click
 
@@ -80,6 +81,7 @@ x402SettlementModes = {
     "settle-metadata": x402SettlementMode.SETTLE_METADATA,
 }
 
+
 def initialize_config(ctx):
     """Interactively initialize OpenGradient config"""
     if ctx.obj:  # Check if config data already exists
@@ -140,7 +142,7 @@ def cli(ctx):
             openai_api_key = ctx.obj.get("openai_api_key")
             anthropic_api_key = ctx.obj.get("anthropic_api_key")
             google_api_key = ctx.obj.get("google_api_key")
-            
+
             ctx.obj["client"] = Client(
                 private_key=ctx.obj["private_key"],
                 rpc_url=DEFAULT_RPC_URL,
@@ -219,9 +221,9 @@ def clear(ctx):
 def set_api_key(ctx, provider: str, key: str):
     """
     Set API key for external LLM providers.
-    
+
     Example usage:
-    
+
     \b
     opengradient config set-api-key --provider openai --key ..
     opengradient config set-api-key --provider anthropic --key ...
@@ -230,7 +232,7 @@ def set_api_key(ctx, provider: str, key: str):
     config_key = f"{provider}_api_key"
     ctx.obj[config_key] = key
     save_og_config(ctx)
-    
+
     click.secho(f"✅ API key for {provider} has been set", fg="green")
     click.echo("You can now use models from this provider in completion and chat commands.")
 
@@ -241,9 +243,9 @@ def set_api_key(ctx, provider: str, key: str):
 def remove_api_key(ctx, provider: str):
     """
     Remove API key for an external LLM provider.
-    
+
     Example usage:
-    
+
     \b
     opengradient config remove-api-key --provider openai
     """
@@ -417,52 +419,68 @@ def infer(ctx, model_cid: str, inference_mode: str, input_data, input_file: Path
     help="Model identifier (local model from LLM enum or external model like 'gpt-4o', 'gemini-2.5-flash-lite', etc.)",
 )
 @click.option(
-    "--mode", 
-    "inference_mode", 
-    type=click.Choice(LlmInferenceModes.keys()), 
-    default="VANILLA", 
-    help="Inference mode (only applies to local models, default: VANILLA)"
+    "--mode",
+    "inference_mode",
+    type=click.Choice(LlmInferenceModes.keys()),
+    default="VANILLA",
+    help="Inference mode (only applies to local models, default: VANILLA)",
 )
 @click.option("--prompt", "-p", required=True, help="Input prompt for the LLM completion")
 @click.option("--max-tokens", type=int, default=100, help="Maximum number of tokens for LLM completion output")
 @click.option("--stop-sequence", multiple=True, help="Stop sequences for LLM")
 @click.option("--temperature", type=float, default=0.0, help="Temperature for LLM inference (0.0 to 1.0)")
 @click.option("--local", is_flag=True, help="Force use of local model even if not in LLM enum")
-@click.option("--x402-settlement-mode", "x402_settlement_mode", type=click.Choice(x402SettlementModes.keys()), default="settle-batch", help="Settlement mode for x402 payload")
+@click.option(
+    "--x402-settlement-mode",
+    "x402_settlement_mode",
+    type=click.Choice(x402SettlementModes.keys()),
+    default="settle-batch",
+    help="Settlement mode for x402 payments: settle (hashes only), settle-batch (batched, default), settle-metadata (full data)",
+)
 @click.pass_context
-def completion(ctx, model_cid: str, inference_mode: str, x402_settlement_mode: str, prompt: str, max_tokens: int, stop_sequence: List[str], temperature: float, local: bool):
+def completion(
+    ctx,
+    model_cid: str,
+    inference_mode: str,
+    x402_settlement_mode: str,
+    prompt: str,
+    max_tokens: int,
+    stop_sequence: List[str],
+    temperature: float,
+    local: bool,
+):
     """
     Run completion inference on an LLM model (local or external).
 
-    This command supports both local OpenGradient models and external providers 
-    (OpenAI, Anthropic, Google, etc.). For external models, make sure to set 
+    This command supports both local OpenGradient models and external providers
+    (OpenAI, Anthropic, Google, etc.). For external models, make sure to set
     the appropriate API key using 'opengradient config set-api-key'.
 
     Example usage:
 
     \b
-    # Local model
-    opengradient completion --model meta-llama/Meta-Llama-3-8B-Instruct --prompt "Hello, how are you?" --max-tokens 50
-    
+    # TEE model
+    opengradient completion --model anthropic/claude-3.5-haiku --prompt "Hello, how are you?" --max-tokens 50
+
     # External OpenAI model
     opengradient completion --model gpt-4o --prompt "Translate to French: Hello world" --max-tokens 50
-    
+
     # External Anthropic model
-    opengradient completion --model claude-haiku-4-5-20251001--prompt "Write a haiku about coding" --max-tokens 100
-    
+    opengradient completion --model claude-haiku-4-5-20251001 --prompt "Write a haiku about coding" --max-tokens 100
+
     # External Google model
     opengradient completion --model gemini-2.5-flash-lite --prompt "Explain quantum computing" --max-tokens 200
     """
     client: Client = ctx.obj["client"]
-    
+
     try:
         is_local = local or model_cid in [llm.value for llm in LLM]
-        
+
         if is_local:
             click.echo(f'Running LLM completion inference for local model "{model_cid}"\n')
         else:
             click.echo(f'Running LLM completion inference for external model "{model_cid}"\n')
-        
+
         completion_output = client.llm_completion(
             model_cid=model_cid,
             inference_mode=LlmInferenceModes[inference_mode],
@@ -475,7 +493,7 @@ def completion(ctx, model_cid: str, inference_mode: str, x402_settlement_mode: s
         )
 
         print_llm_completion_result(model_cid, completion_output.transaction_hash, completion_output.completion_output, is_local)
-        
+
     except Exception as e:
         click.echo(f"Error running LLM completion: {str(e)}")
 
@@ -485,7 +503,7 @@ def print_llm_completion_result(model_cid, tx_hash, llm_output, is_local=True):
     click.echo("──────────────────────────────────────")
     click.echo("Model: ", nl=False)
     click.secho(model_cid, fg="cyan", bold=True)
-    
+
     if is_local and tx_hash != "external":
         click.echo("Transaction hash: ", nl=False)
         click.secho(tx_hash, fg="cyan", bold=True)
@@ -495,7 +513,7 @@ def print_llm_completion_result(model_cid, tx_hash, llm_output, is_local=True):
     else:
         click.echo("Source: ", nl=False)
         click.secho("External Provider", fg="cyan", bold=True)
-    
+
     click.echo("──────────────────────────────────────")
     click.secho("LLM Output:", fg="yellow", bold=True)
     click.echo()
@@ -512,11 +530,11 @@ def print_llm_completion_result(model_cid, tx_hash, llm_output, is_local=True):
     help="Model identifier (local model from LLM enum or external model like 'gpt-4o', 'gemini-2.5-flash-lite', etc.)",
 )
 @click.option(
-    "--mode", 
-    "inference_mode", 
-    type=click.Choice(LlmInferenceModes.keys()), 
-    default="VANILLA", 
-    help="Inference mode (only applies to local models, default: VANILLA)"
+    "--mode",
+    "inference_mode",
+    type=click.Choice(LlmInferenceModes.keys()),
+    default="VANILLA",
+    help="Inference mode (only applies to local models, default: VANILLA)",
 )
 @click.option("--messages", type=str, required=False, help="Input messages for the chat inference in JSON format")
 @click.option(
@@ -530,14 +548,17 @@ def print_llm_completion_result(model_cid, tx_hash, llm_output, is_local=True):
 @click.option("--temperature", type=float, default=0.0, help="Temperature for LLM inference (0.0 to 1.0)")
 @click.option("--tools", type=str, default=None, help="Tool configurations in JSON format")
 @click.option(
-    "--tools-file", 
-    type=click.Path(exists=True, path_type=Path), 
-    required=False, 
-    help="Path to JSON file containing tool configurations"
+    "--tools-file", type=click.Path(exists=True, path_type=Path), required=False, help="Path to JSON file containing tool configurations"
 )
 @click.option("--tool-choice", type=str, default="", help="Specific tool choice for the LLM")
 @click.option("--local", is_flag=True, help="Force use of local model even if not in LLM enum")
-@click.option("--x402-settlement-mode", type=click.Choice(x402SettlementModes.keys()), default="settle-batch", help="Settlement mode for x402 payload")
+@click.option(
+    "--x402-settlement-mode",
+    type=click.Choice(x402SettlementModes.keys()),
+    default="settle-batch",
+    help="Settlement mode for x402 payments: settle (hashes only), settle-batch (batched, default), settle-metadata (full data)",
+)
+@click.option("--stream", is_flag=True, default=False, help="Stream the output from the LLM")
 @click.pass_context
 def chat(
     ctx,
@@ -553,6 +574,7 @@ def chat(
     tool_choice: Optional[str],
     x402_settlement_mode: Optional[str],
     local: bool,
+    stream: bool,
 ):
     """
     Run chat inference on an LLM model (local or external).
@@ -563,25 +585,28 @@ def chat(
     Example usage:
 
     \b
-    # Local model
-    opengradient chat --model meta-llama/Meta-Llama-3-8B-Instruct --messages '[{"role":"user","content":"hello"}]' --max-tokens 50
-    
+    # TEE model
+    opengradient chat --model anthropic/claude-3.5-haiku --messages '[{"role":"user","content":"hello"}]' --max-tokens 50
+
     # External OpenAI model with tools
     opengradient chat --model gpt-4o --messages-file messages.json --tools-file tools.json --max-tokens 200
-    
+
     # External Anthropic model
     opengradient chat --model claude-haiku-4-5-20251001 --messages '[{"role":"user","content":"Write a poem"}]' --max-tokens 100
+
+    # Stream output
+    opengradient chat --model anthropic/claude-3.5-haiku --messages '[{"role":"user","content":"How are clouds formed?"}]' --max-tokens 250 --stream
     """
     client: Client = ctx.obj["client"]
-    
+
     try:
         is_local = local or model_cid in [llm.value for llm in LLM]
-        
+
         if is_local:
             click.echo(f'Running LLM chat inference for local model "{model_cid}"\n')
         else:
             click.echo(f'Running LLM chat inference for external model "{model_cid}"\n')
-        
+
         # Parse messages
         if not messages and not messages_file:
             click.echo("Must specify either messages or messages-file")
@@ -637,7 +662,7 @@ def chat(
         if not tools and not tools_file:
             parsed_tools = None
 
-        completion_output = client.llm_chat(
+        result = client.llm_chat(
             model_cid=model_cid,
             inference_mode=LlmInferenceModes[inference_mode],
             messages=messages,
@@ -648,16 +673,17 @@ def chat(
             tool_choice=tool_choice,
             local_model=local,
             x402_settlement_mode=x402_settlement_mode,
+            stream=stream,
         )
 
-        print_llm_chat_result(
-            model_cid, 
-            completion_output.transaction_hash, 
-            completion_output.finish_reason, 
-            completion_output.chat_output,
-            is_local
-        )
-        
+        # Handle response based on streaming flag
+        if stream:
+            print_streaming_chat_result(model_cid, result, is_local)
+        else:
+            print_llm_chat_result(
+                model_cid, result.transaction_hash, result.finish_reason, result.chat_output, is_local
+            )
+
     except Exception as e:
         click.echo(f"Error running LLM chat inference: {str(e)}")
 
@@ -667,7 +693,7 @@ def print_llm_chat_result(model_cid, tx_hash, finish_reason, chat_output, is_loc
     click.echo("──────────────────────────────────────")
     click.echo("Model: ", nl=False)
     click.secho(model_cid, fg="cyan", bold=True)
-    
+
     if is_local and tx_hash != "external":
         click.echo("Transaction hash: ", nl=False)
         click.secho(tx_hash, fg="cyan", bold=True)
@@ -677,7 +703,7 @@ def print_llm_chat_result(model_cid, tx_hash, finish_reason, chat_output, is_loc
     else:
         click.echo("Source: ", nl=False)
         click.secho("External Provider", fg="cyan", bold=True)
-    
+
     click.echo("──────────────────────────────────────")
     click.secho("Finish Reason: ", fg="yellow", bold=True)
     click.echo()
@@ -691,6 +717,80 @@ def print_llm_chat_result(model_cid, tx_hash, finish_reason, chat_output, is_loc
     click.echo()
 
 
+def print_streaming_chat_result(model_cid, stream, is_local=True):
+    """Handle streaming chat response with typed chunks - prints in real-time"""
+    click.secho("🌊 Streaming LLM Chat", fg="green", bold=True)
+    click.echo("──────────────────────────────────────")
+    click.echo("Model: ", nl=False)
+    click.secho(model_cid, fg="cyan", bold=True)
+    
+    if is_local:
+        click.echo("Source: ", nl=False)
+        click.secho("OpenGradient TEE", fg="cyan", bold=True)
+    else:
+        click.echo("Source: ", nl=False)
+        click.secho("External Provider", fg="cyan", bold=True)
+    
+    click.echo("──────────────────────────────────────")
+    click.secho("Response:", fg="yellow", bold=True)
+    click.echo()
+    
+    try:
+        content_parts = []
+        chunk_count = 0
+        
+        for chunk in stream:
+            chunk_count += 1
+            
+            if chunk.choices[0].delta.content:
+                content = chunk.choices[0].delta.content
+                sys.stdout.write(content)
+                sys.stdout.flush()
+                content_parts.append(content)
+            
+            # Handle tool calls
+            if chunk.choices[0].delta.tool_calls:
+                sys.stdout.write("\n")
+                sys.stdout.flush()
+                click.secho("Tool Calls:", fg="yellow", bold=True)
+                for tool_call in chunk.choices[0].delta.tool_calls:
+                    click.echo(f"  Function: {tool_call['function']['name']}")
+                    click.echo(f"  Arguments: {tool_call['function']['arguments']}")
+            
+            # Print final info when stream completes
+            if chunk.is_final:
+                sys.stdout.write("\n\n")
+                sys.stdout.flush()
+                click.echo("──────────────────────────────────────")
+                
+                if chunk.usage:
+                    click.secho("Token Usage:", fg="cyan")
+                    click.echo(f"  Prompt tokens: {chunk.usage.prompt_tokens}")
+                    click.echo(f"  Completion tokens: {chunk.usage.completion_tokens}")
+                    click.echo(f"  Total tokens: {chunk.usage.total_tokens}")
+                    click.echo()
+                
+                if chunk.choices[0].finish_reason:
+                    click.echo("Finish reason: ", nl=False)
+                    click.secho(chunk.choices[0].finish_reason, fg="green")
+                
+                click.echo("──────────────────────────────────────")
+                click.echo(f"Chunks received: {chunk_count}")
+                click.echo(f"Content length: {len(''.join(content_parts))} characters")
+                click.echo()
+        
+    except KeyboardInterrupt:
+        sys.stdout.write("\n")
+        sys.stdout.flush()
+        click.secho("Stream interrupted by user", fg="yellow")
+        click.echo()
+    except Exception as e:
+        sys.stdout.write("\n")
+        sys.stdout.flush()
+        click.secho(f"Streaming error: {str(e)}", fg="red", bold=True)
+        click.echo()
+
+
 @cli.command()
 def create_account():
     """Create a new test account for OpenGradient inference and model management"""