robosystems-client 0.1.13__tar.gz → 0.1.14__tar.gz

{robosystems_client-0.1.13 → robosystems_client-0.1.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: robosystems-client
-Version: 0.1.13
+Version: 0.1.14
 Summary: Python Client for RoboSystems financial graph database API
 Author: Harbinger FinLab
 License: MIT

{robosystems_client-0.1.13 → robosystems_client-0.1.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "robosystems-client"
-version = "0.1.13"
+version = "0.1.14"
 description = "Python Client for RoboSystems financial graph database API"
 readme = "README.md"
 requires-python = ">=3.10"

{robosystems_client-0.1.13 → robosystems_client-0.1.14}/robosystems_client/api/copy/copy_data_to_graph.py

@@ -1,5 +1,5 @@
 from http import HTTPStatus
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, cast
 
 import httpx
 
@@ -50,11 +50,32 @@ def _get_kwargs(
 
 def _parse_response(
   *, client: Union[AuthenticatedClient, Client], response: httpx.Response
-) -> Optional[Union[CopyResponse, HTTPValidationError]]:
+) -> Optional[Union[Any, CopyResponse, HTTPValidationError]]:
   if response.status_code == 200:
     response_200 = CopyResponse.from_dict(response.json())
 
     return response_200
+  if response.status_code == 202:
+    response_202 = cast(Any, None)
+    return response_202
+  if response.status_code == 400:
+    response_400 = cast(Any, None)
+    return response_400
+  if response.status_code == 403:
+    response_403 = cast(Any, None)
+    return response_403
+  if response.status_code == 408:
+    response_408 = cast(Any, None)
+    return response_408
+  if response.status_code == 429:
+    response_429 = cast(Any, None)
+    return response_429
+  if response.status_code == 500:
+    response_500 = cast(Any, None)
+    return response_500
+  if response.status_code == 503:
+    response_503 = cast(Any, None)
+    return response_503
   if response.status_code == 422:
     response_422 = HTTPValidationError.from_dict(response.json())
 
@@ -67,7 +88,7 @@ def _parse_response(
 
 def _build_response(
   *, client: Union[AuthenticatedClient, Client], response: httpx.Response
-) -> Response[Union[CopyResponse, HTTPValidationError]]:
+) -> Response[Union[Any, CopyResponse, HTTPValidationError]]:
   return Response(
     status_code=HTTPStatus(response.status_code),
     content=response.content,
@@ -83,7 +104,7 @@ def sync_detailed(
   body: Union["DataFrameCopyRequest", "S3CopyRequest", "URLCopyRequest"],
   authorization: Union[None, Unset, str] = UNSET,
   auth_token: Union[None, Unset, str] = UNSET,
-) -> Response[Union[CopyResponse, HTTPValidationError]]:
+) -> Response[Union[Any, CopyResponse, HTTPValidationError]]:
   """Copy Data to Graph
 
    Copy data from external sources into the graph database.
@@ -105,10 +126,46 @@ def sync_detailed(
   - Premium: 100GB max file size, 60 min timeout
 
   **Copy Options:**
-  - `ignore_errors`: Skip duplicate/invalid rows (enables upsert-like behavior)
+  - `ignore_errors`: Skip duplicate/invalid rows (enables upsert-like behavior). Note: When enabled,
+  row counts may not be accurately reported
   - `extended_timeout`: Use extended timeout for large datasets
   - `validate_schema`: Validate source schema against target table
 
+  **Asynchronous Execution with SSE:**
+  For large data imports, this endpoint returns immediately with an operation ID
+  and SSE monitoring endpoint. Connect to the returned stream URL for real-time updates:
+
+  ```javascript
+  const eventSource = new EventSource('/v1/operations/{operation_id}/stream');
+  eventSource.onmessage = (event) => {
+    const data = JSON.parse(event.data);
+    console.log('Progress:', data.message);
+  };
+  ```
+
+  **SSE Events Emitted:**
+  - `operation_started`: Copy operation begins
+  - `operation_progress`: Progress updates during data transfer
+  - `operation_completed`: Copy successful with statistics
+  - `operation_error`: Copy failed with error details
+
+  **SSE Connection Limits:**
+  - Maximum 5 concurrent SSE connections per user
+  - Rate limited to 10 new connections per minute
+  - Automatic circuit breaker for Redis failures
+  - Graceful degradation if event system unavailable
+
+  **Error Handling:**
+  - `403 Forbidden`: Attempted copy to shared repository
+  - `408 Request Timeout`: Operation exceeded timeout limit
+  - `429 Too Many Requests`: Rate limit exceeded
+  - `503 Service Unavailable`: Circuit breaker open or service unavailable
+  - Clients should implement exponential backoff on errors
+
+  **Note:**
+  Copy operations are FREE - no credit consumption required.
+  All copy operations are performed asynchronously with progress monitoring.
+
   Args:
       graph_id (str): Target graph identifier (user graphs only - shared repositories not
           allowed)
@@ -121,7 +178,7 @@ def sync_detailed(
       httpx.TimeoutException: If the request takes longer than Client.timeout.
 
   Returns:
-      Response[Union[CopyResponse, HTTPValidationError]]
+      Response[Union[Any, CopyResponse, HTTPValidationError]]
   """
 
   kwargs = _get_kwargs(
@@ -145,7 +202,7 @@ def sync(
   body: Union["DataFrameCopyRequest", "S3CopyRequest", "URLCopyRequest"],
   authorization: Union[None, Unset, str] = UNSET,
   auth_token: Union[None, Unset, str] = UNSET,
-) -> Optional[Union[CopyResponse, HTTPValidationError]]:
+) -> Optional[Union[Any, CopyResponse, HTTPValidationError]]:
   """Copy Data to Graph
 
    Copy data from external sources into the graph database.
@@ -167,10 +224,46 @@ def sync(
   - Premium: 100GB max file size, 60 min timeout
 
   **Copy Options:**
-  - `ignore_errors`: Skip duplicate/invalid rows (enables upsert-like behavior)
+  - `ignore_errors`: Skip duplicate/invalid rows (enables upsert-like behavior). Note: When enabled,
+  row counts may not be accurately reported
   - `extended_timeout`: Use extended timeout for large datasets
   - `validate_schema`: Validate source schema against target table
 
+  **Asynchronous Execution with SSE:**
+  For large data imports, this endpoint returns immediately with an operation ID
+  and SSE monitoring endpoint. Connect to the returned stream URL for real-time updates:
+
+  ```javascript
+  const eventSource = new EventSource('/v1/operations/{operation_id}/stream');
+  eventSource.onmessage = (event) => {
+    const data = JSON.parse(event.data);
+    console.log('Progress:', data.message);
+  };
+  ```
+
+  **SSE Events Emitted:**
+  - `operation_started`: Copy operation begins
+  - `operation_progress`: Progress updates during data transfer
+  - `operation_completed`: Copy successful with statistics
+  - `operation_error`: Copy failed with error details
+
+  **SSE Connection Limits:**
+  - Maximum 5 concurrent SSE connections per user
+  - Rate limited to 10 new connections per minute
+  - Automatic circuit breaker for Redis failures
+  - Graceful degradation if event system unavailable
+
+  **Error Handling:**
+  - `403 Forbidden`: Attempted copy to shared repository
+  - `408 Request Timeout`: Operation exceeded timeout limit
+  - `429 Too Many Requests`: Rate limit exceeded
+  - `503 Service Unavailable`: Circuit breaker open or service unavailable
+  - Clients should implement exponential backoff on errors
+
+  **Note:**
+  Copy operations are FREE - no credit consumption required.
+  All copy operations are performed asynchronously with progress monitoring.
+
   Args:
       graph_id (str): Target graph identifier (user graphs only - shared repositories not
           allowed)
@@ -183,7 +276,7 @@ def sync(
       httpx.TimeoutException: If the request takes longer than Client.timeout.
 
   Returns:
-      Union[CopyResponse, HTTPValidationError]
+      Union[Any, CopyResponse, HTTPValidationError]
   """
 
   return sync_detailed(
@@ -202,7 +295,7 @@ async def asyncio_detailed(
   body: Union["DataFrameCopyRequest", "S3CopyRequest", "URLCopyRequest"],
   authorization: Union[None, Unset, str] = UNSET,
   auth_token: Union[None, Unset, str] = UNSET,
-) -> Response[Union[CopyResponse, HTTPValidationError]]:
+) -> Response[Union[Any, CopyResponse, HTTPValidationError]]:
   """Copy Data to Graph
 
    Copy data from external sources into the graph database.
@@ -224,10 +317,46 @@ async def asyncio_detailed(
   - Premium: 100GB max file size, 60 min timeout
 
   **Copy Options:**
-  - `ignore_errors`: Skip duplicate/invalid rows (enables upsert-like behavior)
+  - `ignore_errors`: Skip duplicate/invalid rows (enables upsert-like behavior). Note: When enabled,
+  row counts may not be accurately reported
   - `extended_timeout`: Use extended timeout for large datasets
   - `validate_schema`: Validate source schema against target table
 
+  **Asynchronous Execution with SSE:**
+  For large data imports, this endpoint returns immediately with an operation ID
+  and SSE monitoring endpoint. Connect to the returned stream URL for real-time updates:
+
+  ```javascript
+  const eventSource = new EventSource('/v1/operations/{operation_id}/stream');
+  eventSource.onmessage = (event) => {
+    const data = JSON.parse(event.data);
+    console.log('Progress:', data.message);
+  };
+  ```
+
+  **SSE Events Emitted:**
+  - `operation_started`: Copy operation begins
+  - `operation_progress`: Progress updates during data transfer
+  - `operation_completed`: Copy successful with statistics
+  - `operation_error`: Copy failed with error details
+
+  **SSE Connection Limits:**
+  - Maximum 5 concurrent SSE connections per user
+  - Rate limited to 10 new connections per minute
+  - Automatic circuit breaker for Redis failures
+  - Graceful degradation if event system unavailable
+
+  **Error Handling:**
+  - `403 Forbidden`: Attempted copy to shared repository
+  - `408 Request Timeout`: Operation exceeded timeout limit
+  - `429 Too Many Requests`: Rate limit exceeded
+  - `503 Service Unavailable`: Circuit breaker open or service unavailable
+  - Clients should implement exponential backoff on errors
+
+  **Note:**
+  Copy operations are FREE - no credit consumption required.
+  All copy operations are performed asynchronously with progress monitoring.
+
   Args:
       graph_id (str): Target graph identifier (user graphs only - shared repositories not
           allowed)
@@ -240,7 +369,7 @@ async def asyncio_detailed(
       httpx.TimeoutException: If the request takes longer than Client.timeout.
 
   Returns:
-      Response[Union[CopyResponse, HTTPValidationError]]
+      Response[Union[Any, CopyResponse, HTTPValidationError]]
   """
 
   kwargs = _get_kwargs(
@@ -262,7 +391,7 @@ async def asyncio(
   body: Union["DataFrameCopyRequest", "S3CopyRequest", "URLCopyRequest"],
   authorization: Union[None, Unset, str] = UNSET,
   auth_token: Union[None, Unset, str] = UNSET,
-) -> Optional[Union[CopyResponse, HTTPValidationError]]:
+) -> Optional[Union[Any, CopyResponse, HTTPValidationError]]:
   """Copy Data to Graph
 
    Copy data from external sources into the graph database.
@@ -284,10 +413,46 @@ async def asyncio(
   - Premium: 100GB max file size, 60 min timeout
 
   **Copy Options:**
-  - `ignore_errors`: Skip duplicate/invalid rows (enables upsert-like behavior)
+  - `ignore_errors`: Skip duplicate/invalid rows (enables upsert-like behavior). Note: When enabled,
+  row counts may not be accurately reported
   - `extended_timeout`: Use extended timeout for large datasets
   - `validate_schema`: Validate source schema against target table
 
+  **Asynchronous Execution with SSE:**
+  For large data imports, this endpoint returns immediately with an operation ID
+  and SSE monitoring endpoint. Connect to the returned stream URL for real-time updates:
+
+  ```javascript
+  const eventSource = new EventSource('/v1/operations/{operation_id}/stream');
+  eventSource.onmessage = (event) => {
+    const data = JSON.parse(event.data);
+    console.log('Progress:', data.message);
+  };
+  ```
+
+  **SSE Events Emitted:**
+  - `operation_started`: Copy operation begins
+  - `operation_progress`: Progress updates during data transfer
+  - `operation_completed`: Copy successful with statistics
+  - `operation_error`: Copy failed with error details
+
+  **SSE Connection Limits:**
+  - Maximum 5 concurrent SSE connections per user
+  - Rate limited to 10 new connections per minute
+  - Automatic circuit breaker for Redis failures
+  - Graceful degradation if event system unavailable
+
+  **Error Handling:**
+  - `403 Forbidden`: Attempted copy to shared repository
+  - `408 Request Timeout`: Operation exceeded timeout limit
+  - `429 Too Many Requests`: Rate limit exceeded
+  - `503 Service Unavailable`: Circuit breaker open or service unavailable
+  - Clients should implement exponential backoff on errors
+
+  **Note:**
+  Copy operations are FREE - no credit consumption required.
+  All copy operations are performed asynchronously with progress monitoring.
+
   Args:
       graph_id (str): Target graph identifier (user graphs only - shared repositories not
           allowed)
@@ -300,7 +465,7 @@ async def asyncio(
       httpx.TimeoutException: If the request takes longer than Client.timeout.
 
   Returns:
-      Union[CopyResponse, HTTPValidationError]
+      Union[Any, CopyResponse, HTTPValidationError]
   """
 
   return (

{robosystems_client-0.1.13 → robosystems_client-0.1.14}/robosystems_client/extensions/README.md

@@ -11,6 +11,7 @@ The RoboSystems Python Client Extensions provide enhanced functionality for the
 
 - **Server-Sent Events (SSE)** streaming with automatic reconnection
 - **Smart Query Execution** with automatic strategy selection
+- **Data Copy Operations** with S3 import and real-time progress tracking
 - **Operation Monitoring** for long-running operations  
 - **Connection Pooling** and intelligent resource management
 - **Result Processing** and format conversion utilities
@@ -84,6 +85,62 @@ async def main():
 asyncio.run(main())
 ```
 
+### Data Copy Operations
+
+```python
+from robosystems_client.extensions import CopyClient, CopyOptions
+from robosystems_client.models.s3_copy_request import S3CopyRequest
+from robosystems_client.models.s3_copy_request_file_format import S3CopyRequestFileFormat
+
+# Initialize copy client
+copy_client = CopyClient({
+    "base_url": "https://api.robosystems.ai",
+    "api_key": "your-api-key",
+})
+
+# Create S3 copy request
+request = S3CopyRequest(
+    table_name="companies",
+    s3_path="s3://my-bucket/data/companies.csv",
+    s3_access_key_id="AWS_ACCESS_KEY",
+    s3_secret_access_key="AWS_SECRET_KEY",
+    s3_region="us-east-1",
+    file_format=S3CopyRequestFileFormat.CSV,
+    ignore_errors=False,  # Stop on first error
+)
+
+# Set up progress callbacks
+def on_progress(message, percent):
+    if percent:
+        print(f"Progress: {message} ({percent}%)")
+    else:
+        print(f"Progress: {message}")
+
+def on_warning(warning):
+    print(f"Warning: {warning}")
+
+options = CopyOptions(
+    on_progress=on_progress,
+    on_warning=on_warning,
+)
+
+# Execute copy with progress monitoring
+result = copy_client.copy_from_s3("your_graph_id", request, options)
+
+# Check results
+if result.status == "completed":
+    print(f"✅ Successfully imported {result.rows_imported:,} rows")
+    stats = copy_client.calculate_statistics(result)
+    if stats:
+        print(f"Throughput: {stats.throughput:.2f} rows/second")
+elif result.status == "partial":
+    print(f"⚠️ Imported {result.rows_imported:,} rows, skipped {result.rows_skipped:,}")
+else:
+    print(f"❌ Copy failed: {result.error}")
+
+copy_client.close()
+```
+
 ## 🔐 Authentication
 
 ### API Key Authentication (Recommended)
@@ -141,6 +198,79 @@ dev_ext = create_extensions(
 
 ## 🛠 Advanced Features
 
+### Copy Operations with Advanced Features
+
+```python
+from robosystems_client.extensions import CopyClient, CopySourceType
+
+# Batch copy multiple tables
+copy_client = CopyClient({
+    "base_url": "https://api.robosystems.ai",
+    "api_key": "your-api-key",
+})
+
+copies = [
+    {
+        "request": S3CopyRequest(
+            table_name="companies",
+            s3_path="s3://bucket/companies.csv",
+            s3_access_key_id="KEY",
+            s3_secret_access_key="SECRET",
+            file_format=S3CopyRequestFileFormat.CSV,
+        ),
+    },
+    {
+        "request": S3CopyRequest(
+            table_name="transactions",
+            s3_path="s3://bucket/transactions.parquet",
+            s3_access_key_id="KEY",
+            s3_secret_access_key="SECRET",
+            file_format=S3CopyRequestFileFormat.PARQUET,
+            ignore_errors=True,  # Continue on errors
+        ),
+    },
+]
+
+# Execute batch copy
+results = copy_client.batch_copy_from_s3("graph_id", copies)
+
+for i, result in enumerate(results):
+    table_name = copies[i]["request"].table_name
+    print(f"{table_name}: {result.status}")
+    if result.rows_imported:
+        print(f"  Imported: {result.rows_imported:,} rows")
+
+# Copy with retry logic for resilient operations
+result = copy_client.copy_with_retry(
+    graph_id="graph_id",
+    request=S3CopyRequest(
+        table_name="large_dataset",
+        s3_path="s3://bucket/large-dataset.csv",
+        s3_access_key_id="KEY",
+        s3_secret_access_key="SECRET",
+        max_file_size_gb=50,
+        extended_timeout=True,
+    ),
+    source_type=CopySourceType.S3,
+    max_retries=3,
+    options=CopyOptions(
+        on_progress=lambda msg, _: print(msg)
+    ),
+)
+
+# Monitor multiple concurrent copy operations
+operation_ids = ["op-123", "op-456", "op-789"]
+results = copy_client.monitor_multiple_copies(operation_ids, options)
+
+for op_id, result in results.items():
+    print(f"Operation {op_id}: {result.status}")
+    if result.status == "completed":
+        stats = copy_client.calculate_statistics(result)
+        print(f"  Throughput: {stats.throughput:.2f} rows/sec")
+
+copy_client.close()
+```
+
 ### Query Builder
 
 Build complex Cypher queries programmatically:
@@ -273,6 +403,87 @@ client.close()
 
 ## 📊 Examples
 
+### Data Import with Real-Time Monitoring
+
+```python
+from robosystems_client.extensions import CopyClient, CopyOptions
+import time
+
+def import_financial_data():
+    """Import financial data with comprehensive monitoring"""
+    
+    copy_client = CopyClient({
+        "base_url": "https://api.robosystems.ai",
+        "api_key": "your-api-key",
+    })
+    
+    # Track progress history
+    progress_history = []
+    warnings_count = 0
+    
+    def on_progress(message, percent):
+        timestamp = time.strftime("%H:%M:%S")
+        progress_history.append({
+            "time": timestamp,
+            "message": message,
+            "percent": percent,
+        })
+        print(f"[{timestamp}] {message}" + (f" ({percent}%)" if percent else ""))
+    
+    def on_warning(warning):
+        nonlocal warnings_count
+        warnings_count += 1
+        print(f"⚠️  Warning #{warnings_count}: {warning}")
+    
+    def on_queue_update(position, wait_time):
+        print(f"📊 Queue position: {position} (ETA: {wait_time}s)")
+    
+    # Configure copy with all callbacks
+    options = CopyOptions(
+        on_progress=on_progress,
+        on_warning=on_warning,
+        on_queue_update=on_queue_update,
+        timeout=1800000,  # 30 minutes
+    )
+    
+    # Execute copy operation
+    start_time = time.time()
+    
+    result = copy_client.copy_s3(
+        graph_id="financial_graph",
+        table_name="quarterly_reports",
+        s3_path="s3://financial-data/reports-2024-q1.parquet",
+        access_key_id="AWS_KEY",
+        secret_access_key="AWS_SECRET",
+        file_format="parquet",
+        ignore_errors=True,  # Continue on validation errors
+    )
+    
+    # Print summary
+    elapsed = time.time() - start_time
+    
+    print("\n" + "="*50)
+    print("📈 IMPORT SUMMARY")
+    print("="*50)
+    print(f"Status: {result.status.upper()}")
+    print(f"Rows Imported: {result.rows_imported or 0:,}")
+    print(f"Rows Skipped: {result.rows_skipped or 0:,}")
+    print(f"Warnings: {warnings_count}")
+    print(f"Execution Time: {elapsed:.2f} seconds")
+    
+    if result.status == "completed":
+        stats = copy_client.calculate_statistics(result)
+        if stats:
+            print(f"Throughput: {stats.throughput:.2f} rows/second")
+            print(f"Data Processed: {stats.bytes_processed / (1024*1024):.2f} MB")
+    
+    copy_client.close()
+    return result
+
+# Run the import
+result = import_financial_data()
+```
+
 ### Financial Data Analysis
 
 ```python

{robosystems_client-0.1.13 → robosystems_client-0.1.14}/robosystems_client/extensions/__init__.py

@@ -20,6 +20,14 @@ from .operation_client import (
   OperationProgress,
   OperationResult,
 )
+from .copy_client import (
+  CopyClient,
+  AsyncCopyClient,
+  CopySourceType,
+  CopyOptions,
+  CopyResult,
+  CopyStatistics,
+)
 from .extensions import (
   RoboSystemsExtensions,
   RoboSystemsExtensionConfig,
@@ -68,6 +76,13 @@ __all__ = [
   "OperationStatus",
   "OperationProgress",
   "OperationResult",
+  # Copy Client
+  "CopyClient",
+  "AsyncCopyClient",
+  "CopySourceType",
+  "CopyOptions",
+  "CopyResult",
+  "CopyStatistics",
   # Utilities
   "QueryBuilder",
   "ResultProcessor",
@@ -106,3 +121,17 @@ def execute_query(graph_id: str, query: str, parameters=None):
 def stream_query(graph_id: str, query: str, parameters=None, chunk_size=None):
   """Stream a query using the default extensions instance"""
   return extensions.query.stream_query(graph_id, query, parameters, chunk_size)
+
+
+def copy_from_s3(
+  graph_id: str,
+  table_name: str,
+  s3_path: str,
+  access_key_id: str,
+  secret_access_key: str,
+  **kwargs,
+):
+  """Copy data from S3 using the default extensions instance"""
+  return extensions.copy_from_s3(
+    graph_id, table_name, s3_path, access_key_id, secret_access_key, **kwargs
+  )