robosystems-client 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

robosystems_client/extensions/README.md

@@ -11,6 +11,7 @@ The RoboSystems Python Client Extensions provide enhanced functionality for the
 
 - **Server-Sent Events (SSE)** streaming with automatic reconnection
 - **Smart Query Execution** with automatic strategy selection
+- **Data Copy Operations** with S3 import and real-time progress tracking
 - **Operation Monitoring** for long-running operations  
 - **Connection Pooling** and intelligent resource management
 - **Result Processing** and format conversion utilities
@@ -84,6 +85,62 @@ async def main():
 asyncio.run(main())
 ```
 
+### Data Copy Operations
+
+```python
+from robosystems_client.extensions import CopyClient, CopyOptions
+from robosystems_client.models.s3_copy_request import S3CopyRequest
+from robosystems_client.models.s3_copy_request_file_format import S3CopyRequestFileFormat
+
+# Initialize copy client
+copy_client = CopyClient({
+    "base_url": "https://api.robosystems.ai",
+    "api_key": "your-api-key",
+})
+
+# Create S3 copy request
+request = S3CopyRequest(
+    table_name="companies",
+    s3_path="s3://my-bucket/data/companies.csv",
+    s3_access_key_id="AWS_ACCESS_KEY",
+    s3_secret_access_key="AWS_SECRET_KEY",
+    s3_region="us-east-1",
+    file_format=S3CopyRequestFileFormat.CSV,
+    ignore_errors=False,  # Stop on first error
+)
+
+# Set up progress callbacks
+def on_progress(message, percent):
+    if percent:
+        print(f"Progress: {message} ({percent}%)")
+    else:
+        print(f"Progress: {message}")
+
+def on_warning(warning):
+    print(f"Warning: {warning}")
+
+options = CopyOptions(
+    on_progress=on_progress,
+    on_warning=on_warning,
+)
+
+# Execute copy with progress monitoring
+result = copy_client.copy_from_s3("your_graph_id", request, options)
+
+# Check results
+if result.status == "completed":
+    print(f"✅ Successfully imported {result.rows_imported:,} rows")
+    stats = copy_client.calculate_statistics(result)
+    if stats:
+        print(f"Throughput: {stats.throughput:.2f} rows/second")
+elif result.status == "partial":
+    print(f"⚠️ Imported {result.rows_imported:,} rows, skipped {result.rows_skipped:,}")
+else:
+    print(f"❌ Copy failed: {result.error}")
+
+copy_client.close()
+```
+
 ## 🔐 Authentication
 
 ### API Key Authentication (Recommended)
@@ -141,6 +198,79 @@ dev_ext = create_extensions(
 
 ## 🛠 Advanced Features
 
+### Copy Operations with Advanced Features
+
+```python
+from robosystems_client.extensions import CopyClient, CopySourceType
+
+# Batch copy multiple tables
+copy_client = CopyClient({
+    "base_url": "https://api.robosystems.ai",
+    "api_key": "your-api-key",
+})
+
+copies = [
+    {
+        "request": S3CopyRequest(
+            table_name="companies",
+            s3_path="s3://bucket/companies.csv",
+            s3_access_key_id="KEY",
+            s3_secret_access_key="SECRET",
+            file_format=S3CopyRequestFileFormat.CSV,
+        ),
+    },
+    {
+        "request": S3CopyRequest(
+            table_name="transactions",
+            s3_path="s3://bucket/transactions.parquet",
+            s3_access_key_id="KEY",
+            s3_secret_access_key="SECRET",
+            file_format=S3CopyRequestFileFormat.PARQUET,
+            ignore_errors=True,  # Continue on errors
+        ),
+    },
+]
+
+# Execute batch copy
+results = copy_client.batch_copy_from_s3("graph_id", copies)
+
+for i, result in enumerate(results):
+    table_name = copies[i]["request"].table_name
+    print(f"{table_name}: {result.status}")
+    if result.rows_imported:
+        print(f"  Imported: {result.rows_imported:,} rows")
+
+# Copy with retry logic for resilient operations
+result = copy_client.copy_with_retry(
+    graph_id="graph_id",
+    request=S3CopyRequest(
+        table_name="large_dataset",
+        s3_path="s3://bucket/large-dataset.csv",
+        s3_access_key_id="KEY",
+        s3_secret_access_key="SECRET",
+        max_file_size_gb=50,
+        extended_timeout=True,
+    ),
+    source_type=CopySourceType.S3,
+    max_retries=3,
+    options=CopyOptions(
+        on_progress=lambda msg, _: print(msg)
+    ),
+)
+
+# Monitor multiple concurrent copy operations
+operation_ids = ["op-123", "op-456", "op-789"]
+results = copy_client.monitor_multiple_copies(operation_ids, options)
+
+for op_id, result in results.items():
+    print(f"Operation {op_id}: {result.status}")
+    if result.status == "completed":
+        stats = copy_client.calculate_statistics(result)
+        print(f"  Throughput: {stats.throughput:.2f} rows/sec")
+
+copy_client.close()
+```
+
 ### Query Builder
 
 Build complex Cypher queries programmatically:
@@ -273,6 +403,87 @@ client.close()
 
 ## 📊 Examples
 
+### Data Import with Real-Time Monitoring
+
+```python
+from robosystems_client.extensions import CopyClient, CopyOptions
+import time
+
+def import_financial_data():
+    """Import financial data with comprehensive monitoring"""
+    
+    copy_client = CopyClient({
+        "base_url": "https://api.robosystems.ai",
+        "api_key": "your-api-key",
+    })
+    
+    # Track progress history
+    progress_history = []
+    warnings_count = 0
+    
+    def on_progress(message, percent):
+        timestamp = time.strftime("%H:%M:%S")
+        progress_history.append({
+            "time": timestamp,
+            "message": message,
+            "percent": percent,
+        })
+        print(f"[{timestamp}] {message}" + (f" ({percent}%)" if percent else ""))
+    
+    def on_warning(warning):
+        nonlocal warnings_count
+        warnings_count += 1
+        print(f"⚠️  Warning #{warnings_count}: {warning}")
+    
+    def on_queue_update(position, wait_time):
+        print(f"📊 Queue position: {position} (ETA: {wait_time}s)")
+    
+    # Configure copy with all callbacks
+    options = CopyOptions(
+        on_progress=on_progress,
+        on_warning=on_warning,
+        on_queue_update=on_queue_update,
+        timeout=1800000,  # 30 minutes
+    )
+    
+    # Execute copy operation
+    start_time = time.time()
+    
+    result = copy_client.copy_s3(
+        graph_id="financial_graph",
+        table_name="quarterly_reports",
+        s3_path="s3://financial-data/reports-2024-q1.parquet",
+        access_key_id="AWS_KEY",
+        secret_access_key="AWS_SECRET",
+        file_format="parquet",
+        ignore_errors=True,  # Continue on validation errors
+    )
+    
+    # Print summary
+    elapsed = time.time() - start_time
+    
+    print("\n" + "="*50)
+    print("📈 IMPORT SUMMARY")
+    print("="*50)
+    print(f"Status: {result.status.upper()}")
+    print(f"Rows Imported: {result.rows_imported or 0:,}")
+    print(f"Rows Skipped: {result.rows_skipped or 0:,}")
+    print(f"Warnings: {warnings_count}")
+    print(f"Execution Time: {elapsed:.2f} seconds")
+    
+    if result.status == "completed":
+        stats = copy_client.calculate_statistics(result)
+        if stats:
+            print(f"Throughput: {stats.throughput:.2f} rows/second")
+            print(f"Data Processed: {stats.bytes_processed / (1024*1024):.2f} MB")
+    
+    copy_client.close()
+    return result
+
+# Run the import
+result = import_financial_data()
+```
+
 ### Financial Data Analysis
 
 ```python

robosystems_client/extensions/__init__.py

@@ -20,6 +20,14 @@ from .operation_client import (
   OperationProgress,
   OperationResult,
 )
+from .copy_client import (
+  CopyClient,
+  AsyncCopyClient,
+  CopySourceType,
+  CopyOptions,
+  CopyResult,
+  CopyStatistics,
+)
 from .extensions import (
   RoboSystemsExtensions,
   RoboSystemsExtensionConfig,
@@ -68,6 +76,13 @@ __all__ = [
   "OperationStatus",
   "OperationProgress",
   "OperationResult",
+  # Copy Client
+  "CopyClient",
+  "AsyncCopyClient",
+  "CopySourceType",
+  "CopyOptions",
+  "CopyResult",
+  "CopyStatistics",
   # Utilities
   "QueryBuilder",
   "ResultProcessor",
@@ -106,3 +121,17 @@ def execute_query(graph_id: str, query: str, parameters=None):
 def stream_query(graph_id: str, query: str, parameters=None, chunk_size=None):
   """Stream a query using the default extensions instance"""
   return extensions.query.stream_query(graph_id, query, parameters, chunk_size)
+
+
+def copy_from_s3(
+  graph_id: str,
+  table_name: str,
+  s3_path: str,
+  access_key_id: str,
+  secret_access_key: str,
+  **kwargs,
+):
+  """Copy data from S3 using the default extensions instance"""
+  return extensions.copy_from_s3(
+    graph_id, table_name, s3_path, access_key_id, secret_access_key, **kwargs
+  )