fustor-fusion-sdk 0.1.2.post3__tar.gz

fustor_fusion_sdk-0.1.2.post3/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: fustor-fusion-sdk
+Version: 0.1.2.post3
+Summary: SDK for Fustor Fusion service
+License-Expression: MIT
+Requires-Dist: fustor-common
+Requires-Dist: fustor-registry-client

fustor_fusion_sdk-0.1.2.post3/README.md

@@ -0,0 +1,39 @@
+# fustor-fusion-sdk
+
+This package provides a Software Development Kit (SDK) for interacting with the Fustor Fusion service. It offers a client and interfaces to facilitate programmatic access and integration with the Fusion service's functionalities, such as data ingestion and processing.
+
+## Features
+
+*   **Client**: A Python client for making requests to the Fustor Fusion API.
+*   **Interfaces**: Defines abstract interfaces for various components of the Fustor Fusion service, allowing for consistent interaction patterns.
+
+## Installation
+
+This package is part of the Fustor monorepo and is typically installed in editable mode within the monorepo's development environment using `uv sync`.
+
+## Usage
+
+Developers can use this SDK to build custom applications or integrations that need to communicate with the Fustor Fusion service. It simplifies the process of sending data to Fusion and interacting with its processing capabilities.
+
+Example (conceptual):
+
+```python
+from fustor_fusion_sdk.client import FusionClient
+from fustor_fusion_sdk.models import IngestDataRequest
+
+# Assuming FusionClient is initialized with the Fusion service URL
+client = FusionClient(base_url="http://localhost:8102")
+
+# Example: Ingest data
+data_to_ingest = IngestDataRequest(
+    session_id="some-session-id",
+    events=[{"key": "value", "timestamp": 1678886400}]
+)
+response = client.ingest_data(data_to_ingest)
+print(response)
+```
+
+## Dependencies
+
+*   `fustor-common`: Provides foundational elements and shared components.
+*   `fustor-registry-client`: (If applicable) Used for interacting with the Fustor Registry service, which might be a dependency for Fusion configuration or session management.

fustor_fusion_sdk-0.1.2.post3/pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "fustor-fusion-sdk"
+dynamic = ["version"]
+description = "SDK for Fustor Fusion service"
+license = "MIT"
+dependencies = [ "fustor-common", "fustor-registry-client",]
+
+[build-system]
+requires = [ "setuptools>=61.0", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = "../.."
+version_scheme = "post-release"
+local_scheme = "dirty-tag"
+
+["project.urls"]
+Homepage = "https://github.com/excelwang/fustor/tree/master/packages/fustor_fusion_sdk"
+"Bug Tracker" = "https://github.com/excelwang/fustor/issues"
+
+[tool.setuptools.packages.find]
+where = [ "src",]

fustor_fusion_sdk-0.1.2.post3/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

fustor_fusion_sdk-0.1.2.post3/src/fustor_fusion_sdk/client.py

@@ -0,0 +1,101 @@
+
+import httpx
+import logging
+from typing import Optional, List, Dict, Any
+
+logger = logging.getLogger(__name__)
+
+def contains_surrogate_characters(text: str) -> bool:
+    """Check if text contains surrogate characters."""
+    try:
+        text.encode('utf-8')
+        return False
+    except UnicodeEncodeError:
+        return True
+
+def sanitize_surrogate_characters(obj: Any) -> Any:
+    """
+    Recursively sanitize an object by replacing surrogate characters with safe alternatives.
+    """
+    if isinstance(obj, str):
+        if contains_surrogate_characters(obj):
+            # Encode with replacement and decode back to handle surrogate characters
+            return obj.encode('utf-8', errors='replace').decode('utf-8')
+        return obj
+    elif isinstance(obj, dict):
+        return {key: sanitize_surrogate_characters(value) for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [sanitize_surrogate_characters(item) for item in obj]
+    elif isinstance(obj, tuple):
+        return tuple(sanitize_surrogate_characters(item) for item in obj)
+    else:
+        return obj
+
+class FusionClient:
+    def __init__(self, base_url: str, api_key: str):
+        self.base_url = base_url
+        self.api_key = api_key
+        self.client = httpx.AsyncClient(base_url=self.base_url, headers={"X-API-Key": self.api_key})
+
+    async def create_session(self, task_id: str) -> Optional[str]:
+        """
+        Creates a new session and returns the session ID.
+        """
+        try:
+            # Sanitize task_id to handle any surrogate characters before JSON serialization
+            payload = {"task_id": task_id}
+            response = await self.client.post("/ingestor-api/v1/sessions/", json=payload)
+            response.raise_for_status()
+            return response.json().get("session_id")
+        except httpx.HTTPStatusError as e:
+            logger.error(f"HTTP error occurred: {e.response.status_code} - {e.response.text}")
+            return None
+        except Exception as e:
+            logger.error(f"An error occurred: {e}")
+            return None
+
+    async def push_events(self, session_id: str, events: List[Dict[str, Any]], source_type: str) -> bool:
+        """
+        Pushes a batch of events to the Fusion service.
+        """
+        try:
+            # Sanitize events to handle any surrogate characters before JSON serialization
+            sanitized_events = [sanitize_surrogate_characters(event) for event in events]
+            sanitized_source_type = sanitize_surrogate_characters(source_type)
+
+            payload = {
+                "session_id": session_id,
+                "events": sanitized_events,
+                "source_type": sanitized_source_type
+            }
+            response = await self.client.post("/ingestor-api/v1/events/", json=payload)
+            response.raise_for_status()
+            return True
+        except httpx.HTTPStatusError as e:
+            logger.error(f"HTTP error occurred during event push: {e.response.status_code} - {e.response.text}")
+            return False
+        except Exception as e:
+            logger.error(f"An error occurred during event push: {e}")
+            return False
+
+    async def send_heartbeat(self, session_id: str) -> bool:
+        """
+        Sends a heartbeat to the Fusion service to keep the session alive.
+        """
+        try:
+            headers = {"session-id": session_id}
+            response = await self.client.post("/ingestor-api/v1/sessions/heartbeat", headers=headers)
+            response.raise_for_status()
+            return True
+        except httpx.HTTPStatusError as e:
+            logger.error(f"HTTP error occurred during heartbeat: {e.response.status_code} - {e.response.text}")
+            return False
+        except Exception as e:
+            logger.error(f"An error occurred during heartbeat: {e}")
+            return False
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.client.aclose()

fustor_fusion_sdk-0.1.2.post3/src/fustor_fusion_sdk/interfaces.py

@@ -0,0 +1,92 @@
+from typing import Dict, Optional, List, Any, Protocol
+from dataclasses import dataclass
+import asyncio
+
+@dataclass
+class SessionInfo:
+    session_id: str
+    datastore_id: int
+    last_activity: float
+    created_at: float
+    task_id: Optional[str] = None
+    allow_concurrent_push: Optional[bool] = None
+    session_timeout_seconds: Optional[int] = None
+    client_ip: Optional[str] = None
+    cleanup_task: Optional[asyncio.Task] = None
+
+class ApiKeyCacheInterface(Protocol):
+    """
+    Interface for managing API key cache.
+    """
+    def set_cache(self, api_keys_data: List[Dict[str, Any]]):
+        ...
+
+    def get_datastore_id(self, api_key: str) -> Optional[int]:
+        ...
+
+from fustor_common.models import DatastoreConfig
+from fustor_registry_client.models import ClientDatastoreConfigResponse
+
+class DatastoreConfigCacheInterface(Protocol):
+    """
+    Interface for managing datastore config cache.
+    """
+    def set_cache(self, datastore_configs_data: List[ClientDatastoreConfigResponse]):
+        ...
+
+    def get_datastore_config(self, datastore_id: int) -> Optional[DatastoreConfig]:
+        ...
+
+from fustor_registry_client.models import ClientDatastoreConfigResponse
+
+class ParserProcessingTaskManagerInterface(Protocol):
+    """
+    Interface for managing datastore processing tasks.
+    """
+    async def start_processing_for_datastore(self, datastore_id: int):
+        ...
+
+    async def stop_processing_for_datastore(self, datastore_id: int):
+        ...
+
+    async def sync_tasks(self, latest_datastore_configs: List[ClientDatastoreConfigResponse]):
+        ...
+
+    async def shutdown(self):
+        ...
+
+class SessionManagerInterface(Protocol):
+    """
+    Interface for managing user sessions.
+    """
+    async def create_session_entry(self, datastore_id: int, session_id: str, 
+                                 task_id: Optional[str] = None, 
+                                 client_ip: Optional[str] = None,
+                                 allow_concurrent_push: Optional[bool] = None,
+                                 session_timeout_seconds: Optional[int] = None) -> SessionInfo:
+        ...
+
+    async def keep_session_alive(self, datastore_id: int, session_id: str, 
+                               client_ip: Optional[str] = None) -> Optional[SessionInfo]:
+        ...
+
+    async def get_session_info(self, datastore_id: int, session_id: str) -> Optional[SessionInfo]:
+        ...
+
+    async def get_datastore_sessions(self, datastore_id: int) -> Dict[str, SessionInfo]:
+        ...
+
+    async def remove_session(self, datastore_id: int, session_id: str) -> bool:
+        ...
+
+    async def cleanup_expired_sessions(self):
+        ...
+
+    async def terminate_session(self, datastore_id: int, session_id: str) -> bool:
+        ...
+
+    async def start_periodic_cleanup(self, interval_seconds: int = 60):
+        ...
+
+    async def stop_periodic_cleanup(self):
+        ...

fustor_fusion_sdk-0.1.2.post3/src/fustor_fusion_sdk.egg-info/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.4
+Name: fustor-fusion-sdk
+Version: 0.1.2.post3
+Summary: SDK for Fustor Fusion service
+License-Expression: MIT
+Requires-Dist: fustor-common
+Requires-Dist: fustor-registry-client

fustor_fusion_sdk-0.1.2.post3/src/fustor_fusion_sdk.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+src/fustor_fusion_sdk/__init__.py
+src/fustor_fusion_sdk/client.py
+src/fustor_fusion_sdk/interfaces.py
+src/fustor_fusion_sdk.egg-info/PKG-INFO
+src/fustor_fusion_sdk.egg-info/SOURCES.txt
+src/fustor_fusion_sdk.egg-info/dependency_links.txt
+src/fustor_fusion_sdk.egg-info/requires.txt
+src/fustor_fusion_sdk.egg-info/top_level.txt
+tests/test_sdk_surrogate_handling.py

fustor_fusion_sdk-0.1.2.post3/src/fustor_fusion_sdk.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

fustor_fusion_sdk-0.1.2.post3/src/fustor_fusion_sdk.egg-info/requires.txt

@@ -0,0 +1,2 @@
+fustor-common
+fustor-registry-client

fustor_fusion_sdk-0.1.2.post3/src/fustor_fusion_sdk.egg-info/top_level.txt

@@ -0,0 +1 @@
+fustor_fusion_sdk

fustor_fusion_sdk-0.1.2.post3/tests/test_sdk_surrogate_handling.py

@@ -0,0 +1,147 @@
+"""
+Test case to verify that the fustor_fusion_sdk correctly handles surrogate characters
+"""
+import pytest
+import asyncio
+from unittest.mock import AsyncMock, patch
+from fustor_fusion_sdk.client import FusionClient, sanitize_surrogate_characters
+
+
+def test_sanitize_surrogate_characters_string():
+    """Test that the sanitize_surrogate_characters function handles strings with surrogate characters."""
+    # Test string with surrogate character
+    test_str = "valid_text_\udca3_invalid_surrogate"
+    result = sanitize_surrogate_characters(test_str)
+    
+    # The result should not contain the invalid surrogate character
+    # Instead it should contain a replacement character (usually )
+    assert "" in result  # Replacement character
+    assert "_invalid_surrogate" in result  # Valid part remains
+    assert "\udca3" not in result  # Original surrogate char is gone
+
+
+def test_sanitize_surrogate_characters_dict():
+    """Test that the sanitize_surrogate_characters function handles dictionaries with surrogate characters."""
+    test_dict = {
+        "valid_key": "valid_value",
+        "key_with_surrogate": "path_with_\udca3_surrogate",
+        "another_valid": "value_without_surrogates"
+    }
+    
+    result = sanitize_surrogate_characters(test_dict)
+    
+    # Valid entries should remain unchanged
+    assert result["valid_key"] == "valid_value"
+    assert result["another_valid"] == "value_without_surrogates"
+    
+    # The entry with surrogate should be cleaned
+    assert "" in result["key_with_surrogate"]
+    assert "_surrogate" in result["key_with_surrogate"]
+    assert "\udca3" not in result["key_with_surrogate"]
+
+
+def test_sanitize_surrogate_characters_nested():
+    """Test that the sanitize_surrogate_characters function handles nested data structures."""
+    test_data = {
+        "level1": {
+            "level2": [
+                "normal_string",
+                "string_with_\udca3_surrogate",
+                {"nested_key": "nested_\udcb3_value"}
+            ]
+        }
+    }
+    
+    result = sanitize_surrogate_characters(test_data)
+    
+    # Check deeply nested values are cleaned
+    assert "" in result["level1"]["level2"][1]  # First surrogate string
+    assert "" in result["level1"]["level2"][2]["nested_key"]  # Nested surrogate
+    assert "\udca3" not in result["level1"]["level2"][1]
+    assert "\udcb3" not in result["level1"]["level2"][2]["nested_key"]
+    
+    # Valid strings remain unchanged
+    assert result["level1"]["level2"][0] == "normal_string"
+
+
+def test_sanitize_surrogate_characters_no_surrogates():
+    """Test that the sanitize_surrogate_characters function doesn't modify strings without surrogates."""
+    test_str = "normal_string_without_surrogates"
+    result = sanitize_surrogate_characters(test_str)
+    assert result == test_str  # Should remain unchanged
+
+
+def test_sanitize_surrogate_characters_other_types():
+    """Test that the sanitize_surrogate_characters function handles non-string types correctly."""
+    test_data = {
+        "string_val": "with_\udca3_surrogate",
+        "int_val": 42,
+        "bool_val": True,
+        "none_val": None,
+        "list_val": ["item1", "item2_\udca4_with_surrogate"]
+    }
+    
+    result = sanitize_surrogate_characters(test_data)
+    
+    # Non-string types should remain unchanged
+    assert result["int_val"] == 42
+    assert result["bool_val"] is True
+    assert result["none_val"] is None
+    
+    # Strings should be cleaned
+    assert "" in result["string_val"]
+    assert "" in result["list_val"][1]
+
+
+@pytest.mark.asyncio
+async def test_push_events_with_surrogate_characters():
+    """Test that push_events properly sanitizes events containing surrogate characters."""
+    client = FusionClient("http://test.com", "fake-api-key")
+
+    # Mock the HTTP client to avoid actual network calls
+    mock_response = AsyncMock()
+    # In httpx, raise_for_status is a sync method, so we mock it differently
+    mock_response.raise_for_status = lambda: None  # This is a sync method that raises on failure
+    mock_response.json.return_value = {"session_id": "test-session"}
+
+    with patch.object(client.client, 'post', return_value=mock_response) as mock_post:
+        # Create test events with surrogate characters
+        test_events = [
+            {
+                "event_type": "update",
+                "event_schema": "test_schema",
+                "table": "test_table",
+                "rows": [{"path": "/valid/path/with_\udca3_surrogate/file.txt", "size": 100}]
+            }
+        ]
+
+        # This should not raise an exception due to surrogate characters
+        result = await client.push_events("test_session", test_events, "test_source")
+
+        # Verify that the call was made
+        assert result is True
+        mock_post.assert_called_once()
+
+        # Get the arguments passed to the post call
+        call_args = mock_post.call_args
+        payload = call_args[1]['json']  # Get the JSON payload
+
+        # The surrogate character should be removed/replaced in the payload
+        event_path = payload['events'][0]['rows'][0]['path']
+        assert "" in event_path  # Replacement character should be present
+        assert "\udca3" not in event_path  # Original surrogate should be removed
+
+
+
+def test_contains_surrogate_characters_detection():
+    """Test that contains_surrogate_characters correctly detects surrogate characters."""
+    from fustor_fusion_sdk.client import contains_surrogate_characters
+    
+    # String with surrogate character should return True
+    assert contains_surrogate_characters("test_\udca3_string") is True
+    
+    # String without surrogate characters should return False
+    assert contains_surrogate_characters("test_string_without_surrogates") is False
+    
+    # Empty string should return False
+    assert contains_surrogate_characters("") is False