fustor-source-oss 0.2__py3-none-any.whl

fustor_source_oss/config.py

@@ -0,0 +1,70 @@
+from pydantic import BaseModel, Field, AnyHttpUrl, ValidationError, model_validator
+from typing import Optional, Literal, Dict, Any
+from enum import Enum
+
+class QueueType(str, Enum):
+    """Defines the supported queue types for incremental synchronization."""
+    POLLING = "polling"  # Simple polling based on object modification time
+    SQS = "sqs"          # AWS SQS for event notifications
+    AMQP = "amqp"        # AMQP (e.g., RabbitMQ) for event notifications
+
+class SQSQueueConfig(BaseModel):
+    """Configuration for AWS SQS queue."""
+    queue_url: str = Field(..., description="URL of the SQS queue.")
+    region_name: Optional[str] = Field(None, description="AWS region of the SQS queue.")
+    visibility_timeout: int = Field(30, ge=0, description="Message visibility timeout in seconds.")
+
+class AMQPQueueConfig(BaseModel):
+    """Configuration for AMQP (RabbitMQ) queue."""
+    host: str = Field(..., description="AMQP broker hostname.")
+    port: int = Field(5672, gt=0, description="AMQP broker port.")
+    username: str = Field("guest", description="AMQP username.")
+    password: str = Field("guest", description="AMQP password.")
+    queue_name: str = Field(..., description="Name of the queue to consume.")
+    
+class PollingQueueConfig(BaseModel):
+    """Configuration for polling-based incremental sync."""
+    interval_seconds: int = Field(30, ge=1, description="Polling interval in seconds.")
+
+class OssDriverParams(BaseModel):
+    """
+    Configuration parameters specific to the OSS Source Driver.
+    These parameters are stored within the 'driver_params' field of the core SourceConfig.
+    """
+    endpoint_url: str = Field(..., description="S3-compatible service endpoint URL.")
+    bucket_name: str = Field(..., min_length=3, description="Name of the S3 bucket.")
+    region_name: Optional[str] = Field(None, description="AWS region of the S3 bucket.")
+    prefix: str = Field("", description="Optional object key prefix to filter files.")
+    recursive: bool = Field(True, description="Whether to recursively list objects within the prefix.")
+    
+    # Incremental synchronization strategy
+    queue_type: QueueType = Field(QueueType.POLLING, description="Strategy for incremental synchronization.")
+    
+    # Specific queue configurations, validated conditionally
+    sqs_queue_config: Optional[SQSQueueConfig] = Field(None, description="SQS queue configuration if queue_type is SQS.")
+    amqp_queue_config: Optional[AMQPQueueConfig] = Field(None, description="AMQP queue configuration if queue_type is AMQP.")
+    polling_queue_config: Optional[PollingQueueConfig] = Field(None, description="Polling queue configuration if queue_type is POLLING.")
+
+    # Custom validator to ensure correct queue config is provided based on queue_type
+    @classmethod
+    def validate_queue_config(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        queue_type = values.get("queue_type")
+        if queue_type == QueueType.SQS:
+            if not values.get("sqs_queue_config"):
+                raise ValueError("sqs_queue_config must be provided for SQS queue_type.")
+            # Ensure other queue configs are not present if SQS is chosen
+            values["amqp_queue_config"] = None
+            values["polling_queue_config"] = None
+        elif queue_type == QueueType.AMQP:
+            if not values.get("amqp_queue_config"):
+                raise ValueError("amqp_queue_config must be provided for AMQP queue_type.")
+            # Ensure other queue configs are not present if AMQP is chosen
+            values["sqs_queue_config"] = None
+            values["polling_queue_config"] = None
+        elif queue_type == QueueType.POLLING:
+            # Polling config is optional, but if present, others are cleared
+            values["sqs_queue_config"] = None
+            values["amqp_queue_config"] = None
+        return values
+    
+    model_validator(mode='before')(validate_queue_config)

fustor_source_oss/driver.py

@@ -0,0 +1,274 @@
+import logging
+import asyncio
+from typing import Dict, Any, Iterator, Tuple, Optional
+import boto3
+from botocore.exceptions import ClientError
+from datetime import datetime, timezone
+
+from fustor_core.drivers import SourceDriver
+from fustor_core.models.config import SourceConfig, PasswdCredential
+from fustor_core.exceptions import DriverError
+from fustor_event_model.models import EventBase, EventType
+
+from .config import OssDriverParams, QueueType
+from .mapper import map_s3_objects_to_events_batch
+
+logger = logging.getLogger(__name__)
+
+class OssSourceDriver(SourceDriver):
+    """
+    Fustor Source Driver for S3-compatible object storage.
+    Supports snapshot (full listing) and polling-based incremental synchronization.
+    """
+
+    def __init__(self, id: str, config: SourceConfig):
+        super().__init__(id, config)
+        self.logger = logging.getLogger(f"{__name__}.{id}")
+
+        # Parse OSS specific parameters from driver_params
+        try:
+            self.oss_params = OssDriverParams(**config.driver_params)
+        except Exception as e:
+            raise DriverError(f"OSS configuration invalid: {e}")
+
+        # Parse credentials
+        if not isinstance(config.credential, PasswdCredential):
+            raise DriverError("OSS driver requires PasswdCredential (Access Key ID / Secret Access Key).")
+        
+        self.ak = config.credential.user
+        self.sk = config.credential.passwd
+
+        self.bucket_name = self.oss_params.bucket_name
+        self.endpoint_url = self.oss_params.endpoint_url
+        self.region_name = self.oss_params.region_name
+        self.prefix = self.oss_params.prefix
+
+        # Initialize S3 client
+        try:
+            self.s3_client = boto3.client(
+                "s3",
+                endpoint_url=self.endpoint_url,
+                aws_access_key_id=self.ak,
+                aws_secret_access_key=self.sk,
+                region_name=self.region_name,
+            )
+            self.logger.info(f"Initialized S3 client for bucket '{self.bucket_name}' at '{self.endpoint_url}'")
+        except Exception as e:
+            raise DriverError(f"Failed to initialize S3 client: {e}")
+
+    async def get_snapshot_iterator(self, **kwargs) -> Iterator[EventBase]:
+        """
+        Performs a one-time, full snapshot of the source data by listing all objects
+        matching the prefix in the specified S3 bucket.
+        """
+        self.logger.info(f"Starting snapshot for bucket '{self.bucket_name}' with prefix '{self.prefix}'")
+        
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+        page_iterator = paginator.paginate(
+            Bucket=self.bucket_name,
+            Prefix=self.prefix,
+            PaginationConfig={"PageSize": kwargs.get("batch_size", 1000)} # Default batch size
+        )
+
+        for page in page_iterator:
+            contents = page.get("Contents", [])
+            if not contents:
+                continue
+
+            # Convert to Fustor EventBase batch
+            try:
+                event_batch = map_s3_objects_to_events_batch(
+                    contents,
+                    self.bucket_name,
+                    "objects", # Logical table name for S3 objects
+                    EventType.INSERT
+                )
+                yield event_batch
+            except Exception as e:
+                self.logger.error(f"Error mapping S3 objects to event batch: {e}", exc_info=True)
+                continue
+        
+        self.logger.info(f"Snapshot completed for bucket '{self.bucket_name}' with prefix '{self.prefix}'")
+
+    def is_position_available(self, position: int) -> bool:
+        """
+        For OSS, positions are timestamps. We assume any past timestamp is "available"
+        as long as the object was not deleted. Polling logic will handle filtering.
+        """
+        return True # For polling, we can always attempt to resume from a timestamp
+
+
+    async def get_message_iterator(self, start_position: int = -1, **kwargs) -> Iterator[EventBase]:
+        """
+        Performs incremental data capture (CDC) for OSS.
+        
+        In polling mode, this method periodically lists objects and filters them
+        by LastModified timestamp greater than start_position.
+        """
+        if self.oss_params.queue_type != QueueType.POLLING:
+            raise DriverError(f"Unsupported queue_type '{self.oss_params.queue_type}' for get_message_iterator. "
+                              "Only POLLING is currently implemented.")
+        
+        polling_interval = self.oss_params.polling_queue_config.interval_seconds \
+                           if self.oss_params.polling_queue_config else 30
+        
+        self.logger.info(f"Starting message polling for bucket '{self.bucket_name}' with prefix '{self.prefix}' "
+                         f"from position {start_position}, interval: {polling_interval}s")
+
+        last_known_position = start_position
+        while True:
+            try:
+                # S3 ListObjectsV2 doesn't support filtering by LastModified directly.
+                # We must list and then filter client-side.
+                # This can be inefficient for very large buckets/prefixes.
+                paginator = self.s3_client.get_paginator("list_objects_v2")
+                page_iterator = paginator.paginate(
+                    Bucket=self.bucket_name,
+                    Prefix=self.prefix,
+                )
+                
+                new_or_modified_objects = []
+                current_max_timestamp = last_known_position
+
+                for page in page_iterator:
+                    contents = page.get("Contents", [])
+                    for s3_obj in contents:
+                        last_modified: datetime = s3_obj.get("LastModified")
+                        if last_modified:
+                            obj_timestamp = int(last_modified.timestamp())
+                            
+                            # Filter for objects modified AFTER the last_known_position
+                            # We use '>=' to potentially include items modified in the same second as the last checkpoint
+                            # The consumer (Agent) should handle exact duplicates if any.
+                            if obj_timestamp >= last_known_position:
+                                new_or_modified_objects.append(s3_obj)
+                                if obj_timestamp > current_max_timestamp:
+                                    current_max_timestamp = obj_timestamp
+                
+                if new_or_modified_objects:
+                    self.logger.debug(f"Found {len(new_or_modified_objects)} new/modified objects.")
+                    # Sort by timestamp to maintain order
+                    new_or_modified_objects.sort(key=lambda x: int(x.get("LastModified", datetime.min).timestamp()))
+
+                    # Group into batches
+                    batch_size = kwargs.get("batch_size", 100) # Use the batch_size from config
+                    for i in range(0, len(new_or_modified_objects), batch_size):
+                        batch_objects = new_or_modified_objects[i:i + batch_size]
+                        
+                        try:
+                            event_batch = map_s3_objects_to_events_batch(
+                                batch_objects,
+                                self.bucket_name,
+                                "objects",
+                                EventType.UPDATE # Treat as update/insert for changes
+                            )
+                            yield event_batch
+                            last_known_position = int(event_batch.index) # Update position after yielding a batch
+                        except Exception as e:
+                            self.logger.error(f"Error mapping S3 objects to event batch during polling: {e}", exc_info=True)
+                            continue
+                else:
+                    self.logger.debug("No new or modified objects found in this polling cycle.")
+                
+                # Update last_known_position to the highest timestamp seen in this cycle
+                # This prevents re-processing objects already processed in this polling interval
+                last_known_position = current_max_timestamp
+                
+            except ClientError as e:
+                error_code = e.response.get("Error", {}).get("Code")
+                if error_code == "NoSuchBucket":
+                    raise DriverError(f"S3 Bucket '{self.bucket_name}' not found: {e}")
+                elif error_code == "AccessDenied":
+                    raise DriverError(f"Access denied to S3 Bucket '{self.bucket_name}': {e}")
+                else:
+                    raise DriverError(f"S3 client error during polling: {e}")
+            except Exception as e:
+                self.logger.error(f"Unexpected error during S3 polling: {e}", exc_info=True)
+                raise DriverError(f"Unexpected error during S3 polling: {e}")
+
+            await asyncio.sleep(polling_interval)
+
+    async def test_connection(self, **kwargs) -> Tuple[bool, str]:
+        """
+        Tests the connection to the S3 service by performing a head_bucket operation.
+        """
+        self.logger.info(f"Testing connection to S3 bucket '{self.bucket_name}' at '{self.endpoint_url}'")
+        try:
+            await asyncio.to_thread(self.s3_client.head_bucket, Bucket=self.bucket_name)
+            self.logger.info("S3 connection test successful.")
+            return True, "Connection successful."
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code")
+            error_message = e.response.get("Error", {}).get("Message")
+            if error_code == "404": # Bucket not found
+                return False, f"Bucket '{self.bucket_name}' not found. Error: {error_message}"
+            elif error_code == "403": # Access denied
+                return False, f"Access denied to bucket '{self.bucket_name}'. Please check credentials and permissions. Error: {error_message}"
+            else:
+                return False, f"S3 connection failed with client error: {error_code} - {error_message}"
+        except Exception as e:
+            self.logger.error(f"S3 connection test failed with unexpected error: {e}", exc_info=True)
+            return False, f"Connection failed: {str(e)}"
+
+    @classmethod
+    async def get_available_fields(cls, **kwargs) -> Dict:
+        """
+        Declares the data fields that this source can provide.
+        For S3, these are common S3 object metadata fields.
+        """
+        return {
+            "type": "object",
+            "properties": {
+                "Key": {"type": "string", "description": "Object key (path)."},
+                "Size": {"type": "integer", "description": "Object size in bytes."}, 
+                "LastModified": {"type": "string", "format": "date-time", "description": "Last modified timestamp."}, 
+                "ETag": {"type": "string", "description": "ETag of the object."}, 
+                "StorageClass": {"type": "string", "description": "Storage class of the object."}, 
+                "OwnerId": {"type": "string", "description": "Canonical user ID of the object owner."}, 
+                "OwnerDisplayName": {"type": "string", "description": "Display name of the object owner."}, 
+            },
+            "required": ["Key", "Size", "LastModified"]
+        }
+    
+    @classmethod
+    async def get_wizard_steps(cls) -> Dict[str, Any]:
+        """
+        Provides configuration wizard steps for UI integration.
+        """
+        return {
+            "steps": [
+                {
+                    "title": "基本连接设置",
+                    "description": "配置 S3-兼容对象存储服务的连接信息。",
+                    "fields": [
+                        {"key": "driver_params.endpoint_url", "label": "Endpoint URL", "type": "text", "required": True, "placeholder": "e.g., https://s3.amazonaws.com or http://minio:9000"},
+                        {"key": "driver_params.bucket_name", "label": "存储桶名称", "type": "text", "required": True, "placeholder": "e.g., my-fustor-data"},
+                        {"key": "driver_params.region_name", "label": "区域名称 (可选)", "type": "text", "placeholder": "e.g., us-east-1"},
+                        {"key": "driver_params.prefix", "label": "路径前缀 (可选)", "type": "text", "placeholder": "e.g., logs/2023/", "help_text": "仅同步以该前缀开头的对象。"},
+                        {"key": "driver_params.recursive", "label": "递归遍历", "type": "boolean", "default": True, "help_text": "是否递归遍历子目录。"},
+                    ]
+                },
+                {
+                    "title": "认证凭证",
+                    "description": "提供访问 S3 存储桶的凭证。",
+                    "fields": [
+                        {"key": "credential.user", "label": "Access Key ID", "type": "text", "required": True, "placeholder": "您的 S3 Access Key ID"},
+                        {"key": "credential.passwd", "label": "Secret Access Key", "type": "password", "required": True, "placeholder": "您的 S3 Secret Access Key"},
+                    ]
+                },
+                {
+                    "title": "增量同步策略",
+                    "description": "选择用于捕获对象变更的策略。'Polling' 适合小规模存储或测试，更推荐使用消息队列。",
+                    "fields": [
+                        {"key": "driver_params.queue_type", "label": "同步策略", "type": "select", "required": True, "options": [
+                            {"value": "polling", "label": "Polling (轮询)"},
+                            {"value": "sqs", "label": "SQS (AWS SQS / 兼容)"},
+                            {"value": "amqp", "label": "AMQP (RabbitMQ / 兼容)"},
+                        ]},
+                        # Conditional fields for SQS
+                        {"key": "driver_params.sqs_queue_config.queue_url", "label": "SQS 队列 URL", "type": "text", "required": True, "condition_key": "driver_params.queue_type", "condition_value": "sqs", "placeholder": "e.g., https://sqs.us-east-1.amazonaws.com/..."}
+                        # ... other SQS/AMQP fields will go here when implemented
+                    ]
+                }
+            ]
+        }

fustor_source_oss/mapper.py

@@ -0,0 +1,109 @@
+from typing import Dict, Any, List
+from datetime import datetime
+from fustor_event_model.models import EventBase, EventType
+
+def map_s3_object_to_event(
+    s3_object: Dict[str, Any], 
+    bucket_name: str, 
+    table_name: str, 
+    event_type: EventType = EventType.INSERT
+) -> EventBase:
+    """
+    Maps a single S3 object dictionary to a Fustor EventBase.
+
+    Args:
+        s3_object: A dictionary representing an S3 object (from boto3 ListObjectsV2).
+        bucket_name: The name of the S3 bucket.
+        table_name: The logical table name for the event (e.g., 'objects').
+        event_type: The type of event (INSERT, UPDATE, DELETE).
+
+    Returns:
+        An EventBase instance.
+    """
+    
+    # Extract relevant metadata from the S3 object
+    key = s3_object.get("Key")
+    if not key:
+        raise ValueError("S3 object must have a 'Key'.")
+
+    size = s3_object.get("Size")
+    last_modified: datetime = s3_object.get("LastModified")
+    etag = s3_object.get("ETag")
+    storage_class = s3_object.get("StorageClass")
+    owner_id = s3_object.get("Owner", {}).get("ID")
+    owner_display_name = s3_object.get("Owner", {}).get("DisplayName")
+
+    # The index will be the Unix timestamp of LastModified for ordering
+    # Use -1 if LastModified is not available, though it usually is for objects.
+    index = int(last_modified.timestamp()) if last_modified else -1
+
+    # Prepare the row data
+    row_data = {
+        "Key": key,
+        "Size": size,
+        "LastModified": last_modified.isoformat() if last_modified else None,
+        "ETag": etag,
+        "StorageClass": storage_class,
+        "OwnerId": owner_id,
+        "OwnerDisplayName": owner_display_name,
+        # Add any other relevant S3 object metadata here
+    }
+
+    # Define the fields present in the row
+    fields = list(row_data.keys())
+
+    return EventBase(
+        event_type=event_type,
+        event_schema=bucket_name,  # Use bucket name as the schema
+        table=table_name,
+        index=index,
+        fields=fields,
+        rows=[row_data], # Each S3 object becomes one row in an event batch
+    )
+
+def map_s3_objects_to_events_batch(
+    s3_objects: List[Dict[str, Any]],
+    bucket_name: str,
+    table_name: str,
+    event_type: EventType = EventType.INSERT
+) -> EventBase:
+    """
+    Maps a list of S3 objects to a single EventBase with multiple rows.
+    This is useful for batching inserts/updates from a paginator.
+    """
+    if not s3_objects:
+        raise ValueError("s3_objects list cannot be empty.")
+
+    # Assume all objects in the batch belong to the same schema and table
+    # and have similar structure for fields.
+    first_object = s3_objects[0]
+    
+    # The index for a batch event could be the largest LastModified timestamp
+    # or the last object's timestamp in the batch. Let's use the last object for now.
+    last_modified = s3_objects[-1].get("LastModified")
+    index = int(last_modified.timestamp()) if last_modified else -1
+
+    rows: List[Dict[str, Any]] = []
+    for s3_obj in s3_objects:
+        row_data = {
+            "Key": s3_obj.get("Key"),
+            "Size": s3_obj.get("Size"),
+            "LastModified": s3_obj.get("LastModified").isoformat() if s3_obj.get("LastModified") else None,
+            "ETag": s3_obj.get("ETag"),
+            "StorageClass": s3_obj.get("StorageClass"),
+            "OwnerId": s3_obj.get("Owner", {}).get("ID"),
+            "OwnerDisplayName": s3_obj.get("Owner", {}).get("DisplayName"),
+        }
+        rows.append(row_data)
+    
+    # All rows in the batch are assumed to have the same fields, derive from first one
+    fields = list(rows[0].keys()) if rows else []
+
+    return EventBase(
+        event_type=event_type,
+        event_schema=bucket_name,
+        table=table_name,
+        index=index,
+        fields=fields,
+        rows=rows,
+    )

fustor_source_oss-0.2.dist-info/METADATA

@@ -0,0 +1,8 @@
+Metadata-Version: 2.4
+Name: fustor-source-oss
+Version: 0.2
+Summary: Fustor Source Driver for S3-compatible object storage (OSS)
+Requires-Python: >=3.11
+Requires-Dist: boto3<2.0.0,>=1.28.0
+Requires-Dist: fustor-core
+Requires-Dist: pydantic<3.0.0,>=2.0.0

fustor_source_oss-0.2.dist-info/RECORD

@@ -0,0 +1,9 @@
+fustor_source_oss/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fustor_source_oss/config.py,sha256=Ozl_Aj02G66BD4maDkimHHUtr0woGKh3JIxIz3-svb4,3924
+fustor_source_oss/driver.py,sha256=tnpD5tqfnLPWTkCLrUMmgDF5t_AJnLiqSS6JBaJSmVw,14381
+fustor_source_oss/mapper.py,sha256=GUJSU3ylJuVWI5V4j7spx0Tr-7YcAQtAUhzVFF8kdJw,3830
+fustor_source_oss-0.2.dist-info/METADATA,sha256=Dl6aJVRZWlSR24dwPby1VpdmZrfr2y35e1W2hz4ux98,253
+fustor_source_oss-0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+fustor_source_oss-0.2.dist-info/entry_points.txt,sha256=3jSvXrC9TTuba3sFWtAhDVpBSZRxsYyj9nhsYRiqo8c,85
+fustor_source_oss-0.2.dist-info/top_level.txt,sha256=hq80-RUZ_j94qqXmO-jGI50cZrWD143nkbuO_xeWGdg,18
+fustor_source_oss-0.2.dist-info/RECORD,,

fustor_source_oss-0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

fustor_source_oss-0.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[fustor_agent.drivers.sources]
+source_oss = fustor_source_oss.driver:OssSourceDriver

fustor_source_oss-0.2.dist-info/top_level.txt

@@ -0,0 +1 @@
+fustor_source_oss