fustor-core 0.1.2.post3__tar.gz

fustor_core-0.1.2.post3/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: fustor-core
+Version: 0.1.2.post3
+Summary: Core components for Fustor services and plugins
+Author-email: Huajin Wang <wanghuajin999@163.com>
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.11.7
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"

fustor_core-0.1.2.post3/README.md

@@ -0,0 +1,22 @@
+# fustor-core
+
+This package contains core components, abstractions, and utilities shared across various Fustor services and plugins within the monorepo. It provides foundational elements such as base classes for drivers, common exceptions, data models, and utility functions.
+
+## Contents
+
+*   `drivers.py`: Defines abstract base classes (ABCs) for `SourceDriver` and `PusherDriver`, establishing the contract for all data source and data pusher implementations.
+*   `exceptions.py`: Contains custom exception classes used throughout the Fustor ecosystem for consistent error handling.
+*   `models/`: Houses Pydantic models for various data structures, ensuring data validation and serialization.
+*   `utils/`: Provides general utility functions that are commonly used by different Fustor components.
+
+## Installation
+
+This package is part of the Fustor monorepo and is typically installed in editable mode within the monorepo's development environment using `uv sync`.
+
+## Usage
+
+Components from `fustor-core` are imported and utilized by other Fustor services (e.g., `agent`, `registry`, `fusion`) and plugin packages (e.g., `source_mysql`, `pusher_fusion`) to ensure consistency and reusability.
+
+## Note on Package Structure
+
+It has been observed that the `__init__.py` file is missing from the `fustor_core` package directory (`packages/core/src/fustor_core/`). While the package's contents are still accessible, this is an unconventional Python package structure and might lead to issues with package discovery or imports in certain environments. It is recommended to add an empty `__init__.py` file to this directory.

fustor_core-0.1.2.post3/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "fustor-core"
+dynamic = ["version"]
+description = "Core components for Fustor services and plugins"
+requires-python = ">=3.11"
+dependencies = [ "pydantic>=2.11.7",]
+[[project.authors]]
+name = "Huajin Wang"
+email = "wanghuajin999@163.com"
+
+[build-system]
+requires = [ "setuptools>=61.0", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = "../.."
+version_scheme = "post-release"
+local_scheme = "dirty-tag"
+
+["project.urls"]
+Homepage = "https://github.com/excelwang/fustor/tree/master/packages/core"
+"Bug Tracker" = "https://github.com/excelwang/fustor/issues"
+
+license = "MIT"
+
+[project.optional-dependencies]
+dev = [ "pytest>=8.0.0", "ruff>=0.1.0", "mypy>=1.0.0", "pytest-asyncio>=0.23.0",]
+
+[tool.setuptools.packages.find]
+where = [ "src",]

fustor_core-0.1.2.post3/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

fustor_core-0.1.2.post3/src/fustor_core/drivers.py

@@ -0,0 +1,216 @@
+"""
+Abstract Base Classes for Fuagent Drivers.
+
+This module defines the formal interface for Source and Pusher drivers.
+All drivers must inherit from the appropriate base class and implement its
+abstract methods.
+"""
+from abc import ABC, abstractmethod
+from typing import (
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Tuple,
+)
+
+# Forward-referencing models to avoid circular imports if drivers.py is imported by models
+from fustor_event_model.models import EventBase # Import EventBase from fustor_event_model
+from fustor_core.models.config import SourceConfig, PusherConfig
+
+
+class PusherDriver(ABC):
+    """
+    Abstract Base Class for all Pusher drivers.
+
+    Defines the contract for drivers that receive data from the Fuagent core.
+    These drivers are expected to be asynchronous.
+    """
+
+    def __init__(self, id: str, config: PusherConfig):
+        """
+        Initializes the driver with its specific configuration.
+        """
+        self.id = id
+        self.config = config
+
+    @abstractmethod
+    async def push(self, events: List[EventBase], **kwargs) -> Dict:
+        """
+        Receives and processes a list of events. This is the primary data-writing method.
+        """
+        raise NotImplementedError
+    
+    async def get_latest_committed_index(self, **kwargs) -> int:
+        """
+        Optional: Gets the last successfully processed index from the pusher endpoint.
+        Used for resumable syncs. A return value of -1 indicates starting from the beginning.
+        """
+        return -1
+    
+    @abstractmethod
+    async def heartbeat(self, **kwargs) -> Dict:
+        """
+        Sends a heartbeat to maintain session state with the pusher endpoint.
+        The `kwargs` will contain `agent_id`, `task_id`, and `session_id`.
+        Returns a dictionary with status information.
+        """
+        raise NotImplementedError
+    
+    @abstractmethod
+    async def create_session(self, task_id: str) -> str:
+        """
+        Creates a new session with the pusher endpoint.
+        Returns the session ID string.
+        """
+        raise NotImplementedError
+
+    async def close(self):
+        """
+        Optional: Gracefully closes any open resources, like network clients.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    async def get_needed_fields(cls, **kwargs) -> Dict:
+        """
+        Declares the data fields required by this pusher.
+        Returns a JSON Schema dictionary. An empty dict means all fields are accepted.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    async def test_connection(cls, **kwargs) -> Tuple[bool, str]:
+        """
+        Optional: Tests the connection to the source service.
+        """
+        return (True, "Connection test not implemented for this driver.")
+
+    @classmethod
+    async def check_privileges(cls, **kwargs) -> Tuple[bool, str]:
+        """
+        Optional: Checks if the provided credentials have sufficient privileges.
+        """
+        return (True, "Privilege check not implemented for this driver.")  
+    
+    @classmethod
+    async def get_wizard_steps(cls) -> Dict[str, Any]:
+        """
+        Optional: Provides configuration wizard steps for UI integration.
+        Returns a dictionary defining the steps.
+        """
+        return {} 
+
+class SourceDriver(ABC):
+    """
+    Abstract Base Class for all Source drivers.
+
+    Defines the contract for drivers that produce data for the Fuagent core.
+    Note the mix of synchronous and asynchronous methods, reflecting the current
+    design of existing drivers (e.g., threading-based fs vs. async network drivers).
+    """
+
+    def __init__(self, id: str, config: SourceConfig):
+        """
+        Initializes the driver with its specific configuration.
+        """
+        self.id = id
+        self.config = config
+
+    @property
+    def is_transient(self) -> bool:
+        """
+        Indicates whether this source driver is transient.
+        Transient sources lose events if not processed immediately.
+        Defaults to False. Drivers that are transient should override this property.
+        """
+        return False
+
+    @abstractmethod
+    def get_snapshot_iterator(self, **kwargs) -> Iterator[EventBase]:
+        """
+        Performs a one-time, full snapshot of the source data.
+        This method returns an iterator that yields new events.
+        """
+        raise NotImplementedError
+
+    def is_position_available(self, position: int) -> bool:
+        """
+        Checks if the driver can resume from a specific position.
+        For transient sources, this should return False since they don't keep historical events.
+        Defaults to True for non-transient sources, but drivers should override this method
+        to provide accurate information about position availability.
+        """
+        if position <= 0: #means from the latest snapshot
+            return False
+        return not self.is_transient
+
+    @abstractmethod
+    def get_message_iterator(self, start_position: int = -1, **kwargs) -> Iterator[EventBase]:
+        """
+        Performs incremental data capture (CDC).
+
+        This method returns an iterator that yields new events.
+        Optionally, a start_position can be provided to resume from a specific point.
+        Use is_position_available() to check if a position can be resumed from.
+        
+        Args:
+            start_position (int): The position to start from, or -1 for latest position
+            **kwargs: Additional implementation-specific parameters
+
+        Returns:
+            Iterator[EventBase]: An iterator that yields new events.
+        """
+        raise NotImplementedError
+
+    async def close(self):
+        """
+        Optional: Gracefully closes any open resources, like database connections or file handles.
+        """
+        pass
+
+    @classmethod
+    @abstractmethod
+    async def get_available_fields(cls, **kwargs) -> Dict:
+        """
+        Declares the data fields that this source can provide.
+        Returns a JSON Schema dictionary.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    async def test_connection(cls, **kwargs) -> Tuple[bool, str]:
+        """
+        Optional: Tests the connection to the source service.
+        """
+        return (True, "Connection test not implemented for this driver.")
+
+    @classmethod
+    async def check_privileges(cls, **kwargs) -> Tuple[bool, str]:
+        """
+        Optional: Checks if the provided credentials have sufficient privileges.
+        """
+        return (True, "Privilege check not implemented for this driver.")
+
+    @classmethod
+    async def check_runtime_params(cls, **kwargs) -> Tuple[bool, str]:
+        """
+        Optional: Checks if the runtime parameters of the underlying source system are edequate for generating events.
+        """
+        return (True, "Runtime parameter check not implemented for this driver.")
+
+    @classmethod
+    async def create_agent_user(cls, **kwargs) -> Tuple[bool, str]:
+        """
+        Optional: Creates a agent user for the source service.
+        """
+        return (True, "Agent user creation not implemented for this driver.")
+    
+    @classmethod
+    async def get_wizard_steps(cls) -> Dict[str, Any]:
+        """
+        Optional: Provides configuration wizard steps for UI integration.
+        Returns a dictionary defining the steps.
+        """
+        return {}

fustor_core-0.1.2.post3/src/fustor_core/exceptions.py

@@ -0,0 +1,42 @@
+from fastapi import HTTPException, status
+
+from typing import Optional, Any, Dict
+
+class fustor_agentException(HTTPException):
+    """Base exception for fustor_agent, mapping to HTTP exceptions."""
+    def __init__(self, status_code: int, detail: Any = None, headers: Optional[Dict[str, Any]] = None):
+        super().__init__(status_code=status_code, detail=detail, headers=headers)
+
+class ConfigError(fustor_agentException):
+    """Raised when there's an issue with configuration (e.g., not found, invalid)."""
+    def __init__(self, detail: str = "Configuration error", headers: Optional[Dict[str, Any]] = None):
+        super().__init__(status_code=status.HTTP_400_BAD_REQUEST, detail=detail, headers=headers)
+
+class NotFoundError(fustor_agentException):
+    """Raised when a requested resource (e.g., config, instance) is not found."""
+    def __init__(self, detail: str = "Resource not found", headers: Optional[Dict[str, Any]] = None):
+        super().__init__(status_code=status.HTTP_404_NOT_FOUND, detail=detail, headers=headers)
+
+class ConflictError(fustor_agentException):
+    """Raised when a resource already exists and cannot be created again."""
+    def __init__(self, detail: str = "Resource already exists", headers: Optional[Dict[str, Any]] = None):
+        super().__init__(status_code=status.HTTP_409_CONFLICT, detail=detail, headers=headers)
+
+class DriverError(fustor_agentException):
+    """Raised when there's an issue with a driver (e.g., connection, invalid parameters)."""
+    def __init__(self, detail: str = "Driver error", headers: Optional[Dict[str, Any]] = None):
+        super().__init__(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=detail, headers=headers)
+
+class StateConflictError(fustor_agentException):
+    """Raised when an operation is attempted in an invalid state."""
+    def __init__(self, detail: str = "Operation not allowed in current state", headers: Optional[Dict[str, Any]] = None):
+        super().__init__(status_code=status.HTTP_409_CONFLICT, detail=detail, headers=headers)
+
+class ValidationError(fustor_agentException):
+    """Raised when input validation fails."""
+    def __init__(self, detail: str = "Validation error", headers: Optional[Dict[str, Any]] = None):
+        super().__init__(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=detail, headers=headers)
+
+class TransientSourceBufferFullError(Exception):
+    """Raised by MemoryEventBus when its buffer is full and the source is transient."""
+    pass

fustor_core-0.1.2.post3/src/fustor_core/models/config.py

@@ -0,0 +1,206 @@
+from pydantic import BaseModel, Field, field_validator, RootModel, ConfigDict
+from typing import List, Optional, Union, TypeAlias, Dict, Any
+from fustor_core.exceptions import ConfigError, NotFoundError
+
+class PasswdCredential(BaseModel):
+    # --- START FIX: Forbid extra fields to prevent incorrect model coercion ---
+    model_config = ConfigDict(extra='forbid')
+    # --- END FIX ---
+
+    user: str = Field(..., description="用户名")
+    passwd: Optional[str] = Field(None, description="密码")
+
+    def to_base64(self) -> str:
+        """为HTTP Basic Auth生成Base64编码的字符串。"""
+        import base64
+        auth_str = f"{self.user}:{self.passwd or ''}"
+        return base64.b64encode(auth_str.encode('utf-8')).decode('utf-8')
+
+    def _get_hashable_data(self):
+        return ("PasswdCredential", self.user, self.passwd)
+
+    def __hash__(self):
+        return hash(self._get_hashable_data())
+
+    def __eq__(self, other):
+        if not isinstance(other, PasswdCredential):
+            return NotImplemented
+        return self._get_hashable_data() == other._get_hashable_data()
+
+class ApiKeyCredential(BaseModel):
+    # --- START FIX: Forbid extra fields to prevent incorrect model coercion ---
+    model_config = ConfigDict(extra='forbid')
+    # --- END FIX ---
+
+    user: Optional[str] = Field(None, description="用户名")
+    key: str = Field(..., description="api key")
+
+    def _get_hashable_data(self):
+        return ("ApiKeyCredential", self.user, self.key)
+
+    def __hash__(self):
+        return hash(self._get_hashable_data())
+
+    def __eq__(self, other):
+        if not isinstance(other, ApiKeyCredential):
+            return NotImplemented
+        return self._get_hashable_data() == other._get_hashable_data()
+
+# Reordered Union to prioritize the more specific ApiKeyCredential
+Credential: TypeAlias = Union[ApiKeyCredential, PasswdCredential]
+
+class FieldMapping(BaseModel):
+    to: str = Field(..., description="供给字段")
+    source: List[str] = Field(..., description="来源字段")
+    required: bool = Field(default=False, description="是否为必填字段")
+
+class SourceConfig(BaseModel):
+    # Assuming 'name' will be the key in the dict, it's not needed inside the model itself.
+    driver: str
+    uri: str
+    credential: Credential
+    max_queue_size: int = Field(default=1000, gt=0, description="事件缓冲区的最大尺寸")
+    max_retries: int = Field(default=10, gt=0, description="驱动在读取事件失败时的最大重试次数")
+    retry_delay_sec: int = Field(default=5, gt=0, description="驱动重试前的等待秒数")
+    disabled: bool = Field(default=True, description="是否禁用此配置")
+    validation_error: Optional[str] = Field(None, exclude=True)
+    driver_params: Dict[str, Any] = Field(default_factory=dict, description="驱动专属参数")
+
+class PusherConfig(BaseModel):
+    # Assuming 'name' will be the key in the dict
+    driver: str
+    endpoint: str
+    credential: Credential
+    batch_size: int = Field(default=100, description="单次推送事件的批处理大小")
+    max_retries: int = Field(default=10, gt=0, description="推送失败时的最大重试次数")
+    retry_delay_sec: int = Field(default=5, gt=0, description="推送重试前的等待秒数")
+    disabled: bool = Field(default=True, description="是否禁用此配置")
+    driver_params: Optional[Dict[str, Any]] = Field(default=None, description="驱动专属参数")
+    
+    @field_validator('batch_size')
+    def batch_size_must_be_positive(cls, v):
+        if v <= 0:
+            raise ConfigError('batch_size must be positive')
+        return v
+
+class SyncConfig(BaseModel):
+    source: str
+    pusher: str
+    disabled: bool = Field(default=True, description="是否禁用此同步任务")
+    # --- START: 核心修改 ---
+    # [REMOVED] The following two fields are obsolete in the new architecture.
+    # checkpoint_interval_events: int = Field(...)
+    # enable_checkpoint: bool = True
+    # --- END: 核心修改 ---
+    fields_mapping: List[FieldMapping] = Field(default_factory=list)
+
+class SourceConfigDict(RootModel[Dict[str, SourceConfig]]):
+    root: Dict[str, SourceConfig] = Field(default_factory=dict)
+
+class PusherConfigDict(RootModel[Dict[str, PusherConfig]]):
+    root: Dict[str, PusherConfig] = Field(default_factory=dict)
+
+class SyncConfigDict(RootModel[Dict[str, SyncConfig]]):
+    root: Dict[str, SyncConfig] = Field(default_factory=dict)
+
+class AppConfig(BaseModel):
+    # FIX: Provide default factories for top-level configuration sections.
+    # This allows the application to start with an empty but valid config
+    # if the config.yaml file is missing or empty.
+    sources: SourceConfigDict = Field(default_factory=SourceConfigDict)
+    pushers: PusherConfigDict = Field(default_factory=PusherConfigDict)
+    syncs: SyncConfigDict = Field(default_factory=SyncConfigDict)
+
+    def get_sources(self) -> Dict[str, SourceConfig]:
+        return self.sources.root
+    
+    def get_pushers(self) -> Dict[str, PusherConfig]:
+        return self.pushers.root
+
+    def get_syncs(self) -> Dict[str, SyncConfig]:
+        return self.syncs.root
+    
+    def get_source(self, id: str) -> Optional[SourceConfig]:
+        return self.get_sources().get(id)
+    
+    def get_pusher(self, id: str) -> Optional[PusherConfig]:
+        return self.get_pushers().get(id)
+
+    def get_sync(self, id: str) -> Optional[SyncConfig]:
+        return self.get_syncs().get(id)
+    
+    def add_source(self, id: str, config: SourceConfig) -> SourceConfig:
+        config_may = self.get_source(id)
+        if config_may:
+            raise ConfigError(f"Source config with name '{id}' already exists.")
+        self.get_sources()[id] = config
+        return config
+
+    def add_pusher(self, id: str, config: PusherConfig) -> PusherConfig:
+        config_may = self.get_pusher(id)
+        if config_may:
+            raise ConfigError(f"Pusher config with name '{id}' already exists.")
+        self.get_pushers()[id] = config
+        return config
+
+    def add_sync(self, id: str, config: SyncConfig) -> SyncConfig:
+        config_may = self.get_sync(id)
+        if config_may:
+            raise ConfigError(f"Sync config with id '{id}' already exists.")
+        
+        # Dependency check
+        if not self.get_source(config.source):
+            raise NotFoundError(f"Dependency source '{config.source}' not found.")
+        if not self.get_pusher(config.pusher):
+            raise NotFoundError(f"Dependency pusher '{config.pusher}' not found.")
+        
+        self.get_syncs()[id] = config
+        return config
+    
+    def delete_source(self, id: str) -> SourceConfig:
+        config = self.get_source(id)
+        if not config:
+            raise NotFoundError(f"Source config with id '{id}' not found.")
+        
+        # Delete dependent syncs first
+        sync_ids_to_delete = [sync_id for sync_id, cfg in self.get_syncs().items() if cfg.source == id]
+        for sync_id in sync_ids_to_delete:
+            self.delete_sync(sync_id)
+            
+        return self.get_sources().pop(id)
+    
+    def delete_pusher(self, id: str) -> PusherConfig:
+        config = self.get_pusher(id)
+        if not config:
+            raise NotFoundError(f"Pusher config with id '{id}' not found.")
+        
+        # Delete dependent syncs first
+        sync_ids_to_delete = [sync_id for sync_id, cfg in self.syncs.root.items() if cfg.pusher == id]
+        for sync_id in sync_ids_to_delete:
+            self.delete_sync(sync_id)
+            
+        return self.get_pushers().pop(id)
+    
+    def delete_sync(self, id: str) -> SyncConfig:
+        config = self.get_sync(id)
+        if not config:
+            raise NotFoundError(f"Sync config with id '{id}' not found.")
+        return self.get_syncs().pop(id)
+
+    def check_sync_is_disabled(self, id: str) -> bool:
+        config = self.get_sync(id)
+        if not config:
+            raise NotFoundError(f"Sync with id '{id}' not found.")
+        
+        if config.disabled:
+            return True
+        
+        source_config = self.sources.root.get(config.source)
+        if not source_config:
+            raise NotFoundError(f"Dependency source '{config.source}' not found for sync '{id}'.")
+            
+        pusher_config = self.pushers.root.get(config.pusher)
+        if not pusher_config:
+            raise NotFoundError(f"Dependency pusher '{config.pusher}' not found for sync '{id}'.")
+            
+        return source_config.disabled or pusher_config.disabled

fustor_core-0.1.2.post3/src/fustor_core/models/log.py

@@ -0,0 +1,11 @@
+from pydantic import BaseModel, Field
+from datetime import datetime
+from typing import Optional
+
+class LogEntry(BaseModel):
+    """定义了单条日志记录的结构化模型"""
+    timestamp: datetime = Field(..., alias="ts")
+    level: str
+    component: str = Field(..., alias="source")
+    message: str = Field(..., alias="msg")
+    line_number: int # 用于分页的关键字段

fustor_core-0.1.2.post3/src/fustor_core/models/states.py

@@ -0,0 +1,38 @@
+from enum import Enum, Flag, auto
+from typing import Optional, Dict, Any
+from pydantic import BaseModel, Field
+
+class EventBusState(str, Enum):
+    IDLE = "IDLE"
+    PRODUCING = "PRODUCING"
+    ERROR = "ERROR"
+
+class SyncState(Flag):
+    """Enumeration for the state of a sync instance."""
+    STOPPED = 0
+    STARTING = auto()
+    SNAPSHOT_SYNC = auto()
+    MESSAGE_SYNC = auto()
+    RUNNING_CONF_OUTDATE = auto()
+    STOPPING = auto()
+    ERROR = auto()
+
+class EventBusInstance(BaseModel):
+    id: str
+    source_name: str
+    state: EventBusState
+    info: str
+    statistics: Dict[str, Any]
+
+class SyncInstanceDTO(BaseModel):
+    id: str
+    state: SyncState
+    info: str
+    statistics: Dict[str, Any]
+    bus_info: Optional[EventBusInstance] = None
+    bus_id: Optional[str] = None
+
+class AgentState(BaseModel):
+    agent_id: str = Field(..., description="The unique identifier for the agent.")
+    sync_tasks: Dict[str, SyncInstanceDTO] = Field(default_factory=dict, description="A dictionary of all sync tasks, keyed by their ID.")
+    event_buses: Dict[str, EventBusInstance] = Field(default_factory=dict, description="A dictionary of all active event buses, keyed by their ID.")

fustor_core-0.1.2.post3/src/fustor_core/utils/retry.py

@@ -0,0 +1,36 @@
+import asyncio
+import logging
+from functools import wraps
+from fustor_core.exceptions import DriverError
+
+logger = logging.getLogger(__name__)
+
+def retry(max_retries_attr: str, delay_sec_attr: str, exceptions: tuple = (DriverError,)):
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(self, *args, **kwargs):
+            max_retries = getattr(self.config, max_retries_attr)
+            delay_sec = getattr(self.config, delay_sec_attr)
+            
+            retries = 0
+            while True:
+                try:
+                    return await func(self, *args, **kwargs)
+                except asyncio.CancelledError:
+                    # Handle cancellation that occurs during function execution
+                    logger.info(f"Function {func.__name__} was cancelled during execution.")
+                    raise
+                except exceptions as e:
+                    retries += 1
+                    if retries >= max_retries:
+                        logger.error(f"Function {func.__name__} failed after {max_retries} retries.")
+                        raise
+                    logger.warning(f"Function {func.__name__} failed. Retrying in {delay_sec:.2f} seconds... ({retries}/{max_retries})")
+                    try:
+                        await asyncio.sleep(delay_sec)
+                    except asyncio.CancelledError:
+                        # Handle cancellation that occurs during the sleep/delay period
+                        logger.info(f"Retry loop for {func.__name__} was cancelled during sleep.")
+                        raise
+        return wrapper
+    return decorator

fustor_core-0.1.2.post3/src/fustor_core.egg-info/PKG-INFO

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: fustor-core
+Version: 0.1.2.post3
+Summary: Core components for Fustor services and plugins
+Author-email: Huajin Wang <wanghuajin999@163.com>
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.11.7
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"

fustor_core-0.1.2.post3/src/fustor_core.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+README.md
+pyproject.toml
+src/fustor_core/__init__.py
+src/fustor_core/drivers.py
+src/fustor_core/exceptions.py
+src/fustor_core.egg-info/PKG-INFO
+src/fustor_core.egg-info/SOURCES.txt
+src/fustor_core.egg-info/dependency_links.txt
+src/fustor_core.egg-info/requires.txt
+src/fustor_core.egg-info/top_level.txt
+src/fustor_core/models/__init__.py
+src/fustor_core/models/config.py
+src/fustor_core/models/log.py
+src/fustor_core/models/states.py
+src/fustor_core/utils/retry.py
+tests/models/test_config.py
+tests/models/test_event.py
+tests/models/test_states.py

fustor_core-0.1.2.post3/src/fustor_core.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

fustor_core-0.1.2.post3/src/fustor_core.egg-info/requires.txt

@@ -0,0 +1,7 @@
+pydantic>=2.11.7
+
+[dev]
+pytest>=8.0.0
+ruff>=0.1.0
+mypy>=1.0.0
+pytest-asyncio>=0.23.0

fustor_core-0.1.2.post3/src/fustor_core.egg-info/top_level.txt

@@ -0,0 +1 @@
+fustor_core

fustor_core-0.1.2.post3/tests/models/test_config.py

@@ -0,0 +1,196 @@
+import pytest
+from fustor_core.models.config import (
+    PasswdCredential, ApiKeyCredential, FieldMapping,
+    SourceConfig, PusherConfig, SyncConfig,
+    AppConfig, ConfigError, NotFoundError
+)
+
+def test_passwd_credential_hash_and_eq():
+    cred1 = PasswdCredential(user="testuser", passwd="testpass")
+    cred2 = PasswdCredential(user="testuser", passwd="testpass")
+    cred3 = PasswdCredential(user="anotheruser", passwd="testpass")
+
+    assert cred1 == cred2
+    assert hash(cred1) == hash(cred2)
+    assert cred1 != cred3
+    assert hash(cred1) != hash(cred3)
+
+def test_api_key_credential_hash_and_eq():
+    cred1 = ApiKeyCredential(user="testuser", key="apikey123")
+    cred2 = ApiKeyCredential(user="testuser", key="apikey123")
+    cred3 = ApiKeyCredential(user="anotheruser", key="anotherkey")
+
+    assert cred1 == cred2
+    assert hash(cred1) == hash(cred2)
+    assert cred1 != cred3
+    assert hash(cred1) != hash(cred3)
+
+def test_pusher_config_batch_size_validation():
+    with pytest.raises(ConfigError, match="batch_size must be positive"):
+        PusherConfig(driver="test", endpoint="http://localhost", credential=PasswdCredential(user="u"), batch_size=0)
+    with pytest.raises(ConfigError, match="batch_size must be positive"):
+        PusherConfig(driver="test", endpoint="http://localhost", credential=PasswdCredential(user="u"), batch_size=-1)
+    
+    config = PusherConfig(driver="test", endpoint="http://localhost", credential=PasswdCredential(user="u"), batch_size=1)
+    assert config.batch_size == 1
+
+def test_app_config_add_get_delete_source():
+    app_config = AppConfig()
+    source_config = SourceConfig(driver="mysql", uri="mysql://host", credential=PasswdCredential(user="u"), disabled=False)
+
+    # Add source
+    app_config.add_source("my_source", source_config)
+    assert app_config.get_source("my_source") == source_config
+
+    # Add duplicate source
+    with pytest.raises(ConfigError, match="Source config with name 'my_source' already exists."):
+        app_config.add_source("my_source", source_config)
+
+    # Delete source
+    deleted_source = app_config.delete_source("my_source")
+    assert deleted_source == source_config
+    assert app_config.get_source("my_source") is None
+
+    # Delete non-existent source
+    with pytest.raises(NotFoundError, match="Source config with id 'non_existent' not found."):
+        app_config.delete_source("non_existent")
+
+def test_app_config_add_get_delete_pusher():
+    app_config = AppConfig()
+    pusher_config = PusherConfig(driver="http", endpoint="http://localhost", credential=PasswdCredential(user="u"), disabled=False)
+
+    # Add pusher
+    app_config.add_pusher("my_pusher", pusher_config)
+    assert app_config.get_pusher("my_pusher") == pusher_config
+
+    # Add duplicate pusher
+    with pytest.raises(ConfigError, match="Pusher config with name 'my_pusher' already exists."):
+        app_config.add_pusher("my_pusher", pusher_config)
+
+    # Delete pusher
+    deleted_pusher = app_config.delete_pusher("my_pusher")
+    assert deleted_pusher == pusher_config
+    assert app_config.get_pusher("my_pusher") is None
+
+    # Delete non-existent pusher
+    with pytest.raises(NotFoundError, match="Pusher config with id 'non_existent' not found."):
+        app_config.delete_pusher("non_existent")
+
+def test_app_config_add_get_delete_sync():
+    app_config = AppConfig()
+    source_config = SourceConfig(driver="mysql", uri="mysql://host", credential=PasswdCredential(user="u"), disabled=False)
+    pusher_config = PusherConfig(driver="http", endpoint="http://localhost", credential=PasswdCredential(user="u"), disabled=False)
+    sync_config = SyncConfig(source="my_source", pusher="my_pusher", disabled=False)
+
+    # Add sync without dependencies
+    with pytest.raises(NotFoundError, match="Dependency source 'my_source' not found."):
+        app_config.add_sync("my_sync", sync_config)
+    
+    app_config.add_source("my_source", source_config)
+    with pytest.raises(NotFoundError, match="Dependency pusher 'my_pusher' not found."):
+        app_config.add_sync("my_sync", sync_config)
+
+    app_config.add_pusher("my_pusher", pusher_config)
+    app_config.add_sync("my_sync", sync_config)
+    assert app_config.get_sync("my_sync") == sync_config
+
+    # Add duplicate sync
+    with pytest.raises(ConfigError, match="Sync config with id 'my_sync' already exists."):
+        app_config.add_sync("my_sync", sync_config)
+
+    # Delete sync
+    deleted_sync = app_config.delete_sync("my_sync")
+    assert deleted_sync == sync_config
+    assert app_config.get_sync("my_sync") is None
+
+    # Delete non-existent sync
+    with pytest.raises(NotFoundError, match="Sync config with id 'non_existent' not found."):
+        app_config.delete_sync("non_existent")
+
+def test_app_config_delete_source_with_dependent_syncs():
+    app_config = AppConfig()
+    source_config = SourceConfig(driver="mysql", uri="mysql://host", credential=PasswdCredential(user="u"), disabled=False)
+    pusher_config = PusherConfig(driver="http", endpoint="http://localhost", credential=PasswdCredential(user="u"), disabled=False)
+    sync_config1 = SyncConfig(source="my_source", pusher="my_pusher", disabled=False)
+    sync_config2 = SyncConfig(source="my_source", pusher="my_pusher", disabled=False) # Another sync using the same source
+
+    app_config.add_source("my_source", source_config)
+    app_config.add_pusher("my_pusher", pusher_config)
+    app_config.add_sync("sync1", sync_config1)
+    app_config.add_sync("sync2", sync_config2)
+
+    assert app_config.get_sync("sync1") is not None
+    assert app_config.get_sync("sync2") is not None
+
+    app_config.delete_source("my_source")
+
+    assert app_config.get_source("my_source") is None
+    assert app_config.get_sync("sync1") is None
+    assert app_config.get_sync("sync2") is None
+
+def test_app_config_delete_pusher_with_dependent_syncs():
+    app_config = AppConfig()
+    source_config = SourceConfig(driver="mysql", uri="mysql://host", credential=PasswdCredential(user="u"), disabled=False)
+    pusher_config = PusherConfig(driver="http", endpoint="http://localhost", credential=PasswdCredential(user="u"), disabled=False)
+    sync_config1 = SyncConfig(source="my_source", pusher="my_pusher", disabled=False)
+    sync_config2 = SyncConfig(source="my_source", pusher="my_pusher", disabled=False) # Another sync using the same pusher
+
+    app_config.add_source("my_source", source_config)
+    app_config.add_pusher("my_pusher", pusher_config)
+    app_config.add_sync("sync1", sync_config1)
+    app_config.add_sync("sync2", sync_config2)
+
+    assert app_config.get_sync("sync1") is not None
+    assert app_config.get_sync("sync2") is not None
+
+    app_config.delete_pusher("my_pusher")
+
+    assert app_config.get_pusher("my_pusher") is None
+    assert app_config.get_sync("sync1") is None
+    assert app_config.get_sync("sync2") is None
+
+def test_app_config_check_sync_is_disabled():
+    app_config = AppConfig()
+    source_config_enabled = SourceConfig(driver="mysql", uri="mysql://host", credential=PasswdCredential(user="u"), disabled=False)
+    source_config_disabled = SourceConfig(driver="mysql", uri="mysql://host", credential=PasswdCredential(user="u"), disabled=True)
+    pusher_config_enabled = PusherConfig(driver="http", endpoint="http://localhost", credential=PasswdCredential(user="u"), disabled=False)
+    pusher_config_disabled = PusherConfig(driver="http", endpoint="http://localhost", credential=PasswdCredential(user="u"), disabled=True)
+
+    app_config.add_source("source_e", source_config_enabled)
+    app_config.add_source("source_d", source_config_disabled)
+    app_config.add_pusher("pusher_e", pusher_config_enabled)
+    app_config.add_pusher("pusher_d", pusher_config_disabled)
+
+    # Sync itself disabled
+    sync_disabled = SyncConfig(source="source_e", pusher="pusher_e", disabled=True)
+    app_config.add_sync("sync_d", sync_disabled)
+    assert app_config.check_sync_is_disabled("sync_d") is True
+
+    # Source disabled
+    sync_source_disabled = SyncConfig(source="source_d", pusher="pusher_e", disabled=False)
+    app_config.add_sync("sync_source_d", sync_source_disabled)
+    assert app_config.check_sync_is_disabled("sync_source_d") is True
+
+    # Pusher disabled
+    sync_pusher_disabled = SyncConfig(source="source_e", pusher="pusher_d", disabled=False)
+    app_config.add_sync("sync_pusher_d", sync_pusher_disabled)
+    assert app_config.check_sync_is_disabled("sync_pusher_d") is True
+
+    # All enabled
+    sync_enabled = SyncConfig(source="source_e", pusher="pusher_e", disabled=False)
+    app_config.add_sync("sync_e", sync_enabled)
+    assert app_config.check_sync_is_disabled("sync_e") is False
+
+    # Non-existent sync
+    with pytest.raises(NotFoundError, match="Sync with id 'non_existent' not found."):
+        app_config.check_sync_is_disabled("non_existent")
+
+    # Missing source dependency
+    sync_missing_source = SyncConfig(source="non_existent_source", pusher="pusher_e", disabled=False)
+    with pytest.raises(NotFoundError, match="Dependency source 'non_existent_source' not found."):
+        app_config.add_sync("sync_missing_source", sync_missing_source)
+
+    # Missing pusher dependency
+    sync_missing_pusher = SyncConfig(source="source_e", pusher="non_existent_pusher", disabled=False)
+    with pytest.raises(NotFoundError, match="Dependency pusher 'non_existent_pusher' not found."):
+        app_config.add_sync("sync_missing_pusher", sync_missing_pusher)

fustor_core-0.1.2.post3/tests/models/test_event.py

@@ -0,0 +1,26 @@
+from fustor_event_model.models import EventType, EventBase, InsertEvent, UpdateEvent, DeleteEvent
+
+class TestEventModels:
+    def test_event_base(self):
+        event = EventBase(fields=["id", "name"], rows=[(1, "test")], event_type=EventType.INSERT, index=123, event_schema="s", table="t")
+        assert event.event_type == EventType.INSERT
+        assert event.fields == ["id", "name"]
+        assert event.rows == [(1, "test")]
+        assert event.index == 123
+
+    def test_insert_event(self):
+        event = InsertEvent(event_schema="public", table="users", rows=[{"id": 1, "name": "test"}], fields=["id", "name"])
+        assert event.event_type == EventType.INSERT
+        assert event.event_schema == "public"
+        assert event.table == "users"
+        assert event.fields == ["id", "name"]
+
+    def test_update_event(self):
+        event = UpdateEvent(event_schema="public", table="users", rows=[{"id": 1, "name": "test_updated"}], fields=["id", "name"])
+        assert event.event_type == EventType.UPDATE
+        assert event.table == "users"
+
+    def test_delete_event(self):
+        event = DeleteEvent(event_schema="public", table="users", rows=[{"id": 1}], fields=["id"])
+        assert event.event_type == EventType.DELETE
+        assert event.table == "users"

fustor_core-0.1.2.post3/tests/models/test_states.py

@@ -0,0 +1,72 @@
+import pytest
+from fustor_core.models.states import EventBusState, SyncState, EventBusInstance, SyncInstanceDTO
+
+def test_event_bus_state_enum():
+    assert EventBusState.IDLE.name == "IDLE"
+    assert EventBusState.PRODUCING.name == "PRODUCING"
+    assert EventBusState.ERROR.name == "ERROR"
+
+def test_sync_state_enum():
+    assert SyncState.STOPPED.name == "STOPPED"
+    # --- REFACTORED: Test for new two-phase states instead of obsolete RUNNING state ---
+    assert SyncState.SNAPSHOT_SYNC.name == "SNAPSHOT_SYNC"
+    assert SyncState.MESSAGE_SYNC.name == "MESSAGE_SYNC"
+    # --- END REFACTOR ---
+    assert SyncState.RUNNING_CONF_OUTDATE.name == "RUNNING_CONF_OUTDATE"
+    assert SyncState.STOPPING.name == "STOPPING"
+    assert SyncState.ERROR.name == "ERROR"
+
+def test_event_bus_instance_dto():
+    dto = EventBusInstance(
+        id="bus-123",
+        source_name="my-source",
+        state=EventBusState.PRODUCING,
+        info="Bus is actively producing events.",
+        statistics={"events_produced": 100, "consumers": 2}
+    )
+    assert dto.id == "bus-123"
+    assert dto.source_name == "my-source"
+    assert dto.state == EventBusState.PRODUCING
+    assert dto.info == "Bus is actively producing events."
+    assert dto.statistics == {"events_produced": 100, "consumers": 2}
+
+def test_sync_instance_dto():
+    bus_dto = EventBusInstance(
+        id="bus-456",
+        source_name="another-source",
+        state=EventBusState.IDLE,
+        info="Bus is idle.",
+        statistics={}
+    )
+    # --- REFACTORED: Use one of the new valid states for the test ---
+    dto = SyncInstanceDTO(
+        id="sync-abc",
+        state=SyncState.MESSAGE_SYNC,
+        info="Sync task is running normally.",
+        bus_info=bus_dto,
+        bus_id="bus-456",
+        statistics={"events_pushed": 50, "last_event_id": "xyz"}
+    )
+    # --- END REFACTOR ---
+    assert dto.id == "sync-abc"
+    assert dto.state == SyncState.MESSAGE_SYNC
+    assert dto.info == "Sync task is running normally."
+    assert dto.bus_info == bus_dto
+    assert dto.bus_id == "bus-456"
+    assert dto.statistics == {"events_pushed": 50, "last_event_id": "xyz"}
+
+def test_sync_instance_dto_no_bus_info():
+    dto = SyncInstanceDTO(
+        id="sync-def",
+        state=SyncState.STOPPED,
+        info="Sync task is stopped.",
+        bus_info=None,
+        bus_id=None,
+        statistics={}
+    )
+    assert dto.id == "sync-def"
+    assert dto.state == SyncState.STOPPED
+    assert dto.info == "Sync task is stopped."
+    assert dto.bus_info is None
+    assert dto.bus_id is None
+    assert dto.statistics == {}