fustor-source-mysql 0.1.9__tar.gz

fustor_source_mysql-0.1.9/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: fustor-source-mysql
+Version: 0.1.9
+Summary: A MySQL source for Fustor Agent
+Author-email: Huajin Wang <wanghuajin999@163.com>
+Requires-Python: >=3.11
+Requires-Dist: fustor-core
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: mysql-replication>=1.0.9
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"

fustor_source_mysql-0.1.9/README.md

@@ -0,0 +1,47 @@
+# fustor-source-mysql
+
+This package provides a `SourceDriver` implementation for the Fustor Agent service, enabling it to extract data from MySQL databases. It supports both consistent snapshot (historical) and real-time change data capture (CDC) via MySQL's binary log.
+
+## Features
+
+*   **Consistent Snapshot Synchronization**: Performs a consistent snapshot of tables using `START TRANSACTION WITH CONSISTENT SNAPSHOT` to capture historical data.
+*   **Real-time Change Data Capture (CDC)**: Streams real-time data changes (INSERT, UPDATE, DELETE) from MySQL's binary log using `pymysqlreplication`.
+*   **Binlog Position Tracking**: Manages and checks binlog positions for resuming streams and determining data availability.
+*   **Connection Management**: Handles connection to MySQL using username/password credentials.
+*   **Runtime Parameter Validation**: Checks essential MySQL global variables like `log_bin` and `binlog_format` to ensure proper CDC setup.
+*   **Agent User Management**: Provides functionality to create a dedicated agent user with necessary replication and select privileges.
+*   **Privilege Checking**: Verifies that the agent user has the required permissions.
+*   **Field Discovery**: Dynamically discovers available fields (columns) from MySQL schemas.
+*   **Shared Instance Model**: Optimizes resource usage by sharing MySQL client instances for identical configurations.
+*   **Wizard Definition**: Provides a comprehensive configuration wizard for UI integration, guiding users through connection, runtime checks, and agent user setup.
+
+## Installation
+
+This package is part of the Fustor monorepo and is typically installed in editable mode within the monorepo's development environment using `uv sync`. It is registered as a `fustor_agent.drivers.sources` entry point.
+
+## Usage
+
+To use the `fustor-source-mysql` driver, configure a Source in your Fustor Agent setup with the driver type `mysql`. You will need to provide the MySQL URI (host:port) and credentials for both an administrative user (for setup and checks) and a dedicated agent user (for data extraction).
+
+Example (conceptual configuration in Fustor Agent):
+
+```yaml
+# ~/.fustor/config.yaml
+sources:
+  my-mysql-source:
+    driver_type: mysql
+    uri: localhost:3306
+    admin_creds: # Used for initial setup and checks, not saved
+      user: admin_user
+      passwd: admin_password
+    credential: # Dedicated agent user for data extraction, saved
+      user: fustor_agent_user
+      passwd: agent_password
+```
+
+## Dependencies
+
+*   `aiomysql`: Asynchronous MySQL client for Python.
+*   `mysql-replication`: Library for reading MySQL binary logs.
+*   `fustor-core`: Provides the `SourceDriver` abstract base class and other core components.
+*   `fustor-event-model`: Provides `EventBase` for event data structures.

fustor_source_mysql-0.1.9/pyproject.toml

@@ -0,0 +1,33 @@
+[project]
+name = "fustor-source-mysql"
+dynamic = ["version"]
+description = "A MySQL source for Fustor Agent"
+requires-python = ">=3.11"
+dependencies = [ "fustor-core", "aiomysql>=0.2.0", "mysql-replication>=1.0.9",]
+[[project.authors]]
+name = "Huajin Wang"
+email = "wanghuajin999@163.com"
+
+[build-system]
+requires = [ "setuptools>=61.0", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = "../.."
+version_scheme = "post-release"
+local_scheme = "dirty-tag"
+
+["project.urls"]
+Homepage = "https://github.com/excelwang/fustor/tree/master/packages/source_mysql"
+"Bug Tracker" = "https://github.com/excelwang/fustor/issues"
+
+license = "MIT"
+
+[project.optional-dependencies]
+dev = [ "pytest>=8.0.0", "ruff>=0.1.0", "mypy>=1.0.0", "pytest-asyncio>=0.23.0",]
+
+[project.entry-points."fustor_agent.drivers.sources"]
+mysql = "fustor_source_mysql:MysqlDriver"
+
+[tool.setuptools.packages.find]
+where = [ "src",]

fustor_source_mysql-0.1.9/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

fustor_source_mysql-0.1.9/src/fustor_source_mysql/__init__.py

@@ -0,0 +1,547 @@
+"Fuagent source driver for MySQL."
+import time
+import pymysql
+from pymysql.cursors import SSCursor
+import uuid
+from typing import Iterator, Optional, Dict, Any, Tuple, List, Set
+from decimal import Decimal
+from datetime import datetime, date, timedelta
+from contextlib import contextmanager
+from pymysqlreplication import BinLogStreamReader
+from pymysqlreplication.row_event import DeleteRowsEvent, UpdateRowsEvent, WriteRowsEvent
+import logging
+import aiomysql
+import threading
+import json
+import os
+
+from fustor_core.drivers import SourceDriver
+from fustor_core.models.config import SourceConfig, PasswdCredential
+from fustor_core.exceptions import DriverError
+from fustor_event_model.models import EventBase, InsertEvent, UpdateEvent, DeleteEvent
+
+logger = logging.getLogger("fustor_agent.driver.mysql")
+
+class MysqlDriver(SourceDriver):
+    _instances: Dict[str, 'MysqlDriver'] = {}
+    _lock = threading.Lock()
+    
+    def __new__(cls, id: str, config: SourceConfig):
+        # Generate unique signature: URI + credential to ensure permission isolation
+        signature = f"{config.uri}#{hash(str(config.credential))}"
+        
+        with MysqlDriver._lock:
+            if signature not in MysqlDriver._instances:
+                instance = super().__new__(cls)
+                MysqlDriver._instances[signature] = instance
+            return MysqlDriver._instances[signature]
+    
+    def __init__(self, id: str, config: SourceConfig):
+        # Prevent re-initialization of shared instances
+        if hasattr(self, '_initialized'):
+            return
+        
+        super().__init__(id, config)
+        self.uri = self.config.uri
+        self.credential: PasswdCredential = self.config.credential
+        self.column_maps: Dict[str, Dict[int, str]] = {}
+        self._load_schema_and_build_map()
+        
+        self._initialized = True
+
+    def _load_schema_and_build_map(self):
+        schema_file_path = os.path.join('.conf', 'schemas', f'source_{self.id}.schema.json')
+        if not os.path.exists(schema_file_path):
+            logger.warning(f"Schema file not found for source '{self.id}' at '{schema_file_path}'. Binlog events will use placeholder column names.")
+            return
+
+        try:
+            with open(schema_file_path, 'r', encoding='utf-8') as f:
+                schema_data = json.load(f)
+            
+            for table_key, table_schema in schema_data.get("properties", {}).items():
+                column_map: Dict[int, str] = {}
+                for col_name, col_props in table_schema.get("properties", {}).items():
+                    col_index = col_props.get("column_index")
+                    if col_index is not None:
+                        column_map[col_index] = col_name
+                self.column_maps[table_key] = column_map
+            logger.info(f"Successfully loaded schema and built column maps for source '{self.id}'.")
+        except (json.JSONDecodeError, IOError) as e:
+            logger.error(f"Failed to load or parse schema file '{schema_file_path}': {e}", exc_info=True)
+
+    def _get_row_with_column_names(self, table_key: str, values: List[Any]) -> Dict[str, Any]:
+        column_map = self.column_maps.get(table_key)
+        if not column_map:
+            return {f"UNKNOWN_COL{i}": val for i, val in enumerate(values)}
+        
+        row_dict = {}
+        for i, val in enumerate(values):
+            col_name = column_map.get(i, f"UNKNOWN_COL{i}")
+            row_dict[col_name] = val
+        return row_dict
+
+    def get_snapshot_iterator(self, **kwargs) -> Iterator[EventBase]:
+        stream_id = f"snapshot-{uuid.uuid4().hex[:6]}"
+        logger.info(f"[{stream_id}] Starting Consistent Snapshot.")
+        
+        snapshot_conn = None
+        try:
+            host, port_str = self.uri.split(':')
+            snapshot_conn = pymysql.connect(
+                host=host, port=int(port_str), user=self.credential.user, passwd=self.credential.passwd or ''
+            )
+            
+            with snapshot_conn.cursor(SSCursor) as cursor:
+                cursor.execute("START TRANSACTION WITH CONSISTENT SNAPSHOT")
+                logger.info(f"[{stream_id}] Transaction started for consistent snapshot.")
+                
+                cursor.execute("SHOW MASTER STATUS")
+                status = cursor.fetchone()
+                if not status:
+                    raise DriverError("Could not get master status to determine snapshot position.")
+                
+                binlog_start_pos_int = _generate_event_index(status[0], status[1])
+                logger.info(f"[{stream_id}] Consistent snapshot locked at position: {binlog_start_pos_int} ({status[0]}:{status[1]})")
+
+                required_fields = kwargs.get("required_fields_tracker").get_fields() if kwargs.get("required_fields_tracker") else set()
+                table_columns: Dict[Tuple[str, str], List[str]] = {}
+                for full_field_name in required_fields:
+                    field_parts = full_field_name.split('.')
+                    if len(field_parts) < 3: continue
+                    schema, table_name, column_name = field_parts[0], field_parts[1], field_parts[2]
+                    key = (schema, table_name)
+                    if key not in table_columns:
+                        table_columns[key] = []
+                    table_columns[key].append(column_name)
+
+                for (schema, table_name), columns in table_columns.items():
+                    if not columns: continue
+                    columns_csv = ', '.join([f"`{col}`" for col in columns])
+                    query = f"SELECT {columns_csv} FROM `{schema}`.`{table_name}`"
+                    
+                    logger.debug(f"[{stream_id}] Executing snapshot query: {query}")
+                    cursor.execute(query)
+                    
+                    batch_size = kwargs.get("batch_size", 100)
+                    while True:
+                        batch = cursor.fetchmany(batch_size)
+                        if not batch:
+                            break
+                        
+                        rows = [{columns[i]: _normalize_row(col) for i, col in enumerate(row)} for row in batch]
+                        if rows:
+                            event = InsertEvent(event_schema, table_name, rows, index=binlog_start_pos_int)
+                            yield event
+                
+                snapshot_conn.commit()
+                logger.info(f"[{stream_id}] Snapshot transaction committed.")
+
+        except Exception as e:
+            if snapshot_conn:
+                snapshot_conn.rollback()
+            logger.error(f"[{stream_id}] Snapshot phase failed, transaction rolled back: {e}", exc_info=True)
+            raise
+        finally:
+            if snapshot_conn:
+                snapshot_conn.close()
+
+    def is_position_available(self, position: int) -> bool:
+        """
+        Checks if the MySQL binlog position is available for resuming.
+        """
+        if position <= 0: #means from the latest snapshot
+            return False
+            
+        try:
+            logger.debug(f"Checking availability of binlog position {position}")
+            with _create_binlog_streamer(self.uri, self.credential, position, "pos-check", None, connect_timeout=5) as checker:
+                pass # If context manager succeeds, position is valid
+            logger.debug(f"Binlog position {position} is available.")
+            return True
+        except Exception as e:
+            # Broad exception to catch various pymysqlreplication errors for lost logs
+            logger.warning(f"Binlog position {position} is not available (Reason: {e}).")
+            return False
+
+    def get_message_iterator(self, start_position: int=-1, **kwargs) -> Iterator[EventBase]:
+        """
+        Performs incremental data capture (CDC).
+        """
+
+        def _iterator_func() -> Iterator[EventBase]:
+            stream_id = f"message-stream-{uuid.uuid4().hex[:6]}"
+           
+            stop_event = kwargs.get("stop_event")
+            required_fields_tracker = kwargs.get("required_fields_tracker")
+            max_retries = self.config.max_retries
+            retry_delay_sec = self.config.retry_delay_sec
+
+            event_id_from = start_position if start_position != -1 else 0
+            attempt = 0
+            while attempt < max_retries:
+                if stop_event and stop_event.is_set(): break
+                try:
+                    with _create_binlog_streamer(self.uri, self.credential, event_id_from, stream_id, stop_event) as streamer:
+                        for binlog_event in streamer:
+                            if stop_event and stop_event.is_set(): break
+                            
+                            if required_fields_tracker and required_fields_tracker.wait_for_change(timeout=0.1):
+                                required_fields_tracker.clear_event()
+
+                            if streamer.log_file is None or streamer.log_pos is None: continue
+                            event_index = _generate_event_index(streamer.log_file, streamer.log_pos)
+                            event = None
+                            if hasattr(binlog_event, 'rows') and binlog_event.rows:
+                                table_key = f"{binlog_event.event_schema}.{binlog_event.table}"
+                                if isinstance(binlog_event, WriteRowsEvent):
+                                    rows = [_normalize_row(self._get_row_with_column_names(table_key, row['values'])) for row in binlog_event.rows]
+                                    event = InsertEvent(binlog_event.event_schema, binlog_event.table, rows, index=event_index)
+                                elif isinstance(binlog_event, UpdateRowsEvent):
+                                    rows = [_normalize_row(self._get_row_with_column_names(table_key, row['after_values'])) for row in binlog_event.rows]
+                                    event = UpdateEvent(binlog_event.event_schema, binlog_event.table, rows, index=event_index)
+                                elif isinstance(binlog_event, DeleteRowsEvent):
+                                    rows = [_normalize_row(self._get_row_with_column_names(table_key, row['values'])) for row in binlog_event.rows]
+                                    event = DeleteEvent(binlog_event.event_schema, binlog_event.table, rows, index=event_index)
+                            
+                            if event:
+                                filtered_event = _filter_event_rows(event, required_fields_tracker.get_fields() if required_fields_tracker else set())
+                                if filtered_event:
+                                    yield filtered_event
+                            
+                            event_id_from = event_index
+                        
+                        if stop_event and stop_event.is_set(): break
+                    break
+                except Exception as e:
+                    attempt += 1
+                    if attempt < max_retries:
+                        logger.warning(f"[{stream_id}] Transient error in binlog stream (attempt {attempt}/{max_retries}): {e}")
+                        time.sleep(retry_delay_sec)
+                    else:
+                        logger.error(f"[{stream_id}] Failed after {max_retries} retries in binlog stream: {e}", exc_info=True)
+                        raise DriverError(f"Binlog streaming failed after {max_retries} retries: {e}")
+
+            logger.info(f"[{stream_id}] Message iterator finished.")
+
+        return _iterator_func()
+
+    @classmethod
+    async def test_connection(cls, **kwargs) -> Tuple[bool, str]:
+        uri = kwargs.get("uri")
+        admin_creds_dict = kwargs.get("admin_creds", {})
+        if not uri or not admin_creds_dict:
+            return (False, "缺少 'uri' 或 'admin_creds' 参数")
+        creds = PasswdCredential(**admin_creds_dict)
+        
+        conn = None
+        try:
+            conn = await _get_connection(uri, creds)
+            logger.info(f"Successfully tested connection to {uri}")
+            return True, "数据库连接成功。"
+        except Exception as e:
+            logger.error(f"MySQL async test_connection failed: {e}", exc_info=True)
+            return False, f"数据库连接失败: {e}"
+        finally:
+            if conn is not None:
+                close = getattr(conn, "close", None)
+                if callable(close):
+                    close()
+
+    @classmethod
+    async def check_runtime_params(cls, **kwargs) -> Tuple[bool, str]:
+        uri = kwargs.get("uri")
+        admin_creds_dict = kwargs.get("admin_creds", {})
+        if not uri or not admin_creds_dict:
+            return (False, "缺少 'uri' 或 'admin_creds' 参数")
+        admin_creds = PasswdCredential(**admin_creds_dict)
+
+        conn = None
+        try:
+            conn = await _get_connection(uri, admin_creds)
+            async with conn.cursor() as cursor:
+                await cursor.execute("SHOW GLOBAL VARIABLES LIKE 'log_bin'")
+                log_bin = await cursor.fetchone()
+                if not log_bin or log_bin[1] != 'ON':
+                    return (False, "配置检查失败: 全局变量 'log_bin' 必须为 'ON'")
+                
+                await cursor.execute("SHOW GLOBAL VARIABLES LIKE 'binlog_format'")
+                binlog_format = await cursor.fetchone()
+                if not binlog_format or binlog_format[1] != 'ROW':
+                    return (False, "配置检查失败: 全局变量 'binlog_format' 必须为 'ROW'")
+            logger.info("Runtime parameters check passed")
+            return True, "运行时参数有效。"
+        except Exception as e:
+            logger.error(f"MySQL check_runtime_params failed: {e}", exc_info=True)
+            return False, f"检查运行时参数失败: {e}"
+        finally:
+            if conn is not None:
+                close = getattr(conn, "close", None)
+                if callable(close):
+                    close()
+
+    @classmethod
+    async def create_agent_user(cls, **kwargs) -> Tuple[bool, str]:
+        uri = kwargs.get("uri")
+        admin_creds_dict = kwargs.get("admin_creds", {})
+        agent_user_dict = kwargs.get("credential", {})
+        if not uri or not admin_creds_dict or not agent_user_dict:
+            return (False, "缺少 'uri', 'admin_creds', 或 'credential' 参数")
+        admin_creds = PasswdCredential(**admin_creds_dict)
+        agent_user = PasswdCredential(**agent_user_dict)
+        
+        conn = None
+        try:
+            conn = await _get_connection(uri, admin_creds)
+            async with conn.cursor() as cursor:
+                await cursor.execute(
+                    "CREATE USER IF NOT EXISTS %s@%s IDENTIFIED BY %s",
+                    (agent_user.user, '%', agent_user.passwd or '')
+                )
+                await cursor.execute(
+                    "GRANT REPLICATION SLAVE, REPLICATION CLIENT, SELECT ON *.* TO %s@%s",
+                    (agent_user.user, '%')
+                )
+                await cursor.execute("FLUSH PRIVILEGES")
+            logger.info(f"User '{agent_user.user}' is ready for replication.")
+            return True, f"用户 '{agent_user.user}' 已成功创建或验证。"
+        except Exception as e:
+            logger.error(f"Failed to create or grant privileges to user '{agent_user.user}': {e}", exc_info=True)
+            return False, f"创建或授权用户 '{agent_user.user}' 失败: {e}"
+        finally:
+            if conn is not None:
+                close = getattr(conn, "close", None)
+                if callable(close):
+                    close()
+
+    @classmethod
+    async def check_privileges(cls, **kwargs) -> Tuple[bool, str]:
+        uri = kwargs.get("uri")
+        admin_creds_dict = kwargs.get("admin_creds", {})
+        agent_user_dict = kwargs.get("credential", {})
+        if not uri or not admin_creds_dict or not agent_user_dict:
+            return (False, "缺少 'uri', 'admin_creds', 或 'credential' 参数")
+        admin_creds = PasswdCredential(**admin_creds_dict)
+        agent_user = PasswdCredential(**agent_user_dict)
+        
+        conn = None
+        try:
+            conn = await _get_connection(uri, admin_creds)
+            async with conn.cursor() as cursor:
+                await cursor.execute(
+                    "SELECT Repl_slave_priv, Repl_client_priv, Select_priv FROM mysql.user WHERE User = %s AND Host = %s",
+                    (agent_user.user, '%')
+                )
+                result = await cursor.fetchone()
+                if not result or result[0] != 'Y' or result[1] != 'Y' or result[2] != 'Y':
+                    msg = f"用户 '{agent_user.user}' 缺少必要的权限 (REPLICATION SLAVE, REPLICATION CLIENT, SELECT)。"
+                    logger.error(msg)
+                    return False, msg
+
+            logger.info(f"User '{agent_user.user}' privileges verified")
+            return True, f"用户 '{agent_user.user}' 权限充足。"
+        except Exception as e:
+            logger.error(f"MySQL check_user_privileges failed for user '{agent_user.user}': {e}", exc_info=True)
+            return False, f"检查用户 '{agent_user.user}' 权限失败: {e}"
+        finally:
+            if conn is not None:
+                close = getattr(conn, "close", None)
+                if callable(close):
+                    close()
+
+    @classmethod
+    async def get_available_fields(cls, **kwargs) -> Dict[str, Any]:
+        uri = kwargs.get("uri")
+        admin_creds_dict = kwargs.get("admin_creds")
+        if not uri or not admin_creds_dict:
+            raise DriverError("get_available_fields requires 'uri' and 'admin_creds'.")
+        
+        creds = PasswdCredential(**admin_creds_dict)
+
+        conn = None
+        try:
+            conn = await _get_connection(uri, creds)
+            available_fields = {}
+            system_schemas = ('information_schema', 'mysql', 'performance_schema', 'sys')
+            
+            async with conn.cursor(aiomysql.DictCursor) as cursor:
+                await cursor.execute("SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, ORDINAL_POSITION FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA NOT IN %s ORDER BY TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION", (system_schemas,))
+                rows = await cursor.fetchall()
+                for row in rows:
+                    composite_key = f"{row['TABLE_SCHEMA']}.{row['TABLE_NAME']}.{row['COLUMN_NAME']}"
+                    available_fields[composite_key] = {"type": "string", "column_index": row['ORDINAL_POSITION'] - 1} 
+            
+            logger.info(f"Successfully retrieved {len(available_fields)} available fields from {uri}.")
+            return {"properties": available_fields}
+        except pymysql.err.OperationalError as e:
+            error_message = f"连接到 MySQL 失败: 访问被拒绝。请检查用户名、密码和主机权限。"
+            logger.debug(f"Original MySQL connection error in mysql driver: {e}", exc_info=True)
+            raise DriverError(error_message) from e
+        except Exception as e:
+            logger.error(f"Error getting available fields: {e}", exc_info=True)
+            raise
+        finally:
+            if conn is not None:
+                conn.close()
+    
+    @classmethod
+    async def get_wizard_steps(cls) -> Dict[str, Any]:
+        return {
+            "steps": [
+                {
+                    "step_id": "connection",
+                    "title": "连接与发现",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "uri": {
+                                "type": "string",
+                                "title": "URI",
+                                "description": "MySQL服务器地址 (例如, localhost:3306)",
+                                "pattern": "^[a-zA-Z0-9._-]+:\\\\d+$"
+                            },
+                            "admin_creds": {
+                                "$ref": "#/components/schemas/PasswdCredential",
+                                "title": "管理员凭证",
+                                "description": "用于执行连接测试、环境检查和创建代理用户的一次性管理员凭证。此凭证不会被保存。"
+                            }
+                        },
+                        "required": ["uri", "admin_creds"]
+                    },
+                    "validations": ["test_connection", "check_params", "discover_fields_no_cache"]
+                },
+                {
+                    "step_id": "agent_setup",
+                    "title": "代理用户与参数",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "credential": {
+                                "$ref": "#/components/schemas/PasswdCredential",
+                                "title": "代理用户凭证",
+                                "description": "为FuAgent创建一个专用的、权限受限的用户，用于日常的数据拉取。此凭证将被保存。"
+                            }
+                        },
+                        "required": ["credential"]
+                    },
+                    "validations": ["create_agent_user", "check_privileges"]
+                }
+            ],
+            "components": {
+                "schemas": {
+                    "PasswdCredential": {
+                        "type": "object",
+                        "title": "用户名/密码凭证",
+                        "properties": {
+                            "user": { "type": "string", "title": "用户名" },
+                            "passwd": { "type": "string", "title": "密码", "format": "password" }
+                        },
+                        "required": ["user"]
+                    }
+                }
+            }
+        }
+
+# --- Module-level helper functions and classes ---
+
+@contextmanager
+def _create_binlog_streamer(
+   uri: str, user_creds: PasswdCredential, event_id_from: int, stream_id: str, stop_event: Optional[threading.Event] = None, connect_timeout: int = 30
+) -> Iterator[BinLogStreamReader]:
+    streamer = None
+    try:
+        host, port_str = uri.split(':')
+        mysql_settings = {
+            "host": host,
+            "port": int(port_str),
+            "user": user_creds.user,
+            "passwd": user_creds.passwd or ''
+        }
+        
+        log_file, log_pos = _parse_event_index(event_id_from)
+
+        server_id = 10086 + int(uuid.uuid4().hex[:8], 16) % 1000
+        streamer = BinLogStreamReader(
+            connection_settings=mysql_settings,
+            server_id=server_id,
+            resume_stream=True,
+            log_file=log_file,
+            log_pos=log_pos,
+            blocking=True,
+            only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent]
+        )
+        logger.info(f"Stream {stream_id}: Started MySQL binlog monitoring from {log_file}:{log_pos} with server_id {server_id}")
+        if stop_event and stop_event.is_set():
+            logger.info(f"Stream {stream_id}: Stop event already set, not starting binlog stream.")
+            return
+        yield streamer
+    except Exception as e:
+        logger.error(f"Stream {stream_id}: Failed to create BinLogStreamReader: {e}", exc_info=True)
+        raise
+    finally:
+        if streamer:
+            streamer.close()
+            logger.info(f"Stream {stream_id}: MySQL binlog stream closed.")
+
+async def _get_connection(uri: str, creds: PasswdCredential) -> aiomysql.Connection:
+    host, port_str = uri.split(':')
+    return await aiomysql.connect(
+        host=host, port=int(port_str), user=creds.user,
+        password=creds.passwd or '', autocommit=True
+    )
+
+def _generate_event_index(log_file: str, log_pos: int) -> int:
+    if not log_file:
+        return 0
+    try:
+        return (int(log_file.split('.')[-1]) << 32) | log_pos
+    except (ValueError, IndexError):
+        logger.warning(f"Invalid log_file format: {log_file}, returning default index 0")
+        return 0
+
+def _parse_event_index(index: int) -> Tuple[Optional[str], int]:
+    if index == 0:
+        return None, 4
+    try:
+        return f"mysql-bin.{index >> 32:06d}", index & 0xFFFFFFFF
+    except Exception as e:
+        logger.error(f"Failed to parse event index {index}: {e}", exc_info=True)
+        return None, 4
+
+def _normalize_row(data):
+    if isinstance(data, dict):
+        return {k: _normalize_row(v) for k, v in data.items()}
+    if isinstance(data, list):
+        return [_normalize_row(item) for item in data]
+    if isinstance(data, (datetime, date, timedelta)):
+        return str(data)
+    if isinstance(data, Decimal):
+        return float(data)
+    return data
+
+def _filter_event_rows(event: EventBase, required_fields: Set[str]) -> Optional[EventBase]:
+    if not required_fields:
+        return event
+    
+    event_prefix = f"{event.event_schema}.{event.table}."
+    if not any(f.startswith(event_prefix) for f in required_fields):
+        return None
+
+    filtered_rows = []
+    for row in event.rows:
+        filtered_row = {}
+        for field_name, field_value in row.items():
+            full_field_name = f"{event.event_schema}.{event.table}.{field_name}"
+            if full_field_name in required_fields:
+                filtered_row[field_name] = field_value
+        
+        if filtered_row:
+            filtered_rows.append(filtered_row)
+    
+    if filtered_rows:
+        new_event = type(event)(event.event_schema, event.table, filtered_rows)
+        new_event.fields = list(filtered_rows[0].keys())
+        new_event.index = event.index
+        return new_event
+        
+    return None

fustor_source_mysql-0.1.9/src/fustor_source_mysql.egg-info/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.4
+Name: fustor-source-mysql
+Version: 0.1.9
+Summary: A MySQL source for Fustor Agent
+Author-email: Huajin Wang <wanghuajin999@163.com>
+Requires-Python: >=3.11
+Requires-Dist: fustor-core
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: mysql-replication>=1.0.9
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: ruff>=0.1.0; extra == "dev"
+Requires-Dist: mypy>=1.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.23.0; extra == "dev"

fustor_source_mysql-0.1.9/src/fustor_source_mysql.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+README.md
+pyproject.toml
+src/fustor_source_mysql/__init__.py
+src/fustor_source_mysql/py.typed
+src/fustor_source_mysql.egg-info/PKG-INFO
+src/fustor_source_mysql.egg-info/SOURCES.txt
+src/fustor_source_mysql.egg-info/dependency_links.txt
+src/fustor_source_mysql.egg-info/entry_points.txt
+src/fustor_source_mysql.egg-info/requires.txt
+src/fustor_source_mysql.egg-info/top_level.txt
+tests/conftest.py
+tests/docker-mysql.yml
+tests/mysql-init.sql

fustor_source_mysql-0.1.9/src/fustor_source_mysql.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

fustor_source_mysql-0.1.9/src/fustor_source_mysql.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[fustor_agent.drivers.sources]
+mysql = fustor_source_mysql:MysqlDriver

fustor_source_mysql-0.1.9/src/fustor_source_mysql.egg-info/requires.txt

@@ -0,0 +1,9 @@
+fustor-core
+aiomysql>=0.2.0
+mysql-replication>=1.0.9
+
+[dev]
+pytest>=8.0.0
+ruff>=0.1.0
+mypy>=1.0.0
+pytest-asyncio>=0.23.0

fustor_source_mysql-0.1.9/src/fustor_source_mysql.egg-info/top_level.txt

@@ -0,0 +1 @@
+fustor_source_mysql

fustor_source_mysql-0.1.9/tests/conftest.py

@@ -0,0 +1,52 @@
+import pytest
+import pytest_asyncio
+import os
+import pymysql
+
+from fustor_agent.app import App
+
+@pytest_asyncio.fixture(scope="function")
+async def test_db_setup(test_app_instance: App):
+    # ... (this fixture is unchanged)
+    source_config = test_app_instance.source_config_service.get_config('test-test')
+    if not source_config:
+        pytest.fail("Source config 'test-test' not found. Ensure config.yaml is correctly set up for tests.")
+
+    mysql_root_password = os.getenv("MYSQL_ROOT_PASSWORD", "")
+    if not mysql_root_password:
+        pytest.skip("MYSQL_ROOT_PASSWORD environment variable not set, skipping integration test.")
+
+    conn = pymysql.connect(
+        host=source_config.uri.split(':')[0],
+        port=int(source_config.uri.split(':')[1]),
+        user="root",
+        password=mysql_root_password,
+        database="testdb"
+    )
+    table_name = "test_snapshot_table"
+    with conn.cursor() as cursor:
+        cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
+        cursor.execute(f"""
+            CREATE TABLE {table_name} (
+                id INT AUTO_INCREMENT PRIMARY KEY,
+                name VARCHAR(255) NOT NULL,
+                value INT
+            );
+        """)
+        cursor.executemany(f"INSERT INTO {table_name} (name, value) VALUES (%s, %s)", [('record_1', 100), ('record_2', 200), ('record_3', 300)])
+    conn.commit()
+    conn.close()
+
+    yield table_name
+
+    conn = pymysql.connect(
+        host=source_config.uri.split(':')[0],
+        port=int(source_config.uri.split(':')[1]),
+        user="root",
+        password=mysql_root_password,
+        database="testdb"
+    )
+    with conn.cursor() as cursor:
+        cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
+    conn.commit()
+    conn.close()

fustor_source_mysql-0.1.9/tests/docker-mysql.yml

@@ -0,0 +1,25 @@
+services:
+  mysql-test:
+    image: mysql:8.2.0
+    command: 
+      --upgrade=FORCE 
+      --log-bin=mysql-bin 
+      --server-id=1
+      --binlog-format=ROW
+      --log_replica_updates=ON
+      --binlog_expire_logs_seconds=0
+    container_name: fuagent-mysql-test
+    environment:
+      MYSQL_ROOT_PASSWORD: testroot
+      MYSQL_DATABASE: testdb
+      MYSQL_USER: testuser
+      MYSQL_PASSWORD: testpass
+    ports:
+      - "3307:3306"
+    healthcheck:
+      test: ["CMD", "sh", "-c", "mysqladmin ping -uroot -ptestroot --protocol=tcp"]
+      interval: 5s
+      timeout: 10s
+      retries: 60
+    volumes:
+      - ./mysql-init.sql:/docker-entrypoint-initdb.d/init.sql

fustor_source_mysql-0.1.9/tests/mysql-init.sql

@@ -0,0 +1,90 @@
+-- 创建测试表
+-- =================================================================
+-- Table structure for user (用户信息表)
+-- =================================================================
+CREATE TABLE `user` (
+  `user_id` INT NOT NULL AUTO_INCREMENT COMMENT '用户主键ID',
+  `email` VARCHAR(128) NULL,
+  `openid_user_id` VARCHAR(50) NULL,
+  PRIMARY KEY (`user_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- =================================================================
+-- Table structure for dataset (数据信息表)
+-- =================================================================
+CREATE TABLE `dataset` (
+  `dataset_id` INT NOT NULL AUTO_INCREMENT COMMENT '数据集主键ID',
+  `user_id` INT NULL COMMENT '外键，关联到用户表',
+  `status` TINYINT(1) NULL,
+  `create_time` DATETIME NULL,
+  `title` VARCHAR(300) NULL,
+  `description` MEDIUMTEXT NULL,
+  `is_deleted` TINYINT(1) DEFAULT 0,
+  `is_checked` INT NULL,
+  `is_review` INT NULL,
+  `backup_status` INT NULL,
+  `path` VARCHAR(60) NULL,
+  PRIMARY KEY (`dataset_id`),
+  CONSTRAINT `fk_dataset_user` FOREIGN KEY (`user_id`) REFERENCES `user` (`user_id`) ON DELETE SET NULL ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- =================================================================
+-- Table structure for file (文件信息表)
+-- =================================================================
+CREATE TABLE `file` (
+  `file_id` INT NOT NULL AUTO_INCREMENT COMMENT '文件主键ID',
+  `dataset_id` INT NULL COMMENT '外键，关联到数据信息表(dataset)',
+  `file_name` VARCHAR(300) NULL,
+  `is_deleted` TINYINT(1) DEFAULT 0,
+  `relative_path` VARCHAR(500) NULL,
+  `file_size` VARCHAR(100) NULL,
+  `file_suffix` VARCHAR(300) NULL,
+  `file_code` VARCHAR(300) NULL,
+  `md5` VARCHAR(300) NULL,
+  `status` VARCHAR(30) NULL,
+  `create_time` DATETIME NULL,
+  PRIMARY KEY (`file_id`),
+  CONSTRAINT `fk_file_dataset` FOREIGN KEY (`dataset_id`) REFERENCES `dataset` (`dataset_id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- 启用 binlog需要依赖容器启动参数配置
+SET GLOBAL binlog_format = ROW;
+
+-- 创建必要的权限（测试专用）
+GRANT REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'testuser'@'%';
+GRANT SELECT, INSERT, UPDATE, DELETE ON testdb.* TO 'testuser'@'%';
+FLUSH PRIVILEGES;
+
+-- =================================================================
+-- 插入测试数据 (5条/表)
+-- =================================================================
+
+-- user表数据
+INSERT INTO `user` (`email`, `openid_user_id`) VALUES
+('user1@example.com', 'openid1'),
+('user2@example.com', 'openid2'),
+('user3@example.com', 'openid3'),
+('user4@example.com', 'openid4'),
+('user5@example.com', 'openid5');
+
+-- dataset表数据
+INSERT INTO `dataset` (
+  `user_id`, `status`, `create_time`, `title`,
+  `description`, `is_checked`, `is_review`, `backup_status`, `path`
+) VALUES
+(1, 1, NOW(), 'Dataset 1', 'Description 1', 1, 0, 0, '/datasets/1'),
+(2, 1, NOW(), 'Dataset 2', 'Description 2', 1, 0, 0, '/datasets/2'),
+(3, 1, NOW(), 'Dataset 3', 'Description 3', 1, 0, 0, '/datasets/3'),
+(4, 1, NOW(), 'Dataset 4', 'Description 4', 1, 0, 0, '/datasets/4'),
+(5, 1, NOW(), 'Dataset 5', 'Description 5', 1, 0, 0, '/datasets/5');
+
+-- file表数据
+INSERT INTO `file` (
+  `dataset_id`, `file_name`, `relative_path`, `file_size`,
+  `file_suffix`, `file_code`, `md5`, `status`, `create_time`
+) VALUES
+(1, 'file1.txt', 'files/1', '1024', 'txt', 'F001', 'md51', 'active', NOW()),
+(2, 'file2.jpg', 'files/2', '2048', 'jpg', 'F002', 'md52', 'active', NOW()),
+(3, 'file3.pdf', 'files/3', '3072', 'pdf', 'F003', 'md53', 'active', NOW()),
+(4, 'file4.png', 'files/4', '4096', 'png', 'F004', 'md54', 'active', NOW()),
+(5, 'file5.doc', 'files/5', '5120', 'doc', 'F005', 'md55', 'active', NOW());