fustor-sender-echo 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,48 @@
1
+ Metadata-Version: 2.4
2
+ Name: fustor-sender-echo
3
+ Version: 0.1.0
4
+ Summary: Echo sender driver for Fustor
5
+ Author: Fustor Team
6
+ Requires-Python: >=3.10
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: fustor-core
9
+
10
+ # fustor-sender-echo
11
+
12
+ This package provides an "echo" sender driver for the Fustor Agent service. It serves as a basic example and debugging tool for `SenderDriver` implementations. Instead of sending data to an external system, it simply logs all received events and control flags to the Fustor Agent's log output.
13
+
14
+ ## Features
15
+
16
+ * **Echo Functionality**: Logs all incoming events, including `realtime` and `snapshot` data, to the console/log.
17
+ * **Control Flag Visibility**: Displays control flags like `is_snapshot_end` and `snapshot_sync_suggested` for debugging data flow.
18
+ * **Session Management**: Implements `create_session` and `heartbeat` to demonstrate session lifecycle.
19
+ * **No Configuration Needed**: The `get_needed_fields` method returns an empty schema, indicating it accepts all fields without specific requirements.
20
+ * **Wizard Definition**: Provides a simple wizard step for UI integration.
21
+
22
+ ## Installation
23
+
24
+ This package is part of the Fustor monorepo and is typically installed in editable mode within the monorepo's development environment using `uv sync`. It is registered as a `fustor_agent.drivers.senders` entry point.
25
+
26
+ ## Usage
27
+
28
+ To use the `fustor-sender-echo` driver, configure a Sender in your Fustor Agent setup with the driver type `echo`. When a pipe involves this sender, all data processed by the Agent will be logged by this driver.
29
+
30
+ This driver is particularly useful for:
31
+ * **Debugging**: Understanding the exact data and control signals being sent by the Fustor Agent.
32
+ * **Development**: As a template for creating new `SenderDriver` implementations.
33
+ * **Testing**: Verifying that the Fustor Agent's data pipe is correctly delivering events.
34
+
35
+ Example (conceptual configuration in Fustor Agent):
36
+
37
+ ```yaml
38
+ # Fustor 主目录下的 agent-config.yaml
39
+ senders:
40
+ my-echo-sender:
41
+ driver_type: echo
42
+ # No specific configuration parameters needed for the echo driver
43
+ ```
44
+
45
+ ## Dependencies
46
+
47
+ * `fustor-core`: Provides the `SenderDriver` abstract base class and other core components.
48
+ * `fustor-event-model`: Provides `EventBase` for event data structures.
@@ -0,0 +1,39 @@
1
+ # fustor-sender-echo
2
+
3
+ This package provides an "echo" sender driver for the Fustor Agent service. It serves as a basic example and debugging tool for `SenderDriver` implementations. Instead of sending data to an external system, it simply logs all received events and control flags to the Fustor Agent's log output.
4
+
5
+ ## Features
6
+
7
+ * **Echo Functionality**: Logs all incoming events, including `realtime` and `snapshot` data, to the console/log.
8
+ * **Control Flag Visibility**: Displays control flags like `is_snapshot_end` and `snapshot_sync_suggested` for debugging data flow.
9
+ * **Session Management**: Implements `create_session` and `heartbeat` to demonstrate session lifecycle.
10
+ * **No Configuration Needed**: The `get_needed_fields` method returns an empty schema, indicating it accepts all fields without specific requirements.
11
+ * **Wizard Definition**: Provides a simple wizard step for UI integration.
12
+
13
+ ## Installation
14
+
15
+ This package is part of the Fustor monorepo and is typically installed in editable mode within the monorepo's development environment using `uv sync`. It is registered as a `fustor_agent.drivers.senders` entry point.
16
+
17
+ ## Usage
18
+
19
+ To use the `fustor-sender-echo` driver, configure a Sender in your Fustor Agent setup with the driver type `echo`. When a pipe involves this sender, all data processed by the Agent will be logged by this driver.
20
+
21
+ This driver is particularly useful for:
22
+ * **Debugging**: Understanding the exact data and control signals being sent by the Fustor Agent.
23
+ * **Development**: As a template for creating new `SenderDriver` implementations.
24
+ * **Testing**: Verifying that the Fustor Agent's data pipe is correctly delivering events.
25
+
26
+ Example (conceptual configuration in Fustor Agent):
27
+
28
+ ```yaml
29
+ # Fustor 主目录下的 agent-config.yaml
30
+ senders:
31
+ my-echo-sender:
32
+ driver_type: echo
33
+ # No specific configuration parameters needed for the echo driver
34
+ ```
35
+
36
+ ## Dependencies
37
+
38
+ * `fustor-core`: Provides the `SenderDriver` abstract base class and other core components.
39
+ * `fustor-event-model`: Provides `EventBase` for event data structures.
@@ -0,0 +1,28 @@
1
+ [project]
2
+ name = "fustor-sender-echo"
3
+ version = "0.1.0"
4
+ description = "Echo sender driver for Fustor"
5
+ requires-python = ">= 3.10"
6
+ dependencies = [
7
+ "fustor-core",
8
+ ]
9
+ authors = [
10
+ { name = "Fustor Team" }
11
+ ]
12
+ readme = "README.md"
13
+
14
+ [build-system]
15
+ requires = ["setuptools>=61.0"]
16
+ build-backend = "setuptools.build_meta"
17
+
18
+ ["project.urls"]
19
+ Homepage = "https://github.com/excelwang/fustor/tree/master/extensions/sender-echo"
20
+ "Bug Tracker" = "https://github.com/excelwang/fustor/issues"
21
+
22
+ license = "MIT"
23
+
24
+ [project.entry-points."fustor_agent.drivers.senders"]
25
+ echo = "fustor_sender_echo:EchoDriver"
26
+
27
+ [tool.setuptools.packages.find]
28
+ where = [ "src",]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,131 @@
1
+ """
2
+ Fustor Echo Sender Driver (Class-based)
3
+ """
4
+ import json
5
+ import logging
6
+ from typing import Any, Dict, List, Optional
7
+
8
+ from fustor_core.transport import Sender
9
+ from fustor_core.models.config import SenderConfig
10
+ from fustor_core.event import EventBase
11
+
12
+
13
+ class EchoDriver(Sender):
14
+ """
15
+ An echo driver that inherits from the Sender ABC.
16
+ It prints batch and cumulative statistics for all received events.
17
+ """
18
+ def __init__(
19
+ self,
20
+ sender_id: str,
21
+ endpoint: str,
22
+ credential: Dict[str, Any],
23
+ config: Optional[Dict[str, Any]] = None
24
+ ):
25
+ """Initializes the driver and its statistics counters."""
26
+ super().__init__(sender_id, endpoint, credential, config)
27
+ self.total_rows = 0
28
+ self.total_size = 0
29
+ self.logger = logging.getLogger(f"fustor_agent.sender.echo.{sender_id}")
30
+ self._snapshot_triggered = False
31
+
32
+ async def connect(self) -> None:
33
+ """Echo driver connection is a no-op."""
34
+ self.logger.info(f"Echo Sender {self.id} ready.")
35
+
36
+ async def _send_events_impl(
37
+ self, events: List[Any], source_type: str = "realtime", is_end: bool = False
38
+ ) -> Dict:
39
+ """
40
+ Implementation of echo sending.
41
+ """
42
+ batch_rows = 0
43
+ for event in events:
44
+ # Handle both EventBase objects and raw dicts
45
+ if hasattr(event, "rows"):
46
+ batch_rows += len(event.rows)
47
+ elif isinstance(event, dict):
48
+ batch_rows += len(event.get("rows", []))
49
+
50
+ self.total_rows += batch_rows
51
+
52
+ # Prepare a summary for logging
53
+ flags = []
54
+ if is_end:
55
+ flags.append("END")
56
+
57
+ flags_str = f" | Flags: {', '.join(flags)}" if flags else ""
58
+
59
+ self.logger.info(
60
+ f"[EchoSender] [{source_type.upper()}] Task: {self.id} | 本批次: {batch_rows}条; 累计: {self.total_rows}条{flags_str}"
61
+ )
62
+
63
+ # For debugging, log the first event's data if available
64
+ if events:
65
+ first_event_rows = None
66
+ if hasattr(events[0], "rows"):
67
+ first_event_rows = events[0].rows
68
+ elif isinstance(events[0], dict):
69
+ first_event_rows = events[0].get("rows", [])
70
+
71
+ if first_event_rows:
72
+ self.logger.info(f"First event data: {json.dumps(first_event_rows[0], ensure_ascii=False)}")
73
+
74
+ # Trigger snapshot only once if the condition is met
75
+ snapshot_needed = False
76
+ if not self._snapshot_triggered:
77
+ snapshot_needed = True
78
+ self._snapshot_triggered = True
79
+ self.logger.info(f"Task '{self.id}' is triggering a one-time snapshot.")
80
+
81
+ return {"success": True, "snapshot_needed": snapshot_needed}
82
+
83
+ async def heartbeat(self, **kwargs) -> Dict[str, Any]:
84
+ """
85
+ Sends a heartbeat to maintain session state.
86
+ """
87
+ self.logger.info(f"[EchoSender] Heartbeat for session {self.session_id} from task {self.id}")
88
+ return {"status": "ok", "role": self.config.get("mock_role", "leader")}
89
+
90
+ async def create_session(
91
+ self,
92
+ task_id: str,
93
+ source_type: Optional[str] = None,
94
+ session_timeout_seconds: Optional[int] = None,
95
+ **kwargs
96
+ ) -> Dict[str, Any]:
97
+ """
98
+ Creates a new session.
99
+ """
100
+ import uuid
101
+ session_id = str(uuid.uuid4())
102
+ self.session_id = session_id
103
+ role = self.config.get("mock_role", "leader")
104
+ timeout = session_timeout_seconds or self.config.get("session_timeout_seconds", 30)
105
+ self.logger.info(f"[EchoSender] Created session {session_id} for task {task_id} with role {role}")
106
+ metadata = {
107
+ "session_id": session_id,
108
+ "role": role,
109
+ "session_timeout_seconds": timeout
110
+ }
111
+ return session_id, metadata
112
+
113
+ async def close_session(self) -> None:
114
+ """Close the current session."""
115
+ if self.session_id:
116
+ self.logger.info(f"[EchoSender] Closing session {self.session_id}")
117
+ self.session_id = None
118
+
119
+ async def close(self) -> None:
120
+ """Close the sender."""
121
+ self.logger.info(f"[EchoSender] Sender {self.id} closed")
122
+ self._snapshot_triggered = False
123
+
124
+ @classmethod
125
+ async def get_needed_fields(cls, **kwargs) -> Dict[str, Any]:
126
+ """
127
+ The echo driver does not need any specific fields, so it returns an empty schema,
128
+ signaling that it accepts all fields.
129
+ """
130
+ return {}
131
+
@@ -0,0 +1,48 @@
1
+ Metadata-Version: 2.4
2
+ Name: fustor-sender-echo
3
+ Version: 0.1.0
4
+ Summary: Echo sender driver for Fustor
5
+ Author: Fustor Team
6
+ Requires-Python: >=3.10
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: fustor-core
9
+
10
+ # fustor-sender-echo
11
+
12
+ This package provides an "echo" sender driver for the Fustor Agent service. It serves as a basic example and debugging tool for `SenderDriver` implementations. Instead of sending data to an external system, it simply logs all received events and control flags to the Fustor Agent's log output.
13
+
14
+ ## Features
15
+
16
+ * **Echo Functionality**: Logs all incoming events, including `realtime` and `snapshot` data, to the console/log.
17
+ * **Control Flag Visibility**: Displays control flags like `is_snapshot_end` and `snapshot_sync_suggested` for debugging data flow.
18
+ * **Session Management**: Implements `create_session` and `heartbeat` to demonstrate session lifecycle.
19
+ * **No Configuration Needed**: The `get_needed_fields` method returns an empty schema, indicating it accepts all fields without specific requirements.
20
+ * **Wizard Definition**: Provides a simple wizard step for UI integration.
21
+
22
+ ## Installation
23
+
24
+ This package is part of the Fustor monorepo and is typically installed in editable mode within the monorepo's development environment using `uv sync`. It is registered as a `fustor_agent.drivers.senders` entry point.
25
+
26
+ ## Usage
27
+
28
+ To use the `fustor-sender-echo` driver, configure a Sender in your Fustor Agent setup with the driver type `echo`. When a pipe involves this sender, all data processed by the Agent will be logged by this driver.
29
+
30
+ This driver is particularly useful for:
31
+ * **Debugging**: Understanding the exact data and control signals being sent by the Fustor Agent.
32
+ * **Development**: As a template for creating new `SenderDriver` implementations.
33
+ * **Testing**: Verifying that the Fustor Agent's data pipe is correctly delivering events.
34
+
35
+ Example (conceptual configuration in Fustor Agent):
36
+
37
+ ```yaml
38
+ # Fustor 主目录下的 agent-config.yaml
39
+ senders:
40
+ my-echo-sender:
41
+ driver_type: echo
42
+ # No specific configuration parameters needed for the echo driver
43
+ ```
44
+
45
+ ## Dependencies
46
+
47
+ * `fustor-core`: Provides the `SenderDriver` abstract base class and other core components.
48
+ * `fustor-event-model`: Provides `EventBase` for event data structures.
@@ -0,0 +1,13 @@
1
+ README.md
2
+ pyproject.toml
3
+ src/fustor_sender_echo/__init__.py
4
+ src/fustor_sender_echo.egg-info/PKG-INFO
5
+ src/fustor_sender_echo.egg-info/SOURCES.txt
6
+ src/fustor_sender_echo.egg-info/dependency_links.txt
7
+ src/fustor_sender_echo.egg-info/entry_points.txt
8
+ src/fustor_sender_echo.egg-info/requires.txt
9
+ src/fustor_sender_echo.egg-info/top_level.txt
10
+ tests/test_echo_driver.py
11
+ tests/test_echo_integration.py
12
+ tests/test_echo_logging.py
13
+ tests/test_echo_snapshot_trigger.py
@@ -0,0 +1,2 @@
1
+ [fustor_agent.drivers.senders]
2
+ echo = fustor_sender_echo:EchoDriver
@@ -0,0 +1 @@
1
+ fustor_sender_echo
@@ -0,0 +1,82 @@
1
+ import pytest
2
+ import json
3
+ import logging
4
+ from io import StringIO
5
+ from fustor_sender_echo import EchoDriver
6
+ from fustor_core.models.config import SenderConfig, PasswdCredential
7
+ from fustor_core.event import InsertEvent
8
+
9
+ @pytest.mark.asyncio
10
+ async def test_echo_driver_push(caplog):
11
+ """Tests the send_events method of the EchoDriver conforms to the new interface."""
12
+ # 1. Arrange
13
+ credential = {"user": "test"}
14
+ config = {"batch_size": 10}
15
+ driver = EchoDriver("test-echo-id", "http://localhost", credential, config)
16
+ events = [InsertEvent(event_schema="test_schema", table="test_table", rows=[{"id": 1, "msg": "hello"}], fields=["id", "msg"])]
17
+
18
+ # 2. Act & 3. Assert - Check logging
19
+ with caplog.at_level(logging.INFO):
20
+ result = await driver.send_events(events, source_type="realtime", is_end=True)
21
+
22
+ # Check for the summary output in logs
23
+ assert "[EchoSender]" in caplog.text
24
+ assert "[REALTIME]" in caplog.text
25
+ assert "Task: test-echo-id" in caplog.text
26
+ assert "本批次: 1条" in caplog.text
27
+ assert "累计: 1条" in caplog.text
28
+ assert "Flags: END" in caplog.text
29
+
30
+ # Check the result dictionary
31
+ assert result == {"success": True, "snapshot_needed": True}
32
+
33
+ @pytest.mark.asyncio
34
+ async def test_echo_driver_get_needed_fields():
35
+ """Tests the get_needed_fields class method."""
36
+ # 1. Arrange & 2. Act
37
+ fields = await EchoDriver.get_needed_fields()
38
+
39
+ # 3. Assert
40
+ assert fields == {}
41
+
42
+
43
+ @pytest.mark.asyncio
44
+ async def test_echo_driver_cumulative_push(caplog):
45
+ """Tests that the driver correctly accumulates row counts over multiple pushes."""
46
+ # 1. Arrange
47
+ credential = {"user": "test"}
48
+ config = {"batch_size": 10}
49
+ driver = EchoDriver("test-echo-id", "http://localhost", credential, config)
50
+
51
+ # First batch
52
+ events1 = [
53
+ InsertEvent(event_schema="test_schema", table="files", rows=[{"id": 1}, {"id": 2}], fields=["id"]),
54
+ ]
55
+
56
+ # Second batch
57
+ events2 = [
58
+ InsertEvent(event_schema="test_schema", table="files", rows=[{"id": 3}], fields=["id"])
59
+ ]
60
+
61
+ # Clear any previous log records
62
+ caplog.clear()
63
+
64
+ # 2. Act & 3. Assert - First push
65
+ with caplog.at_level(logging.INFO):
66
+ result1 = await driver.send_events(events1)
67
+
68
+ # Check the logs for first push
69
+ assert "本批次: 2条" in caplog.text
70
+ assert "累计: 2条" in caplog.text
71
+ assert result1 == {"success": True, "snapshot_needed": True}
72
+
73
+ # Clear logs for second push
74
+ caplog.clear()
75
+
76
+ # 2. Act & 3. Assert - Second push
77
+ result2 = await driver.send_events(events2)
78
+
79
+ # Check the logs for second push
80
+ assert "本批次: 1条" in caplog.text
81
+ assert "累计: 3条" in caplog.text # 2 (from first) + 1 (from second)
82
+ assert result2 == {"success": True, "snapshot_needed": False}
@@ -0,0 +1,45 @@
1
+ """
2
+ Integration test to verify that the echo sender can trigger snapshot sync.
3
+ """
4
+ import asyncio
5
+ import pytest
6
+ from fustor_sender_echo import EchoDriver
7
+ from fustor_core.event import UpdateEvent
8
+
9
+ @pytest.mark.asyncio
10
+ async def test_echo_pipe_instance_triggers_snapshot():
11
+ """Integration test to verify echo sender triggers snapshot."""
12
+ # 1. Arrange - Create configurations
13
+ credential = {"user": "echo-user"}
14
+ config = {"batch_size": 100}
15
+
16
+ # Create echo sender and verify it returns snapshot_needed=True
17
+ echo_driver = EchoDriver("echo-sender", "http://localhost", credential, config)
18
+
19
+ # Mock an event to simulate push
20
+ mock_events = [UpdateEvent(event_schema="test", table="files", rows=[{"file_path": "/tmp/test.txt", "size": 100}], fields=["file_path", "size"])]
21
+
22
+ # send_events should return snapshot_needed=True on first call
23
+ result = await echo_driver.send_events(mock_events, source_type="realtime")
24
+
25
+ # Verify that snapshot sync would be triggered
26
+ assert result == {"success": True, "snapshot_needed": True}, "Echo sender should request snapshot on first call"
27
+
28
+
29
+ @pytest.mark.asyncio
30
+ async def test_snapshot_trigger_once_and_only_once():
31
+ """
32
+ Tests that the EchoSender triggers a snapshot on the first push, and not on subsequent pushes.
33
+ """
34
+ credential = {"user": "test"}
35
+ config = {"batch_size": 10}
36
+ driver = EchoDriver("test-driver", "http://localhost", credential, config)
37
+ events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1}], fields=["id"])]
38
+
39
+ # First push should trigger a snapshot
40
+ result1 = await driver.send_events(events, source_type="realtime")
41
+ assert result1 == {"success": True, "snapshot_needed": True}, "Should trigger snapshot on the first push"
42
+
43
+ # Second push should NOT trigger a snapshot
44
+ result2 = await driver.send_events(events, source_type="realtime")
45
+ assert result2 == {"success": True, "snapshot_needed": False}, "Should NOT trigger snapshot on the second push"
@@ -0,0 +1,192 @@
1
+ """
2
+ Test cases specifically for echo sender logging functionality.
3
+ This test verifies the echo sender can write to logs for various event types.
4
+ """
5
+ import pytest
6
+ import logging
7
+ from fustor_sender_echo import EchoDriver
8
+ from fustor_core.models.config import SenderConfig, PasswdCredential
9
+ from fustor_core.event import UpdateEvent, DeleteEvent, InsertEvent
10
+
11
+
12
+ @pytest.mark.asyncio
13
+ async def test_echo_driver_logs_update_events(caplog):
14
+ """Test that echo driver properly logs UpdateEvent data."""
15
+ # 1. Arrange
16
+ credential = {"user": "test"}
17
+ config = {"batch_size": 10}
18
+ driver = EchoDriver("test-update-echo", "http://localhost", credential, config)
19
+
20
+ # Create UpdateEvent similar to what fs would generate
21
+ update_events = [
22
+ UpdateEvent(
23
+ event_schema="/home/test",
24
+ table="files",
25
+ rows=[
26
+ {
27
+ "file_path": "/home/test/file1.txt",
28
+ "size": 1024,
29
+ "modified_time": 1700000000.0,
30
+ "created_time": 1699999000.0,
31
+ "is_dir": False
32
+ },
33
+ {
34
+ "file_path": "/home/test/subdir",
35
+ "size": 4096,
36
+ "modified_time": 1700000100.0,
37
+ "created_time": 1699999100.0,
38
+ "is_dir": True
39
+ }
40
+ ],
41
+ fields=["file_path", "size", "modified_time", "created_time", "is_dir"]
42
+ )
43
+ ]
44
+
45
+ # 2. Act & 3. Assert - Check logging
46
+ with caplog.at_level(logging.INFO):
47
+ result = await driver.send_events(update_events, source_type="realtime")
48
+
49
+ # Verify logging content
50
+ assert "[EchoSender]" in caplog.text
51
+ assert "Task: test-update-echo" in caplog.text
52
+ assert result == {"success": True, "snapshot_needed": True}
53
+
54
+
55
+ @pytest.mark.asyncio
56
+ async def test_echo_driver_logs_delete_events(caplog):
57
+ """Test that echo driver properly logs DeleteEvent data."""
58
+ # 1. Arrange
59
+ credential = {"user": "test"}
60
+ config = {"batch_size": 10}
61
+ driver = EchoDriver("test-delete-echo", "http://localhost", credential, config)
62
+
63
+ # Create DeleteEvent similar to what fs would generate
64
+ delete_events = [
65
+ DeleteEvent(
66
+ event_schema="/home/test",
67
+ table="files",
68
+ rows=[
69
+ {"file_path": "/home/test/old_file.txt"},
70
+ {"file_path": "/home/test/old_dir"}
71
+ ],
72
+ fields=["file_path"]
73
+ )
74
+ ]
75
+
76
+ # 2. Act & 3. Assert - Check logging
77
+ with caplog.at_level(logging.INFO):
78
+ result = await driver.send_events(delete_events, source_type="realtime")
79
+
80
+ # Verify logging content
81
+ assert "[EchoSender]" in caplog.text
82
+ assert "Task: test-delete-echo" in caplog.text
83
+ assert "本批次: 2条" in caplog.text # 2 rows in the delete event
84
+ assert result == {"success": True, "snapshot_needed": True}
85
+
86
+
87
+ @pytest.mark.asyncio
88
+ async def test_echo_driver_logs_with_snapshot_end_flag(caplog):
89
+ """Test that echo driver properly logs with snapshot end flag."""
90
+ # 1. Arrange
91
+ credential = {"user": "test"}
92
+ config = {"batch_size": 10}
93
+ driver = EchoDriver("test-snapshot-echo", "http://localhost", credential, config)
94
+
95
+ events = [
96
+ UpdateEvent(
97
+ event_schema="test_schema",
98
+ table="test_table",
99
+ rows=[{"id": 1, "name": "test"}],
100
+ fields=["id", "name"]
101
+ )
102
+ ]
103
+
104
+ # 2. Act & 3. Assert - Check logging with end flag
105
+ with caplog.at_level(logging.INFO):
106
+ result = await driver.send_events(
107
+ events,
108
+ source_type="realtime",
109
+ is_end=True
110
+ )
111
+
112
+ # Verify logging content includes end flag
113
+ assert "[EchoSender]" in caplog.text
114
+ assert "Task: test-snapshot-echo" in caplog.text
115
+ assert "本批次: 1条" in caplog.text
116
+ assert "累计: 1条" in caplog.text
117
+ assert "Flags: END" in caplog.text
118
+ assert result == {"success": True, "snapshot_needed": True}
119
+
120
+
121
+ @pytest.mark.asyncio
122
+ async def test_echo_driver_logs_with_multiple_flags(caplog):
123
+ """Test that echo driver properly logs with multiple control flags."""
124
+ # 1. Arrange
125
+ credential = {"user": "test"}
126
+ config = {"batch_size": 10}
127
+ driver = EchoDriver("test-flags-echo", "http://localhost", credential, config)
128
+
129
+ events = [
130
+ InsertEvent(
131
+ event_schema="test_schema",
132
+ table="test_table",
133
+ rows=[{"id": 1, "name": "test"}],
134
+ fields=["id", "name"]
135
+ )
136
+ ]
137
+
138
+ # 2. Act & 3. Assert - Check logging with end flag
139
+ with caplog.at_level(logging.INFO):
140
+ result = await driver.send_events(
141
+ events,
142
+ source_type="realtime",
143
+ is_end=True
144
+ )
145
+
146
+ # Verify logging content includes flag
147
+ assert "[EchoSender]" in caplog.text
148
+ assert "Flags: END" in caplog.text
149
+ assert result == {"success": True, "snapshot_needed": True}
150
+
151
+
152
+ @pytest.mark.asyncio
153
+ async def test_echo_driver_logs_first_event_data(caplog):
154
+ """Test that echo driver logs the first event's data in JSON format."""
155
+ # 1. Arrange
156
+ credential = {"user": "test"}
157
+ config = {"batch_size": 10}
158
+ driver = EchoDriver("test-data-echo", "http://localhost", credential, config)
159
+
160
+ # Create an event with structured data
161
+ events = [
162
+ UpdateEvent(
163
+ event_schema="test_schema",
164
+ table="files",
165
+ rows=[
166
+ {
167
+ "file_path": "/home/test/file.txt",
168
+ "size": 2048,
169
+ "modified_time": 1700000000.0,
170
+ "created_time": 1699999000.0,
171
+ "is_dir": False
172
+ },
173
+ {
174
+ "file_path": "/home/test/another.txt",
175
+ "size": 4096,
176
+ "modified_time": 1700000100.0,
177
+ "created_time": 1699999100.0,
178
+ "is_dir": False
179
+ }
180
+ ],
181
+ fields=["file_path", "size", "modified_time", "created_time", "is_dir"]
182
+ )
183
+ ]
184
+
185
+ # 2. Act & 3. Assert - Check that first event data is logged
186
+ with caplog.at_level(logging.INFO):
187
+ await driver.send_events(events, source_type="realtime")
188
+
189
+ # Verify that the first event's data appears in logs (in JSON format)
190
+ assert "/home/test/file.txt" in caplog.text # First event's file path
191
+ assert "2048" in caplog.text # First event's size
192
+ assert "false" in caplog.text # First event's is_dir value (JSON format)
@@ -0,0 +1,131 @@
1
+ """
2
+ Test cases to verify that the echo sender can trigger snapshot sync.
3
+ This test simulates the scenario where the echo sender returns snapshot_needed=True
4
+ for echo tasks, which should trigger the _run_message_sync method in the pipe instance.
5
+ """
6
+ import pytest
7
+ import asyncio
8
+ from unittest.mock import AsyncMock, MagicMock, patch
9
+
10
+ from fustor_sender_echo import EchoDriver
11
+ from fustor_core.models.config import SenderConfig, PasswdCredential
12
+ from fustor_core.event import UpdateEvent
13
+
14
+ @pytest.mark.asyncio
15
+ async def test_echo_sender_requests_snapshot_on_first_push():
16
+ """Test that echo sender requests snapshot on the first push."""
17
+ # 1. Arrange
18
+ credential = {"user": "test"}
19
+ config = {"batch_size": 10}
20
+ driver = EchoDriver("test-echo", "http://localhost", credential, config)
21
+
22
+ # Simulate events
23
+ events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1, "name": "test"}], fields=["id", "name"])]
24
+
25
+ # 2. Act
26
+ result = await driver.send_events(events, source_type="snapshot")
27
+
28
+ # 3. Assert
29
+ assert result == {"success": True, "snapshot_needed": True}
30
+
31
+
32
+ @pytest.mark.asyncio
33
+ async def test_echo_sender_requests_snapshot_on_first_push_for_any_task():
34
+ """Test that echo sender requests snapshot on the first push, regardless of task name."""
35
+ # 1. Arrange
36
+ credential = {"user": "test"}
37
+ config = {"batch_size": 10}
38
+ driver = EchoDriver("test-echo", "http://localhost", credential, config)
39
+
40
+ # Simulate events
41
+ events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1, "name": "test"}], fields=["id", "name"])]
42
+
43
+ # 2. Act
44
+ result = await driver.send_events(events, source_type="realtime")
45
+
46
+ # 3. Assert
47
+ assert result == {"success": True, "snapshot_needed": True}
48
+
49
+
50
+ @pytest.mark.asyncio
51
+ async def test_echo_sender_requests_snapshot_on_first_push_with_missing_task_id():
52
+ """Test that echo sender requests snapshot on the first push even when task_id is not provided."""
53
+ # 1. Arrange
54
+ credential = {"user": "test"}
55
+ config = {"batch_size": 10}
56
+ driver = EchoDriver("test-echo", "http://localhost", credential, config)
57
+
58
+ # Simulate events
59
+ events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1, "name": "test"}], fields=["id", "name"])]
60
+
61
+ # 2. Act
62
+ result = await driver.send_events(events) # Use defaults
63
+
64
+ # 3. Assert
65
+ assert result == {"success": True, "snapshot_needed": True}
66
+
67
+
68
+ @pytest.mark.asyncio
69
+ async def test_echo_sender_logs_properly():
70
+ """Test that echo sender still logs properly while triggering snapshots."""
71
+ import logging
72
+ from io import StringIO
73
+
74
+ # 1. Arrange
75
+ credential = {"user": "test"}
76
+ config = {"batch_size": 10}
77
+ driver = EchoDriver("test-echo", "http://localhost", credential, config)
78
+
79
+ # Capture logs
80
+ log_stream = StringIO()
81
+ handler = logging.StreamHandler(log_stream)
82
+ logger = logging.getLogger(f"fustor_agent.sender.echo.test-echo")
83
+ logger.addHandler(handler)
84
+ logger.setLevel(logging.INFO) # Ensure INFO logs are captured
85
+ events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1, "name": "test"}], fields=["id", "name"])]
86
+
87
+ # 2. Act
88
+ result = await driver.send_events(events, source_type="snapshot", is_end=True)
89
+
90
+ # 3. Assert
91
+ assert result == {"success": True, "snapshot_needed": True}
92
+ log_output = log_stream.getvalue()
93
+ assert "[EchoSender]" in log_output
94
+ assert "[SNAPSHOT]" in log_output
95
+ assert "Task: test-echo" in log_output
96
+ assert "本批次: 1条" in log_output
97
+ assert "累计: 1条" in log_output
98
+ assert "Flags: END" in log_output
99
+ assert "First event data" in log_output
100
+
101
+ # Cleanup
102
+ logger.removeHandler(handler)
103
+
104
+
105
+ @pytest.mark.asyncio
106
+ async def test_echo_sender_maintains_statistics():
107
+ """Test that echo sender maintains cumulative statistics while triggering snapshots."""
108
+ # 1. Arrange
109
+ credential = {"user": "test"}
110
+ config = {"batch_size": 10}
111
+ driver = EchoDriver("test-stats", "http://localhost", credential, config)
112
+
113
+ # First batch of events
114
+ events1 = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1}, {"id": 2}], fields=["id"])]
115
+
116
+ # Second batch of events
117
+ events2 = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 3}], fields=["id"])]
118
+
119
+ # 2. Act
120
+ result1 = await driver.send_events(events1, source_type="realtime")
121
+ result2 = await driver.send_events(events2, source_type="realtime")
122
+
123
+ # 3. Assert
124
+ # First call should return snapshot_needed=True
125
+ assert result1 == {"success": True, "snapshot_needed": True}
126
+
127
+ # Second call should return snapshot_needed=False because the trigger is one-time
128
+ assert result2 == {"success": True, "snapshot_needed": False}
129
+
130
+ # Statistics should be cumulative (2 from first batch + 1 from second batch = 3 total)
131
+ assert driver.total_rows == 3