fustor-sender-echo 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fustor_sender_echo-0.1.0/PKG-INFO +48 -0
- fustor_sender_echo-0.1.0/README.md +39 -0
- fustor_sender_echo-0.1.0/pyproject.toml +28 -0
- fustor_sender_echo-0.1.0/setup.cfg +4 -0
- fustor_sender_echo-0.1.0/src/fustor_sender_echo/__init__.py +131 -0
- fustor_sender_echo-0.1.0/src/fustor_sender_echo.egg-info/PKG-INFO +48 -0
- fustor_sender_echo-0.1.0/src/fustor_sender_echo.egg-info/SOURCES.txt +13 -0
- fustor_sender_echo-0.1.0/src/fustor_sender_echo.egg-info/dependency_links.txt +1 -0
- fustor_sender_echo-0.1.0/src/fustor_sender_echo.egg-info/entry_points.txt +2 -0
- fustor_sender_echo-0.1.0/src/fustor_sender_echo.egg-info/requires.txt +1 -0
- fustor_sender_echo-0.1.0/src/fustor_sender_echo.egg-info/top_level.txt +1 -0
- fustor_sender_echo-0.1.0/tests/test_echo_driver.py +82 -0
- fustor_sender_echo-0.1.0/tests/test_echo_integration.py +45 -0
- fustor_sender_echo-0.1.0/tests/test_echo_logging.py +192 -0
- fustor_sender_echo-0.1.0/tests/test_echo_snapshot_trigger.py +131 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fustor-sender-echo
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Echo sender driver for Fustor
|
|
5
|
+
Author: Fustor Team
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: fustor-core
|
|
9
|
+
|
|
10
|
+
# fustor-sender-echo
|
|
11
|
+
|
|
12
|
+
This package provides an "echo" sender driver for the Fustor Agent service. It serves as a basic example and debugging tool for `SenderDriver` implementations. Instead of sending data to an external system, it simply logs all received events and control flags to the Fustor Agent's log output.
|
|
13
|
+
|
|
14
|
+
## Features
|
|
15
|
+
|
|
16
|
+
* **Echo Functionality**: Logs all incoming events, including `realtime` and `snapshot` data, to the console/log.
|
|
17
|
+
* **Control Flag Visibility**: Displays control flags like `is_snapshot_end` and `snapshot_sync_suggested` for debugging data flow.
|
|
18
|
+
* **Session Management**: Implements `create_session` and `heartbeat` to demonstrate session lifecycle.
|
|
19
|
+
* **No Configuration Needed**: The `get_needed_fields` method returns an empty schema, indicating it accepts all fields without specific requirements.
|
|
20
|
+
* **Wizard Definition**: Provides a simple wizard step for UI integration.
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
|
|
24
|
+
This package is part of the Fustor monorepo and is typically installed in editable mode within the monorepo's development environment using `uv sync`. It is registered as a `fustor_agent.drivers.senders` entry point.
|
|
25
|
+
|
|
26
|
+
## Usage
|
|
27
|
+
|
|
28
|
+
To use the `fustor-sender-echo` driver, configure a Sender in your Fustor Agent setup with the driver type `echo`. When a pipe involves this sender, all data processed by the Agent will be logged by this driver.
|
|
29
|
+
|
|
30
|
+
This driver is particularly useful for:
|
|
31
|
+
* **Debugging**: Understanding the exact data and control signals being sent by the Fustor Agent.
|
|
32
|
+
* **Development**: As a template for creating new `SenderDriver` implementations.
|
|
33
|
+
* **Testing**: Verifying that the Fustor Agent's data pipe is correctly delivering events.
|
|
34
|
+
|
|
35
|
+
Example (conceptual configuration in Fustor Agent):
|
|
36
|
+
|
|
37
|
+
```yaml
|
|
38
|
+
# Fustor 主目录下的 agent-config.yaml
|
|
39
|
+
senders:
|
|
40
|
+
my-echo-sender:
|
|
41
|
+
driver_type: echo
|
|
42
|
+
# No specific configuration parameters needed for the echo driver
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Dependencies
|
|
46
|
+
|
|
47
|
+
* `fustor-core`: Provides the `SenderDriver` abstract base class and other core components.
|
|
48
|
+
* `fustor-event-model`: Provides `EventBase` for event data structures.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# fustor-sender-echo
|
|
2
|
+
|
|
3
|
+
This package provides an "echo" sender driver for the Fustor Agent service. It serves as a basic example and debugging tool for `SenderDriver` implementations. Instead of sending data to an external system, it simply logs all received events and control flags to the Fustor Agent's log output.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
* **Echo Functionality**: Logs all incoming events, including `realtime` and `snapshot` data, to the console/log.
|
|
8
|
+
* **Control Flag Visibility**: Displays control flags like `is_snapshot_end` and `snapshot_sync_suggested` for debugging data flow.
|
|
9
|
+
* **Session Management**: Implements `create_session` and `heartbeat` to demonstrate session lifecycle.
|
|
10
|
+
* **No Configuration Needed**: The `get_needed_fields` method returns an empty schema, indicating it accepts all fields without specific requirements.
|
|
11
|
+
* **Wizard Definition**: Provides a simple wizard step for UI integration.
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
This package is part of the Fustor monorepo and is typically installed in editable mode within the monorepo's development environment using `uv sync`. It is registered as a `fustor_agent.drivers.senders` entry point.
|
|
16
|
+
|
|
17
|
+
## Usage
|
|
18
|
+
|
|
19
|
+
To use the `fustor-sender-echo` driver, configure a Sender in your Fustor Agent setup with the driver type `echo`. When a pipe involves this sender, all data processed by the Agent will be logged by this driver.
|
|
20
|
+
|
|
21
|
+
This driver is particularly useful for:
|
|
22
|
+
* **Debugging**: Understanding the exact data and control signals being sent by the Fustor Agent.
|
|
23
|
+
* **Development**: As a template for creating new `SenderDriver` implementations.
|
|
24
|
+
* **Testing**: Verifying that the Fustor Agent's data pipe is correctly delivering events.
|
|
25
|
+
|
|
26
|
+
Example (conceptual configuration in Fustor Agent):
|
|
27
|
+
|
|
28
|
+
```yaml
|
|
29
|
+
# Fustor 主目录下的 agent-config.yaml
|
|
30
|
+
senders:
|
|
31
|
+
my-echo-sender:
|
|
32
|
+
driver_type: echo
|
|
33
|
+
# No specific configuration parameters needed for the echo driver
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Dependencies
|
|
37
|
+
|
|
38
|
+
* `fustor-core`: Provides the `SenderDriver` abstract base class and other core components.
|
|
39
|
+
* `fustor-event-model`: Provides `EventBase` for event data structures.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "fustor-sender-echo"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Echo sender driver for Fustor"
|
|
5
|
+
requires-python = ">= 3.10"
|
|
6
|
+
dependencies = [
|
|
7
|
+
"fustor-core",
|
|
8
|
+
]
|
|
9
|
+
authors = [
|
|
10
|
+
{ name = "Fustor Team" }
|
|
11
|
+
]
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
|
|
14
|
+
[build-system]
|
|
15
|
+
requires = ["setuptools>=61.0"]
|
|
16
|
+
build-backend = "setuptools.build_meta"
|
|
17
|
+
|
|
18
|
+
["project.urls"]
|
|
19
|
+
Homepage = "https://github.com/excelwang/fustor/tree/master/extensions/sender-echo"
|
|
20
|
+
"Bug Tracker" = "https://github.com/excelwang/fustor/issues"
|
|
21
|
+
|
|
22
|
+
license = "MIT"
|
|
23
|
+
|
|
24
|
+
[project.entry-points."fustor_agent.drivers.senders"]
|
|
25
|
+
echo = "fustor_sender_echo:EchoDriver"
|
|
26
|
+
|
|
27
|
+
[tool.setuptools.packages.find]
|
|
28
|
+
where = [ "src",]
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Fustor Echo Sender Driver (Class-based)
|
|
3
|
+
"""
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
from typing import Any, Dict, List, Optional
|
|
7
|
+
|
|
8
|
+
from fustor_core.transport import Sender
|
|
9
|
+
from fustor_core.models.config import SenderConfig
|
|
10
|
+
from fustor_core.event import EventBase
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class EchoDriver(Sender):
|
|
14
|
+
"""
|
|
15
|
+
An echo driver that inherits from the Sender ABC.
|
|
16
|
+
It prints batch and cumulative statistics for all received events.
|
|
17
|
+
"""
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
sender_id: str,
|
|
21
|
+
endpoint: str,
|
|
22
|
+
credential: Dict[str, Any],
|
|
23
|
+
config: Optional[Dict[str, Any]] = None
|
|
24
|
+
):
|
|
25
|
+
"""Initializes the driver and its statistics counters."""
|
|
26
|
+
super().__init__(sender_id, endpoint, credential, config)
|
|
27
|
+
self.total_rows = 0
|
|
28
|
+
self.total_size = 0
|
|
29
|
+
self.logger = logging.getLogger(f"fustor_agent.sender.echo.{sender_id}")
|
|
30
|
+
self._snapshot_triggered = False
|
|
31
|
+
|
|
32
|
+
async def connect(self) -> None:
|
|
33
|
+
"""Echo driver connection is a no-op."""
|
|
34
|
+
self.logger.info(f"Echo Sender {self.id} ready.")
|
|
35
|
+
|
|
36
|
+
async def _send_events_impl(
|
|
37
|
+
self, events: List[Any], source_type: str = "realtime", is_end: bool = False
|
|
38
|
+
) -> Dict:
|
|
39
|
+
"""
|
|
40
|
+
Implementation of echo sending.
|
|
41
|
+
"""
|
|
42
|
+
batch_rows = 0
|
|
43
|
+
for event in events:
|
|
44
|
+
# Handle both EventBase objects and raw dicts
|
|
45
|
+
if hasattr(event, "rows"):
|
|
46
|
+
batch_rows += len(event.rows)
|
|
47
|
+
elif isinstance(event, dict):
|
|
48
|
+
batch_rows += len(event.get("rows", []))
|
|
49
|
+
|
|
50
|
+
self.total_rows += batch_rows
|
|
51
|
+
|
|
52
|
+
# Prepare a summary for logging
|
|
53
|
+
flags = []
|
|
54
|
+
if is_end:
|
|
55
|
+
flags.append("END")
|
|
56
|
+
|
|
57
|
+
flags_str = f" | Flags: {', '.join(flags)}" if flags else ""
|
|
58
|
+
|
|
59
|
+
self.logger.info(
|
|
60
|
+
f"[EchoSender] [{source_type.upper()}] Task: {self.id} | 本批次: {batch_rows}条; 累计: {self.total_rows}条{flags_str}"
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# For debugging, log the first event's data if available
|
|
64
|
+
if events:
|
|
65
|
+
first_event_rows = None
|
|
66
|
+
if hasattr(events[0], "rows"):
|
|
67
|
+
first_event_rows = events[0].rows
|
|
68
|
+
elif isinstance(events[0], dict):
|
|
69
|
+
first_event_rows = events[0].get("rows", [])
|
|
70
|
+
|
|
71
|
+
if first_event_rows:
|
|
72
|
+
self.logger.info(f"First event data: {json.dumps(first_event_rows[0], ensure_ascii=False)}")
|
|
73
|
+
|
|
74
|
+
# Trigger snapshot only once if the condition is met
|
|
75
|
+
snapshot_needed = False
|
|
76
|
+
if not self._snapshot_triggered:
|
|
77
|
+
snapshot_needed = True
|
|
78
|
+
self._snapshot_triggered = True
|
|
79
|
+
self.logger.info(f"Task '{self.id}' is triggering a one-time snapshot.")
|
|
80
|
+
|
|
81
|
+
return {"success": True, "snapshot_needed": snapshot_needed}
|
|
82
|
+
|
|
83
|
+
async def heartbeat(self, **kwargs) -> Dict[str, Any]:
|
|
84
|
+
"""
|
|
85
|
+
Sends a heartbeat to maintain session state.
|
|
86
|
+
"""
|
|
87
|
+
self.logger.info(f"[EchoSender] Heartbeat for session {self.session_id} from task {self.id}")
|
|
88
|
+
return {"status": "ok", "role": self.config.get("mock_role", "leader")}
|
|
89
|
+
|
|
90
|
+
async def create_session(
|
|
91
|
+
self,
|
|
92
|
+
task_id: str,
|
|
93
|
+
source_type: Optional[str] = None,
|
|
94
|
+
session_timeout_seconds: Optional[int] = None,
|
|
95
|
+
**kwargs
|
|
96
|
+
) -> Dict[str, Any]:
|
|
97
|
+
"""
|
|
98
|
+
Creates a new session.
|
|
99
|
+
"""
|
|
100
|
+
import uuid
|
|
101
|
+
session_id = str(uuid.uuid4())
|
|
102
|
+
self.session_id = session_id
|
|
103
|
+
role = self.config.get("mock_role", "leader")
|
|
104
|
+
timeout = session_timeout_seconds or self.config.get("session_timeout_seconds", 30)
|
|
105
|
+
self.logger.info(f"[EchoSender] Created session {session_id} for task {task_id} with role {role}")
|
|
106
|
+
metadata = {
|
|
107
|
+
"session_id": session_id,
|
|
108
|
+
"role": role,
|
|
109
|
+
"session_timeout_seconds": timeout
|
|
110
|
+
}
|
|
111
|
+
return session_id, metadata
|
|
112
|
+
|
|
113
|
+
async def close_session(self) -> None:
|
|
114
|
+
"""Close the current session."""
|
|
115
|
+
if self.session_id:
|
|
116
|
+
self.logger.info(f"[EchoSender] Closing session {self.session_id}")
|
|
117
|
+
self.session_id = None
|
|
118
|
+
|
|
119
|
+
async def close(self) -> None:
|
|
120
|
+
"""Close the sender."""
|
|
121
|
+
self.logger.info(f"[EchoSender] Sender {self.id} closed")
|
|
122
|
+
self._snapshot_triggered = False
|
|
123
|
+
|
|
124
|
+
@classmethod
|
|
125
|
+
async def get_needed_fields(cls, **kwargs) -> Dict[str, Any]:
|
|
126
|
+
"""
|
|
127
|
+
The echo driver does not need any specific fields, so it returns an empty schema,
|
|
128
|
+
signaling that it accepts all fields.
|
|
129
|
+
"""
|
|
130
|
+
return {}
|
|
131
|
+
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fustor-sender-echo
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Echo sender driver for Fustor
|
|
5
|
+
Author: Fustor Team
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: fustor-core
|
|
9
|
+
|
|
10
|
+
# fustor-sender-echo
|
|
11
|
+
|
|
12
|
+
This package provides an "echo" sender driver for the Fustor Agent service. It serves as a basic example and debugging tool for `SenderDriver` implementations. Instead of sending data to an external system, it simply logs all received events and control flags to the Fustor Agent's log output.
|
|
13
|
+
|
|
14
|
+
## Features
|
|
15
|
+
|
|
16
|
+
* **Echo Functionality**: Logs all incoming events, including `realtime` and `snapshot` data, to the console/log.
|
|
17
|
+
* **Control Flag Visibility**: Displays control flags like `is_snapshot_end` and `snapshot_sync_suggested` for debugging data flow.
|
|
18
|
+
* **Session Management**: Implements `create_session` and `heartbeat` to demonstrate session lifecycle.
|
|
19
|
+
* **No Configuration Needed**: The `get_needed_fields` method returns an empty schema, indicating it accepts all fields without specific requirements.
|
|
20
|
+
* **Wizard Definition**: Provides a simple wizard step for UI integration.
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
|
|
24
|
+
This package is part of the Fustor monorepo and is typically installed in editable mode within the monorepo's development environment using `uv sync`. It is registered as a `fustor_agent.drivers.senders` entry point.
|
|
25
|
+
|
|
26
|
+
## Usage
|
|
27
|
+
|
|
28
|
+
To use the `fustor-sender-echo` driver, configure a Sender in your Fustor Agent setup with the driver type `echo`. When a pipe involves this sender, all data processed by the Agent will be logged by this driver.
|
|
29
|
+
|
|
30
|
+
This driver is particularly useful for:
|
|
31
|
+
* **Debugging**: Understanding the exact data and control signals being sent by the Fustor Agent.
|
|
32
|
+
* **Development**: As a template for creating new `SenderDriver` implementations.
|
|
33
|
+
* **Testing**: Verifying that the Fustor Agent's data pipe is correctly delivering events.
|
|
34
|
+
|
|
35
|
+
Example (conceptual configuration in Fustor Agent):
|
|
36
|
+
|
|
37
|
+
```yaml
|
|
38
|
+
# Fustor 主目录下的 agent-config.yaml
|
|
39
|
+
senders:
|
|
40
|
+
my-echo-sender:
|
|
41
|
+
driver_type: echo
|
|
42
|
+
# No specific configuration parameters needed for the echo driver
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Dependencies
|
|
46
|
+
|
|
47
|
+
* `fustor-core`: Provides the `SenderDriver` abstract base class and other core components.
|
|
48
|
+
* `fustor-event-model`: Provides `EventBase` for event data structures.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
src/fustor_sender_echo/__init__.py
|
|
4
|
+
src/fustor_sender_echo.egg-info/PKG-INFO
|
|
5
|
+
src/fustor_sender_echo.egg-info/SOURCES.txt
|
|
6
|
+
src/fustor_sender_echo.egg-info/dependency_links.txt
|
|
7
|
+
src/fustor_sender_echo.egg-info/entry_points.txt
|
|
8
|
+
src/fustor_sender_echo.egg-info/requires.txt
|
|
9
|
+
src/fustor_sender_echo.egg-info/top_level.txt
|
|
10
|
+
tests/test_echo_driver.py
|
|
11
|
+
tests/test_echo_integration.py
|
|
12
|
+
tests/test_echo_logging.py
|
|
13
|
+
tests/test_echo_snapshot_trigger.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fustor-core
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fustor_sender_echo
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
import json
|
|
3
|
+
import logging
|
|
4
|
+
from io import StringIO
|
|
5
|
+
from fustor_sender_echo import EchoDriver
|
|
6
|
+
from fustor_core.models.config import SenderConfig, PasswdCredential
|
|
7
|
+
from fustor_core.event import InsertEvent
|
|
8
|
+
|
|
9
|
+
@pytest.mark.asyncio
|
|
10
|
+
async def test_echo_driver_push(caplog):
|
|
11
|
+
"""Tests the send_events method of the EchoDriver conforms to the new interface."""
|
|
12
|
+
# 1. Arrange
|
|
13
|
+
credential = {"user": "test"}
|
|
14
|
+
config = {"batch_size": 10}
|
|
15
|
+
driver = EchoDriver("test-echo-id", "http://localhost", credential, config)
|
|
16
|
+
events = [InsertEvent(event_schema="test_schema", table="test_table", rows=[{"id": 1, "msg": "hello"}], fields=["id", "msg"])]
|
|
17
|
+
|
|
18
|
+
# 2. Act & 3. Assert - Check logging
|
|
19
|
+
with caplog.at_level(logging.INFO):
|
|
20
|
+
result = await driver.send_events(events, source_type="realtime", is_end=True)
|
|
21
|
+
|
|
22
|
+
# Check for the summary output in logs
|
|
23
|
+
assert "[EchoSender]" in caplog.text
|
|
24
|
+
assert "[REALTIME]" in caplog.text
|
|
25
|
+
assert "Task: test-echo-id" in caplog.text
|
|
26
|
+
assert "本批次: 1条" in caplog.text
|
|
27
|
+
assert "累计: 1条" in caplog.text
|
|
28
|
+
assert "Flags: END" in caplog.text
|
|
29
|
+
|
|
30
|
+
# Check the result dictionary
|
|
31
|
+
assert result == {"success": True, "snapshot_needed": True}
|
|
32
|
+
|
|
33
|
+
@pytest.mark.asyncio
|
|
34
|
+
async def test_echo_driver_get_needed_fields():
|
|
35
|
+
"""Tests the get_needed_fields class method."""
|
|
36
|
+
# 1. Arrange & 2. Act
|
|
37
|
+
fields = await EchoDriver.get_needed_fields()
|
|
38
|
+
|
|
39
|
+
# 3. Assert
|
|
40
|
+
assert fields == {}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@pytest.mark.asyncio
|
|
44
|
+
async def test_echo_driver_cumulative_push(caplog):
|
|
45
|
+
"""Tests that the driver correctly accumulates row counts over multiple pushes."""
|
|
46
|
+
# 1. Arrange
|
|
47
|
+
credential = {"user": "test"}
|
|
48
|
+
config = {"batch_size": 10}
|
|
49
|
+
driver = EchoDriver("test-echo-id", "http://localhost", credential, config)
|
|
50
|
+
|
|
51
|
+
# First batch
|
|
52
|
+
events1 = [
|
|
53
|
+
InsertEvent(event_schema="test_schema", table="files", rows=[{"id": 1}, {"id": 2}], fields=["id"]),
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
# Second batch
|
|
57
|
+
events2 = [
|
|
58
|
+
InsertEvent(event_schema="test_schema", table="files", rows=[{"id": 3}], fields=["id"])
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
# Clear any previous log records
|
|
62
|
+
caplog.clear()
|
|
63
|
+
|
|
64
|
+
# 2. Act & 3. Assert - First push
|
|
65
|
+
with caplog.at_level(logging.INFO):
|
|
66
|
+
result1 = await driver.send_events(events1)
|
|
67
|
+
|
|
68
|
+
# Check the logs for first push
|
|
69
|
+
assert "本批次: 2条" in caplog.text
|
|
70
|
+
assert "累计: 2条" in caplog.text
|
|
71
|
+
assert result1 == {"success": True, "snapshot_needed": True}
|
|
72
|
+
|
|
73
|
+
# Clear logs for second push
|
|
74
|
+
caplog.clear()
|
|
75
|
+
|
|
76
|
+
# 2. Act & 3. Assert - Second push
|
|
77
|
+
result2 = await driver.send_events(events2)
|
|
78
|
+
|
|
79
|
+
# Check the logs for second push
|
|
80
|
+
assert "本批次: 1条" in caplog.text
|
|
81
|
+
assert "累计: 3条" in caplog.text # 2 (from first) + 1 (from second)
|
|
82
|
+
assert result2 == {"success": True, "snapshot_needed": False}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Integration test to verify that the echo sender can trigger snapshot sync.
|
|
3
|
+
"""
|
|
4
|
+
import asyncio
|
|
5
|
+
import pytest
|
|
6
|
+
from fustor_sender_echo import EchoDriver
|
|
7
|
+
from fustor_core.event import UpdateEvent
|
|
8
|
+
|
|
9
|
+
@pytest.mark.asyncio
|
|
10
|
+
async def test_echo_pipe_instance_triggers_snapshot():
|
|
11
|
+
"""Integration test to verify echo sender triggers snapshot."""
|
|
12
|
+
# 1. Arrange - Create configurations
|
|
13
|
+
credential = {"user": "echo-user"}
|
|
14
|
+
config = {"batch_size": 100}
|
|
15
|
+
|
|
16
|
+
# Create echo sender and verify it returns snapshot_needed=True
|
|
17
|
+
echo_driver = EchoDriver("echo-sender", "http://localhost", credential, config)
|
|
18
|
+
|
|
19
|
+
# Mock an event to simulate push
|
|
20
|
+
mock_events = [UpdateEvent(event_schema="test", table="files", rows=[{"file_path": "/tmp/test.txt", "size": 100}], fields=["file_path", "size"])]
|
|
21
|
+
|
|
22
|
+
# send_events should return snapshot_needed=True on first call
|
|
23
|
+
result = await echo_driver.send_events(mock_events, source_type="realtime")
|
|
24
|
+
|
|
25
|
+
# Verify that snapshot sync would be triggered
|
|
26
|
+
assert result == {"success": True, "snapshot_needed": True}, "Echo sender should request snapshot on first call"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@pytest.mark.asyncio
|
|
30
|
+
async def test_snapshot_trigger_once_and_only_once():
|
|
31
|
+
"""
|
|
32
|
+
Tests that the EchoSender triggers a snapshot on the first push, and not on subsequent pushes.
|
|
33
|
+
"""
|
|
34
|
+
credential = {"user": "test"}
|
|
35
|
+
config = {"batch_size": 10}
|
|
36
|
+
driver = EchoDriver("test-driver", "http://localhost", credential, config)
|
|
37
|
+
events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1}], fields=["id"])]
|
|
38
|
+
|
|
39
|
+
# First push should trigger a snapshot
|
|
40
|
+
result1 = await driver.send_events(events, source_type="realtime")
|
|
41
|
+
assert result1 == {"success": True, "snapshot_needed": True}, "Should trigger snapshot on the first push"
|
|
42
|
+
|
|
43
|
+
# Second push should NOT trigger a snapshot
|
|
44
|
+
result2 = await driver.send_events(events, source_type="realtime")
|
|
45
|
+
assert result2 == {"success": True, "snapshot_needed": False}, "Should NOT trigger snapshot on the second push"
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Test cases specifically for echo sender logging functionality.
|
|
3
|
+
This test verifies the echo sender can write to logs for various event types.
|
|
4
|
+
"""
|
|
5
|
+
import pytest
|
|
6
|
+
import logging
|
|
7
|
+
from fustor_sender_echo import EchoDriver
|
|
8
|
+
from fustor_core.models.config import SenderConfig, PasswdCredential
|
|
9
|
+
from fustor_core.event import UpdateEvent, DeleteEvent, InsertEvent
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@pytest.mark.asyncio
|
|
13
|
+
async def test_echo_driver_logs_update_events(caplog):
|
|
14
|
+
"""Test that echo driver properly logs UpdateEvent data."""
|
|
15
|
+
# 1. Arrange
|
|
16
|
+
credential = {"user": "test"}
|
|
17
|
+
config = {"batch_size": 10}
|
|
18
|
+
driver = EchoDriver("test-update-echo", "http://localhost", credential, config)
|
|
19
|
+
|
|
20
|
+
# Create UpdateEvent similar to what fs would generate
|
|
21
|
+
update_events = [
|
|
22
|
+
UpdateEvent(
|
|
23
|
+
event_schema="/home/test",
|
|
24
|
+
table="files",
|
|
25
|
+
rows=[
|
|
26
|
+
{
|
|
27
|
+
"file_path": "/home/test/file1.txt",
|
|
28
|
+
"size": 1024,
|
|
29
|
+
"modified_time": 1700000000.0,
|
|
30
|
+
"created_time": 1699999000.0,
|
|
31
|
+
"is_dir": False
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"file_path": "/home/test/subdir",
|
|
35
|
+
"size": 4096,
|
|
36
|
+
"modified_time": 1700000100.0,
|
|
37
|
+
"created_time": 1699999100.0,
|
|
38
|
+
"is_dir": True
|
|
39
|
+
}
|
|
40
|
+
],
|
|
41
|
+
fields=["file_path", "size", "modified_time", "created_time", "is_dir"]
|
|
42
|
+
)
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
# 2. Act & 3. Assert - Check logging
|
|
46
|
+
with caplog.at_level(logging.INFO):
|
|
47
|
+
result = await driver.send_events(update_events, source_type="realtime")
|
|
48
|
+
|
|
49
|
+
# Verify logging content
|
|
50
|
+
assert "[EchoSender]" in caplog.text
|
|
51
|
+
assert "Task: test-update-echo" in caplog.text
|
|
52
|
+
assert result == {"success": True, "snapshot_needed": True}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@pytest.mark.asyncio
|
|
56
|
+
async def test_echo_driver_logs_delete_events(caplog):
|
|
57
|
+
"""Test that echo driver properly logs DeleteEvent data."""
|
|
58
|
+
# 1. Arrange
|
|
59
|
+
credential = {"user": "test"}
|
|
60
|
+
config = {"batch_size": 10}
|
|
61
|
+
driver = EchoDriver("test-delete-echo", "http://localhost", credential, config)
|
|
62
|
+
|
|
63
|
+
# Create DeleteEvent similar to what fs would generate
|
|
64
|
+
delete_events = [
|
|
65
|
+
DeleteEvent(
|
|
66
|
+
event_schema="/home/test",
|
|
67
|
+
table="files",
|
|
68
|
+
rows=[
|
|
69
|
+
{"file_path": "/home/test/old_file.txt"},
|
|
70
|
+
{"file_path": "/home/test/old_dir"}
|
|
71
|
+
],
|
|
72
|
+
fields=["file_path"]
|
|
73
|
+
)
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
# 2. Act & 3. Assert - Check logging
|
|
77
|
+
with caplog.at_level(logging.INFO):
|
|
78
|
+
result = await driver.send_events(delete_events, source_type="realtime")
|
|
79
|
+
|
|
80
|
+
# Verify logging content
|
|
81
|
+
assert "[EchoSender]" in caplog.text
|
|
82
|
+
assert "Task: test-delete-echo" in caplog.text
|
|
83
|
+
assert "本批次: 2条" in caplog.text # 2 rows in the delete event
|
|
84
|
+
assert result == {"success": True, "snapshot_needed": True}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@pytest.mark.asyncio
|
|
88
|
+
async def test_echo_driver_logs_with_snapshot_end_flag(caplog):
|
|
89
|
+
"""Test that echo driver properly logs with snapshot end flag."""
|
|
90
|
+
# 1. Arrange
|
|
91
|
+
credential = {"user": "test"}
|
|
92
|
+
config = {"batch_size": 10}
|
|
93
|
+
driver = EchoDriver("test-snapshot-echo", "http://localhost", credential, config)
|
|
94
|
+
|
|
95
|
+
events = [
|
|
96
|
+
UpdateEvent(
|
|
97
|
+
event_schema="test_schema",
|
|
98
|
+
table="test_table",
|
|
99
|
+
rows=[{"id": 1, "name": "test"}],
|
|
100
|
+
fields=["id", "name"]
|
|
101
|
+
)
|
|
102
|
+
]
|
|
103
|
+
|
|
104
|
+
# 2. Act & 3. Assert - Check logging with end flag
|
|
105
|
+
with caplog.at_level(logging.INFO):
|
|
106
|
+
result = await driver.send_events(
|
|
107
|
+
events,
|
|
108
|
+
source_type="realtime",
|
|
109
|
+
is_end=True
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
# Verify logging content includes end flag
|
|
113
|
+
assert "[EchoSender]" in caplog.text
|
|
114
|
+
assert "Task: test-snapshot-echo" in caplog.text
|
|
115
|
+
assert "本批次: 1条" in caplog.text
|
|
116
|
+
assert "累计: 1条" in caplog.text
|
|
117
|
+
assert "Flags: END" in caplog.text
|
|
118
|
+
assert result == {"success": True, "snapshot_needed": True}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@pytest.mark.asyncio
|
|
122
|
+
async def test_echo_driver_logs_with_multiple_flags(caplog):
|
|
123
|
+
"""Test that echo driver properly logs with multiple control flags."""
|
|
124
|
+
# 1. Arrange
|
|
125
|
+
credential = {"user": "test"}
|
|
126
|
+
config = {"batch_size": 10}
|
|
127
|
+
driver = EchoDriver("test-flags-echo", "http://localhost", credential, config)
|
|
128
|
+
|
|
129
|
+
events = [
|
|
130
|
+
InsertEvent(
|
|
131
|
+
event_schema="test_schema",
|
|
132
|
+
table="test_table",
|
|
133
|
+
rows=[{"id": 1, "name": "test"}],
|
|
134
|
+
fields=["id", "name"]
|
|
135
|
+
)
|
|
136
|
+
]
|
|
137
|
+
|
|
138
|
+
# 2. Act & 3. Assert - Check logging with end flag
|
|
139
|
+
with caplog.at_level(logging.INFO):
|
|
140
|
+
result = await driver.send_events(
|
|
141
|
+
events,
|
|
142
|
+
source_type="realtime",
|
|
143
|
+
is_end=True
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
# Verify logging content includes flag
|
|
147
|
+
assert "[EchoSender]" in caplog.text
|
|
148
|
+
assert "Flags: END" in caplog.text
|
|
149
|
+
assert result == {"success": True, "snapshot_needed": True}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@pytest.mark.asyncio
|
|
153
|
+
async def test_echo_driver_logs_first_event_data(caplog):
|
|
154
|
+
"""Test that echo driver logs the first event's data in JSON format."""
|
|
155
|
+
# 1. Arrange
|
|
156
|
+
credential = {"user": "test"}
|
|
157
|
+
config = {"batch_size": 10}
|
|
158
|
+
driver = EchoDriver("test-data-echo", "http://localhost", credential, config)
|
|
159
|
+
|
|
160
|
+
# Create an event with structured data
|
|
161
|
+
events = [
|
|
162
|
+
UpdateEvent(
|
|
163
|
+
event_schema="test_schema",
|
|
164
|
+
table="files",
|
|
165
|
+
rows=[
|
|
166
|
+
{
|
|
167
|
+
"file_path": "/home/test/file.txt",
|
|
168
|
+
"size": 2048,
|
|
169
|
+
"modified_time": 1700000000.0,
|
|
170
|
+
"created_time": 1699999000.0,
|
|
171
|
+
"is_dir": False
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
"file_path": "/home/test/another.txt",
|
|
175
|
+
"size": 4096,
|
|
176
|
+
"modified_time": 1700000100.0,
|
|
177
|
+
"created_time": 1699999100.0,
|
|
178
|
+
"is_dir": False
|
|
179
|
+
}
|
|
180
|
+
],
|
|
181
|
+
fields=["file_path", "size", "modified_time", "created_time", "is_dir"]
|
|
182
|
+
)
|
|
183
|
+
]
|
|
184
|
+
|
|
185
|
+
# 2. Act & 3. Assert - Check that first event data is logged
|
|
186
|
+
with caplog.at_level(logging.INFO):
|
|
187
|
+
await driver.send_events(events, source_type="realtime")
|
|
188
|
+
|
|
189
|
+
# Verify that the first event's data appears in logs (in JSON format)
|
|
190
|
+
assert "/home/test/file.txt" in caplog.text # First event's file path
|
|
191
|
+
assert "2048" in caplog.text # First event's size
|
|
192
|
+
assert "false" in caplog.text # First event's is_dir value (JSON format)
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Test cases to verify that the echo sender can trigger snapshot sync.
|
|
3
|
+
This test simulates the scenario where the echo sender returns snapshot_needed=True
|
|
4
|
+
for echo tasks, which should trigger the _run_message_sync method in the pipe instance.
|
|
5
|
+
"""
|
|
6
|
+
import pytest
|
|
7
|
+
import asyncio
|
|
8
|
+
from unittest.mock import AsyncMock, MagicMock, patch
|
|
9
|
+
|
|
10
|
+
from fustor_sender_echo import EchoDriver
|
|
11
|
+
from fustor_core.models.config import SenderConfig, PasswdCredential
|
|
12
|
+
from fustor_core.event import UpdateEvent
|
|
13
|
+
|
|
14
|
+
@pytest.mark.asyncio
|
|
15
|
+
async def test_echo_sender_requests_snapshot_on_first_push():
|
|
16
|
+
"""Test that echo sender requests snapshot on the first push."""
|
|
17
|
+
# 1. Arrange
|
|
18
|
+
credential = {"user": "test"}
|
|
19
|
+
config = {"batch_size": 10}
|
|
20
|
+
driver = EchoDriver("test-echo", "http://localhost", credential, config)
|
|
21
|
+
|
|
22
|
+
# Simulate events
|
|
23
|
+
events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1, "name": "test"}], fields=["id", "name"])]
|
|
24
|
+
|
|
25
|
+
# 2. Act
|
|
26
|
+
result = await driver.send_events(events, source_type="snapshot")
|
|
27
|
+
|
|
28
|
+
# 3. Assert
|
|
29
|
+
assert result == {"success": True, "snapshot_needed": True}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@pytest.mark.asyncio
|
|
33
|
+
async def test_echo_sender_requests_snapshot_on_first_push_for_any_task():
|
|
34
|
+
"""Test that echo sender requests snapshot on the first push, regardless of task name."""
|
|
35
|
+
# 1. Arrange
|
|
36
|
+
credential = {"user": "test"}
|
|
37
|
+
config = {"batch_size": 10}
|
|
38
|
+
driver = EchoDriver("test-echo", "http://localhost", credential, config)
|
|
39
|
+
|
|
40
|
+
# Simulate events
|
|
41
|
+
events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1, "name": "test"}], fields=["id", "name"])]
|
|
42
|
+
|
|
43
|
+
# 2. Act
|
|
44
|
+
result = await driver.send_events(events, source_type="realtime")
|
|
45
|
+
|
|
46
|
+
# 3. Assert
|
|
47
|
+
assert result == {"success": True, "snapshot_needed": True}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@pytest.mark.asyncio
|
|
51
|
+
async def test_echo_sender_requests_snapshot_on_first_push_with_missing_task_id():
|
|
52
|
+
"""Test that echo sender requests snapshot on the first push even when task_id is not provided."""
|
|
53
|
+
# 1. Arrange
|
|
54
|
+
credential = {"user": "test"}
|
|
55
|
+
config = {"batch_size": 10}
|
|
56
|
+
driver = EchoDriver("test-echo", "http://localhost", credential, config)
|
|
57
|
+
|
|
58
|
+
# Simulate events
|
|
59
|
+
events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1, "name": "test"}], fields=["id", "name"])]
|
|
60
|
+
|
|
61
|
+
# 2. Act
|
|
62
|
+
result = await driver.send_events(events) # Use defaults
|
|
63
|
+
|
|
64
|
+
# 3. Assert
|
|
65
|
+
assert result == {"success": True, "snapshot_needed": True}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@pytest.mark.asyncio
|
|
69
|
+
async def test_echo_sender_logs_properly():
|
|
70
|
+
"""Test that echo sender still logs properly while triggering snapshots."""
|
|
71
|
+
import logging
|
|
72
|
+
from io import StringIO
|
|
73
|
+
|
|
74
|
+
# 1. Arrange
|
|
75
|
+
credential = {"user": "test"}
|
|
76
|
+
config = {"batch_size": 10}
|
|
77
|
+
driver = EchoDriver("test-echo", "http://localhost", credential, config)
|
|
78
|
+
|
|
79
|
+
# Capture logs
|
|
80
|
+
log_stream = StringIO()
|
|
81
|
+
handler = logging.StreamHandler(log_stream)
|
|
82
|
+
logger = logging.getLogger(f"fustor_agent.sender.echo.test-echo")
|
|
83
|
+
logger.addHandler(handler)
|
|
84
|
+
logger.setLevel(logging.INFO) # Ensure INFO logs are captured
|
|
85
|
+
events = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1, "name": "test"}], fields=["id", "name"])]
|
|
86
|
+
|
|
87
|
+
# 2. Act
|
|
88
|
+
result = await driver.send_events(events, source_type="snapshot", is_end=True)
|
|
89
|
+
|
|
90
|
+
# 3. Assert
|
|
91
|
+
assert result == {"success": True, "snapshot_needed": True}
|
|
92
|
+
log_output = log_stream.getvalue()
|
|
93
|
+
assert "[EchoSender]" in log_output
|
|
94
|
+
assert "[SNAPSHOT]" in log_output
|
|
95
|
+
assert "Task: test-echo" in log_output
|
|
96
|
+
assert "本批次: 1条" in log_output
|
|
97
|
+
assert "累计: 1条" in log_output
|
|
98
|
+
assert "Flags: END" in log_output
|
|
99
|
+
assert "First event data" in log_output
|
|
100
|
+
|
|
101
|
+
# Cleanup
|
|
102
|
+
logger.removeHandler(handler)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@pytest.mark.asyncio
|
|
106
|
+
async def test_echo_sender_maintains_statistics():
|
|
107
|
+
"""Test that echo sender maintains cumulative statistics while triggering snapshots."""
|
|
108
|
+
# 1. Arrange
|
|
109
|
+
credential = {"user": "test"}
|
|
110
|
+
config = {"batch_size": 10}
|
|
111
|
+
driver = EchoDriver("test-stats", "http://localhost", credential, config)
|
|
112
|
+
|
|
113
|
+
# First batch of events
|
|
114
|
+
events1 = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 1}, {"id": 2}], fields=["id"])]
|
|
115
|
+
|
|
116
|
+
# Second batch of events
|
|
117
|
+
events2 = [UpdateEvent(event_schema="test", table="test", rows=[{"id": 3}], fields=["id"])]
|
|
118
|
+
|
|
119
|
+
# 2. Act
|
|
120
|
+
result1 = await driver.send_events(events1, source_type="realtime")
|
|
121
|
+
result2 = await driver.send_events(events2, source_type="realtime")
|
|
122
|
+
|
|
123
|
+
# 3. Assert
|
|
124
|
+
# First call should return snapshot_needed=True
|
|
125
|
+
assert result1 == {"success": True, "snapshot_needed": True}
|
|
126
|
+
|
|
127
|
+
# Second call should return snapshot_needed=False because the trigger is one-time
|
|
128
|
+
assert result2 == {"success": True, "snapshot_needed": False}
|
|
129
|
+
|
|
130
|
+
# Statistics should be cumulative (2 from first batch + 1 from second batch = 3 total)
|
|
131
|
+
assert driver.total_rows == 3
|