spidermanager-sdk 0.1.3.dev0__tar.gz → 0.1.4.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {spidermanager_sdk-0.1.3.dev0/src/spidermanager_sdk.egg-info → spidermanager_sdk-0.1.4.dev0}/PKG-INFO +2 -2
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/README.md +1 -1
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/_version.py +3 -3
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/transport.py +1 -1
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0/src/spidermanager_sdk.egg-info}/PKG-INFO +2 -2
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk.egg-info/SOURCES.txt +1 -0
- spidermanager_sdk-0.1.4.dev0/test_transport.py +27 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/.github/workflows/publish.yml +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/.gitignore +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/LICENSE +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/pyproject.toml +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/setup.cfg +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/__init__.py +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/aio.py +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/buffer.py +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/client.py +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk.egg-info/dependency_links.txt +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk.egg-info/requires.txt +0 -0
- {spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: spidermanager-sdk
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4.dev0
|
|
4
4
|
Summary: 极简 Python SDK,将爬虫采集数据通过 HTTP 异步中转至 SpiderManager 后端
|
|
5
5
|
Author: SpiderManager Team
|
|
6
6
|
License: MIT
|
|
@@ -93,5 +93,5 @@ FlushBuffer (内存缓冲, 线程安全)
|
|
|
93
93
|
↓ 条数阈值 / 定时器触发
|
|
94
94
|
HttpTransport.send_batch()
|
|
95
95
|
↓
|
|
96
|
-
POST /api/
|
|
96
|
+
POST /api/tasks/data/ingest?task_id=xxx
|
|
97
97
|
```
|
{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/_version.py
RENAMED
|
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
|
|
|
28
28
|
commit_id: COMMIT_ID
|
|
29
29
|
__commit_id__: COMMIT_ID
|
|
30
30
|
|
|
31
|
-
__version__ = version = '0.1.
|
|
32
|
-
__version_tuple__ = version_tuple = (0, 1,
|
|
31
|
+
__version__ = version = '0.1.4.dev0'
|
|
32
|
+
__version_tuple__ = version_tuple = (0, 1, 4, 'dev0')
|
|
33
33
|
|
|
34
|
-
__commit_id__ = commit_id = '
|
|
34
|
+
__commit_id__ = commit_id = 'g5b7006393'
|
{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/transport.py
RENAMED
|
@@ -17,7 +17,7 @@ import httpx
|
|
|
17
17
|
logger = logging.getLogger("spidermanager_sdk.transport")
|
|
18
18
|
|
|
19
19
|
# 上报接口固定路径
|
|
20
|
-
_INGEST_PATH: str = "/api/
|
|
20
|
+
_INGEST_PATH: str = "/api/tasks/data/ingest"
|
|
21
21
|
|
|
22
22
|
# 默认超时配置(连接 / 读取 / 写入 / 总计)
|
|
23
23
|
_DEFAULT_TIMEOUT = httpx.Timeout(connect=5.0, read=10.0, write=10.0, pool=30.0)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: spidermanager-sdk
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4.dev0
|
|
4
4
|
Summary: 极简 Python SDK,将爬虫采集数据通过 HTTP 异步中转至 SpiderManager 后端
|
|
5
5
|
Author: SpiderManager Team
|
|
6
6
|
License: MIT
|
|
@@ -93,5 +93,5 @@ FlushBuffer (内存缓冲, 线程安全)
|
|
|
93
93
|
↓ 条数阈值 / 定时器触发
|
|
94
94
|
HttpTransport.send_batch()
|
|
95
95
|
↓
|
|
96
|
-
POST /api/
|
|
96
|
+
POST /api/tasks/data/ingest?task_id=xxx
|
|
97
97
|
```
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
from unittest.mock import patch, MagicMock
|
|
3
|
+
from spidermanager_sdk.transport import HttpTransport, _INGEST_PATH
|
|
4
|
+
|
|
5
|
+
class TestTransport(unittest.TestCase):
|
|
6
|
+
@patch("httpx.Client.post")
|
|
7
|
+
def test_send_batch_url(self, mock_post):
|
|
8
|
+
mock_response = MagicMock()
|
|
9
|
+
mock_response.status_code = 200
|
|
10
|
+
mock_post.return_value = mock_response
|
|
11
|
+
|
|
12
|
+
transport = HttpTransport(api_url="http://test_backend:8000", task_id="test_task_123")
|
|
13
|
+
|
|
14
|
+
# 验证发送批量数据时使用的 URL 是否正确
|
|
15
|
+
success = transport.send_batch("test_table", [{"col1": "val1"}])
|
|
16
|
+
|
|
17
|
+
self.assertTrue(success)
|
|
18
|
+
mock_post.assert_called_once()
|
|
19
|
+
args, kwargs = mock_post.call_args
|
|
20
|
+
|
|
21
|
+
self.assertEqual(args[0], "/api/tasks/data/ingest")
|
|
22
|
+
self.assertEqual(kwargs["json"], {"table_name": "test_table", "data": [{"col1": "val1"}]})
|
|
23
|
+
self.assertEqual(kwargs["params"], {"task_id": "test_task_123"})
|
|
24
|
+
print("Test passed: URL is correctly formed as /api/tasks/data/ingest")
|
|
25
|
+
|
|
26
|
+
if __name__ == "__main__":
|
|
27
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/buffer.py
RENAMED
|
File without changes
|
{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.4.dev0}/src/spidermanager_sdk/client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|