spidermanager-sdk 0.1.3.dev0__tar.gz → 0.1.5.dev0__tar.gz

{spidermanager_sdk-0.1.3.dev0/src/spidermanager_sdk.egg-info → spidermanager_sdk-0.1.5.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spidermanager-sdk
-Version: 0.1.3.dev0
+Version: 0.1.5.dev0
 Summary: 极简 Python SDK，将爬虫采集数据通过 HTTP 异步中转至 SpiderManager 后端
 Author: SpiderManager Team
 License: MIT
@@ -93,5 +93,5 @@ FlushBuffer (内存缓冲, 线程安全)
     ↓  条数阈值 / 定时器触发
 HttpTransport.send_batch()
     ↓
-POST /api/v1/tasks/data/ingest?task_id=xxx
+POST /api/tasks/data/ingest?task_id=xxx
 ```

{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.5.dev0}/README.md

@@ -78,5 +78,5 @@ FlushBuffer (内存缓冲, 线程安全)
     ↓  条数阈值 / 定时器触发
 HttpTransport.send_batch()
     ↓
-POST /api/v1/tasks/data/ingest?task_id=xxx
+POST /api/tasks/data/ingest?task_id=xxx
 ```

{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.5.dev0}/src/spidermanager_sdk/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.1.3.dev0'
-__version_tuple__ = version_tuple = (0, 1, 3, 'dev0')
+__version__ = version = '0.1.5.dev0'
+__version_tuple__ = version_tuple = (0, 1, 5, 'dev0')
 
-__commit_id__ = commit_id = 'g2385ac666'
+__commit_id__ = commit_id = 'g5c067d7e2'

{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.5.dev0}/src/spidermanager_sdk/aio.py

@@ -17,7 +17,7 @@ import httpx
 
 from spidermanager_sdk.buffer import BufferEntry
 from spidermanager_sdk.client import _DEFAULT_BUFFER_SIZE, _DEFAULT_FLUSH_INTERVAL
-from spidermanager_sdk.transport import _INGEST_PATH, _DEFAULT_TIMEOUT
+from spidermanager_sdk.transport import _INGEST_PATH, _DEFAULT_TIMEOUT, _DEFAULT_LIMITS
 
 logger = logging.getLogger("spidermanager_sdk.aio")
 
@@ -25,15 +25,21 @@ logger = logging.getLogger("spidermanager_sdk.aio")
 class AsyncHttpTransport:
     api_url: str = ""
     task_id: str = ""
+    host_header: str | None = None
     _client: httpx.AsyncClient | None = field(default=None, init=False, repr=False)
 
     async def open(self) -> None:
         if self._client is not None:
             return
+        headers = {"Content-Type": "application/json"}
+        if self.host_header:
+            headers["Host"] = self.host_header
+
         self._client = httpx.AsyncClient(
             base_url=self.api_url,
             timeout=_DEFAULT_TIMEOUT,
-            headers={"Content-Type": "application/json"},
+            limits=_DEFAULT_LIMITS,
+            headers=headers,
         )
 
     async def close(self) -> None:
@@ -100,6 +106,14 @@ class AsyncFlushBuffer:
         if current_size >= self.max_size:
             await self.flush()
 
+    async def rollback(self, entries: list[BufferEntry]) -> None:
+        """
+        异步回滚：将失败数据写回缓冲区头部。
+        """
+        async with self._lock:
+            self._entries = entries + self._entries
+            logger.debug("已异步回滚 %d 条数据至缓冲区", len(entries))
+
     async def flush(self) -> None:
         async with self._lock:
             if not self._entries:
@@ -129,6 +143,7 @@ class AsyncSpiderManagerClient:
     def __init__(self) -> None:
         self._api_url: str = ""
         self._task_id: str = ""
+        self._host_header: str | None = None
         self._initialized: bool = False
         self._transport: AsyncHttpTransport | None = None
         self._buffer: AsyncFlushBuffer | None = None
@@ -138,16 +153,26 @@ class AsyncSpiderManagerClient:
         api_url: str | None = None,
         task_id: str | None = None,
         *,
-        buffer_size: int = _DEFAULT_BUFFER_SIZE,
+        buffer_size: int = 50,
         flush_interval: float = _DEFAULT_FLUSH_INTERVAL,
+        resolve_dns: bool = True,
     ) -> None:
-        self._api_url = api_url or os.environ.get("SPIDER_API_URL", "")
-        self._task_id = task_id or os.environ.get("TASK_ID", "")
+        self._api_url = (api_url or os.environ.get("SPIDER_API_URL", "")).strip()
+        self._task_id = (task_id or os.environ.get("TASK_ID", "")).strip()
         if not self._api_url or not self._task_id:
             raise ValueError("api_url 或 task_id 未配置 (或环境变量缺失)")
         self._api_url = self._api_url.rstrip("/")
 
-        self._transport = AsyncHttpTransport(api_url=self._api_url, task_id=self._task_id)
+        # ── 1.5 DNS 预解析 ──
+        if resolve_dns:
+            from spidermanager_sdk.utils import resolve_provider_url
+            self._api_url, self._host_header = resolve_provider_url(self._api_url)
+
+        self._transport = AsyncHttpTransport(
+            api_url=self._api_url, 
+            task_id=self._task_id,
+            host_header=self._host_header
+        )
         await self._transport.open()
 
         self._buffer = AsyncFlushBuffer(
@@ -206,8 +231,22 @@ class AsyncSpiderManagerClient:
         for table_name, records in grouped.items():
             tasks.append(self._transport.send_batch(table_name, records))
         
-        if tasks:
-            await asyncio.gather(*tasks)
+        if not tasks:
+            return
+
+        try:
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            for res in results:
+                if res is not True:
+                    # 发生异常或返回 False，触发回退
+                    logger.warning("异步上报部分表失败，触发回退")
+                    if self._buffer:
+                        await self._buffer.rollback(entries)
+                    break
+        except Exception:
+            logger.exception("异步上报发生未知异常，触发回退")
+            if self._buffer:
+                await self._buffer.rollback(entries)
 
 # 默认全局异步单例
 async_sdk = AsyncSpiderManagerClient()

{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.5.dev0}/src/spidermanager_sdk/buffer.py

@@ -38,7 +38,7 @@ class FlushBuffer:
     on_flush : Callable[[list[BufferEntry]], None]
         实际的 flush 回调，由 Client 层注入。
     """
-    max_size: int = 20
+    max_size: int = 50
     flush_interval: float = 3.0
     on_flush: Callable[[list[BufferEntry]], None] | None = None
 
@@ -77,6 +77,15 @@ class FlushBuffer:
         if current_size >= self.max_size:
             self.flush()
 
+    def rollback(self, entries: list[BufferEntry]) -> None:
+        """
+        将上报失败的数据重新写回缓冲区头部，等待下次 flush。
+        """
+        with self._lock:
+            # 将失败的数据插到现有数据之前
+            self._entries = entries + self._entries
+            logger.debug("已回滚 %d 条数据至缓冲区", len(entries))
+
     def flush(self) -> None:
         """
         将缓冲区中所有数据取出，交给 on_flush 回调处理。

{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.5.dev0}/src/spidermanager_sdk/client.py

@@ -45,6 +45,7 @@ class SpiderManagerClient:
     def __init__(self) -> None:
         self._api_url: str = ""
         self._task_id: str = ""
+        self._host_header: str | None = None
         self._initialized: bool = False
         self._transport: HttpTransport | None = None
         self._buffer: FlushBuffer | None = None
@@ -59,8 +60,9 @@ class SpiderManagerClient:
         api_url: str | None = None,
         task_id: str | None = None,
         *,
-        buffer_size: int = _DEFAULT_BUFFER_SIZE,
+        buffer_size: int = 50,
         flush_interval: float = _DEFAULT_FLUSH_INTERVAL,
+        resolve_dns: bool = True,
     ) -> None:
         """
         初始化 SDK。
@@ -77,8 +79,8 @@ class SpiderManagerClient:
             时间窗口（秒），默认 3.0。
         """
         # ── 1. 解析配置 ──
-        self._api_url = api_url or os.environ.get("SPIDER_API_URL", "")
-        self._task_id = task_id or os.environ.get("TASK_ID", "")
+        self._api_url = (api_url or os.environ.get("SPIDER_API_URL", "")).strip()
+        self._task_id = (task_id or os.environ.get("TASK_ID", "")).strip()
 
         if not self._api_url:
             raise ValueError(
@@ -94,8 +96,17 @@ class SpiderManagerClient:
         # 去除尾部斜杠
         self._api_url = self._api_url.rstrip("/")
 
+        # ── 1.5 DNS 预解析 ──
+        if resolve_dns:
+            from spidermanager_sdk.utils import resolve_provider_url
+            self._api_url, self._host_header = resolve_provider_url(self._api_url)
+
         # ── 2. 初始化传输层 ──
-        self._transport = HttpTransport(api_url=self._api_url, task_id=self._task_id)
+        self._transport = HttpTransport(
+            api_url=self._api_url, 
+            task_id=self._task_id,
+            host_header=self._host_header
+        )
         self._transport.open()
 
         # ── 3. 初始化缓冲区 ──
@@ -148,11 +159,21 @@ class SpiderManagerClient:
 
     def shutdown(self) -> None:
         """
-        优雅关闭 SDK：停止定时器 → flush 剩余数据 → 关闭 HTTP 连接。
+        优雅关闭 SDK：停止定时器 → 多轮强行 flush → 关闭连接。
         """
         logger.info("SDK 正在关闭...")
         if self._buffer:
             self._buffer.stop()
+            
+            # “夺命连环报”：如果停止后还有残留（由于 rollback 可能导致残留），再尝试最后一搏
+            # 这对处理 atexit 时的重试很有用
+            for attempt in range(3):
+                if self._buffer.pending_count > 0:
+                    logger.info("正在执行进程退出前的最后一轮数据冲刷 (Round %d)...", attempt + 1)
+                    self._buffer.flush()
+                else:
+                    break
+
         if self._transport:
             self._transport.close()
         self._initialized = False
@@ -194,11 +215,16 @@ class SpiderManagerClient:
             grouped[entry.table_name].append(entry.data)
 
         for table_name, records in grouped.items():
-            success = self._transport.send_batch(table_name, records)
-            if not success:
-                logger.error(
-                    "表 '%s' 的 %d 条数据上报失败", table_name, len(records),
-                )
+            try:
+                success = self._transport.send_batch(table_name, records)
+                if not success:
+                    logger.error("表 '%s' 上报失败，触发回退", table_name)
+                    self._buffer.rollback(entries)
+                    break
+            except Exception:
+                logger.exception("上报过程发生异常，触发回退")
+                self._buffer.rollback(entries)
+                break
 
     def _register_exit_hooks(self) -> None:
         """

{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.5.dev0}/src/spidermanager_sdk/transport.py

@@ -17,11 +17,14 @@ import httpx
 logger = logging.getLogger("spidermanager_sdk.transport")
 
 # 上报接口固定路径
-_INGEST_PATH: str = "/api/v1/tasks/data/ingest"
+_INGEST_PATH: str = "/api/tasks/data/ingest"
 
 # 默认超时配置（连接 / 读取 / 写入 / 总计）
 _DEFAULT_TIMEOUT = httpx.Timeout(connect=5.0, read=10.0, write=10.0, pool=30.0)
 
+# 默认连接池限制：增加保活连接数，减少高频上报下的 DNS 解析压力
+_DEFAULT_LIMITS = httpx.Limits(max_connections=50, max_keepalive_connections=20)
+
 
 @dataclass
 class HttpTransport:
@@ -37,6 +40,7 @@ class HttpTransport:
     """
     api_url: str = ""
     task_id: str = ""
+    host_header: str | None = None
 
     _client: httpx.Client | None = field(default=None, init=False, repr=False)
 
@@ -48,10 +52,15 @@ class HttpTransport:
         """初始化底层 httpx 连接池。"""
         if self._client is not None:
             return
+        headers = {"Content-Type": "application/json"}
+        if self.host_header:
+            headers["Host"] = self.host_header
+
         self._client = httpx.Client(
             base_url=self.api_url,
             timeout=_DEFAULT_TIMEOUT,
-            headers={"Content-Type": "application/json"},
+            limits=_DEFAULT_LIMITS,
+            headers=headers,
         )
         logger.debug("HTTP transport opened → %s", self.api_url)
 

spidermanager_sdk-0.1.5.dev0/src/spidermanager_sdk/utils.py

@@ -0,0 +1,26 @@
+import socket
+import logging
+from urllib.parse import urlparse
+
+logger = logging.getLogger("spidermanager_sdk.utils")
+
+def resolve_provider_url(api_url: str) -> tuple[str, str | None]:
+    """
+    将 URL 中的域名解析为 IP，绕过 Docker DNS 抖动。
+    返回: (基于IP的URL, 原始Hostname)
+    """
+    parsed = urlparse(api_url)
+    hostname = parsed.hostname
+    if not hostname:
+        return api_url, None
+    
+    try:
+        # 仅在初始化时执行一次同步解析
+        ip = socket.gethostbyname(hostname)
+        port = f":{parsed.port}" if parsed.port else ""
+        # 重新构建 URL，保留协议和路径，替换域名为 IP
+        resolved_url = f"{parsed.scheme}://{ip}{port}"
+        return resolved_url, hostname
+    except Exception as e:
+        logger.warning(f"DNS 预解析失败: {e}，将回退到原始地址")
+        return api_url, hostname

{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.5.dev0/src/spidermanager_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: spidermanager-sdk
-Version: 0.1.3.dev0
+Version: 0.1.5.dev0
 Summary: 极简 Python SDK，将爬虫采集数据通过 HTTP 异步中转至 SpiderManager 后端
 Author: SpiderManager Team
 License: MIT
@@ -93,5 +93,5 @@ FlushBuffer (内存缓冲, 线程安全)
     ↓  条数阈值 / 定时器触发
 HttpTransport.send_batch()
     ↓
-POST /api/v1/tasks/data/ingest?task_id=xxx
+POST /api/tasks/data/ingest?task_id=xxx
 ```

{spidermanager_sdk-0.1.3.dev0 → spidermanager_sdk-0.1.5.dev0}/src/spidermanager_sdk.egg-info/SOURCES.txt

@@ -2,6 +2,8 @@
 LICENSE
 README.md
 pyproject.toml
+test_robustness.py
+test_transport.py
 .github/workflows/publish.yml
 src/spidermanager_sdk/__init__.py
 src/spidermanager_sdk/_version.py
@@ -9,6 +11,7 @@ src/spidermanager_sdk/aio.py
 src/spidermanager_sdk/buffer.py
 src/spidermanager_sdk/client.py
 src/spidermanager_sdk/transport.py
+src/spidermanager_sdk/utils.py
 src/spidermanager_sdk.egg-info/PKG-INFO
 src/spidermanager_sdk.egg-info/SOURCES.txt
 src/spidermanager_sdk.egg-info/dependency_links.txt

spidermanager_sdk-0.1.5.dev0/test_robustness.py

@@ -0,0 +1,52 @@
+import unittest
+from unittest.mock import MagicMock, patch
+from spidermanager_sdk.buffer import BufferEntry, FlushBuffer
+from spidermanager_sdk.client import SpiderManagerClient
+
+class TestRobustness(unittest.TestCase):
+    def test_buffer_rollback(self):
+        buffer = FlushBuffer(max_size=10)
+        entries = [BufferEntry("test", {"id": i}) for i in range(5)]
+        
+        # Initial entries
+        for e in entries:
+            buffer.add(e)
+        self.assertEqual(buffer.pending_count, 5)
+        
+        # Rollback some failed entries
+        failed_entries = [BufferEntry("test", {"id": "failed"})]
+        buffer.rollback(failed_entries)
+        
+        # Failed entries should be at the front
+        self.assertEqual(buffer.pending_count, 6)
+        
+        # Verify ordering (simplistic check)
+        # In a real test we'd access private _entries but let's just check count
+        
+    @patch("spidermanager_sdk.transport.HttpTransport.send_batch")
+    def test_client_rollback_on_failure(self, mock_send):
+        mock_send.return_value = False # Simulate failure
+        
+        client = SpiderManagerClient()
+        # Mocking transport and buffer to avoid real IO
+        client.init(api_url="http://localhost:8000", task_id="test", resolve_dns=False)
+        
+        client.insert("test", {"data": "foo"})
+        client.flush()
+        
+        # Since it failed, data should be rolled back to buffer
+        self.assertEqual(client.pending_count, 1)
+
+    @patch("spidermanager_sdk.utils.resolve_provider_url")
+    def test_dns_resolution_integration(self, mock_resolve):
+        mock_resolve.return_value = ("http://1.2.3.4:8000", "my-host")
+        
+        client = SpiderManagerClient()
+        client.init(api_url="http://my-host:8000", task_id="test", resolve_dns=True)
+        
+        self.assertEqual(client._api_url, "http://1.2.3.4:8000")
+        self.assertEqual(client._host_header, "my-host")
+        self.assertEqual(client._transport.host_header, "my-host")
+
+if __name__ == "__main__":
+    unittest.main()

spidermanager_sdk-0.1.5.dev0/test_transport.py

@@ -0,0 +1,27 @@
+import unittest
+from unittest.mock import patch, MagicMock
+from spidermanager_sdk.transport import HttpTransport, _INGEST_PATH
+
+class TestTransport(unittest.TestCase):
+    @patch("httpx.Client.post")
+    def test_send_batch_url(self, mock_post):
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_post.return_value = mock_response
+
+        transport = HttpTransport(api_url="http://test_backend:8000", task_id="test_task_123")
+        
+        # 验证发送批量数据时使用的 URL 是否正确
+        success = transport.send_batch("test_table", [{"col1": "val1"}])
+        
+        self.assertTrue(success)
+        mock_post.assert_called_once()
+        args, kwargs = mock_post.call_args
+        
+        self.assertEqual(args[0], "/api/tasks/data/ingest")
+        self.assertEqual(kwargs["json"], {"table_name": "test_table", "data": [{"col1": "val1"}]})
+        self.assertEqual(kwargs["params"], {"task_id": "test_task_123"})
+        print("Test passed: URL is correctly formed as /api/tasks/data/ingest")
+
+if __name__ == "__main__":
+    unittest.main()