xparse-client 0.2.20__py3-none-any.whl → 0.3.0b1__py3-none-any.whl

tests/unit/connectors/test_ftp.py

@@ -0,0 +1,525 @@
+"""FtpSource 测试
+
+测试 FTP 数据源连接器。使用 mock 模式测试，无需真实 FTP 服务器。
+
+运行方式：
+    pytest tests/unit/connectors/test_ftp.py -v
+"""
+
+from ftplib import error_perm
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from xparse_client.connectors import FtpSource
+from xparse_client.exceptions import SourceError
+
+# ============================================================================
+# Mock FTP 测试
+# ============================================================================
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_initialization(mock_ftp_class):
+    """测试 FtpSource 初始化"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="testuser",
+        password="testpass"
+    )
+
+    assert source.host == "ftp.example.com"
+    assert source.port == 21
+    assert source.username == "testuser"
+    assert source.password == "testpass"
+    assert source.pattern is None
+    assert source.recursive is False
+
+    # 验证连接和登录
+    mock_ftp_class.assert_called_once_with()
+    mock_ftp.connect.assert_called_once_with("ftp.example.com", 21)
+    mock_ftp.login.assert_called_once_with("testuser", "testpass")
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_list_files_basic(mock_ftp_class):
+    """测试列出文件（非递归）"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    # Mock pwd() 返回当前目录
+    mock_ftp.pwd.return_value = "/"
+
+    # Mock mlsd() 返回文件列表
+    mock_ftp.mlsd.return_value = [
+        ("file1.pdf", {"type": "file"}),
+        ("file2.docx", {"type": "file"}),
+        ("subdir", {"type": "dir"}),
+        (".", {"type": "dir"}),
+        ("..", {"type": "dir"}),
+    ]
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test",
+        recursive=False
+    )
+
+    files = source.list_files()
+
+    # 非递归，只列出文件，不包含目录和 . ..
+    assert len(files) == 2
+    assert 'file1.pdf' in files
+    assert 'file2.docx' in files
+    assert 'subdir' not in files
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_list_files_with_pattern(mock_ftp_class):
+    """测试带 pattern 的文件列表"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    mock_ftp.pwd.return_value = "/"
+    mock_ftp.mlsd.return_value = [
+        ("file1.pdf", {"type": "file"}),
+        ("file2.docx", {"type": "file"}),
+        ("file3.txt", {"type": "file"}),
+    ]
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test",
+        pattern=["*.pdf"]
+    )
+
+    files = source.list_files()
+
+    assert len(files) == 1
+    assert 'file1.pdf' in files
+    assert 'file2.docx' not in files
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_list_files_recursive(mock_ftp_class):
+    """测试递归列出文件"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    mock_ftp.pwd.side_effect = ["/", "/", "/subdir", "/subdir", "/"]
+
+    # Mock mlsd() 根据当前目录返回不同内容
+    call_count = [0]
+
+    def mock_mlsd():
+        call_count[0] += 1
+        if call_count[0] == 1:
+            # 根目录
+            return [
+                ("file1.pdf", {"type": "file"}),
+                ("subdir", {"type": "dir"}),
+            ]
+        else:
+            # subdir 目录
+            return [
+                ("file2.pdf", {"type": "file"}),
+            ]
+
+    mock_ftp.mlsd.side_effect = mock_mlsd
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test",
+        recursive=True
+    )
+
+    files = source.list_files()
+
+    # 应该包含两级目录的文件
+    assert len(files) == 2
+    assert 'file1.pdf' in files
+    assert 'subdir/file2.pdf' in files
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_list_files_fallback_list(mock_ftp_class):
+    """测试 mlsd 失败时回退到 LIST 命令"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    mock_ftp.pwd.return_value = "/"
+    # mlsd 失败
+    mock_ftp.mlsd.side_effect = Exception("mlsd not supported")
+
+    # Mock retrlines 返回 LIST 格式输出
+    def mock_retrlines(cmd, callback):
+        if cmd == "LIST":
+            callback("-rw-r--r-- 1 user group 1024 Jan 01 12:00 file1.pdf")
+            callback("-rw-r--r-- 1 user group 2048 Jan 02 13:00 file2.docx")
+            callback("drwxr-xr-x 2 user group 4096 Jan 03 14:00 subdir")
+
+    mock_ftp.retrlines.side_effect = mock_retrlines
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    files = source.list_files()
+
+    assert len(files) == 2
+    assert 'file1.pdf' in files
+    assert 'file2.docx' in files
+    assert 'subdir' not in files
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_read_file(mock_ftp_class):
+    """测试读取文件"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    # Mock retrbinary 读取文件
+    test_content = b'FTP file content'
+
+    def mock_retrbinary(cmd, callback):
+        callback(test_content)
+
+    mock_ftp.retrbinary.side_effect = mock_retrbinary
+
+    # Mock sendcmd 返回修改时间
+    mock_ftp.sendcmd.return_value = '213 20240128120000'
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    file_bytes, data_source = source.read_file('test.pdf')
+
+    assert file_bytes == test_content
+    assert data_source['url'] == 'ftp://ftp.example.com:21/test.pdf'
+    assert data_source['record_locator']['protocol'] == 'ftp'
+    assert data_source['record_locator']['server'] == 'ftp.example.com:21'
+    assert 'version' in data_source
+    assert 'date_modified' in data_source
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_read_file_no_mdtm(mock_ftp_class):
+    """测试读取文件（服务器不支持 MDTM）"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    test_content = b'Content'
+
+    def mock_retrbinary(cmd, callback):
+        callback(test_content)
+
+    mock_ftp.retrbinary.side_effect = mock_retrbinary
+    # sendcmd 失败
+    mock_ftp.sendcmd.side_effect = Exception("MDTM not supported")
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    file_bytes, data_source = source.read_file('test.pdf')
+
+    assert file_bytes == test_content
+    # version 应该是 None 或 0
+    assert data_source['version'] is not None
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_read_file_not_found(mock_ftp_class):
+    """测试读取不存在的文件"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    # Mock retrbinary 抛出异常
+    mock_ftp.retrbinary.side_effect = error_perm("550 File not found")
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    with pytest.raises(SourceError) as exc_info:
+        source.read_file('nonexistent.pdf')
+
+    assert "读取 FTP 文件失败" in str(exc_info.value)
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_connection_error(mock_ftp_class):
+    """测试连接错误"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    # Mock connect 失败
+    mock_ftp.connect.side_effect = OSError("Connection refused")
+
+    with pytest.raises(SourceError) as exc_info:
+        FtpSource(
+            host="invalid-host",
+            port=21,
+            username="test",
+            password="test"
+        )
+
+    assert "FTP 连接失败" in str(exc_info.value)
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_authentication_error(mock_ftp_class):
+    """测试认证错误"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    # Mock login 失败
+    mock_ftp.login.side_effect = error_perm("530 Login incorrect")
+
+    with pytest.raises(SourceError) as exc_info:
+        FtpSource(
+            host="ftp.example.com",
+            port=21,
+            username="wrong",
+            password="wrong"
+        )
+
+    assert "FTP 连接失败" in str(exc_info.value)
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_context_manager(mock_ftp_class):
+    """测试上下文管理器"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    with FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    ) as source:
+        assert source is not None
+
+    # 验证连接被关闭
+    mock_ftp.quit.assert_called_once()
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_close(mock_ftp_class):
+    """测试手动关闭连接"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    source.close()
+
+    # 验证 quit 被调用
+    mock_ftp.quit.assert_called_once()
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_close_with_error(mock_ftp_class):
+    """测试关闭连接时出错（回退到 close）"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    # quit 失败
+    mock_ftp.quit.side_effect = Exception("quit failed")
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    source.close()  # 不应该抛出异常
+
+    # 验证回退到 close
+    mock_ftp.close.assert_called_once()
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_custom_pattern_and_recursive(mock_ftp_class):
+    """测试自定义 pattern 和 recursive"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test",
+        pattern=["*.pdf", "*.docx"],
+        recursive=True
+    )
+
+    assert source.pattern == ["*.pdf", "*.docx"]
+    assert source.recursive is True
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_repr(mock_ftp_class):
+    """测试 __repr__"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    assert repr(source) == "<FtpSource host=ftp.example.com:21>"
+
+
+# ============================================================================
+# MLSD/NLST 回退测试
+# ============================================================================
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_get_dir_items_mlsd_to_list_fallback(mock_ftp_class):
+    """测试 MLSD 失败回退到 LIST 的场景"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    # Mock MLSD 失败
+    mock_ftp.mlsd.side_effect = Exception("MLSD not supported")
+
+    # Mock LIST 成功（需要支持 retrlines 回调）
+    def mock_retrlines(cmd, callback):
+        # 模拟 LIST 输出
+        callback("drwxr-xr-x   2 user group     4096 Jan 28 12:00 dir1")
+        callback("-rw-r--r--   1 user group     1024 Jan 28 12:00 file1.txt")
+
+    mock_ftp.retrlines.side_effect = mock_retrlines
+
+    items = source._get_dir_items()
+
+    # 验证返回的结果
+    assert len(items) == 2
+    assert ("dir1", True) in items
+    assert ("file1.txt", False) in items
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_get_dir_items_list_to_nlst_fallback(mock_ftp_class):
+    """测试 LIST 失败回退到 NLST 的场景"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    # Mock MLSD 和 LIST 都失败
+    mock_ftp.mlsd.side_effect = Exception("MLSD not supported")
+    mock_ftp.retrlines.side_effect = Exception("LIST not supported")
+
+    # Mock NLST 成功
+    mock_ftp.nlst.return_value = ["dir1", "file1.txt", ".", ".."]
+
+    # Mock cwd 判断是否为目录
+    def mock_cwd(path):
+        if path == "dir1":
+            return  # 成功，说明是目录
+        elif path == "..":
+            return  # 返回上级目录
+        else:
+            raise Exception("Not a directory")
+
+    mock_ftp.cwd.side_effect = mock_cwd
+
+    items = source._get_dir_items()
+
+    # 验证返回的结果（. 和 .. 会被过滤）
+    assert len(items) == 2
+    assert ("dir1", True) in items
+    assert ("file1.txt", False) in items
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_get_dir_items_all_methods_fail(mock_ftp_class):
+    """测试所有方法都失败的场景"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    # Mock 所有方法都失败
+    mock_ftp.mlsd.side_effect = Exception("MLSD not supported")
+    mock_ftp.retrlines.side_effect = Exception("LIST not supported")
+    mock_ftp.nlst.side_effect = Exception("NLST not supported")
+
+    items = source._get_dir_items()
+
+    # 应该返回空列表
+    assert items == []
+
+
+@patch('xparse_client.connectors.sources.ftp.ftplib.FTP')
+def test_ftp_source_close_quit_error(mock_ftp_class):
+    """测试 close 时 quit 失败的场景"""
+    mock_ftp = MagicMock()
+    mock_ftp_class.return_value = mock_ftp
+
+    source = FtpSource(
+        host="ftp.example.com",
+        port=21,
+        username="test",
+        password="test"
+    )
+
+    # Mock quit 失败
+    mock_ftp.quit.side_effect = Exception("Quit error")
+
+    # 应该不抛出异常（错误被捕获）
+    source.close()
+
+    # 验证 quit 被调用
+    mock_ftp.quit.assert_called_once()