crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

tests/simple_command_test.py

@@ -1,120 +1,120 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-简单测试所有 crawlo 命令
-"""
-
-import sys
-import os
-import subprocess
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-def run_command(cmd, cwd=None):
-    """运行命令并返回结果"""
-    try:
-        result = subprocess.run(
-            cmd, 
-            shell=True, 
-            cwd=cwd, 
-            capture_output=True, 
-            text=True, 
-            timeout=30
-        )
-        return result.returncode, result.stdout, result.stderr
-    except subprocess.TimeoutExpired:
-        return -1, "", "Command timed out"
-    except Exception as e:
-        return -1, "", str(e)
-
-def test_help_command():
-    """测试 help 命令"""
-    print("测试 help 命令...")
-    
-    # 测试 -h 参数
-    code, stdout, stderr = run_command("python -m crawlo.cli -h")
-    assert code == 0, f"help 命令失败: {stderr}"
-    assert "Crawlo" in stdout, "help 输出不包含框架名称"
-    
-    # 测试 --help 参数
-    code, stdout, stderr = run_command("python -m crawlo.cli --help")
-    assert code == 0, f"help 命令失败: {stderr}"
-    assert "Crawlo" in stdout, "help 输出不包含框架名称"
-    
-    print("✅ help 命令测试通过")
-
-def test_version_command():
-    """测试 version 命令"""
-    print("测试 version 命令...")
-    
-    # 测试 -v 参数
-    code, stdout, stderr = run_command("python -m crawlo.cli -v")
-    assert code == 0, f"version 命令失败: {stderr}"
-    assert "Crawlo" in stdout, "version 输出不包含框架名称"
-    
-    # 测试 --version 参数
-    code, stdout, stderr = run_command("python -m crawlo.cli --version")
-    assert code == 0, f"version 命令失败: {stderr}"
-    assert "Crawlo" in stdout, "version 输出不包含框架名称"
-    
-    print("✅ version 命令测试通过")
-
-def test_command_help():
-    """测试各命令的帮助信息"""
-    print("测试各命令的帮助信息...")
-    
-    commands = ["startproject", "genspider", "run", "check", "list", "stats"]
-    
-    for command in commands:
-        code, stdout, stderr = run_command(f"python -m crawlo.cli {command} --help")
-        # 命令帮助通常返回非0状态码，但我们检查输出
-        assert len(stdout) > 0 or len(stderr) > 0, f"{command} 命令帮助无输出"
-        print(f"✅ {command} 命令帮助测试通过")
-
-def test_invalid_command():
-    """测试无效命令"""
-    print("测试无效命令...")
-    
-    code, stdout, stderr = run_command("python -m crawlo.cli invalid_command")
-    assert code != 0, "无效命令应该返回非0状态码"
-    assert "Unknown command" in stderr or "Unknown command" in stdout, "应该提示未知命令"
-    
-    print("✅ 无效命令测试通过")
-
-def main():
-    """主函数"""
-    print("开始简单测试所有 crawlo 命令...")
-    print("=" * 50)
-    
-    try:
-        # 测试 help 命令
-        test_help_command()
-        print()
-        
-        # 测试 version 命令
-        test_version_command()
-        print()
-        
-        # 测试各命令的帮助信息
-        test_command_help()
-        print()
-        
-        # 测试无效命令
-        test_invalid_command()
-        print()
-        
-        print("=" * 50)
-        print("所有命令简单测试通过！")
-        
-    except Exception as e:
-        print("=" * 50)
-        print(f"测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-        return 1
-    
-    return 0
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+简单测试所有 crawlo 命令
+"""
+
+import sys
+import os
+import subprocess
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+def run_command(cmd, cwd=None):
+    """运行命令并返回结果"""
+    try:
+        result = subprocess.run(
+            cmd, 
+            shell=True, 
+            cwd=cwd, 
+            capture_output=True, 
+            text=True, 
+            timeout=30
+        )
+        return result.returncode, result.stdout, result.stderr
+    except subprocess.TimeoutExpired:
+        return -1, "", "Command timed out"
+    except Exception as e:
+        return -1, "", str(e)
+
+def test_help_command():
+    """测试 help 命令"""
+    print("测试 help 命令...")
+    
+    # 测试 -h 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli -h")
+    assert code == 0, f"help 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "help 输出不包含框架名称"
+    
+    # 测试 --help 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli --help")
+    assert code == 0, f"help 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "help 输出不包含框架名称"
+    
+    print("✅ help 命令测试通过")
+
+def test_version_command():
+    """测试 version 命令"""
+    print("测试 version 命令...")
+    
+    # 测试 -v 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli -v")
+    assert code == 0, f"version 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "version 输出不包含框架名称"
+    
+    # 测试 --version 参数
+    code, stdout, stderr = run_command("python -m crawlo.cli --version")
+    assert code == 0, f"version 命令失败: {stderr}"
+    assert "Crawlo" in stdout, "version 输出不包含框架名称"
+    
+    print("✅ version 命令测试通过")
+
+def test_command_help():
+    """测试各命令的帮助信息"""
+    print("测试各命令的帮助信息...")
+    
+    commands = ["startproject", "genspider", "run", "check", "list", "stats"]
+    
+    for command in commands:
+        code, stdout, stderr = run_command(f"python -m crawlo.cli {command} --help")
+        # 命令帮助通常返回非0状态码，但我们检查输出
+        assert len(stdout) > 0 or len(stderr) > 0, f"{command} 命令帮助无输出"
+        print(f"✅ {command} 命令帮助测试通过")
+
+def test_invalid_command():
+    """测试无效命令"""
+    print("测试无效命令...")
+    
+    code, stdout, stderr = run_command("python -m crawlo.cli invalid_command")
+    assert code != 0, "无效命令应该返回非0状态码"
+    assert "Unknown command" in stderr or "Unknown command" in stdout, "应该提示未知命令"
+    
+    print("✅ 无效命令测试通过")
+
+def main():
+    """主函数"""
+    print("开始简单测试所有 crawlo 命令...")
+    print("=" * 50)
+    
+    try:
+        # 测试 help 命令
+        test_help_command()
+        print()
+        
+        # 测试 version 命令
+        test_version_command()
+        print()
+        
+        # 测试各命令的帮助信息
+        test_command_help()
+        print()
+        
+        # 测试无效命令
+        test_invalid_command()
+        print()
+        
+        print("=" * 50)
+        print("所有命令简单测试通过！")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+if __name__ == "__main__":
     sys.exit(main())

tests/simple_crawlo_test.py

@@ -1,127 +1,127 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-简单的Crawlo性能测试
-"""
-import asyncio
-import time
-import sys
-import os
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo import Spider, Request
-from crawlo.crawler import CrawlerProcess
-from crawlo.items import Item, Field
-
-class NewsItem(Item):
-    title = Field()
-    publish_time = Field()
-    url = Field()
-    source = Field()
-    content = Field()
-
-class OfweekSimpleSpider(Spider):
-    name = "ofweek_simple"
-    
-    def start_requests(self):
-        headers = {
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
-            "Cache-Control": "no-cache",
-            "Connection": "keep-alive",
-            "Pragma": "no-cache",
-            "Referer": "https://ee.ofweek.com/CATList-2800-8100-ee-2.html",
-            "Sec-Fetch-Dest": "document",
-            "Sec-Fetch-Mode": "navigate",
-            "Sec-Fetch-Site": "same-origin",
-            "Sec-Fetch-User": "?1",
-            "Upgrade-Insecure-Requests": "1",
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
-        }
-        cookies = {
-            "__utmz": "57425525.1730117117.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
-            "Hm_lvt_abe9900db162c6d089cdbfd107db0f03": "1739244841",
-            "Hm_lvt_af50e2fc51af73da7720fb324b88a975": "1740100727",
-            "JSESSIONID": "FEA96D3B5FC31350B2285E711BF2A541",
-            "Hm_lvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477622",
-            "HMACCOUNT": "08DF0D235A291EAA",
-            "__utma": "57425525.2080994505.1730117117.1747970718.1757477622.50",
-            "__utmc": "57425525",
-            "__utmt": "1",
-            "__utmb": "57425525.2.10.1757477622",
-            "Hm_lpvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477628",
-            "index_burying_point": "c64d6c31e69d560efe319cc9f8be279f"
-        }
-
-        # 使用较少的页数进行测试
-        max_page = 5
-        for page in range(1, max_page + 1):
-            url = f"https://ee.ofweek.com/CATList-2800-8100-ee-{page}.html"
-            yield Request(url=url, callback=self.parse, headers=headers, cookies=cookies)
-
-    def parse(self, response):
-        rows = response.xpath("//div[@class=\"main_left\"]/div[@class=\"list_model\"]/div[@class=\"model_right model_right2\"]")
-        
-        for row in rows:
-            try:
-                url = row.xpath("./h3/a/@href").extract_first()
-                title = row.xpath("./h3/a/text()").extract_first()
-
-                if not url or not title:
-                    continue
-
-                absolute_url = response.urljoin(url)
-                if not absolute_url.startswith(("http://", "https://")):
-                    continue
-
-                yield Request(
-                    url=absolute_url,
-                    meta={"title": title.strip() if title else "", "parent_url": response.url},
-                    callback=self.parse_detail
-                )
-            except Exception:
-                continue
-
-    def parse_detail(self, response):
-        title = response.meta.get("title", "")
-        content_elements = response.xpath("//div[@class=\"TRS_Editor\"]|//*[@id=\"articleC\"]")
-        if content_elements:
-            content = content_elements.xpath(".//text()").extract()
-            content = "\n".join([text.strip() for text in content if text.strip()])
-        else:
-            content = ""
-
-        publish_time = response.xpath("//div[@class=\"time fl\"]/text()").extract_first()
-        if publish_time:
-            publish_time = publish_time.strip()
-
-        source = response.xpath("//div[@class=\"source-name\"]/text()").extract_first()
-
-        item = NewsItem()
-        item["title"] = title.strip() if title else ""
-        item["publish_time"] = publish_time if publish_time else ""
-        item["url"] = response.url
-        item["source"] = source if source else ""
-        item["content"] = content
-
-        yield item
-
-async def main():
-    start_time = time.time()
-    
-    process = CrawlerProcess(settings={
-        "CONCURRENCY": 8,
-        "DOWNLOAD_DELAY": 0.1,
-        "LOG_LEVEL": "ERROR",  # 减少日志输出以提高性能
-    })
-    await process.crawl(OfweekSimpleSpider)
-    
-    end_time = time.time()
-    execution_time = end_time - start_time
-    
-    print(f"Crawlo执行时间: {execution_time:.2f}秒")
-
-if __name__ == "__main__":
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+简单的Crawlo性能测试
+"""
+import asyncio
+import time
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo import Spider, Request
+from crawlo.crawler import CrawlerProcess
+from crawlo.items import Item, Field
+
+class NewsItem(Item):
+    title = Field()
+    publish_time = Field()
+    url = Field()
+    source = Field()
+    content = Field()
+
+class OfweekSimpleSpider(Spider):
+    name = "ofweek_simple"
+    
+    def start_requests(self):
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://ee.ofweek.com/CATList-2800-8100-ee-2.html",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+        }
+        cookies = {
+            "__utmz": "57425525.1730117117.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
+            "Hm_lvt_abe9900db162c6d089cdbfd107db0f03": "1739244841",
+            "Hm_lvt_af50e2fc51af73da7720fb324b88a975": "1740100727",
+            "JSESSIONID": "FEA96D3B5FC31350B2285E711BF2A541",
+            "Hm_lvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477622",
+            "HMACCOUNT": "08DF0D235A291EAA",
+            "__utma": "57425525.2080994505.1730117117.1747970718.1757477622.50",
+            "__utmc": "57425525",
+            "__utmt": "1",
+            "__utmb": "57425525.2.10.1757477622",
+            "Hm_lpvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477628",
+            "index_burying_point": "c64d6c31e69d560efe319cc9f8be279f"
+        }
+
+        # 使用较少的页数进行测试
+        max_page = 5
+        for page in range(1, max_page + 1):
+            url = f"https://ee.ofweek.com/CATList-2800-8100-ee-{page}.html"
+            yield Request(url=url, callback=self.parse, headers=headers, cookies=cookies)
+
+    def parse(self, response):
+        rows = response.xpath("//div[@class=\"main_left\"]/div[@class=\"list_model\"]/div[@class=\"model_right model_right2\"]")
+        
+        for row in rows:
+            try:
+                url = row.xpath("./h3/a/@href").extract_first()
+                title = row.xpath("./h3/a/text()").extract_first()
+
+                if not url or not title:
+                    continue
+
+                absolute_url = response.urljoin(url)
+                if not absolute_url.startswith(("http://", "https://")):
+                    continue
+
+                yield Request(
+                    url=absolute_url,
+                    meta={"title": title.strip() if title else "", "parent_url": response.url},
+                    callback=self.parse_detail
+                )
+            except Exception:
+                continue
+
+    def parse_detail(self, response):
+        title = response.meta.get("title", "")
+        content_elements = response.xpath("//div[@class=\"TRS_Editor\"]|//*[@id=\"articleC\"]")
+        if content_elements:
+            content = content_elements.xpath(".//text()").extract()
+            content = "\n".join([text.strip() for text in content if text.strip()])
+        else:
+            content = ""
+
+        publish_time = response.xpath("//div[@class=\"time fl\"]/text()").extract_first()
+        if publish_time:
+            publish_time = publish_time.strip()
+
+        source = response.xpath("//div[@class=\"source-name\"]/text()").extract_first()
+
+        item = NewsItem()
+        item["title"] = title.strip() if title else ""
+        item["publish_time"] = publish_time if publish_time else ""
+        item["url"] = response.url
+        item["source"] = source if source else ""
+        item["content"] = content
+
+        yield item
+
+async def main():
+    start_time = time.time()
+    
+    process = CrawlerProcess(settings={
+        "CONCURRENCY": 8,
+        "DOWNLOAD_DELAY": 0.1,
+        "LOG_LEVEL": "ERROR",  # 减少日志输出以提高性能
+    })
+    await process.crawl(OfweekSimpleSpider)
+    
+    end_time = time.time()
+    execution_time = end_time - start_time
+    
+    print(f"Crawlo执行时间: {execution_time:.2f}秒")
+
+if __name__ == "__main__":
     asyncio.run(main())

tests/simple_follow_test.py

@@ -1,39 +1,39 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-Response.follow 方法简单测试
-"""
-import sys
-import os
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-# 直接测试 urljoin 方法
-from urllib.parse import urljoin
-
-
-def test_urljoin():
-    """测试 urljoin 方法"""
-    base_url = "https://example.com/test"
-    
-    # 测试绝对URL
-    absolute_url = urljoin(base_url, "https://other.com/page")
-    print(f"绝对URL: {absolute_url}")
-    assert absolute_url == "https://other.com/page"
-    
-    # 测试相对URL
-    relative_url = urljoin(base_url, "/relative/path")
-    print(f"相对URL: {relative_url}")
-    assert relative_url == "https://example.com/relative/path"
-    
-    # 测试复杂相对URL
-    complex_url = urljoin(base_url, "../other/path")
-    print(f"复杂相对URL: {complex_url}")
-    assert complex_url == "https://example.com/other/path"
-    
-    print("所有测试通过！")
-
-
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response.follow 方法简单测试
+"""
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+# 直接测试 urljoin 方法
+from urllib.parse import urljoin
+
+
+def test_urljoin():
+    """测试 urljoin 方法"""
+    base_url = "https://example.com/test"
+    
+    # 测试绝对URL
+    absolute_url = urljoin(base_url, "https://other.com/page")
+    print(f"绝对URL: {absolute_url}")
+    assert absolute_url == "https://other.com/page"
+    
+    # 测试相对URL
+    relative_url = urljoin(base_url, "/relative/path")
+    print(f"相对URL: {relative_url}")
+    assert relative_url == "https://example.com/relative/path"
+    
+    # 测试复杂相对URL
+    complex_url = urljoin(base_url, "../other/path")
+    print(f"复杂相对URL: {complex_url}")
+    assert complex_url == "https://example.com/other/path"
+    
+    print("所有测试通过！")
+
+
+if __name__ == '__main__':
     test_urljoin()