crawlo 1.4.4__py3-none-any.whl → 1.4.6__py3-none-any.whl

tests/ofweek_scrapy/ofweek_scrapy/spiders/ofweek_spider.py

@@ -0,0 +1,162 @@
+import scrapy
+from urllib.parse import urljoin
+
+class OfweekSpider(scrapy.Spider):
+    name = 'ofweek'
+    allowed_domains = ['ee.ofweek.com']
+    
+    def start_requests(self):
+        """生成初始请求"""
+        headers = {
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Pragma": "no-cache",
+            "Referer": "https://ee.ofweek.com/CATList-2800-8100-ee-2.html",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "same-origin",
+            "Sec-Fetch-User": "?1",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+        }
+        cookies = {
+            "__utmz": "57425525.1730117117.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)",
+            "Hm_lvt_abe9900db162c6d089cdbfd107db0f03": "1739244841",
+            "Hm_lvt_af50e2fc51af73da7720fb324b88a975": "1740100727",
+            "JSESSIONID": "FEA96D3B5FC31350B2285E711BF2A541",
+            "Hm_lvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477622",
+            "HMACCOUNT": "08DF0D235A291EAA",
+            "__utma": "57425525.2080994505.1730117117.1747970718.1757477622.50",
+            "__utmc": "57425525",
+            "__utmt": "1",
+            "__utmb": "57425525.2.10.1757477622",
+            "Hm_lpvt_28a416fcfc17063eb9c4f9bb1a1f5cda": "1757477628",
+            "index_burying_point": "c64d6c31e69d560efe319cc9f8be279f"
+        }
+
+        # 使用与Crawlo相同的页数进行公平测试
+        max_page = 50  # 原来是1851，现在改为50页进行测试
+        start_urls = []
+        for page in range(1, max_page + 1):
+            url = f'https://ee.ofweek.com/CATList-2800-8100-ee-{page}.html'
+            start_urls.append(url)
+
+        self.logger.info(f"生成了 {len(start_urls)} 个起始URL")
+
+        # 生成请求
+        for url in start_urls:
+            self.logger.info(f"添加起始URL: {url}")
+            try:
+                yield scrapy.Request(
+                    url=url,
+                    callback=self.parse,
+                    headers=headers,
+                    cookies=cookies,
+                    dont_filter=True
+                )
+            except Exception as e:
+                self.logger.error(f"创建请求失败: {url}, 错误: {e}")
+        
+        self.logger.info("start_requests方法执行完成")
+
+    def parse(self, response):
+        """解析列表页"""
+        self.logger.info(f'正在解析页面: {response.url}')
+
+        # 检查响应状态
+        if response.status != 200:
+            self.logger.warning(f"页面返回非200状态码: {response.status}, URL: {response.url}")
+            return
+
+        # ================== 数据提取 ==================
+        try:
+            rows = response.xpath('//div[@class="main_left"]/div[@class="list_model"]/div[@class="model_right model_right2"]')
+            self.logger.info(f"在页面 {response.url} 中找到 {len(rows)} 个条目")
+
+            for row in rows:
+                try:
+                    # 提取URL和标题
+                    url = row.xpath('./h3/a/@href').extract_first()
+                    title = row.xpath('./h3/a/text()').extract_first()
+
+                    # 容错处理
+                    if not url:
+                        self.logger.warning(f"条目缺少URL，跳过")
+                        continue
+
+                    if not title:
+                        self.logger.warning(f"条目缺少标题，跳过")
+                        continue
+
+                    # 确保 URL 是绝对路径
+                    absolute_url = response.urljoin(url)
+
+                    # 验证URL格式
+                    if not absolute_url.startswith(('http://', 'https://')):
+                        self.logger.warning(f"无效的URL格式，跳过: {absolute_url}")
+                        continue
+
+                    self.logger.info(f"提取到详情页链接: {absolute_url}, 标题: {title}")
+                    yield scrapy.Request(
+                        url=absolute_url,
+                        meta={
+                            "title": title.strip() if title else '',
+                            "parent_url": response.url
+                        },
+                        callback=self.parse_detail
+                    )
+                except Exception as e:
+                    self.logger.error(f"处理条目时出错: {e}")
+                    continue
+
+        except Exception as e:
+            self.logger.error(f"解析页面 {response.url} 时出错: {e}")
+
+    def parse_detail(self, response):
+        """解析详情页"""
+        self.logger.info(f'正在解析详情页: {response.url}')
+
+        # 检查响应状态
+        if response.status != 200:
+            self.logger.warning(f"详情页返回非200状态码: {response.status}, URL: {response.url}")
+            return
+
+        try:
+            title = response.meta.get('title', '')
+
+            # 提取内容，增加容错处理
+            content_elements = response.xpath('//div[@class="TRS_Editor"]|//*[@id="articleC"]')
+            if content_elements:
+                content = content_elements.xpath('.//text()').extract()
+                content = '\n'.join([text.strip() for text in content if text.strip()])
+            else:
+                content = ''
+                self.logger.warning(f"未找到内容区域: {response.url}")
+
+            # 提取发布时间
+            publish_time = response.xpath('//div[@class="time fl"]/text()').extract_first()
+            if publish_time:
+                publish_time = publish_time.strip()
+
+            source = response.xpath('//div[@class="source-name"]/text()').extract_first()
+
+            # 创建数据项
+            item = {
+                'title': title.strip() if title else '',
+                'publish_time': publish_time if publish_time else '',
+                'url': response.url,
+                'source': source if source else '',
+                'content': content
+            }
+
+            # 验证必要字段
+            if not item['title']:
+                self.logger.warning(f"详情页缺少标题: {response.url}")
+
+            self.logger.info(f"成功提取详情页数据: {item['title']}")
+            yield item
+
+        except Exception as e:
+            self.logger.error(f"解析详情页 {response.url} 时出错: {e}")

tests/ofweek_scrapy/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = ofweek_scrapy.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = ofweek_scrapy

tests/performance_comparison.py

@@ -79,7 +79,6 @@ class OfweekPerformanceSpider(Spider):
             yield Request(url=url, callback=self.parse, headers=headers, cookies=cookies)
 
     def parse(self, response):
-        from urllib.parse import urljoin
         rows = response.xpath("//div[@class=\"main_left\"]/div[@class=\"list_model\"]/div[@class=\"model_right model_right2\"]")
         
         for row in rows:
@@ -90,7 +89,7 @@ class OfweekPerformanceSpider(Spider):
                 if not url or not title:
                     continue
 
-                absolute_url = urljoin(response.url, url)
+                absolute_url = response.urljoin(url)
                 if not absolute_url.startswith(("http://", "https://")):
                     continue
 
@@ -171,11 +170,11 @@ def run_scrapy_test():
         # 运行Scrapy测试
         result = subprocess.run([
             'scrapy', 'runspider', 
-            '/Users/oscar/projects/Crawlo/tests/scrapy_comparison/ofweek_scrapy.py',
+            'D:/dowell/projects/Crawlo/tests/scrapy_comparison/ofweek_scrapy.py',
             '-s', 'CONCURRENT_REQUESTS=8',
             '-s', 'DOWNLOAD_DELAY=0.1',
             '-s', 'LOG_LEVEL=ERROR'
-        ], capture_output=True, text=True, timeout=300, cwd='/Users/oscar/projects/Crawlo')
+        ], capture_output=True, text=True, timeout=300, cwd='D:\dowell\projects\Crawlo')
         
         end_time = time.time()
         execution_time = end_time - start_time
@@ -205,7 +204,7 @@ def main():
     print("=== Crawlo vs Scrapy 性能对比测试 ===")
     
     # 创建测试目录
-    os.makedirs('/Users/oscar/projects/Crawlo/tests/scrapy_comparison', exist_ok=True)
+    os.makedirs(os.path.join('D:\dowell\projects\Crawlo', 'tests', 'scrapy_comparison'), exist_ok=True)
     
     # 运行测试
     crawlo_time, crawlo_pages = run_crawlo_test()

tests/simple_crawlo_test.py

@@ -62,7 +62,6 @@ class OfweekSimpleSpider(Spider):
             yield Request(url=url, callback=self.parse, headers=headers, cookies=cookies)
 
     def parse(self, response):
-        from urllib.parse import urljoin
         rows = response.xpath("//div[@class=\"main_left\"]/div[@class=\"list_model\"]/div[@class=\"model_right model_right2\"]")
         
         for row in rows:
@@ -73,7 +72,7 @@ class OfweekSimpleSpider(Spider):
                 if not url or not title:
                     continue
 
-                absolute_url = urljoin(response.url, url)
+                absolute_url = response.urljoin(url)
                 if not absolute_url.startswith(("http://", "https://")):
                     continue
 

tests/simple_follow_test.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Response.follow 方法简单测试
+"""
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+# 直接测试 urljoin 方法
+from urllib.parse import urljoin
+
+
+def test_urljoin():
+    """测试 urljoin 方法"""
+    base_url = "https://example.com/test"
+    
+    # 测试绝对URL
+    absolute_url = urljoin(base_url, "https://other.com/page")
+    print(f"绝对URL: {absolute_url}")
+    assert absolute_url == "https://other.com/page"
+    
+    # 测试相对URL
+    relative_url = urljoin(base_url, "/relative/path")
+    print(f"相对URL: {relative_url}")
+    assert relative_url == "https://example.com/relative/path"
+    
+    # 测试复杂相对URL
+    complex_url = urljoin(base_url, "../other/path")
+    print(f"复杂相对URL: {complex_url}")
+    assert complex_url == "https://example.com/other/path"
+    
+    print("所有测试通过！")
+
+
+if __name__ == '__main__':
+    test_urljoin()

tests/simple_response_selector_test.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+简化测试Response类中的选择器方法
+"""
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+# 直接导入Response类
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'crawlo', 'network'))
+from response import Response
+
+
+def test_response_selector_methods():
+    """测试Response类中的选择器方法"""
+    print("测试Response类中的选择器方法...")
+    print("=" * 50)
+    
+    # 创建测试HTML响应
+    html_content = """
+    <html>
+    <head>
+        <title>测试页面</title>
+    </head>
+    <body>
+        <div class="content">
+            <h1>主标题</h1>
+            <p class="intro">介绍段落</p>
+            <ul class="list">
+                <li>项目1</li>
+                <li>项目2</li>
+                <li>项目3</li>
+            </ul>
+            <a href="https://example.com" class="link">链接文本</a>
+            <img src="image.jpg" alt="图片描述" class="image">
+        </div>
+    </body>
+    </html>
+    """
+    
+    # 创建Response对象
+    response = Response(
+        url="https://example.com/test",
+        body=html_content.encode('utf-8'),
+        headers={"content-type": "text/html; charset=utf-8"}
+    )
+    
+    # 测试 extract_text (CSS选择器)
+    print("1. 测试 extract_text (CSS选择器):")
+    title_text = response.extract_text('title')
+    print(f"   标题文本: {title_text}")
+    
+    h1_text = response.extract_text('.content h1')
+    print(f"   H1文本: {h1_text}")
+    print()
+    
+    # 测试 extract_text (XPath选择器)
+    print("2. 测试 extract_text (XPath选择器):")
+    title_text_xpath = response.extract_text('//title')
+    print(f"   标题文本: {title_text_xpath}")
+    
+    h1_text_xpath = response.extract_text('//div[@class="content"]/h1')
+    print(f"   H1文本: {h1_text_xpath}")
+    print()
+    
+    # 测试 extract_texts
+    print("3. 测试 extract_texts:")
+    li_texts = response.extract_texts('.list li')
+    print(f"   列表项文本: {li_texts}")
+    print()
+    
+    # 测试 extract_attr
+    print("4. 测试 extract_attr:")
+    link_href = response.extract_attr('.link', 'href')
+    print(f"   链接href: {link_href}")
+    
+    img_alt = response.extract_attr('.image', 'alt')
+    print(f"   图片alt: {img_alt}")
+    print()
+    
+    # 测试 extract_attrs
+    print("5. 测试 extract_attrs:")
+    all_links = response.extract_attrs('a', 'href')
+    print(f"   所有链接href: {all_links}")
+    
+    all_images = response.extract_attrs('img', 'src')
+    print(f"   所有图片src: {all_images}")
+    print()
+    
+    print("所有测试完成！")
+
+
+if __name__ == '__main__':
+    test_response_selector_methods()

tests/simple_selector_helper_test.py

@@ -0,0 +1,155 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+简化选择器辅助工具测试
+"""
+from parsel import Selector
+
+
+# 直接复制工具函数用于测试
+def extract_text(elements, join_str=" "):
+    """
+    从元素列表中提取文本并拼接
+    """
+    texts = []
+    for element in elements:
+        if hasattr(element, 'xpath'):
+            element_texts = element.xpath('.//text()').getall()
+        else:
+            element_texts = [str(element)]
+        for text in element_texts:
+            cleaned = text.strip()
+            if cleaned:
+                texts.append(cleaned)
+    return join_str.join(texts)
+
+
+def extract_texts(elements, join_str=" "):
+    """
+    从元素列表中提取多个文本列表
+    """
+    result = []
+    for element in elements:
+        if hasattr(element, 'xpath'):
+            texts = element.xpath('.//text()').getall()
+        else:
+            texts = [str(element)]
+        clean_texts = [text.strip() for text in texts if text.strip()]
+        if clean_texts:
+            result.append(join_str.join(clean_texts))
+    return result
+
+
+def extract_attr(elements, attr_name, default=None):
+    """
+    从元素列表中提取单个元素的属性值
+    """
+    if hasattr(elements, 'attrib'):
+        return elements.attrib.get(attr_name, default)
+    elif len(elements) > 0 and hasattr(elements[0], 'attrib'):
+        return elements[0].attrib.get(attr_name, default)
+    return default
+
+
+def extract_attrs(elements, attr_name):
+    """
+    从元素列表中提取多个元素的属性值列表
+    """
+    result = []
+    for element in elements:
+        if hasattr(element, 'attrib'):
+            attr_value = element.attrib.get(attr_name)
+            if attr_value is not None:
+                result.append(attr_value)
+    return result
+
+
+def is_xpath(query):
+    """
+    判断查询语句是否为XPath
+    """
+    return query.startswith(('/', '//', './'))
+
+
+def test_selector_helper():
+    """测试选择器辅助工具"""
+    print("测试选择器辅助工具...")
+    print("=" * 50)
+    
+    # 创建测试HTML
+    html_content = """
+    <html>
+    <head>
+        <title>测试页面</title>
+    </head>
+    <body>
+        <div class="content">
+            <h1>主标题</h1>
+            <p class="intro">介绍段落</p>
+            <ul class="list">
+                <li>项目1</li>
+                <li>项目2</li>
+                <li>项目3</li>
+            </ul>
+            <a href="https://example.com" class="link">链接文本</a>
+            <img src="image.jpg" alt="图片描述" class="image">
+        </div>
+    </body>
+    </html>
+    """
+    
+    selector = Selector(text=html_content)
+    
+    # 测试 is_xpath
+    print("1. 测试 is_xpath:")
+    print(f"   '/' 开头: {is_xpath('/')}")
+    print(f"   '//' 开头: {is_xpath('//title')}")
+    print(f"   './' 开头: {is_xpath('./div')}")
+    print(f"   'title' 开头: {is_xpath('title')}")
+    print()
+    
+    # 测试 extract_text
+    print("2. 测试 extract_text:")
+    title_elements = selector.css('title')
+    title_text = extract_text(title_elements)
+    print(f"   标题文本: {title_text}")
+    
+    h1_elements = selector.css('.content h1')
+    h1_text = extract_text(h1_elements)
+    print(f"   H1文本: {h1_text}")
+    print()
+    
+    # 测试 extract_texts
+    print("3. 测试 extract_texts:")
+    li_elements = selector.css('.list li')
+    li_texts = extract_texts(li_elements)
+    print(f"   列表项文本: {li_texts}")
+    print()
+    
+    # 测试 extract_attr
+    print("4. 测试 extract_attr:")
+    link_elements = selector.css('.link')
+    link_href = extract_attr(link_elements, 'href')
+    print(f"   链接href: {link_href}")
+    
+    img_elements = selector.css('.image')
+    img_alt = extract_attr(img_elements, 'alt')
+    print(f"   图片alt: {img_alt}")
+    print()
+    
+    # 测试 extract_attrs
+    print("5. 测试 extract_attrs:")
+    all_links = selector.css('a')
+    all_hrefs = extract_attrs(all_links, 'href')
+    print(f"   所有链接href: {all_hrefs}")
+    
+    all_images = selector.css('img')
+    all_srcs = extract_attrs(all_images, 'src')
+    print(f"   所有图片src: {all_srcs}")
+    print()
+    
+    print("所有测试完成！")
+
+
+if __name__ == '__main__':
+    test_selector_helper()