crawlo 1.0.4__tar.gz → 1.0.6__tar.gz

{crawlo-1.0.4 → crawlo-1.0.6}/LICENSE

@@ -1,23 +1,23 @@
-MIT License
-
-Modifications:
-
-Copyright (c) 2020 crawl-coder <2251018029@qq.com>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+MIT License
+
+Modifications:
+
+Copyright (c) 2020 crawl-coder <2251018029@qq.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.

{crawlo-1.0.4 → crawlo-1.0.6}/MANIFEST.in

@@ -1,17 +1,17 @@
-include README.md
-include LICENSE
-include requirements.txt  # 如果根目录有全局requirements.txt
-include VERSION           # 如果根目录有全局VERSION文件
-
-# 包内文件包含
-recursive-include crawlo/utils/js *
-recursive-include crawlo/templates *
-
-# 测试文件（如果需要在分发包中包含测试）
-recursive-include tests *
-
-# 排除项
-global-exclude __pycache__ *.py[cod] .DS_Store *.so
-global-exclude *.bak *.swp *.orig *.rej
-prune samples              # 排除示例目录
+include README.md
+include LICENSE
+include requirements.txt  # 如果根目录有全局requirements.txt
+include VERSION           # 如果根目录有全局VERSION文件
+
+# 包内文件包含
+recursive-include crawlo/utils/js *
+recursive-include crawlo/templates *
+
+# 测试文件（如果需要在分发包中包含测试）
+recursive-include tests *
+
+# 排除项
+global-exclude __pycache__ *.py[cod] .DS_Store *.so
+global-exclude *.bak *.swp *.orig *.rej
+prune samples              # 排除示例目录
 prune docs                 # 排除文档目录

{crawlo-1.0.4/crawlo.egg-info → crawlo-1.0.6}/PKG-INFO

@@ -1,48 +1,49 @@
-Metadata-Version: 2.4
-Name: crawlo
-Version: 1.0.4
-Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
-Home-page: https://github.com/crawl-coder/Crawlo.git
-Author: crawl-coder
-Author-email: crawlo@qq.com
-License: MIT
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.6
-Description-Content-Type: text/markdown
-Requires-Dist: aiohttp>=3.12.14
-Requires-Dist: aiomysql>=0.2.0
-Requires-Dist: aioredis>=2.0.1
-Requires-Dist: asyncmy>=0.2.10
-Requires-Dist: cssselect>=1.2.0
-Requires-Dist: dateparser>=1.2.2
-Requires-Dist: httpx[http2]>=0.27.0
-Requires-Dist: lxml>=5.2.1
-Requires-Dist: motor>=3.7.0
-Requires-Dist: parsel>=1.9.1
-Requires-Dist: pydantic>=2.11.7
-Requires-Dist: pymongo>=4.11
-Requires-Dist: PyMySQL>=1.1.1
-Requires-Dist: python-dateutil>=2.9.0.post0
-Requires-Dist: redis>=6.2.0
-Requires-Dist: requests>=2.32.4
-Requires-Dist: six>=1.17.0
-Requires-Dist: ujson>=5.9.0
-Requires-Dist: urllib3>=2.5.0
-Requires-Dist: w3lib>=2.1.2
-Provides-Extra: render
-Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
-Requires-Dist: playwright; extra == "render"
-Requires-Dist: selenium>=3.141.0; extra == "render"
-Provides-Extra: all
-Requires-Dist: bitarray>=1.5.3; extra == "all"
-Requires-Dist: PyExecJS>=1.5.1; extra == "all"
-Requires-Dist: pymongo>=3.10.1; extra == "all"
-Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
-Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
-Requires-Dist: playwright; extra == "all"
-Requires-Dist: selenium>=3.141.0; extra == "all"
-
-# Crawlo
-Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。
+Metadata-Version: 2.4
+Name: crawlo
+Version: 1.0.6
+Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
+Home-page: https://github.com/crawl-coder/Crawlo.git
+Author: crawl-coder
+Author-email: crawlo@qq.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp>=3.12.14
+Requires-Dist: aiomysql>=0.2.0
+Requires-Dist: aioredis>=2.0.1
+Requires-Dist: asyncmy>=0.2.10
+Requires-Dist: cssselect>=1.2.0
+Requires-Dist: dateparser>=1.2.2
+Requires-Dist: httpx[http2]>=0.27.0
+Requires-Dist: curl-cffi>=0.13.0
+Requires-Dist: lxml>=5.2.1
+Requires-Dist: motor>=3.7.0
+Requires-Dist: parsel>=1.9.1
+Requires-Dist: pydantic>=2.11.7
+Requires-Dist: pymongo>=4.11
+Requires-Dist: PyMySQL>=1.1.1
+Requires-Dist: python-dateutil>=2.9.0.post0
+Requires-Dist: redis>=6.2.0
+Requires-Dist: requests>=2.32.4
+Requires-Dist: six>=1.17.0
+Requires-Dist: ujson>=5.9.0
+Requires-Dist: urllib3>=2.5.0
+Requires-Dist: w3lib>=2.1.2
+Provides-Extra: render
+Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
+Requires-Dist: playwright; extra == "render"
+Requires-Dist: selenium>=3.141.0; extra == "render"
+Provides-Extra: all
+Requires-Dist: bitarray>=1.5.3; extra == "all"
+Requires-Dist: PyExecJS>=1.5.1; extra == "all"
+Requires-Dist: pymongo>=3.10.1; extra == "all"
+Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
+Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: selenium>=3.141.0; extra == "all"
+
+# Crawlo
+Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。

{crawlo-1.0.4 → crawlo-1.0.6}/README.md

@@ -1,2 +1,2 @@
-# Crawlo
-Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。
+# Crawlo
+Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取与数据管道。

crawlo-1.0.6/crawlo/__init__.py

@@ -0,0 +1,25 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo - 一个异步爬虫框架
+"""
+from crawlo.spider import Spider
+from crawlo.items.items import Item
+from crawlo.network.request import Request
+from crawlo.network.response import Response
+from crawlo.downloader import DownloaderBase
+from crawlo.middleware import BaseMiddleware
+
+# 版本号
+from crawlo.__version__ import __version__
+
+# 可选：定义对外暴露的接口
+__all__ = [
+    'Spider',
+    'Item',
+    'Request',
+    'Response',
+    'DownloaderBase',
+    'BaseMiddleware',
+    '__version__',
+]

crawlo-1.0.6/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.6"

crawlo-1.0.6/crawlo/cli.py

@@ -0,0 +1,41 @@
+# crawlo/cli.py
+# !/usr/bin/python
+# -*- coding: UTF-8 -*-
+import sys
+import argparse
+from crawlo.commands import get_commands
+
+
+def main():
+    # 获取所有可用命令
+    commands = get_commands()
+
+    parser = argparse.ArgumentParser(
+        description="Crawlo: A lightweight web crawler framework.",
+        usage="crawlo <command> [options]"
+    )
+    parser.add_argument('command', help='Available commands: ' + ', '.join(commands.keys()))
+    # 注意：这里不添加具体参数，由子命令解析
+
+    # 只解析命令
+    args, unknown = parser.parse_known_args()
+
+    if args.command not in commands:
+        print(f"Unknown command: {args.command}")
+        print(f"Available commands: {', '.join(commands.keys())}")
+        sys.exit(1)
+
+    # 动态导入并执行命令
+    try:
+        module = __import__(commands[args.command], fromlist=['main'])
+        sys.exit(module.main(unknown))
+    except ImportError as e:
+        print(f"Failed to load command '{args.command}': {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Command '{args.command}' failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

crawlo-1.0.6/crawlo/commands/__init__.py

@@ -0,0 +1,10 @@
+# crawlo/commands/__init__.py
+# 定义可用的命令
+_commands = {
+    'startproject': 'crawlo.commands.startproject',
+    'genspider': 'crawlo.commands.genspider',
+    'run': 'crawlo.commands.run',
+}
+
+def get_commands():
+    return _commands

crawlo-1.0.6/crawlo/commands/genspider.py

@@ -0,0 +1,111 @@
+import os
+import sys
+from pathlib import Path
+import configparser
+import importlib
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def main(args):
+    if len(args) < 2:
+        print("Usage: crawlo genspider <spider_name> <domain>")
+        return 1
+
+    spider_name = args[0]
+    domain = args[1]
+
+    # 查找项目根目录
+    project_root = None
+    current = Path.cwd()
+    while True:
+        cfg_file = current / 'crawlo.cfg'
+        if cfg_file.exists():
+            project_root = current
+            break
+        parent = current.parent
+        if parent == current:
+            break
+        current = parent
+
+    if not project_root:
+        print("Error: Not a crawlo project. crawlo.cfg not found.")
+        return 1
+
+    # 将项目根目录加入 sys.path
+    if str(project_root) not in sys.path:
+        sys.path.insert(0, str(project_root))
+
+    # 从 crawlo.cfg 读取 settings 模块，获取项目包名
+    config = configparser.ConfigParser()
+    try:
+        config.read(cfg_file, encoding='utf-8')
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]  # e.g., myproject.settings -> myproject
+    except Exception as e:
+        print(f"Error reading crawlo.cfg: {e}")
+        return 1
+
+    # 确定 items 模块的路径
+    items_module_path = f"{project_package}.items"
+
+    # 尝试导入 items 模块
+    try:
+        items_module = importlib.import_module(items_module_path)
+        # 获取模块中所有大写开头的类
+        item_classes = [cls for cls in items_module.__dict__.values()
+                        if isinstance(cls, type) and cls.__name__.isupper()]
+
+        # 如果找到了类，使用第一个作为默认
+        if item_classes:
+            default_item_class = item_classes[0].__name__
+        else:
+            default_item_class = "ExampleItem"  # 回退到示例
+    except ImportError as e:
+        print(f"Error importing items module '{items_module_path}': {e}")
+        default_item_class = "ExampleItem"
+
+    # 创建爬虫文件
+    spiders_dir = project_root / project_package / 'spiders'
+    if not spiders_dir.exists():
+        spiders_dir.mkdir(parents=True)
+
+    spider_file = spiders_dir / f'{spider_name}.py'
+    if spider_file.exists():
+        print(f"Error: Spider '{spider_name}' already exists.")
+        return 1
+
+    # ✅ 修正模板路径
+    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
+
+    if not tmpl_path.exists():
+        print(f"Error: Template file not found at {tmpl_path}")
+        return 1
+
+    # ✅ 生成正确的类名
+    class_name = f"{spider_name.capitalize()}Spider"
+
+    context = {
+        'spider_name': spider_name,
+        'domain': domain,
+        'project_name': project_package,
+        'item_class': default_item_class,
+        'class_name': class_name  # ✅ 添加处理好的类名
+    }
+
+    content = _render_template(tmpl_path, context)
+
+    with open(spider_file, 'w', encoding='utf-8') as f:
+        f.write(content)
+
+    print(f"Spider '{spider_name}' created in {spider_file}")
+    return 0

crawlo-1.0.6/crawlo/commands/run.py

@@ -0,0 +1,149 @@
+# crawlo/commands/run.py
+import asyncio
+import importlib
+import sys
+from pathlib import Path
+import configparser
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.project import get_settings
+from crawlo.utils.log import get_logger
+from crawlo.utils.spider_loader import SpiderLoader
+
+logger = get_logger(__name__)
+
+
+def main(args):
+    """
+    运行指定爬虫的主函数
+    用法: crawlo run <spider_name>
+    """
+    if len(args) < 1:
+        print("Usage: crawlo run <spider_name>")
+        print("Example: crawlo run baidu")
+        return 1
+
+    spider_name = args[0]
+
+    try:
+        # 1. 获取项目根目录
+        project_root = get_settings()
+
+        # 将项目根目录添加到 Python 路径
+        if str(project_root) not in sys.path:
+            sys.path.insert(0, str(project_root))
+
+        # 2. 读取配置文件获取项目包名
+        cfg_file = project_root / 'crawlo.cfg'
+        if not cfg_file.exists():
+            print(f"❌ Error: crawlo.cfg not found in {project_root}")
+            return 1
+
+        config = configparser.ConfigParser()
+        config.read(cfg_file, encoding='utf-8')
+
+        if not config.has_section('settings') or not config.has_option('settings', 'default'):
+            print("❌ Error: Missing [settings] section or 'default' option in crawlo.cfg")
+            return 1
+
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]
+
+        # 3. 查找并加载指定名称的 Spider
+        spider_class = find_spider_by_name(project_package, spider_name)
+        if spider_class is None:
+            return 1
+
+        # 4. 创建 CrawlerProcess 并运行单个爬虫
+        settings = get_settings()
+        process = CrawlerProcess(settings)
+
+        print(f"🚀 Starting spider: {spider_class.name}")
+        print(f"📁 Project: {project_package}")
+        print(f"🕷️  Class: {spider_class.__name__}")
+        print("-" * 50)
+
+        # 运行单个爬虫
+        asyncio.run(process.crawl(spider_class))
+
+        print("-" * 50)
+        print("✅ Spider completed successfully!")
+        return 0
+
+    except Exception as e:
+        print(f"❌ Error running spider: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+
+def find_spider_by_name(project_package: str, target_spider_name: str):
+    """使用 SpiderLoader 查找爬虫"""
+    loader = SpiderLoader(project_package)
+    spider_class = loader.load(target_spider_name)
+
+    if spider_class is None:
+        print(f"❌ Error: Spider with name '{target_spider_name}' not found")
+        print("💡 Available spiders:")
+        available_spiders = loader.list()
+        for spider_name in available_spiders:
+            print(f"   - {spider_name}")
+        return None
+
+    return spider_class
+
+
+def list_available_spiders(project_package: str):
+    """
+    列出所有可用的爬虫
+    """
+    spiders_dir = Path.cwd() / project_package / 'spiders'
+    if not spiders_dir.exists():
+        print("   No spiders directory found")
+        return
+
+    spider_count = 0
+    for py_file in spiders_dir.glob("*.py"):
+        if py_file.name.startswith('_'):
+            continue
+
+        module_name = py_file.stem
+        spider_module_path = f"{project_package}.spiders.{module_name}"
+
+        try:
+            module = importlib.import_module(spider_module_path)
+        except ImportError:
+            continue
+
+        # 查找模块中所有 Spider 子类
+        from crawlo.spider import Spider
+        for attr_name in dir(module):
+            attr_value = getattr(module, attr_name)
+            if (isinstance(attr_value, type) and
+                    issubclass(attr_value, Spider) and
+                    attr_value != Spider and
+                    hasattr(attr_value, 'name')):
+                print(f"   - {attr_value.name} (class: {attr_value.__name__}, module: {module_name})")
+                spider_count += 1
+
+    if spider_count == 0:
+        print("   No spiders found")
+
+
+def run_spider_by_name(spider_name: str, project_root: Path = None):
+    """
+    直接在代码中通过 spider name 运行爬虫
+    """
+    if project_root:
+        if str(project_root) not in sys.path:
+            sys.path.insert(0, str(project_root))
+
+    args = [spider_name]
+    return main(args)
+
+
+if __name__ == '__main__':
+    # 允许直接运行: python -m crawlo.commands.run <spider_name>
+    import sys
+
+    sys.exit(main(sys.argv[1:]))

crawlo-1.0.6/crawlo/commands/startproject.py

@@ -0,0 +1,101 @@
+# crawlo/commands/startproject.py
+import os
+import shutil
+from pathlib import Path
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def _copytree_with_templates(src, dst, context):
+    """
+    递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
+    """
+    src_path = Path(src)
+    dst_path = Path(dst)
+    dst_path.mkdir(parents=True, exist_ok=True)
+
+    for item in src_path.rglob('*'):
+        rel_path = item.relative_to(src_path)
+        dst_item = dst_path / rel_path
+
+        if item.is_dir():
+            # 创建目标目录
+            dst_item.mkdir(parents=True, exist_ok=True)
+        else:
+            if item.suffix == '.tmpl':
+                # 渲染模板文件，并去掉 .tmpl 后缀
+                rendered_content = _render_template(item, context)
+                final_dst = dst_item.with_suffix('') # 去掉 .tmpl
+                final_dst.parent.mkdir(parents=True, exist_ok=True) # 确保父目录存在
+                with open(final_dst, 'w', encoding='utf-8') as f:
+                    f.write(rendered_content)
+            else:
+                # 普通文件，直接复制
+                shutil.copy2(item, dst_item)
+
+
+def main(args):
+    if len(args) != 1:
+        print("Usage: crawlo startproject <project_name>")
+        return 1
+
+    project_name = args[0]
+    project_dir = Path(project_name)
+
+    if project_dir.exists():
+        print(f"Error: Directory '{project_dir}' already exists.")
+        return 1
+
+    context = {'project_name': project_name}
+    template_dir = TEMPLATES_DIR / 'project'
+
+    try:
+        # 1. 创建项目根目录
+        project_dir.mkdir()
+
+        # 2. 处理 crawlo.cfg.tmpl：单独渲染并写入项目根目录
+        cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'  # ✅ 使用 templates/ 目录下的模板
+        if cfg_template.exists():
+            cfg_content = _render_template(cfg_template, context)
+            (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
+        else:
+            print("Warning: crawlo.cfg.tmpl not found in templates.")
+
+        # 3. 复制所有其他模板文件到项目包内 (project_dir / project_name)
+        package_dir = project_dir / project_name
+        # 这会复制 __init__.py.tmpl, items.py.tmpl, settings.py.tmpl, spiders/ 等
+        # 并将它们渲染为 .py 文件
+        _copytree_with_templates(template_dir, package_dir, context)
+
+        # 4. 创建 logs 目录
+        (project_dir / 'logs').mkdir(exist_ok=True)
+
+        print(f"""
+        ✔ 项目 '{project_name}' 创建成功！
+
+        进入项目目录:
+            cd {project_name}
+
+        创建一个爬虫:
+            crawlo genspider example example.com
+
+        运行爬虫:
+            crawlo run example
+        """)
+        return 0
+
+    except Exception as e:
+        print(f"Error creating project: {e}")
+        # 如果出错，尝试清理已创建的目录
+        if project_dir.exists():
+            shutil.rmtree(project_dir, ignore_errors=True)
+        return 1

{crawlo-1.0.4 → crawlo-1.0.6}/crawlo/core/__init__.py

@@ -1,2 +1,2 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-