crawlo 1.0.0__py3-none-any.whl

crawlo/templates/project_template/setting.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+"""爬虫配置文件"""
+# import os
+# import sys
+#
+# # MYSQL
+# MYSQL_IP = "localhost"
+# MYSQL_PORT = 3306
+# MYSQL_DB = ""
+# MYSQL_USER_NAME = ""
+# MYSQL_USER_PASS = ""
+#
+# # MONGODB
+# MONGO_IP = "localhost"
+# MONGO_PORT = 27017
+# MONGO_DB = ""
+# MONGO_USER_NAME = ""
+# MONGO_USER_PASS = ""
+# MONGO_URL = "
+#
+# # REDIS
+# # ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"]
+# REDISDB_IP_PORTS = "localhost:6379"
+# REDISDB_USER_PASS = ""
+# REDISDB_DB = 0
+# # 连接redis时携带的其他参数，如ssl=True
+# REDISDB_KWARGS = dict()
+# # 适用于redis哨兵模式
+# REDISDB_SERVICE_NAME = ""
+#
+# # 数据入库的pipeline，可自定义，默认MysqlPipeline
+# ITEM_PIPELINES = [
+#     "feapder.pipelines.mysql_pipeline.MysqlPipeline",
+#     # "feapder.pipelines.mongo_pipeline.MongoPipeline",
+#     # "feapder.pipelines.console_pipeline.ConsolePipeline",
+# ]
+# EXPORT_DATA_MAX_FAILED_TIMES = 10  # 导出数据时最大的失败次数，包括保存和更新，超过这个次数报警
+# EXPORT_DATA_MAX_RETRY_TIMES = 10  # 导出数据时最大的重试次数，包括保存和更新，超过这个次数则放弃重试
+#
+# # 爬虫相关
+# # COLLECTOR
+# COLLECTOR_TASK_COUNT = 32  # 每次获取任务数量，追求速度推荐32
+#
+# # SPIDER
+# SPIDER_THREAD_COUNT = 1  # 爬虫并发数，追求速度推荐32
+# # 下载时间间隔 单位秒。 支持随机 如 SPIDER_SLEEP_TIME = [2, 5] 则间隔为 2~5秒之间的随机数，包含2和5
+# SPIDER_SLEEP_TIME = 0
+# SPIDER_MAX_RETRY_TIMES = 10  # 每个请求最大重试次数
+# KEEP_ALIVE = False  # 爬虫是否常驻
+
+# 下载
+# DOWNLOADER = "feapder.network.downloader.RequestsDownloader"  # 请求下载器
+# SESSION_DOWNLOADER = "feapder.network.downloader.RequestsSessionDownloader"
+# RENDER_DOWNLOADER = "feapder.network.downloader.SeleniumDownloader"  # 渲染下载器
+# # RENDER_DOWNLOADER="feapder.network.downloader.PlaywrightDownloader"
+# MAKE_ABSOLUTE_LINKS = True  # 自动转成绝对连接
+
+# # 浏览器渲染
+# WEBDRIVER = dict(
+#     pool_size=1,  # 浏览器的数量
+#     load_images=True,  # 是否加载图片
+#     user_agent=None,  # 字符串 或 无参函数，返回值为user_agent
+#     proxy=None,  # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址
+#     headless=False,  # 是否为无头浏览器
+#     driver_type="CHROME",  # CHROME、EDGE、PHANTOMJS、FIREFOX
+#     timeout=30,  # 请求超时时间
+#     window_size=(1024, 800),  # 窗口大小
+#     executable_path=None,  # 浏览器路径，默认为默认路径
+#     render_time=0,  # 渲染时长，即打开网页等待指定时间后再获取源码
+#     custom_argument=[
+#         "--ignore-certificate-errors",
+#         "--disable-blink-features=AutomationControlled",
+#     ],  # 自定义浏览器渲染参数
+#     xhr_url_regexes=None,  # 拦截xhr接口，支持正则，数组类型
+#     auto_install_driver=True,  # 自动下载浏览器驱动 支持chrome 和 firefox
+#     download_path=None,  # 下载文件的路径
+#     use_stealth_js=False,  # 使用stealth.min.js隐藏浏览器特征
+# )
+#
+# PLAYWRIGHT = dict(
+#     user_agent=None,  # 字符串 或 无参函数，返回值为user_agent
+#     proxy=None,  # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址
+#     headless=False,  # 是否为无头浏览器
+#     driver_type="chromium",  # chromium、firefox、webkit
+#     timeout=30,  # 请求超时时间
+#     window_size=(1024, 800),  # 窗口大小
+#     executable_path=None,  # 浏览器路径，默认为默认路径
+#     download_path=None,  # 下载文件的路径
+#     render_time=0,  # 渲染时长，即打开网页等待指定时间后再获取源码
+#     wait_until="networkidle",  # 等待页面加载完成的事件,可选值："commit", "domcontentloaded", "load", "networkidle"
+#     use_stealth_js=False,  # 使用stealth.min.js隐藏浏览器特征
+#     page_on_event_callback=None,  # page.on() 事件的回调 如 page_on_event_callback={"dialog": lambda dialog: dialog.accept()}
+#     storage_state_path=None,  # 保存浏览器状态的路径
+#     url_regexes=None,  # 拦截接口，支持正则，数组类型
+#     save_all=False,  # 是否保存所有拦截的接口, 配合url_regexes使用，为False时只保存最后一次拦截的接口
+# )
+#
+# # 爬虫启动时，重新抓取失败的requests
+# RETRY_FAILED_REQUESTS = False
+# # 爬虫启动时，重新入库失败的item
+# RETRY_FAILED_ITEMS = False
+# # 保存失败的request
+# SAVE_FAILED_REQUEST = True
+# # request防丢机制。（指定的REQUEST_LOST_TIMEOUT时间内request还没做完，会重新下发 重做）
+# REQUEST_LOST_TIMEOUT = 600  # 10分钟
+# # request网络请求超时时间
+# REQUEST_TIMEOUT = 22  # 等待服务器响应的超时时间，浮点数，或(connect timeout, read timeout)元组
+# # item在内存队列中最大缓存数量
+# ITEM_MAX_CACHED_COUNT = 5000
+# # item每批入库的最大数量
+# ITEM_UPLOAD_BATCH_MAX_SIZE = 1000
+# # item入库时间间隔
+# ITEM_UPLOAD_INTERVAL = 1
+# # 内存任务队列最大缓存的任务数，默认不限制；仅对AirSpider有效。
+# TASK_MAX_CACHED_SIZE = 0
+#
+# # 下载缓存 利用redis缓存，但由于内存大小限制，所以建议仅供开发调试代码时使用，防止每次debug都需要网络请求
+# RESPONSE_CACHED_ENABLE = False  # 是否启用下载缓存 成本高的数据或容易变需求的数据，建议设置为True
+# RESPONSE_CACHED_EXPIRE_TIME = 3600  # 缓存时间 秒
+# RESPONSE_CACHED_USED = False  # 是否使用缓存 补采数据时可设置为True
+#
+# # 设置代理
+# PROXY_EXTRACT_API = None  # 代理提取API ，返回的代理分割符为\r\n
+# PROXY_ENABLE = True
+# PROXY_MAX_FAILED_TIMES = 5  # 代理最大失败次数，超过则不使用，自动删除
+# PROXY_POOL = "feapder.network.proxy_pool.ProxyPool"  # 代理池
+#
+# # 随机headers
+# RANDOM_HEADERS = True
+# # UserAgent类型 支持 'chrome', 'opera', 'firefox', 'internetexplorer', 'safari'，'mobile' 若不指定则随机类型
+# USER_AGENT_TYPE = "chrome"
+# # 默认使用的浏览器头
+# DEFAULT_USERAGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36"
+# # requests 使用session
+# USE_SESSION = False
+#
+# # 去重
+# ITEM_FILTER_ENABLE = False  # item 去重
+# REQUEST_FILTER_ENABLE = False  # request 去重
+# ITEM_FILTER_SETTING = dict(
+#     filter_type=1  # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、轻量去重（LiteFilter）= 4
+# )
+# REQUEST_FILTER_SETTING = dict(
+#     filter_type=3,  # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、 轻量去重（LiteFilter）= 4
+#     expire_time=2592000,  # 过期时间1个月
+# )
+#
+# # 报警 支持钉钉、飞书、企业微信、邮件
+# # 钉钉报警
+# DINGDING_WARNING_URL = ""  # 钉钉机器人api
+# DINGDING_WARNING_PHONE = ""  # 被@的群成员手机号，支持列表，可指定多个。
+# DINGDING_WARNING_USER_ID = ""  # 被@的群成员userId，支持列表，可指定多个
+# DINGDING_WARNING_ALL = False  # 是否提示所有人， 默认为False
+# DINGDING_WARNING_SECRET = None  # 加签密钥
+# # 飞书报警
+# # https://open.feishu.cn/document/ukTMukTMukTM/ucTM5YjL3ETO24yNxkjN#e1cdee9f
+# FEISHU_WARNING_URL = ""  # 飞书机器人api
+# FEISHU_WARNING_USER = None  # 报警人 {"open_id":"ou_xxxxx", "name":"xxxx"} 或 [{"open_id":"ou_xxxxx", "name":"xxxx"}]
+# FEISHU_WARNING_ALL = False  # 是否提示所有人， 默认为False
+# # 邮件报警
+# EMAIL_SENDER = ""  # 发件人
+# EMAIL_PASSWORD = ""  # 授权码
+# EMAIL_RECEIVER = ""  # 收件人 支持列表，可指定多个
+# EMAIL_SMTPSERVER = "smtp.163.com"  # 邮件服务器 默认为163邮箱
+# # 企业微信报警
+# WECHAT_WARNING_URL = ""  # 企业微信机器人api
+# WECHAT_WARNING_PHONE = ""  # 报警人 将会在群内@此人, 支持列表，可指定多人
+# WECHAT_WARNING_ALL = False  # 是否提示所有人， 默认为False
+# # 时间间隔
+# WARNING_INTERVAL = 3600  # 相同报警的报警时间间隔，防止刷屏; 0表示不去重
+# WARNING_LEVEL = "DEBUG"  # 报警级别， DEBUG / INFO / ERROR
+# WARNING_FAILED_COUNT = 1000  # 任务失败数 超过WARNING_FAILED_COUNT则报警
+#
+# LOG_NAME = os.path.basename(os.getcwd())
+# LOG_PATH = "log/%s.log" % LOG_NAME  # log存储路径
+# LOG_LEVEL = "DEBUG"
+# LOG_COLOR = True  # 是否带有颜色
+# LOG_IS_WRITE_TO_CONSOLE = True  # 是否打印到控制台
+# LOG_IS_WRITE_TO_FILE = False  # 是否写文件
+# LOG_MODE = "w"  # 写文件的模式
+# LOG_MAX_BYTES = 10 * 1024 * 1024  # 每个日志文件的最大字节数
+# LOG_BACKUP_COUNT = 20  # 日志文件保留数量
+# LOG_ENCODING = "utf8"  # 日志文件编码
+# OTHERS_LOG_LEVAL = "ERROR"  # 第三方库的log等级
+#
+# # 切换工作路径为当前项目路径
+# project_path = os.path.abspath(os.path.dirname(__file__))
+# os.chdir(project_path)  # 切换工作路经
+# sys.path.insert(0, project_path)
+# print("当前工作路径为 " + os.getcwd())

crawlo/templates/spider_template.tmpl

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+Created on {DATE}
+---------
+@summary:
+---------
+@author: {USER}
+"""
+
+import crawlo
+
+
+class ${spider_name}(crawlo.Spider):
+    # 自定义数据库，若项目中有setting.py文件，此自定义可删除
+    __custom_setting__ = dict(
+        REDISDB_IP_PORTS="localhost:6379", REDISDB_USER_PASS="", REDISDB_DB=0
+    )
+
+    def start_requests(self):
+        yield feapder.Request("https://spidertools.cn")
+
+    def parse(self, request, response):
+        # 提取网站title
+        print(response.xpath("//title/text()").extract_first())
+        # 提取网站描述
+        print(response.xpath("//meta[@name='description']/@content").extract_first())
+        print("网站地址: ", response.url)
+
+
+if __name__ == "__main__":
+    ${spider_name}(redis_key="xxx:xxx").start()

crawlo/utils/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 13:57
+# @Author  :   oscar
+# @Desc    :   None
+"""

crawlo/utils/date_tools.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-05-17 10:20
+# @Author  :   crawl-coder
+# @Desc    :   时间工具
+"""
+from datetime import datetime
+
+
+def now():
+    return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+
+def date_delta(start, end):
+    start = datetime.strptime(start, '%Y-%m-%d %H:%M:%S')
+    end = datetime.strptime(end, '%Y-%m-%d %H:%M:%S')
+    delta = end - start
+    seconds = delta.total_seconds()
+    return int(seconds)

crawlo/utils/func_tools.py

@@ -0,0 +1,22 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Callable
+from inspect import isgenerator, isasyncgen
+from crawlo.exceptions import TransformTypeError
+
+
+async def transform(func: Callable):
+    try:
+        if isgenerator(func):
+            for f in func:
+                yield f
+        elif isasyncgen(func):
+            async for f in func:
+                yield f
+        else:
+            raise TransformTypeError(
+                f'callback return type error: {type(func)} must be `generator` or `async generator`'
+            )
+    except Exception as exp:
+        yield exp
+

crawlo/utils/log.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2024-04-11 09:03
+# @Author  :   oscar
+# @Desc    :   None
+"""
+from logging import Formatter, StreamHandler, Logger, INFO
+
+LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s： %(message)s'
+
+
+class LoggerManager(object):
+    logger_cache = {}
+
+    def __init__(self):
+        pass
+
+    @classmethod
+    def get_logger(cls, name: str = 'default', level=None, log_format: str = LOG_FORMAT):
+        key = (name, level)
+
+        def gen_logger():
+            log_formatter = Formatter(log_format)
+            handler = StreamHandler()
+            handler.setFormatter(log_formatter)
+            handler.setLevel(level or INFO)
+
+            _logger = Logger(name=name)
+            _logger.addHandler(handler)
+            _logger.setLevel(level or INFO)
+            cls.logger_cache[key] = _logger
+            return _logger
+
+        return cls.logger_cache.get(key, None) or gen_logger()
+
+
+get_logger = LoggerManager.get_logger
+

crawlo/utils/pqueue.py

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from asyncio import PriorityQueue, TimeoutError
+
+
+class SpiderPriorityQueue(PriorityQueue):
+    def __init__(self, maxsize=0):
+        super(SpiderPriorityQueue, self).__init__(maxsize=maxsize)
+
+    async def get(self):
+        fut = super().get()
+        try:
+            return await asyncio.wait_for(fut, timeout=0.1)
+        except TimeoutError:
+            return None

crawlo/utils/project.py

@@ -0,0 +1,58 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import os
+import sys
+from importlib import import_module
+from inspect import iscoroutinefunction
+from typing import Callable
+
+from crawlo.settings.setting_manager import SettingManager
+
+
+def _get_closest(path='.'):
+    path = os.path.abspath(path)
+    return path
+
+
+def _init_env():
+    closest = _get_closest()
+    if closest:
+        project_dir = os.path.dirname(closest)
+        sys.path.append(project_dir)
+
+
+def get_settings(settings='settings'):
+    _settings = SettingManager()
+    _init_env()
+    _settings.set_settings(settings)
+    return _settings
+
+
+def merge_settings(spider, settings):
+    if hasattr(spider, 'custom_settings'):
+        custom_settings = getattr(spider, 'custom_settings')
+        settings.update_attributes(custom_settings)
+
+
+def load_class(_path):
+    if not isinstance(_path, str):
+        if callable(_path):
+            return _path
+        else:
+            raise TypeError(f"args expect str or object, got {_path}")
+
+    module_name, class_name = _path.rsplit('.', 1)
+    module = import_module(module_name)
+
+    try:
+        cls = getattr(module, class_name)
+    except AttributeError:
+        raise NameError(f"Module {module_name!r} has no class named {class_name!r}")
+    return cls
+
+
+async def common_call(func: Callable, *args, **kwargs):
+    if iscoroutinefunction(func):
+        return await func(*args, **kwargs)
+    else:
+        return func(*args, **kwargs)

crawlo/utils/system.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import platform
+
+system_name = platform.system().lower()
+if system_name == 'windows':
+    import asyncio
+    asyncio.set_event_loop_policy(
+        asyncio.WindowsSelectorEventLoopPolicy()
+    )
+

crawlo-1.0.0.dist-info/METADATA

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: crawlo
+Version: 1.0.0
+Summary: feapder是一款支持异步的python爬虫框架
+Home-page: https://github.com/crawl-coder/Crawlo.git
+Author: crawl-coder
+Author-email: crawlo@qq.com
+License: MIT
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: aiohttp>=3.12.6
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: DBUtils>=2.0
+Requires-Dist: parsel>=1.10.0
+Requires-Dist: pymysql>=1.1.1
+Requires-Dist: ujson>=5.10.0
+Provides-Extra: render
+Requires-Dist: webdriver-manager>=4.0.0; extra == "render"
+Requires-Dist: playwright; extra == "render"
+Requires-Dist: selenium>=3.141.0; extra == "render"
+Provides-Extra: all
+Requires-Dist: bitarray>=1.5.3; extra == "all"
+Requires-Dist: PyExecJS>=1.5.1; extra == "all"
+Requires-Dist: pymongo>=3.10.1; extra == "all"
+Requires-Dist: redis-py-cluster>=2.1.0; extra == "all"
+Requires-Dist: webdriver-manager>=4.0.0; extra == "all"
+Requires-Dist: playwright; extra == "all"
+Requires-Dist: selenium>=3.141.0; extra == "all"
+Dynamic: license-file
+
+# Crawlo
+异步通用爬虫框架

crawlo-1.0.0.dist-info/RECORD

@@ -0,0 +1,59 @@
+crawlo/__init__.py,sha256=yzg6DkuCNkz27Tn5YDdaboZ1Ah96FwV6eGNEDO48lxA,170
+crawlo/__version__.py,sha256=j4YpqEiBQsl3obE5LH55xbuejIMjUwUNp-k4JXOjEFU,23
+crawlo/crawler.py,sha256=lIx3-_TKXdCDhpfw4lYcizpyoclapjAHfGchBJ9DmTU,3679
+crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
+crawlo/exceptions.py,sha256=trxM2c0jw50QsGSoFAKC2RrKpapOFHQDq0wQuLWqmKE,980
+crawlo/stats_collector.py,sha256=jhAW8k0SzjqelkpiWpfGmMw2DBkgTjpwnObqTNDOp6A,1286
+crawlo/subscriber.py,sha256=4stxeXqNK6RB7oqo0wKJdUw2Ym3b6UBMhZ4pRrjSMEU,1001
+crawlo/task_manager.py,sha256=D9m-nqnGj-FZPtGk4CdwZX3Gw7IWyYvTS7CHpRGWc_w,748
+crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
+crawlo/core/engine.py,sha256=mQfzY4Bm0Ysb5bPjoH3lMVFbn9WJuQ1UjBTReuOljzQ,5873
+crawlo/core/processor.py,sha256=5RQPfffE8AnKnuajL9J2oc7QAeoSjATxwdxRU3_EoKM,1183
+crawlo/core/scheduler.py,sha256=2z8qzZmj6zn9w5pE1yux6Vi7fDMq8Cto6n9Wyyo_DP8,1074
+crawlo/downloader/__init__.py,sha256=72u2Hef4HaMfs9VCqEjbMtiaRXbaXmgNiJn6qy09LHs,2384
+crawlo/downloader/aiohttp_downloader.py,sha256=4C2BDloKzwss16kfD7tH0WPugPbSSFxl-5-_DLWB0vM,3676
+crawlo/downloader/httpx_downloader.py,sha256=ra6Ae_lv8pNyvLzPQYBgTNuBdMVBYi86kNt2OdZlcSo,1704
+crawlo/extension/__init__.py,sha256=O2BVK1U3WwmurZb-PaYVz3g1tZ_iYUjCwilmUKf6844,1170
+crawlo/extension/log_interval.py,sha256=FOWeTOuWtOpCz2UPV5F_--QIa8yomltSpjxbw3F7bkU,1971
+crawlo/extension/log_stats.py,sha256=ZIIB6WKvYSxDIw7Mr52QkOSiX-tMGWU8Ifpd1EXiTUY,1677
+crawlo/items/__init__.py,sha256=Ir04T486XuJJ-OCItxqO-78Jyt7B8JGLBsrgCO605PU,535
+crawlo/items/items.py,sha256=7w0uojKu1GzY5SDfrlJtAhqVv6alHQjwkBkthSoJ-04,2856
+crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
+crawlo/middleware/default_header.py,sha256=OVW4vpRPp3Y6qYXtiEYlGqVjCYcbuv1Iecc7zEgwCsI,1099
+crawlo/middleware/download_delay.py,sha256=P2eyAJXwdLdC4yYuLhvKZVa1b5YQvQD0GpsR8aDW8-8,994
+crawlo/middleware/middleware_manager.py,sha256=T4axTY89Z0BOwaWDWcUTABeDNTvyPFiyrbwj-H4sbSA,6629
+crawlo/middleware/request_ignore.py,sha256=jdybWFVXuA5YsAPfZJFzLTWkYhEAewNgxuhFqczPW9M,1027
+crawlo/middleware/response_code.py,sha256=vgXWv3mMu_v9URvhKA9myIFH4u6L4EwNme80wL4DCGc,677
+crawlo/middleware/response_filter.py,sha256=O2gkV_Yjart8kmmXTGzrtZnb_Uuefap4uL2Cu01iRs4,863
+crawlo/middleware/retry.py,sha256=sCRO5uh9cG8Ui3PODJEg_6PlxDD_lAoopaF2P7u6bUo,3148
+crawlo/network/__init__.py,sha256=DVz1JpasjxCgOlXvm76gz-S18OXr4emG_J39yi5iVuA,130
+crawlo/network/request.py,sha256=YokEk6fEBQF2ii-Oe18ZzVc262Ih6KcUZPLOpxcs6Mg,1474
+crawlo/network/response.py,sha256=QikuOCgxS5yMh4Uh0QuMpqWfyR78vossCs-Va-sQ2YE,2993
+crawlo/pipelines/__init__.py,sha256=IbXJ6B8LqxVVjeLNgL_12AxV6zbV8hNRQxAfMLjjSaw,273
+crawlo/pipelines/console_pipeline.py,sha256=1zorg1xrIiCctiTkd38jm7Q-miKMnRFnJBjv7Tg0gyo,537
+crawlo/pipelines/mongo_pipeline.py,sha256=DfXD10Ee-3q685EdS6du3Nn-f0EaBKJgdgsVlpcYhd8,67
+crawlo/pipelines/mysql_pipeline.py,sha256=QAZToJAmy3xyTLOh7DkwCfxMDWQTLiTOLdfvZhkMP6E,67
+crawlo/pipelines/pipeline_manager.py,sha256=FvpMuHwB5DWGzPsy1mZFdcyxzPN2mVL2aivUWlzrHx4,2183
+crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
+crawlo/settings/default_settings.py,sha256=4J8Kzc3dk09ogy5_FeaGA1KHUzHPMYg9LFIVFIrhuXY,979
+crawlo/settings/setting_manager.py,sha256=4xXOzKwZCgAp8ybwvVcs2R--CsOD7c6dBIkj6DJHB3c,2998
+crawlo/spider/__init__.py,sha256=9haJYfxX4Gx2iV3KK5Fry2fO9R-gEmAkR_pQjZTIErE,905
+crawlo/templates/item_template.tmpl,sha256=0bGFnlwJRqstxMNEj1H_pEICybwoueRhs31QaDPXrS0,372
+crawlo/templates/spider_template.tmpl,sha256=JzphuA87Yl_F1xR9zOIi_ZSazyT8eSNPxYYPMv3Uiko,835
+crawlo/templates/project_template/main.py,sha256=BcCP294ycCPsHi_AMN7OAJtcrLvQdf91meH93PqbQgs,626
+crawlo/templates/project_template/setting.py,sha256=Ce4nMbrdhL1ioRdTcB0vV_vK_50cfnwVqSvt49QsNkA,9395
+crawlo/templates/project_template/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/templates/project_template/spiders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+crawlo/utils/__init__.py,sha256=XCYumI8wJ1jU_Myn_K0LT-LVygPDUCdETCbXM3EWvlo,130
+crawlo/utils/date_tools.py,sha256=bW3j9LxxuAUwykzuzaOYR3iY8U0g7wBPZL1ELf4lXVo,482
+crawlo/utils/func_tools.py,sha256=sWnoa2SDAMK2NBdoIO2o3Ew6SqeJXcSNEzmWZUJcE6o,616
+crawlo/utils/log.py,sha256=LU0J3boPCL-Kynx3wR_CAryRgScNmPPn4pBitLrrsX4,1028
+crawlo/utils/pqueue.py,sha256=W1n23t9roVvDB84jQ4j7sk_Z4FoD8q3LbRRbH4B6Ej4,446
+crawlo/utils/project.py,sha256=JHMV7SLDWMaPCwuwVY9KQqvu2VT1ZlXEQvpIzzNaSeo,1506
+crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
+crawlo-1.0.0.dist-info/licenses/LICENSE,sha256=f96mrub4oLJnuMcNwMjHWMksci5brQ2a2Fu0R7YEgnk,1125
+crawlo-1.0.0.dist-info/METADATA,sha256=wS6yr3avxrdzm4x4WPN7fBKgO5jobpQ8UpvrCUC1aQw,1266
+crawlo-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.0.0.dist-info/entry_points.txt,sha256=GD9PBhKQN83EaxPYtz7NhcGeZeh3bdr2jWbTixOs-lw,59
+crawlo-1.0.0.dist-info/top_level.txt,sha256=Dwuv-Y1aGSJD3mjFrCdNGQ8EHroMj7RgVcxDdcczx4k,7
+crawlo-1.0.0.dist-info/RECORD,,

crawlo-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

crawlo-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+crawlo = crawlo.commands.cmdline:execute

crawlo-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,23 @@
+MIT License
+
+Modifications:
+
+Copyright (c) 2020 crawl-coder <2251018029@qq.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

crawlo-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+crawlo