crawlo 1.0.2__py3-none-any.whl → 1.0.4__py3-none-any.whl

crawlo/settings/default_settings.py

@@ -1,89 +1,94 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-# 默认项目名称
-import os
-
-PROJECT_NAME = 'crawlo'
-VERSION = 1.0
-# 并发数
-CONCURRENCY = 8
-
-# 下载超时时长
-DOWNLOAD_TIMEOUT = 60
-
-INTERVAL = 5
-
-# --------------------------------------------------- delay ------------------------------------------------------------
-# 下载延迟，默认关闭
-DOWNLOAD_DELAY = 0
-# 下载延迟范围
-RANDOM_RANGE = (0.75, 1.25)
-# 是否需要随机
-RANDOMNESS = True
-
-# --------------------------------------------------- retry ------------------------------------------------------------
-MAX_RETRY_TIMES = 2
-IGNORE_HTTP_CODES = [403, 404]
-RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
-# 允许通过的状态码
-ALLOWED_CODES = []
-# 请求优先级设置
-RETRY_PRIORITY = -1
-#
-DEPTH_PRIORITY = 1
-
-STATS_DUMP = True
-# ssl 验证
-VERIFY_SSL = True
-# 是否使用同一个session
-USE_SESSION = True
-# 日志级别
-LOG_LEVEL = 'DEBUG'
-# 选择下载器
-DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"  # HttpXDownloader
-
-EXTENSIONS = []
-
-# --------------------------------------------------- 公共MySQL配置 -----------------------------------------------------
-MYSQL_HOST = '127.0.0.1'
-MYSQL_PORT = 3306
-MYSQL_USER = 'scrapy_user'
-MYSQL_PASSWORD = 'your_password'
-MYSQL_DB = 'scrapy_data'
-MYSQL_TABLE = 'crawled_data'
-
-# asyncmy专属配置
-MYSQL_POOL_MIN = 5  # 连接池最小连接数
-MYSQL_POOL_MAX = 20  # 连接池最大连接数
-MYSQL_ECHO = False
-
-# 批量插入大小
-MYSQL_BATCH_SIZE = 100
-
-# --------------------------------------------------- MongoDB 基础配置 -----------------------------------------------------
-MONGO_URI = 'mongodb://user:password@host:27017'
-MONGO_DATABASE = 'scrapy_data'
-MONGO_COLLECTION = 'crawled_items'  # 可选，默认使用spider名称
-
-# 连接池优化配置（仅方案二需要）
-MONGO_MAX_POOL_SIZE = 200  # 最大连接数
-MONGO_MIN_POOL_SIZE = 20  # 最小保持连接数
-
-# 启用管道
-PIPELINES = [
-    'crawlo.pipelines.console_pipeline.ConsolePipeline',
-]
-
-# filter
-REQUEST_DIR = '.'
-FILTER_DEBUG = True
-FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
-
-# redis filter
-CLEANUP_FP = False
-DECODE_RESPONSES = True
-REDIS_KEY = 'request_fingerprint'
-REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
-REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', 'oscar&0503')
-REDIS_PORT = os.getenv('REDIS_PORT', 6379)
-REDIS_URL = f'redis://:{REDIS_PASSWORD or ""}@{REDIS_HOST}:{REDIS_PORT}/0'
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+# 默认项目名称
+import os
+
+PROJECT_NAME = 'crawlo'
+VERSION = 1.0
+# 并发数
+CONCURRENCY = 8
+
+# 下载超时时长
+DOWNLOAD_TIMEOUT = 60
+
+INTERVAL = 5
+
+# --------------------------------------------------- delay ------------------------------------------------------------
+# 下载延迟，默认关闭
+DOWNLOAD_DELAY = 0
+# 下载延迟范围
+RANDOM_RANGE = (0.75, 1.25)
+# 是否需要随机
+RANDOMNESS = True
+
+# --------------------------------------------------- retry ------------------------------------------------------------
+MAX_RETRY_TIMES = 2
+IGNORE_HTTP_CODES = [403, 404]
+RETRY_HTTP_CODES = [408, 429, 500, 502, 503, 504, 522, 524]
+# 允许通过的状态码
+ALLOWED_CODES = []
+# 请求优先级设置
+RETRY_PRIORITY = -1
+#
+DEPTH_PRIORITY = 1
+
+STATS_DUMP = True
+# ssl 验证
+VERIFY_SSL = True
+# 是否使用同一个session
+USE_SESSION = True
+# 日志级别
+LOG_LEVEL = 'DEBUG'
+# 选择下载器
+DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"  # HttpXDownloader
+
+# --------------------------------------------------- 公共MySQL配置 -----------------------------------------------------
+MYSQL_HOST = '127.0.0.1'
+MYSQL_PORT = 3306
+MYSQL_USER = 'scrapy_user'
+MYSQL_PASSWORD = 'your_password'
+MYSQL_DB = 'scrapy_data'
+MYSQL_TABLE = 'crawled_data'
+
+# asyncmy专属配置
+MYSQL_POOL_MIN = 5  # 连接池最小连接数
+MYSQL_POOL_MAX = 20  # 连接池最大连接数
+MYSQL_ECHO = False
+
+# 批量插入大小
+MYSQL_BATCH_SIZE = 100
+
+# --------------------------------------------------- MongoDB 基础配置 -----------------------------------------------------
+MONGO_URI = 'mongodb://user:password@host:27017'
+MONGO_DATABASE = 'scrapy_data'
+MONGO_COLLECTION = 'crawled_items'  # 可选，默认使用spider名称
+
+# 连接池优化配置（仅方案二需要）
+MONGO_MAX_POOL_SIZE = 200  # 最大连接数
+MONGO_MIN_POOL_SIZE = 20  # 最小保持连接数
+
+# 启用管道
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+]
+
+
+EXTENSIONS = [
+    'crawlo.extension.log_interval.LogIntervalExtension',
+    'crawlo.extension.log_stats.LogStats'
+]
+
+# filter
+REQUEST_DIR = '.'
+FILTER_DEBUG = True
+FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
+
+# redis filter
+REDIS_TTL = 0
+CLEANUP_FP = 0
+DECODE_RESPONSES = True
+REDIS_KEY = 'request_fingerprint'
+REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
+REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', 'oscar&0503')
+REDIS_PORT = os.getenv('REDIS_PORT', 6379)
+REDIS_URL = f'redis://:{REDIS_PASSWORD or ""}@{REDIS_HOST}:{REDIS_PORT}/0'

crawlo/settings/setting_manager.py

@@ -1,100 +1,100 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-import json
-from copy import deepcopy
-from importlib import import_module
-from collections.abc import MutableMapping
-
-from crawlo.settings import default_settings
-
-
-class SettingManager(MutableMapping):
-
-    def __init__(self, values=None):
-        self.attributes = {}
-        self.set_settings(default_settings)
-        self.update_attributes(values)
-
-    def get(self, key, default=None):
-        """安全获取值，不触发递归"""
-        value = self.attributes.get(key, default)
-        return value if value is not None else default
-
-    def get_int(self, key, default=0):
-        return int(self.get(key, default=default))
-
-    def get_float(self, key, default=0.0):
-        return float(self.get(key, default=default))
-
-    def get_bool(self, key, default=False):
-        got = self.get(key, default=default)
-        if isinstance(got, bool):
-            return got
-        if isinstance(got, (int, float)):
-            return bool(got)
-        got_lower = str(got).strip().lower()
-        if got_lower in ('1', 'true'):
-            return True
-        if got_lower in ('0', 'false'):
-            return False
-        raise ValueError(
-            f"Unsupported value for boolean setting: {got}. "
-            "Supported values are: 0/1, True/False, '0'/'1', 'True'/'False' (case-insensitive)."
-        )
-
-    def get_list(self, key, default=None):
-        values = self.get(key, default or [])
-        if isinstance(values, str):
-            return [v.strip() for v in values.split(',') if v.strip()]
-        try:
-            return list(values)
-        except TypeError:
-            return [values]
-
-    def get_dict(self, key, default=None):
-        value = self.get(key, default or {})
-        if isinstance(value, str):
-            value = json.loads(value)
-        try:
-            return dict(value)
-        except TypeError:
-            return value
-
-    def set(self, key, value):
-        self.attributes[key] = value
-
-    def set_settings(self, module):
-        if isinstance(module, str):
-            module = import_module(module)
-        for key in dir(module):
-            if key.isupper():
-                self.set(key, getattr(module, key))
-
-    # 实现 MutableMapping 必须的方法
-    def __getitem__(self, item):
-        return self.attributes[item]
-
-    def __setitem__(self, key, value):
-        self.set(key, value)
-
-    def __delitem__(self, key):
-        del self.attributes[key]
-
-    def __iter__(self):
-        return iter(self.attributes)
-
-    def __len__(self):
-        return len(self.attributes)
-
-    def __str__(self):
-        return f'<Settings: {self.attributes}>'
-
-    __repr__ = __str__
-
-    def update_attributes(self, attributes):
-        if attributes is not None:
-            for key, value in attributes.items():
-                self.set(key, value)
-
-    def copy(self):
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+import json
+from copy import deepcopy
+from importlib import import_module
+from collections.abc import MutableMapping
+
+from crawlo.settings import default_settings
+
+
+class SettingManager(MutableMapping):
+
+    def __init__(self, values=None):
+        self.attributes = {}
+        self.set_settings(default_settings)
+        self.update_attributes(values)
+
+    def get(self, key, default=None):
+        """安全获取值，不触发递归"""
+        value = self.attributes.get(key, default)
+        return value if value is not None else default
+
+    def get_int(self, key, default=0):
+        return int(self.get(key, default=default))
+
+    def get_float(self, key, default=0.0):
+        return float(self.get(key, default=default))
+
+    def get_bool(self, key, default=False):
+        got = self.get(key, default=default)
+        if isinstance(got, bool):
+            return got
+        if isinstance(got, (int, float)):
+            return bool(got)
+        got_lower = str(got).strip().lower()
+        if got_lower in ('1', 'true'):
+            return True
+        if got_lower in ('0', 'false'):
+            return False
+        raise ValueError(
+            f"Unsupported value for boolean setting: {got}. "
+            "Supported values are: 0/1, True/False, '0'/'1', 'True'/'False' (case-insensitive)."
+        )
+
+    def get_list(self, key, default=None):
+        values = self.get(key, default or [])
+        if isinstance(values, str):
+            return [v.strip() for v in values.split(',') if v.strip()]
+        try:
+            return list(values)
+        except TypeError:
+            return [values]
+
+    def get_dict(self, key, default=None):
+        value = self.get(key, default or {})
+        if isinstance(value, str):
+            value = json.loads(value)
+        try:
+            return dict(value)
+        except TypeError:
+            return value
+
+    def set(self, key, value):
+        self.attributes[key] = value
+
+    def set_settings(self, module):
+        if isinstance(module, str):
+            module = import_module(module)
+        for key in dir(module):
+            if key.isupper():
+                self.set(key, getattr(module, key))
+
+    # 实现 MutableMapping 必须的方法
+    def __getitem__(self, item):
+        return self.attributes[item]
+
+    def __setitem__(self, key, value):
+        self.set(key, value)
+
+    def __delitem__(self, key):
+        del self.attributes[key]
+
+    def __iter__(self):
+        return iter(self.attributes)
+
+    def __len__(self):
+        return len(self.attributes)
+
+    def __str__(self):
+        return f'<Settings: {self.attributes}>'
+
+    __repr__ = __str__
+
+    def update_attributes(self, attributes):
+        if attributes is not None:
+            for key, value in attributes.items():
+                self.set(key, value)
+
+    def copy(self):
         return deepcopy(self)

crawlo/spider/__init__.py

@@ -1,36 +1,36 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo import Request
-
-
-class Spider(object):
-    def __init__(self):
-        if not hasattr(self, 'start_urls'):
-            self.start_urls = []
-        self.crawler = None
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls()
-        o.crawler = crawler
-        return o
-
-    def start_requests(self):
-        if self.start_urls:
-            for url in self.start_urls:
-                yield Request(url=url, dont_filter=True)
-        else:
-            if hasattr(self, 'start_url') and isinstance(getattr(self, 'start_url'), str):
-                yield Request(getattr(self, 'start_url'), dont_filter=True)
-
-    def parse(self, response):
-        raise NotImplementedError
-
-    async def spider_opened(self):
-        pass
-
-    async def spider_closed(self):
-        pass
-
-    def __str__(self):
-        return self.__class__.__name__
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import Request
+
+
+class Spider(object):
+    def __init__(self):
+        if not hasattr(self, 'start_urls'):
+            self.start_urls = []
+        self.crawler = None
+
+    @classmethod
+    def create_instance(cls, crawler):
+        o = cls()
+        o.crawler = crawler
+        return o
+
+    def start_requests(self):
+        if self.start_urls:
+            for url in self.start_urls:
+                yield Request(url=url, dont_filter=True)
+        else:
+            if hasattr(self, 'start_url') and isinstance(getattr(self, 'start_url'), str):
+                yield Request(getattr(self, 'start_url'), dont_filter=True)
+
+    def parse(self, response):
+        raise NotImplementedError
+
+    async def spider_opened(self):
+        pass
+
+    async def spider_closed(self):
+        pass
+
+    def __str__(self):
+        return self.__class__.__name__

crawlo/stats_collector.py

@@ -1,47 +1,59 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-05-17 09:57
-# @Author  :   crawl-coder
-# @Desc    :   统计信息收集器
-"""
-from pprint import pformat
-from crawlo.utils.log import get_logger
-
-
-class StatsCollector(object):
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self._dump = self.crawler.settings.get_bool('STATS_DUMP')
-        self._stats = {}
-        self.logger = get_logger(self.__class__.__name__, "INFO")
-
-    def inc_value(self, key, count=1, start=0):
-        self._stats[key] = self._stats.setdefault(key, start) + count
-
-    def get_value(self, key, default=None):
-        return self._stats.get(key, default)
-
-    def get_stats(self):
-        return self._stats
-
-    def set_stats(self, stats):
-        self._stats = stats
-
-    def clear_stats(self):
-        self._stats.clear()
-
-    def close_spider(self, spider_name, reason):
-        self._stats['reason'] = reason
-        if self._dump:
-            self.logger.info(f'{spider_name} stats: \n{pformat(self._stats)}')
-
-    def __getitem__(self, item):
-        return self._stats[item]
-
-    def __setitem__(self, key, value):
-        self._stats[key] = value
-
-    def __delitem__(self, key):
-        del self._stats[key]
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-05-17 09:57
+# @Author  :   crawl-coder
+# @Desc    :   统计信息收集器
+"""
+from pprint import pformat
+from crawlo.utils.log import get_logger
+
+
+class StatsCollector(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self._dump = self.crawler.settings.get_bool('STATS_DUMP')
+        self._stats = {}
+        self.logger = get_logger(self.__class__.__name__, "INFO")
+
+    def inc_value(self, key, count=1, start=0):
+        self._stats[key] = self._stats.setdefault(key, start) + count
+
+    def get_value(self, key, default=None):
+        return self._stats.get(key, default)
+
+    def get_stats(self):
+        return self._stats
+
+    def set_stats(self, stats):
+        self._stats = stats
+
+    def clear_stats(self):
+        self._stats.clear()
+
+    def close_spider(self, spider, reason):
+        self._stats['reason'] = reason
+
+        # 首选：使用 spider.name
+        # 次选：使用实例的类名
+        # 最后：使用一个完全未知的占位符
+        spider_name = (
+                getattr(spider, 'name', None) or
+                spider.__class__.__name__ or
+                '<Unknown>'
+        )
+
+        self._stats['spider_name'] = spider_name
+
+        if self._dump:
+            self.logger.info(f'{spider_name} stats: \n{pformat(self._stats)}')
+
+    def __getitem__(self, item):
+        return self._stats[item]
+
+    def __setitem__(self, key, value):
+        self._stats[key] = value
+
+    def __delitem__(self, key):
+        del self._stats[key]