crawlo 1.2.9__py3-none-any.whl → 1.3.1__py3-none-any.whl

crawlo/utils/log.py

@@ -8,28 +8,20 @@ from logging import (
     INFO,
     getLevelName,
 )
-# 导入日志轮转处理器
-from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
 
 LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s'
 
 
 class LoggerManager:
+    """日志管理器，提供统一的日志配置和获取接口"""
     logger_cache = {}
     _default_filename = None
     _default_level = DEBUG  # 设置为最低级别，由handler控制实际输出
-    _default_file_level = INFO
-    _default_console_level = INFO
+    _default_file_level = DEBUG  # 默认为DEBUG级别，确保所有INFO级别日志都能写入文件
+    _default_console_level = DEBUG  # 默认为DEBUG级别
     _default_log_format = LOG_FORMAT
     _default_encoding = 'utf-8'
-    # 添加日志轮转相关默认配置
-    _default_log_max_bytes = 10 * 1024 * 1024  # 10MB
-    _default_log_backup_count = 5
-    _default_log_when = 'midnight'
-    _default_log_interval = 1
-    _default_log_use_rotation = False
-    _default_log_rotation_type = 'size'  # 'size' or 'time'
-    _default_log_rotation_suffix = None  # 轮转文件后缀格式
+    _configured = False  # 标记是否已配置
 
     @classmethod
     def _to_level(cls, level):
@@ -67,41 +59,26 @@ class LoggerManager:
         get_val = settings.get if hasattr(settings, 'get') else (lambda k, d=None: kwargs.get(k, d))
 
         filename = get_val('LOG_FILE')
-        level = get_val('LOG_LEVEL', 'DEBUG')  # 默认为DEBUG级别
-        file_level = get_val('LOG_FILE_LEVEL', 'INFO')
-        console_level = get_val('LOG_CONSOLE_LEVEL', 'INFO')  # 保留LOG_CONSOLE_LEVEL配置
+        level = get_val('LOG_LEVEL', 'INFO')  # 默认为INFO级别
+        file_level = get_val('LOG_FILE_LEVEL', level)  # 默认继承LOG_LEVEL的值
+        # 根据项目规范，已完全移除LOG_CONSOLE_LEVEL支持，统一使用LOG_LEVEL控制控制台和文件的日志输出级别
         log_format = get_val('LOG_FORMAT', LOG_FORMAT)
         encoding = get_val('LOG_ENCODING', 'utf-8')
-        # 获取日志轮转配置
-        use_rotation = get_val('LOG_USE_ROTATION', False)
-        rotation_type = get_val('LOG_ROTATION_TYPE', 'size')
-        max_bytes = get_val('LOG_MAX_BYTES', cls._default_log_max_bytes)
-        backup_count = get_val('LOG_BACKUP_COUNT', cls._default_log_backup_count)
-        when = get_val('LOG_WHEN', cls._default_log_when)
-        interval = get_val('LOG_INTERVAL', cls._default_log_interval)
-        rotation_suffix = get_val('LOG_ROTATION_SUFFIX', cls._default_log_rotation_suffix)  # 轮转文件后缀
 
         cls._default_filename = filename
         cls._default_level = cls._to_level(level)
         cls._default_file_level = cls._to_level(file_level)
-        cls._default_console_level = cls._to_level(console_level)
+        # 控制台日志级别直接使用LOG_LEVEL的值，不再支持LOG_CONSOLE_LEVEL
+        cls._default_console_level = cls._default_level
         cls._default_log_format = log_format
         cls._default_encoding = encoding
-        # 设置日志轮转配置
-        cls._default_log_use_rotation = use_rotation
-        cls._default_log_rotation_type = rotation_type
-        cls._default_log_max_bytes = max_bytes
-        cls._default_log_backup_count = backup_count
-        cls._default_log_when = when
-        cls._default_log_interval = interval
-        cls._default_log_rotation_suffix = rotation_suffix
-        
-        # 移除对根日志记录器级别的修改，避免副作用
+
+        cls._configured = True
 
     @classmethod
     def get_logger(cls, name='default', level=None, filename=None):
         """
-        简化接口，只暴露必要参数
+        获取logger实例
         """
         # 确定最终参数
         # 如果传入了level参数，则使用它，否则使用默认级别
@@ -110,7 +87,7 @@ class LoggerManager:
         else:
             # Logger级别设置为DEBUG（最低级别），由handler控制实际输出
             final_level = DEBUG
-            
+
         final_filename = filename if filename is not None else cls._default_filename
 
         # 安全的字符串化 key，避免任何 unhashable 类型
@@ -146,32 +123,9 @@ class LoggerManager:
                 if log_dir and not os.path.exists(log_dir):
                     os.makedirs(log_dir, exist_ok=True)
 
-                # 检查是否启用日志轮转
-                if cls._default_log_use_rotation:
-                    if cls._default_log_rotation_type == 'size':
-                        # 使用大小轮转
-                        fh = RotatingFileHandler(
-                            final_filename, 
-                            maxBytes=cls._default_log_max_bytes,
-                            backupCount=cls._default_log_backup_count,
-                            encoding=cls._default_encoding
-                        )
-                    else:
-                        # 使用时间轮转
-                        fh = TimedRotatingFileHandler(
-                            final_filename,
-                            when=cls._default_log_when,
-                            interval=cls._default_log_interval,
-                            backupCount=cls._default_log_backup_count,
-                            encoding=cls._default_encoding
-                        )
-                        # 如果提供了自定义后缀格式，则设置
-                        if cls._default_log_rotation_suffix:
-                            fh.suffix = cls._default_log_rotation_suffix
-                else:
-                    # 使用普通文件处理器（默认行为，会追加到文件）
-                    fh = FileHandler(final_filename, mode='a', encoding=cls._default_encoding)
-                    
+                # 使用普通文件处理器（移除日志轮转功能）
+                fh = FileHandler(final_filename, mode='a', encoding=cls._default_encoding)
+
                 fh.setFormatter(formatter)
                 fh.setLevel(cls._default_file_level)
                 _logger.addHandler(fh)
@@ -183,6 +137,11 @@ class LoggerManager:
         cls.logger_cache[key] = _logger
         return _logger
 
+    @classmethod
+    def is_configured(cls):
+        """检查日志系统是否已配置"""
+        return cls._configured
+
 
 # 全局快捷函数
 get_logger = LoggerManager.get_logger

{crawlo-1.2.9.dist-info → crawlo-1.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.2.9
+Version: 1.3.1
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder
@@ -56,7 +56,7 @@ Requires-Dist: selenium>=3.141.0; extra == "all"
   
   <p align="center">
     <a href="https://www.python.org/downloads/">
-      <img src="https://img.shields.io/badge/python-3.8%2B-blue" alt="Python Version">
+      <img src="https://img.shields.io/badge/python-%3C%3D3.12-blue" alt="Python Version">
     </a>
     <a href="LICENSE">
       <img src="https://img.shields.io/badge/license-MIT-green" alt="License">
@@ -429,13 +429,22 @@ Crawlo框架的中间件、管道和扩展组件采用模块化设计，框架
 
 用户可以通过`CUSTOM_MIDDLEWARES`配置自定义中间件：
 
-```python
+``python
 # settings.py
 CUSTOM_MIDDLEWARES = [
     'myproject.middlewares.CustomMiddleware',
 ]
 ```
 
+> **注意**：DefaultHeaderMiddleware 和 OffsiteMiddleware 需要相应的配置才能启用：
+> - DefaultHeaderMiddleware 需要配置 `DEFAULT_REQUEST_HEADERS` 或 `USER_AGENT` 参数
+> - OffsiteMiddleware 需要配置 `ALLOWED_DOMAINS` 参数
+> 
+> 如果未配置相应参数，这些中间件会因为 NotConfiguredError 而被禁用。
+
+> **注意**：中间件的顺序很重要。SimpleProxyMiddleware 通常放在列表末尾，
+> 这样可以在所有默认中间件处理后再应用代理设置。
+
 #### 管道配置
 
 框架默认加载以下管道：
@@ -930,7 +939,7 @@ Crawlo框架内置了多种中间件，其中代理中间件有两种实现：
 
 如果需要使用简化版代理中间件，可以在配置文件中替换默认的代理中间件：
 
-```python
+``python
 # settings.py
 MIDDLEWARES = [
     # 注释掉复杂版代理中间件

{crawlo-1.2.9.dist-info → crawlo-1.3.1.dist-info}/RECORD

@@ -1,13 +1,13 @@
 crawlo/__init__.py,sha256=qZzTmb7hw5h_qcP2EYGUZcoSScxlKZFJ76CjSeS7UfA,1381
-crawlo/__version__.py,sha256=S2h29KjM2QfA587KjqKjzd2pK0GLkBmHIq8WDOutUiI,21
+crawlo/__version__.py,sha256=P2DFKJQEJRlJhF0IW0Lwt4G4uMYyJJ5ymhv-XrCcPGo,22
 crawlo/cli.py,sha256=OXprmcTUbFK02ptw_Gq8Gk4-ZCU-WEMJgzU1ztgP6Bk,2327
 crawlo/config.py,sha256=dNoNyTkXLe2msQ7bZx3YTQItk1m49nIg5-g89FQDNwE,9486
 crawlo/config_validator.py,sha256=gsiLqf5swWd9ISDvoLqCdG7iSXr-ZdBPD4iT6ug1ua4,11239
-crawlo/crawler.py,sha256=cXvzTF3W6LeZNll4hbMLK-E1xDpZL5M9fgtd_8Gk6eI,39863
+crawlo/crawler.py,sha256=Fiu9O_eFHKCfzgzFe0O9gpzWGyneY-imI8-9O4hiWqU,42608
 crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
 crawlo/exceptions.py,sha256=sMay0wnWLfc_FXWslqxm60qz6b66LXs3EdN_w8ygE9k,1166
-crawlo/mode_manager.py,sha256=D8LOgqkqN4yglCKtXw56mq41r30FvxCUb5JOitSTp_U,7502
-crawlo/project.py,sha256=vHqDrGFgRThzFuF2RxDtSHxPerruR3liwXuGC3cVda8,10796
+crawlo/mode_manager.py,sha256=IN3CsWW1pzKnzHxvULxJTTx3Vw2kxJilfeLuDWfxm4Q,4890
+crawlo/project.py,sha256=swSTcan4Ky7sYfCatpNLKsVxztmPkIVwjdo3u6dgcpI,11128
 crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
 crawlo/subscriber.py,sha256=D3hzE7Pc_zJjc-zR7lct5pt32bz6LsDYeC8uHlS4Hso,4986
 crawlo/task_manager.py,sha256=19InAxS9oJ9EMj20Aw2urN-v6BeC22dkgPuW-B9-4UI,819
@@ -16,12 +16,12 @@ crawlo/commands/check.py,sha256=7pD43s97DD-fSLO9OEOuNcNr7o-2g94rJULL8fUzdaI,2260
 crawlo/commands/genspider.py,sha256=HhtvBLkIuhYtJUzom6PquItiC22vU9LNpOkjDUiqdM4,4937
 crawlo/commands/help.py,sha256=gwfHibRpdYDmZO6waUMOEn8SMJ_ubdjL-prD5fiuVY8,4973
 crawlo/commands/list.py,sha256=BqlPjBa5FLotjAlyZ3-nGmXg5cWcCNbHi8U5znb2_D8,5722
-crawlo/commands/run.py,sha256=b_HxEmaze0jpPNIrPbm5lyPYZ20--eUVKJZpXoATf0M,12088
+crawlo/commands/run.py,sha256=KcJ4h4D7lavB6qQDpYMrbgJMgY5vCSLHaLckos5EUNY,11793
 crawlo/commands/startproject.py,sha256=aqKRJarKqTf5XjJnGXwjRpp0uYF16LreFbwwQLGpK-0,16070
 crawlo/commands/stats.py,sha256=8wTubR1RQ1JPTlpOKArcGcQ39bM-0cuH27lDpndnwPQ,6014
-crawlo/commands/utils.py,sha256=pXiFzwVIVXdSPO2Fty_u19P1lsE8HmuE8gTMamKZZUs,5047
+crawlo/commands/utils.py,sha256=Psfu2tKrmDloMq0WnfXLaxx0lJFitMZ-FWS3HAIrziQ,5382
 crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
-crawlo/core/engine.py,sha256=NIWbfRL_Lzjl6yYQ6LSfRJmDHBxjtk6x3nOWrnzaHME,14569
+crawlo/core/engine.py,sha256=Hy0K_g9My6aQ3CPkxAcCiPsumdwh4O8qRhmFlNoErd4,14496
 crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
 crawlo/core/scheduler.py,sha256=D-YzXVvnP6DEkovmz9hThhzIe2UgRrQLNt9pJCPEPwY,12593
 crawlo/data/__init__.py,sha256=8MgDxcMhx-emFARcLAw_ODOZNz0neYBcx7kEbzothd8,116
@@ -37,7 +37,7 @@ crawlo/extension/__init__.py,sha256=FbOwJ4jh60xCbSh7P9CUGJsGAC-VH4MyOtCftRMlxbk,
 crawlo/extension/health_check.py,sha256=0GveZgUtFwjYEKlm3qbwIvCmb4FR0qrIKc8cEF1yQV8,5516
 crawlo/extension/log_interval.py,sha256=VCIeNqXcWDnxj4m6l77cjqgRzV8LfsPMb22X0Xc1Vwc,2417
 crawlo/extension/log_stats.py,sha256=vrChs3bj_Dvay3kxxkBOp4-w0K-IG-2XZ0PoSUahTPs,2908
-crawlo/extension/logging_extension.py,sha256=ET6VAu1J2qNMz4NnG1G3zQLRhbsvV7l6xRIuQLE6DaE,1626
+crawlo/extension/logging_extension.py,sha256=RfL1wI4nz-1Xtg420Ktp7RXnOPnZSHwO0Zpg1w4fO4M,1726
 crawlo/extension/memory_monitor.py,sha256=4aszl3C0GMQbqFhGZjZq5iQuXQR1sOz06VHjjEHgkyE,4290
 crawlo/extension/performance_profiler.py,sha256=EPiNuXuPPDU0Jtgy8arYHpr_8ASK13cCI2BytdJnu_I,4899
 crawlo/extension/request_recorder.py,sha256=RC23yzXClnVv9j2ljQvjBkUfWznfnDHsrQejKhE9y5E,4074
@@ -51,8 +51,8 @@ crawlo/items/items.py,sha256=e-3nXI9ckD64vcDxxQiAU6ufbtJMs09gbZQcYjxgwHY,3374
 crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
 crawlo/middleware/default_header.py,sha256=wQ7BrUHd-hRosFoKsReV9hwNNr_jwK6V0ZfxL6MOGrk,5032
 crawlo/middleware/download_delay.py,sha256=zt9R5g2HWErWA_MAOnGcw_D8l6HD769Kyaw-Hv-vcTc,3438
-crawlo/middleware/middleware_manager.py,sha256=_JnJ50u3u4zFKkNZDae8M8FHcK-V2LkbcX7DkBdtJ2M,6279
-crawlo/middleware/offsite.py,sha256=b3BMwNKGR41YGJGHt1S0H7yXujEkztVvonUQGO05hoM,4026
+crawlo/middleware/middleware_manager.py,sha256=9Sj9rrWK6R9NZq9eT38sWRGuBKLKfjSgEAxu-5NCWgU,6278
+crawlo/middleware/offsite.py,sha256=cR0nVAygxAGbkmyI8yqiRtrZWTRLWddUiToMYGmhrfs,4084
 crawlo/middleware/proxy.py,sha256=NquB6tqHAgHs3-2_1_5220kJYfjNG5JyHRJyo_2j4wo,15636
 crawlo/middleware/request_ignore.py,sha256=xcyZ1c7r_HhbzR3r9pfjsLGW7L7FBVeYvlNt8cpP2wY,2577
 crawlo/middleware/response_code.py,sha256=-Aa9Mm9nJN-WdddN7iTanJRMA83_LYYgSEz3XLQGvMo,4934
@@ -70,16 +70,16 @@ crawlo/pipelines/database_dedup_pipeline.py,sha256=Ao_5jvVPl5QikxXhPeIrcB7_3tinR
 crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
 crawlo/pipelines/memory_dedup_pipeline.py,sha256=oIksbIrmSw9s9jMh6JJMfVbv6hzseVMV_g9S8UHQUP4,3837
 crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
-crawlo/pipelines/mysql_pipeline.py,sha256=G2DMhdh0ihBBolIul4YVTDz2JbrZGJauDtWF-gqRW0w,13473
-crawlo/pipelines/pipeline_manager.py,sha256=DuIRl6FYbrKKUCrtSHnq4pnRImpZPYtg1YRcgUd1xTk,2425
+crawlo/pipelines/mysql_pipeline.py,sha256=CKll3rNFXc0-BGQ_0A6QOSm2-ymHtdjdybX6bSB8i2g,13500
+crawlo/pipelines/pipeline_manager.py,sha256=R4zXYaMUseZsv5d7vbsNPFuXGjO_KWyDve0bHpqsL7Y,3079
 crawlo/pipelines/redis_dedup_pipeline.py,sha256=POYRiWAOp1pqDW9iTPJ8h3VcpLALeLrpw74MvJJqPiM,6342
 crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
 crawlo/queue/queue_manager.py,sha256=XqS_oVbNQJWdtokOuDDPK-FzMrVdnZ3UKp1MF_DMJww,14941
 crawlo/queue/redis_priority_queue.py,sha256=k1OChSMRovSMkbbJ9388axfhpYeMevuJTe-3N1oYhbA,13126
 crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
-crawlo/settings/default_settings.py,sha256=98URrj6QBrx_pmJ1yvK-MSAW8VrZ-pl0FfiZEHV0ZnI,9183
-crawlo/settings/setting_manager.py,sha256=55iJJambnFliZxwhNd7BtlgCV6eviuPRBrrIgyrmZgw,5659
+crawlo/settings/default_settings.py,sha256=VKaCb8JnHx_B-Zi3hN6Mt0QIdv0YRhFlhVEZ2fhx84o,9193
+crawlo/settings/setting_manager.py,sha256=4uuMpGVYzxjmQjvlGqfZ8hDaoSh34OAoL0LCATsMCkI,7512
 crawlo/spider/__init__.py,sha256=ZnSAL9PXLZSIH-Jdv-P6RuWmQUdukr8KPLQK6SXZZaU,20435
 crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
 crawlo/templates/run.py.tmpl,sha256=v_g-LQMYJ6pC8TZgyWj0yB2yTTKrwy9lEJufAYCXyxY,1228
@@ -117,7 +117,7 @@ crawlo/utils/error_handler.py,sha256=q6NqHxjYrKdswfmhshMYMmfBIr0M2YWPYxts4ScHl4Y
 crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
 crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
 crawlo/utils/large_scale_helper.py,sha256=Kxdy3WMuqjzQTyCc6z4xEYxXDi4xnYKJzsVwaBYZrrg,12108
-crawlo/utils/log.py,sha256=AcRx6_9U6fY-6GEZP77Il2o3FZZm5Cx0MFubIK9R3oI,7501
+crawlo/utils/log.py,sha256=vAMACdX8N3kTIRegmKDE1oVImESufeGEskoRmNQQkJo,5281
 crawlo/utils/performance_monitor.py,sha256=Q9xxuXBIfFoig_U-FQPOUuPAh1axO3MzYgpielDyku0,9547
 crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
 crawlo/utils/redis_connection_pool.py,sha256=amGjhaKpodMrw9X56qxZ6f3OTZhjrI89sSVGqgwAQGU,11050
@@ -212,8 +212,8 @@ tests/test_tools.py,sha256=9t9FXZ61MfdB70nck9NYzCq97yd3SLVlLiMybEAlClk,5345
 tests/test_user_agents.py,sha256=rUotyuE2iJDi2LQBrUh980U-dAMTs4ARPMJxICOoQFY,3231
 tests/tools_example.py,sha256=MtIypR-OFiWwi-skurwmq4fM0cGTt-GUX4hSekYs7BY,7739
 tests/verify_distributed.py,sha256=krnYYA5Qx9xXDMWc9YF5DxPSplGvawDg2n0l-3CAqoM,3928
-crawlo-1.2.9.dist-info/METADATA,sha256=wyjBzrx9hUZzlt0r1JPu33ouyrm_iJtphQAi2THb7LA,26298
-crawlo-1.2.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.2.9.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.2.9.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.2.9.dist-info/RECORD,,
+crawlo-1.3.1.dist-info/METADATA,sha256=6m0OV78Iso8DwQFiIB0_mWkYSDymuhjw69F9aVKwtCo,26813
+crawlo-1.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.3.1.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.3.1.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.3.1.dist-info/RECORD,,