crawlo 1.0.2__py3-none-any.whl → 1.0.4__py3-none-any.whl

tests/baidu_spider/settings.py

@@ -0,0 +1,80 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+
+PROJECT_NAME = 'baidu_spider'
+
+CONCURRENCY = 4
+
+USE_SESSION = True
+
+# 下载延迟
+DOWNLOAD_DELAY = 0.5
+RANDOMNESS = False
+
+# --------------------------------------------------- 公共MySQL配置 -----------------------------------------------------
+MYSQL_HOST = '43.139.14.225'
+MYSQL_PORT = 3306
+MYSQL_USER = 'picker'
+MYSQL_PASSWORD = 'kmcNbbz6TbSihttZ'
+MYSQL_DB = 'stock_pro'
+MYSQL_TABLE = 'articles'  # 可选，默认使用spider名称
+MYSQL_BATCH_SIZE = 500
+
+# asyncmy专属配置
+MYSQL_POOL_MIN = 5  # 连接池最小连接数
+MYSQL_POOL_MAX = 20  # 连接池最大连接数
+
+# 选择下载器
+# DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"
+
+MIDDLEWARES = [
+    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
+    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
+    'crawlo.middleware.response_filter.ResponseFilterMiddleware',
+    'crawlo.middleware.retry.RetryMiddleware',
+    'crawlo.middleware.response_code.ResponseCodeMiddleware',
+    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
+    # 'baidu_spider.middleware.TestMiddleWare',
+    # 'baidu_spider.middleware.TestMiddleWare2'
+]
+
+EXTENSIONS = [
+    'crawlo.extension.log_interval.LogIntervalExtension',
+    'crawlo.extension.log_stats.LogStats',
+]
+
+PIPELINES = [
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
+    'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',  # 或 AiomysqlMySQLPipeline
+    # 'crawlo.pipelines.mysql_batch_pipline.AsyncmyMySQLPipeline',  # 或 AiomysqlMySQLPipeline
+    # 'baidu_spider.pipeline.TestPipeline',
+    # 'baidu_spider.pipeline.MongoPipeline',
+]
+
+USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36'
+DEFAULT_HEADERS = {
+    "accept": "application/json, text/javascript, */*; q=0.01",
+    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "cache-control": "no-cache",
+    "pragma": "no-cache",
+    "priority": "u=1, i",
+    "sec-ch-ua": "\"Chromium\";v=\"136\", \"Google Chrome\";v=\"136\", \"Not.A/Brand\";v=\"99\"",
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": "\"macOS\"",
+    "sec-fetch-dest": "empty",
+    "sec-fetch-mode": "cors",
+    "sec-fetch-site": "same-origin",
+    # "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
+    "x-requested-with": "XMLHttpRequest"
+}
+
+# --------------------------------------DB ---------------------------------------------
+Mongo_Params = ''
+MONGODB_DB = 'news'
+
+REDIS_TTL = 0
+CLEANUP_FP = False
+
+FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'
+# FILTER_CLASS = 'crawlo.filters.redis_filter.RedisFilter'
+# FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFileFilter'

tests/baidu_spider/spiders/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-05-11 12:20
+# @Author  :   oscar
+# @Desc    :   None
+"""

tests/baidu_spider/spiders/bai_du.py

@@ -0,0 +1,61 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 13:05
+# @Author  :   oscar
+# @Desc    :   None
+"""
+import asyncio
+from crawlo import Request
+from crawlo.spider import Spider
+
+from items import BauDuItem
+
+
+class BaiDuSpider(Spider):
+    start_urls = ["https://www.baidu.com/", "https://www.baidu.com/"]
+
+    custom_settings = {
+        'CONCURRENCY': 1
+    }
+
+    name = "bai_du"
+
+    # headers = {
+    #     "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
+    # }
+    #
+    user_gent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
+
+    async def parse(self, response):
+        for i in range(5):
+            url = f"https://www.baidu.com"
+            # url = f"https://www.httpbin.org/404"
+            r = Request(url=url, callback=self.parse_page, dont_filter=True)
+            yield r
+
+    async def parse_page(self, response):
+        for i in range(5):
+            url = f"https://www.baidu.com"
+            meta = {'test': 'hhhh'}
+            r = Request(url=url, callback=self.parse_detail, meta=meta, dont_filter=False)
+            yield r
+
+    def parse_detail(self, response):
+        item = BauDuItem()
+        item['title'] = response.xpath('//title/text()').get()
+
+        item['url'] = response.url
+
+        yield item
+
+    async def spider_opened(self):
+        pass
+
+    async def spider_closed(self):
+        pass
+
+
+if __name__ == '__main__':
+    b = BaiDuSpider()
+    b.start_requests()

tests/baidu_spider/spiders/sina.py

@@ -0,0 +1,79 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+# @Time    :    2025-02-05 13:05
+# @Author  :   oscar
+# @Desc    :   None
+"""
+import time
+
+from crawlo import Request
+from crawlo.spider import Spider
+from crawlo.utils.date_tools import timestamp_to_datetime, format_datetime
+
+from tests.baidu_spider.items import ArticleItem
+
+
+class SinaSpider(Spider):
+    # 获取当前时间戳，并减去 10 分钟（600 秒）
+    current_time_minus_10min = int(time.time()) - 6000
+    # 构造 URL
+    url = f'https://news.10jqka.com.cn/tapp/news/push/stock/?page=1&tag=&track=website&ctime={current_time_minus_10min}'
+
+    start_urls = [url]
+    name = 'sina'
+    # mysql_table = 'news_10jqka'
+
+    allowed_domains = ['*']
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield Request(url=url, callback=self.parse, dont_filter=True)
+
+    async def parse(self, response):
+        jsonp_str = response.json()
+        rows = jsonp_str.get('data', {}).get('list', [])
+        for row in rows:
+            article_id = row.get('id')
+            title = row.get('title')
+            digest = row.get('digest')
+            short = row.get('short')
+            detail_url = row.get('url')
+            tag = row.get('tag')
+            ctime = row.get('ctime')
+            source = row.get('source')
+            meta = {
+                'article_id': article_id,
+                'title': title,
+                'digest': digest,
+                'short': short,
+                'detail_url': detail_url,
+                'source': source,
+                'tag': tag,
+                'ctime': timestamp_to_datetime(int(ctime))
+            }
+
+            yield Request(url=detail_url, callback=self.parse_detail, encoding='gbk', meta=meta)
+
+    @staticmethod
+    async def parse_detail(response):
+        item = ArticleItem()
+        meta = response.meta
+        content = ''.join(response.xpath('//*[@id="contentApp"]/p/text()').extract()).strip()
+        ctime = meta.get('ctime')
+        item['article_id'] = meta.get('article_id')
+        item['title'] = meta.get('title')
+        item['digest'] = content
+        item['short'] = meta.get('short')
+        item['url'] = meta.get('detail_url')
+        item['tag'] = meta.get('tag').strip()
+        item['ctime'] = format_datetime(ctime)
+        item['source'] = meta.get('source')
+
+        yield item
+
+    async def spider_opened(self):
+        pass
+
+    async def spider_closed(self):
+        pass

crawlo/filters/redis_filter.py

@@ -1,120 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import redis
-
-from crawlo import Request
-from crawlo.filters import BaseFilter
-from crawlo.utils.log import get_logger
-from crawlo.utils.request import request_fingerprint
-
-
-class RedisFilter(BaseFilter):
-    """使用Redis集合实现的同步请求去重过滤器"""
-
-    def __init__(
-            self,
-            redis_key: str,
-            client: redis.Redis,
-            stats: dict,
-            debug: bool,
-            log_level: str,
-            save_fp: bool
-    ):
-        """
-        初始化过滤器
-
-        :param redis_key: Redis存储键名
-        :param client: redis客户端实例
-        :param stats: 统计字典
-        :param debug: 是否启用调试模式
-        :param log_level: 日志级别
-        :param save_fp: 是否保留指纹数据
-        """
-        self.logger = get_logger(self.__class__.__name__, log_level)
-        super().__init__(self.logger, stats, debug)
-
-        self.redis_key = redis_key
-        self.redis = client
-        self.save_fp = save_fp
-
-    @classmethod
-    def create_instance(cls, crawler) -> 'BaseFilter':
-        """工厂方法创建实例"""
-        redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
-        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', True)
-
-        try:
-            # 添加连接池配置
-            redis_client = redis.from_url(
-                redis_url,
-                decode_responses=decode_responses,
-                socket_timeout=5,  # 超时设置
-                socket_connect_timeout=5,
-                max_connections=20  # 连接池大小
-            )
-            # 测试连接是否有效
-            redis_client.ping()
-        except redis.RedisError as e:
-            raise RuntimeError(f"Redis连接失败: {str(e)}")
-
-        return cls(
-            redis_key=f"{crawler.settings.get('PROJECT_NAME')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
-            client=redis_client,
-            stats=crawler.stats,
-            save_fp=crawler.settings.get_bool('SAVE_FP', False),
-            debug=crawler.settings.get_bool('FILTER_DEBUG', False),
-            log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
-        )
-
-    def requested(self, request: Request) -> bool:
-        """
-        检查请求是否已存在
-
-        :param request: 请求对象
-        :return: 是否重复
-        """
-        fp = request_fingerprint(request)
-        try:
-            if self.redis.sismember(self.redis_key, fp):
-                self.logger.debug(f"重复请求: {fp}")
-                return True
-
-            self.add_fingerprint(fp)
-            return False
-        except redis.RedisError as e:
-            self.logger.error(f"Redis操作失败: {str(e)}")
-            raise
-
-    def add_fingerprint(self, fp: str) -> None:
-        """添加指纹到Redis集合"""
-        try:
-            self.redis.sadd(self.redis_key, fp)
-            self.logger.debug(f"新增指纹: {fp}")
-        except redis.RedisError as e:
-            self.logger.error(f"指纹添加失败: {str(e)}")
-            raise
-
-    def __contains__(self, item) -> bool:
-        """支持 in 操作符检查 (必须返回bool类型)"""
-        try:
-            # 显式将redis返回的0/1转换为bool
-            return bool(self.redis.sismember(self.redis_key, item))
-        except redis.RedisError as e:
-            self.logger.error(f"Redis查询失败: {str(e)}")
-            raise
-
-    def close(self) -> None:
-        """同步清理方法（注意不是异步的closed）"""
-        if not self.save_fp:
-            try:
-                count = self.redis.delete(self.redis_key)
-                self.logger.info(f"已清理Redis键 {self.redis_key}, 删除数量: {count}")
-            except redis.RedisError as e:
-                self.logger.error(f"清理失败: {str(e)}")
-            finally:
-                # 同步客户端需要手动关闭连接池
-                self.redis.close()
-
-    async def closed(self):
-        """兼容异步接口的同步实现"""
-        self.close()

crawlo-1.0.2.dist-info/RECORD

@@ -1,68 +0,0 @@
-crawlo/__init__.py,sha256=BoRtaB19VFlByP3JKzXQbmg4Jb6i6yVnpTR3jCSrCig,208
-crawlo/__version__.py,sha256=FlR3yFykEvKzEITpEQq_qx6Uq29lYlhxcnSHeRP1LgI,23
-crawlo/crawler.py,sha256=XLiDyFyoiJr8BzhLbCB15zbccVHQiiLGDM1zFUkdrAI,8544
-crawlo/event.py,sha256=ZhoPW5CglCEuZNFEwviSCBIw0pT5O6jT98bqYrDFd3E,324
-crawlo/exceptions.py,sha256=trxM2c0jw50QsGSoFAKC2RrKpapOFHQDq0wQuLWqmKE,980
-crawlo/stats_collector.py,sha256=jhAW8k0SzjqelkpiWpfGmMw2DBkgTjpwnObqTNDOp6A,1286
-crawlo/subscriber.py,sha256=udlHeTR0ymGQhCDxVUGwUzeeeR4TYCEJrJwFnkgr0cU,3836
-crawlo/task_manager.py,sha256=D9m-nqnGj-FZPtGk4CdwZX3Gw7IWyYvTS7CHpRGWc_w,748
-crawlo/core/__init__.py,sha256=JYSAn15r8yWgRK_Nc69t_8tZCyb70MiPZKssA8wrYz0,43
-crawlo/core/engine.py,sha256=JFHooPp-5cfHSyxEh87nOOR5NMaPLVDfNSqAsbtx4PM,6030
-crawlo/core/processor.py,sha256=oHLs-cno0bJGTNc9NGD2S7_2-grI3ruvggO0SY2mf3Q,1180
-crawlo/core/scheduler.py,sha256=ZwPoU_QRjs9wwrxdt-MGPwsSmKhvvhgmcnBllDrXnhg,2014
-crawlo/downloader/__init__.py,sha256=72u2Hef4HaMfs9VCqEjbMtiaRXbaXmgNiJn6qy09LHs,2384
-crawlo/downloader/aiohttp_downloader.py,sha256=4C2BDloKzwss16kfD7tH0WPugPbSSFxl-5-_DLWB0vM,3676
-crawlo/downloader/httpx_downloader.py,sha256=ra6Ae_lv8pNyvLzPQYBgTNuBdMVBYi86kNt2OdZlcSo,1704
-crawlo/downloader/playwright_downloader.py,sha256=mEGlSd6A6sN0Wyq-TDkownIElOgxnwVfY3rS5wtLoYY,6726
-crawlo/extension/__init__.py,sha256=O2BVK1U3WwmurZb-PaYVz3g1tZ_iYUjCwilmUKf6844,1170
-crawlo/extension/log_interval.py,sha256=FOWeTOuWtOpCz2UPV5F_--QIa8yomltSpjxbw3F7bkU,1971
-crawlo/extension/log_stats.py,sha256=-V7ipdIfYMQdp1ZDc4kvNEAIHIR74U1ZHV5FhlLyGGU,1786
-crawlo/filters/__init__.py,sha256=9fJQRVkxWWPChajYbAGe1O6UYB639xWt0hiLUGBs4hQ,1014
-crawlo/filters/aioredis_filter.py,sha256=H_HAFfE9mHnPrzQcjyXXCseQ77iN4nT9a3lukHiu8M8,4874
-crawlo/filters/memory_filter.py,sha256=pk2o0kbX0zrGLJ6poKhSa-cfOmsp68fA65hXEGQCQ5M,6895
-crawlo/filters/redis_filter.py,sha256=m1nRsf_3slnWSb80RtTEURi5kwjdI0xotoFxnscx974,4211
-crawlo/items/__init__.py,sha256=o5BSpS1Byivr-bpdfFgc9GCoGi8ThNuPJiTW7lz85-I,2125
-crawlo/items/items.py,sha256=myOOjWaSByKW0r8SxIQ0bxS3PXjUDE1c-Pe38z6fSF8,4108
-crawlo/middleware/__init__.py,sha256=PSwpRLdBUopaQzBp1S0zK_TZbrRagQ4yzvgyLy4tBk8,570
-crawlo/middleware/default_header.py,sha256=OVW4vpRPp3Y6qYXtiEYlGqVjCYcbuv1Iecc7zEgwCsI,1099
-crawlo/middleware/download_delay.py,sha256=P2eyAJXwdLdC4yYuLhvKZVa1b5YQvQD0GpsR8aDW8-8,994
-crawlo/middleware/middleware_manager.py,sha256=T4axTY89Z0BOwaWDWcUTABeDNTvyPFiyrbwj-H4sbSA,6629
-crawlo/middleware/request_ignore.py,sha256=jdybWFVXuA5YsAPfZJFzLTWkYhEAewNgxuhFqczPW9M,1027
-crawlo/middleware/response_code.py,sha256=vgXWv3mMu_v9URvhKA9myIFH4u6L4EwNme80wL4DCGc,677
-crawlo/middleware/response_filter.py,sha256=O2gkV_Yjart8kmmXTGzrtZnb_Uuefap4uL2Cu01iRs4,863
-crawlo/middleware/retry.py,sha256=9SnE7l3Nhh143AqCiL0gfE6dl_gF1Kr6CjoNxvMH_Ps,3465
-crawlo/network/__init__.py,sha256=DVz1JpasjxCgOlXvm76gz-S18OXr4emG_J39yi5iVuA,130
-crawlo/network/request.py,sha256=yUAL6oecm1TniD9dsBmRedEaEHzh3rtr_6p3dMK2EfQ,8195
-crawlo/network/response.py,sha256=amnLEExKq11dLh6m_YdSqdKLl0srh1eeY5uPz15fwyo,6055
-crawlo/pipelines/__init__.py,sha256=IbXJ6B8LqxVVjeLNgL_12AxV6zbV8hNRQxAfMLjjSaw,273
-crawlo/pipelines/console_pipeline.py,sha256=bwe5hZgaVSWmh3R8XpOaaeAjJme-Ttrpo6G6f1cnLIg,1287
-crawlo/pipelines/mongo_pipeline.py,sha256=Yr48D0T61-_Y-EpgWXf7BUn9w8e-Pj5P07QDSPZ0pYU,4558
-crawlo/pipelines/mysql_batch_pipline.py,sha256=7KXd0IUV0h3IViD8R0iruyWv5XdZR1pANB8EY9z6iMI,5022
-crawlo/pipelines/mysql_pipeline.py,sha256=TzyaBg1oBj9pgzuVUrb5VVtkiwZH6gqP_6IYM2QCkLs,8052
-crawlo/pipelines/pipeline_manager.py,sha256=k-Rg0os0Havrov99D-Jn3ROpnz154K30tf7aARE5W3k,2174
-crawlo/settings/__init__.py,sha256=NgYFLfk_Bw7h6KSoepJn_lMBSqVbCHebjKxaE3_eMgw,130
-crawlo/settings/default_settings.py,sha256=JS1QKYe7jkdFlOjqZ-eOeOcVS3AXCZynoNH95GuEnds,2556
-crawlo/settings/setting_manager.py,sha256=4xXOzKwZCgAp8ybwvVcs2R--CsOD7c6dBIkj6DJHB3c,2998
-crawlo/spider/__init__.py,sha256=1tmKkr2-oJi0w9r2ho9nn6Z_VDn18pjXHXU0Hv2eheY,941
-crawlo/templates/item_template.tmpl,sha256=0bGFnlwJRqstxMNEj1H_pEICybwoueRhs31QaDPXrS0,372
-crawlo/templates/spider_template.tmpl,sha256=JzphuA87Yl_F1xR9zOIi_ZSazyT8eSNPxYYPMv3Uiko,835
-crawlo/templates/project_template/main.py,sha256=BcCP294ycCPsHi_AMN7OAJtcrLvQdf91meH93PqbQgs,626
-crawlo/templates/project_template/setting.py,sha256=Ce4nMbrdhL1ioRdTcB0vV_vK_50cfnwVqSvt49QsNkA,9395
-crawlo/templates/project_template/items/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/templates/project_template/spiders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-crawlo/utils/__init__.py,sha256=XCYumI8wJ1jU_Myn_K0LT-LVygPDUCdETCbXM3EWvlo,130
-crawlo/utils/concurrency_manager.py,sha256=o-_cfeUHdlBOM3eAXF857MtekSrRcVTBJ2jWZvY6weQ,5230
-crawlo/utils/date_tools.py,sha256=Y7pnGNn1-5vkiHtydAgmQ-qR3pSO30k5WEYigOPifPQ,5496
-crawlo/utils/func_tools.py,sha256=y-TYP9H3X67MS_foWy9Z2LIS6GP7Y4Cy3T168ulq3Jc,2451
-crawlo/utils/log.py,sha256=LU0J3boPCL-Kynx3wR_CAryRgScNmPPn4pBitLrrsX4,1028
-crawlo/utils/pqueue.py,sha256=HDgX4HAkc7RqYUtX6q51tzI1ZRTACf8P_4jLqC4-uC0,5559
-crawlo/utils/project.py,sha256=FfBaMfxcau4yL59O-DfD7FAii8k6gXWQmQ1YU6aaUCE,1544
-crawlo/utils/request.py,sha256=BEBtxwejvQw5euEiSclHCbqkNcBwUMY7KPGGkvj8BjE,2936
-crawlo/utils/system.py,sha256=24zGmtHNhDFMGVo7ftMV-Pqg6_5d63zsyNey9udvJJk,248
-crawlo/utils/tools.py,sha256=U7xGKgXe2PmMyvNxZ1vlJYcv4Ei1WhIBMf8XcJZ7XCY,9764
-crawlo/utils/url.py,sha256=RKe_iqdjafsNcp-P2GVLYpsL1qbxiuZLiFc-SqOQkcs,1521
-crawlo-1.0.2.dist-info/METADATA,sha256=kzNv4kckIbSZCADon5g2Ik5QJhubNxWra0-aXX6bMug,1784
-crawlo-1.0.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.0.2.dist-info/entry_points.txt,sha256=GD9PBhKQN83EaxPYtz7NhcGeZeh3bdr2jWbTixOs-lw,59
-crawlo-1.0.2.dist-info/top_level.txt,sha256=Dwuv-Y1aGSJD3mjFrCdNGQ8EHroMj7RgVcxDdcczx4k,7
-crawlo-1.0.2.dist-info/RECORD,,