crawlo 1.0.4__py3-none-any.whl → 1.0.5__py3-none-any.whl

tests/baidu_spider/settings.py

@@ -1,80 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-
-PROJECT_NAME = 'baidu_spider'
-
-CONCURRENCY = 4
-
-USE_SESSION = True
-
-# 下载延迟
-DOWNLOAD_DELAY = 0.5
-RANDOMNESS = False
-
-# --------------------------------------------------- 公共MySQL配置 -----------------------------------------------------
-MYSQL_HOST = '43.139.14.225'
-MYSQL_PORT = 3306
-MYSQL_USER = 'picker'
-MYSQL_PASSWORD = 'kmcNbbz6TbSihttZ'
-MYSQL_DB = 'stock_pro'
-MYSQL_TABLE = 'articles'  # 可选，默认使用spider名称
-MYSQL_BATCH_SIZE = 500
-
-# asyncmy专属配置
-MYSQL_POOL_MIN = 5  # 连接池最小连接数
-MYSQL_POOL_MAX = 20  # 连接池最大连接数
-
-# 选择下载器
-# DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"
-
-MIDDLEWARES = [
-    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
-    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
-    'crawlo.middleware.response_filter.ResponseFilterMiddleware',
-    'crawlo.middleware.retry.RetryMiddleware',
-    'crawlo.middleware.response_code.ResponseCodeMiddleware',
-    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
-    # 'baidu_spider.middleware.TestMiddleWare',
-    # 'baidu_spider.middleware.TestMiddleWare2'
-]
-
-EXTENSIONS = [
-    'crawlo.extension.log_interval.LogIntervalExtension',
-    'crawlo.extension.log_stats.LogStats',
-]
-
-PIPELINES = [
-    'crawlo.pipelines.console_pipeline.ConsolePipeline',
-    'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',  # 或 AiomysqlMySQLPipeline
-    # 'crawlo.pipelines.mysql_batch_pipline.AsyncmyMySQLPipeline',  # 或 AiomysqlMySQLPipeline
-    # 'baidu_spider.pipeline.TestPipeline',
-    # 'baidu_spider.pipeline.MongoPipeline',
-]
-
-USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36'
-DEFAULT_HEADERS = {
-    "accept": "application/json, text/javascript, */*; q=0.01",
-    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
-    "cache-control": "no-cache",
-    "pragma": "no-cache",
-    "priority": "u=1, i",
-    "sec-ch-ua": "\"Chromium\";v=\"136\", \"Google Chrome\";v=\"136\", \"Not.A/Brand\";v=\"99\"",
-    "sec-ch-ua-mobile": "?0",
-    "sec-ch-ua-platform": "\"macOS\"",
-    "sec-fetch-dest": "empty",
-    "sec-fetch-mode": "cors",
-    "sec-fetch-site": "same-origin",
-    # "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
-    "x-requested-with": "XMLHttpRequest"
-}
-
-# --------------------------------------DB ---------------------------------------------
-Mongo_Params = ''
-MONGODB_DB = 'news'
-
-REDIS_TTL = 0
-CLEANUP_FP = False
-
-FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'
-# FILTER_CLASS = 'crawlo.filters.redis_filter.RedisFilter'
-# FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFileFilter'

tests/baidu_spider/spiders/__init__.py

@@ -1,7 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-05-11 12:20
-# @Author  :   oscar
-# @Desc    :   None
-"""

tests/baidu_spider/spiders/bai_du.py

@@ -1,61 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 13:05
-# @Author  :   oscar
-# @Desc    :   None
-"""
-import asyncio
-from crawlo import Request
-from crawlo.spider import Spider
-
-from items import BauDuItem
-
-
-class BaiDuSpider(Spider):
-    start_urls = ["https://www.baidu.com/", "https://www.baidu.com/"]
-
-    custom_settings = {
-        'CONCURRENCY': 1
-    }
-
-    name = "bai_du"
-
-    # headers = {
-    #     "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
-    # }
-    #
-    user_gent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
-
-    async def parse(self, response):
-        for i in range(5):
-            url = f"https://www.baidu.com"
-            # url = f"https://www.httpbin.org/404"
-            r = Request(url=url, callback=self.parse_page, dont_filter=True)
-            yield r
-
-    async def parse_page(self, response):
-        for i in range(5):
-            url = f"https://www.baidu.com"
-            meta = {'test': 'hhhh'}
-            r = Request(url=url, callback=self.parse_detail, meta=meta, dont_filter=False)
-            yield r
-
-    def parse_detail(self, response):
-        item = BauDuItem()
-        item['title'] = response.xpath('//title/text()').get()
-
-        item['url'] = response.url
-
-        yield item
-
-    async def spider_opened(self):
-        pass
-
-    async def spider_closed(self):
-        pass
-
-
-if __name__ == '__main__':
-    b = BaiDuSpider()
-    b.start_requests()

tests/baidu_spider/spiders/sina.py

@@ -1,79 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 13:05
-# @Author  :   oscar
-# @Desc    :   None
-"""
-import time
-
-from crawlo import Request
-from crawlo.spider import Spider
-from crawlo.utils.date_tools import timestamp_to_datetime, format_datetime
-
-from tests.baidu_spider.items import ArticleItem
-
-
-class SinaSpider(Spider):
-    # 获取当前时间戳，并减去 10 分钟（600 秒）
-    current_time_minus_10min = int(time.time()) - 6000
-    # 构造 URL
-    url = f'https://news.10jqka.com.cn/tapp/news/push/stock/?page=1&tag=&track=website&ctime={current_time_minus_10min}'
-
-    start_urls = [url]
-    name = 'sina'
-    # mysql_table = 'news_10jqka'
-
-    allowed_domains = ['*']
-
-    def start_requests(self):
-        for url in self.start_urls:
-            yield Request(url=url, callback=self.parse, dont_filter=True)
-
-    async def parse(self, response):
-        jsonp_str = response.json()
-        rows = jsonp_str.get('data', {}).get('list', [])
-        for row in rows:
-            article_id = row.get('id')
-            title = row.get('title')
-            digest = row.get('digest')
-            short = row.get('short')
-            detail_url = row.get('url')
-            tag = row.get('tag')
-            ctime = row.get('ctime')
-            source = row.get('source')
-            meta = {
-                'article_id': article_id,
-                'title': title,
-                'digest': digest,
-                'short': short,
-                'detail_url': detail_url,
-                'source': source,
-                'tag': tag,
-                'ctime': timestamp_to_datetime(int(ctime))
-            }
-
-            yield Request(url=detail_url, callback=self.parse_detail, encoding='gbk', meta=meta)
-
-    @staticmethod
-    async def parse_detail(response):
-        item = ArticleItem()
-        meta = response.meta
-        content = ''.join(response.xpath('//*[@id="contentApp"]/p/text()').extract()).strip()
-        ctime = meta.get('ctime')
-        item['article_id'] = meta.get('article_id')
-        item['title'] = meta.get('title')
-        item['digest'] = content
-        item['short'] = meta.get('short')
-        item['url'] = meta.get('detail_url')
-        item['tag'] = meta.get('tag').strip()
-        item['ctime'] = format_datetime(ctime)
-        item['source'] = meta.get('source')
-
-        yield item
-
-    async def spider_opened(self):
-        pass
-
-    async def spider_closed(self):
-        pass