crawlo 1.0.3__py3-none-any.whl → 1.0.5__py3-none-any.whl

tests/baidu_spider/pipeline.py

@@ -1,55 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import pymongo
-
-from motor.motor_asyncio import AsyncIOMotorClient
-
-from random import randint
-
-from crawlo.event import spider_closed
-from crawlo.exceptions import ItemDiscard
-from crawlo.utils.log import get_logger
-
-
-class TestPipeline(object):
-
-    async def process_item(self, item, spider):
-        if randint(1, 3) == 1:
-            raise ItemDiscard('重复数据')
-        return item
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs):
-        return cls()
-
-
-class MongoPipeline(object):
-
-    def __init__(self, conn, col):
-        self.conn = conn
-        self.col = col
-
-        self.logger = get_logger(self.__class__.__name__)
-
-    @classmethod
-    def create_instance(cls, crawler):
-        settings = crawler.settings
-        mongo_params = settings.get('MONGODB_PARAMS', None)
-        db_name = settings.get('MONGODB_DB', None)
-        project_name = settings.get('PROJECT_NAME', None)
-
-        conn = AsyncIOMotorClient(**mongo_params) if mongo_params else AsyncIOMotorClient()
-
-        col = conn[db_name][project_name]
-        o = cls(conn, col)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-        return o
-
-    async def process_item(self, item, spider):
-        await self.col.insert_one(item.to_dict())
-        return item
-
-    async def spider_closed(self):
-        self.logger.info('MongoDB closed.')
-        self.conn.close()
-

tests/baidu_spider/request_fingerprints.txt

@@ -1,9 +0,0 @@
-48e727ce8566d65e5233eaac29498b03e2908dd78a90dad7fdd7510e8ada9e32
-c5e78d04ca9b1113e6a8076792aaa7b09ff4b040bd790c684689745aa7edb1ae
-8287f5ad5e2f06687e88cc31d64fdbd3b1b56cee71fbc3344ad8cea852ea9dd3
-f84661b1d15a6e96c6a77a6484c173be3fb502f73e256e8f72f98982674a7992
-f5c1693afa1293e758331a8e95aa6277ffa49105ccd0d79115d8e85375863adc
-f6f2175b1ae909ac0dd41aa2ed735b8305dde6f92d51dd2e411a0c695cfc4843
-3ba6793c55838d267567f6b65b3406bbad30e89e187d3fbe88e6ae55db24dd9d
-488f1d28fe532f1113f634dfa58a2bccae1d34af5421e4064e2ae024d5010280
-dbf802098ea25af78c5751fdc750624296e79d9e1d968e33e5956860ebb5ecc7

tests/baidu_spider/run.py

@@ -1,27 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 13:12
-# @Author  :   oscar
-# @Desc    :   None
-"""
-import asyncio
-from crawlo.crawler import CrawlerProcess
-
-# from crawlo.utils import system as _
-from tests.baidu_spider.spiders.bai_du import BaiDuSpider
-from crawlo.utils.project import get_settings
-from tests.baidu_spider.spiders.sina import SinaSpider
-
-
-async def main():
-    settings = get_settings()
-    process = CrawlerProcess(settings)
-    # await process.crawl(BaiDuSpider)
-    await process.crawl(SinaSpider)
-
-    await process.start()
-
-if __name__ == '__main__':
-    asyncio.run(main())
-    # 观看到第18集

tests/baidu_spider/settings.py

@@ -1,78 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-
-PROJECT_NAME = 'baidu_spider'
-
-CONCURRENCY = 4
-
-USE_SESSION = True
-
-# 下载延迟
-DOWNLOAD_DELAY = 0.5
-RANDOMNESS = False
-
-# --------------------------------------------------- 公共MySQL配置 -----------------------------------------------------
-MYSQL_HOST = '43.139.14.225'
-MYSQL_PORT = 3306
-MYSQL_USER = 'picker'
-MYSQL_PASSWORD = 'kmcNbbz6TbSihttZ'
-MYSQL_DB = 'stock_pro'
-MYSQL_TABLE = 'articles'  # 可选，默认使用spider名称
-MYSQL_BATCH_SIZE = 500
-
-# asyncmy专属配置
-MYSQL_POOL_MIN = 5  # 连接池最小连接数
-MYSQL_POOL_MAX = 20  # 连接池最大连接数
-
-# 选择下载器
-# DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"
-
-MIDDLEWARES = [
-    'crawlo.middleware.download_delay.DownloadDelayMiddleware',
-    'crawlo.middleware.default_header.DefaultHeaderMiddleware',
-    'crawlo.middleware.response_filter.ResponseFilterMiddleware',
-    'crawlo.middleware.retry.RetryMiddleware',
-    'crawlo.middleware.response_code.ResponseCodeMiddleware',
-    'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',
-    # 'baidu_spider.middleware.TestMiddleWare',
-    # 'baidu_spider.middleware.TestMiddleWare2'
-]
-
-EXTENSIONS = [
-    'crawlo.extension.log_interval.LogIntervalExtension',
-    'crawlo.extension.log_stats.LogStats',
-]
-
-PIPELINES = [
-    'crawlo.pipelines.console_pipeline.ConsolePipeline',
-    'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',  # 或 AiomysqlMySQLPipeline
-    # 'crawlo.pipelines.mysql_batch_pipline.AsyncmyMySQLPipeline',  # 或 AiomysqlMySQLPipeline
-    # 'baidu_spider.pipeline.TestPipeline',
-    # 'baidu_spider.pipeline.MongoPipeline',
-]
-
-USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36'
-DEFAULT_HEADERS = {
-    "accept": "application/json, text/javascript, */*; q=0.01",
-    "accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
-    "cache-control": "no-cache",
-    "pragma": "no-cache",
-    "priority": "u=1, i",
-    "sec-ch-ua": "\"Chromium\";v=\"136\", \"Google Chrome\";v=\"136\", \"Not.A/Brand\";v=\"99\"",
-    "sec-ch-ua-mobile": "?0",
-    "sec-ch-ua-platform": "\"macOS\"",
-    "sec-fetch-dest": "empty",
-    "sec-fetch-mode": "cors",
-    "sec-fetch-site": "same-origin",
-    # "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
-    "x-requested-with": "XMLHttpRequest"
-}
-
-# --------------------------------------DB ---------------------------------------------
-Mongo_Params = ''
-MONGODB_DB = 'news'
-
-CLEANUP_FP = True
-
-FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter'
-# FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFileFilter'

tests/baidu_spider/spiders/__init__.py

@@ -1,7 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-05-11 12:20
-# @Author  :   oscar
-# @Desc    :   None
-"""

tests/baidu_spider/spiders/bai_du.py

@@ -1,61 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 13:05
-# @Author  :   oscar
-# @Desc    :   None
-"""
-import asyncio
-from crawlo import Request
-from crawlo.spider import Spider
-
-from items import BauDuItem
-
-
-class BaiDuSpider(Spider):
-    start_urls = ["https://www.baidu.com/", "https://www.baidu.com/"]
-
-    custom_settings = {
-        'CONCURRENCY': 1
-    }
-
-    name = "bai_du"
-
-    # headers = {
-    #     "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
-    # }
-    #
-    user_gent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"
-
-    async def parse(self, response):
-        for i in range(5):
-            url = f"https://www.baidu.com"
-            # url = f"https://www.httpbin.org/404"
-            r = Request(url=url, callback=self.parse_page, dont_filter=True)
-            yield r
-
-    async def parse_page(self, response):
-        for i in range(5):
-            url = f"https://www.baidu.com"
-            meta = {'test': 'hhhh'}
-            r = Request(url=url, callback=self.parse_detail, meta=meta, dont_filter=False)
-            yield r
-
-    def parse_detail(self, response):
-        item = BauDuItem()
-        item['title'] = response.xpath('//title/text()').get()
-
-        item['url'] = response.url
-
-        yield item
-
-    async def spider_opened(self):
-        pass
-
-    async def spider_closed(self):
-        pass
-
-
-if __name__ == '__main__':
-    b = BaiDuSpider()
-    b.start_requests()

tests/baidu_spider/spiders/sina.py

@@ -1,79 +0,0 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-02-05 13:05
-# @Author  :   oscar
-# @Desc    :   None
-"""
-import time
-
-from crawlo import Request
-from crawlo.spider import Spider
-from crawlo.utils.date_tools import timestamp_to_datetime, format_datetime
-
-from tests.baidu_spider.items import ArticleItem
-
-
-class SinaSpider(Spider):
-    # 获取当前时间戳，并减去 10 分钟（600 秒）
-    current_time_minus_10min = int(time.time()) - 6000
-    # 构造 URL
-    url = f'https://news.10jqka.com.cn/tapp/news/push/stock/?page=1&tag=&track=website&ctime={current_time_minus_10min}'
-
-    start_urls = [url]
-    name = 'sina'
-    # mysql_table = 'news_10jqka'
-
-    allowed_domains = ['*']
-
-    def start_requests(self):
-        for url in self.start_urls:
-            yield Request(url=url, callback=self.parse, dont_filter=True)
-
-    async def parse(self, response):
-        jsonp_str = response.json()
-        rows = jsonp_str.get('data', {}).get('list', [])
-        for row in rows:
-            article_id = row.get('id')
-            title = row.get('title')
-            digest = row.get('digest')
-            short = row.get('short')
-            detail_url = row.get('url')
-            tag = row.get('tag')
-            ctime = row.get('ctime')
-            source = row.get('source')
-            meta = {
-                'article_id': article_id,
-                'title': title,
-                'digest': digest,
-                'short': short,
-                'detail_url': detail_url,
-                'source': source,
-                'tag': tag,
-                'ctime': timestamp_to_datetime(int(ctime))
-            }
-
-            yield Request(url=detail_url, callback=self.parse_detail, encoding='gbk', meta=meta)
-
-    @staticmethod
-    async def parse_detail(response):
-        item = ArticleItem()
-        meta = response.meta
-        content = ''.join(response.xpath('//*[@id="contentApp"]/p/text()').extract()).strip()
-        ctime = meta.get('ctime')
-        item['article_id'] = meta.get('article_id')
-        item['title'] = meta.get('title')
-        item['digest'] = content
-        item['short'] = meta.get('short')
-        item['url'] = meta.get('detail_url')
-        item['tag'] = meta.get('tag').strip()
-        item['ctime'] = format_datetime(ctime)
-        item['source'] = meta.get('source')
-
-        yield item
-
-    async def spider_opened(self):
-        pass
-
-    async def spider_closed(self):
-        pass