fast-crawl-core 0.0.1__tar.gz

fast_crawl_core-0.0.1/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.1
+Name: fast-crawl-core
+Version: 0.0.1
+Summary: fast crawl core for zsodata
+Home-page: http://www.zsodata.com
+Author: zsodata
+Author-email: team@zso.io
+License: BSD License
+Platform: all
+Requires-Python: >=3.7
+Requires-Dist: zcbot-url-parser
+
+pip uninstall fast-crawl-core
+
+pip install --upgrade fast-crawl-core -i https://pypi.python.org/simple
+pip show fast-crawl-core

fast_crawl_core-0.0.1/README.rst

@@ -0,0 +1,4 @@
+pip uninstall fast-crawl-core
+
+pip install --upgrade fast-crawl-core -i https://pypi.python.org/simple
+pip show fast-crawl-core

fast_crawl_core-0.0.1/fast_crawl_core/__init__.py

@@ -0,0 +1 @@
+from __future__ import absolute_import

fast_crawl_core-0.0.1/fast_crawl_core/client/mongo_client.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+import pytz
+from motor.motor_asyncio import AsyncIOMotorClient
+
+from ..util import cfg
+from ..util.exceptions import NoConfigException
+from ..util.decator import singleton
+
+
+@singleton
+class Mongo(object):
+    """
+    Mongo异步客户端简易封装（使用motor）
+    """
+
+    def __init__(self, mongo_uri: str = None, mongo_db: str = None, *args, **kwargs):
+        self.mongo_uri = mongo_uri or cfg.get('ZCBOT_CORE_MONGO_URL')
+        self.mongo_db = mongo_db or cfg.get('ZCBOT_CORE_MONGO_DB')
+        if not self.mongo_uri:
+            raise NoConfigException('mongodb uri not config!')
+        if not self.mongo_db:
+            raise NoConfigException('mongodb database not config!')
+
+        self.client = AsyncIOMotorClient(self.mongo_uri, tz_aware=True, tzinfo=pytz.timezone('Asia/Shanghai'))
+        self.db = self.client[self.mongo_db]
+
+    # 获取集合
+    def get_collection(self, coll):
+        return self.db.get_collection(coll)
+
+    # 查询对象
+    async def get(self, collection, query={}):
+        result = await self.db[collection].find_one(query)
+        return result
+
+    # 统计数量
+    async def count(self, collection, query={}):
+        return await self.db[collection].count_documents(query)
+
+    # 查询列表
+    async def list(self, collection, query={}, fields=None, sort=[]):
+        if fields:
+            cursor = self.db[collection].find(query, fields, sort=sort)
+        else:
+            cursor = self.db[collection].find(query, sort=sort)
+        return await cursor.to_list(length=None)
+
+    # 查询去重列表
+    async def distinct(self, collection, dist_key, query={}, fields=None):
+        return await self.db[collection].find(query, fields).distinct(dist_key)
+
+    # 聚合查询
+    async def aggregate(self, collection, pipeline=[]):
+        cursor = self.db[collection].aggregate(pipeline, allowDiskUse=True)
+        return await cursor.to_list(length=None)
+
+    # 查询分页列表
+    async def list_with_page(self, collection, query={}, page_size=10000, fields=None):
+        rows = list()
+        total = await self.db[collection].count_documents(query)
+        if total > 0 and page_size > 0:
+            total_page = round(total / page_size)
+            for page in range(0, total_page):
+                if fields:
+                    cursor = self.db[collection].find(query, fields).skip(page_size * page).limit(page)
+                else:
+                    cursor = self.db[collection].find(query).skip(page_size * page).limit(page)
+                curr_batch = await cursor.to_list(length=None)
+                if curr_batch:
+                    rows.append(curr_batch)
+        return rows
+
+    # 插入或更新
+    async def insert_or_update(self, collection, data, id_key='_id'):
+        return await self.db[collection].update_one({id_key: data[id_key]}, {'$set': data}, upsert=True)
+
+    # 更新
+    async def update(self, collection, filter, data, multi=False):
+        if multi:
+            return await self.db[collection].update_many(filter, {'$set': data})
+        return await self.db[collection].update_one(filter, {'$set': data})
+
+    # 以主键更新
+    async def update_by_pk(self, collection, pk_val, data):
+        return await self.db[collection].update_one({'_id': pk_val}, {'$set': data})
+
+    # 批量更新
+    async def batch_update(self, collection, filter, datas, multi=False):
+        if multi:
+            return await self.db[collection].update_many(filter, datas)
+        return await self.db[collection].update_one(filter, datas)
+
+    # 删除
+    async def delete(self, collection, filter):
+        return await self.db[collection].delete_many(filter)
+
+    # 批量写入
+    async def bulk_write(self, collection, bulk_list):
+        if bulk_list:
+            return await self.db[collection].bulk_write(bulk_list, ordered=False, bypass_document_validation=True)
+
+    # 关闭链接
+    def close(self):
+        self.client.close()

fast_crawl_core-0.0.1/fast_crawl_core/client/portainer_client.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+import json
+import aiohttp
+from ..util import logger
+from ..util.decator import singleton
+from ..model.entity import PortainerNode, BatchSpider
+
+LOGGER = logger.get('容器')
+
+
+@singleton
+class PortainerClient(object):
+    """
+    Portainer异步客户端简易封装（使用aiohttp）
+    """
+
+    def __init__(self):
+        self.user_agent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3775.400 QQBrowser/10.6.4208.400'
+        # 客户端
+        self.create_api = '{}/api/endpoints/{}/docker/containers/create?name={}'
+        self.start_api = '{}/api/endpoints/{}/docker/containers/{}/start'
+        self.stop_api = '{}/api/endpoints/{}/docker/containers/{}/kill'
+
+    async def create_container(self, node: PortainerNode, spider: BatchSpider, container_name: str, param_data):
+        container_conf = {
+            "Cmd": [
+                "/bin/sh",
+                "-c",
+                spider.param % param_data
+            ],
+            "Env": spider.env,
+            "Image": spider.dockerImage
+        }
+
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                url=self.create_api.format(node.apiBaseUrl, node.endpointId, container_name),
+                headers={
+                    'X-API-Key': node.apiToken,
+                    'User-Agent': self.user_agent,
+                    'Content-Type': 'application/json;charset=UTF-8',
+                },
+                data=json.dumps(container_conf)
+            ) as resp:
+                text = await resp.text()
+                if resp.status in [200, 201]:
+                    data = json.loads(text) or {}
+                    if data.get('Id', None):
+                        id = data.get('Id', '')
+                        LOGGER.info(f'创建成功: id={id}, endpoint={node.endpointId} name={container_name}')
+                        return id
+
+                LOGGER.error(f'创建失败: result={text}, endpoint={node.endpointId} name={container_name}')
+
+    async def start(self, node: PortainerNode, container_id: str):
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                url=self.start_api.format(node.apiBaseUrl, node.endpointId, container_id),
+                headers={
+                    'X-API-Key': node.apiToken,
+                    'User-Agent': self.user_agent,
+                    'Content-Type': 'application/json;charset=UTF-8',
+                }
+            ) as resp:
+                if resp.status in [204, 200]:
+                    LOGGER.info(f'启动成功: id={container_id}')
+                    return container_id
+
+                LOGGER.error(f'启动失败: id={container_id}')
+
+    async def stop(self, node: PortainerNode, container_id: str):
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                url=self.stop_api.format(node.apiBaseUrl, node.endpointId, container_id),
+                headers={
+                    'X-API-Key': node.apiToken,
+                    'User-Agent': self.user_agent,
+                    'Content-Type': 'application/json;charset=UTF-8',
+                }
+            ) as resp:
+                if resp.status in [204, 200]:
+                    LOGGER.info(f'停止成功: id={container_id}')
+                    return container_id
+
+                LOGGER.error(f'停止失败: id={container_id}')

fast_crawl_core-0.0.1/fast_crawl_core/client/redis_client.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+import redis.asyncio as aioredis
+from ..util import cfg
+from ..util.decator import singleton
+from ..util.exceptions import NoConfigException
+
+
+@singleton
+class Redis(object):
+    """
+    Redis异步客户端简易封装（单例，使用redis.asyncio）
+    """
+
+    def __init__(self, redis_uri: str = None, redis_db: int = None, decode_responses=True):
+        self.redis_uri = redis_uri or cfg.get('ZCBOT_CORE_REDIS_URL')
+        self.redis_db = redis_db or cfg.get('ZCBOT_CORE_REDIS_DB')
+        if not self.redis_uri:
+            raise NoConfigException('redis uri not config!')
+        if not self.redis_db:
+            raise NoConfigException('redis database not config!')
+
+        self.client = aioredis.Redis.from_url(url=self.redis_uri, db=self.redis_db, decode_responses=decode_responses)

fast_crawl_core-0.0.1/fast_crawl_core/dao/auth_user_spider_group.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+from typing import List
+from ..client.mongo_client import Mongo
+
+
+# 获取支持网站平台
+async def get_platforms_by_group(group_code: str, enable: int = None):
+    _query = {}
+    if group_code:
+        _query["groupCode"] = group_code
+    if enable is not None:
+        _query["enable"] = enable
+
+    return await Mongo().list('zcbot_auth_user_spider_group', query=_query)
+
+
+async def get_platforms_by_group_code(group_code: str, enable):
+    _query = {
+        "groupCode": group_code
+    }
+    _fields = {
+        "spiders": 0,
+        "_id": 0
+    }
+    return await Mongo().aggregate('zcbot_platforms', [
+        {
+            '$lookup': {
+                'from': 'zcbot_auth_user_spider_group',
+                'localField': '_id',
+                'foreignField': 'platCode',
+                'as': 'spider'
+            }
+        },
+        {
+            '$match': {'spider.groupCode': group_code}
+        },
+        {
+            '$project': {'spider': 0}
+        },
+        {'$sort': {'sort': 1}}
+    ])
+
+
+# 根据爬虫组编号，获取可选的爬虫清单
+async def get_spider_group(tenant_code: str = None, group_code: str = None, plat_code: str = None, enable: int = None):
+    _query = {}
+    if tenant_code:
+        _query["_id"] = f'{tenant_code}:{group_code}:{plat_code}:group'
+    if group_code:
+        _query["groupCode"] = group_code
+    if plat_code:
+        _query["platCode"] = plat_code
+    if enable is not None:
+        _query["enable"] = enable
+
+    return await Mongo().get('zcbot_auth_user_spider_group', query=_query)
+
+
+# 根据爬虫组编号，获取爬虫组清单
+async def get_spider_group_list(group_code: str = None, plat_codes: List[str] = None, enable: int = None):
+    _query = {}
+    if group_code:
+        _query["groupCode"] = group_code
+    if plat_codes and len(plat_codes):
+        _query["platCode"] = {"$in": plat_codes}
+    if enable is not None:
+        _query["enable"] = enable
+
+    return await Mongo().list('zcbot_auth_user_spider_group', query=_query)

fast_crawl_core-0.0.1/fast_crawl_core/dao/batch_spider.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+from typing import Optional
+from ..client.mongo_client import Mongo
+from ..model.entity import BatchSpider
+from ..model.enums import CommonStatus
+
+
+def _build_task_group_key(task_group, plat_code):
+    return {'_id': f'{task_group}:{plat_code}:group'}
+
+
+def _build_stream_spider_key(task_type, plat_code):
+    return f'{task_type}:{plat_code}'
+
+
+# 获取爬虫任务组配置
+async def get_spider_group(task_group, plat_code):
+    return await Mongo().get('zcbot_batch_spider_group', _build_task_group_key(task_group, plat_code))
+
+
+# 获取portainer爬虫配置
+async def get_batch_spider(spider_id: str) -> Optional[BatchSpider]:
+    row = await Mongo().get('zcbot_batch_spider', {'spiderId': spider_id, 'status': CommonStatus.ON.name})
+    if not row:
+        return None
+    return BatchSpider(**row)
+
+
+# 获取支持网站平台
+async def get_platforms_by_spider_group(task_type: str):
+    return await Mongo().aggregate('zcbot_platforms', [
+        {
+            '$lookup': {
+                'from': 'zcbot_batch_spider_group',
+                'localField': '_id',
+                'foreignField': 'plat_code',
+                'as': 'spider'
+            }
+        },
+        {
+            '$match': {'spider.task_type': task_type.lower()}
+        },
+        {
+            '$project': {'spider': 0}
+        },
+        {'$sort': {'sort': 1}}
+    ])

fast_crawl_core-0.0.1/fast_crawl_core/dao/batch_spider_group.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+from typing import List
+from ..client.mongo_client import Mongo
+
+
+# 获取支持网站平台
+async def get_platforms_by_group(group_code: str, enable: int = None):
+    _query = {}
+    if group_code:
+        _query["groupCode"] = group_code
+    if enable is not None:
+        _query["enable"] = enable
+
+    return await Mongo().list('zcbot_batch_spider_group', query=_query)
+
+
+async def get_platforms_by_group_code(group_code: str, enable):
+    _query = {
+        "groupCode": group_code
+    }
+    _fields = {
+        "spiders": 0,
+        "_id": 0
+    }
+    return await Mongo().aggregate('zcbot_platforms', [
+        {
+            '$lookup': {
+                'from': 'zcbot_batch_spider_group',
+                'localField': '_id',
+                'foreignField': 'platCode',
+                'as': 'spider'
+            }
+        },
+        {
+            '$match': {'spider.groupCode': group_code}
+        },
+        {
+            '$project': {'spider': 0}
+        },
+        {'$sort': {'sort': 1}}
+    ])
+
+
+# 根据爬虫组编号，获取可选的爬虫清单
+async def get_spider_group(group_code: str = None, plat_code: str = None, enable: int = None):
+    _query = {}
+    if group_code:
+        _query["groupCode"] = group_code
+    if plat_code:
+        _query["platCode"] = plat_code
+    if enable is not None:
+        _query["enable"] = enable
+
+    return await Mongo().get('zcbot_batch_spider_group', query=_query)
+
+
+# 根据爬虫组编号，获取爬虫组清单
+async def get_spider_group_list(group_code: str = None, plat_codes: List[str] = None, enable: int = None):
+    _query = {}
+    if group_code:
+        _query["groupCode"] = group_code
+    if plat_codes and len(plat_codes):
+        _query["platCode"] = {"$in": plat_codes}
+    if enable is not None:
+        _query["enable"] = enable
+
+    return await Mongo().list('zcbot_batch_spider_group', query=_query)

fast_crawl_core-0.0.1/fast_crawl_core/dao/meta.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from ..client.mongo_client import Mongo
+from ..model.base import Rule
+from ..client.redis_client import Redis
+
+
+# 获取链接分拣规则配置
+async def get_url_parse_rule(host: str = None):
+    if host:
+        return Rule(**await Mongo().get(collection='zcbot_url_parse_rule', query={'_id': host}))
+
+    return [Rule(**x) for x in await Mongo().list(collection='zcbot_url_parse_rule')]
+
+
+# 获取支持网站平台
+async def get_platforms():
+    return await Mongo().list(collection='zcbot_platforms', sort=[('sort', 1)])
+
+
+# 获取文件命名规则配置
+async def get_file_name_config(config_id):
+    return await Mongo().get(collection='zcbot_file_name_config', query={'_id': config_id})
+
+
+def create_pipeline():
+    pipe = Redis().client.pipeline()
+    return pipe
+
+
+async def run_pipeline(pipe):
+    await pipe.execute()
+    del pipe

fast_crawl_core-0.0.1/fast_crawl_core/dao/node.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+from typing import Optional, List
+from ..client.mongo_client import Mongo
+from ..model.entity import PortainerNode
+from ..model.enums import CommonStatus
+
+
+# 获取节点信息
+async def get_node(node_id) -> Optional[PortainerNode]:
+    rs = await Mongo().get(collection='zcbot_portainer_node', query={'_id': node_id})
+    if not rs:
+        return None
+
+    return PortainerNode(**rs)
+
+
+# 获取节点信息列表
+async def get_node_list(query: dict = None, fields: dict = None, include_disable: bool = False) -> List[PortainerNode]:
+    _query = query or {}
+    _fields = fields or {}
+    if not include_disable:
+        _query['status'] = CommonStatus.ON.name
+    rows = await Mongo().list(collection='zcbot_portainer_node', query=_query, fields=_fields)
+    if rows:
+        return [PortainerNode(**row) for row in rows]
+
+    return []

fast_crawl_core-0.0.1/fast_crawl_core/dao/spider.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+from typing import Optional, List
+from ..client.mongo_client import Mongo
+from ..model.entity import BatchSpider
+from ..model.enums import CommonStatus
+
+
+def _build_task_group_key(task_group, plat_code):
+    return {'_id': f'{task_group}:{plat_code}:group'}
+
+
+def _build_stream_spider_key(task_type, plat_code):
+    return f'{task_type}:{plat_code}'
+
+
+# 获取爬虫任务组配置
+async def get_spider_group(task_group, plat_code):
+    return await Mongo().get('zcbot_spider_group', _build_task_group_key(task_group, plat_code))
+
+
+# 获取portainer爬虫配置
+async def get_spider(spider_id: str, status: CommonStatus = CommonStatus.ON) -> Optional[BatchSpider]:
+    row = await Mongo().get('zcbot_spider', {'spiderId': spider_id, 'status': status.name})
+    if not row:
+        return None
+    return BatchSpider(**row)
+
+
+# 获取支持网站平台
+async def get_platforms_by_spider_group(task_type: str):
+    return await Mongo().aggregate('zcbot_platforms', [
+        {
+            '$lookup': {
+                'from': 'zcbot_spider_group',
+                'localField': '_id',
+                'foreignField': 'plat_code',
+                'as': 'spider'
+            }
+        },
+        {
+            '$match': {'spider.task_type': task_type.lower()}
+        },
+        {
+            '$project': {'spider': 0}
+        },
+        {'$sort': {'sort': 1}}
+    ])
+
+
+async def get_spiders_by_id(spider_ids: List):
+    _query = {
+        "_id": {
+            "$in": spider_ids
+        },
+        "status": "ON"
+    }
+    rs = await Mongo().list("zcbot_spider", query=_query)
+    return rs

fast_crawl_core-0.0.1/fast_crawl_core/dao/spider_group.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+from typing import List
+from ..client.mongo_client import Mongo
+
+
+# 获取支持网站平台
+async def get_platforms_by_group(group_code: str, enable: int = None):
+    _query = {}
+    if group_code:
+        _query["groupCode"] = group_code
+    if enable is not None:
+        _query["enable"] = enable
+
+    return await Mongo().list('zcbot_spider_group', query=_query)
+
+
+async def get_platforms_by_group_code(group_code: str, enable):
+    _query = {
+        "groupCode": group_code
+    }
+    _fields = {
+        "spiders": 0,
+        "_id": 0
+    }
+    return await Mongo().aggregate('zcbot_platforms', [
+        {
+            '$lookup': {
+                'from': 'zcbot_spider_group',
+                'localField': '_id',
+                'foreignField': 'platCode',
+                'as': 'spider'
+            }
+        },
+        {
+            '$match': {'spider.groupCode': group_code}
+        },
+        {
+            '$project': {'spider': 0}
+        },
+        {'$sort': {'sort': 1}}
+    ])
+
+
+# 根据爬虫组编号，获取可选的爬虫清单
+async def get_spider_group(group_code: str = None, plat_code: str = None, enable: int = None):
+    _query = {}
+    if group_code:
+        _query["groupCode"] = group_code
+    if plat_code:
+        _query["platCode"] = plat_code
+    if enable is not None:
+        _query["enable"] = enable
+
+    return await Mongo().get('zcbot_spider_group', query=_query)
+
+
+# 根据爬虫组编号，获取爬虫组清单
+async def get_spider_group_list(group_code: str = None, plat_codes: List[str] = None, enable: int = None):
+    _query = {}
+    if group_code:
+        _query["groupCode"] = group_code
+    if plat_codes and len(plat_codes):
+        _query["platCode"] = {"$in": plat_codes}
+    if enable is not None:
+        _query["enable"] = enable
+
+    return await Mongo().list('zcbot_spider_group', query=_query)

fast_crawl_core-0.0.1/fast_crawl_core/dao/stream_spider.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+from typing import Optional, List
+from ..client.mongo_client import Mongo
+from ..model.entity import StreamSpider, StreamSpiderModel
+from ..model.enums import CommonStatus
+
+
+# 获取portainer爬虫配置
+async def get_stream_spiders(spider_ids: List[str], status: CommonStatus) -> List[StreamSpider]:
+    rows = await Mongo().list("zcbot_stream_spider", {'spiderId': {'$in': spider_ids}, 'status': status.name})
+    if not rows:
+        return []
+    return [StreamSpider(**x) for x in rows]
+
+
+async def get_stream_spider(spider_id: str, status: CommonStatus) -> Optional[StreamSpiderModel]:
+    row = await Mongo().get('zcbot_stream_spider', {'spiderId': spider_id, 'status': status.name})
+    if not row:
+        return None
+    return StreamSpiderModel(**row)
+
+
+# 获取支持网站平台
+async def get_platforms_by_spider_id(spider_ids: List[str], status: CommonStatus):
+    return await Mongo().aggregate('zcbot_platforms', [
+        {
+            '$lookup': {
+                'from': 'zcbot_stream_spider',
+                'localField': '_id',
+                'foreignField': 'plat_code',
+                'as': 'spider'
+            }
+        },
+        {
+            '$match': {'spider._id': {'$in': spider_ids}, 'status': status.name}
+        },
+        {
+            '$project': {'spider': 0}
+        },
+        {'$sort': {'sort': 1}}
+    ])