cobweb-launcher 1.2.49__py3-none-any.whl → 1.3.2__py3-none-any.whl

cobweb_new/pipelines/pipeline_console.py

@@ -0,0 +1,22 @@
+from cobweb.base import ConsoleItem, logger
+from cobweb.constant import LogTemplate
+from cobweb.pipelines import Pipeline
+
+
+class Console(Pipeline):
+
+    def build(self, item: ConsoleItem):
+        return {
+            "seed": item.seed.to_dict,
+            "data": item.to_dict
+        }
+
+    def upload(self, table, datas):
+        for data in datas:
+            parse_detail = LogTemplate.log_info(data["data"])
+            if len(parse_detail) > 500:
+                parse_detail = parse_detail[:500] + " ...\n" + " " * 12 + "-- Text is too long and details are omitted!"
+            logger.info(LogTemplate.console_item.format(
+                seed_detail=LogTemplate.log_info(data["seed"]),
+                parse_detail=parse_detail
+            ))

cobweb_new/pipelines/pipeline_loghub.py

@@ -0,0 +1,34 @@
+import json
+
+from cobweb import setting
+from cobweb.base import BaseItem
+from cobweb.pipelines import Pipeline
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+
+class Loghub(Pipeline):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = LogClient(**setting.LOGHUB_CONFIG)
+
+    def build(self, item: BaseItem):
+        log_item = LogItem()
+        temp = item.to_dict
+        for key, value in temp.items():
+            if not isinstance(value, str):
+                temp[key] = json.dumps(value, ensure_ascii=False)
+        contents = sorted(temp.items())
+        log_item.set_contents(contents)
+        return log_item
+
+    def upload(self, table, datas):
+        request = PutLogsRequest(
+            project=setting.LOGHUB_PROJECT,
+            logstore=table,
+            topic=setting.LOGHUB_TOPIC,
+            source=setting.LOGHUB_SOURCE,
+            logitems=datas,
+            compress=True
+        )
+        self.client.put_logs(request=request)

cobweb_new/setting.py

@@ -0,0 +1,95 @@
+import os
+
+# redis db config
+REDIS_CONFIG = {
+    "host": os.getenv("REDIS_HOST"),
+    "password": os.getenv("REDIS_PASSWORD"),
+    "port": int(os.getenv("REDIS_PORT", 6379)),
+    "db": int(os.getenv("REDIS_DB", 0)),
+}
+
+# loghub db config
+LOGHUB_TOPIC = os.getenv("LOGHUB_TOPIC")
+LOGHUB_SOURCE = os.getenv("LOGHUB_SOURCE")
+LOGHUB_PROJECT = os.getenv("LOGHUB_PROJECT")
+LOGHUB_CONFIG = {
+    "endpoint": os.getenv("LOGHUB_ENDPOINT"),
+    "accessKeyId": os.getenv("LOGHUB_ACCESS_KEY"),
+    "accessKey": os.getenv("LOGHUB_SECRET_KEY")
+}
+
+# oss util config
+OSS_BUCKET = os.getenv("OSS_BUCKET")
+OSS_ENDPOINT = os.getenv("OSS_ENDPOINT")
+OSS_ACCESS_KEY = os.getenv("OSS_ACCESS_KEY")
+OSS_SECRET_KEY = os.getenv("OSS_SECRET_KEY")
+OSS_CHUNK_SIZE = 10 * 1024 ** 2
+OSS_MIN_UPLOAD_SIZE = 1024
+
+
+# 采集器选择
+CRAWLER = "cobweb.crawlers.Crawler"
+
+# 数据存储链路
+PIPELINE = "cobweb.pipelines.pipeline_console.Console"
+
+
+# Launcher 等待时间
+
+BEFORE_SCHEDULER_WAIT_SECONDS = 60  # 调度前等待时间,只作用于单次任务
+
+SCHEDULING_WAIT_TIME = 15  # SCHEDULER ITEM 调度等待时间
+INSERTING_WAIT_TIME = 30   # INSERT ITEM 等待时间
+REMOVING_WAIT_TIME = 5     # REMOVE ITEM 等待时间
+
+TODO_QUEUE_FULL_WAIT_SECONDS = 5  # 队列已满时等待时间
+NEW_QUEUE_WAIT_SECONDS = 30   # new队列等待时间
+DONE_QUEUE_WAIT_SECONDS = 5   # done队列等待时间
+UPLOAD_QUEUE_WAIT_SECONDS = 15   # upload队列等待时间
+SEED_RESET_SECONDS = 30   # 种子重制时间
+
+
+# Launcher 队列长度
+SCHEDULING_SIZE = 100   # 调度队列长度
+INSERTING_SIZE = 100    # INSERT 长度
+REMOVING_SIZE = 100     # REMOVE 长度
+
+# SEED = Queue()          # 添加任务种子队列
+# TODO = Queue()          # 任务种子队列
+# REQUEST = Queue()       # 请求队列
+# DOWNLOAD = Queue()      # 下载任务队列
+# RESPONSE = Queue()      # 响应队列
+# DONE = Queue()          # 下载完成队列
+# UPLOAD = Queue()        # 任务上传队列
+# DELETE = Queue()        # 任务删除队列
+
+SEED_QUEUE_SIZE = 100       # TODO 队列长度
+TODO_QUEUE_SIZE = 100       # TODO 队列长度
+REQUEST_QUEUE_SIZE = 100    # new队列长度
+DOWNLOAD_QUEUE_SIZE = 100   # done队列长度
+RESPONSE_QUEUE_SIZE = 100   # upload队列长度
+DONE_QUEUE_SIZE = 100       # upload队列长度
+UPLOAD_QUEUE_SIZE = 100     # upload队列长度
+DELETE_QUEUE_SIZE = 100     # upload队列长度
+
+# DONE_MODEL IN (0, 1), 种子完成模式
+DONE_MODEL = 0   # 0:种子消费成功直接从队列移除，失败则添加至失败队列；1:种子消费成功添加至成功队列，失败添加至失败队列
+
+# spider
+SPIDER_THREAD_NUM = 10
+SPIDER_MAX_RETRIES = 5
+SPIDER_TIME_SLEEP = 10
+
+SPIDER_MAX_COUNT = 1000   # 在规定时间窗口内最大采集数
+TIME_WINDOW = 60   # 频控固定时间窗口（秒）
+
+# 任务模式
+TASK_MODEL = 0  # 0:单次，1:常驻
+
+
+# bloom过滤器
+CAPACITY = 100000000
+ERROR_RATE = 0.001
+FILTER_FIELD = "url"
+# 文件下载响应类型过滤
+# FILE_FILTER_CONTENT_TYPE = ["text/html", "application/xhtml+xml"]

cobweb_new/utils/__init__.py

@@ -0,0 +1,5 @@
+from .oss import OssUtil
+from .tools import *
+from .bloom import BloomFilter
+from .task_queue import TaskQueue
+

cobweb_new/utils/bloom.py

@@ -0,0 +1,58 @@
+import math
+import time
+
+import mmh3
+import redis
+from cobweb import setting
+
+
+class BloomFilter:
+
+    def __init__(self, key, redis_config=None, capacity=None, error_rate=None):
+        redis_config = redis_config or setting.REDIS_CONFIG
+        capacity = capacity or setting.CAPACITY
+        error_rate = error_rate or setting.ERROR_RATE
+        redis_config['db'] = 3
+
+        self.key = key
+
+        pool = redis.ConnectionPool(**redis_config)
+        self._client = redis.Redis(connection_pool=pool)
+        self.bit_size = self.get_bit_size(capacity, error_rate)
+        self.hash_count = self.get_hash_count(self.bit_size, capacity)
+        self._init_bloom_key()
+
+    def add(self, value):
+        for seed in range(self.hash_count):
+            result = mmh3.hash(value, seed) % self.bit_size
+            self._client.setbit(self.key, result, 1)
+        return True
+
+    def exists(self, value):
+        if not self._client.exists(self.key):
+            return False
+        for seed in range(self.hash_count):
+            result = mmh3.hash(value, seed) % self.bit_size
+            if not self._client.getbit(self.key, result):
+                return False
+        return True
+
+    def _init_bloom_key(self):
+        lua_script = """
+        redis.call("SETBIT", KEYS[1], ARGV[1], ARGV[2])
+        redis.call("EXPIRE", KEYS[1], 604800)
+        """
+        if self._client.exists(self.key):
+            return True
+        execute = self._client.register_script(lua_script)
+        execute(keys=[self.key], args=[self.bit_size-1, 1])
+
+    @classmethod
+    def get_bit_size(cls, n, p):
+        return int(-(n * math.log(p)) / (math.log(2) ** 2))
+
+    @classmethod
+    def get_hash_count(cls, m, n):
+        return int((m / n) * math.log(2))
+
+

cobweb_new/utils/oss.py

@@ -0,0 +1,94 @@
+from typing import List
+from cobweb import setting
+from requests import Response
+from oss2 import Auth, Bucket, models, PartIterator
+from cobweb.exceptions import oss_db_exception
+from cobweb.base.decorators import decorator_oss_db
+
+
+class OssUtil:
+
+    def __init__(
+        self,
+        bucket=None,
+        endpoint=None,
+        access_key=None,
+        secret_key=None,
+        chunk_size=None,
+        min_upload_size=None,
+        **kwargs
+    ):
+        self.bucket = bucket or setting.OSS_BUCKET
+        self.endpoint = endpoint or setting.OSS_ENDPOINT
+        self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
+        self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
+
+        self._auth = Auth(
+            access_key_id=access_key or setting.OSS_ACCESS_KEY,
+            access_key_secret=secret_key or setting.OSS_SECRET_KEY
+        )
+        self._client = Bucket(
+            auth=self._auth,
+            endpoint=self.endpoint,
+            bucket_name=self.bucket,
+            **kwargs
+        )
+
+    def exists(self, key: str) -> bool:
+        return self._client.object_exists(key)
+
+    def head(self, key: str) -> models.HeadObjectResult:
+        return self._client.head_object(key)
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
+    def init_part(self, key) -> models.InitMultipartUploadResult:
+        """初始化分片上传"""
+        return self._client.init_multipart_upload(key)
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
+    def put(self, key, data) -> models.PutObjectResult:
+        """文件上传"""
+        return self._client.put_object(key, data)
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
+    def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
+        """分片上传"""
+        return self._client.upload_part(key, upload_id, position, data)
+
+    def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
+        """获取分片列表"""
+        return [part_info for part_info in PartIterator(self._client, key, upload_id)]
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
+    def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
+        """合并分片"""
+        headers = None if parts else {"x-oss-complete-all": "yes"}
+        return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
+
+    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
+    def append(self, key, position, data) -> models.AppendObjectResult:
+        """追加上传"""
+        return self._client.append_object(key, position, data)
+
+    def iter_data(self, data, chunk_size=None):
+        chunk_size = chunk_size or self.chunk_size
+        if isinstance(data, Response):
+            for part_data in data.iter_content(chunk_size):
+                yield part_data
+        if isinstance(data, bytes):
+            for i in range(0, len(data), chunk_size):
+                yield data[i:i + chunk_size]
+
+    def assemble(self, ready_data, data, chunk_size=None):
+        upload_data = b""
+        ready_data = ready_data + data
+        chunk_size = chunk_size or self.chunk_size
+        if len(ready_data) >= chunk_size:
+            upload_data = ready_data[:chunk_size]
+            ready_data = ready_data[chunk_size:]
+        return ready_data, upload_data
+
+    def content_length(self, key: str) -> int:
+        head = self.head(key)
+        return head.content_length
+

cobweb_new/utils/tools.py

@@ -0,0 +1,42 @@
+import re
+import hashlib
+from typing import Union
+from importlib import import_module
+
+
+def md5(text: Union[str, bytes]) -> str:
+    if isinstance(text, str):
+        text = text.encode('utf-8')
+    return hashlib.md5(text).hexdigest()
+
+
+def build_path(site, url, file_type):
+    return f"{site}/{md5(url)}.{file_type}"
+
+
+def format_size(content_length: int) -> str:
+    units = ["KB", "MB", "GB", "TB"]
+    for i in range(4):
+        num = content_length / (1024 ** (i + 1))
+        if num < 1024:
+            return f"{round(num, 2)} {units[i]}"
+
+
+def dynamic_load_class(model_info):
+    if isinstance(model_info, str):
+        if "import" in model_info:
+            model_path, class_name = re.search(
+                r"from (.*?) import (.*?)$", model_info
+            ).groups()
+            model = import_module(model_path)
+            class_object = getattr(model, class_name)
+        else:
+            model_path, class_name = model_info.rsplit(".", 1)
+            model = import_module(model_path)
+            class_object = getattr(model, class_name)
+        return class_object
+    raise TypeError()
+
+
+# def download_log_info(item:dict) -> str:
+#     return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])

cobweb/schedulers/scheduler_api.py

@@ -1,72 +0,0 @@
-import threading
-import time
-
-# from cobweb.base import Seed
-from cobweb.db import ApiDB
-
-
-class ApiScheduler:
-
-    def __init__(self, task, project, scheduler_wait_seconds=30):
-        self._todo_key = "{%s:%s}:todo" % (project, task)
-        self._download_key = "{%s:%s}:download" % (project, task)
-        self._heartbeat_key = "heartbeat:%s_%s" % (project, task)
-        self._speed_control_key = "speed_control:%s_%s" % (project, task)
-        self._reset_lock_key = "lock:reset:%s_%s" % (project, task)
-        self._db = ApiDB()
-
-        self.scheduler_wait_seconds = scheduler_wait_seconds
-        self.working = threading.Event()
-
-    @property
-    def heartbeat(self):
-        return self._db.exists(self._heartbeat_key)
-
-    def set_heartbeat(self):
-        return self._db.setex(self._heartbeat_key, 5)
-
-    def schedule(self, key, count):
-        if not self._db.zcount(key, 0, "(1000"):
-            time.sleep(self.scheduler_wait_seconds)
-        else:
-            source = int(time.time())
-            members = self._db.members(key, source, count=count, _min=0, _max="(1000")
-            for member, priority in members:
-                # seed = Seed(member, priority=priority)
-                yield member, priority
-
-    def insert(self, key, items):
-        if items:
-            self._db.zadd(key, items, nx=True)
-
-    def reset(self, keys, reset_time=30):
-        if self._db.lock(self._reset_lock_key, t=120):
-
-            if isinstance(keys, str):
-                keys = [keys]
-
-            _min = reset_time - int(time.time()) if self.heartbeat else "-inf"
-
-            for key in keys:
-                if self._db.exists(key):
-                    self._db.members(key, 0, _min=_min, _max="(0")
-
-            if not self.heartbeat:
-                self.working.set()
-                time.sleep(10)
-
-            self._db.delete(self._reset_lock_key)
-
-    def refresh(self, key, items: dict[str, int]):
-        refresh_time = int(time.time())
-        its = {k: -refresh_time - v / 1000 for k, v in items.items()}
-        if its:
-            self._db.zadd(key, item=its, xx=True)
-
-    def delete(self, key, values):
-        if values:
-            self._db.zrem(key, *values)
-
-
-
-

cobweb_launcher-1.2.49.dist-info/RECORD

@@ -1,44 +0,0 @@
-cobweb/__init__.py,sha256=CBd2oByCfc5EmH2dCZYVHkxXYZG-oWrLyTtZU5sEoP0,96
-cobweb/constant.py,sha256=zy3XYsc1qp2B76_Fn_hVQ8eGHlPBd3OFlZK2cryE6FY,2839
-cobweb/setting.py,sha256=47HZsw40HLpsmOmvij1lyQALPQQCN_tWlKZ0wbn2MtM,2216
-cobweb/base/__init__.py,sha256=4gwWWQ0Q8cYG9cD7Lwf4XMqRGc5M_mapS3IczR6zeCE,222
-cobweb/base/basic.py,sha256=Z56SSLB3I2IGHWCCcSy0Qbfzj8Qbg_po3gP32q1jh4k,7741
-cobweb/base/common_queue.py,sha256=W7PPZZFl52j3Mc916T0imHj7oAUelA6aKJwW-FecDPE,872
-cobweb/base/decorators.py,sha256=wDCaQ94aAZGxks9Ljc0aXq6omDXT1_yzFy83ZW6VbVI,930
-cobweb/base/dotting.py,sha256=lfFXXqnVP__hxlW3qH5Bnuq69KtnFaQLbcz1M8e2Ajg,1239
-cobweb/base/item.py,sha256=hYheVTV2Bozp4iciJpE2ZwBIXkaqBg4QQkRccP8yoVk,1049
-cobweb/base/log.py,sha256=L01hXdk3L2qEm9X1FOXQ9VmWIoHSELe0cyZvrdAN61A,2003
-cobweb/base/request.py,sha256=tEkgMVUfdQI-kZuzWuiit9P_q4Q9-_RZh9aXXpc0314,2352
-cobweb/base/response.py,sha256=eB1DWMXFCpn3cJ3yzgCRU1WeZAdayGDohRgdjdMUFN4,406
-cobweb/base/seed.py,sha256=Uz_VBRlAxNYQcFHk3tsZFMlU96yPOedHaWGTvk-zKd8,2908
-cobweb/crawlers/__init__.py,sha256=msvkB9mTpsgyj8JfNMsmwAcpy5kWk_2NrO1Adw2Hkw0,29
-cobweb/crawlers/base_crawler.py,sha256=ee_WSDnPQpPTk6wlFuY2UEx5L3hcsAZFcr6i3GLSry8,5751
-cobweb/crawlers/crawler.py,sha256=dKyQJOVh1cnuazSryb9vA-IGJJnhw-gxAAlIqsEn6-M,8832
-cobweb/crawlers/file_crawler.py,sha256=2Sjbdgxzqd41WykKUQE3QQlGai3T8k-pmHNmPlTchjQ,4454
-cobweb/db/__init__.py,sha256=uZwSkd105EAwYo95oZQXAfofUKHVIAZZIPpNMy-hm2Q,56
-cobweb/db/api_db.py,sha256=bDc5dJQxq4z04h70KUTHd0OqUOEY7Cm3wcNJZtTvJIM,3015
-cobweb/db/redis_db.py,sha256=fumNZJiio-uQqRcSrymx8eJ1PqsdOwITe_Y-9JOXxrQ,4298
-cobweb/exceptions/__init__.py,sha256=E9SHnJBbhD7fOgPFMswqyOf8SKRDrI_i25L0bSpohvk,32
-cobweb/exceptions/oss_db_exception.py,sha256=iP_AImjNHT3-Iv49zCFQ3rdLnlvuHa3h2BXApgrOYpA,636
-cobweb/launchers/__init__.py,sha256=qMuVlQcjErVK67HyKFZEsXf_rfZD5ODjx1QucSCKMOM,114
-cobweb/launchers/launcher.py,sha256=sPts-xlgxoeIfl1fn1XR2XVZxLzt7He9xrYDfTHRAGo,7029
-cobweb/launchers/launcher_air.py,sha256=KAk_M8F3029cXYe7m4nn3Nzyi89lbxJ2cqZjqW8iZ0E,2832
-cobweb/launchers/launcher_api.py,sha256=YFqCTRvKn6icBLWTR1VxkU0WEIte2F7fv_LgPkifqdo,7885
-cobweb/launchers/launcher_pro.py,sha256=B5FdxvuENRL3XrMl74ENdP1uNgnZOaYCUUfBfM0t3io,7842
-cobweb/pipelines/__init__.py,sha256=zSUsGtx6smbs2iXBXvYynReKSgky-3gjqaAtKVnA_OU,105
-cobweb/pipelines/pipeline.py,sha256=4TJLX0sUHRxYndF5A4Vs5btUGI-wigkOcFvhTW1hLXI,2009
-cobweb/pipelines/pipeline_console.py,sha256=NEh-4zhuVAQOqwXLsqeb-rcNZ9_KXFUpL3otUTL5qBs,754
-cobweb/pipelines/pipeline_loghub.py,sha256=xZ6D55BGdiM71WUv83jyLGbEyUwhBHLJRZoXthBxxTs,1019
-cobweb/schedulers/__init__.py,sha256=y7Lv_7b0zfTl0OhIONb_8u1K1C9gVlBA-xz_XG_kI9g,85
-cobweb/schedulers/scheduler_api.py,sha256=pFEdS1H4zuzxwMhCV-G7CoLz-rEOPv4EVo3xZUXTyDo,2199
-cobweb/schedulers/scheduler_redis.py,sha256=E5fjc3nNld8GbUhUGT7uY4smRejj2J2ZIzp2g6lhxFM,2205
-cobweb/utils/__init__.py,sha256=Ev2LZZ1-S56iQYDqFZrqadizEv4Gk8Of-DraH-_WnKY,109
-cobweb/utils/bloom.py,sha256=vng-YbKgh9HbtpAWYf_nkUSbfVTOj40aqUUejRYlsCU,1752
-cobweb/utils/dotting.py,sha256=_QsC-LCSh1QYk9XmYIWBRlRTs-E9RfqKSkkRkedV2Eo,2082
-cobweb/utils/oss.py,sha256=gyt8-UB07tVphZLQXMOf-JTJwU-mWq8KZkOXKkAf3uk,3513
-cobweb/utils/tools.py,sha256=5JEaaAwYoV9Sdla2UBIJn6faUBuXmxUMagm9ck6FVqs,1253
-cobweb_launcher-1.2.49.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
-cobweb_launcher-1.2.49.dist-info/METADATA,sha256=VmuY5V3Hhjw9bZ5JqQfwJi8GfPWq5g6VDb6G39q36IM,6510
-cobweb_launcher-1.2.49.dist-info/WHEEL,sha256=ewwEueio1C2XeHTvT17n8dZUJgOvyCWCt0WVNLClP9o,92
-cobweb_launcher-1.2.49.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
-cobweb_launcher-1.2.49.dist-info/RECORD,,

cobweb_launcher-1.2.49.dist-info/top_level.txt

@@ -1 +0,0 @@
-cobweb