cobweb-launcher 0.0.1__py3-none-any.whl

cobweb/distributed/models.py

@@ -0,0 +1,140 @@
+import time
+from hashlib import md5
+from base.log import log
+from base.bbb import Queue, Seed
+from base.interface import SchedulerInterface, StorerInterface
+# from pympler import asizeof
+
+
+class Scheduler:
+
+    def schedule_seed(self, ready_seed_length, get_scheduler_lock, add_seed):
+
+        if not issubclass(self.__class__, SchedulerInterface):
+            raise Exception("not have schedule function!")
+
+        if self.__class__.__name__ == "Default":
+            self.stop = True
+            return None
+
+        while not self.stop:
+            length = ready_seed_length()
+            if length > self.size:
+                time.sleep(15)
+
+            elif get_scheduler_lock():
+                seeds = self.schedule()
+                add_seed(seeds)
+
+        log.info(f"close thread: schedule_seed")
+
+    def schedule_task(self, stop, get_seed, ready_seed_length):
+        time.sleep(3)
+        while not stop.is_set():
+
+            if not ready_seed_length():
+                time.sleep(15)
+                continue
+
+            if self.queue.length >= self.length:
+                time.sleep(3)
+                continue
+
+            seeds = get_seed(self.length)
+            self.queue.push(seeds)
+        log.info(f"close thread: schedule_task")
+
+
+class Spider:
+
+    def __init__(self, queue, max_retries=5):
+        self.spider_in_progress = Queue()
+        self.max_retries = max_retries
+        self.queue = queue
+
+    def spider_task(self, stop, func, item, del_seed):
+        while not stop.is_set():
+            seed = self.queue.pop()
+            if not seed:
+                time.sleep(3)
+                continue
+            elif seed._retry > self.max_retries:
+                del_seed(seed, spider_status=False)
+                continue
+            try:
+                self.spider_in_progress.push(1)
+                log.info("spider seed: " + str(seed))
+                status = None
+                store_queue = None
+                store_data = list()
+                for it in func(item, seed):
+                    if getattr(it, "table_name", None):
+                        if not store_queue:
+                            store_queue = it.queue()
+                        store_data.append(it.struct_data)
+                    elif isinstance(it, Seed):
+                        self.queue.push(it)
+                    elif any(isinstance(it, t) for t in (list, tuple)):
+                        self.queue.push([s if isinstance(s, Seed) else Seed(s) for s in it])
+                    elif isinstance(it, bool):
+                        status = it
+
+                if store_queue and store_data:
+                    store_data.append(seed)
+                    store_queue.push(store_data)
+
+                if status is not None:
+                    if status:
+                        del_seed(seed, spider_status=True)
+                    else:
+                        seed._retry += 1
+                        self.queue.push(seed)
+
+            except Exception as e:
+                seed._retry += 1
+                self.queue.push(seed)
+                log.info(str(e))
+            finally:
+                self.spider_in_progress.pop()
+        log.info(f"close thread: spider")
+
+
+class Storer:
+
+    def store_task(self, stop, last, reset_seed, set_storer):
+
+        if not issubclass(self.__class__, StorerInterface):
+            return None
+
+        if not getattr(self, "store", None):
+            raise Exception("not have store function!")
+
+        storer_name = self.__class__.__name__ + self.table
+        store_key_id = md5(storer_name.encode()).hexdigest()
+
+        while not stop.is_set():
+
+            if last.is_set() or self.queue.length > self.length:
+                seeds, data_list = [], []
+
+                while True:
+                    data = self.queue.pop()
+                    if not data:
+                        break
+                    if isinstance(data, Seed):
+                        seeds.append(data)
+                        if len(data_list) >= self.length:
+                            break
+                        continue
+                    data_list.append(data)
+
+                if data_list:
+                    if self.store(data_list):
+                        set_storer(store_key_id, seeds)
+                    else:
+                        reset_seed(seeds)
+                    continue
+
+            time.sleep(3)
+
+        log.info(f"close thread: {storer_name}")

cobweb/single/models.py

@@ -0,0 +1,104 @@
+import time
+# from pympler import asizeof
+from single.nest import Queue
+from single.nest import struct_queue_name
+from single.nest import SchedulerInterface, StorerInterface
+
+
+# class Transceiver:
+class Distributor:
+
+    def __init__(self):
+        self.seed_queue = Queue()
+
+    @property
+    def queue_names(self):
+        return tuple(self.__dict__.keys())
+
+    @property
+    def used_memory(self):
+        return asizeof.asizeof(self)
+
+    def create_queue(self, queue_name: str):
+        self.__setattr__(queue_name, Queue())
+
+    def get_queue(self, queue_name: str):
+        return self.__getattribute__(queue_name)
+
+    def deal_item(self, item):
+        icn = item.__class__.__name__
+        if icn == "Seed":
+            self.seed_queue.push(item)
+        elif getattr(item, "table_name", None):
+            queue_name = struct_queue_name(icn, item.table_name)
+            getattr(self, queue_name).push(item.serialization)
+
+    def distribute(self, callback, *args, **kwargs):
+        iterable = callback(*args, **kwargs)
+        if not iterable:
+            return None
+        for result in iterable:
+            self.deal_item(result)
+        return True
+
+
+class Scheduler:
+
+    def schedule_task(self, distribute):
+
+        if not issubclass(self.__class__, SchedulerInterface):
+            return None
+
+        if not getattr(self, "schedule", None):
+            raise Exception("not have schedule function!")
+
+        while not self.stop:
+
+            if self.queue.length < self.length:
+                distribute(self.schedule)
+
+            else:
+                print("------------")
+                time.sleep(15)
+
+
+class Spider:
+
+    def __init__(self, queue):
+        self.queue = queue
+        self.spider_in_progress = Queue()
+
+    def spider_task(self, stop_event, distribute, func, item):
+        while not stop_event.is_set():
+            seed = self.queue.pop()
+            if not seed:
+                time.sleep(3)
+                continue
+            try:
+                self.spider_in_progress.push(1)
+                distribute(func, item, seed)
+            except Exception as e:
+                print(e)
+            finally:
+                self.spider_in_progress.pop()
+
+
+class Storer:
+
+    def store_task(self, stop_event, last_event, distribute):
+
+        if not issubclass(self.__class__, StorerInterface):
+            return None
+
+        if not getattr(self, "store", None):
+            raise Exception("not have store function!")
+
+        while not stop_event.is_set():
+            if last_event.is_set() or self.queue.length > self.length:
+                data_list = []
+                data_length = min(self.queue.length, self.length)
+                for _ in range(data_length):
+                    data = self.queue.pop()
+                    data_list.append(data)
+                if data_list:
+                    distribute(self.store, data_list)

cobweb/single/nest.py

@@ -0,0 +1,153 @@
+import time
+import threading
+
+from single.nest import Seed, DBItem
+from single.nest import struct_queue_name, restore_table_name
+from single.nest import Distributor, Scheduler, Spider, Storer
+
+
+def init_task_seed(seeds):
+    if not seeds:
+        return None
+    if isinstance(seeds, list) or isinstance(seeds, tuple):
+        for seed in seeds:
+            yield Seed(seed)
+    elif isinstance(seeds, str) or isinstance(seeds, dict):
+        yield Seed(seeds)
+
+
+def parse_storer_info(storer_info):
+    storer_data = {}
+    storer_info_list = []
+    if storer_info.__class__.__name__ == 'StorerInfo':
+        storer_info_list.append(storer_info)
+    elif isinstance(storer_info, tuple) or isinstance(storer_info, list):
+        storer_info_list = storer_info
+    for info in storer_info_list:
+        db_name = info.DB.__name__
+        storer_data.setdefault(db_name, {"StorerDB": info.DB, "db_args_list": []})
+        storer_data[db_name]["db_args_list"].append(info[1:])
+    return storer_data
+
+
+def check(stop_event, last_event, distributor, scheduler, spider, storer_list):
+    while True:
+        time.sleep(3)
+        if (
+                scheduler.stop and
+                not distributor.seed_queue.length and
+                not spider.spider_in_progress.length
+        ):
+            last_event.set()
+            time.sleep(10)
+            storer_queue_empty = True
+            for storer in storer_list:
+                if storer.queue.length:
+                    storer_queue_empty = False
+                    break
+            if storer_queue_empty:
+                break
+        last_event.clear()
+    stop_event.set()
+
+
+def cobweb(task):
+    """
+    任务启动装饰器
+    :param task: 任务配置信息
+    """
+    def decorator(func):
+        """
+        func(Item, seed)
+            Item:
+                Item.Textfile()
+                Item.Console()
+        """
+        # project task_name start_seed spider_num queue_length scheduler_info storer_info
+
+        storer_list = []
+
+        # 程序结束事件
+        last_event = threading.Event()
+        # 暂停采集事件
+        stop_event = threading.Event()
+
+        # 创建分发器
+        distributor = Distributor()
+
+        # 调度器动态继承
+        SchedulerDB, table, sql, length, size = task.SchedulerInfo
+        SchedulerTmp = type('Scheduler', (Scheduler, SchedulerDB), {})
+
+        # 初始化调度器
+        scheduler = SchedulerTmp(table=table, sql=sql, length=length, size=size, queue=distributor.seed_queue)
+
+        # 初始化采集器
+        spider = Spider(queue=distributor.seed_queue)
+
+        # 解析存储器信息
+        storer_data = parse_storer_info(task.storer_info)
+
+        # sds
+        item = type("item", (object,), {})
+        for db_name in storer_data.keys():
+            # 存储器动态继承
+            StorerDB = storer_data[db_name]["StorerDB"]
+            StorerTmp = type('Storer', (Storer, StorerDB), {})
+            db_args_list = storer_data[db_name]["db_args_list"]
+            for storer_db_args in db_args_list:
+                table, fields, length = storer_db_args
+                if not getattr(item, db_name, None):
+                    instance = type(db_name, (DBItem,), {})
+                    setattr(item, db_name, instance)
+                # 创建存储xxx
+                getattr(item, db_name).init_item(table, fields)
+                # 创建存储队列
+                storer_queue = struct_queue_name(db_name, table)
+                distributor.create_queue(queue_name=storer_queue)
+                queue = distributor.get_queue(queue_name=storer_queue)
+                # 初始话存储器
+                table_name = restore_table_name(table_name=table)
+                storer = StorerTmp(table=table_name, fields=fields, length=length, queue=queue)
+                storer_list.append(storer)
+
+        # 推送初始种子
+        distributor.distribute(init_task_seed, seeds=task.start_seed)
+
+        # 启动调度器
+        threading.Thread(
+            target=scheduler.schedule_task,
+            args=(distributor.distribute,),
+            name="single_scheduler_task"
+        ).start()
+
+        # 启动采集器
+        for index in range(task.spider_num):
+            threading.Thread(
+                target=spider.spider_task,
+                args=(stop_event, distributor.distribute, func, item),
+                name=f"single_spider_task:{index}"
+            ).start()
+
+        # 启动存储器
+        for storer in storer_list:
+            threading.Thread(
+                target=storer.store_task,
+                args=(stop_event, last_event, distributor.distribute),
+                name=f"single_store_task:{storer.table}",
+            ).start()
+
+        threading.Thread(
+            target=check, name="check",
+            args=(
+                stop_event, last_event, distributor,
+                scheduler, spider, storer_list
+            )
+        ).start()
+
+        # return starter(task, func)
+    return decorator
+
+
+
+

cobweb_launcher-0.0.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Juannie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cobweb_launcher-0.0.1.dist-info/METADATA

@@ -0,0 +1,45 @@
+Metadata-Version: 2.1
+Name: cobweb-launcher
+Version: 0.0.1
+Summary: spider_hole
+Home-page: https://github.com/Juannie-PP/cobweb
+Author: Juannie-PP
+Author-email: 2604868278@qq.com
+License: MIT
+Keywords: cobweb
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests (>=2.19.1)
+Requires-Dist: oss2 (>=2.18.1)
+Requires-Dist: redis (>=4.4.4)
+Requires-Dist: aliyun-log-python-sdk
+
+# cobweb
+
+> 通用爬虫框架： 1.单机模式采集框架；2.分布式采集框架
+> 
+>  5部分
+>  
+> 1. starter -- 启动器
+> 
+> 2. scheduler -- 调度器
+> 
+> 3. distributor -- 分发器
+> 
+> 4. storer -- 存储器
+> 
+> 5. utils -- 工具函数
+> 
+
+need deal
+- 队列优化完善，使用queue的机制wait()同步各模块执行？
+- 日志功能完善，单机模式调度和保存数据写入文件，结构化输出各任务日志
+- 去重过滤（布隆过滤器等）
+- 防丢失（单机模式可以通过日志文件进行检查种子）
+- 自定义数据库的功能
+- excel、mysql、redis数据完善
+
+
+![img.png](https://image-luyuan.oss-cn-hangzhou.aliyuncs.com/image/D2388CDC-B9E5-4CE4-9F2C-7D173763B6A8.png)

cobweb_launcher-0.0.1.dist-info/RECORD

@@ -0,0 +1,37 @@
+cobweb/__init__.py,sha256=E6b7Sfy6WJTinGMY8xUpnx_zAH1U_TGmdjt_wOEpi8I,57
+cobweb/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/base/bbb.py,sha256=iX2Xn5AukRussL8xHKTGFfgODWTMaFA3JnvQWd02XDQ,5615
+cobweb/base/config.py,sha256=qZY26tziCeENdopGyVSTCyd_8B30S37GbWx0-_g7EiY,5357
+cobweb/base/decorators.py,sha256=gb2puZLRHv_348ZTIAFLhKuJWKZ6a9_tzduCS8p1-UQ,3089
+cobweb/base/hash_table.py,sha256=-EPHRMNOHHbdjxZJUuDxghUfwrbMA05sqrldHAgrIco,1885
+cobweb/base/interface.py,sha256=LM6C0eh-d1b2CxjtiHKfP8I3XhhlYQR5r-3MD6TMIc4,1037
+cobweb/base/log.py,sha256=Gb3_y4IzTo5pJohTggBCU9rK6-ZN3hgTOHkoXHyN6CU,2384
+cobweb/base/queue_tmp.py,sha256=NS4qBHKq2o-R78Jpv5xp7TtOHIMg8y0livilTVK49M8,1527
+cobweb/base/request.py,sha256=dHTR7qMHbIIW4ggpTAg4io1TBAYH77teYU4bmcWPXH0,2318
+cobweb/base/task.py,sha256=ztgNh4_tgy95pe3REBfMLKkwf7HaShvp-fdRIWJiJXo,1230
+cobweb/base/utils.py,sha256=NSSgCBE4u1yTpXZrjg8RIepYedo4ZdM38rhDObVfRhI,325
+cobweb/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/db/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/db/base/client_db.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+cobweb/db/base/oss_db.py,sha256=LYRsh26-Fttc6sjc9oOVsCd93jZSja6dtRc8G2Em-1E,3812
+cobweb/db/base/redis_db.py,sha256=2-YMrTpiLNGwA7_bt62HYKevVeJr1Y_n88JKjPg1V3s,7636
+cobweb/db/base/redis_dbv3.py,sha256=u-Tmexl0nrYVVRCCbxAjcH6fyRx7CP4J7iW4BdO7q98,8354
+cobweb/db/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/db/scheduler/default.py,sha256=m-zzC2cbHGxplEW5OoB9Vj3nJm30Xl0sAQ94wpPb7Yw,122
+cobweb/db/scheduler/textfile.py,sha256=EiOxV8h99ouIr2HvmpM9B90QY3hqGNPMeQEnps_RG-c,869
+cobweb/db/storer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/db/storer/console.py,sha256=QbHnZ3ou0AR6c35iVy2hYEbKlYYTnFaD8-bUODeNt14,210
+cobweb/db/storer/loghub.py,sha256=Pb0OwVteIllYjL2cIyBlc1WWau1PHn9K1rlAGrr0M3k,1815
+cobweb/db/storer/redis.py,sha256=jK_RirqgSaV4aIWSuySIm5f1ZfZiULqFj2kman2H-Qw,440
+cobweb/db/storer/textfile.py,sha256=yAvtbPkScjZ298H25kWsI0MDg2JuI2Im4m2qmPEUNTM,443
+cobweb/distributed/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/distributed/launcher.py,sha256=IV2jd1hLyt1YyhyN3leSMrtbN5APsmvqblr2NXy18xg,6163
+cobweb/distributed/models.py,sha256=7ypYQaiHP91LbPE0u5Lb-9LDazg74UW7KacK4-ai1tM,4438
+cobweb/single/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cobweb/single/models.py,sha256=lu8teNWnWcUwZFra8XmqyhzOAf3UyuEztwBr1Ne6pUs,2898
+cobweb/single/nest.py,sha256=mL8q9a5BjtoeUyzXCIVw_vyUsNY8ltbvQpYIIpZEDFU,5012
+cobweb_launcher-0.0.1.dist-info/LICENSE,sha256=z1rxSIGOyzcSb3orZxFPxzx-0C1vTocmswqBNxpKfEk,1063
+cobweb_launcher-0.0.1.dist-info/METADATA,sha256=jyujfg_Grc9wymEL9UlDkJvU1zS_N7IC7ZxIpirY6jM,1208
+cobweb_launcher-0.0.1.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
+cobweb_launcher-0.0.1.dist-info/top_level.txt,sha256=4GETBGNsKqiCUezmT-mJn7tjhcDlu7nLIV5gGgHBW4I,7
+cobweb_launcher-0.0.1.dist-info/RECORD,,

cobweb_launcher-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.40.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

cobweb_launcher-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+cobweb