cobweb-launcher 1.2.25__py3-none-any.whl → 3.2.18__py3-none-any.whl

cobweb/launchers/uploader.py

@@ -0,0 +1,68 @@
+import time
+import threading
+from typing import Callable, Type
+from cobweb.pipelines import Pipeline
+from cobweb.base import TaskQueue, logger, Status
+from cobweb.utils import check_pause
+
+
+class Uploader(threading.Thread):
+
+    def __init__(
+            self,
+            task: str,
+            project: str,
+            stop: threading.Event,
+            pause: threading.Event,
+            task_queue: TaskQueue,
+            callback_register: Callable,
+            SpiderPipeline: Type[Pipeline]
+    ):
+        super().__init__()
+        self.task = task
+        self.project = project
+
+        self.stop = stop
+        self.pause = pause
+
+        self.task_queue = task_queue
+        self.callback_register = callback_register
+
+        from cobweb import setting
+
+        self.upload_size = setting.UPLOAD_QUEUE_MAX_SIZE
+        self.wait_seconds = setting.UPLOAD_QUEUE_WAIT_SECONDS
+
+        self.pipeline = SpiderPipeline(task=self.task, project=self.project)
+
+        logger.debug(f"Uploader instance attrs: {self.__dict__}")
+
+    @check_pause
+    def upload_data(self):
+        try:
+            data_info, task_ids = dict(), set()
+            if task_list := self.task_queue.get_task_by_status(
+                    status=Status.UPLOAD, limit=self.upload_size
+            ):
+                for task_item in task_list:
+                    upload_data = self.pipeline.build(task_item.data)
+                    data_info.setdefault(task_item.data.table, []).append(upload_data)
+                    task_ids.add(task_item.task_id)
+
+                for table, datas in data_info.items():
+                    try:
+                        self.pipeline.upload(table, datas)
+                    except Exception as e:
+                        logger.info(e)
+
+            self.task_queue.remove(task_ids)
+        except Exception as e:
+            logger.info(e)
+
+        if self.task_queue.status_length(status=Status.UPLOAD) < self.upload_size:
+            time.sleep(self.wait_seconds)
+
+    def run(self):
+        self.callback_register(self.upload_data, tag="Uploader")
+
+

cobweb/log_dots/__init__.py

@@ -0,0 +1,2 @@
+from .dot import Dot
+from .loghub_dot import LoghubDot

cobweb/log_dots/dot.py

@@ -0,0 +1,258 @@
+import json
+import os
+import time
+from threading import Event
+from requests import RequestException, Response as requests_Response
+from cobweb.base import Queue, Request, Seed, Response, BaseItem, logger
+
+
+class Dot:
+
+    def __init__(self, stop: Event, project: str, task: str) -> None:
+        self._stop = stop
+        self._queue = Queue()
+        self.project = project
+        self.task = task
+
+    def logging(self, topic, msg):
+        log_data = {
+            "stage": topic,
+            "message": msg,
+            "project": self.project,
+            "task": self.task,
+        }
+        self._queue.push(log_data)
+
+    def _build_request_log(self, request_item: Request):
+        seed: Seed = request_item.seed
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        stage_cost = request_time - start_time if request_time and start_time else 0
+        cost = request_time - start_time if request_time and start_time else 0
+
+        instance_id = os.getenv("INSTANCE_ID")
+
+        request_settings = json.dumps(
+            request_item.request_settings,
+            ensure_ascii=False, separators=(',', ':')
+        )
+
+        log_data = {
+            "stage": "request",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "request": repr(request_item),
+            "instance_id": instance_id,
+            "request_settings": request_settings,
+            "get_time": get_time,
+            "start_time": start_time,
+            "stage_cost": stage_cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(request_time)),
+        }
+
+        self._queue.push(log_data)
+
+    def _build_download_log(self, response_item: Response):
+        seed: Seed = response_item.seed
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        download_time = seed.params.download_time
+        stage_cost = download_time - request_time if request_time and download_time else 0
+        cost = download_time - start_time if download_time and start_time else 0
+
+        instance_id = os.getenv("INSTANCE_ID")
+
+        log_data = {
+            "stage": "download",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "response": repr(response_item),
+            "get_time": get_time,
+            "start_time": start_time,
+            "request_time": request_time,
+            "download_time": download_time,
+            "stage_cost": stage_cost,
+            "cost": cost,
+            "instance_id": instance_id,
+            "proxy": seed.params.proxy or '-',
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(download_time)),
+        }
+
+        if (response := response_item.response) and isinstance(response, requests_Response):
+            log_data['request_info'] = {
+                'method': response.request.method,
+                'url': response.request.url,
+                'headers': dict(response.request.headers),
+                'body': response.request.body or "-",
+            }
+            log_data['response_info'] = {
+                "status_code": response.status_code,
+                "reason": response.reason,
+                "headers": dict(response.headers),
+                # "content": response.text[:500],  # 截取内容
+                "content_type": response.headers.get('content-type', '-'),
+                "content_length": response.headers.get('content-length', '-'),
+                "server": response.headers.get('server', '-'),
+                "date": response.headers.get('date', '-'),
+            }
+
+        self._queue.push(log_data)
+
+    def _build_parse_log(self, parse_item: BaseItem):
+        seed: Seed = parse_item.seed
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        response_time = seed.params.response_time
+        parse_time = seed.params.parse_time
+        instance_id = os.getenv("INSTANCE_ID")
+
+        pre_time = request_time or response_time
+        stage_cost = parse_time - pre_time if parse_time and pre_time else 0
+        cost = parse_time - start_time if parse_time and start_time else 0
+
+        log_data = {
+            "stage": "parse",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "parse": repr(parse_item),
+            "get_time": get_time,
+            "start_time": start_time,
+            "instance_id": instance_id,
+            "parse_time": parse_time,
+            "stage_cost": stage_cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(parse_time)),
+        }
+
+        self._queue.push(log_data)
+
+    def _build_http_error_log(self, seed: Seed, e: RequestException):
+        status_code = getattr(e.response, 'status_code', '-')
+        instance_id = os.getenv("INSTANCE_ID")
+        request_info = {
+            'method': getattr(e.request, 'method', '-'),
+            'url': getattr(e.request, 'url', '-'),
+            'headers': dict(getattr(e.request, 'headers', {})),
+            'body': getattr(e.request, 'body', '-'),
+        }
+
+        response_info = {
+            'status_code': getattr(e.response, 'status_code', '-'),
+            'reason': getattr(e.response, 'reason', '-'),
+            'headers': dict(getattr(e.response, 'headers', {})),
+            # 'content': getattr(e.response, 'text', '')[:500],
+            'content_type': e.response.headers.get('content-type', '-') if e.response else '-',
+            'content_length': e.response.headers.get('content-length', '-') if e.response else '-',
+            'server': e.response.headers.get('server', '-') if e.response else '-',
+            'date': e.response.headers.get('date', '-') if e.response else '-',
+        }
+        retry = seed.params.retry
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        failed_time = seed.params.failed_time
+        cost = failed_time - start_time if failed_time and start_time else 0
+
+        log_data = {
+            "stage": "http_error",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "status_code": status_code,
+            "request_info": request_info,
+            "response_info": response_info,
+            "instance_id": instance_id,
+            "retry": retry,
+            "proxy": seed.params.proxy or '-',
+            "exception_type": type(e).__name__,
+            "exception_message": str(e),
+            "traceback": seed.params.traceback or '-',
+            "get_time": get_time,
+            "start_time": start_time,
+            "error_time": failed_time,
+            "stage_cost": cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(failed_time)),
+        }
+
+        self._queue.push(log_data)
+
+    def _build_exception_log(self, seed: Seed, e: Exception):
+        retry = seed.params.retry
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        failed_time = seed.params.failed_time
+        cost = failed_time - start_time if failed_time and start_time else 0
+        instance_id = os.getenv("INSTANCE_ID")
+        log_data = {
+            "stage": "exception",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "instance_id": instance_id,
+            "retry": retry,
+            "exception_type": type(e).__name__,
+            "exception_message": str(e),
+            "traceback": seed.params.traceback or '-',
+            "proxy": seed.params.proxy or '-',
+            "get_time": get_time,
+            "start_time": start_time,
+            "error_time": failed_time,
+            "stage_cost": cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(failed_time)),
+        }
+        self._queue.push(log_data)
+
+    def _build_finish_log(self, seed: Seed, status: bool):
+        retry = seed.params.retry
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        response_time = seed.params.response_time
+        parse_time = seed.params.parse_time
+        finsh_time = seed.params.finsh_time
+
+        cost = finsh_time - start_time if finsh_time and start_time else 0
+        instance_id = os.getenv("instance_id")
+        log_data = {
+            "retry": retry,
+            "stage": "finish",
+            "status": "succeed" if status else "failed",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "instance_id": instance_id,
+            "traceback": seed.params.traceback or '-',
+            "proxy": seed.params.proxy or '-',
+            "get_time": get_time,
+            "start_time": start_time,
+            "request_time": request_time or '-',
+            "response_time": response_time or '-',
+            "parse_time": parse_time or '-',
+            "finsh_time": finsh_time or '-',
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(finsh_time)),
+        }
+        self._queue.push(log_data)
+
+    def _build_run(self):
+        while not self._stop.is_set():
+            try:
+                items = []
+                start_time = int(time.time())
+
+                while len(items) < 1000:
+                    log_item = self._queue.pop()
+                    print(log_item)
+                    if not log_item or (int(time.time()) - start_time > 10):
+                        break
+
+            except Exception as e:
+                logger.info(str(e))

cobweb/log_dots/loghub_dot.py

@@ -0,0 +1,53 @@
+import json
+import os
+import time
+
+from aliyun.log import LogClient, PutLogsRequest, LogItem
+from .dot import Dot, logger, Event
+
+
+class LoghubDot(Dot):
+
+    def __init__(self, stop: Event, project: str, task: str):
+        super().__init__(stop, project, task)
+        self._client = LogClient(
+            endpoint=os.getenv("DOT_LOGHUB_ENDPOINT"),
+            accessKeyId=os.getenv("DOT_LOGHUB_ACCESS_KEY"),
+            accessKey=os.getenv("DOT_LOGHUB_SECRET_KEY")
+        )
+
+    def _build_run(self):
+        while not self._stop.is_set():
+            try:
+                items = []
+                start_time = int(time.time())
+
+                while len(items) < 1000 and (int(time.time()) - start_time) < 10:
+                    log_data = self._queue.pop()
+
+                    if not log_data:
+                        break
+
+                    for key, value in log_data.items():
+                        if not isinstance(value, str):
+                            log_data[key] = json.dumps(value, ensure_ascii=False)
+                        else:
+                            log_data[key] = value
+
+                    log_item = LogItem()
+                    contents = sorted(log_data.items())
+                    log_item.set_contents(contents)
+                    items.append(log_item)
+
+                if items:
+                    request = PutLogsRequest(
+                        project="databee-download-log",
+                        logstore="log",
+                        topic="cobweb",
+                        logitems=items,
+                        compress=True
+                    )
+                    self._client.put_logs(request=request)
+            except Exception as e:
+                logger.info(str(e))
+

cobweb/pipelines/__init__.py

@@ -1,3 +1,3 @@
 from .pipeline import Pipeline
-from .pipeline_console import Console
 from .pipeline_loghub import Loghub
+from .pipeline_csv import CSV

cobweb/pipelines/pipeline.py

@@ -1,28 +1,12 @@
-import time
-import threading
-
 from abc import ABC, abstractmethod
-from cobweb.base import BaseItem, Queue, logger
-
+from cobweb.base import BaseItem
 
-class Pipeline(threading.Thread, ABC):
 
-    def __init__(
-            self,
-            stop: threading.Event,
-            pause: threading.Event,
-            upload: Queue, done: Queue,
-            upload_size: int,
-            wait_seconds: int
-    ):
-        super().__init__()
-        self._stop = stop
-        self._pause = pause
-        self._upload = upload
-        self._done = done
+class Pipeline(ABC):
 
-        self.upload_size = upload_size
-        self.wait_seconds = wait_seconds
+    def __init__(self, task, project):
+        self.task = task
+        self.project = project
 
     @abstractmethod
     def build(self, item: BaseItem) -> dict:
@@ -32,38 +16,4 @@ class Pipeline(threading.Thread, ABC):
     def upload(self, table: str, data: list) -> bool:
         pass
 
-    def run(self):
-        while not self._stop.is_set():
-            if not self._upload.length:
-                time.sleep(self.wait_seconds)
-                continue
-            if self._upload.length < self.upload_size:
-                time.sleep(self.wait_seconds)
-            status = True
-            data_info, seeds = {}, []
-            try:
-                for _ in range(self.upload_size):
-                    item = self._upload.pop()
-                    if not item:
-                        break
-                    seeds.append(item.seed)
-                    data = self.build(item)
-                    data_info.setdefault(item.table, []).append(data)
-                for table, datas in data_info.items():
-                    try:
-                        self.upload(table, datas)
-                    except Exception as e:
-                        logger.info(e)
-                        status = False
-            except Exception as e:
-                logger.info(e)
-                status = False
-            if not status:
-                for seed in seeds:
-                    seed.params.seed_status = "deal model: fail"
-            if seeds:
-                self._done.push(seeds)
-
-        logger.info("upload pipeline close!")
-
 

cobweb/pipelines/pipeline_csv.py

@@ -0,0 +1,25 @@
+import os
+import csv
+
+from cobweb.base import BaseItem
+from cobweb.pipelines import Pipeline
+
+
+class CSV(Pipeline):
+
+    def __init__(self, *args, **kwargs):
+        super(CSV, self).__init__(*args, **kwargs)
+        self.log_path = rf"{os.getcwd()}\{self.project}\{self.task}\%s.csv"
+
+    def build(self, item: BaseItem):
+        return item.to_dict
+
+    def upload(self, table, datas):
+        fieldnames = datas[0].keys()
+        file_path = self.log_path % table
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        with open(file_path, mode='a', encoding='utf-8', newline="") as file:
+            writer = csv.DictWriter(file, fieldnames=fieldnames)
+            if file.tell() == 0:  # 判断文件是否为空
+                writer.writeheader()
+            writer.writerows(datas)

cobweb/pipelines/pipeline_loghub.py

@@ -1,16 +1,24 @@
+import os
 import json
 
-from cobweb import setting
 from cobweb.base import BaseItem
 from cobweb.pipelines import Pipeline
 from aliyun.log import LogClient, LogItem, PutLogsRequest
+from collections import defaultdict
 
 
 class Loghub(Pipeline):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.client = LogClient(**setting.LOGHUB_CONFIG)
+        self.client = LogClient(
+            endpoint=os.getenv("LOGHUB_ENDPOINT"),
+            accessKeyId=os.getenv("LOGHUB_ACCESS_KEY"),
+            accessKey=os.getenv("LOGHUB_SECRET_KEY")
+        )
+        self.project = os.getenv("LOGHUB_PROJECT")
+        self.source = os.getenv("LOGHUB_SOURCE")
+        self.topic = os.getenv("LOGHUB_TOPIC")
 
     def build(self, item: BaseItem):
         log_item = LogItem()
@@ -20,15 +28,27 @@ class Loghub(Pipeline):
                 temp[key] = json.dumps(value, ensure_ascii=False)
         contents = sorted(temp.items())
         log_item.set_contents(contents)
-        return log_item
+        return (
+            log_item,
+            item.baseitem_topic or self.topic,
+            item.baseitem_source or self.source,
+            item.baseitem_project or self.project,
+        )
 
     def upload(self, table, datas):
-        request = PutLogsRequest(
-            project=setting.LOGHUB_PROJECT,
-            logstore=table,
-            topic=setting.LOGHUB_TOPIC,
-            source=setting.LOGHUB_SOURCE,
-            logitems=datas,
-            compress=True
-        )
-        self.client.put_logs(request=request)
+
+        upload_items = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
+
+        for log_item, topic, source, project in datas:
+            upload_items[project][source][topic].append(log_item)
+
+        for request in [
+            PutLogsRequest(
+                logstore=table, project=project,
+                topic=topic, source=source,
+                logitems=log_items, compress=True
+            ) for project, sources in upload_items.items()
+            for source, topics in sources.items()
+            for topic, log_items in topics.items()
+        ]:
+            self.client.put_logs(request=request)

cobweb/schedulers/__init__.py

@@ -0,0 +1 @@
+from .scheduler_with_redis import RedisScheduler

cobweb/schedulers/scheduler.py

@@ -0,0 +1,66 @@
+import threading
+
+from typing import Callable
+from cobweb.base import TaskQueue
+from abc import ABC, abstractmethod
+
+
+class Scheduler(ABC, threading.Thread):
+
+    # __LAUNCHER_FUNC__ = ["_reset", "_scheduler", "_insert", "_refresh", "_delete"]
+
+    def __init__(
+            self,
+            task,
+            project,
+            stop: threading.Event,
+            pause: threading.Event,
+            task_queue: TaskQueue,
+            callback_register: Callable
+    ):
+        super().__init__()
+        self.task = task
+        self.project = project
+        from cobweb import setting
+
+        self.task_model = setting.TASK_MODEL
+        self.seed_reset_seconds = setting.SEED_RESET_SECONDS
+        self.scheduler_wait_seconds = setting.SCHEDULER_WAIT_SECONDS
+        self.new_queue_wait_seconds = setting.NEW_QUEUE_WAIT_SECONDS
+        self.done_queue_wait_seconds = setting.DONE_QUEUE_WAIT_SECONDS
+        self.todo_queue_full_wait_seconds = setting.TODO_QUEUE_FULL_WAIT_SECONDS
+        self.before_scheduler_wait_seconds = setting.BEFORE_SCHEDULER_WAIT_SECONDS
+
+        self.todo_queue_size = setting.TODO_QUEUE_SIZE
+        self.new_queue_max_size = setting.NEW_QUEUE_MAX_SIZE
+        self.done_queue_max_size = setting.DONE_QUEUE_MAX_SIZE
+        self.upload_queue_max_size = setting.UPLOAD_QUEUE_MAX_SIZE
+
+        self.stop = stop
+        self.pause = pause
+
+        self.task_queue = task_queue
+
+        self.callback_register = callback_register
+
+    @abstractmethod
+    def reset(self):
+        ...
+
+    @abstractmethod
+    def schedule(self):
+        ...
+
+    @abstractmethod
+    def insert(self):
+        ...
+
+    @abstractmethod
+    def refresh(self):
+        ...
+
+    @abstractmethod
+    def delete(self):
+        ...
+
+