cobweb-launcher 1.2.25__py3-none-any.whl → 3.2.18__py3-none-any.whl

cobweb/utils/dotting.py

@@ -0,0 +1,300 @@
+import os
+import json
+import time
+from threading import Event
+from requests import RequestException, Response as requests_Response
+
+from cobweb.base import Queue, Request, Seed, Response, BaseItem, logger
+from aliyun.log import LogClient, LogItem, PutLogsRequest
+
+
+class LoghubDot:
+
+    def __init__(self, stop: Event, project: str, task: str) -> None:
+        self._stop = stop
+        self._queue = Queue()
+        self._client = LogClient(
+            endpoint=os.getenv("LOGHUB_ENDPOINT"),
+            accessKeyId=os.getenv("LOGHUB_ACCESS_KEY"),
+            accessKey=os.getenv("LOGHUB_SECRET_KEY")
+        )
+        self.project = project
+        self.task = task
+
+    def logging(self, topic, msg):
+        log_item = LogItem()
+        log_data = {
+            "stage": topic,
+            "message": msg,
+            "project": self.project,
+            "task": self.task,
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_request_log(self, request_item: Request):
+        log_item = LogItem()
+
+        seed: Seed = request_item.seed
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        stage_cost = request_time - start_time
+        cost = request_time - start_time
+
+        request_settings = json.dumps(
+            request_item.request_settings,
+            ensure_ascii=False, separators=(',', ':')
+        )
+
+        log_data = {
+            "stage": "request",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "request": repr(request_item),
+            "request_settings": request_settings,
+            "get_time": get_time,
+            "start_time": start_time,
+            "stage_cost": stage_cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(request_time)),
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_download_log(self, response_item: Response):
+        """
+        构建下载阶段的日志项
+
+        Args:
+            response_item: 响应对象
+        """
+        log_item = LogItem()
+
+        seed: Seed = response_item.seed
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        download_time = seed.params.download_time
+        stage_cost = download_time - request_time
+        cost = download_time - start_time
+
+        log_data = {
+            "stage": "download",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "response": repr(response_item),
+            "get_time": get_time,
+            "start_time": start_time,
+            "request_time": request_time,
+            "download_time": download_time,
+            "stage_cost": stage_cost,
+            "cost": cost,
+            "proxy": seed.params.proxy or '-',
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(download_time)),
+        }
+
+        response = response_item.response
+        if isinstance(response, requests_Response):
+            log_data['request_info'] = {
+                'method': response.request.method,
+                'url': response.request.url,
+                'headers': dict(response.request.headers),
+                'body': response.request.body or "-",
+            }
+            log_data['response_info'] = {
+                "status_code": response.status_code,
+                "reason": response.reason,
+                "headers": dict(response.headers),
+                "content": response.text[:500],  # 截取内容
+                "content_type": response.headers.get('content-type', '-'),
+                "content_length": response.headers.get('content-length', '-'),
+                "server": response.headers.get('server', '-'),
+                "date": response.headers.get('date', '-'),
+            }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_parse_log(self, parse_item: BaseItem):
+        log_item = LogItem()
+
+        seed: Seed = parse_item.seed
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        request_time = seed.params.request_time
+        response_time = seed.params.response_time
+        parse_time = seed.params.parse_time
+
+        pre_time = request_time or response_time
+        stage_cost = parse_time - pre_time
+        cost = parse_time - start_time
+
+        log_data = {
+            "stage": "parse",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "parse": repr(parse_item),
+            "get_time": get_time,
+            "start_time": start_time,
+            "parse_time": parse_time,
+            "stage_cost": stage_cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(parse_time)),
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_http_error_log(self, seed: Seed, e: RequestException):
+        log_item = LogItem()
+
+        status_code = getattr(e.response, 'status_code', '-')
+
+        request_info = {
+            'method': getattr(e.request, 'method', '-'),
+            'url': getattr(e.request, 'url', '-'),
+            'headers': dict(getattr(e.request, 'headers', {})),
+            'body': getattr(e.request, 'body', '-'),
+        }
+
+        response_info = {
+            'status_code': getattr(e.response, 'status_code', '-'),
+            'reason': getattr(e.response, 'reason', '-'),
+            'headers': dict(getattr(e.response, 'headers', {})),
+            'content': getattr(e.response, 'text', '')[:500],
+            'content_type': e.response.headers.get('content-type', '-') if e.response else '-',
+            'content_length': e.response.headers.get('content-length', '-') if e.response else '-',
+            'server': e.response.headers.get('server', '-') if e.response else '-',
+            'date': e.response.headers.get('date', '-') if e.response else '-',
+        }
+        retry = seed.params.retry
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        failed_time = seed.params.failed_time
+        cost = failed_time - start_time
+
+        log_data = {
+            "stage": "http_error",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string, 
+            "status_code": status_code,
+            "request_info": request_info,
+            "response_info": response_info,
+            "retry": retry,
+            "proxy": seed.params.proxy or '-',
+            "exception_type": type(e).__name__,
+            "exception_message": str(e),
+            "traceback": seed.params.traceback or '-',
+            "get_time": get_time,
+            "start_time": start_time,
+            "error_time": failed_time,
+            "stage_cost": cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(failed_time)),
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_exception_log(self, seed: Seed, e: Exception):
+        log_item = LogItem()
+
+        retry = seed.params.retry
+        get_time = seed.params.get_time
+        start_time = seed.params.start_time
+        failed_time = seed.params.failed_time
+        cost = failed_time - start_time
+
+        log_data = {
+            "stage": "exception",
+            "project": self.project,
+            "task": self.task,
+            "seed": seed.to_string,
+            "retry": retry,
+            "exception_type": type(e).__name__,
+            "exception_message": str(e),
+            "traceback": seed.params.traceback or '-',
+            "proxy": seed.params.proxy or '-',
+            "get_time": get_time,
+            "start_time": start_time,
+            "error_time": failed_time,
+            "stage_cost": cost,
+            "cost": cost,
+            "time": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(failed_time)),
+        }
+
+        for key, value in log_data.items():
+            if not isinstance(value, str):
+                log_data[key] = json.dumps(value, ensure_ascii=False)
+            else:
+                log_data[key] = value
+
+        contents = sorted(log_data.items())
+        log_item.set_contents(contents)
+        self._queue.push(log_item)
+
+    def _build_run(self):
+        while not self._stop.is_set():
+            try:
+                items = []
+                start_time = int(time.time())
+
+                while len(items) < 1000:
+                    log_item = self._queue.pop()
+                    if not log_item or (int(time.time()) - start_time > 10):
+                        break
+                    items.append(log_item)
+
+                if items:
+                    request = PutLogsRequest(
+                        project="databee-download-log",
+                        logstore="log",
+                        topic="cobweb",
+                        logitems=items,
+                        compress=True
+                    )
+                    self._client.put_logs(request=request)
+            except Exception as e:
+                logger.info(str(e))

cobweb/utils/oss.py

@@ -1,94 +1,113 @@
-from typing import List
-from cobweb import setting
-from requests import Response
-from oss2 import Auth, Bucket, models, PartIterator
-from cobweb.exceptions import oss_db_exception
-from cobweb.base.decorators import decorator_oss_db
-
-
-class OssUtil:
-
-    def __init__(
-        self,
-        bucket=None,
-        endpoint=None,
-        access_key=None,
-        secret_key=None,
-        chunk_size=None,
-        min_upload_size=None,
-        **kwargs
-    ):
-        self.bucket = bucket or setting.OSS_BUCKET
-        self.endpoint = endpoint or setting.OSS_ENDPOINT
-        self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
-        self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
-
-        self._auth = Auth(
-            access_key_id=access_key or setting.OSS_ACCESS_KEY,
-            access_key_secret=secret_key or setting.OSS_SECRET_KEY
-        )
-        self._client = Bucket(
-            auth=self._auth,
-            endpoint=self.endpoint,
-            bucket_name=self.bucket,
-            **kwargs
-        )
-
-    def exists(self, key: str) -> bool:
-        return self._client.object_exists(key)
-
-    def head(self, key: str) -> models.HeadObjectResult:
-        return self._client.head_object(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
-    def init_part(self, key) -> models.InitMultipartUploadResult:
-        """初始化分片上传"""
-        return self._client.init_multipart_upload(key)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
-    def put(self, key, data) -> models.PutObjectResult:
-        """文件上传"""
-        return self._client.put_object(key, data)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
-    def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
-        """分片上传"""
-        return self._client.upload_part(key, upload_id, position, data)
-
-    def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
-        """获取分片列表"""
-        return [part_info for part_info in PartIterator(self._client, key, upload_id)]
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
-    def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
-        """合并分片"""
-        headers = None if parts else {"x-oss-complete-all": "yes"}
-        return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
-
-    @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
-    def append(self, key, position, data) -> models.AppendObjectResult:
-        """追加上传"""
-        return self._client.append_object(key, position, data)
-
-    def iter_data(self, data, chunk_size=None):
-        chunk_size = chunk_size or self.chunk_size
-        if isinstance(data, Response):
-            for part_data in data.iter_content(chunk_size):
-                yield part_data
-        if isinstance(data, bytes):
-            for i in range(0, len(data), chunk_size):
-                yield data[i:i + chunk_size]
-
-    def assemble(self, ready_data, data, chunk_size=None):
-        upload_data = b""
-        ready_data = ready_data + data
-        chunk_size = chunk_size or self.chunk_size
-        if len(ready_data) >= chunk_size:
-            upload_data = ready_data[:chunk_size]
-            ready_data = ready_data[chunk_size:]
-        return ready_data, upload_data
-
-    def content_length(self, key: str) -> int:
-        head = self.head(key)
-        return head.content_length
-
+#
+# from cobweb import setting
+# from requests import Response
+# from oss2 import Auth, Bucket, models, PartIterator
+# from cobweb.exceptions import oss_db_exception
+# from cobweb.utils.decorators import decorator_oss_db
+#
+#
+# class OssUtil:
+#
+#     def __init__(
+#         self,
+#         bucket=None,
+#         endpoint=None,
+#         access_key=None,
+#         secret_key=None,
+#         chunk_size=None,
+#         min_upload_size=None,
+#         **kwargs
+#     ):
+#         self.bucket = bucket or setting.OSS_BUCKET
+#         self.endpoint = endpoint or setting.OSS_ENDPOINT
+#         self.chunk_size = int(chunk_size or setting.OSS_CHUNK_SIZE)
+#         self.min_upload_size = int(min_upload_size or setting.OSS_MIN_UPLOAD_SIZE)
+#
+#         self.failed_count = 0
+#         self._kw = kwargs
+#
+#         self._auth = Auth(
+#             access_key_id=access_key or setting.OSS_ACCESS_KEY,
+#             access_key_secret=secret_key or setting.OSS_SECRET_KEY
+#         )
+#         self._client = Bucket(
+#             auth=self._auth,
+#             endpoint=self.endpoint,
+#             bucket_name=self.bucket,
+#             **self._kw
+#         )
+#
+#     def failed(self):
+#         self.failed_count += 1
+#         if self.failed_count >= 5:
+#             self._client = Bucket(
+#                 auth=self._auth,
+#                 endpoint=self.endpoint,
+#                 bucket_name=self.bucket,
+#                 **self._kw
+#             )
+#
+#     def exists(self, key: str) -> bool:
+#         try:
+#             result = self._client.object_exists(key)
+#             self.failed_count = 0
+#             return result
+#         except Exception as e:
+#             self.failed()
+#             raise e
+#
+#     def head(self, key: str) -> models.HeadObjectResult:
+#         return self._client.head_object(key)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBInitPartError)
+#     def init_part(self, key) -> models.InitMultipartUploadResult:
+#         """初始化分片上传"""
+#         return self._client.init_multipart_upload(key)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBPutObjError)
+#     def put(self, key, data) -> models.PutObjectResult:
+#         """文件上传"""
+#         return self._client.put_object(key, data)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBPutPartError)
+#     def put_part(self, key, upload_id, position, data) -> models.PutObjectResult:
+#         """分片上传"""
+#         return self._client.upload_part(key, upload_id, position, data)
+#
+#     def list_part(self, key, upload_id):  # -> List[models.ListPartsResult]:
+#         """获取分片列表"""
+#         return [part_info for part_info in PartIterator(self._client, key, upload_id)]
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBMergeError)
+#     def merge(self, key, upload_id, parts=None) -> models.PutObjectResult:
+#         """合并分片"""
+#         headers = None if parts else {"x-oss-complete-all": "yes"}
+#         return self._client.complete_multipart_upload(key, upload_id, parts, headers=headers)
+#
+#     @decorator_oss_db(exception=oss_db_exception.OssDBAppendObjError)
+#     def append(self, key, position, data) -> models.AppendObjectResult:
+#         """追加上传"""
+#         return self._client.append_object(key, position, data)
+#
+#     def iter_data(self, data, chunk_size=None):
+#         chunk_size = chunk_size or self.chunk_size
+#         if isinstance(data, Response):
+#             for part_data in data.iter_content(chunk_size):
+#                 yield part_data
+#         if isinstance(data, bytes):
+#             for i in range(0, len(data), chunk_size):
+#                 yield data[i:i + chunk_size]
+#
+#     def assemble(self, ready_data, data, chunk_size=None):
+#         upload_data = b""
+#         ready_data = ready_data + data
+#         chunk_size = chunk_size or self.chunk_size
+#         if len(ready_data) >= chunk_size:
+#             upload_data = ready_data[:chunk_size]
+#             ready_data = ready_data[chunk_size:]
+#         return ready_data, upload_data
+#
+#     def content_length(self, key: str) -> int:
+#         head = self.head(key)
+#         return head.content_length
+#

cobweb/utils/tools.py

@@ -1,5 +1,6 @@
 import re
 import hashlib
+import inspect
 from typing import Union
 from importlib import import_module
 
@@ -10,18 +11,6 @@ def md5(text: Union[str, bytes]) -> str:
     return hashlib.md5(text).hexdigest()
 
 
-def build_path(site, url, file_type):
-    return f"{site}/{md5(url)}.{file_type}"
-
-
-def format_size(content_length: int) -> str:
-    units = ["KB", "MB", "GB", "TB"]
-    for i in range(4):
-        num = content_length / (1024 ** (i + 1))
-        if num < 1024:
-            return f"{round(num, 2)} {units[i]}"
-
-
 def dynamic_load_class(model_info):
     if isinstance(model_info, str):
         if "import" in model_info:
@@ -35,8 +24,7 @@ def dynamic_load_class(model_info):
             model = import_module(model_path)
             class_object = getattr(model, class_name)
         return class_object
+    elif inspect.isclass(model_info):
+        return model_info
     raise TypeError()
 
-
-# def download_log_info(item:dict) -> str:
-#     return "\n".join([" " * 12 + f"{str(k).ljust(14)}:    {str(v)}" for k, v in item.items()])