bricks-py 0.0.2__tar.gz → 0.0.4__tar.gz

{bricks-py-0.0.2 → bricks-py-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bricks-py
-Version: 0.0.2
+Version: 0.0.4
 Summary: quickly build your crawler
 Author: Kem
 Author-email: 531144129@qq.com

{bricks-py-0.0.2 → bricks-py-0.0.4}/README.md

@@ -16,7 +16,7 @@ Bricks 拥有以下特性
 安装
 
 ```
-pip install bricks-py==0.0.1
+pip install bricks-py
 ```
 
 # 简单上手

{bricks-py-0.0.2 → bricks-py-0.0.4}/bricks/const.py

@@ -9,7 +9,7 @@ import uuid
 MACHINE_ID = hashlib.sha256(uuid.UUID(int=uuid.getnode()).hex[-12:].encode()).hexdigest()
 
 # 当前框架版本
-VERSION = "0.0.2"
+VERSION = "0.0.4"
 
 # 事件类型
 ERROR_OCCURRED = 'ERROR_OCCURRED'

{bricks-py-0.0.2 → bricks-py-0.0.4}/bricks/core/genesis.py

@@ -199,6 +199,7 @@ class Pangu(Chaos):
                     pass
 
                 except signals.Retry:
+                    context.signpost['retry'] = True
                     context.retry()
 
                 except signals.Success:

{bricks-py-0.0.2 → bricks-py-0.0.4}/bricks/downloader/genesis.py

@@ -108,12 +108,17 @@ class Downloader(metaclass=genesis.MetaClass):
         # 获取请求头中的Content-Type
         content_type = request.headers.get('Content-Type', '').lower()
 
-        if not content_type or not isinstance(request.body, str):
+        # 如果 body 本来就是字符串 / bytes -> 直接使用, 不需要转换
+        if isinstance(request.body, (str, bytes)):
             return {
                 "data": request.body,
-                "type": "unknown"
+                "type": "raw"
             }
 
+        # 没有传 content-type, 并且 body 不为字符串, 默认设置为 application/json
+        if not content_type:
+            content_type = 'application/json'
+
         # 根据Content-Type判断并处理请求体
         if 'application/json' in content_type:
             try:
@@ -143,4 +148,4 @@ class Downloader(metaclass=genesis.MetaClass):
             }
 
         else:
-            raise ValueError(f"Invalid JSON format, raw: {request.body}")
+            raise ValueError(f"Unsupported Content-Type: {content_type}")

{bricks-py-0.0.2 → bricks-py-0.0.4}/bricks/lib/context.py

@@ -67,6 +67,7 @@ class Flow(Context):
         super().__init__(form, target, **kwargs)
         self.doing: deque = deque([self])
         self.pending: deque = deque([])
+        self.signpost: dict = self.install("signpost", {}, True)
 
     def _set_next(self, value):
         if isinstance(value, Node):

{bricks-py-0.0.2 → bricks-py-0.0.4}/bricks/lib/proxies.py

@@ -35,6 +35,7 @@ class MetaClass(type):
     def __new__(cls, name, bases, dct):  # noqa
         def wrapper(raw_method):
             def inner(self, *args, **kwargs):
+                self: BaseProxy
                 proxy = raw_method(self, *args, **kwargs)
                 proxy.proxy = self.fmt(proxy=proxy.proxy)
                 proxy.auth = self.auth

{bricks-py-0.0.2 → bricks-py-0.0.4}/bricks/spider/air.py

@@ -137,6 +137,7 @@ class Spider(Pangu):
         return {
             self.on_consume: self.on_seeds,
             self.on_seeds: self.on_request,
+            self.on_retry: self.on_request,
             self.on_request: self.on_response,
             self.on_response: self.on_pipeline,
             self.on_pipeline: None
@@ -535,7 +536,7 @@ class Spider(Pangu):
                 }
             )
             ret = prepared.func(*prepared.args, **prepared.kwargs)
-            context.flow({"next": self.on_request})
+            context.flow()
             return ret
 
         return wrapper

{bricks-py-0.0.2 → bricks-py-0.0.4}/bricks/spider/form.py

@@ -28,69 +28,26 @@ FORMAT_REGEX = re.compile(r'{(\w+)(?::(\w+))?}')
 class Node:
 
     @classmethod
-    def format(cls, value, base, errors: str = "raise"):
+    def format(cls, value, base: dict, errors: str = "raise"):
         if isinstance(value, str):
-            # 使用正则表达式提取占位符和类型
-            # try:
-            #     return value.format(**base)
-            # except ValueError:
-            #     placeholders = FORMAT_REGEX.findall(value)
-            #
-            #     # 只有一个
-            #     if len(placeholders) == 1:
-            #         placeholder, type_str = placeholders[0]
-            #         try:
-            #             placeholder_value = base[placeholder]
-            #             return cls.convert(placeholder_value, type_str)
-            #         except KeyError as e:
-            #             if errors == "raise":
-            #                 raise ValueError(f"Missing key in base: {e}")
-            #
-            #             elif errors == 'ignore':
-            #                 return value
-            #
-            #             else:
-            #                 return ""
-            #
-            #     elif len(placeholders) == 0:
-            #         raise
-            #     else:
-            #         for placeholder, type_str in placeholders:
-            #             try:
-            #                 placeholder_value = base[placeholder]
-            #             except KeyError as e:
-            #                 if errors == "raise":
-            #                     raise ValueError(f"Missing key in base: {e}")
-            #                 elif errors == 'ignore':
-            #                     placeholder_value = f"{{{placeholder}:{type_str}}}"
-            #                 else:
-            #                     placeholder_value = ""
-            #
-            #             value = value.replace(f"{{{placeholder}:{type_str}}}", str(placeholder_value))
-            #
-            #         else:
-            #             return value
-            #
-            # except KeyError as e:
-            #     if errors == "raise":
-            #         raise ValueError(f"Missing key in base: {e}")
-            #
-            #     elif errors == 'ignore':
-            #         return value
-            #
-            #     else:
-            #         return ""
-            # 查找所有占位符
+            while True:
+                try:
+                    return value.format(**base)
+                except ValueError:
+
+                    placeholders = FORMAT_REGEX.findall(value)
+                    # 有多个, 那最终肯定还是字符串
+                    convert_value = len(placeholders) == 1
+                    for placeholder, type_str in placeholders:
+
+                        if placeholder not in base:
+                            if errors == 'raise':
+                                raise ValueError(f"Missing key in base: {placeholder}")
+                            elif errors == 'ignore':
+                                return value
+                            else:
+                                base.setdefault(placeholder, "")
 
-            try:
-                return value.format(**base)
-            except ValueError:
-
-                placeholders = FORMAT_REGEX.findall(value)
-                # 有多个, 那最终肯定还是字符串
-                convert_value = len(placeholders) == 1
-                for placeholder, type_str in placeholders:
-                    if placeholder in base:
                         placeholder_value = base[placeholder]
                         if type_str:
                             placeholder_value = cls.convert(placeholder_value, type_str)
@@ -101,23 +58,17 @@ class Node:
                         if convert_value:
                             value = cls.convert(value, type(placeholder_value))
 
-                    elif errors == 'raise':
-                        raise ValueError(f"Missing key in base: {placeholder}")
-                    elif errors == 'ignore':
-                        return value
-                    else:
-                        return ""
-                return value
+                    return value
 
-            except KeyError as e:
-                if errors == "raise":
-                    raise ValueError(f"Missing key in base: {e}")
+                except KeyError as e:
+                    if errors == "raise":
+                        raise ValueError(f"Missing key in base: {e}")
 
-                elif errors == 'ignore':
-                    return value
+                    elif errors == 'ignore':
+                        return value
 
-                else:
-                    return ""
+                    else:
+                        base.setdefault(e.args[0], "")
 
         elif isinstance(value, list):
             return [cls.format(item, base, errors=errors) for item in value]
@@ -251,6 +202,7 @@ class Spider(air.Spider):
             self.on_consume: self.on_flow,
             self.on_seeds: self.on_request,
             self.on_request: self.on_flow,
+            self.on_retry: self.on_flow,
             self.on_response: self.on_flow,
             self.on_pipeline: self.on_flow,
         }
@@ -260,23 +212,43 @@ class Spider(air.Spider):
         raise NotImplementedError
 
     def on_flow(self, context: Context):
-        context.signpost = context.install("signpost", 0, True)
         if not self.config.spider:
             logger.warning('没有配置 Spider 节点流程..')
             raise signals.Exit
 
+        context.signpost.setdefault("cursor", 0)
+        # 这是重试回来了
+        if context.signpost.pop('retry', False):
+            # 找到之前下载节点的位置
+            bookmark = context.signpost.get('bookmark', 0)
+            # 没有下载节点 / 下载节点就在第一个 -> cursor 指向最起点
+            if bookmark == 0:
+                context.signpost['cursor'] = 1
+
+            # 找到下载节点前面不是 Task 的节点, 但是如果是两个
+            else:
+                for i in range(bookmark - 1, -1, -1):
+                    node = self.config.spider[i]
+                    if not isinstance(node, Task):
+                        context.signpost['cursor'] = i + 1
+                        break
+                else:
+                    context.signpost['cursor'] = bookmark
+
         while True:
             try:
-                node: Union[Download, Task, Parse, Pipeline] = self.config.spider[context.signpost]
+                node: Union[Download, Task, Parse, Pipeline] = self.config.spider[context.signpost['cursor']]
                 context.node = node
             except IndexError:
                 context.flow({"next": None})
                 raise signals.Switch
             else:
-                context.signpost += 1
+                context.signpost['cursor'] += 1
 
                 # 种子 -> Request
                 if isinstance(node, Download):
+                    # 记录下载节点的位置
+                    context.signpost['bookmark'] = context.signpost['cursor'] - 1
                     context.flow({"next": self.on_seeds})
                     raise signals.Switch
 

{bricks-py-0.0.2 → bricks-py-0.0.4}/bricks/utils/pandora.py

@@ -116,6 +116,10 @@ def prepare(func, args=None, kwargs: dict = None, annotations: dict = None, name
         if name in kwargs:
             value = kwargs[name]
 
+        # 参数在 namespace 里面 -> 从 namespace 里面取
+        elif param.name in namespace:
+            value = namespace[param.name]
+
         # 参数类型存在于 annotations, 并且还可以从 args 里面取值, 并且刚好取到的对应的值也是当前类型 -> 直接从 args 里面取
         elif param.annotation in annotations and index < len(args) and type(args[index]) == param.annotation:
             value = args[index]
@@ -130,20 +134,16 @@ def prepare(func, args=None, kwargs: dict = None, annotations: dict = None, name
             value = args[index]
             index += 1
 
-        elif param.name in namespace:
-
-            value = namespace[param.name]
-
         elif param.default != inspect.Parameter.empty:
             continue
 
         # 没有传这个参数, 并且也没有可以备选的 annotations  -> 报错
         else:
             raise TypeError(f"missing required argument: {name}, signature: {dict(parameters)}")
-        if param.kind in [inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD]:
+        if param.kind in [inspect.Parameter.POSITIONAL_ONLY]:
             new_args.append(value)
 
-        if param.kind == inspect.Parameter.KEYWORD_ONLY:
+        if param.kind in [inspect.Parameter.KEYWORD_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD]:
             new_kwargs[name] = value
 
     return prepared(func=func, args=new_args, kwargs=new_kwargs)

{bricks-py-0.0.2 → bricks-py-0.0.4}/bricks_py.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bricks-py
-Version: 0.0.2
+Version: 0.0.4
 Summary: quickly build your crawler
 Author: Kem
 Author-email: 531144129@qq.com

{bricks-py-0.0.2 → bricks-py-0.0.4}/demos/air_spider_demo.py

@@ -85,27 +85,27 @@ if __name__ == '__main__':
     spider = MySpider(
         # # 设置代理模式 1, 该模式适用于: 你已经将代理提取至 Redis 的 proxy 里面
         # # 这样设置的话就会自动去取
-        # proxy={
-        #     "ref": "bricks.lib.proxies.RedisProxy",  # 指向 Redis
-        #     "key": "proxy",  # 指向代理 Key
-        #     # 这个不写默认指向本地 Redis, 无密码的
-        #     "options": {
-        #         "host": "127.0.0.1",
-        #         "port": 6379,
-        #         "password": "xsxsxax"
-        #     },
-        #     "threshold": 100,  # 一个代理最多使用多少次, 到这个次数之后就会归还到Redis, 然后重新拿, 默认不归还
-        #     "scheme": "socks5"  # 代理协议, 默认是 http
-        # }
-
-        # 设置代理模式 2, 该模式适用于: 指向固定代理, 如 http://127.0.0.1:7890
-        # 这样设置的话就会自动去取
         proxy={
-            "ref": "bricks.lib.proxies.CustomProxy",  # 指向 Redis
-            "key": "127.0.0.1:7890",  # 指向代理 Key
+            "ref": "bricks.lib.proxies.RedisProxy",  # 指向 Redis
+            "key": "proxy",  # 指向代理 Key
+            # 这个不写默认指向本地 Redis, 无密码的
+            "options": {
+                "host": "127.0.0.1",
+                "port": 6379,
+                # "password": "xsxsxax"
+            },
             "threshold": 100,  # 一个代理最多使用多少次, 到这个次数之后就会归还到Redis, 然后重新拿, 默认不归还
-            "scheme": "http"  # 代理协议, 默认是 http
-        },
+            "scheme": "socks5"  # 代理协议, 默认是 http
+        }
+
+        # # 设置代理模式 2, 该模式适用于: 指向固定代理, 如 http://127.0.0.1:7890
+        # # 这样设置的话就会自动去取
+        # proxy={
+        #     "ref": "bricks.lib.proxies.CustomProxy",  # 指向 Redis
+        #     "key": "127.0.0.1:7890",  # 指向代理 Key
+        #     "threshold": 100,  # 一个代理最多使用多少次, 到这个次数之后就会归还到Redis, 然后重新拿, 默认不归还
+        #     # "scheme": "http"  # 代理协议, 默认是 http
+        # },
 
         # # 设置代理模式 3, 该模式适用于: 你有一个提取 api,访问就会获取代理
         # # 这样设置的话就会自动去取