buildz 0.6.12__tar.gz → 0.6.14__tar.gz

{buildz-0.6.12/buildz.egg-info → buildz-0.6.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildz
-Version: 0.6.12
+Version: 0.6.14
 Summary: 配置读写（基于json格式进行简化）、ioc、以及其他工具代码。a json-base file format's read and write code by python, and codes to read and product object from configure file in such format(ioc), and other tool codes
 Home-page: https://github.com/buildCodeZ/buildz
 Author: Zzz
@@ -31,6 +31,8 @@ License-File: LICENSE
 运行python -m buildz查看帮助
 
 持续更新中。。。
+2024/09/25:
+修复bug，增强html的搜索功能，增强auto配置功能
 2024/09/24:
 增加模块html:html页面解析
 增加模块auto:自动化调用

{buildz-0.6.12 → buildz-0.6.14}/README.md

@@ -18,6 +18,8 @@
 运行python -m buildz查看帮助
 
 持续更新中。。。
+2024/09/25:
+修复bug，增强html的搜索功能，增强auto配置功能
 2024/09/24:
 增加模块html:html页面解析
 增加模块auto:自动化调用

{buildz-0.6.12 → buildz-0.6.14}/buildz/auto/cache.py

@@ -6,7 +6,7 @@ from ..base import Base
 from ..ioc import wrap
 import os,re
 
-@wrap.obj(id="cache_modify")
+@wrap.obj(id="cache.modify")
 @wrap.obj_args("ref, cache", "ref, log")
 class Update(Base):
     """
@@ -38,7 +38,10 @@ class Update(Base):
                 err = f"'{key}' not found in cache"
                 self.log.error(err)
                 raise Exception(err)
-            s = s.replace(match, val)
+            if s == match:
+                s = val
+            else:
+                s = s.replace(match, str(val))
         return s
 
 pass
@@ -53,8 +56,8 @@ class Save(Base):
         self.log = log
     def call(self, maps, fp):
         fp = xf.get(maps, "cache.save", None)
-        self.log.debug(f"try save cache to '{fp}'")
         if fp is None:
+            self.log.warn(f"cache not save cause 'cache.save' is None")
             return
         fz.makefdir(fp)
         rst  = self.cache.data
@@ -81,7 +84,7 @@ class Cache(Base):
         data = {}
         if os.path.isfile(fp):
             self.log.info(f"load cache from {fp}")
-            data = xf.loadf(fp)
+            data = xf.flush_maps(xf.loadf(fp),visit_list=False)
         self.data.update(data)
         return True
 

buildz-0.6.14/buildz/auto/config.py

@@ -0,0 +1,34 @@
+#
+from .. import xf
+from .. import ioc
+from ..base import Base
+from ..ioc import wrap
+import os
+@wrap.obj(id="buildz.auto.config.load")
+class Config(Base):
+    def call(self, maps, fp):
+        configs = xf.g(maps, configs=[])
+        if type(configs)!=list:
+            configs = [configs]
+        for cfp in configs:
+            if not os.path.isfile(cfp):
+                self.log.error(f"config file not exist: {cfp}")
+                return False
+            dt = xf.loadf(cfp)
+            xf.fill(dt, maps, replace=0)
+        return True
+
+pass
+#wrap.add_datas("[(env, env.buildz.auto.deal), buildz.auto.deal, auto.deal]")
+@wrap.obj_args("[env, buildz.auto.deal, auto.deal]")
+@wrap.obj(id="autoz.deal")
+class DfDeal(Base):
+    def init(self, id):
+        self.id = id
+    def call(self, data):
+        print(f"[ERROR] implement obj with id '{self.id}' by yourself")
+
+pass
+
+
+

{buildz-0.6.12 → buildz-0.6.14}/buildz/auto/deal_type.py

@@ -5,9 +5,10 @@ from ..base import Base
 from ..ioc import wrap
 class DealType(Base):
     def init(self, deals = {}):
-        print(f"deals: {deals}")
         self.deals = deals
     def call(self, data):
+        if type(data)==str:
+            data = xf.loadf(data)
         _type = xf.g(data, type = None)
         return self.deals[_type](data)
 

{buildz-0.6.12 → buildz-0.6.14}/buildz/auto/init.py

@@ -9,4 +9,4 @@
 
 
 
-from . import deal, deal_type, factory, cache, verify, log, save
+from . import deal, deal_type, factory, cache, verify, log, save, config,request

{buildz-0.6.12 → buildz-0.6.14}/buildz/auto/log.py

@@ -23,6 +23,8 @@ class Log(Base):
             fz.write(msg.encode("utf-8"), fp, 'ab')
     def info(self, *args):
         self.log("INFO", *args)
+    def warn(self, *args):
+        self.log("WARN", *args)
     def debug(self, *args):
         self.log("DEBUG", *args)
     def error(self, *args):

buildz-0.6.14/buildz/auto/request.py

@@ -0,0 +1,131 @@
+#
+
+from ..tools import *
+from buildz.ioc import wrap
+import json
+
+try:
+    import requests as rq
+    session = rq.session()
+except:
+    rq = None
+    session = None
+
+pass
+class Request(Base):
+    def init(self, http_type, use_session, log, upd):
+        self.type = http_type
+        self.use_session = use_session
+        self.log = log
+        self.upd = upd
+        self.fc = None
+        self.fcs = None
+        self.key_data = None
+        self.build()
+    def build(self):
+        fcs = {}
+        global rq, session
+        if rq is None:
+            return
+        obj = rq
+        if self.use_session:
+            obj = session
+        fcs['get'] = [obj.get, "params"]
+        fcs['post'] = [obj.post, "data"]
+        fcs['json'] = [obj.post, "json"]
+        fcs['put'] = [obj.put, "json"]
+        fcs['delete'] = [obj.delete, 'json']
+        self.fcs = fcs
+        self.fc = self.fcs[self.type][0]
+        self.key_data = self.fcs[self.type][1]
+    def get_set(self, data, kd, maps, km):
+        v = xf.get(data, kd)
+        if v is not None:
+            maps[km] = v
+    def req(self, data):
+        url = xf.g(data, url=None)
+        maps = {}
+        self.get_set(data, "data", maps, self.key_data)
+        self.get_set(data, "cookies", maps, "cookies")
+        self.get_set(data, "headers", maps, "headers")
+        self.get_set(data, "proxies", maps, "proxies")
+        return self.fc(url, **maps)
+    def rsp(self, rp, data):
+        url = xf.g(data, url=None)
+        xf.s(data, status_code = rp.status_code)
+        xf.s(data, result_code = rp.status_code)
+        try:
+            xf.s(data, result_content=rp.content)
+            s = xf.decode(rp.content, "utf-8")
+            xf.s(data, result_text=s)
+        except Exception as exp:
+            self.log.warn(f"exp in deal response on '{url}': {exp}")
+        try:
+            obj = json.loads(s)
+            xf.s(data, result=obj)
+        except Exception as exp:
+            #self.log.warn(f"exp in deal response on '{url}': {exp}")
+            pass
+        try:
+            xf.s(data, result_cookies=dict(rp.cookies))
+        except Exception as exp:
+            self.log.warn(f"exp in deal response on '{url}': {exp}")
+        try:
+            xf.s(data, result_headers = dict(rp.headers))
+        except Exception as exp:
+            self.log.warn(f"exp in deal response on '{url}': {exp}")
+        return True
+    def call(self, data, fc=None):
+        if self.fc is None:
+            self.log.error("install requests to use this(pip install requests)")
+            return False
+        if self.upd is not None:
+            data = self.upd(data)
+        url = xf.g(data, url=None)
+        try:
+            rp = self.req(data)
+        except Exception as exp:
+            self.log.error("error in request '{url}' with method {self.type}: {exp}")
+            return False
+        self.rsp(rp, data)
+        if fc is None:
+            return True
+        return fc(data)
+
+pass
+
+
+@wrap.obj(id="request.post")
+@wrap.obj_args("post", "env, request.session, false", "ref, log", "ref, cache.modify")
+class Post(Request):
+    pass
+
+pass
+
+@wrap.obj(id="request.json")
+@wrap.obj_args("json", "env, request.session, false", "ref, log", "ref, cache.modify")
+class Json(Request):
+    pass
+
+pass
+
+@wrap.obj(id="request.get")
+@wrap.obj_args("get", "env, request.session, false", "ref, log", "ref, cache.modify")
+class Get(Request):
+    pass
+
+pass
+
+@wrap.obj(id="request.put")
+@wrap.obj_args("put", "env, request.session, false", "ref, log", "ref, cache.modify")
+class Put(Request):
+    pass
+
+pass
+
+@wrap.obj(id="request.delete")
+@wrap.obj_args("delete", "env, request.session, false", "ref, log", "ref, cache.modify")
+class Delete(Request):
+    pass
+
+pass

{buildz-0.6.12 → buildz-0.6.14}/buildz/auto/run.py

@@ -2,6 +2,7 @@ from .. import xf
 from .. import ioc
 from ..base import Base
 from . import init
+import os
 class Run(Base):
     def init(self, fps=None):
         mg = ioc.build()
@@ -10,6 +11,9 @@ class Run(Base):
         self.mg = mg
     def call(self, fp):
         maps = xf.loadf(fp)
+        config = self.mg.get("buildz.auto.config.load")
+        if not config(maps, fp):
+            return
         calls = xf.g(maps, calls = [])
         for deal in calls:
             fc = self.mg.get(deal)

buildz-0.6.14/buildz/auto/save.py

@@ -0,0 +1,33 @@
+#
+from ..tools import *
+from buildz.ioc import wrap
+@wrap.obj(id="save")
+@wrap.obj_args("ref, cache", "ref, log", "ref, cache.modify")
+class Save(Base):
+    def init(self, cache, log, upd):
+        self.cache = cache
+        self.log = log
+        self.upd = upd
+    def call(self, data, fc=None):
+        data = self.upd(data)
+        save = xf.g(data, save={})
+        for k, v in save.items():
+            if type(v)==str:
+                v = ["key"]+v.split(".")
+            if type(v)!=list:
+                v = ["key", v]
+            tp = v[0]
+            v = v[1:]
+            if tp=="eval":
+                val = eval(v[0])
+            elif tp == "exec":
+                exec(v[0])
+                val = self.val
+            else:
+                val = xf.gets(data, v)
+            self.cache.set(k, val)
+        return True
+
+pass
+
+

buildz-0.6.14/buildz/auto/test/cache.js

@@ -0,0 +1,16 @@
+{
+    host: www.baidu.com,
+    test: {
+        url: tmpurl,
+        val: 1,
+        code: 1.99
+    },
+    result: {
+        code: 1.99
+    },
+    test2: {
+        headers: {
+            content_type: "text/html"
+        }
+    }
+}

buildz-0.6.14/buildz/auto/test/config.js

@@ -0,0 +1,10 @@
+calls: [log, cache, def.deal.type, list, cache.save]
+cache: cache.js
+cache.save: cache.js
+log: %Y%m%d_log.txt
+def.deal: {
+    types: {
+        http.get: [request, verify, save]
+        get: [request.get, verify, save]
+    }
+}

buildz-0.6.14/buildz/auto/test/item1.js

@@ -0,0 +1,16 @@
+note: test1
+url: test
+type: http.get
+data: {
+    url: "#{test.url}"
+    val: "#{test.val}"
+}
+save: {
+    test.url: data.url
+}
+result: {
+    code: 1.99
+}
+verify: [
+    "result.code, [>, 1]"
+]

buildz-0.6.14/buildz/auto/test/item2.js

@@ -0,0 +1,15 @@
+note: test2
+url: "https://#{host}/"
+type: get
+headers: {
+    User-Agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+}
+data: {
+}
+verify: [
+    "status_code, 200"
+    "result_headers.Content-Type, text/html"
+]
+save: {
+    test2.headers.content_type: result_headers.Content-Type
+}

buildz-0.6.14/buildz/auto/test/test.js

@@ -0,0 +1,27 @@
+configs: config.js
+datas: [
+    //写一个配置
+    {
+        note: test
+        url: test
+        type: http.get
+        data: {
+            url: "#{test.url}"
+        }
+        save: {
+            test.url: data.url
+        }
+        result: {
+            code: 1.99
+        }
+        verify: [
+            "result.code, [>, 1]"
+        ]
+        save: {
+            result.code: result.code
+        }
+    }
+    // 或者配置所在的文件
+    item1.js
+    item2.js
+]

{buildz-0.6.12 → buildz-0.6.14}/buildz/auto/test/test.py

@@ -2,7 +2,7 @@
 from buildz.tools import *
 from buildz.ioc import wrap
 @wrap.obj(id="request")
-@wrap.obj_args("ref, log", "ref, cache_modify")
+@wrap.obj_args("ref, log", "ref, cache.modify")
 class Req(Base):
     def init(self, log, upd):
         self.upd = upd

buildz-0.6.14/buildz/auto/verify.py

@@ -0,0 +1,78 @@
+#
+
+from ..tools import *
+from buildz.ioc import wrap
+@wrap.obj(id="verify")
+@wrap.obj_args("ref, log", "ref, cache.modify")
+class Verify(Base):
+    def init(self, log, upd):
+        self.log = log
+        self.upd = upd
+        self.opts = {}
+        self.opts[">"] = self.make_val_cmp(lambda val,v:val>v)
+        self.opts["<"] = self.make_val_cmp(lambda val,v:val<v)
+        self.opts[">="] = self.make_val_cmp(lambda val,v:val>=v)
+        self.opts["<="] = self.make_val_cmp(lambda val,v:val<=v)
+        self.opts["="] = self.eq
+        self.opts["!="] = self.neq
+    def eq(self, val, v):
+        if type(val)!=type(v):
+            if type(val)==str or type(v)==str:
+                return str(val)==str(v)
+        return val==v
+    def neq(self, val, v):
+        if type(val)!=type(v):
+            if type(val)==str or type(v)==str:
+                return str(val)!=str(v)
+        return val!=v
+    def make_val_cmp(self, opt):
+        def fc(val, v):
+            if val is None:
+                return False
+            if type(val)==str:
+                val = float(val)
+            if type(v)==str:
+                v = float(v)
+            return opt(val, v)
+        return fc
+    def match(self, v, val, result, data):
+        tp,v = v
+        if tp in self.opts:
+            return self.opts[tp](val,v)
+        if tp=="eval":
+            return eval(v)
+        elif tp == "exec":
+            exec(v)
+            return self.val
+        else:
+            err = f"not impl match type: {tp}"
+            self.log.error(err)
+            raise Exception(err)
+    def call(self, data, fc=None):
+        data = self.upd(data)
+        note = xf.g(data, note=data)
+        result = xf.g(data, result = {})
+        vs = xf.g(data, verify=[])
+        for it in vs:
+            if type(it)==str:
+                it = xf.loads(it)
+            k,v=it
+            bak = v
+            if k == "$":
+                if type(v)!=list:
+                    v = ["eval", v]
+                val = None
+            else:
+                val = xf.gets(data, k.split("."))
+            if type(v)!=list:
+                v = ["=", v]
+            jg = self.match(v, val, result, data)
+            if not jg:
+                self.log.error(f"verify failed in {note}, key: '{k}', match: {bak}, val: {val}")
+                return False
+        if fc is None:
+            return True
+        return fc(data)
+
+pass
+            

buildz-0.6.14/buildz/html/test/test.py

@@ -0,0 +1,15 @@
+
+from buildz.html import parse
+from buildz.tools import *
+#demo.html太大，已经删除
+fp = "demo.html"
+s = fz.read(fp, "rb").decode("utf-8")
+
+obj = parse(s)
+
+# 可以进行多次查询调用，后一次查询在前一次的结果集里做查询
+rst = obj.finds("(class,toplist1-tr_1MWDu)").finds("tag=a,target=_blank").data()
+rst = obj.searchs("class,toplist1-tr_1MWDu").finds("(tag,a),(tag,a),(target,_blank)").data()
+rst = obj.searchs("class,toplist1-tr_1MWDu").searchs("tag,a", "tag, a", "target, _blank").data()
+
+print(f"search rst: {rst}")

{buildz-0.6.12 → buildz-0.6.14}/buildz/html/xml.py

@@ -1,16 +1,47 @@
 from html.parser import HTMLParser
 from .. import xf
+from ..base import Base
 import re
+class SearchResult(Base):
+    def str(self):
+        return str(self.arr)
+    def init(self, arr):
+        self.arr = arr
+        self.size = len(arr)
+    def run(self, fc_name, *a, **b):
+        rst = []
+        for it in self.arr:
+            fc = getattr(it, fc_name)
+            rst+=fc(*a,**b).data()
+        return SearchResult(rst)
+    def searchs(self, *args, **maps):
+        return self.run("searchs", *args, **maps)
+    def finds(self, s):
+        return self.run("finds", s)
+    def tags(self, _tag):
+        return self.searchs(tag=_tag)
+    def call(self):
+        return self.data()
+    def data(self):
+        return self.arr
+
+pass
 class HtmlTag:
+    def data(self):
+        return self.to_maps()
     def to_maps(self):
         nodes = [n.to_maps() for n in self.nodes]
-        rst = {'tag': self.tag, 'attrs': self.attrs, 'texts': self.texts, 'text': self.text, 'childs': nodes}
+        rst = {'tag': self.tag, 'attrs': self.attrs, 'texts': self.texts, 'text': self.text, 'nodes': nodes}
         return rst
     def __str__(self):
         return xf.dumps(self.to_maps())
     def __repr__(self):
         return self.__str__()
     def match(self, val, pt):
+        tag = self.tag
+        text = self.text
+        texts = self.texts
+        attrs = self.attrs
         if type(pt)==list:
             tp = pt[0]
             v = pt[1]
@@ -28,20 +59,24 @@ class HtmlTag:
         elif tp=="<=":
             return val<=v
         elif tp == "re":
-            return (re.findall(v, val))>0
-        elif tp == 'sh':
-            #v.replace("${it}", val)
+            return len(re.findall(v, val))>0
+        elif tp == 'eval':
             return eval(v)
         else:
             raise Exception(f"not impl match type: '{tp}'")
-    def check(self, maps):
+    def check(self, args, maps):
         #print(f"[TESTZ] check: {maps}, attrs: {self.attrs}")
-        for k,v in maps.items():
+        arr = args+list(maps.items())
+        for k,v in arr:
             if k == 'tag':
                 if not self.match(self.tag, v):
                     return False
+            elif k == 'text':
+                if not self.match(self.text, v):
+                    return False
             elif k == "$":
-                return self.match(None, ["sh", v])
+                if not self.match(None, ["eval", v]):
+                    return False
             else:
                 if k not in self.attrs or not self.match(self.attrs[k], v):
                     return False
@@ -54,22 +89,35 @@ class HtmlTag:
         self.nodes = []
         self.text = None
         self.texts = []
-    def _search(self, rst, maps):
-        if self.check(maps):
+    def _search(self, rst, args, maps):
+        if self.check(args, maps):
             rst.append(self)
         for n in self.nodes:
-            n._search(rst, maps)
-    def searchs(self, **maps):
+            n._search(rst, args, maps)
+    def searchs(self, *args, **maps):
+        tmp = []
+        for k in args:
+            if type(k)!=list:
+                k = xf.loads(k)
+            tmp.append(k)
         rst = []
-        self._search(rst, maps)
-        return rst
+        self._search(rst, tmp, maps)
+        return SearchResult(rst)
     def tags(self, _tag):
-        return self.searchs(tag = _tag)
+        return SearchResult(self.searchs(tag = _tag))
     def finds(self, s):
         rst = []
-        maps = xf.loads(s)
-        self._search(rst, maps)
-        return rst
+        args = []
+        maps = {}
+        dt = xf.loads(s)
+        if type(dt)==list:
+            args = dt
+        else:
+            maps = dt
+        if len(args)==2 and type(args[0])!=list:
+            args = [args]
+        self._search(rst, args, maps)
+        return SearchResult(rst)
     def add_node(self, node):
         self.nodes.append(node)
     def add_text(self, text):

{buildz-0.6.12 → buildz-0.6.14}/buildz/xf/mapz.py

@@ -126,8 +126,11 @@ def sets(maps, keys, val):
     if type(keys) != list:
         keys = [keys]
     for key in keys[:-1]:
-        if key not in maps:
-            maps[key] = {}
+        if type(maps)==dict:
+            if key not in maps:
+                maps[key] = {}
+        else:
+            key = int(key)
         maps = maps[key]
     maps[keys[-1]] = val
 
@@ -136,8 +139,13 @@ def gets(maps, keys, default = None):
     if type(keys) != list:
         keys = [keys]
     for key in keys:
-        if key not in maps:
-            return default
+        if type(maps)==dict:
+            if key not in maps:
+                return default
+        else:
+            key = int(key)
+            if key>=len(maps):
+                return default
         maps = maps[key]
     return maps
 
@@ -203,4 +211,21 @@ fill=deep_fill
 def maps(**kv):
     return kv
 
-pass
+pass
+
+def flush_maps(maps, fc_key = lambda x:x.split(".") if type(x)==str else [x], visit_list=False):
+    if type(maps)==list:
+        if not visit_list:
+            return maps
+        rst = [flush_maps(it, fc_key, visit_list) for it in maps]
+        return rst
+    if type(maps)!=dict:
+        return maps
+    rst = {}
+    for k,v in maps.items():
+        ks = fc_key(k)
+        v = flush_maps(v, fc_key, visit_list)
+        sets(rst, ks, v)
+    return rst
+
+pass