buildz 0.6.58__tar.gz → 0.6.59__tar.gz

{buildz-0.6.58/buildz.egg-info → buildz-0.6.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildz
-Version: 0.6.58
+Version: 0.6.59
 Summary: 配置读写（基于json格式进行简化）、ioc、以及其他工具代码。a json-base file format's read and write code by python, and codes to read and product object from configure file in such format(ioc), and other tool codes
 Home-page: https://github.com/buildCodeZ/buildz
 Author: Zzz

{buildz-0.6.58 → buildz-0.6.59}/buildz/__init__.py

@@ -1,6 +1,6 @@
 #coding=utf-8
 
-__version__="0.6.34"
+__version__="0.6.59"
 
 # 小号多
 __author__ = "Zzz, emails: 1174534295@qq.com, 1309458652@qq.com"

{buildz-0.6.58 → buildz-0.6.59}/buildz/netz/mhttp/__init__.py

@@ -3,3 +3,4 @@ from .caps import CapsDealer,CapsProxy
 from .proxy import ProxyDealer,Proxy
 from .mhttp import *
 from .record import MsgRecord,MsgLog
+from .gateway import GWDealer,Gateway

{buildz-0.6.58 → buildz-0.6.59}/buildz/netz/mhttp/caps.py

@@ -1,5 +1,5 @@
 
-from . import proxy,mhttp
+from . import proxy,mhttp,mhttps
 import socket, threading
 log = mhttp.log
 from buildz import xf
@@ -72,9 +72,7 @@ class CapsDealer(proxy.ProxyDealer):
     def context(self, hostname):
         if hostname not in self.contexts:
             fp_cert, fp_prv, pwd = self.sign(hostname)
-            context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-            context.load_cert_chain(fp_cert, fp_prv, password=pwd)
-            self.contexts[hostname]=context
+            self.contexts[hostname]=mhttps.load_server_context(fp_cert, fp_prv, pwd)
         return self.contexts[hostname]
     def deal_channel(self, skt_cli, skt_srv):
         #self.wskt.closefile()
@@ -92,7 +90,7 @@ class CapsDealer(proxy.ProxyDealer):
                 line, headers, data_size = mhttp.http_recv(skt_cli)
                 if line is None:
                     continue
-                self.default_deal(skt_cli, line, headers, data_size, skt_srv)
+                self.deals[None](skt_cli, line, headers, data_size, skt_srv)
         except Exception as exp:
             log.debug(f"channel exp: {exp}")
             log.warn(f"traceback: {traceback.format_exc()}")
@@ -110,29 +108,9 @@ class CapsProxy(proxy.Proxy):
         # context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
         # context.load_cert_chain(fp_sign, fp_prv, password=password)
         self.ca = fp_sign, fp_prv, password
-        if cafile is not None or capath is not None or cadata is not None:
-            # 导入根证书，需要单个根证书文件cafile或者根证书文件夹capath或者根证书数据cadata
-            # cadata是证书数据，不清楚能不能多个证书字节码拼在一起
-            srv_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-            srv_context.load_verify_locations(cafile=cafile, capath=capath, cadata=cadata)
-            srv_context.check_hostname = check_hostname
-        else:
-            # 不会校验服务端https证书是否有效，可能有风险？
-            srv_context = ssl._create_unverified_context(ssl.PROTOCOL_TLS_CLIENT)
-        #self.context = context
-        self.srv_context = srv_context
+        self.srv_context = mhttps.load_verify_context(cafile, capath, cadata,check_hostname)
         if dp_cache is None:
             dp_cache = "./res/certs"
         self.dp_cache = dp_cache
-    def call(self):
-        self.running=True
-        skt = socket.socket()
-        skt.bind(self.addr)
-        skt.listen(self.listen)
-        self.skt = skt
-        while self.running:
-            skt,addr = self.skt.accept()
-            deal = CapsDealer(skt, self.ca, self.srv_context, record=self.record.clone(), dp_cache = self.dp_cache)
-            th = threading.Thread(target=deal,daemon=True)
-            th.start()
-            self.ths.append(th)
+    def make_dealer(self, skt, addr):
+        return CapsDealer(skt, self.ca, self.srv_context, record=self.record.clone(), dp_cache = self.dp_cache)

buildz-0.6.59/buildz/netz/mhttp/gateway.py

@@ -0,0 +1,90 @@
+
+from . import proxy,mhttp, mhttps
+import socket, threading
+log = mhttp.log
+from buildz import xf
+import ssl,re
+from os.path import join, isfile
+import traceback
+from ...iocz.conf import conf as confz
+from ... import dz
+class GWDealer(proxy.ProxyDealer):
+    conf = confz.Conf()
+    conf.ikey(0, 'match', need=1)
+    conf.ikey(1, 'replace', need=1)
+    conf.ikey(2, 'ssl')
+    conf.ikey(3, 'type', 'match_type,http_type'.split(","))
+    """
+    rules:[
+        [match, replace, [opt|ssl], [opt|match_type]]
+        {match=match, replace=relace, ssl=false, match_type='GET'}
+    ]
+    """
+    def init(self, skt, rules, verify_context, context, channel_read_size=1024000, record=None):
+        self.context = context
+        if context is not None:
+            skt = context.wrap_socket(skt, server_side=True)
+        rules = [self.conf(k)[0] for k in rules]
+        self.rules = rules
+        self.verify_context = verify_context
+        super().init(skt, channel_read_size, record=record, default_deal = self.maps_deal)
+    def match(self, line, headers):
+        http_type, url, protocol = line
+        for item in self.rules:
+            pt, rep, mark_ssl, match_type = dz.g(item, match=None, replace=None, ssl=False, type=http_type)
+            if match_type != http_type:
+                continue
+            rst = re.findall(pt, url)
+            if len(rst)==0:
+                continue
+            match = rst[0]
+            i = url.find(match)
+            pfx, sfx = url[:i], url[i+len(match):]
+            rurl = pfx+rep+sfx
+            log.debug(f"url match: {rst} for {url}, replace to {rurl} with i={i}")
+            return rurl, mark_ssl
+        return None, False
+    def ssl_connect(self, addr):
+        return mhttps.wrap(self.verify_context, mhttp.WSocket.Connect(addr), False)
+    def maps_deal(self, skt_cli, line, headers, data_size, skt=None):
+        http_type, url, protocol = line
+        rel_url, mark_ssl = self.match(line, headers)
+        if rel_url is None:
+            if data_size>0:
+                data = skt_cli.read(data_size)
+            if mhttp.check_chunked(headers):
+                self.simple_chunked(skt_cli)
+            return self.error(skt_cli)
+        if mark_ssl:
+            prev = "https"
+        else:
+            prev = "http"
+        host = rel_url.split("?")[0].split("/")[0]
+        rel_url = prev+"://"+rel_url
+        for key in "Host,Origin".split(","):
+            if key in headers:
+                headers[key] = host
+        if 'Referer' in headers:
+            ref = headers['Referer']
+            prev, src_host, path = mhttp.spt_url(ref)
+            ref = mhttp.comb_url(prev, host, path)
+            headers["Referer"] = ref
+            log.info(f"[REF] host: {host}, ref: {ref}")
+        line = http_type, rel_url, protocol
+        fc_connect = None
+        if mark_ssl:
+            fc_connect = self.ssl_connect
+        return self.default_deal(skt_cli, line, headers, data_size, skt, fc_connect)
+
+
+class Gateway(proxy.Proxy):
+    '''
+        类似nginx的前向代理
+    '''
+    def init(self, addr, rules, fp_cert=None, fp_prv=None, password = None,listen=5, record=None, cafile=None, capath=None, cadata=None,check_hostname=True):
+        super().init(addr, listen, record)
+        self.rules = rules
+        self.verify_context = mhttps.load_verify_context(cafile, capath, cadata,check_hostname)
+        self.context = None if fp_cert is None else mhttps.load_server_context(fp_cert, fp_prv, password)
+    def make_dealer(self, skt, addr):
+        return GWDealer(skt, self.rules, self.verify_context, self.context, record=self.record.clone())

{buildz-0.6.58 → buildz-0.6.59}/buildz/netz/mhttp/mhttp.py

@@ -62,17 +62,21 @@ def http_decode(line):
     #log.debug(f"http_docde: {a,b,c}")
     return a,b,c.strip()
 pass
-def http_encode_send(http_type, url, headers=None, data_size=0, protocol = "HTTP/1.1"):
+def spt_url(url):
     index = url.find("://")
     prev = url[:index]
     url = url[index+3:]
-    index = url.find("/")
-    if index>=0:
-        addr = url[:index]
-        url = url[index:]
-    else:
-        addr = url
-        url = "/"
+    addr = url.split("?")[0].split("/")[0]
+    url = url[len(addr):]
+    if len(url)==0 or url[0]!="/":
+        url = "/"+url
+    return prev, addr, url
+
+pass
+def comb_url(prev, addr, url):
+    return prev+"://"+addr+url
+def http_encode_send(http_type, url, headers=None, data_size=0, protocol = "HTTP/1.1"):
+    prev, addr, url = spt_url(url)
     addrs = addr.split(":")
     ip = addrs[0]
     if prev.upper()=="HTTP":
@@ -171,20 +175,24 @@ class SendDone(Base):
         if not self.to_rel:
             return
         self.skt.close()
-def http_send_head(http_type, url, headers=None, data_size=0,protocol='HTTP/1.1', skt=None, caches=None):
+def http_send_head(http_type, url, headers=None, data_size=0,protocol='HTTP/1.1', skt=None, caches=None, fc_connect=None):
     if caches is None:
         caches = {}
     bts, addr,keep_alive = http_encode_send(http_type, url, headers, data_size,protocol=protocol)
+    pfx = spt_url(url)[0].lower()
+    if fc_connect is None:
+        fc_connect = WSocket.Connect
     #log.debug(f"request addr: {addr}")
     #log.debug(f"http_send: {len(bts)}")
     retry=0
     use_caches=skt is None and keep_alive
     need_rel = keep_alive
+    addr_key = tuple([pfx]+list(addr))
     if skt is None:
-        if not keep_alive or addr not in caches:
-            skt = WSocket.Connect(addr)
-        elif addr in caches:
-            skt = caches[addr]
+        if not keep_alive or addr_key not in caches:
+            skt = fc_connect(addr)
+        else:
+            skt = caches[addr_key]
             retry=1
     else:
         need_rel=False
@@ -193,12 +201,12 @@ def http_send_head(http_type, url, headers=None, data_size=0,protocol='HTTP/1.1'
         skt.send(bts)
     except:
         if retry:
-            skt=WSocket.Connect(addr)
+            skt=fc_connect(addr)
             skt.send(bts)
         else:
             raise
     if use_caches:
-        caches[addr]=skt
+        caches[addr_key]=skt
     return skt,fc_done
 
 class HttpMonitor(Base):

buildz-0.6.59/buildz/netz/mhttp/mhttps.py

@@ -0,0 +1,28 @@
+#
+
+import ssl
+from . import mhttp
+
+def load_verify_context(cafile=None, capath=None, cadata=None,check_hostname=True):
+    if cafile is not None or capath is not None or cadata is not None:
+        # 导入根证书，需要单个根证书文件cafile或者根证书文件夹capath或者根证书数据cadata
+        # cadata是证书数据，不清楚能不能多个证书字节码拼在一起
+        srv_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        srv_context.load_verify_locations(cafile=cafile, capath=capath, cadata=cadata)
+        srv_context.check_hostname = check_hostname
+    else:
+        # 不会校验服务端https证书是否有效，可能有风险？
+        srv_context = ssl._create_unverified_context(ssl.PROTOCOL_TLS_CLIENT)
+    return srv_context
+
+pass
+
+def load_server_context(fp_cert, fp_prv, pwd=None):
+    context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    context.load_cert_chain(fp_cert, fp_prv, password=pwd)
+    return context
+
+def wrap(context, wrap_cli, server_side, is_wrap = True):
+    if is_wrap:
+        wrap_cli = wrap_cli.skt
+    return mhttp.WSocket.Bind(context.wrap_socket(wrap_cli, server_side=server_side))

{buildz-0.6.58 → buildz-0.6.59}/buildz/netz/mhttp/proxy.py

@@ -8,29 +8,36 @@ class ProxyDealer(Base):
     """
         http和https代理
     """
-    def init(self, skt, channel_read_size=1024000, record=None, protocol = "HTTP/1.1"):
+    def init(self, skt, channel_read_size=1024000, record=None, protocol = "HTTP/1.1", default_deal=None):
         self.wskt = mhttp.WSocket.Bind(skt)
         self.channel_read_size = channel_read_size
         self.deals = {}
-        self.deals[None] = self.default_deal
+        if default_deal is None:
+            default_deal = self.default_deal
+        self.deals[None] = default_deal
         self.deals['CONNECT'] = self.connect_deal
         self.skts = {}
         self.protocol = protocol
         if record is None:
             record = _record.MsgLog(log)
         self.record = record
-    def simple_chunked(self, skt_read, skt_send):
+    def simple_chunked(self, skt_read, skt_send=None):
         while True:
             n, dt = mhttp.chunked_data(skt_read)
             self.record.add_chunk(dt)
             bts = mhttp.chunked_encode(dt)
-            skt_send.send(bts)
+            if skt_send is not None:
+                skt_send.send(bts)
             if n==0:
                 break
-    def default_deal(self, skt_cli, line, headers, data_size, skt=None):
+    def do_connect(self, addr):
+        return mhttp.WSocket.Connect(addr)
+    def default_deal(self, skt_cli, line, headers, data_size, skt=None, fc_connect=None):
+        if fc_connect is None:
+            fc_connect = self.do_connect
         self.record.request(line, headers, data_size)
         http_type, url, protocol = line
-        skt,fc_done = mhttp.http_send_head(http_type, url, headers, data_size,protocol, skt, self.skts)
+        skt,fc_done = mhttp.http_send_head(http_type, url, headers, data_size,protocol, skt, self.skts, fc_connect)
         if data_size>0:
             data = skt_cli.read(data_size)
             skt.send(data)
@@ -42,6 +49,7 @@ class ProxyDealer(Base):
         self.record.response(line, headers, data_size)
         protocol, code, rsp_text = line
         bts = mhttp.http_encode_rsp(code, rsp_text, headers, data_size, protocol)
+        print(f"[TESTZ] bts send: {bts}")
         skt_cli.send(bts)
         if data_size>0:
             data = skt.read(data_size)
@@ -58,6 +66,14 @@ class ProxyDealer(Base):
                 skt.close()
             except:
                 pass
+    def error(self, skt_cli, code=404, txt="Not Found", data=None):
+        if data is None:
+            data = txt
+        if type(data)==str:
+            data = data.encode("utf-8")
+        bts = mhttp.http_encode_rsp(code, txt, data_size=len(data), protocol = self.protocol)
+        skt_cli.send(bts+data)
+        log.debug(f"error: {code}, txt={txt}")
     def connect_deal(self, skt_cli, line, headers, data_size, skt=None):
         self.record.request(line, headers, data_size)
         if data_size>0:
@@ -78,14 +94,15 @@ class ProxyDealer(Base):
             except:
                 code,txt=404,"Not Found"
         self.record.response([self.protocol,code, txt],{},0).finish()
+        if code!=200:
+            return self.error(skt_cli, code, txt)
         bts = mhttp.http_encode_rsp(code, txt, protocol = self.protocol)
         skt_cli.send(bts)
-        if code==200:
-            try:
-                self.deal_channel(skt_cli, skt)
-            finally:
-                if need_close:
-                    skt.close()
+        try:
+            self.deal_channel(skt_cli, skt)
+        finally:
+            if need_close:
+                skt.close()
     def deal_channel(self, skt_cli, skt_srv):
         return self.direct_channel(skt_cli, skt_srv)
     def direct_channel(self, skt_cli, skt_srv):
@@ -147,6 +164,8 @@ class Proxy(Base):
         self.record = record
     def close(self):
         self.skt.close()
+    def make_dealer(self, skt, addr):
+        return ProxyDealer(skt, record=self.record.clone())
     def call(self):
         self.running=True
         skt = socket.socket()
@@ -155,7 +174,7 @@ class Proxy(Base):
         self.skt = skt
         while self.running:
             skt,addr = self.skt.accept()
-            deal = ProxyDealer(skt,record=self.record.clone())
+            deal = self.make_dealer(skt, addr)
             th = threading.Thread(target=deal,daemon=True)
             th.start()
             self.ths.append(th)

buildz-0.6.59/buildz/netz/test/test_gw.py

@@ -0,0 +1,33 @@
+
+
+from buildz import xf,fz,pathz,logz,pyz
+from buildz.netz import mhttp
+import threading
+confs = r"""
+[
+    {match=^/baidu, replace='www.baidu.com',ssl=true}
+]
+"""
+rules = xf.loads(confs)
+path = pathz.Path()
+path.set("res", "./res")
+log = logz.simple(path.res("gw/%Y%m%d.log"))
+def gw():
+    import sys,time
+    ip = '127.0.0.1'
+    port = 9999
+    args = sys.argv[1:]
+    if len(args)>0:
+        ip = args.pop(0)
+    if len(args)>0:
+        port = int(args.pop(0))
+    px = mhttp.Gateway((ip, port), rules, record=mhttp.MsgLog(log))
+    th = threading.Thread(target=px,daemon=True)
+    th.start()
+    print(f"start on {(ip, port)}")
+    while px.running:
+        time.sleep(1)
+
+#pyz.lc(locals(), caps)
+pyz.lc(locals(), gw)
+

{buildz-0.6.58 → buildz-0.6.59/buildz.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buildz
-Version: 0.6.58
+Version: 0.6.59
 Summary: 配置读写（基于json格式进行简化）、ioc、以及其他工具代码。a json-base file format's read and write code by python, and codes to read and product object from configure file in such format(ioc), and other tool codes
 Home-page: https://github.com/buildCodeZ/buildz
 Author: Zzz

{buildz-0.6.58 → buildz-0.6.59}/buildz.egg-info/SOURCES.txt

@@ -219,7 +219,9 @@ buildz/iocz/wrap/wraps.py
 buildz/netz/__init__.py
 buildz/netz/mhttp/__init__.py
 buildz/netz/mhttp/caps.py
+buildz/netz/mhttp/gateway.py
 buildz/netz/mhttp/mhttp.py
+buildz/netz/mhttp/mhttps.py
 buildz/netz/mhttp/proxy.py
 buildz/netz/mhttp/record.py
 buildz/netz/sslz/__init__.py
@@ -228,6 +230,7 @@ buildz/netz/sslz/gen.pyi
 buildz/netz/test/__main__.py
 buildz/netz/test/test.py
 buildz/netz/test/test_cli.py
+buildz/netz/test/test_gw.py
 buildz/tz/__init__.py
 buildz/tz/myers_diff.py
 buildz/tz/test_xfind.py

{buildz-0.6.58 → buildz-0.6.59}/setup.py

@@ -7,7 +7,7 @@ from setuptools import setup, find_packages
 
 setup(
     name = 'buildz',
-    version = '0.6.58',
+    version = '0.6.59',
     keywords='buildz',
     long_description=open('README.md', 'r', encoding="utf-8").read(),
     long_description_content_type='text/markdown',