jtt-utils 1.0.0__tar.gz

jtt-utils-1.0.0/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.1
+Name: jtt-utils
+Version: 1.0.0
+Summary: jtt common tools
+Home-page: UNKNOWN
+Author: candyabc
+Author-email: hfcandyabc@163.com
+License: maxwin
+Platform: any
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+
+jtt common tools
+

jtt-utils-1.0.0/README.md

@@ -0,0 +1,72 @@
+## D:\work\mwwork\JTT\jtt-tm-util
+#### jtt 常用工具
+* 
+
+### 设计
+
+
+### 设定
+*
+
+###产生分发包
+``
+python setup.py sdist
+twine upload dist/*
+``
+### 运行
+* python setup.py sdist build
+* python setup.py install
+
+
+### 修改记录
+* 2020/03/23
+```text
+  v 0.0.4
+  修改consul reader 增加read_service 的key参数,向上兼容 
+
+```
+* 2020/05/27
+```text
+  v 0.0.5
+  修改consul reader 增加service_as_url向上兼容 
+
+```
+
+* 2020/06/11
+```text
+  v 0.0.6
+  
+  修改sync_basedata 增加read employee
+```
+
+* 2020/06/17
+```text
+  v 0.0.7
+  
+  修改sync_basedata 的bug
+```
+
+* 2023/03/16
+```text
+v 0.1.0
+
+fix update line_config bug。
+```
+* 2023/03/17
+```text
+v 0.1.1
+
+kong register add host param
+```
+* 2023/04/20
+```text
+v 0.1.2
+sync basedata 同步支持 key_type='set' 方式
+```
+
+* 2023/04/20
+```text
+v 0.1.3
+fix : consul read datebase driver 的错误
+add read_dbdriver func
+```

jtt-utils-1.0.0/jtt_tm_utils/__init__.py

@@ -0,0 +1,12 @@
+__version__ ='1.0.0'
+__author__ = 'candyabc'
+__copyright__ = 'candyabc'
+__email__ = 'hfcandyabc@163.com'
+__license__ = 'maxwin'
+
+
+from .sync_basedata import data_manager
+
+from .consul_handle import consul_reader
+from .kong import Kong
+# __all__=("JttTaskManager","JttTask")

jtt-utils-1.0.0/jtt_tm_utils/consul_handle.py

@@ -0,0 +1,245 @@
+from consul import Consul
+
+import sys
+
+import collections
+
+from .log import logger
+
+CService =collections.namedtuple('CService','ip port agent_ip serviceid')
+CRedisService = collections.namedtuple('CRedisService','ip port db')
+CDatabaseService = collections.namedtuple('CDatabaseService','dbdriver ip port dbname user pwd')
+# CMongoService = collections.namedtuple('CMongoService','host port username password authSource')
+def cservice_to_address(service):
+    return '%s:%s' % (service.ip,service.port)
+
+class ConsulAgent:
+    def __init__(self,node_name,ip,services=None):
+        self.node_name = node_name
+        self.ip = ip
+        self._services = services
+
+    def combine_address(self,port):
+        return '%s:%s' %(self.ip,port)
+
+    def combine_service_id(self,name,port):
+        return '%s:%s:%s' %(self.node_name,name,port)
+
+class ConsulReader():
+    def __init__(self):
+        self._consul =None
+        self._agent =None
+
+    @property
+    def consul(self):
+        if self._consul is None:
+            self._consul =Consul()
+        return self._consul
+
+    @property
+    def agent(self):
+        if self._agent is None:
+            try:
+                agent_self = self.consul.agent.self()
+                self._agent = ConsulAgent(agent_self['Member'].get('Name'),
+                                          agent_self['Member'].get('Addr'),
+                                          self.consul.catalog.services()[1])
+
+            except Exception as e:
+                logger.error('读取consul 服务失败,error:%s' % (e))
+
+                sys.exit(-1)
+
+        return self._agent
+
+    def read_key(self,key,raise_error=False):
+        v =self.consul.kv.get(key)[1]
+        if v is None:
+            if raise_error:
+                raise Exception('consul: not has key %s' % key)
+            else:
+                sys.exit('consul: not has key %s' % key)
+        return v['Value'].decode()
+
+    def read_key_default(self,key,default):
+        try:
+            return self.read_key(key,True)
+        except:
+            return default
+
+    def put_key(self,key,value,**args):
+        self.consul.kv.put(key,value,**args)
+
+    def read_services(self,service_name,tag,raise_error=False,key =None,**args):
+        try:
+            srvs = self.consul.catalog.service(service_name, tag=tag,**args)[1]
+
+            if key is not None:
+                port = self.read_key('%s_%s_%s' %(service_name,tag,key),raise_error=raise_error)
+            else:
+                port =None
+            re =[]
+            for srv in srvs or []:
+                re.append(CService(ip=srv['ServiceAddress'],
+                         port=port or srv['ServicePort'],
+                         agent_ip=srv['Address'],
+                                   serviceid = srv['ServiceID']))
+            return re
+        except Exception as e:
+            logger.error('consul:%s,%s error:%s' % (service_name,tag,str(e)))
+            self._consul =None
+            if raise_error:
+                raise Exception('consul:%s,%s, error:%s' % (service_name,tag,str(e)))
+            else:
+                sys.exit('consul:%s,%s, error:%s' % (service_name,tag,str(e)))
+
+    def read_service(self,service_name,tag,raise_error=False,key =None,**args):
+        srvs = self.read_services(service_name,tag,raise_error,key,**args)
+        if len(srvs) <= 0:
+            logger.error('consul:%s,%s error:config is not exists' % (service_name,tag))
+            if raise_error:
+                raise Exception('consul:%s,%s error:config is not exists' % (service_name,tag))
+            else:
+                sys.exit('consul:%s,%s error:config is not exists' % (service_name,tag))
+        return srvs[0]
+
+    def read_service_as_address(self,service_name,tag,raise_error=False,key =None,**args):
+        srv = self.read_service(service_name,tag,raise_error,key,**args)
+        return cservice_to_address(srv)
+
+    def read_redis(self,tag,as_slave=False):
+        try:
+            if as_slave:
+                agent_self = self.consul.agent.self()
+                member = agent_self['Member']
+                node_name = member.get('NodeName')
+                srv = self.read_service('redis', tag, raise_error=True, near=node_name)
+            else:
+                srv = self.read_service('redis',tag,raise_error=True)
+
+            try:
+                db = self.read_key('redis_%s_db' % tag)
+            except:
+                db =0
+
+            return CRedisService(srv.ip,srv.port,db)
+
+        except Exception as e:
+            logger.error('consul:redis error:%s' % str(e))
+            sys.exit('consul:redis error:%s' % str(e))
+
+    def redis_as_url(self,tag,**args):
+        credis = self.read_redis(tag,**args)
+        return 'redis://%s:%s/%s' % (credis.ip,credis.port,credis.db)
+
+    def service_as_url(self,service_name,tag,raise_error=False,key =None,protocol ='http',**args):
+        service = self.read_service(service_name,tag,raise_error,key,**args)
+        return '%s://%s' % (protocol,cservice_to_address(service))
+
+    def kafka_servers(self,tag=None):
+        srvs =self.read_services('kafka',tag=tag)
+        if len(srvs)<=0:
+            logger.error('consul not set kafka')
+            sys.exit('consul not set kafka')
+        addresses = [cservice_to_address(srv) for srv in srvs]
+        return ','.join(addresses)
+
+    def read_dbdriver(self,dbname,**args):
+        return self.read_key('%s_dbdriver' % dbname,**args)
+    
+    def read_database(self,dbname,dbapi =None):
+        dbsrv =self.read_service('database',dbname)
+        driver_key = self.read_key('%s_dbdriver' % dbname)
+        if dbapi ==None:
+            if driver_key =='mssql':
+                dbapi ='pymssql'
+            elif driver_key =='mysql':
+                dbapi ='pymysql'
+            else:
+                raise Exception(f'not set dbapi driver for {driver_key}')
+            
+
+        # if dbapi is None:
+        #     db_driver = 'mssql+pymssql' if driver_key=='mssql' else 'mysql+mysqldb'
+        # else:
+        db_driver = '%s+%s' %(driver_key,dbapi)
+        
+        return CDatabaseService(db_driver,
+                                dbsrv.ip,
+                                dbsrv.port,
+                                self.read_key('%s_db_name'% dbname),
+                                self.read_key('%s_login_user' % dbname),
+                                self.read_key('%s_login_pw' % dbname))
+
+    #dbdriver ip port dbname user pwd
+    def read_database_keys(self,dbname):
+        return {'dbdriver' : self.read_key('%s_dbdriver' % dbname),
+                'dbname':self.read_key('%s_db_name' % dbname),
+                'user': self.read_key('%s_login_user' % dbname),
+                'pwd':  self.read_key('%s_login_pw' % dbname)}
+
+    def database_as_url(self,dbname,**args):
+        dbsrv = self.read_database(dbname,**args)
+        return "{dbdriver}://{us}:{pw}@{host}:{port}/{name}".format(
+                 dbdriver =dbsrv.dbdriver,us=dbsrv.user,pw=dbsrv.pwd,
+                 host=dbsrv.ip,port=dbsrv.port,name=dbsrv.dbname)
+
+    def database_as_json(self,dbname):
+        dbsrv = self.read_database(dbname)
+        return {'host':dbsrv.ip,
+                       'port':dbsrv.port,
+                       'db':dbsrv.dbname,
+                       'user':dbsrv.user,
+                       'password':dbsrv.pwd}
+
+    def read_mongo(self,dbname):
+        dbsrv = self.read_service('mongo', dbname)
+        return {'host':dbsrv.ip,
+                'port':dbsrv.port,
+                'username':self.read_key('mongo_%s_user' % dbname),
+                'password':self.read_key('mongo_%s_pw' % dbname),
+                'authSource': dbname}
+
+    def mongo_as_url(self,dbname):
+        dbconfig = self.read_mongo(dbname)
+        return 'mongodb://{username}:{password}@{host}:{port}/{authSource}'.format(**dbconfig)
+
+    def register_service(self,name,port,host=None,tags =None,check=None,token=None,service_id=None):
+        if host is None:
+            host = self.agent.ip
+        try:
+            if service_id is None:
+                service_id = self.agent.combine_service_id( name, port )
+
+            ret= self.consul.agent.service.register(name, service_id=service_id, address=host,
+                                        port=port, tags=tags, check=check, token=token)
+            if ret:
+                logger.info('註冊%s(%s) 到consul成功！' % (name, service_id))
+            else:
+                logger.error('注册 %s(%s) 服务失败' % (name, service_id))
+        except Exception as e:
+            logger.error('注册 %s(%s) 服务失败,error:%s' % (name,service_id,str(e)))
+
+            sys.exit('注册 %s(%s)服务失败,error:%s' % (name,service_id, e))
+
+    def deregister_service(self,name,port,service_id=None):
+        try:
+            if service_id is None:
+                service_id=self.agent.combine_service_id( name, port )
+
+            ret= self.consul.agent.service.deregister(service_id)
+
+            if ret:
+                logger.info('删除%s(%s)成功！' % (name, service_id))
+            else:
+                logger.error('删除%s(%s)服务失败！' % (name, service_id))
+        except Exception as e:
+            logger.error('删除 %s(%s)服务失败,error:%s' % (name,service_id, str(e)))
+
+            sys.exit('删除 %s(%s)服务失败,error:%s' % (name,service_id, e))
+
+    def catalog_register(self):
+        self.consul.catalog.register()
+consul_reader = ConsulReader()
+
+

jtt-utils-1.0.0/jtt_tm_utils/kong.py

@@ -0,0 +1,177 @@
+from .consul_handle import consul_reader
+from .log import logger
+import requests
+import sys
+
+
+class Kong:
+    def __init__(self):
+
+        self.admin_uri = consul_reader.service_as_url('kong',tag='admin')
+        srv = consul_reader.read_service('kong',tag='kong')
+        self.ip = srv.ip
+        self.port = srv.port
+
+    def register_upstream_service(self,upstream_name,port, base_path,auth=None,host =None,**args):
+        if host is None:
+            address = consul_reader.agent.combine_address(port)
+        else:
+            address ='%s:%s' %(host,port)
+        self.register_upstream(upstream_name,address,**args)
+        self.register_service(upstream_name,base_path,auth)
+
+    def register_service(self,upstream_name,base_path, auth=None):
+        '''
+        :param kong_admin: 用于注册服务
+        :param service_host: 本地服务的host，用于设定upstream_url
+        :param service: 服务api，如：rightmanage/v1.0
+        :param kong_uris: 空的host uris，为空时值为service,如：rightmanage/v1.0
+        :param auth: 认证类型，可为str或list，如：'jwt'，或：['jwt','key']
+        :param health: 如果service_host 是upstreams，则可以注册health
+        :return:
+        '''
+        ''''
+        # 注册 服务API
+        resp = requests.post('http://{kong}/apis/'.format(kong=KONG),
+                             json={'name': 'rightmanage_v1.0',
+                                   'uris': '/rightmanage/v1.0',
+                                   'upstream_url': 'http://{server}/rightmanage/v1.0'.format(server=SERVER)})
+        print('注册 服务API', resp.text)
+
+        resp = requests.post('http://{kong}/apis/rightmanage_v1.0/plugins'.format(kong=KONG),
+                             json={"name": "jwt"})
+        print('注册服务API JWT', resp.text)
+        '''
+        api_url = '{kong}/apis/'.format(kong=self.admin_uri)
+        # 如果kong_uris有值，则需要用它注册
+        api_name = base_path.replace('/','_').replace('.','_').strip('_')
+
+
+        upstream_url = 'http://{upstream_name}{base_path}'.format(upstream_name=upstream_name,
+                                                                   base_path=base_path)
+        try:
+            # 删除旧的api
+            resp = requests.delete('{kong}/apis/{api_name}'.format(kong=self.admin_uri,
+                                                            api_name=api_name))
+            logger.info('del api(%s) %s,%s '% (api_name,resp.status_code,resp.text))
+            resp = requests.post(api_url,
+                                 json={'name': api_name,
+                                       'uris': base_path,
+                                       'upstream_url': upstream_url,
+                                       'preserve_host': True})
+            logger.info('注册%s服务 to kong 成功,%s '% (base_path, resp.text))
+        except Exception as e:
+            logger.error('注册 %s服务失败,error:%s' % (base_path, e))
+
+            sys.exit(-1)
+        if auth:
+            if isinstance(auth, str):
+                auth =[auth]
+
+            if 'jwt' in auth:
+                auth_data =  {"name": "jwt",
+                             "config.uri_param_names": "jwt",
+                             "config.claims_to_verify": "exp",
+                             "config.key_claim_name": "iss",
+                             "config.secret_is_base64": "false",
+                             "config.cookie_names": "sessionid"
+                             }
+
+            elif 'key' in auth:
+                auth_data =  {"name": "key-auth",
+                             "config.key_names": "apikey"
+                             }
+
+
+            else:
+                raise Exception('不支持的auth(%s)' % auth)
+
+            resp = requests.post('{kong}/apis/{api_name}/plugins'.format(kong=self.admin_uri,
+                                                                         api_name=api_name),
+                                 json=auth_data)
+            if resp.status_code != 201:
+                # 在0.12.1之前的kong 不支持config.cookie_names
+                if 'jwt' in auth:
+                    auth_data.pop('config.cookie_names')
+                    resp = requests.post('{kong}/apis/{api_name}/plugins'.format(kong=self.admin_uri,
+                                                                                 api_name=api_name),
+                                         json=auth_data)
+            if resp.status_code == 201:
+                logger.info('注册%s服务API %s 成功,%s' % (base_path,auth, resp.text))
+            else:
+                logger.error('注册%s服务API %s 失败,%s,%s' % (base_path,auth, resp.status_code, resp.text))
+
+    def register_upstream(self,upstream_name,service_address,health =None,weight=100):
+        '''
+
+        :param upstream_name: monitor-srv
+        :param service_host: ip:port
+        :param weight: 100
+        :param health: 如： ordermng/v1.0/health, realtime-srv/v1.0/health
+        :return:
+        '''
+        ''''
+        # create an upstream
+        $ curl -X POST http://kong:8001/upstreams \
+            --data "name=address.v1.service"
+
+        # add two targets to the upstream
+        $ curl -X POST http://kong:8001/upstreams/address.v1.service/targets \
+            --data "target=192.168.101.75:80"
+            --data "weight=100"
+        $ curl -X POST http://kong:8001/upstreams/address.v1.service/targets \
+            --data "target=192.168.100.76:80"
+            --data "weight=50"
+
+        # create a Service targeting the Blue upstream
+        $ curl -X POST http://kong:8001/apis/ \
+            --data "name=service-api" \
+            --data "uris=/aa"
+            --data "upstream_url=http://address.v1.service"
+        '''
+        upstream_api_url = '{kong}/upstreams/'.format(kong=self.admin_uri)
+        # add upstream
+        try:
+            '''
+            healthchecks.active.http_path - 在向目标发出HTTP GET请求时应该使用的路径。默认值是“/”。
+            healthchecks.active.timeout - 用于探测的HTTP GET请求的连接超时限制。默认值是1秒。
+            healthchecks.active.concurrency - 在主动健康检查中并发检查的目标数量。
+            你还需要为运行的探针指定间隔的值：
+
+            healthchecks.active.healthy.interval - 健康目标的主动健康检查间隔时间（以秒为单位）。0的值表明不执行对健康目标的主动探测。
+            healthchecks.active.unhealthy.interval - 对不健康目标的主动健康检查间隔时间（以秒为单位）。0值表示不应该执行不健康目标的主动探测。
+            这允许您调整主动健康检查的行为，无论您是否希望探测健康和不健康的目标在相同的时间间隔内运行，或者一个比另一个更频繁。
+
+            最后，您需要配置Kong应该如何解释探头，通过设置健康计数器上的各种阈值，一旦到达，就会触发状态变化。计数器阈值字段是：
+
+            healthchecks.active.healthy.successes - 在主动探测中成功的数量（由healthchecks.active.healthy.http_statuses定义）来确认目标的健康
+            healthchecks.active.unhealthy.tcp_failures - 在主动探测中TCP故障的数量，以确认目标是不健康的。
+            healthchecks.active.unhealthy.timeouts - 在主动探测中超时的数量，以确认目标是不健康的。
+            healthchecks.active.unhealthy.http_failures - 在主动探测中出现的HTTP故障数量（由healthchecks.active.healthy.http_statuses定义）来确认目标是不健康的。
+           '''
+            upstream = {'name': upstream_name,
+                        'healthchecks.active.http_path': health or '/',
+                        'healthchecks.active.healthy.interval': 0 if health is None else 5,
+                        'healthchecks.active.healthy.successes': 1,
+                        'healthchecks.active.unhealthy.interval': 5,
+                        'healthchecks.active.unhealthy.tcp_failures': 1,
+                        'healthchecks.active.unhealthy.timeouts': 3,
+                        'healthchecks.active.unhealthy.http_failures': 1
+                        }
+            resp = requests.post(upstream_api_url, json=upstream)
+            if resp.status_code == 409:
+                resp = requests.patch(upstream_api_url + upstream_name, json=upstream)
+            logger.info('add %s upstream to kong 成功,%s,%s' % (upstream_name,resp.status_code,resp.text))
+        except Exception as e:
+            logger.error('add %s upstream失败,error:%s' % (upstream_name, e))
+            sys.exit(-1)
+        targets_api_url = '{kong}/upstreams/{upstream_name}/targets'.format(kong=self.admin_uri,
+                                                                            upstream_name=upstream_name)
+        try:
+            resp = requests.post(targets_api_url,
+                                 json={'target': service_address, 'weight': weight})
+            logger.info('add %s target to kong 成功,%s,%s' % (upstream_name,resp.status_code,resp.text))
+        except Exception as e:
+            logger.error('add target for %s失败,error:%s' % (upstream_name, e))
+
+            sys.exit(-1)

jtt-utils-1.0.0/jtt_tm_utils/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger('jtt_util')

jtt-utils-1.0.0/jtt_tm_utils/sync_basedata.py

@@ -0,0 +1,266 @@
+import asyncio
+import json
+from datetime import datetime
+from collections import defaultdict
+from .log import logger
+
+
+DEFAULT_NAMES=[('vehicle','carid,dvrid1,fleetid,vtid'),
+               'accredituser']
+
+UPDATE_ASSOCIATIONS={'vehicle':['vehicle.','device.']}
+
+NullObj ={}
+
+def data_isnull(data):
+    return data ==NullObj
+
+async def get_last_modifytime(rds_r):
+    # 有可能新旧同步程序同时运行，需取最新变更值
+    async def get_last_time(rds_key):
+        try:
+            last_mdy_time =await rds_r.hget(rds_key, 'lastmodifytime')
+        except Exception as e:
+            logger.error('redis error:%s'%str(e))
+            last_mdy_time = None
+        last_mdy_time = last_mdy_time.decode()\
+            if last_mdy_time is not None else '20010101010101010101'
+        return datetime.strptime(last_mdy_time, '%Y%m%d%H%M%S%f')
+
+    return await get_last_time('sync_record')
+
+async def read_data_as_dict(rds,rds_key,fields=None):
+    if fields is None:
+        data = await rds.hgetall(rds_key)
+        if data:
+            return {k.decode(): v.decode() for k, v in data.items()}
+    else:
+        data = await rds.hmget(rds_key, *fields)
+        if data:
+            obj = {k: v.decode() for (k, v) in zip(fields, data) if v is not None}
+            return obj
+    return NullObj
+
+class SyncBasedataManage():
+    def __init__(self):
+        self._tasks =[]
+        self._redis =None
+        self.mappings ={}
+        self.names =[]
+        self.cache =defaultdict(dict)
+        # self.sync_task =None
+
+        # self.clear_all()
+
+    def update_associations(self,key):
+        re =[key]
+        conditions = UPDATE_ASSOCIATIONS.get(key)
+        if conditions:
+            for condition in conditions:
+                re+=list(filter(lambda name:name.startswith(condition),self.names))
+
+        return re
+
+    def config(self,redis,names=DEFAULT_NAMES,loop=None):
+        def make_func(name):
+            def _method(self,key):
+                return self.get_bdobj(name,key )
+
+            return _method
+
+        self._redis =redis
+        for cname in names:
+            if type(cname)==tuple:
+                name, fields = cname
+                self.mappings[name]=fields.split(',')
+
+            else:
+                name =cname
+            self.names.append(name)
+            _method_name = 'get_%s' % name.replace('.', '_')
+            if not hasattr(self.__class__,_method_name):
+                _method = make_func(name)
+                setattr(self.__class__, _method_name, _method)
+        # self.names =names
+        if loop is not None:
+            self._tasks.append(loop.create_task(self.sync_data()))
+
+    @property
+    def rds(self):
+        assert self._redis is not None, '%s :please config redis first.' % self.__class__.__name__
+        return self._redis
+
+    def clear_all(self):
+        self.cache.clear()
+
+
+    async def sync_data(self):
+        last_modifytime =await get_last_modifytime(self.rds)
+        logger.info('sync_data get last modifytime:%s' % last_modifytime)
+        # last_modifytime =last_modifytime+timedelta(microseconds=-1)
+        # 避免频繁重启时，频繁访问redis
+        await asyncio.sleep(5)
+
+        while True:
+            try:
+                logger.debug('SyncBasedataManage.sync_data')
+                last_modifytime_local =await get_last_modifytime(self.rds)
+                if last_modifytime_local > last_modifytime:
+                    updatedatas = await self.rds.zrangebyscore('bd_updatekeys',int(last_modifytime.timestamp()), float('inf'))
+                    for ud in updatedatas:
+                        ud_key = ud.decode().split('_{')[0] ##可能有'bdupd.line_config.%s_{"time": %s的情况，导致split到错误位置
+                        if ud_key.startswith('bdupd.'):
+                            [name,key] = ud_key.split('.')[-2:]
+                            if name =='basedata' and key =='recreateall':
+                                logger.info('ready recreatecall')
+                                handled = self.custom_reload()
+                                if not handled:
+                                    self.clear_all()
+
+                            else:
+                                if name =='line_config': # line.config的 update name 是line_config 所以要调整一下
+                                    name = 'line.config'
+                                elif name =='vehiclegroup_vehicles':
+                                    name ='vehiclegroup.vehicles'
+                                logger.info(f'notify update {name} {key}')
+                                bd_names = self.update_associations(name)
+                                for bd_name in bd_names:
+                                    handled =self.custom_reload_item(bd_name,key)
+                                    if not handled :
+                                        self.remove_bdobj(bd_name,key)
+
+                    last_modifytime = last_modifytime_local
+            except Exception as e:
+                logger.error('sync_basedata error:%s' % str(e))
+            finally:
+                await asyncio.sleep(20)
+    def custom_reload(self):
+        return False
+
+    def custom_reload_item(self,name,key):
+        return False
+
+    async def get_bdobj(self,name,key,key_type='hash'):
+        assert name in self.names,'not set config %s' % name
+        key = str(key)
+        obj =(self.cache.get(name) or {}).get(key)
+        # obj = self.cache[name].get(key)
+        if obj is None:
+            logger.info(f"get bdobj from redis {name},{key}")
+            rds_key = 'bd:%s:%s' % (name, key)
+            fields =self.mappings.get(name)
+            obj =None
+            if key_type =='hash':
+                obj = await read_data_as_dict(self.rds,rds_key,fields)
+            elif key_type =='string':
+                data = await self.rds.get(rds_key)
+                if data is not None:
+                    try:
+                        obj = json.loads(data.decode())
+                    except Exception as e:
+                        logger.error('key:%s load error:%s' %(rds_key,str(e)))
+            elif key_type =='set':
+                data = await self.rds.smembers(rds_key)
+                if data is not None:
+                    try:
+                        obj =[item.decode() for item in data]
+                    except Exception as e:
+                        logger.error('key:%s load error:%s' % (rds_key, str(e)))
+            else:
+                logger.error('key:%s not support key_type:%s' %(rds_key,key_type))
+            if data_isnull(obj):
+                logger.warning('the %s(%s) is not exist!' % (name, key))
+                obj =None
+            # if self.cache.get(name) is None:
+            #     self.cache[name] = {}
+            self.add_bdobj(name,key,obj)
+            return obj
+        return obj
+
+
+    def add_bdobj(self,name,key,obj):
+        self.cache[name][key] = obj
+
+    def remove_bdobj(self,name,key):
+        if self.cache.get(name):
+            try:
+                logger.info(f'remove {name} {key}')
+                self.cache[name].pop(key)
+            except:
+                pass
+
+
+    async def event_config(self,carid,evcode):
+        config = await self.get_vehicle_config(carid)
+        if config is not None:
+            data = config.get('event_params_%s' % evcode)
+            if data:
+                return json.loads(data)
+        return NullObj
+
+    async def get_e_stop(self,imsi):
+        return await self.get_bdobj('estop',imsi,'string')
+
+    async def get_vehicletype(self,id):
+        return await self.get_bdobj('vehicletype',id,'string')
+
+class MapSyncManager(SyncBasedataManage):
+    def __init__(self,has_map_dvrid=False):
+        super().__init__()
+        self.has_map_dvrid =has_map_dvrid
+        self.dvrid_carid_map ={}
+        # self.has_dvrid = has_map_dvrid
+
+
+    def add_bdobj(self,name,key,obj):
+        super().add_bdobj(name,key,obj)
+        if self.has_map_dvrid and name =='vehicle' and obj is not None and obj.get('dvrid1') is not None:
+            self.dvrid_carid_map[obj['dvrid1']]=key
+
+    def remove_bdobj(self,name,key):
+        if self.has_map_dvrid and name == 'vehicle':
+            obj = self.cache[name].get(key)
+            if obj is not None and obj.get('dvrid1') is not None:
+                try:
+                    self.dvrid_carid_map.pop(obj['dvrid1'])
+                except:
+                    pass
+
+        super().remove_bdobj(name, key)
+
+    def clear_all(self):
+        super().clear_all()
+        self.dvrid_carid_map.clear()
+
+    def custom_reload(self):
+
+        vt_keys = self.cache['vehicle'].keys()
+
+        self.clear_all()
+
+        for key in vt_keys:
+            asyncio.ensure_future(self.get_bdobj('vehicle',key))
+
+        return True
+
+    def custom_reload_item(self,name,key):
+        # logger.info(f'custom_reload_item{name},{key}')
+        self.remove_bdobj(name,key)
+        if name =='vehicle' and self.has_map_dvrid:
+            asyncio.ensure_future(self.get_bdobj(name,key))
+        return True
+
+
+# managers={'default':SyncBasedataManage(),
+#           'map_dvrid1':MapSyncManager(True)}
+# def get_manager():
+#     import os
+#     mode = os.getenv('SYNC_MANAGER','default')
+#     if mode =='default':
+#         return SyncBasedataManage()
+#     elif mode =='map_dvrid1':
+#         return MapSyncManager(True)
+
+data_manager=MapSyncManager()
+
+

jtt-utils-1.0.0/jtt_tm_utils/timeutil.py

@@ -0,0 +1,23 @@
+from datetime import datetime
+def linux_timestamp():
+    return int(datetime.now().timestamp())
+
+def timestamp_ms():
+    return datetime.now().timestamp()*1000
+
+def time_to_microsec_str(d=None):
+    if d is None:
+        d = datetime.now()
+    return d.strftime('%Y%m%d%H%M%S%f')
+
+def time_to_sql_str(d=None):
+    if d is None:
+        d = datetime.now()
+
+    return d.strftime('%Y-%m-%d %H:%M:%S')
+
+def date_to_sql_str(d=None):
+    if d is None:
+        d = datetime.now()
+
+    return d.strftime('%Y-%m-%d')

jtt-utils-1.0.0/jtt_utils.egg-info/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 2.1
+Name: jtt-utils
+Version: 1.0.0
+Summary: jtt common tools
+Home-page: UNKNOWN
+Author: candyabc
+Author-email: hfcandyabc@163.com
+License: maxwin
+Platform: any
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+
+jtt common tools
+

jtt-utils-1.0.0/jtt_utils.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+setup.cfg
+setup.py
+jtt_tm_utils/__init__.py
+jtt_tm_utils/consul_handle.py
+jtt_tm_utils/kong.py
+jtt_tm_utils/log.py
+jtt_tm_utils/sync_basedata.py
+jtt_tm_utils/timeutil.py
+jtt_utils.egg-info/PKG-INFO
+jtt_utils.egg-info/SOURCES.txt
+jtt_utils.egg-info/dependency_links.txt
+jtt_utils.egg-info/not-zip-safe
+jtt_utils.egg-info/top_level.txt

jtt-utils-1.0.0/jtt_utils.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

jtt-utils-1.0.0/jtt_utils.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

jtt-utils-1.0.0/jtt_utils.egg-info/top_level.txt

@@ -0,0 +1 @@
+jtt_tm_utils

jtt-utils-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

jtt-utils-1.0.0/setup.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import re
+import os
+
+
+from setuptools import setup, find_packages
+
+
+def fpath(name):
+    return os.path.join(os.path.dirname(__file__), name)
+
+
+def read(fname):
+    return open(fpath(fname)).read()
+
+
+# def desc():
+#     info = read('README.rst')
+#     try:
+#         return info + '\n\n' + read('doc/changelog.rst')
+#     except IOError:
+#         return info
+
+# grep flask_admin/__init__.py since python 3.x cannot import it before using 2to3
+file_text = read(fpath('jtt_tm_utils/__init__.py'))
+
+
+def grep(attrname):
+    pattern = r"{0}\W*=\W*'([^']+)'".format(attrname)
+    strval, = re.findall(pattern, file_text)
+    return strval
+
+
+# entry_points = """
+# [console_scripts]
+# pycli = mw_uml_generator.cli:cli
+# """
+install_requires = [
+]
+
+setup(
+    name='jtt-utils',
+    version=grep('__version__'),
+    url='',
+    license=grep('__license__'),
+    author=grep('__author__'),
+    author_email=grep('__email__'),
+    description='jtt common tools',
+    long_description='jtt common tools',
+    packages=find_packages(),
+    # entry_points=entry_points,
+    include_package_data=True,
+    zip_safe=False,
+    platforms='any',
+    install_requires=install_requires,
+
+    tests_require=[
+    ],
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Environment :: Web Environment',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: BSD License',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+    ]
+)
+
+