fast-web-core 1.0.0__tar.gz

fast-web-core-1.0.0/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.1
+Name: fast-web-core
+Version: 1.0.0
+Summary: fast web core for zsodata
+Home-page: http://www.zsodata.com
+Author: zsodata
+Author-email: team@zso.io
+License: BSD License
+Platform: all
+Requires-Python: >=3.7
+
+pip uninstall fast-web-core
+
+
+注意：
+mongo依赖版本：pymongo==4.7.3 and mongo version 7.0
+如果需要使用旧版，切换到分支：mongo4
+
+重点新增：
+异步mongo，异步redis
+
+pip install --upgrade fast-web-core -i https://pypi.python.org/simple
+pip show fast-web-core

fast-web-core-1.0.0/README.rst

@@ -0,0 +1,12 @@
+pip uninstall fast-web-core
+
+
+注意：
+mongo依赖版本：pymongo==4.7.3 and mongo version 7.0
+如果需要使用旧版，切换到分支：mongo4
+
+重点新增：
+异步mongo，异步redis
+
+pip install --upgrade fast-web-core -i https://pypi.python.org/simple
+pip show fast-web-core

fast-web-core-1.0.0/fast_web_core/__init__.py

@@ -0,0 +1,9 @@
+from __future__ import absolute_import
+
+
+def init_core_modules(*args, **kwargs) -> None:
+    """
+    Init core handlers, middlewares and services for fastapi application
+    :return: None
+    """
+    pass

fast-web-core-1.0.0/fast_web_core/auth/auth_cache_pool.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+import json
+import logging
+from typing import List, Set, Optional, Dict
+from ..lib import cfg
+from ..client.redis_client import Redis
+from ..model.items import AuthUser, AuthApp
+
+logger = logging.getLogger(__name__)
+rds = Redis(redis_uri=cfg.get('AUTH_REDIS_URL', None), redis_db=cfg.get('AUTH_REDIS_DB', None)).client
+default_expire = 43200
+
+
+def save_cached_app(access_token: str, app: AuthApp, expire: int = None):
+    if app:
+        _expire = expire or default_expire
+        rds.set(f'share:access:{access_token}', app.json(), ex=_expire)
+
+
+def get_cached_user(access_token: str) -> Optional[AuthUser]:
+    rs = rds.get(f'share:access:{access_token}')
+    if rs:
+        js_user = json.loads(rs)
+        # 容错
+        if isinstance(js_user, str):
+            js_user = json.loads(js_user)
+        # 特征验证（user对象需要有id字段）
+        if js_user and js_user.get('id', None):
+            user = AuthUser(**js_user)
+            user.userName = js_user.get('username', '')
+            return user
+
+    return None
+
+
+def get_cached_app(access_token: str) -> Optional[AuthApp]:
+    rs = rds.get(f'share:access:{access_token}')
+    if rs:
+        js_app = json.loads(rs)
+        # 容错
+        if isinstance(js_app, str):
+            js_app = json.loads(js_app)
+        # 特征验证（APP对象需要有appId字段）
+        if js_app and js_app.get('appId', None):
+            app = AuthApp(**js_app)
+            return app
+
+    return None
+
+
+def get_cached_access_dict(access_token: str) -> Optional[Dict]:
+    rs = rds.get(f'share:access:{access_token}')
+    if rs:
+        item = json.loads(rs)
+        # 容错
+        if isinstance(item, str):
+            item = json.loads(item)
+        return item
+
+    return None
+
+
+def has_access(access_token: str) -> bool:
+    return rds.exists(f'share:access:{access_token}')
+
+
+def save_cached_permission(access_token: str, permissions: Set[str], expire: int = None):
+    if permissions:
+        key = f'share:permissions:{access_token}'
+        for per in permissions:
+            rds.sadd(key, per)
+        _expire = expire or default_expire
+        rds.expire(key, _expire)
+
+
+def has_permission(access_token: str, permission: str) -> bool:
+    # 后者兼容Java版本异常
+    _key = f'share:permissions:{access_token}'
+    return rds.sismember(_key, f'{permission}') or rds.sismember(_key, f'"{permission}"')
+
+
+def get_cached_permission(access_token: str) -> List[str]:
+    return rds.smembers(f'share:permissions:{access_token}')
+
+
+def clean_cache(access_token: str):
+    rds.delete(f'share:access:{access_token}')
+    rds.delete(f'share:permissions:{access_token}')

fast-web-core-1.0.0/fast_web_core/auth/share_auth.py

@@ -0,0 +1,223 @@
+# -*- coding: utf-8 -*-
+import logging
+import re
+import json
+import threading
+from typing import Optional, List
+from starlette.routing import BaseRoute
+from fastapi.routing import APIRoute
+from starlette.requests import Request
+
+from ..lib import cfg
+from ..auth import auth_cache_pool
+from ..client.redis_client import Redis
+from ..exception.exceptions import NoAuthException
+from ..model.items import AuthUser, AuthApp
+from ..context.context_vars import tenant_context
+
+logger = logging.getLogger(__name__)
+
+FIX_WHITELIST = [
+]
+
+
+class ShareAuth(object):
+    _instance_lock = threading.Lock()
+    _biz_inited = False
+    _routes_inited = False
+
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(ShareAuth, "_instance"):
+            with ShareAuth._instance_lock:
+                if not hasattr(ShareAuth, "_instance"):
+                    ShareAuth._instance = object.__new__(cls)
+        return ShareAuth._instance
+
+    def __init__(self):
+        if not self._biz_inited:
+            self._biz_inited = True
+            self.rds = Redis(redis_uri=cfg.get('AUTH_REDIS_URL', None), redis_db=cfg.get('AUTH_REDIS_DB', None)).client
+            # route path regex -> tags
+            self.regex_to_tags_map = dict()
+            # no auth path whitelist
+            self.auth_whitelist = []
+
+    def get_access_token(self, request: Request):
+        """
+        解析请求头鉴权信息
+        :param request:
+        :return:
+        """
+        if request and (request.headers or request.query_params):
+            access_token = request.headers.get('Authorization', '').replace('Bearer ', '')
+            if not access_token:
+                access_token = request.cookies.get('access_token')
+            if not access_token:
+                access_token = request.query_params.get('access_token')
+            return access_token
+
+        return ''
+
+    def reload(self, routes: Optional[List[BaseRoute]], white_list=[], force: bool = False):
+        if not force and self._routes_inited:
+            return self
+        # 构造路由匹配正则与权限颗粒(tags)映射
+        _tmp_regex_to_tags_map = dict()
+        for route in routes:
+            if isinstance(route, APIRoute) and route.path_regex:
+                _tmp_regex_to_tags_map[route.path_regex] = route.tags
+        self.regex_to_tags_map.clear()
+        self.regex_to_tags_map = _tmp_regex_to_tags_map
+        logger.info(f'reload path to tags map: {len(self.regex_to_tags_map)}')
+
+        # 构造权限白名单
+        _auth_whitelist = list()
+        for row in FIX_WHITELIST:
+            _auth_whitelist.append(re.compile(row))
+        for row in white_list:
+            _auth_whitelist.append(re.compile(row))
+        self.auth_whitelist.clear()
+        self.auth_whitelist = _auth_whitelist
+        logger.info(f'reload no auth whitelist: {len(self.auth_whitelist)}')
+        #
+        self._routes_inited = True
+
+        return self
+
+    def access_check(self, request: Request):
+        """
+        登录校验
+        :param request:
+        :return:
+        """
+        access_token = self.get_access_token(request)
+        if not access_token:
+            raise NoAuthException('请先登录')
+
+        if not auth_cache_pool.has_access(access_token):
+            raise NoAuthException('登录信息已失效')
+
+        return access_token
+
+    def auth_check(self, request: Request):
+        """
+        权限校验
+        :param request:
+        :return:
+        """
+        if not self.regex_to_tags_map:
+            logger.debug(f'no path to tags map')
+            return False
+        # 优先处理白名单
+        for white_regex in self.auth_whitelist:
+            if white_regex.match(request.url.path):
+                return True
+        # 先校验登录
+        access_token = self.access_check(request)
+        for regex in self.regex_to_tags_map.keys():
+            if regex.match(request.url.path):
+                # 匹配到路由
+                tags = self.regex_to_tags_map.get(regex)
+                if tags:
+                    # 需要登录且需要鉴权
+                    auth_rs = self._rds_permissions_check(tags, access_token)
+                    if not auth_rs:
+                        raise NoAuthException('权限不足')
+                    return True
+                else:
+                    # 仅登录无需鉴权
+                    return True
+
+        return False
+
+    def _rds_permissions_check(self, tags, access_token) -> bool:
+        # redis权限查询
+        for tag in tags:
+            # 新权限颗粒 xx.xx.xx.xx
+            has = auth_cache_pool.has_permission(access_token, tag)
+            if not has:
+                # 兼容Java版权限颗粒 "xx.xx.xx.xx"
+                has = auth_cache_pool.has_permission(access_token, json.dumps(tag))
+                if not has:
+                    return False
+
+        return True
+
+    def get_auth_user(self, request: Request) -> Optional[AuthUser]:
+        """
+        获取当前登录的用户信息
+        :param request:
+        :return:
+        """
+        access_token = self.get_access_token(request)
+        if not access_token:
+            raise NoAuthException('请先登录')
+        if not auth_cache_pool.has_access(access_token):
+            raise NoAuthException('登录信息已失效')
+
+        auth_user = auth_cache_pool.get_cached_user(access_token)
+        # 仅超级管理员可以切换租户
+        if auth_user and (auth_user.superAdmin == 1 or auth_user.superAdmin == '1'):
+            tenant_code = request.headers.get('TenantCode', None)
+            if tenant_code:
+                # 切换租户
+                auth_user.tenantCode = tenant_code
+
+        # 设置租户上下文
+        if auth_user and auth_user.tenantCode:
+            tenant_context.set(auth_user.tenantCode)
+
+        return auth_user
+
+    def get_auth_app(self, request: Request) -> Optional[AuthApp]:
+        """
+        获取当前登录的应用信息
+        """
+        access_token = self.get_access_token(request)
+        if not access_token:
+            raise NoAuthException('请先鉴权认证')
+        if not auth_cache_pool.has_access(access_token):
+            raise NoAuthException('鉴权认证信息已失效')
+
+        auth_app = auth_cache_pool.get_cached_app(access_token=access_token)
+        # 设置租户上下文
+        if auth_app and auth_app.tenantCode:
+            tenant_context.set(auth_app.tenantCode)
+
+        return auth_app
+
+    def get_auth_tenant_code(self, request: Request) -> Optional[str]:
+        """
+        获取授权用户所属租户
+        :param request:
+        :return:
+        """
+        access_token = self.get_access_token(request)
+        if access_token:
+            access_item = auth_cache_pool.get_cached_access_dict(access_token)
+            if access_item:
+                # 仅超级管理员可以切换租户
+                super_admin = access_item.get('superAdmin', None)
+                if super_admin == 1 or super_admin == '1':
+                    tenant_code = request.headers.get('TenantCode', None)
+                    if tenant_code:
+                        return tenant_code
+
+                return access_item.get('tenantCode', '')
+
+        return ''
+
+
+# 用于handler来Depends，获取授权用户
+async def authed_user(request: Request) -> Optional[AuthUser]:
+    return ShareAuth().get_auth_user(request)
+
+
+# 用于handler来Depends，获取授权APP信息
+async def authed_app(request: Request) -> Optional[AuthApp]:
+    return ShareAuth().get_auth_app(request)
+
+
+# 用于handler来Depends，获取授权用户
+async def access_token(request: Request) -> Optional[str]:
+    return ShareAuth().get_access_token(request)

fast-web-core-1.0.0/fast_web_core/client/async_mongo_client.py

@@ -0,0 +1,233 @@
+# encoding: utf-8
+# version: pymongo==4.7.3, motor==3.5.0
+
+from ..exception.exceptions import BizException
+from ..exception.exceptions import NoConfigException
+from ..lib import cfg, logger
+
+LOGGER = logger.get('AsyncMongoClient')
+
+
+class AsyncMongo(object):
+    def __init__(self, mongo_uri: str = None, mongo_db: str = None):
+        import motor.motor_asyncio
+        self.mongo_uri = mongo_uri or cfg.get('MONGO_URL')
+        self.mongo_db = mongo_db or cfg.get('MONGO_DB')
+        if not self.mongo_uri:
+            raise NoConfigException('mongodb uri not config!')
+        if not self.mongo_db:
+            raise NoConfigException('mongodb database not config!')
+
+        self.client = motor.motor_asyncio.AsyncIOMotorClient(self.mongo_uri)
+        self.db = self.client[self.mongo_db]
+
+        LOGGER.info(self.client)
+
+    # 获取集合
+    def get_collection(self, coll):
+        # 非异步
+        return self.db.get_collection(coll)
+
+    # 查询对象
+    async def get(self, collection, query={}):
+        return await self.db[collection].find_one(query)
+
+    # 统计数量
+    async def count(self, collection, query={}):
+        return await self.db[collection].count_documents(query)
+
+    # 查询列表
+    async def list(self, collection, query={}, fields=None, sort=[], batch_size: int = 2000):
+        result = []
+        cursor = self.db[collection].find(filter=query, projection=fields, sort=sort)
+        cursor.batch_size(batch_size)
+        async for document in cursor:
+            result.append(document)
+        await cursor.close()
+
+        return result
+
+    async def list_with_cursor(self, collection, query={}, fields=None, sort=[], batch_size: int = 2000):
+        """
+        样例:
+            cursor = await aio_mongo.list_with_cursor(collection)
+            async for document in cursor:
+                print("document =", document)
+            await cursor.close()
+        注意：应该在使用完毕后主动关闭 cursor
+        """
+        cursor = self.db[collection].find(filter=query, projection=fields, sort=sort)
+        cursor.batch_size(batch_size)
+
+        return cursor
+
+    # 分页查询
+    async def page(self, collection, query={}, page_no=1, page_size=20, fields=None, sort=[], batch_size: int = 2000):
+        total = await self.db[collection].count_documents(query) or 0
+        cursor = self.db[collection].find(query, fields, sort=sort).skip(page_size * (page_no - 1)).limit(page_size)
+        cursor.batch_size(batch_size)
+        rows = []
+        async for document in cursor:
+            rows.append(document)
+        await cursor.close()
+
+        return rows, total
+
+    # 查询列表前N个
+    async def top(self, collection, query={}, sort=[], limit=1, fields=None, batch_size: int = 2000):
+        cursor = self.db[collection].find(filter=query, projection=fields, sort=sort, limit=limit)
+        cursor.batch_size(batch_size)
+        rows = []
+        async for document in cursor:
+            if limit == 1:
+                return document
+            rows.append(document)
+        await cursor.close()
+
+        return rows
+
+    # 查询去重列表
+    async def distinct(self, collection, dist_key, query={}, fields=None):
+        return await self.db[collection].find(query, fields).distinct(dist_key)
+
+    # 含分页聚合查询
+    async def aggregate_page(self, collection, pipelines, page_no=1, page_size=20):
+        skip = page_size * (page_no - 1)
+        if pipelines:
+            pipelines.append({'$facet': {'total': [{'$count': 'count'}], 'rows': [{'$skip': skip}, {'$limit': page_size}]}})
+            pipelines.append({'$project': {'data': '$rows', 'total': {'$arrayElemAt': ['$total.count', 0]}}})
+
+            cursor = self.db[collection].aggregate(pipelines, session=None, allowDiskUse=True)
+            cursor.batch_size(page_size)
+            async for rs in cursor:
+                # rs: dict as {"data": [], "total": int}
+                if rs and 'data' in rs and 'total' in rs:
+                    await cursor.close()
+                    return rs.get('data'), rs.get('total')
+            await cursor.close()
+        return [], 0
+
+    # 聚合查询
+    async def aggregate(self, collection, pipelines=[], batch_size: int = 2000):
+        """
+        样例:
+            cursor = await aio_mongo.aggregate(collection="collection_name", pipelines=[])
+            async for row in cursor:
+                print(row)
+
+            await cursor.close()
+        注意：应该在使用完毕后主动关闭 cursor
+        """
+        cursor = self.db[collection].aggregate(pipelines, session=None, allowDiskUse=True)
+        cursor.batch_size(batch_size)
+        return cursor
+
+    # 查询分页列表,还没找到实际应用
+    async def list_with_page(self, collection, query={}, page_size=10000, fields=None):
+        # 没有用到
+        rows = list()
+        total = await self.db[collection].count_documents(query)
+        if total > 0 and page_size > 0:
+            total_page = round(total / page_size) + 1
+            for page in range(0, total_page):
+                if fields:
+                    cursor = self.db[collection].find(query, fields).skip(page_size * page).limit(page)
+                else:
+                    cursor = self.db[collection].find(query).skip(page_size * page).limit(page)
+                async for document in cursor:
+                    rows.append(document)
+                await cursor.close()
+        return rows
+
+    # 插入或更新
+    async def insert_or_update(self, collection, data, id_key='_id', update=None, upsert=True, multi=False):
+        if not multi:
+            if data and not update:
+                result = await self.db[collection].update_one({id_key: data[id_key]}, {'$set': data}, upsert=upsert)
+            elif not data and update:
+                result = await self.db[collection].update_one({id_key: data[id_key]}, update, upsert=upsert)
+            else:
+                # all([data, update]) or not all([data, update])
+                raise BizException("data和update不能同时存在或同时为空")
+        else:
+            if data and not update:
+                result = await self.db[collection].update_many({id_key: data[id_key]}, {'$set': data}, upsert=upsert)
+            elif not data and update:
+                result = await self.db[collection].update_many({id_key: data[id_key]}, update, upsert=upsert)
+            else:
+                raise BizException("data和update不能同时存在或同时为空")
+
+        return result
+
+    # 插入
+    async def insert(self, collection, data):
+        return await self.db[collection].insert_one(data)
+
+    # 更新
+    async def update(self, collection, filter, data=None, update=None, multi=False):
+        if multi:
+            if data and not update:
+                result = await self.db[collection].update_many(filter, {'$set': data})
+            elif not data and update:
+                result = await self.db[collection].update_many(filter, update)
+            else:
+                raise BizException("data和update不能同时存在或同时为空")
+        else:
+            if data and not update:
+                result = await self.db[collection].update_one(filter, {'$set': data})
+            elif not data and update:
+                result = await self.db[collection].update_one(filter, update)
+            else:
+                raise BizException("data和update不能同时存在或同时为空")
+
+        return result
+
+    # 原生保存方法
+    async def save(self, collection, filter: dict, save_data: dict, upsert=True):
+        return await self.db[collection].update_one(filter, {'$set': save_data}, upsert=upsert)
+
+    # 以主键更新
+    async def update_by_pk(self, collection, pk_val, data=None, update=None, upsert=False):
+        if data and not update:
+            result = await self.db[collection].update_one({'_id': pk_val}, {'$set': data}, upsert=upsert)
+        elif not data and update:
+            result = await self.db[collection].update_one({'_id': pk_val}, update, upsert=upsert)
+        else:
+            raise BizException("data和update不能同时存在或同时为空")
+        return result
+
+    # 批量更新
+    async def batch_update(self, collection, filter, datas=None, update=None, *args, **kwargs):
+        if datas and not update:
+            result = await self.db[collection].update_many(filter, {'$set': datas})
+        elif not datas and update:
+            result = await self.db[collection].update_many(filter, update)
+        else:
+            raise BizException("data和update不能同时存在或同时为空")
+        return result
+
+    # 删除
+    async def delete(self, collection, filter):
+        return await self.db[collection].delete_many(filter)
+
+    # 插入或更新
+    async def bulk_write(self, collection, bulk_list: list, batch_size: int = 1000):
+        result = None
+        if bulk_list:
+            bulk_lists = [bulk_list[i: i + batch_size] for i in range(0, len(bulk_list), batch_size)]
+            for _bulk_list in bulk_lists:
+                result = await self.db[collection].bulk_write(_bulk_list, ordered=False, bypass_document_validation=True)
+        return result
+
+    # 创建索引
+    async def create_index(self, collection, fields):
+        return await self.db[collection].create_index(fields)
+
+    def close(self):
+        if self.client:
+            try:
+                self.client.close()
+                print("close successful")
+            except Exception as e:
+                print("close mongo client catch err:", e)
+

fast-web-core-1.0.0/fast_web_core/client/async_mulit_tenant_mongo_client.py

@@ -0,0 +1,69 @@
+# encoding: utf-8
+import asyncio
+import threading
+from typing import Optional
+from ..client.async_mongo_client import AsyncMongo
+from ..lib import cfg, logger
+
+
+LOGGER = logger.get('AsyncMultiTenantMongo')
+
+
+class AsyncMultiTenantMongo(AsyncMongo):
+    """
+    异步多租户Mongo客户端简易封装
+    """
+    def __init__(self, tenant_code=None, mongo_url=None, mongo_db=None, db_prefix=None):
+        if not mongo_url:
+            mongo_url = cfg.get('MONGO_URL') or 'mongodb://localhost:27017'
+        if not mongo_db:
+            if tenant_code and (db_prefix or cfg.get('MONGO_DB_BIZ_PREFIX', None)):
+                mongo_db = f"{db_prefix or cfg.get('MONGO_DB_BIZ_PREFIX', None)}{tenant_code}"
+        if not mongo_db:
+            raise Exception('mongodb database not specified')
+        super().__init__(mongo_url, mongo_db)
+        LOGGER.info(f'[{tenant_code}] tenant mongodb inited~')
+
+
+class AsyncMultiTenantMongoHolder(object):
+    """
+    异步多租户Mongo客户端持有器（租户数据库连接池）
+    """
+    _instance_lock = threading.Lock()
+    _instance_async_lock = asyncio.Lock()
+    _tenant_instance_dict = dict()
+
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(AsyncMultiTenantMongoHolder, "_instance"):
+
+            if not hasattr(AsyncMultiTenantMongoHolder, "_instance"):
+                AsyncMultiTenantMongoHolder._instance = object.__new__(cls)
+
+        return AsyncMultiTenantMongoHolder._instance
+
+    @staticmethod
+    def get_tenant_mongo(tenant_code: str) -> Optional[AsyncMultiTenantMongo]:
+        if not tenant_code:
+            return None
+
+        # 有实例则直接返回
+        if tenant_code in AsyncMultiTenantMongoHolder._tenant_instance_dict:
+            return AsyncMultiTenantMongoHolder._tenant_instance_dict.get(tenant_code)
+
+        # 无实例则加锁创建
+        AsyncMultiTenantMongoHolder._instance_lock.acquire()
+        try:
+            # 双重锁校验
+            if tenant_code not in AsyncMultiTenantMongoHolder._tenant_instance_dict:
+                # 初始化新实例
+                inst = AsyncMultiTenantMongo(tenant_code=tenant_code)
+                AsyncMultiTenantMongoHolder._tenant_instance_dict[tenant_code] = inst
+            if AsyncMultiTenantMongoHolder._instance_lock.locked():
+                AsyncMultiTenantMongoHolder._instance_lock.release()
+            return AsyncMultiTenantMongoHolder._tenant_instance_dict.get(tenant_code)
+        finally:
+            try:
+                if AsyncMultiTenantMongoHolder._instance_lock.locked():
+                    AsyncMultiTenantMongoHolder._instance_lock.release()
+            except Exception as e:
+                LOGGER.info(e)