valar 0.0.12__py3-none-any.whl → 0.0.14__py3-none-any.whl

valar/data/mon/__init__.py

@@ -0,0 +1,84 @@
+from bson import ObjectId
+from django.db.models import QuerySet
+from pymongo.results import InsertOneResult, UpdateResult
+
+from django.conf import settings
+import pymongo
+
+from ..query import Query
+
+try:
+    MONGO = settings.MONGO_SETTINGS
+except AttributeError:
+    MONGO = {
+        'host': 'localhost',
+        'port': 27017,
+    }
+host, port, username, password = MONGO.get('host'), MONGO.get('port'), MONGO.get('username'), MONGO.get('password')
+
+if username and password:
+    uri = f'mongodb://{username}:{password}@{host}:{port}/'
+else:
+    uri = f'mongodb://{host}:{port}/'
+mongo_params = {
+    'maxPoolSize': 10,
+    'minPoolSize': 0,
+    'maxIdleTimeMS': 10000,
+    'connectTimeoutMS': 10000,
+    'socketTimeoutMS': 10000,
+    'serverSelectionTimeoutMS': 10000,
+}
+
+def get_mongo_client():
+    client = pymongo.MongoClient(uri, **mongo_params)
+    client['admin'].command('ping')
+    return client
+
+
+
+class MongoDao:
+    def __init__(self, ref):
+        self.ref = ref
+        db_name = settings.BASE_APP
+        col_name = ref.replace('.', '_')
+        self.client = get_mongo_client()
+        self.collection = self.client[db_name][col_name]
+    def save_one(self, item):
+        _id = item.get('id', None)
+        _id = None if isinstance(_id, int) else _id
+        if _id is None:
+            bean:InsertOneResult = self.collection.insert_one(item)
+            _id = bean.inserted_id
+            self.collection.update_one({'_id': _id}, {'$set': {'sort':str(_id)}})
+        else:
+            del item['id']
+            _id = ObjectId(_id)
+            self.collection.update_one({'_id': _id}, {'$set': item})
+        return self.collection.find_one({'_id': _id})
+
+
+    def update_many(self, query, template):
+        self.collection.update_many(query.mon_conditions(), {'$set': template})
+
+
+    def delete_one(self, _id):
+        self.collection.delete_one({'_id': ObjectId(_id)})
+
+    def delete_many(self, query):
+        self.collection.delete_many(query.mon_conditions())
+
+
+    def find_one(self, _id):
+        return self.collection.find_one({'_id': ObjectId(_id)})
+
+    def find_many(self, query: Query, size=0, page=1) -> [QuerySet, int]:
+        skip = (page - 1) * size
+        condition = query.mon_conditions()
+        total = self.collection.count_documents(condition)
+        cursor = self.collection.find(condition).sort(query.orders).skip(skip).limit(size)
+        return [cursor, total]
+
+
+    def meta(self):
+        one = self.collection.find_one()
+        return one

valar/data/orm/__init__.py

@@ -0,0 +1,87 @@
+import datetime
+from django.apps import apps
+from django.core.paginator import Paginator
+from django.db.models import Manager, QuerySet
+
+from ..orm.detacher import detach_props, save_detached
+from ..orm.meta_loader import load_meta, load_view, load_meta_field
+from ..models import VModel
+from ..query import Query
+
+
+def load_model(entity=None):
+    mapping = {}
+    for mod in apps.get_models():
+        if issubclass(mod, VModel):
+            path, name = mod.__module__, mod.__name__
+            print(path)
+            app = path.replace('.models', '').replace('src.valar.','')
+            key = '%s.%s' % (app, name)
+            verbose_name = mod._meta.verbose_name
+            mapping[key] = [mod, verbose_name]
+    return mapping.get(entity) if entity else mapping
+
+class OrmDao:
+    def __init__(self, entity):
+        self.entity = entity
+        param = load_model(entity)
+        if param is None:
+            raise Exception('no entity named %s' % entity)
+        self.model = param[0]
+        self.name: str = param[1]
+        self.manager: Manager = self.model.objects
+        self.meta_fields = {}
+        self.model_fields = {}
+        for field in self.model._meta.get_fields():
+            _field = load_meta_field(field)
+            prop = _field['prop']
+            self.model_fields[prop] = field
+            self.meta_fields[prop] = _field
+
+
+    def save_one(self, item):
+        _item = detach_props(item, self.meta_fields.values())
+        _id = item.get('id',0)
+        query_set = self.manager.filter(id=_id)
+        if len(query_set):
+            del item['id']
+            item['modify_time'] = datetime.datetime.now()
+            query_set.update(**item)
+            bean = query_set.first()
+        else:
+            bean = self.manager.create(**item)
+            bean.sort = bean.id
+            bean.save()
+        save_detached(bean, _item, self.model_fields)
+        return bean
+
+    def update_many(self, query: Query, template):
+        self.find_many(query).update(**template)
+
+    def delete_one(self, _id):
+        self.manager.filter(id=_id).delete()
+
+    def delete_many(self, query: Query):
+        self.find_many(query)[0].delete()
+
+    def find_one(self, _id):
+        return self.manager.filter(id=_id).first()
+
+    def find_many(self, query: Query, size=0, page=1)->[QuerySet, int]:
+        includes, excludes = query.orm_conditions()
+        query_set = self.manager.filter(includes).exclude(excludes).order_by(*query.orm_orders())
+        total = query_set.count()
+        if size:
+            paginator = Paginator(query_set, size)
+            query_set = paginator.page(page).object_list
+        return query_set, total
+
+
+    def meta(self, code:str = 'default'):
+        omit = [ 'sort', 'create_time', 'modify_time', 'saved']
+        fields = [ self.meta_fields[prop] for prop in self.meta_fields if prop not in omit]
+        view =  load_view(self.entity, code, self.name, fields)
+        return load_meta(view)
+
+
+

valar/data/orm/detacher.py

@@ -0,0 +1,61 @@
+from django.db.models import (ManyToOneRel, ForeignKey, ManyToManyRel, ManyToManyField, OneToOneField, CharField,
+                              OneToOneRel, IntegerField, BooleanField, FloatField, FileField, JSONField, DateField,
+                              DateTimeField, TimeField, QuerySet)
+
+from ..models import VModel
+
+
+def detach_props(item, fields):
+    keys = [field['prop'] for field in fields if
+            field['domain'] in ['ManyToOneRel', 'ManyToManyField', 'ManyToManyRel', 'OneToOneRel', 'OneToOneField']]
+    _item = {}
+    for key in keys:
+        value = item.get(key)
+        if value is not None:
+            _item[key] = value
+            del item[key]
+    return _item
+
+def save_detached(bean, _item, model_fields ):
+    for prop in _item:
+        value = _item[prop]
+        field = model_fields.get(prop)
+        clazz = type(field)
+        if clazz == ManyToManyField:
+            m2m = getattr(bean, prop)
+            m2m.clear()
+            m2m.add(*value)
+        elif clazz == ManyToOneRel:
+            getattr(bean, field.get_accessor_name()).clear()
+            remote_model: VModel = field.related_model
+            new_set: QuerySet = remote_model.objects.filter(id__in=value)
+            remote_field: ForeignKey = field.remote_field
+            k = remote_field.get_attname()
+            new_set.update(**{k: bean.id})
+        elif clazz == ManyToManyRel:
+            getattr(bean, field.get_accessor_name()).clear()
+            remote_model: VModel = field.related_model
+            remote_items: QuerySet = remote_model.objects.filter(id__in=value)
+            remote_field: ManyToManyField = field.remote_field
+            remote_field_prop = remote_field.get_attname()
+            for _bean in remote_items:
+                bean_set = getattr(_bean, remote_field_prop)
+                bean_set.add(bean)
+        elif clazz == OneToOneRel:
+            remote_model: VModel = field.related_model
+            remote_field: OneToOneField = field.remote_field
+            remote_field_prop = remote_field.get_attname()
+            _bean = remote_model.objects.get(id=value)
+            __bean = remote_model.objects.filter(**{remote_field_prop: bean.id}).first()
+            if __bean:
+                setattr(__bean, remote_field_prop, None)
+                __bean.save()
+            setattr(_bean, remote_field_prop, bean.id)
+            _bean.save()
+        elif clazz == OneToOneField:
+            __bean = field.model.objects.filter(**{prop: value}).first()
+            if __bean:
+                setattr(__bean, prop, None)
+                __bean.save()
+            setattr(bean, prop, value)
+            bean.save()

valar/data/orm/meta.py

@@ -0,0 +1,24 @@
+mf_common = ['prop','name','domain']
+
+field_props = {
+    'data.Meta': {
+        'default': ('pick', ['entity','name']),
+    },
+    'data.MetaView': {
+        'list': ('pick', ['meta_id','code','view_name']),
+        'control':('omit', ['name','meta_id', 'code', 'view_name', 'metafield'])
+    },
+    'data.MetaField': {
+        'tool': ('pick',[*mf_common,'tool','refer','format']),
+        'rest': ('pick',[*mf_common,'not_null','read_only','unit','sortable','allow_search','allow_download','allow_upload','allow_update']),
+        'table': ('pick',[*mf_common,'column_width','fixed','align','edit_on_table','hide_on_table','header_color','cell_color']),
+        'form': ('pick',[*mf_common,'hide_on_form','hide_on_form_insert','hide_on_form_edit','hide_on_form_branch','hide_on_form_leaf','span']),
+    }
+}
+
+
+field_default = {
+    'data.MetaField':{
+
+    }
+}

valar/data/orm/meta_loader.py

@@ -0,0 +1,135 @@
+from django.db.models import (ManyToOneRel, ForeignKey, ManyToManyRel, ManyToManyField, OneToOneField, CharField,
+                              OneToOneRel, IntegerField, BooleanField, FloatField, FileField, JSONField, DateField,
+                              TextField,DateTimeField, TimeField)
+
+from ..orm.meta import field_props
+from ..models import Meta, MetaView, VModel, MetaField
+
+
+
+
+
+def __save_model(model):
+    model.save()
+    model.sort = model.id
+    model.save()
+
+
+
+def load_view(entity, code, name, fields):
+    meta = Meta.objects.filter(entity=entity).first()
+    if meta is None:
+        meta = Meta(entity=entity, name=name)
+        __save_model(meta)
+    view = MetaView.objects.filter(meta__entity=entity, code=code).first()
+    if view is None:
+        view = MetaView(meta=meta, code=code, view_name=code.upper())
+        __save_model(view)
+    if view.metafield_set.count() == 0:
+        t, p = field_props.get(entity, {}).get(code,('omit',[]))
+        _fields = [f for f in fields if f['prop'] not in p] if t=='omit' else [f for f in fields if f['prop']  in p]
+        for f in _fields:
+            f['view'] = view
+            field = MetaField.objects.create(**f)
+            __save_model(field)
+    return view
+
+def load_meta(view):
+    _view = view.full
+    _meta = _view['meta']
+    _fields = _view['metafield_set']
+    clear_item(_view, 'meta_id', 'metafield', 'metafield_set', 'meta')
+    _view['meta_name'] = _meta['name']
+    _view['entity'] = _meta['entity']
+    _view['fields'] = {}
+    for _field in _fields:
+        clear_item(_field, 'view_id')
+        prop = _field['prop']
+        _view['fields'][prop] = _field
+    return _view
+
+
+
+def clear_item(item, *keys):
+    del item['saved']
+    del item['sort']
+    del item['id']
+    del item['create_time']
+    del item['modify_time']
+    for key in keys:
+        del item[key]
+
+
+
+def get_refer(model, multiple = False):
+    module, name = model.__module__, model.__name__
+    entity = '%s.%s' % (module.replace('.models', '').split('.')[-1], name)
+    return {
+        "entity": entity,
+        "value": "id", "label": 'name', "display": "id",
+        "strict": False, "remote": False, "multiple": multiple,
+        "includes": {}, "excludes": {}, "root": 0,
+    }
+
+def get_align(clazz):
+    if clazz in [FloatField, IntegerField, ManyToManyRel, ManyToManyField, ManyToOneRel]:
+        return 'right'
+    elif clazz in [BooleanField,FileField,JSONField,DateField,DateTimeField,TimeField]:
+        return 'center'
+    return 'left'
+
+def get_format(field):
+    clazz = type(field)
+    if clazz == CharField:
+        return {'maxlength': field.max_length, "type": "text"}
+    if clazz == TextField:
+        return {'maxlength': None, "type": "textarea"}
+    elif clazz == FloatField:
+        return {'min': None, 'max': None, 'step': 1, 'precision': None, 'step_strictly': False}
+    elif clazz == IntegerField:
+        return {'min': None, 'max': None, 'step': 1, 'precision': 0, 'step_strictly': True}
+    elif clazz == FileField:
+        return {'max': 5, 'accept': "*", "width": "800px", "height": "auto", 'locked': False}
+    else:
+        return {}
+
+
+
+def load_meta_field(field):
+    clazz = type(field)
+    if clazz in [ManyToOneRel, ManyToManyField, ManyToManyRel]:
+        prop = field.name
+        domain = clazz.__name__
+        model: VModel= field.related_model
+        label = model._meta.verbose_name
+        refer = get_refer(model, True)
+    elif clazz in [ForeignKey]:
+        prop = field.name + "_id"
+        domain = field.get_internal_type()
+        model: VModel = field.related_model
+        label = field.verbose_name
+        refer = get_refer(model)
+    elif clazz in [OneToOneRel, OneToOneField]:
+        prop = field.name + "_id"
+        domain = clazz.__name__
+        model: VModel = field.related_model
+        label = model._meta.verbose_name
+        refer = get_refer(model)
+    else:
+        prop = field.name
+        domain = field.get_internal_type()
+        label = field.verbose_name
+        refer = {}
+    not_null = not field.null
+    align = get_align(clazz)
+    _format = get_format(field)
+    return {
+        "prop": prop,
+        "label":label,
+        "name":label,
+        "domain":domain,
+        "refer":refer,
+        "format":_format,
+        "not_null":not_null,
+        "align":align,
+    }

valar/data/orm/values.py

@@ -0,0 +1,93 @@
+from django.db.models import (ManyToOneRel,  ManyToManyRel, OneToOneField, CharField,ManyToManyField,
+                              OneToOneRel, IntegerField, BooleanField, FloatField, FileField, JSONField, DateField,
+                              DateTimeField, TimeField, TextField, QuerySet)
+from django.db.models.fields.related import ForeignKey
+
+from ..models import VModel, MetaField
+
+
+def get_props(model:VModel, code):
+
+    simple_props,  referred_fields, date_props = [], {}, {}
+    fields = model._meta.get_fields()
+    for field in fields:
+        clazz = type(field)
+        prop = field.name
+        if clazz in [ForeignKey,ManyToOneRel,
+                     ManyToManyRel,ManyToManyField,
+                     OneToOneRel, OneToOneField]:
+            referred_fields[prop] = {
+                'value': 'id', 'label': 'name', 'display': 'id',
+                'field': field
+            }
+        else:
+            simple_props.append(prop)
+        if clazz in [ForeignKey, OneToOneField, OneToOneRel]:
+            simple_props.append(prop)
+        if clazz in [DateField, DateTimeField]:
+            date_props[prop] = clazz
+
+    module, name = model.__module__, model.__name__
+    entity = '%s.%s' % (module.replace('.models', '').split('.')[-1], name)
+    if code:
+        field_set = MetaField.objects.filter(view__code=code, view__meta__entity=entity)
+        customs = field_set.filter(domain='Custom').values('prop')
+        for row in customs:
+            simple_props.append(row['prop'])
+        refers =field_set.filter(prop__in=referred_fields.keys()).values('prop','refer__value','refer__label','refer__display')
+        for row in refers:
+            prop, value, label, display = row['prop'], row['refer__value'], row['refer__label'], row['refer__display']
+            referred_fields[prop]['value'] = value
+            referred_fields[prop]['label'] = label
+            referred_fields[prop]['display'] = display
+    return simple_props, referred_fields, date_props
+
+
+def to_dict(query_set: QuerySet, code=None):
+    model = query_set.model
+    simple_props, referred_fields, date_props = get_props(model, code)
+    values = query_set.values(*[*simple_props]) if len(query_set) else []
+    results = list(values)
+    pks = []
+    for row in results:
+        pks.append(row['id'])
+        for prop in date_props:
+            formating = '%Y-%m-%d' if date_props[prop] == DateField else '%Y-%m-%d %H:%M:%S'
+            val = row[prop]
+            row[prop] =val.strftime(formating) if val else val
+    for prop in referred_fields:
+        _field = referred_fields[prop]
+        value, label, display, field = _field['value'], _field['label'], _field['display'], _field['field']
+        clazz = type(field)
+        keys = {'id', value, display, label}
+        if clazz in [ForeignKey, OneToOneField, OneToOneRel]:
+            related_model = field.related_model
+            related_pks = set([row.get(prop) for row in results if row.get(prop)])
+            related_items = related_model.objects.filter(id__in=related_pks).values(*keys)
+            mapping = {item['id']: item  for item in related_items}
+            for row in results:
+                value = row.get(prop)
+                if value:
+                    row[prop] = mapping[value]
+                    row[f'{prop}_id'] = value
+        elif clazz in [ ManyToManyField, ManyToOneRel, ManyToManyRel]:
+            _prop = f'{prop}__id'
+            linkage = model.objects.filter(id__in=pks).exclude(**{f'{_prop}__isnull':True}).values('id',_prop)
+            row_mapping = {}
+            _pks = set()
+            for link in linkage:
+                _id, _pk = link['id'], link[_prop]
+                _pks.add(_pk)
+                array = row_mapping.get(_id,[])
+                array.append(_pk)
+                row_mapping[_id] = array
+            related_model = field.related_model
+            related_items = related_model.objects.filter(id__in=_pks).values(*keys)
+            mapping = {item['id']: item  for item in related_items}
+            for row in results:
+                _id = row.get('id')
+                __ids = row_mapping.get(_id, [])
+                __set = [ mapping[__id] for __id in __ids]
+                row[prop] = __ids
+                row[f'{prop}_set'] = __set
+    return results

valar/data/query.py

@@ -0,0 +1,48 @@
+from functools import reduce
+
+
+from django.db.models import Q
+
+
+
+
+class Condition:
+    def __init__(self, condition):
+        self.includes = condition.get('includes', {})
+        self.excludes = condition.get('excludes', {})
+
+
+
+class Query:
+    def __init__(self, body=None):
+        if body is None:
+            body = {}
+        self.template = body.get('template',{})
+        self.condition = Condition(body.get('condition',{}))
+        self.search = [ Condition(condition) for condition in body.get('search',[])]
+        self.orders = body.get('orders',{})
+
+    def orm_orders(self):
+        array = []
+        for key in self.orders :
+            value = self.orders .get(key)
+            prefix = '-' if value == -1 else ''
+            array.append(f'{prefix}{key}')
+        return array
+
+    def orm_conditions(self):
+        includes, excludes = self.condition.includes, self.condition.excludes
+        if len(self.search):
+            inc = [ Q(**{**includes, **sea.includes}) for sea in self.search]
+            exc = [ Q(**{**excludes, **sea.excludes}) for sea in self.search]
+            def fun(x, y):return x | y
+            return [reduce(fun, inc), reduce(fun, exc)]
+        else:
+            return [Q(**self.condition.includes), Q(**self.condition.excludes)]
+
+    def mon_conditions(self):
+        includes, excludes = self.condition.includes, self.condition.excludes
+        return {}
+
+
+    # def _mon_conditions(self):

valar/data/urls.py

@@ -0,0 +1,15 @@
+from django.urls import path
+
+from ..data import views
+
+urlpatterns = [
+    path('save_one', views.save_one),
+    path('save_many', views.save_many),
+    path('update_many', views.update_many),
+    path('delete_one', views.delete_one),
+    path('delete_many', views.delete_many),
+    path('find_one', views.find_one),
+    path('find_many', views.find_many),
+    path('meta', views.meta),
+
+]

valar/data/utils.py

@@ -0,0 +1,70 @@
+from django.db.models import QuerySet
+from pymongo.synchronous.cursor import Cursor
+
+from .orm.values import to_dict
+from ..data.mon import MongoDao
+from ..data.orm import OrmDao
+
+
+
+
+
+
+def get_dao(db,entity):
+    if db == 'orm':
+        return OrmDao(entity)
+    elif db == 'mon':
+        return MongoDao(entity)
+    else:
+        raise ValueError('Invalid database %s' % db)
+
+def transform(db, results, code=None):
+    if isinstance(results, Cursor):
+        return [to_item(db, bean) for bean in results]
+    elif isinstance(results, QuerySet):
+        return to_dict(results, code)
+    else:
+        return to_item(db, results)
+
+def to_item(db, bean):
+    if db == 'orm':
+        return bean.full
+    elif db == 'mon':
+        bean['id'] = str(bean['_id'])
+        del bean['_id']
+        return bean
+
+
+def array2tree(data, mapping):
+    mapping = mapping or {}
+    lookup = {}
+    for array in data:
+        for i in range(len(array)):
+            key = '/'.join(array[0:i+1])
+            item = mapping.get(key, {})
+            value = item.get('value', array[i])
+            label = item.get('label', value)
+            display = item.get('display')
+            item = lookup.get(key, {'value': value,'label':label,'display':display})
+            if i < len(array) -1:
+                item['children'] = item.get('children', [])
+            lookup[key] = item
+            if i > 0:
+                parent =   '/'.join(array[0:i])
+                lookup[parent]['children'].append(lookup[key])
+    return [lookup[root] for root in [*set([array[0] for array in data])]]
+
+
+
+    # for parent, child in data:
+    #     if parent not in lookup:
+    #         lookup[parent] = {'label': parent, 'value':parent, 'children': []}
+    #     if child not in lookup:
+    #         lookup[child] = {'label': child, 'value':child, 'children': []}
+    #     lookup[parent]['children'].append(lookup[child])
+    # children_set = {child for _, child in data}
+    # root_nodes = [node for name, node in lookup.items() if name not in children_set]
+    # return root_nodes
+
+
+