deepfos 1.1.60__py3-none-any.whl

deepfos/core/dimension/sysdimension.py

@@ -0,0 +1,168 @@
+from typing import Union
+
+import pandas as pd
+
+from deepfos.core.logictable import MetaTable
+from deepfos.lib.utils import unpack_expr
+from deepfos.element.dimension import Dimension as EleDimension
+from deepfos.element.datatable import DataTableClickHouse, DataTableMySQL
+from ._base import DimensionBase, MemberBase, MemberContainer, NAME_DFLT
+from .dimexpr import DimExprAnalysor
+
+# -----------------------------------------------------------------------------
+# typing
+T_AbsDataTable = Union[DataTableMySQL, DataTableClickHouse]
+
+
+# -----------------------------------------------------------------------------
+# core
+class AsMbrContainer:
+    def __init__(self, func, include_self=False):
+        self.func = func
+        self.include_self = include_self
+
+    def __get__(self, instance, owner=None):
+        with instance.tbl_dim.temporary_lock(name__in=instance.name_list):
+            with self.func(instance):
+                rslt = instance.data_tbl.data
+        if self.include_self:
+            # noinspection PyProtectedMember
+            rslt = rslt.append(instance._data)
+        return SysDimMemberContainer(rslt, *instance.split(), hierarchy=self.name)
+
+    def __set_name__(self, owner, name):
+        self.name = name
+
+
+class SysDimension(DimensionBase):
+    """
+    系统维度
+
+    绕过维度元素，使用维度底层数据表创建的维度
+
+    Args:
+        dimname: 维度名称
+        tbl_closure: 维度绑定的closure表，记录维度层级关系
+        tbl_dim: 维度数据表
+        name_only: 是否仅需要维度名 (Cube用)
+        folder_id: 维度所在文件夹ID
+        path: 维度所在绝对路径
+
+    """
+    def __init__(
+        self,
+        dimname: str,
+        tbl_closure: T_AbsDataTable = None,
+        tbl_dim: T_AbsDataTable = None,
+        name_only: bool = False,
+        folder_id: str = None,
+        path: str = None,
+        server_name: str = None,
+    ):
+        super().__init__(dimname)
+        self._name_only = name_only
+        if tbl_closure is None or tbl_dim is None:
+            tbl_dim, tbl_closure = self._get_datatable(folder_id, path, server_name)
+
+        self._create_tree(tbl_closure, tbl_dim, name_only)
+
+    def _create_tree(self, tbl_closure, tbl_dim, name_only: bool):
+        # 组建table_dimension和table_closure的关联关系
+        # 父表为table_closure，以不同字段关联两张table_dimension
+        self.tbl_closure = MetaTable(tbl_closure.table_name, tuple(), {"datatable": tbl_closure})
+        self.tbl_dim = MetaTable(
+            tbl_dim.table_name, tuple(),
+            {"datatable": tbl_dim, "parent": {"cls": self.tbl_closure, 'on': ('id',), "alias": ('ancestor',)}}
+        )
+        self.data_tbl = MetaTable(
+            tbl_dim.table_name + '_data', tuple(),
+            {
+                "datatable": tbl_dim,
+                "parent": {"cls": self.tbl_closure, 'alias': ('descendants',), 'on': ('id',)}
+            }
+        )
+        if name_only:
+            self.data_tbl.fields = (NAME_DFLT,)
+
+    def _get_datatable(self, folder_id: str, path: str, server_name: str = None):
+        # 获取table_dimension和table_closure的真实表名
+        dim = EleDimension(self.name, folder_id=folder_id, path=path, server_name=server_name)
+        return dim.table_dimension, dim.table_closure
+
+    def __getitem__(self, item):
+        if self._name_only:
+            data = pd.DataFrame(data={NAME_DFLT: [item]})
+            return SysDimMember(item, self.tbl_closure, self.tbl_dim, self.data_tbl, data)
+        return SysDimMember(item, self.tbl_closure, self.tbl_dim, self.data_tbl)
+
+
+_base = lambda self: self.data_tbl.temporary_lock(is_base__eq=1)
+_children = lambda self: self.tbl_closure.temporary_lock(distance__eq=1)
+_descendant = lambda self: self.tbl_closure.temporary_lock(distance__ne=0)
+
+
+class SysDimMember(MemberBase):
+    def __init__(self, name, tbl_closure, tbl_dim, data_tbl, data=None):
+        super().__init__(name)
+        self.tbl_closure = tbl_closure
+        self.tbl_dim = tbl_dim
+        self.data_tbl = data_tbl
+        self.name_list = (name, ) if isinstance(name, str) else name
+        if data is None:
+            self._data = self._fetch_data()
+        else:
+            self._data = data
+        if len(self.name_list) != len(self._data):
+            unknown = set(self.name_list) - set(self._data.name)
+            raise ValueError(f"Dimension {unknown!r} does not exist.")
+
+    def _fetch_data(self):
+        with self.tbl_dim.temporary_lock(name__in=self.name_list):
+            return self.tbl_dim.data
+
+    Base = AsMbrContainer(_base)
+    IBase = AsMbrContainer(_base, include_self=True)
+    Children = AsMbrContainer(_children)
+    IChildren = AsMbrContainer(_children, include_self=True)
+    Descendant = AsMbrContainer(_descendant)
+    IDescendant = AsMbrContainer(_descendant, include_self=True)
+
+    @property
+    def members(self):
+        return list(self._data.itertuples(index=False))
+
+    def __str__(self):
+        return ';'.join(self.name_list)
+
+    def where(self, method, **kwargs):
+        return SysDimMemberContainer(self._data, *self.split()).where(method, **kwargs)
+
+    def remove(self, *to_remove):
+        return SysDimMemberContainer(self._data, *self.split()).remove(*to_remove)
+
+    def split(self):
+        """分解为仅行单个成员的SysDimMember"""
+        if len(self.name_list) == 1:
+            return [self]
+
+        rtn = []
+        for idx, name in enumerate(self.name_list):
+            rtn.append(self.__class__(
+                name, self.tbl_dim, self.tbl_closure, self.data_tbl,
+                data=self._data.iloc[idx:idx+1]))
+        return rtn
+
+
+class SysDimMemberContainer(MemberContainer):
+    def __init__(self, data, *anchor_mbr, hierarchy=None):
+        super().__init__(*anchor_mbr, hierarchy=hierarchy)
+        self.__data = data
+
+    def _get_all_member(self):
+        return list(self.__data.itertuples(index=False))
+
+
+def read_expr(dim_expr):
+    dimname, expr = unpack_expr(dim_expr)
+    dim = SysDimension(dimname)
+    return DimExprAnalysor(dim, expr).solve()

deepfos/core/logictable/__init__.py

@@ -0,0 +1,5 @@
+from .sqlcondition import SQLCondition
+from .tablemodel import BaseTable, MetaTable
+from .nodemixin import TreeRenderer
+
+__all__ = ['SQLCondition', 'BaseTable', 'MetaTable', 'TreeRenderer']

deepfos/core/logictable/_cache.py

@@ -0,0 +1,141 @@
+from typing import Iterable, Optional
+from collections import UserDict, defaultdict
+import itertools
+
+import pandas as pd
+from loguru import logger
+from .sqlcondition import SQLCondition
+
+
+class Cache(UserDict):
+    def __init__(self, *args, **kwds):
+        super(Cache, self).__init__(*args, **kwds)
+        self.__ref_count = defaultdict(int)
+        self.__idx_map = {}
+        self._counter = itertools.count(1).__next__
+
+    def pop_by_score(self, score_func=None, highest=False):
+        if not self:
+            raise KeyError('dictionary is empty')
+
+        score_func = score_func or self._score_idx_plus_ref
+
+        sorted_keys = sorted(self.__idx_map.items(), key=score_func, reverse=highest)
+        pop_key = sorted_keys[0][0]
+        pop_val = self.pop(pop_key)
+        self.__idx_map.pop(pop_key)
+        return pop_key, pop_val
+
+    def _score_idx_plus_ref(self, key_with_idx, weight_idx=0.8, weight_ref=1):
+        key, idx = key_with_idx
+        ref_count = self.__ref_count[key]
+        return idx * weight_idx + ref_count * weight_ref
+
+    def __getitem__(self, item):
+        val = super(Cache, self).__getitem__(item)
+        self.__ref_count[item] += 1
+        return val
+
+    def __setitem__(self, key, value):
+        rslt = super(Cache, self).__setitem__(key, value)
+        self.__idx_map[key] = self._counter()
+        return rslt
+
+
+class Fields:
+    def __init__(
+        self,
+        fields: Optional[Iterable[str]],
+        extra_fields: Optional[Iterable[str]] = None
+    ):
+        if fields is None:
+            self.flds = None
+        else:
+            extra_fields = extra_fields or []
+            all_flds = {*fields, *extra_fields}
+            if any(fld.strip() == '*' for fld in all_flds):
+                self.flds = None
+            else:
+                self.flds = all_flds
+
+    def __le__(self, other: 'Fields'):
+        if other.flds is None:
+            return True
+        elif self.flds is None:
+            return other.flds is None
+        else:
+            return self.flds <= other.flds
+
+
+class DataProxy:
+    def __init__(self, max_size=10):
+        self._data_cache = Cache()
+        self._cond_cache = dict()
+        self._fields_cache = {}
+        if not max_size >= 1:
+            raise ValueError("Max size must be positive.")
+        self._max_size = max_size
+
+    def make_cache(
+        self,
+        key: SQLCondition,
+        fields: Optional[Iterable[str]],
+        value: pd.DataFrame,
+        force_update: bool = False
+    ):
+        series_code = key.serialized
+
+        if not force_update and series_code in self._cond_cache \
+                and Fields(fields) <= self._fields_cache[series_code]:
+            logger.debug("Found a better version in cache. Skip update.")
+            return
+
+        self._data_cache[series_code] = value
+        self._cond_cache[series_code] = key
+        self._fields_cache[series_code] = Fields(fields)
+
+        if len(self._data_cache) > self._max_size:
+            rm_key, _ = self._data_cache.pop_by_score()
+            logger.debug(f'Removed cache with lowest score:\n{_}')
+            self._cond_cache.pop(rm_key)
+            self._fields_cache.pop(rm_key)
+
+    def get_data(self, key: SQLCondition, fields: Optional[Iterable[str]]):
+        series_code = key.serialized
+        if series_code in self._data_cache:
+            fields_memo = Fields(fields)
+            if fields_memo <= self._fields_cache[series_code]:
+                logger.debug("Found perfect-match from cache.")
+                if fields_memo.flds is None:
+                    return self._data_cache[series_code]
+                else:
+                    return self._data_cache[series_code][list(fields)]
+
+        for sql_cnd in self._cond_cache.values():
+            # 不用小于号是为了避免重复调用__eq__
+            if key <= sql_cnd:
+                series_code = sql_cnd.serialized
+                fields_memo = Fields(fields, key.all_fields)
+
+                if fields_memo <= self._fields_cache[series_code]:
+                    cache = self._data_cache[sql_cnd.serialized]
+                    data = self.query_from_cache(cache, key)
+                    if fields_memo.flds is None:
+                        return data
+                    else:
+                        return data[list(fields)]
+        return None
+
+    @staticmethod
+    def query_from_cache(cache: pd.DataFrame, key: SQLCondition):
+        logger.debug(f"Query {key!r} from cache:\n{cache}")
+        query_str = key.to_pandasql()
+        if query_str is None:
+            data = cache
+        else:
+            data = cache.query(query_str)
+        fields = key.fields
+        if fields:
+            data = data.merge(key.val_list, on=fields, how='inner')
+        logger.debug(f"Got DATA from cache:\n{data}")
+        return data.reset_index(drop=True)