lidb 1.2.0__py3-none-any.whl

lidb/svc/data.py

@@ -0,0 +1,138 @@
+# Copyright (c) ZhangYundi.
+# Licensed under the MIT License. 
+# Created on 2025/10/11 11:01
+# Description:
+
+import queue
+import time
+from collections.abc import Callable
+
+import polars as pl
+import threading
+import logair
+
+class DataService:
+
+    def __init__(self, cache_size: int = 5):
+        self._max_cache_size = cache_size
+        self._cache = queue.Queue(maxsize=self._max_cache_size)
+        self._cache_dict: dict[str, dict[str, pl.DataFrame]] = dict()  # 用于快速查找的字典
+        self.stop_event = threading.Event()
+        self._data_thread = None
+        self.is_running = False
+        self._fn = None
+
+    def put_data(self, key: str, data: dict[str, pl.DataFrame]):
+        self._cache.put(key)
+        self._cache_dict[key] = data
+
+    def get_data(self) -> pl.DataFrame:
+        try:
+            key = self._cache.get_nowait()
+            data = self._cache_dict.pop(key)
+            return key, data, False
+        except queue.Empty:
+            return "", None, True
+
+    def _data_loading_worder(self,
+                             keys: list[str],
+                             iter_conf: dict[str, list[str]],):
+        logger = logair.get_logger(f"{__name__}.{self.__class__.__name__}.worker")
+        logger.info(f"Data loading worker started for {len(keys)} keys.")
+
+        def worker(key, work_id: int):
+            result = dict()
+            try:
+                for name, iters in iter_conf.items():
+                    data = self._fn(key=key, iterables=iters)
+                    result[name] = data
+                self.put_data(key, result)
+                logger.info(f"{key}(WorkerID: {work_id}) Loaded data.")
+            except Exception as e:
+                logger.warning(f"Failed to load data for {key}(WorkerID: {work_id}): {e}")
+
+        for i, k in enumerate(keys):
+            worker(key=k, work_id=i + 1)
+        self.stop_event.set()
+
+    def start(self,
+              fn: Callable,
+              keys: list[str],
+              iter_conf: dict[str, list[str]],
+              max_cache_size: int,):
+        """
+
+        Parameters
+        ----------
+        fn: 获取数据的函数，参数为 key 和 iterables 以及其它参数
+        keys
+        iter_conf
+        max_cache_size
+
+        Returns
+        -------
+
+        """
+        logger = logair.get_logger(f"{__name__}.{self.__class__.__name__}")
+        self._fn = fn
+        self._max_cache_size = max_cache_size
+        # 先确保之前的服务已经完全停止
+        if self.is_running:
+            logger.warning("DataService is already running")
+            self.stop()
+            # return
+        # 重新初始化缓存和 stop_event
+        self._cache = queue.Queue(maxsize=self._max_cache_size)
+        self._cache_dict.clear()
+        self.stop_event.clear()
+
+        logger.info(f"Starting DataService({self._max_cache_size}) for {len(keys)} key...")
+        # 启动后台数据加载线程
+        self._data_thread = threading.Thread(
+            target=self._data_loading_worder,
+            args=(keys,
+                  iter_conf,),
+            daemon=True,  # 设置为守护线程，主程序退出时自动结束
+        )
+        self.is_running = True
+        self._data_thread.start()
+        logger.info("DataService started successfully.")
+
+    def stop(self):
+        """停止数据服务"""
+        logger = logair.get_logger(f"{__name__}.{self.__class__.__name__}")
+        if not self.is_running:
+            logger.warning("Data service is not running")
+            return
+        logger.info("Stopping data service...")
+        self.stop_event.set()
+        if self._data_thread and self._data_thread.is_alive():
+            self._data_thread.join(timeout=10)
+        self.is_running = False
+        logger.info("Data service stopped")
+
+    def do(self, consumer: callable, wait_secs: float = 3):
+        """
+        消费数据
+        Parameters
+        ----------
+        consumer:
+        wait_secs
+
+        Returns
+        -------
+
+        """
+        while self.is_running:
+            key, data, is_empty = self.get_data()
+            if is_empty:
+                if self.stop_event.is_set():
+                    self.stop()
+                    break
+                else:
+                    time.sleep(wait_secs)
+                    continue
+            consumer(dict(key=key, data=data))
+
+
+D = DataService()

lidb/table.py

@@ -0,0 +1,129 @@
+# Copyright (c) ZhangYundi.
+# Licensed under the MIT License. 
+# Created on 2025/11/10 13:43
+# Description: 只有一张表单，没有分区的dataset特例, 所有数据都在一张表中
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from enum import Enum
+
+import xcals
+from functools import partial
+import polars as pl
+from datetime import datetime
+import logair
+import uuid
+from .database import tb_path, scan
+
+import ygo
+
+
+class TableMode(Enum):
+
+    F = "full" # 全量更新
+    I = "increment" # 增量更新
+
+
+
+class Table:
+
+    def __init__(self,
+                 fn: Callable[..., pl.DataFrame],
+                 tb: str,
+                 update_time: str,
+                 mode: TableMode = TableMode.F):
+        self.fn = fn
+        self.tb = tb
+        self.update_time = update_time
+        self._data_dir = tb_path(self.tb)
+        self.logger = logair.get_logger(__name__)
+        self.verbose = False
+        self.mode = mode
+
+    def __call__(self, *args, **kwargs):
+        fn = partial(self.fn, *args, **kwargs)
+        table = Table(fn,
+                      tb=self.tb,
+                      update_time=self.update_time,
+                      mode=self.mode)
+        return table
+
+    def _log(self, msg: str, lvl: str = "info"):
+        """统一日志输出方法"""
+        if self.verbose:
+            getattr(self.logger, lvl)(f"{self.tb}: {msg}")
+
+    def _do_job(self):
+        """获取数据并且保存数据"""
+        data = ygo.delay(self.fn)(this=self)()
+        if data is None:
+            self.logger.error("No data.")
+            return
+        if data.is_empty():
+            self.logger.warning("No data.")
+            return
+        if self.mode == TableMode.I:
+            time_uuid = uuid.uuid1()
+            data_file = self._data_dir / f"{time_uuid}.parquet"
+            data.write_parquet(data_file)
+        elif self.mode == TableMode.F:
+            data_file = self._data_dir / "0.parquet"
+            data.write_parquet(data_file)
+        else:
+            self.logger.error(f"Invalid table mode: {self.mode}")
+
+
+    def update(self, verbose: bool = False):
+        """更新最新数据: 全量更新, 覆盖旧数据"""
+        self.verbose = verbose
+        existed = self._data_dir.exists()
+        if not existed:
+            self._data_dir.mkdir(parents=True, exist_ok=True)
+            self._log("Creating new data.", "info")
+            self._do_job()
+        else:
+            modified_time = self.modified_time
+            if modified_time is not None:
+                modified_datetime = modified_time.strftime("%Y-%m-%d %H:%M:%S")
+                modified_d, modified_t = modified_datetime.split(" ")
+                if self._updated(data_date=modified_d, data_time=modified_t):
+                    return
+            self._log("Updating.", "info")
+            self._do_job()
+            self._log("Updated.", "info")
+
+    def get_value(self, eager: bool = True) -> pl.DataFrame | pl.LazyFrame:
+        """获取数据"""
+        self.update(verbose=True)
+        df = scan(self._data_dir)
+        if eager:
+            return df.collect()
+        return df
+
+    def _updated(self, data_date: str, data_time: str) -> bool:
+        """判断是否需要更新数据"""
+        recent_tradeday = xcals.get_recent_tradeday()
+        prev_tradeday = xcals.shift_tradeday(recent_tradeday, -1)
+        now = xcals.now()
+        latest_update_date = recent_tradeday if now >= self.update_time else prev_tradeday
+        return f"{data_date} {data_time}" >= f"{latest_update_date} {self.update_time}"
+
+    @property
+    def latest_file(self):
+        if not self._data_dir.exists():
+            return
+        parquet_files = list(self._data_dir.glob("*.parquet"))
+        if not parquet_files:
+            return
+        latest_file = max(parquet_files, key=lambda x: x.stat().st_mtime)
+        return latest_file
+
+    @property
+    def modified_time(self):
+        """获取文件修改时间"""
+        latest_file = self.latest_file
+        if latest_file is None:
+            return
+        mtime = self.latest_file.stat().st_mtime
+        return datetime.fromtimestamp(mtime)

lidb-1.2.0.dist-info/METADATA

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: lidb
+Version: 1.2.0
+Summary: Light database for quantor
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: dynaconf>=3.2.11
+Requires-Dist: polars>=1.31.0
+Requires-Dist: sqlparse>=0.5.3
+Requires-Dist: logair>=1.0.1
+Requires-Dist: clickhouse-df>=0.1.5
+Requires-Dist: connectorx>=0.4.3
+Requires-Dist: pymysql>=1.1.2
+Requires-Dist: xcals>=0.0.4
+Requires-Dist: ygo>=1.2.8
+Requires-Dist: lark>=1.3.1
+Requires-Dist: numpy>=2.3.1
+Requires-Dist: tqdm>=4.67.1

lidb-1.2.0.dist-info/RECORD

@@ -0,0 +1,24 @@
+lidb/__init__.py,sha256=-EUd5pO1o7cBs__gvTsxquuHRBpiVn7mVGkL9miyc2k,504
+lidb/database.py,sha256=DnPXRXvUO6g0kuMo3LPl6eKo_HbD3JNW1qzoaJ14Sgo,7533
+lidb/dataset.py,sha256=j3yFtokbNILVhjV-etAJunnbgfxYAu68Dkr2cgtCYSc,15766
+lidb/init.py,sha256=jLHpeL5mIM4YjdMYAndZlDilMiKXJMr_51Ke3ZSJWCM,1170
+lidb/parse.py,sha256=f7vfj6Nguw1WzUVEUb7fs2Oh-_2YQzB_atJhm3WGC28,3379
+lidb/table.py,sha256=-85U2N1ECDtZTTCJtgOM8XBKyueIgBmYRF5DocPvkh8,4167
+lidb/qdf/__init__.py,sha256=gYiSxijoPQZmbgATQX4GsutjolPpN82Kea0eQz6zGyg,1037
+lidb/qdf/errors.py,sha256=lJhhjDRdQOOKUFGlLQ9ELK4AexXBwYQSYus_V-kc5K8,1180
+lidb/qdf/expr.py,sha256=kBzXwjL_PVsJUL9FIHJ2W_G_OVRqFR-kS2mUHTt9thM,10412
+lidb/qdf/lazy.py,sha256=I08IvSkSC84qJkgtZ7nwvG_4UH07jaHBKRp7qQnwqbs,6937
+lidb/qdf/lazy2.py,sha256=ADKQaxmo-BlndhLY-idWCFypZF1icxKNHNMWEfmWy-Q,6294
+lidb/qdf/qdf.py,sha256=tfPnnQvh8uQZT4aOqJi6bDyDoJwLObvQrFeM2Ilz6vM,6236
+lidb/qdf/udf/__init__.py,sha256=yIySmkWjtJ-Lj_PMP5O4EnXGDjMAPQL40NmFCekKXBw,313
+lidb/qdf/udf/base_udf.py,sha256=ZjRF2UIrZFgznbm1gxFpdf4V92oO84IaakLeeSNF44U,3444
+lidb/qdf/udf/cs_udf.py,sha256=qlBZd2c1enIdGp_DrNyQWzH3cth4ZpLBIE1hGZuJXbA,3528
+lidb/qdf/udf/d_udf.py,sha256=SYfuI_HzKoxKP6iPwm94HRqerzl-JeZzAtzHZpdKdZw,5614
+lidb/qdf/udf/itd_udf.py,sha256=O_OOdSTEaeCoqjtlKnpvNF-_10QoamJL_tw2xEZCYVw,6747
+lidb/qdf/udf/ts_udf.py,sha256=Ag6-ffhmIugkA-st2QY-GP4hclQZcRG8SB-bVa7k5cc,5674
+lidb/svc/__init__.py,sha256=9vQo7gCm5LRgWSiq_UU2hlbwvXi0FlGYt2UDVZixx_U,141
+lidb/svc/data.py,sha256=tLOI_YylnsVejyqv9l-KgPetkPO0QzybOf1PEeFSZNI,4380
+lidb-1.2.0.dist-info/METADATA,sha256=fj1SvELa0jivjl6dcyut8IHbE7V00h5o6mGJkZa04S0,506
+lidb-1.2.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lidb-1.2.0.dist-info/top_level.txt,sha256=NgXJNwt6ld6oLXtW1vOPaEh-VO5R0JEX_KmGIJR4ueE,5
+lidb-1.2.0.dist-info/RECORD,,

lidb-1.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

lidb-1.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+lidb