ygo 1.0.10__py3-none-any.whl → 1.1.0__py3-none-any.whl

ycat/updator.py

@@ -1,101 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
----------------------------------------------
-Created on 2025/5/23 01:34
-@author: ZhangYundi
-@email: yundi.xxii@outlook.com
----------------------------------------------
-"""
-import os
-from datetime import datetime, timedelta
-from pathlib import Path
-
-import ygo
-import ylog
-from .client import CATDB
-
-DATE_FORMAT = "%Y-%m-%d"
-
-
-class Updator:
-    """
-    数据更新器
-    路径：{ycat.CATDB}/updator/{name}
-    """
-
-    def __init__(self, name: str, update_time="16:30"):
-        """
-        数据更新器
-        :param name: 数据更新器名称
-        :param update_time: 数据更新时间，默认16:30
-        """
-        self.name = name
-        self._tb_path = Path(CATDB) / "updator" / name
-        os.makedirs(self._tb_path, exist_ok=True)
-        self._update_time = update_time
-        self.present = datetime.now().today()
-
-        if self.present.strftime("%H:%M") >= self._update_time:
-            self.last_date = self.present.strftime(DATE_FORMAT)
-        else:
-            self.last_date = (self.present - timedelta(days=1)).strftime(DATE_FORMAT)
-
-        self._tasks = list()
-        self._last_run_file = self._tb_path / f".last_run"
-        self.logger = ylog.get_logger("updator")
-
-    @property
-    def last_update_date(self):
-        return self._read_last_run_date()
-
-    def _read_last_run_date(self):
-        if self._last_run_file.exists():
-            with open(self._last_run_file, "r") as f:
-                return f.read().strip()
-        return
-
-    def _write_last_run_date(self, date_str: str):
-        with open(self._last_run_file, "w") as f:
-            f.write(date_str)
-
-    def wrap_fn(self, task_name: str, update_fn: callable):
-        """包装函数，添加异常处理"""
-        try:
-            update_fn()
-            return 0
-        except Exception as e:
-            self.logger.error(ygo.FailTaskError(task_name=task_name, error=e))
-            return 1
-
-    def add_task(self, task_name: str, update_fn: callable):
-        """添加任务"""
-        self._tasks.append((task_name, ygo.delay(self.wrap_fn)(task_name=task_name, update_fn=update_fn)))
-
-    def do(self,
-           overwrite: bool = False,
-           n_jobs: int = 10,
-           backend: str = "threading"):
-        """
-        执行任务
-        :param overwrite: 是否覆盖现有数据
-        :param n_jobs: 并发数
-        :param backend: loky/threading/multiprocessing
-        :return:
-        """
-        if not overwrite:
-            local_last_date = self._read_last_run_date()
-            if local_last_date is not None:
-                if local_last_date >= self.last_date:
-                    self.logger.info(f"[{self.name}] 已是最新数据，跳过更新")
-                    return
-        self.logger.info(f"[{self.name}] 更新数据")
-        failed_num = 0
-        with ygo.pool(n_jobs=n_jobs, backend=backend) as go:
-            for task_name, task in self._tasks:
-                go.submit(task, job_name=task_name)()
-            for status in go.do():
-                failed_num += status
-        if failed_num < 1:
-            self._write_last_run_date(self.last_date)
-            self.logger.info(f"[{self.name}] 更新成功，最新数据日期：{self.last_date}")
-        self.logger.info(f"[{self.name}] 更新完成，失败任务数：{str(failed_num).zfill(2)}/{str(len(self._tasks)).zfill(2)}")

ygo-1.0.10.dist-info/METADATA

@@ -1,102 +0,0 @@
-Metadata-Version: 2.4
-Name: ygo
-Version: 1.0.10
-Project-URL: homepage, https://github.com/link-yundi/ygo
-Project-URL: repository, https://github.com/link-yundi/ygo
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: clickhouse-df>=0.1.5
-Requires-Dist: clickhouse-driver>=0.2.9
-Requires-Dist: connectorx>=0.3.3
-Requires-Dist: dynaconf>=3.2.11
-Requires-Dist: exchange-calendars>=4.2.8
-Requires-Dist: joblib>=1.4.2
-Requires-Dist: lark>=1.2.2
-Requires-Dist: lightgbm>=4.6.0
-Requires-Dist: loguru>=0.7.3
-Requires-Dist: mlflow>=2.17.2
-Requires-Dist: pandas>=2.0.3
-Requires-Dist: polars>=1.8.2
-Requires-Dist: pyarrow>=17.0.0
-Requires-Dist: pymysql>=1.1.1
-Requires-Dist: sqlalchemy>=2.0.40
-Requires-Dist: sqlparse>=0.5.3
-Requires-Dist: toolz>=1.0.0
-Requires-Dist: torch>=2.5.1
-Requires-Dist: tqdm>=4.67.1
-Dynamic: license-file
-
-# ygo
-并发执行(加入进度条)以及延迟调用（基于joblib），以及获取对应函数的相关信息
-
-### 安装
-```shell
-pip install -U git+https://github.com/link-yundi/ygo.git
-```
-
-### 示例
-
-```
-├── a
-│   ├── __init__.py
-│   └── b
-│       ├── __init__.py
-│       └── c.py
-└── test.py
-
-c.py 中定义了目标函数
-def test_fn(a, b=2):
-    return a+b
-```
-
-#### 场景1: 并发
-
-```python
-import ygo
-import ylog
-from a.b.c import test_fn
-
-with ygo.pool(job_name="test parallel", show_progress=True) as go:
-    for i in range(10):
-        go.submit(test_fn)(a=i, b=2*i)
-    for res in go.do():
-        ylog.info(res)
-```
-
-#### 场景2: 延迟调用
-
-```
->>> fn = delay(test_fn)(a=1, b=2)
->>> fn()
-3
->>> # 逐步传递参数
->>> fn1 = delay(lambda a, b, c: a+b+c)(a=1)
->>> fn2 = delay(fn1)(b=2)
->>> fn2(c=3)
-6
->>> # 参数更改
->>> fn1 = delay(lambda a, b, c: a+b+c)(a=1, b=2)
->>> fn2 = delay(fn1)(c=3, b=5)
->>> fn2()
-9
-```
-
-#### 场景3: 获取目标函数信息
-
-```
->>> ygo.fn_info(test_fn)
-=============================================================
-    a.b.c.test_fn(a, b=2)
-=============================================================
-    def test_fn(a, b=2):
-    return a+b
-```
-
-#### 场景4: 通过字符串解析函数并执行
-
-```
->>> ygo.fn_from_str("a.b.c.test_fn")(a=1, b=5)
-6
-```
-

ygo-1.0.10.dist-info/RECORD

@@ -1,24 +0,0 @@
-ycat/__init__.py,sha256=8cVsQXL476hDIU_yDxfoAC9ILC4_Tnp1Y1dDTs6vkXA,529
-ycat/client.py,sha256=Z5lwybzXv6MADbbooDT-rhdr9JMI_t2TDAXt0ghongQ,5451
-ycat/parse.py,sha256=piz_iciF7JFnn8v_qtUEHP6NZ_MWJidsA5gBpLtuZzw,2195
-ycat/updator.py,sha256=dV2vhYVlaoK8eRQggAkNkZuu9zbOYFajicyggMt-r78,3393
-ycat/qdf/__init__.py,sha256=8fIBr0FUAYGc33CYnWILY0Ur2DXdctWjw28S5qDWhD4,7572
-ycat/qdf/errors.py,sha256=lJhhjDRdQOOKUFGlLQ9ELK4AexXBwYQSYus_V-kc5K8,1180
-ycat/qdf/expr.py,sha256=ck_BHMCV29Q8-szci1_v4ud964QI7JoRRcmA0ppupsc,8454
-ycat/qdf/qdf.py,sha256=XcnGyyfuRY1HqaG56kC5tB6psrIXqo9QVQtgH3mhips,7322
-ycat/qdf/udf/__init__.py,sha256=DdrSGaCB__5C1YL0vd_5rjIB3KLrAKn3h3k9k50L0jA,313
-ycat/qdf/udf/base_udf.py,sha256=6VDaCIGNLJxZ7UsoIDWtTH6PzUDj89b8FiwN-TEat2g,3437
-ycat/qdf/udf/cs_udf.py,sha256=HT3EKBwAhOxOFDQnpfwb4YcMTT3-lqFXkdysdn5_FI4,3179
-ycat/qdf/udf/d_udf.py,sha256=L9mkX6yDpQPwXvQTAebDepjEkirCqur1DfV2Fnl8KA0,5352
-ycat/qdf/udf/ind_udf.py,sha256=hDCKfcLFCgIhdC9dQ5GYxLemZaOE6K7kQyAnjUrwePM,6482
-ycat/qdf/udf/ts_udf.py,sha256=uUuZnKMY-V_uInP0nsBMblDpxY3ld3EwvrXTwWMqeig,5410
-ygo/__init__.py,sha256=kQK7CwVCz8NJTj5eS9Xrt_G1kPHvDIbe2sTzHgWITxI,590
-ygo/exceptions.py,sha256=0OYDYt_9KKo8mF2XBG5QkCMr3-ASp69VDSPOEwlIsrI,660
-ygo/ygo.py,sha256=kcXI5vzndNOJqEEEZOeWbn61O47gW72UDiUWN1v9AYc,11290
-ygo-1.0.10.dist-info/licenses/LICENSE,sha256=6AKUWQ1xe-jwPSFv_H6FMQLNNWb7AYqzuEUTwlP2S8M,1067
-ylog/__init__.py,sha256=AoRCQ-o4gWAcJ8svw30wM5UJyccx45WhYIndrrkNv8o,428
-ylog/core.py,sha256=d6QCFRDTvlyxgvS6JphUGOgX5Mgx9qPv9wB3g-4YOJw,9225
-ygo-1.0.10.dist-info/METADATA,sha256=J0M_naeBSfTGAcep-85CtjOz-Tz0z2l8h_VKd7RLIq4,2235
-ygo-1.0.10.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-ygo-1.0.10.dist-info/top_level.txt,sha256=jEbfiz5fX4iSzDg8_Npdv5SIC_Kphmb1m3vuyD9ZC1E,14
-ygo-1.0.10.dist-info/RECORD,,