expr-tracker 0.1.7__py3-none-any.whl

expr_tracker/__init__.py

@@ -0,0 +1,9 @@
+from . import tracker
+from .alert import alert
+
+init = tracker.init
+finish = tracker.finish
+log = tracker.log
+info = tracker.info
+
+__all__ = ["init", "finish", "log", "info", "alert"]

expr_tracker/_compat.py

@@ -0,0 +1,31 @@
+from typing import Any, Type
+
+from pydantic import BaseModel
+from pydantic.version import VERSION as PYDANTIC_VERSION
+from typing_extensions import Literal
+
+PYDANTIC_VERSION_MINOR_TUPLE = tuple(int(x) for x in PYDANTIC_VERSION.split(".")[:2])
+PYDANTIC_V2 = PYDANTIC_VERSION_MINOR_TUPLE[0] == 2
+
+Url: Type[Any]
+
+if PYDANTIC_V2:
+    from pydantic_core import PydanticUndefinedType
+    from pydantic_core import Url as Url
+
+    UndefinedType = PydanticUndefinedType
+
+    def _model_dump(
+        model: BaseModel, mode: Literal["json", "python"] = "json", **kwargs: Any
+    ) -> Any:
+        return model.model_dump(mode=mode, **kwargs)
+else:
+    from pydantic import AnyUrl as Url  # noqa: F401
+    from pydantic.fields import (  # type: ignore[no-redef, attr-defined]
+        UndefinedType as UndefinedType,  # noqa: F401
+    )
+
+    def _model_dump(
+        model: BaseModel, mode: Literal["json", "python"] = "json", **kwargs: Any
+    ) -> Any:
+        return model.dict(**kwargs)

expr_tracker/alert.py

@@ -0,0 +1,73 @@
+from typing import Literal
+from loguru import logger
+
+
+def ignore_exception(func):
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            logger.warning(f"Error occurred in {func.__name__}: {e}")
+            return None
+
+    return wrapper
+
+
+class LarkBackend:
+    def __init__(self, **kwargs):
+        from slark import Lark
+
+        self.lark = Lark(**kwargs)
+
+    def publish_alert(
+        self,
+        title: str,
+        text: str,
+        subtitle: str | None = None,
+        level: Literal["info", "warning", "error"] = "info",
+        traceback: str | None = None,
+    ):
+        if level == "info" or level == "warning":
+            self.lark.webhook.post_success_card(
+                msg=text, title=title, subtitle=subtitle
+            )
+        elif level == "error":
+            self.lark.webhook.post_error_card(
+                msg=text, traceback=traceback or "", title=title, subtitle=subtitle
+            )
+        else:
+            raise ValueError(
+                f"Unsupported level: {level}. Supported levels are 'info', 'warning', and 'error'."
+            )
+
+
+@ignore_exception
+def alert(
+    title: str,
+    text: str,
+    subtitle: str | None = None,
+    traceback: str | None = None,
+    level: Literal["info", "warning", "error"] = "info",
+    backends: list[Literal["lark", "slack", "email"]] = ["lark"],
+):
+    """
+    Send an alert message to specified backends.
+
+    Args:
+        title (str): The title of the alert.
+        text (str): The main content of the alert.
+        subtitle (str | None): Optional subtitle for the alert.
+        level (Literal["info", "warning", "error"]): The severity level of the alert.
+        backends (list[Literal["lark", "slack", "email"]]): List of backends to send the alert to.
+    """
+    for backend in backends:
+        if backend == "lark":
+            lark_backend = LarkBackend()
+            if lark_backend.lark._webhook_url is None:
+                logger.warning(
+                    "Lark webhook URL is not set. Please set the WEBHOOK_URL environment variable."
+                )
+                return
+            lark_backend.publish_alert(title, text, subtitle, level, traceback)
+        else:
+            raise NotImplementedError(f"Backend '{backend}' is not implemented.")

expr_tracker/cli.py

@@ -0,0 +1,15 @@
+import click
+import expr_tracker as et
+
+
+@click.group()
+def main():
+    pass
+
+
+@main.command()
+@click.argument("msg")
+@click.option("--title", default="Alert", help="Title of the alert")
+@click.option("--level", default="info", help="Level of the alert")
+def alert(msg: str, title: str = "Alert", level: str = "info"):
+    et.alert(title=title, text=msg, level=level)

expr_tracker/encoders.py

@@ -0,0 +1,344 @@
+import dataclasses
+import datetime
+from collections import defaultdict, deque
+from decimal import Decimal
+from enum import Enum
+from ipaddress import (
+    IPv4Address,
+    IPv4Interface,
+    IPv4Network,
+    IPv6Address,
+    IPv6Interface,
+    IPv6Network,
+)
+from pathlib import Path, PurePath
+from re import Pattern
+from types import GeneratorType
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
+from uuid import UUID
+
+from expr_tracker.types import IncEx
+from pydantic import BaseModel
+from pydantic.color import Color
+from pydantic.networks import AnyUrl, NameEmail
+from pydantic.types import SecretBytes, SecretStr
+from typing_extensions import Annotated, Doc
+
+from ._compat import PYDANTIC_V2, UndefinedType, Url, _model_dump
+
+
+# Taken from Pydantic v1 as is
+def isoformat(o: Union[datetime.date, datetime.time]) -> str:
+    return o.isoformat()
+
+
+# Taken from Pydantic v1 as is
+# TODO: pv2 should this return strings instead?
+def decimal_encoder(dec_value: Decimal) -> Union[int, float]:
+    """
+    Encodes a Decimal as int of there's no exponent, otherwise float
+
+    This is useful when we use ConstrainedDecimal to represent Numeric(x,0)
+    where a integer (but not int typed) is used. Encoding this as a float
+    results in failed round-tripping between encode and parse.
+    Our Id type is a prime example of this.
+
+    >>> decimal_encoder(Decimal("1.0"))
+    1.0
+
+    >>> decimal_encoder(Decimal("1"))
+    1
+    """
+    if dec_value.as_tuple().exponent >= 0:  # type: ignore[operator]
+        return int(dec_value)
+    else:
+        return float(dec_value)
+
+
+ENCODERS_BY_TYPE: Dict[Type[Any], Callable[[Any], Any]] = {
+    bytes: lambda o: o.decode(),
+    Color: str,
+    datetime.date: isoformat,
+    datetime.datetime: isoformat,
+    datetime.time: isoformat,
+    datetime.timedelta: lambda td: td.total_seconds(),
+    Decimal: decimal_encoder,
+    Enum: lambda o: o.value,
+    frozenset: list,
+    deque: list,
+    GeneratorType: list,
+    IPv4Address: str,
+    IPv4Interface: str,
+    IPv4Network: str,
+    IPv6Address: str,
+    IPv6Interface: str,
+    IPv6Network: str,
+    NameEmail: str,
+    Path: str,
+    Pattern: lambda o: o.pattern,
+    SecretBytes: str,
+    SecretStr: str,
+    set: list,
+    UUID: str,
+    Url: str,
+    AnyUrl: str,
+}
+
+
+def generate_encoders_by_class_tuples(
+    type_encoder_map: Dict[Any, Callable[[Any], Any]],
+) -> Dict[Callable[[Any], Any], Tuple[Any, ...]]:
+    encoders_by_class_tuples: Dict[Callable[[Any], Any], Tuple[Any, ...]] = defaultdict(
+        tuple
+    )
+    for type_, encoder in type_encoder_map.items():
+        encoders_by_class_tuples[encoder] += (type_,)
+    return encoders_by_class_tuples
+
+
+encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE)
+
+
+def jsonable_encoder(
+    obj: Annotated[
+        Any,
+        Doc(
+            """
+            The input object to convert to JSON.
+            """
+        ),
+    ],
+    include: Annotated[
+        Optional[IncEx],
+        Doc(
+            """
+            Pydantic's `include` parameter, passed to Pydantic models to set the
+            fields to include.
+            """
+        ),
+    ] = None,
+    exclude: Annotated[
+        Optional[IncEx],
+        Doc(
+            """
+            Pydantic's `exclude` parameter, passed to Pydantic models to set the
+            fields to exclude.
+            """
+        ),
+    ] = None,
+    by_alias: Annotated[
+        bool,
+        Doc(
+            """
+            Pydantic's `by_alias` parameter, passed to Pydantic models to define if
+            the output should use the alias names (when provided) or the Python
+            attribute names. In an API, if you set an alias, it's probably because you
+            want to use it in the result, so you probably want to leave this set to
+            `True`.
+            """
+        ),
+    ] = True,
+    exclude_unset: Annotated[
+        bool,
+        Doc(
+            """
+            Pydantic's `exclude_unset` parameter, passed to Pydantic models to define
+            if it should exclude from the output the fields that were not explicitly
+            set (and that only had their default values).
+            """
+        ),
+    ] = False,
+    exclude_defaults: Annotated[
+        bool,
+        Doc(
+            """
+            Pydantic's `exclude_defaults` parameter, passed to Pydantic models to define
+            if it should exclude from the output the fields that had the same default
+            value, even when they were explicitly set.
+            """
+        ),
+    ] = False,
+    exclude_none: Annotated[
+        bool,
+        Doc(
+            """
+            Pydantic's `exclude_none` parameter, passed to Pydantic models to define
+            if it should exclude from the output any fields that have a `None` value.
+            """
+        ),
+    ] = False,
+    custom_encoder: Annotated[
+        Optional[Dict[Any, Callable[[Any], Any]]],
+        Doc(
+            """
+            Pydantic's `custom_encoder` parameter, passed to Pydantic models to define
+            a custom encoder.
+            """
+        ),
+    ] = None,
+    sqlalchemy_safe: Annotated[
+        bool,
+        Doc(
+            """
+            Exclude from the output any fields that start with the name `_sa`.
+
+            This is mainly a hack for compatibility with SQLAlchemy objects, they
+            store internal SQLAlchemy-specific state in attributes named with `_sa`,
+            and those objects can't (and shouldn't be) serialized to JSON.
+            """
+        ),
+    ] = True,
+) -> Any:
+    """
+    Convert any object to something that can be encoded in JSON.
+
+    This is used internally by FastAPI to make sure anything you return can be
+    encoded as JSON before it is sent to the client.
+
+    You can also use it yourself, for example to convert objects before saving them
+    in a database that supports only JSON.
+
+    Read more about it in the
+    [FastAPI docs for JSON Compatible Encoder](https://fastapi.tiangolo.com/tutorial/encoder/).
+    """
+    custom_encoder = custom_encoder or {}
+    if custom_encoder:
+        if type(obj) in custom_encoder:
+            return custom_encoder[type(obj)](obj)
+        else:
+            for encoder_type, encoder_instance in custom_encoder.items():
+                if isinstance(obj, encoder_type):
+                    return encoder_instance(obj)
+    if include is not None and not isinstance(include, (set, dict)):
+        include = set(include)
+    if exclude is not None and not isinstance(exclude, (set, dict)):
+        exclude = set(exclude)
+    if isinstance(obj, BaseModel):
+        # TODO: remove when deprecating Pydantic v1
+        encoders: Dict[Any, Any] = {}
+        if not PYDANTIC_V2:
+            encoders = getattr(obj.__config__, "json_encoders", {})  # type: ignore[attr-defined]
+            if custom_encoder:
+                encoders = {**encoders, **custom_encoder}
+        obj_dict = _model_dump(
+            obj,
+            mode="json",
+            include=include,
+            exclude=exclude,
+            by_alias=by_alias,
+            exclude_unset=exclude_unset,
+            exclude_none=exclude_none,
+            exclude_defaults=exclude_defaults,
+        )
+        if "__root__" in obj_dict:
+            obj_dict = obj_dict["__root__"]
+        return jsonable_encoder(
+            obj_dict,
+            exclude_none=exclude_none,
+            exclude_defaults=exclude_defaults,
+            # TODO: remove when deprecating Pydantic v1
+            custom_encoder=encoders,
+            sqlalchemy_safe=sqlalchemy_safe,
+        )
+    if dataclasses.is_dataclass(obj):
+        assert not isinstance(obj, type)
+        obj_dict = dataclasses.asdict(obj)
+        return jsonable_encoder(
+            obj_dict,
+            include=include,
+            exclude=exclude,
+            by_alias=by_alias,
+            exclude_unset=exclude_unset,
+            exclude_defaults=exclude_defaults,
+            exclude_none=exclude_none,
+            custom_encoder=custom_encoder,
+            sqlalchemy_safe=sqlalchemy_safe,
+        )
+    if isinstance(obj, Enum):
+        return obj.value
+    if isinstance(obj, PurePath):
+        return str(obj)
+    if isinstance(obj, (str, int, float, type(None))):
+        return obj
+    if isinstance(obj, UndefinedType):
+        return None
+    if isinstance(obj, dict):
+        encoded_dict = {}
+        allowed_keys = set(obj.keys())
+        if include is not None:
+            allowed_keys &= set(include)
+        if exclude is not None:
+            allowed_keys -= set(exclude)
+        for key, value in obj.items():
+            if (
+                (
+                    not sqlalchemy_safe
+                    or (not isinstance(key, str))
+                    or (not key.startswith("_sa"))
+                )
+                and (value is not None or not exclude_none)
+                and key in allowed_keys
+            ):
+                encoded_key = jsonable_encoder(
+                    key,
+                    by_alias=by_alias,
+                    exclude_unset=exclude_unset,
+                    exclude_none=exclude_none,
+                    custom_encoder=custom_encoder,
+                    sqlalchemy_safe=sqlalchemy_safe,
+                )
+                encoded_value = jsonable_encoder(
+                    value,
+                    by_alias=by_alias,
+                    exclude_unset=exclude_unset,
+                    exclude_none=exclude_none,
+                    custom_encoder=custom_encoder,
+                    sqlalchemy_safe=sqlalchemy_safe,
+                )
+                encoded_dict[encoded_key] = encoded_value
+        return encoded_dict
+    if isinstance(obj, (list, set, frozenset, GeneratorType, tuple, deque)):
+        encoded_list = []
+        for item in obj:
+            encoded_list.append(
+                jsonable_encoder(
+                    item,
+                    include=include,
+                    exclude=exclude,
+                    by_alias=by_alias,
+                    exclude_unset=exclude_unset,
+                    exclude_defaults=exclude_defaults,
+                    exclude_none=exclude_none,
+                    custom_encoder=custom_encoder,
+                    sqlalchemy_safe=sqlalchemy_safe,
+                )
+            )
+        return encoded_list
+
+    if type(obj) in ENCODERS_BY_TYPE:
+        return ENCODERS_BY_TYPE[type(obj)](obj)
+    for encoder, classes_tuple in encoders_by_class_tuples.items():
+        if isinstance(obj, classes_tuple):
+            return encoder(obj)
+
+    try:
+        data = dict(obj)
+    except Exception as e:
+        errors: List[Exception] = []
+        errors.append(e)
+        try:
+            data = vars(obj)
+        except Exception as e:
+            errors.append(e)
+            raise ValueError(errors) from e
+    return jsonable_encoder(
+        data,
+        include=include,
+        exclude=exclude,
+        by_alias=by_alias,
+        exclude_unset=exclude_unset,
+        exclude_defaults=exclude_defaults,
+        exclude_none=exclude_none,
+        custom_encoder=custom_encoder,
+        sqlalchemy_safe=sqlalchemy_safe,
+    )

expr_tracker/jsonl.py

@@ -0,0 +1,209 @@
+import atexit
+import json
+import threading
+import time
+from pathlib import Path
+
+import jsonlines
+from loguru import logger
+
+from expr_tracker.encoders import jsonable_encoder
+
+DEFAULT_BUFFER_SIZE = 50
+DEFAULT_BUFFER_INTERVAL = 1.0
+DEFAULT_MAX_BUFFER_SECONDS = 5.0
+
+
+class JsonlTracker:
+    def __init__(self):
+        self.buffer = []
+        self.buffer_size = DEFAULT_BUFFER_SIZE
+        self.buffer_interval = DEFAULT_BUFFER_INTERVAL
+        self.max_buffer_seconds = DEFAULT_MAX_BUFFER_SECONDS
+        self.log_fp = None
+        self._lock = threading.RLock()
+        self._last_log_time = None
+        self._first_buffered_time = None
+        self._flush_timer = None
+
+    def init(
+        self,
+        project: str,
+        name: str | None = None,
+        config: dict | None = None,
+        dir: str | None = None,
+        print_to_screen: bool = False,
+        print_handle=print,
+        buffer_size: int = DEFAULT_BUFFER_SIZE,
+        buffer_interval: float | None = DEFAULT_BUFFER_INTERVAL,
+        max_buffer_seconds: float | None = DEFAULT_MAX_BUFFER_SECONDS,
+        **kwargs,
+    ):
+        """初始化 jsonl backend。
+
+        缓冲策略（按 log 频率自适应）：
+        - ``buffer_size``: buffer 中记录数达到该值立即写盘。
+        - ``buffer_interval``: 相邻两次 ``log()`` 的间隔 >= 该值时认为不是高频写入，
+          直接写盘（低延迟）；小于该值才认为是高频写入，先攒在内存里。
+          设为 ``None`` 表示关闭该判断（只按 buffer_size 攒批）。
+        - ``max_buffer_seconds``: 记录在 buffer 中的最长停留时间，超时后由后台定时器
+          强制写盘，避免高频写入突然停止时数据长期滞留内存。设为 ``None`` 关闭。
+        """
+        self.project = project
+        self.name = name
+        if dir is None:
+            dir = "./tracker/jsonl"
+        self.log_dir = Path(dir) / self.project / self.name
+        self.config_fp = self.log_dir / "config.json"
+        self.log_fp = self.log_dir / "metrics.jsonl"
+
+        # 初始化 Buffer 配置
+        self._cancel_timer()
+        with self._lock:
+            self.buffer = []
+            self.buffer_size = max(1, int(buffer_size))
+            self.buffer_interval = (
+                None if buffer_interval is None else float(buffer_interval)
+            )
+            self.max_buffer_seconds = (
+                None if max_buffer_seconds is None else float(max_buffer_seconds)
+            )
+            self._last_log_time = None
+            self._first_buffered_time = None
+
+        self.log_dir.mkdir(parents=True, exist_ok=True)
+
+        if self.config_fp.exists():
+            logger.warning(
+                f"Config file {self.config_fp} already exists. It will be overwritten."
+            )
+
+        if config is not None:
+            # Config 通常只写一次，直接写入即可
+            with open(self.config_fp, "w") as f:
+                json.dump(jsonable_encoder(config), f, indent=4)
+
+        self.print_to_screen = print_to_screen
+        self.print_handle = print_handle
+
+        # 优化：流式计算行数，避免一次性加载大文件到内存 (对 BlobFuse 友好)
+        self.current_step = 0
+        if self.log_fp.exists():
+            try:
+                with open(self.log_fp, "rb") as f:
+                    self.current_step = sum(1 for _ in f)
+            except Exception as e:
+                logger.warning(f"Could not count existing lines in {self.log_fp}: {e}")
+
+        # 注册退出钩子：确保程序意外终止时也能写入剩余数据
+        atexit.register(self.flush)
+
+    def log(self, metrics: dict, step: int | None = None):
+        now = time.monotonic()
+
+        with self._lock:
+            if step is not None:
+                self.current_step = step
+
+            record = {"_step": self.current_step, **metrics}
+
+            # 1. 写入内存 Buffer
+            self.buffer.append(record)
+            if self._first_buffered_time is None:
+                self._first_buffered_time = now
+
+            # 2. 根据「距离上一次 log 的时间间隔」判断是否为高频写入
+            interval = None if self._last_log_time is None else now - self._last_log_time
+            self._last_log_time = now
+
+            should_flush = self._should_flush(now, interval)
+            self.current_step += 1
+
+        # 3. 屏幕打印（放在锁外，避免 print handle 阻塞其他线程）
+        if self.print_to_screen:
+            self.print_handle(f"{record}")
+
+        # 4. 立即写盘，或安排一次超时写盘
+        if should_flush:
+            self.flush()
+        else:
+            self._schedule_timer()
+
+    def _should_flush(self, now: float, interval: float | None) -> bool:
+        """判断当前是否应该立即写盘（需在持锁状态下调用）"""
+        # Buffer 已满
+        if len(self.buffer) >= self.buffer_size:
+            return True
+        # 首次 log：直接落盘，尽快产生文件内容
+        if interval is None:
+            return True
+        # 低频写入：距离上次 log 已经过去足够久，没必要继续攒批
+        if self.buffer_interval is not None and interval >= self.buffer_interval:
+            return True
+        # 高频写入，但最早的记录已在内存中停留过久
+        return (
+            self.max_buffer_seconds is not None
+            and self._first_buffered_time is not None
+            and now - self._first_buffered_time >= self.max_buffer_seconds
+        )
+
+    def _schedule_timer(self):
+        """为 buffer 中最早的记录安排一次超时写盘"""
+        if self.max_buffer_seconds is None:
+            return
+        with self._lock:
+            if self._flush_timer is not None or not self.buffer:
+                return
+            now = time.monotonic()
+            elapsed = now - (self._first_buffered_time or now)
+            delay = max(0.0, self.max_buffer_seconds - elapsed)
+            timer = threading.Timer(delay, self._on_timer)
+            timer.daemon = True
+            self._flush_timer = timer
+        timer.start()
+
+    def _on_timer(self):
+        with self._lock:
+            self._flush_timer = None
+        self.flush()
+
+    def _cancel_timer(self):
+        with self._lock:
+            timer = self._flush_timer
+            self._flush_timer = None
+        if timer is not None:
+            timer.cancel()
+
+    def flush(self):
+        """强制将内存中的 Buffer 写入磁盘"""
+        self._cancel_timer()
+
+        with self._lock:
+            self._first_buffered_time = None
+            if not self.buffer:
+                return
+            records, self.buffer = self.buffer, []
+            log_fp = self.log_fp
+
+        # 确保目录存在 (防止运行时目录被删)
+        if log_fp and not log_fp.parent.exists():
+            log_fp.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            # 批量追加写入
+            with jsonlines.open(log_fp, mode="a") as writer:
+                writer.write_all(records)
+        except Exception as e:
+            logger.error(f"Failed to flush metrics to {log_fp}: {e}")
+            # 写入失败：放回 buffer 头部，等待下次 flush 重试
+            with self._lock:
+                self.buffer[:0] = records
+                self._first_buffered_time = time.monotonic()
+            self._schedule_timer()
+
+    def finish(self):
+        """结束时显式调用"""
+        self._cancel_timer()
+        self.flush()
+        # 如果手动调用了 finish，取消 atexit 注册，防止重复调用
+        atexit.unregister(self.flush)

expr_tracker/tracker.py

@@ -0,0 +1,167 @@
+import os
+from contextvars import ContextVar
+from typing import Literal
+
+from loguru import logger
+
+_tracker: ContextVar["Tracker | None"] = ContextVar("tracker", default=None)
+
+
+def get_backend(backend: str | object):
+    if isinstance(backend, str):
+        backend = backend.lower()
+        if backend == "wandb":
+            try:
+                import wandb
+
+                wandb.login(
+                    key=os.getenv("WANDB_API_KEY", None),
+                    host=os.getenv("WANDB_HOST", None),
+                )
+                return wandb
+            except ImportError:
+                raise ImportError("WandB backend is not installed.")
+        elif backend == "jsonl":
+            from .jsonl import JsonlTracker
+
+            return JsonlTracker()
+        elif backend == "trackio":
+            try:
+                import trackio
+
+                return trackio
+            except ImportError:
+                raise ImportError("TrackIO backend is not installed.")
+    else:
+        return backend
+
+
+class Tracker:
+    def __init__(
+        self,
+        project: str,
+        name: str | None = None,
+        entity: str | None = None,
+        dir: str | None = None,
+        notes: str | None = None,
+        tags: list[str] | None = None,
+        resume: bool | Literal["allow", "never", "must", "auto"] | None = "allow",
+        config: dict | None = None,
+        backends: list[Literal["wandb", "jsonl", "trackio"]] = [
+            "wandb",
+            "trackio",
+            "jsonl",
+        ],
+        backend_kwargs: dict[str, dict] | None = None,
+        **kwargs,
+    ):
+        if kwargs:
+            logger.info(
+                f"Unrecognized keyword arguments passed to Tracker: {kwargs}. "
+                "These will be ignored."
+            )
+        self.backend = {b: get_backend(b) for b in backends}
+        if config is None:
+            config = {}
+        if backend_kwargs is None:
+            backend_kwargs = {}
+        for b, t in self.backend.items():
+            if b == "trackio":
+                t.init(
+                    project=project,
+                    name=name,
+                    config=config.update(
+                        {
+                            "trackio.notes": notes,
+                            "trackio.tags": tags,
+                            "trackio.entity": entity,
+                            "trackio.resume": resume,
+                        }
+                    ),
+                    **backend_kwargs.get(b, {}),
+                )
+            else:
+                t.init(
+                    project=project,
+                    name=name,
+                    entity=entity,
+                    dir=dir,
+                    notes=notes,
+                    tags=tags,
+                    resume=resume,
+                    id=name,
+                    config=config,
+                    **backend_kwargs.get(b, {}),
+                )
+
+
+def init(
+    project: str,
+    name: str | None = None,
+    entity: str | None = None,
+    dir: str | None = None,
+    notes: str | None = None,
+    tags: list[str] | None = None,
+    resume: bool | Literal["allow", "never", "must", "auto"] | None = "allow",
+    config: dict | None = None,
+    backends: list[Literal["wandb", "jsonl", "trackio"]] = ["wandb", "jsonl"],
+    backend_kwargs: dict[str, dict] | None = None,
+    **kwargs,
+) -> Tracker:
+    """
+    Initialize the tracker with the given parameters.
+    """
+    if _tracker.get() is not None:
+        raise RuntimeError("Tracker is already initialized. Call finish() first.")
+
+    tracker = Tracker(
+        project=project,
+        name=name,
+        entity=entity,
+        dir=dir,
+        notes=notes,
+        tags=tags,
+        resume=resume,
+        config=config,
+        backends=backends,
+        backend_kwargs=backend_kwargs,
+        **kwargs,
+    )
+    _tracker.set(tracker)
+    return tracker
+
+
+def log(metrics: dict, step: int | None = None):
+    tracker = _tracker.get()
+    if tracker is None:
+        raise RuntimeError("Tracker is not initialized. Call init() first.")
+    for backend, t in tracker.backend.items():
+        try:
+            t.log(metrics, step=step)
+        except Exception as e:
+            logger.warning(f"Failed to log metrics to {backend}: {e}")
+
+
+def finish():
+    tracker = _tracker.get()
+    if tracker is None:
+        raise RuntimeError("Tracker is not initialized. Call init() first.")
+    for backend, t in tracker.backend.items():
+        try:
+            t.finish()
+        except Exception as e:
+            logger.warning(f"Failed to finish tracker for {backend}: {e}")
+    _tracker.set(None)
+
+
+def info():
+    tracker = _tracker.get()
+    if tracker is None:
+        raise RuntimeError("Tracker is not initialized. Call init() first.")
+    info = {}
+    for backend, t in tracker.backend.items():
+        if backend == "wandb":
+            info["wandb"] = {"url": t.run.url}
+        if backend == "jsonl":
+            info["jsonl"] = {"log_dir": t.log_dir.as_posix()}
+    return info

expr_tracker/types.py

@@ -0,0 +1,10 @@
+import types
+from enum import Enum
+from typing import Any, Callable, Dict, Set, Type, TypeVar, Union
+
+from pydantic import BaseModel
+
+DecoratedCallable = TypeVar("DecoratedCallable", bound=Callable[..., Any])
+UnionType = getattr(types, "UnionType", Union)
+ModelNameMap = Dict[Union[Type[BaseModel], Type[Enum]], str]
+IncEx = Union[Set[int], Set[str], Dict[int, Any], Dict[str, Any]]

expr_tracker-0.1.7.dist-info/METADATA

@@ -0,0 +1,67 @@
+Metadata-Version: 2.4
+Name: expr_tracker
+Version: 0.1.7
+Summary: Add your description here
+Author-email: HSPK <whxway@whu.edu.cn>
+Requires-Python: >=3.10
+Requires-Dist: jsonlines>=4.0.0
+Requires-Dist: loguru>=0.7.3
+Requires-Dist: slark>=0.1.28
+Requires-Dist: wandb>=0.21.0
+Description-Content-Type: text/markdown
+
+# Experiment Tracker
+
+A simple experiment tracker that supports `wandb`, `trackio` and local `jsonl` storage. Features include:
+
+## Features
+1. **Multi-Backend Support**: Compatible with `wandb`, `trackio`, and local `jsonl` storage.
+2. **Alert System**: Send alerts via Lark, with email and other platforms to be added.
+3. **Resume Functionality**: Allows resuming experiments using project and name as unique identifiers.
+4. **Simple API**: Provides a straightforward API similar to `wandb`, making it easy to integrate into existing workflows.
+
+## Usage
+
+Add dependency to your project:
+
+```bash
+uv add expr_tracker
+```
+
+Simple usage example:
+```python
+import expr_tracker as et
+
+et.init(project="my_project", name="my_experiment", backends=["wandb", "jsonl"])
+et.log({"accuracy": 0.95, "loss": 0.05})
+et.alert("Experiment completed!", text="Your experiment has finished successfully.", subtitle="Experiment Status")
+et.finish()
+```
+
+### JSONL buffering
+
+The `jsonl` backend adapts its buffering to how often you call `log()`, so that
+high-frequency logging doesn't hammer the disk (helpful on network mounts such as
+BlobFuse) while low-frequency logging still lands on disk immediately:
+
+- If the gap since the previous `log()` call is `>= buffer_interval` (default `1.0s`),
+  the call is considered low-frequency and is written straight through.
+- Otherwise records are batched in memory and flushed once `buffer_size` (default `50`)
+  records accumulate.
+- A background timer flushes records that have been buffered for more than
+  `max_buffer_seconds` (default `5.0s`), so a burst that suddenly stops is never stranded
+  in memory. `finish()` and the `atexit` hook flush any remainder.
+
+Set `buffer_interval=None` to disable the frequency check, or `max_buffer_seconds=None`
+to disable the background timer. Tune them via `backend_kwargs`:
+
+```python
+et.init(
+    project="my_project",
+    name="my_experiment",
+    backends=["jsonl"],
+    backend_kwargs={
+        "jsonl": {"buffer_size": 200, "buffer_interval": 0.5, "max_buffer_seconds": 10},
+    },
+)
+```

expr_tracker-0.1.7.dist-info/RECORD

@@ -0,0 +1,12 @@
+expr_tracker/__init__.py,sha256=I-Y4K79ZSh0thhpQehfFVG2CvWpXKoQSwNFn05UN4uY,184
+expr_tracker/_compat.py,sha256=qf8oYIK_lkoLzcPp6wPCYLJtVatCl7ROYRNPwc5F1Pg,981
+expr_tracker/alert.py,sha256=6egNe-DGp1U96kjRJ8bN4sV3Se_kMKX5q-rSj9kW3Kc,2342
+expr_tracker/cli.py,sha256=Yhyh_rIjCjzkC2ri6q9ldVhCsTlrOlpp-jRgh4foyFU,368
+expr_tracker/encoders.py,sha256=OUsoS5Hd499zOZPb5QdojhUKFmCAGzYQD4OwEqSHvEg,11124
+expr_tracker/jsonl.py,sha256=OQHwuihKCbRFlUY-lPLduoeCcS8W4Y5wTLUpHyEBBh0,7646
+expr_tracker/tracker.py,sha256=iGdsr-KzjHcUJnQR6EgMwfgra53KwhimHfV-mVA89qQ,5023
+expr_tracker/types.py,sha256=nFb36sK3DSoqoyo7Miwy3meKK5UdFBgkAgLSzQlUVyI,383
+expr_tracker-0.1.7.dist-info/METADATA,sha256=61rVcbtCL5W75bzEFmedfa1XdHo_aGnhOancBdB0UPM,2379
+expr_tracker-0.1.7.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+expr_tracker-0.1.7.dist-info/entry_points.txt,sha256=GfZCAG2JFsd5CxTjRsGJAhYInpBCuLjrU3csmUVBDSs,45
+expr_tracker-0.1.7.dist-info/RECORD,,

expr_tracker-0.1.7.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

expr_tracker-0.1.7.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+et = expr_tracker.cli:main