speedy-utils 1.0.4__py3-none-any.whl → 1.0.9__py3-none-any.whl

speedy_utils-1.0.9.dist-info/METADATA

@@ -0,0 +1,287 @@
+Metadata-Version: 2.3
+Name: speedy-utils
+Version: 1.0.9
+Summary: Fast and easy-to-use package for data science
+Author: AnhVTH
+Author-email: anhvth.226@gmail.com
+Requires-Python: >=3.9
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: bump2version
+Requires-Dist: cachetools
+Requires-Dist: debugpy
+Requires-Dist: fastcore
+Requires-Dist: fastprogress
+Requires-Dist: freezegun (>=1.5.1,<2.0.0)
+Requires-Dist: ipdb
+Requires-Dist: ipywidgets
+Requires-Dist: json-repair (>=0.40.0,<0.41.0)
+Requires-Dist: jupyterlab
+Requires-Dist: loguru
+Requires-Dist: matplotlib
+Requires-Dist: numpy
+Requires-Dist: packaging (>=23.2,<25)
+Requires-Dist: pandas
+Requires-Dist: pydantic
+Requires-Dist: requests
+Requires-Dist: scikit-learn
+Requires-Dist: tabulate
+Requires-Dist: tqdm
+Requires-Dist: xxhash
+Project-URL: Homepage, https://github.com/anhvth/speedy
+Description-Content-Type: text/markdown
+
+# Speedy Utils
+
+![PyPI](https://img.shields.io/pypi/v/speedy-utils)
+![Python Versions](https://img.shields.io/pypi/pyversions/speedy-utils)
+![License](https://img.shields.io/pypi/l/speedy-utils)
+
+**Speedy Utils** is a Python utility library designed to streamline common programming tasks such as caching, parallel processing, file I/O, and data manipulation. It provides a collection of decorators, functions, and classes to enhance productivity and performance in your Python projects.
+
+## Table of Contents
+
+- [Features](#features)
+- [Installation](#installation)
+- [Usage](#usage)
+  - [Caching](#caching)
+  - [Parallel Processing](#parallel-processing)
+  - [File I/O](#file-io)
+  - [Data Manipulation](#data-manipulation)
+  - [Utility Functions](#utility-functions)
+- [Testing](#testing)
+
+## Features
+
+- **Caching Mechanisms**: Disk-based and in-memory caching to optimize function calls.
+- **Parallel Processing**: Multi-threading, multi-processing, and asynchronous multi-threading utilities.
+- **File I/O**: Simplified JSON, JSONL, and pickle file handling with support for various file extensions.
+- **Data Manipulation**: Utilities for flattening lists and dictionaries, converting data types, and more.
+- **Timing Utilities**: Tools to measure and log execution time of functions and processes.
+- **Pretty Printing**: Enhanced printing functions for structured data, including HTML tables for Jupyter notebooks.
+
+## Installation
+
+You can install **Speedy Utils** via [PyPI](https://pypi.org/project/speedy-utils/) using `pip`:
+
+```bash
+uv pip install speedy-utils
+
+```
+
+Alternatively, install directly from the repository:
+
+```bash
+uv pip install git+https://github.com/anhvth/speedy
+cd speedy-utils
+pip install .
+```
+
+## Updating from previous versions
+
+To update from previous versions or switch to v1.x, first uninstall any old
+packages, then install the latest version:
+
+```bash
+pip uninstall speedy_llm_utils speedy_utils
+pip install -e ./  # for local development
+# or
+pip install speedy_utils -U  # for PyPI upgrade
+```
+
+## Usage
+
+Below are examples demonstrating how to utilize various features of **Speedy Utils**.
+
+### Caching
+
+#### Memoize Decorator
+
+Cache the results of function calls to disk to avoid redundant computations.
+
+```python
+from speedy_utils import memoize
+
+@memoize
+def expensive_function(x):
+    # Simulate an expensive computation
+    import time
+    time.sleep(2)
+    return x * x
+
+result = expensive_function(4)  # Takes ~2 seconds
+result = expensive_function(4)  # Retrieved from cache instantly
+```
+
+#### In-Memory Memoization
+
+Cache function results in memory for faster access within the same runtime.
+
+```python
+from speedy_utils import imemoize
+
+@imemoize
+def compute_sum(a, b):
+    return a + b
+
+result = compute_sum(5, 7)  # Computed and cached
+result = compute_sum(5, 7)  # Retrieved from in-memory cache
+```
+
+### Parallel Processing
+
+#### Multi-threading
+
+Execute functions concurrently using multiple threads. This approach is straightforward and automatically handles both notebook and Python script executions. In a notebook environment, it delegates the running thread to a separate process. If interrupted, it immediately stops this process, avoiding thread dependency issues where threads continue running until all tasks are completed.
+
+```python
+from speedy_utils import multi_thread
+
+def process_item(item):
+    # Your processing logic
+    return item * 2
+
+items = [1, 2, 3, 4, 5]
+results = multi_thread(process_item, items, workers=3)
+print(results)  # [2, 4, 6, 8, 10]
+```
+
+### File I/O
+
+#### Dumping Data
+
+Save data in JSON, JSONL, or pickle formats.
+
+```python
+from speedy_utils import dump_json_or_pickle, dump_jsonl
+
+data = {"name": "Alice", "age": 30}
+
+# Save as JSON
+dump_json_or_pickle(data, "data.json")
+
+# Save as JSONL
+dump_jsonl([data, {"name": "Bob", "age": 25}], "data.jsonl")
+
+# Save as Pickle
+dump_json_or_pickle(data, "data.pkl")
+```
+
+#### Loading Data
+
+Load data based on file extensions.
+
+```python
+from speedy_utils import load_json_or_pickle, load_by_ext
+
+# Load JSON
+data = load_json_or_pickle("data.json")
+
+# Load JSONL
+data_list = load_json_or_pickle("data.jsonl")
+
+# Load Pickle
+data = load_json_or_pickle("data.pkl")
+
+# Load based on extension with parallel processing
+loaded_data = load_by_ext(["data.json", "data.pkl"])
+```
+
+### Data Manipulation
+
+#### Flattening Lists and Dictionaries
+
+```python
+from speedy_utils import flatten_list, flatten_dict
+
+nested_list = [[1, 2], [3, 4], [5]]
+flat_list = flatten_list(nested_list)
+print(flat_list)  # [1, 2, 3, 4, 5]
+
+nested_dict = {"a": {"b": 1, "c": 2}, "d": 3}
+flat_dict = flatten_dict(nested_dict)
+print(flat_dict)  # {'a.b': 1, 'a.c': 2, 'd': 3}
+```
+
+#### Converting to Built-in Python Types
+
+```python
+from speedy_utils import convert_to_builtin_python
+from pydantic import BaseModel
+
+class User(BaseModel):
+    name: str
+    age: int
+
+user = User(name="Charlie", age=28)
+builtin_user = convert_to_builtin_python(user)
+print(builtin_user)  # {'name': 'Charlie', 'age': 28}
+```
+
+### Utility Functions
+
+#### Pretty Printing
+
+```python
+from speedy_utils import fprint, print_table
+
+data = {"name": "Dana", "age": 22, "city": "New York"}
+
+# Pretty print as table
+fprint(data)
+
+# Print as table using tabulate
+print_table(data)
+```
+
+#### Timing Utilities
+
+```python
+from speedy_utils import timef, Clock
+
+@timef
+def slow_function():
+    import time
+    time.sleep(3)
+    return "Done"
+
+result = slow_function()  # Prints execution time
+
+# Using Clock
+clock = Clock()
+# ... your code ...
+clock.log()
+```
+
+## Testing
+
+The project includes a comprehensive test suite using `unittest`. To run the tests, execute the following command in the project root directory:
+
+```bash
+python test.py
+```
+
+Ensure all dependencies are installed before running tests:
+
+```bash
+pip install -r requirements.txt
+```
+
+Run the script to parse and display the arguments:
+
+```bash
+python speedy_utils/common/dataclass_parser.py
+```
+
+Example output:
+
+| Field              | Value                                 |
+|--------------------|---------------------------------------|
+| from_peft          | ./outputs/llm_hn_qw32b/hn_results_r3/ |
+
+Please ensure your code adheres to the project's coding standards and includes appropriate tests.
+

speedy_utils-1.0.9.dist-info/RECORD

@@ -0,0 +1,30 @@
+llm_utils/__init__.py,sha256=ibEVUPkL11M4htL-3uXkSyyUZiIO-TZD6IzWVmi8QYw,697
+llm_utils/chat_format/__init__.py,sha256=8dBIUqFJvkgQYedxBtcyxt-4tt8JxAKVap2JlTXmgaM,737
+llm_utils/chat_format/display.py,sha256=a3zWzo47SUf4i-uic-dwf-vxtu6gZWLbnJrszjjZjQ8,9801
+llm_utils/chat_format/transform.py,sha256=328V18FOgRQzljAl9Mh8NF4Tl-N3cZZIPmAwHQspXCY,5461
+llm_utils/chat_format/utils.py,sha256=xTxN4HrLHcRO2PfCTR43nH1M5zCa7v0kTTdzAcGkZg0,1229
+llm_utils/group_messages.py,sha256=wyiZzs7O8yK2lyIakV2x-1CrrWVT12sjnP1vVnmPet4,3606
+llm_utils/lm/__init__.py,sha256=vXFILZLBmmpg39cy5XniQPSMzoFQCE3wdfz39EtqDKU,71
+llm_utils/lm/lm.py,sha256=wLVX7-kE8odyS-mLew0rihswxWxoEn7smxFGHoPKv_4,10739
+llm_utils/lm/utils.py,sha256=-fDNueiXKQI6RDoNHJYNyORomf2XlCf2doJZ3GEV2Io,4762
+llm_utils/scripts/vllm_load_balancer.py,sha256=MgMnnoKWJQc-l2fspUSkyA9wxL1RkXd7wdBLJNQBlr4,17384
+llm_utils/scripts/vllm_serve.py,sha256=uFS5kNXZ7kZ9rQms63LnliGEVV3rATT6dEppGTgoR0s,13910
+speedy_utils/__init__.py,sha256=I2bSfDIE9yRF77tnHW0vqfExDA2m1gUx4AH8C9XmGtg,1707
+speedy_utils/all.py,sha256=A9jiKGjo950eg1pscS9x38OWAjKGyusoAN5mrfweY4E,3090
+speedy_utils/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+speedy_utils/common/clock.py,sha256=3n4FkCW0dz46O8By09V5Pve1DSMgpLDRbWEVRryryeQ,7423
+speedy_utils/common/function_decorator.py,sha256=r_r42qCWuNcu0_aH7musf2BWvcJfgZrD81G28mDcolw,2226
+speedy_utils/common/logger.py,sha256=NIOlhhACpcc0BUTSJ8oDYrLp23J2gW_KJFyRVdLN2tY,6432
+speedy_utils/common/report_manager.py,sha256=dgGfS_fHbZiQMsLzkgnj0OfB758t1x6B4MhjsetSl9A,3930
+speedy_utils/common/utils_cache.py,sha256=gXX5qTXpCG3qDgjnOsSvxM4LkQurmcsg4QRv_zOBG1E,8378
+speedy_utils/common/utils_io.py,sha256=d7PKz5tOPrwHDr7GXuYmILvjXJOFEwfzAEIuUcYaI60,4790
+speedy_utils/common/utils_misc.py,sha256=nsQOu2jcplcFHVQ1CnOjEpNcdxIINveGxB493Cqo63U,1812
+speedy_utils/common/utils_print.py,sha256=QRaL2QPbks4Mtol_gJy3ZdahgUfzUEtcOp4--lBlzYI,6709
+speedy_utils/multi_worker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+speedy_utils/multi_worker/process.py,sha256=XwQlffxzRFnCVeKjDNBZDwFfUQHiJiuFA12MRGJVru8,6708
+speedy_utils/multi_worker/thread.py,sha256=9pXjvgjD0s0Hp0cZ6I3M0ndp1OlYZ1yvqbs_bcun_Kw,12775
+speedy_utils/scripts/mpython.py,sha256=ZzkBWI5Xw3vPoMx8xQt2x4mOFRjtwWqfvAJ5_ngyWgw,3816
+speedy_utils-1.0.9.dist-info/METADATA,sha256=bLbYeo_uDW0hasjtbmuZoslF-S6qkO-gURfbZhnEuIo,7391
+speedy_utils-1.0.9.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+speedy_utils-1.0.9.dist-info/entry_points.txt,sha256=rP43satgw1uHcKUAlmVxS-MTAQImL-03-WwLIB5a300,165
+speedy_utils-1.0.9.dist-info/RECORD,,

{speedy_utils-1.0.4.dist-info → speedy_utils-1.0.9.dist-info}/WHEEL

@@ -1,5 +1,4 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any
-

speedy_utils-1.0.9.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+mpython=speedy_utils.scripts.mpython:main
+svllm=llm_utils.scripts.vllm_serve:main
+svllm-lb=llm_utils.scripts.vllm_load_balancer:run_load_balancer
+

speedy/__init__.py

@@ -1,53 +0,0 @@
-from .common.clock import Clock, timef
-from .common.utils_cache import (
-    ICACHE,
-    SPEED_CACHE_DIR,
-    identify,
-    imemoize,
-    imemoize_v2,
-    memoize,
-    memoize_method,
-    memoize_v2,
-)
-from .common.utils_io import (
-    dump_json_or_pickle,
-    dump_jsonl,
-    load_by_ext,
-    load_json_or_pickle,
-)
-from .common.utils_misc import (
-    convert_to_builtin_python,
-    flatten_list,
-    get_arg_names,
-    is_interactive,
-    mkdir_or_exist,
-)
-from .common.utils_print import fprint, print_table
-from .multi_worker import async_multi_thread, multi_process, multi_thread
-
-__all__ = [
-    "SPEED_CACHE_DIR",
-    "ICACHE",
-    "mkdir_or_exist",
-    "dump_jsonl",
-    "dump_json_or_pickle",
-    "timef",  # Ensure timef is moved to an appropriate module or included here
-    "load_json_or_pickle",
-    "load_by_ext",
-    "identify",
-    "memoize",
-    "imemoize",
-    "imemoize_v2",
-    "flatten_list",
-    "fprint",
-    "get_arg_names",
-    "memoize_v2",
-    "is_interactive",
-    "print_table",
-    "convert_to_builtin_python",
-    "Clock",
-    "multi_thread",
-    "multi_process",
-    "async_multi_thread",
-    "memoize_method",
-]

speedy/common/clock.py

@@ -1,68 +0,0 @@
-import time
-from loguru import logger
-
-__all__ = ["Clock"]
-
-
-def timef(func):
-    def wrapper(*args, **kwargs):
-        start_time = time.time()
-        result = func(*args, **kwargs)
-        end_time = time.time()
-        execution_time = end_time - start_time
-        print(f"{func.__name__} took {execution_time:0.2f} seconds to execute.")
-        return result
-
-    return wrapper
-
-
-class Clock:
-    def __init__(self, start_now=True):
-        self.start_time = None
-        self.time_table = {}
-        self.last_check = None
-        if start_now:
-            self.start()
-        self.pbar_counter = 0
-        self.last_print = time.time()
-
-    def start(self):
-        self.start_time = time.time() if self.start_time is None else self.start_time
-        self.last_check = self.start_time
-
-    def since_start(self):
-        if self.start_time is None:
-            raise ValueError("Clock has not been started.")
-        return time.time() - self.start_time
-
-    def log(self, custom_logger=None):
-        msg = f"Time elapsed: {self.since_start():.2f} seconds."
-        if custom_logger:
-            custom_logger(msg)
-        else:
-            logger.info(msg)
-
-    def since_last_check(self):
-        now = time.time()
-        elapsed = now - self.last_check
-        self.last_check = now
-        return elapsed
-
-    def update(self, name):
-        if not name in self.time_table:
-            self.time_table[name] = 0
-        self.time_table[name] += self.since_last_check()
-
-    def print_table(self, every=1):
-        now = time.time()
-        if now - self.last_print > every:
-            self.pbar_counter += 1
-            total_time = sum(self.time_table.values())
-            desc = "Time table: "
-            for name, t in self.time_table.items():
-                percentage = (t / total_time) * 100
-                desc += "{}: avg_time: {:.2f} s ({:.2f}%), total: {} s".format(
-                    name, t, percentage, total_time
-                )
-            logger.info(desc)
-            self.last_print = now

speedy/common/utils_cache.py

@@ -1,170 +0,0 @@
-# utils/utils_cache.py
-
-import functools
-import inspect
-import os
-import os.path as osp
-import pickle
-import traceback
-from typing import Any, Callable, Dict, List, Optional
-
-import xxhash
-from loguru import logger
-
-from .utils_io import dump_json_or_pickle, load_json_or_pickle
-from .utils_misc import mkdir_or_exist
-
-SPEED_CACHE_DIR = osp.join(osp.expanduser("~"), ".cache/av")
-ICACHE: Dict[str, Any] = {}
-
-
-def identify(x: Any) -> str:
-    """Return an hex digest of the input."""
-    return xxhash.xxh64(pickle.dumps(x), seed=0).hexdigest()
-
-
-def memoize(
-    func: Callable,
-    ignore_self: bool = True,
-    cache_dir: str = SPEED_CACHE_DIR,
-    cache_type: str = ".pkl",
-    verbose: bool = False,
-    cache_key: Optional[str] = None,
-) -> Callable:
-    """Cache result of function call on disk."""
-    assert cache_type in [".pkl", ".json"]
-    if os.environ.get("AV_MEMOIZE_DISABLE", "0") == "1":
-        logger.info("Memoize is disabled")
-        return func
-
-    @functools.wraps(func)
-    def memoized_func(*args, **kwargs):
-        try:
-            arg_names = inspect.getfullargspec(func).args
-            func_source = inspect.getsource(func).replace(" ", "")
-            if cache_key is not None:
-                logger.info(f"Use cache_key={kwargs[cache_key]}")
-                fid = [func_source, kwargs[cache_key]]
-                func_id = identify(fid)
-            elif len(arg_names) > 0 and arg_names[0] == "self" and ignore_self:
-                func_id = identify((func_source, args[1:], kwargs))
-            else:
-                func_id = identify((func_source, args, kwargs))
-
-            cache_path = osp.join(
-                cache_dir, "funcs", func.__name__, f"{func_id}{cache_type}"
-            )
-            mkdir_or_exist(os.path.dirname(cache_path))
-            if osp.exists(cache_path):
-                if verbose:
-                    logger.info(f"Load from cache file: {cache_path}")
-                result = load_json_or_pickle(cache_path)
-            else:
-                result = func(*args, **kwargs)
-                dump_json_or_pickle(result, cache_path)
-            return result
-        except Exception as e:
-            traceback.print_exc()
-            logger.warning(f"Exception: {e}, using default function call")
-            return func(*args, **kwargs)
-
-    return memoized_func
-
-
-def imemoize(func: Callable) -> Callable:
-    """Memoize a function into memory."""
-
-    @functools.wraps(func)
-    def _f(*args, **kwargs):
-        ident_name = identify((inspect.getsource(func), args, kwargs))
-        try:
-            return ICACHE[ident_name]
-        except KeyError:
-            result = func(*args, **kwargs)
-            ICACHE[ident_name] = result
-            return result
-
-    return _f
-
-
-def imemoize_v2(keys: List[str]) -> Callable:
-    """Memoize a function into memory based on specified keys."""
-
-    def decorator(func: Callable) -> Callable:
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            arg_names = inspect.getfullargspec(func).args
-            args_dict = dict(zip(arg_names, args))
-            all_args = {**args_dict, **kwargs}
-            key_values = {key: all_args[key] for key in keys if key in all_args}
-            if not key_values:
-                return func(*args, **kwargs)
-
-            ident_name = identify((func.__name__, tuple(sorted(key_values.items()))))
-            try:
-                return ICACHE[ident_name]
-            except KeyError:
-                result = func(*args, **kwargs)
-                ICACHE[ident_name] = result
-                return result
-
-        return wrapper
-
-    return decorator
-
-
-def memoize_v2(keys: List[str], cache_dir: str = SPEED_CACHE_DIR) -> Callable:
-    """Decorator to memoize function results based on specific keys."""
-
-    def decorator(func: Callable) -> Callable:
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            args_key_values = {}
-            for i, arg in enumerate(args):
-                arg_name = inspect.getfullargspec(func).args[i]
-                args_key_values[arg_name] = arg
-            args_key_values.update(kwargs)
-
-            values = [args_key_values[key] for key in keys if key in args_key_values]
-            if not values:
-                return func(*args, **kwargs)
-
-            key_id = identify(values)
-            func_source = inspect.getsource(func).replace(" ", "")
-            func_id = identify(func_source)
-            key_names = "_".join(keys)
-            cache_path = osp.join(
-                cache_dir, f"{func.__name__}_{func_id}", f"{key_names}_{key_id}.pkl"
-            )
-            if osp.exists(cache_path):
-                return load_json_or_pickle(cache_path)
-            result = func(*args, **kwargs)
-            dump_json_or_pickle(result, cache_path)
-            return result
-
-        return wrapper
-
-    return decorator
-
-
-def memoize_method(method):
-    """
-    Decorator function to memoize (cache) results of a class method.
-
-    This decorator caches the output of the wrapped method based on its input arguments
-    (both positional and keyword). If the method is called again with the same arguments,
-    it returns the cached result instead of executing the method again.
-
-    Args:
-        method (Callable): The decorated method whose result will be memoized.
-    """
-    cache = {}
-
-    def cached_method(cls, *args, **kwargs):
-        cache_key = identify([args, kwargs])
-        logger.debug("HIT" if cache_key in cache else "MISS")
-        if cache_key not in cache:
-            cache[cache_key] = method(cls, *args, **kwargs)
-        return cache[cache_key]
-
-    return cached_method