speedy-utils 1.2.12__tar.gz → 2.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- speedy_utils-1.2.12/src/speedy_utils/common/__init__.py → speedy_utils-2.0.2/.codex +0 -0
- speedy_utils-2.0.2/CLAUDE.md +180 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/PKG-INFO +105 -59
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/README.md +102 -50
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/debug/integration_test.py +13 -13
- speedy_utils-2.0.2/debug/test_qwen3_generate.md +87 -0
- speedy_utils-2.0.2/debug/test_qwen3_generate.py +12 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/docs/GENERATE_QUICKREF.md +10 -2
- speedy_utils-2.0.2/docs/MIGRATION_THINKING.md +73 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/docs/TOKENIZATION.md +32 -5
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/docs/TOKENIZATION_IMPLEMENTATION.md +1 -1
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/examples/generate_example.py +17 -17
- speedy_utils-2.0.2/examples/llm_public_api_example.ipynb +273 -0
- speedy_utils-2.0.2/examples/model_specific_thinking_example.py +24 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/examples/tokenization_example.py +1 -1
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/notebooks/llm_utils/llm_as_a_judge.ipynb +1 -1
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/pyproject.toml +10 -17
- speedy_utils-2.0.2/pyrightconfig.json +8 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/scripts/bug_simple.py +1 -1
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/scripts/test.py +1 -1
- speedy_utils-2.0.2/scripts/test_locals.py +17 -0
- speedy_utils-2.0.2/src/datasets_utils/viz_chat.py +983 -0
- speedy_utils-2.0.2/src/llm_utils/__init__.py +75 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/chat_format/display.py +40 -8
- speedy_utils-2.0.2/src/llm_utils/lm/__init__.py +21 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/lm/base_prompt_builder.py +31 -4
- speedy_utils-2.0.2/src/llm_utils/lm/llm.py +863 -0
- speedy_utils-2.0.2/src/llm_utils/lm/llm_qwen3.py +705 -0
- speedy_utils-2.0.2/src/llm_utils/lm/llm_signature.py +86 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/lm/lm_base.py +10 -10
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/lm/openai_memoize.py +46 -2
- speedy_utils-2.0.2/src/llm_utils/lm/utils.py +71 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/scripts/sp_chat.py +80 -36
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/__imports.py +106 -110
- speedy_utils-2.0.2/src/speedy_utils/__init__.py +227 -0
- {speedy_utils-1.2.12/src/speedy_utils/scripts → speedy_utils-2.0.2/src/speedy_utils/common}/__init__.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/utils_error.py +9 -3
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/utils_io.py +5 -2
- speedy_utils-2.0.2/src/speedy_utils/multi_worker/__init__.py +11 -0
- speedy_utils-2.0.2/src/speedy_utils/multi_worker/_multi_process.py +819 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/multi_worker/common.py +8 -113
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/multi_worker/dataset_sharding.py +9 -24
- speedy_utils-2.0.2/src/speedy_utils/multi_worker/process.py +71 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/multi_worker/thread.py +208 -56
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/scripts/mpython.py +63 -1
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/vision_utils/__init__.py +7 -7
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/vision_utils/io_utils.py +36 -57
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/vision_utils/plot.py +2 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/import_time_report.py +0 -4
- speedy_utils-2.0.2/tests/llm_utils/test_llm_call_contract.py +442 -0
- speedy_utils-2.0.2/tests/llm_utils/test_llm_mixins.py +835 -0
- speedy_utils-2.0.2/tests/llm_utils/test_llm_signature.py +168 -0
- speedy_utils-2.0.2/tests/llm_utils/test_openai_memoize_completion_cache.py +116 -0
- speedy_utils-2.0.2/tests/llm_utils/test_openai_proxy_localhost.py +29 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/llm_utils/test_sp_chat.py +3 -1
- speedy_utils-2.0.2/tests/test_import_time.py +80 -0
- speedy_utils-2.0.2/tests/test_mpython.py +99 -0
- speedy_utils-2.0.2/tests/test_multi_process.py +193 -0
- speedy_utils-2.0.2/tests/test_multi_thread.py +91 -0
- speedy_utils-2.0.2/tests/test_process.py +383 -0
- speedy_utils-2.0.2/tests/test_process_update.py +129 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/test_thread.py +11 -14
- speedy_utils-2.0.2/tests/test_tokenization.py +161 -0
- speedy_utils-2.0.2/tests/test_viz_chat.py +532 -0
- speedy_utils-2.0.2/tools/uv_test.sh +9 -0
- speedy_utils-2.0.2/uv.lock +9807 -0
- speedy_utils-1.2.12/CLAUDE.md +0 -79
- speedy_utils-1.2.12/docs/IMPLEMENTATION.md +0 -149
- speedy_utils-1.2.12/docs/MIGRATION_THINKING.md +0 -278
- speedy_utils-1.2.12/docs/QUICKSTART.md +0 -109
- speedy_utils-1.2.12/docs/SKILL.md +0 -415
- speedy_utils-1.2.12/docs/zero_copy_sharing.md +0 -235
- speedy_utils-1.2.12/documents/llm_utils_recipe/README.md +0 -176
- speedy_utils-1.2.12/examples/llm_ray_example.py +0 -73
- speedy_utils-1.2.12/examples/model_specific_thinking_example.py +0 -187
- speedy_utils-1.2.12/examples/pytorch_large_model.py +0 -285
- speedy_utils-1.2.12/examples/shared_kwargs_example.py +0 -200
- speedy_utils-1.2.12/examples/temperature_range_example.py +0 -122
- speedy_utils-1.2.12/examples/test_parallel_gpu.py +0 -61
- speedy_utils-1.2.12/examples/test_share_ray.py +0 -24
- speedy_utils-1.2.12/experiments/test_read_image.py +0 -37
- speedy_utils-1.2.12/notebooks/README.ipynb +0 -389
- speedy_utils-1.2.12/notebooks/parallel_gpu_pool.ipynb +0 -89
- speedy_utils-1.2.12/notebooks/ray_tutorial.ipynb +0 -120
- speedy_utils-1.2.12/scripts/bug.py +0 -16
- speedy_utils-1.2.12/scripts/test_both_backends.py +0 -25
- speedy_utils-1.2.12/scripts/test_locals.py +0 -18
- speedy_utils-1.2.12/scripts/test_ray_locals.py +0 -10
- speedy_utils-1.2.12/src/llm_utils/__init__.py +0 -114
- speedy_utils-1.2.12/src/llm_utils/llm_ray.py +0 -376
- speedy_utils-1.2.12/src/llm_utils/lm/__init__.py +0 -42
- speedy_utils-1.2.12/src/llm_utils/lm/async_lm/__init__.py +0 -8
- speedy_utils-1.2.12/src/llm_utils/lm/async_lm/_utils.py +0 -203
- speedy_utils-1.2.12/src/llm_utils/lm/async_lm/async_llm_task.py +0 -514
- speedy_utils-1.2.12/src/llm_utils/lm/async_lm/async_lm.py +0 -475
- speedy_utils-1.2.12/src/llm_utils/lm/async_lm/async_lm_base.py +0 -246
- speedy_utils-1.2.12/src/llm_utils/lm/async_lm/lm_specific.py +0 -167
- speedy_utils-1.2.12/src/llm_utils/lm/async_lm/model_specific.py +0 -268
- speedy_utils-1.2.12/src/llm_utils/lm/llm.py +0 -674
- speedy_utils-1.2.12/src/llm_utils/lm/llm_signature.py +0 -37
- speedy_utils-1.2.12/src/llm_utils/lm/mixins.py +0 -697
- speedy_utils-1.2.12/src/llm_utils/lm/utils.py +0 -357
- speedy_utils-1.2.12/src/llm_utils/scripts/README.md +0 -48
- speedy_utils-1.2.12/src/llm_utils/scripts/fast_vllm.py +0 -131
- speedy_utils-1.2.12/src/llm_utils/scripts/vllm_load_balancer.py +0 -972
- speedy_utils-1.2.12/src/llm_utils/scripts/vllm_serve.py +0 -398
- speedy_utils-1.2.12/src/llm_utils/vector_cache/__init__.py +0 -31
- speedy_utils-1.2.12/src/llm_utils/vector_cache/cli.py +0 -200
- speedy_utils-1.2.12/src/llm_utils/vector_cache/core.py +0 -778
- speedy_utils-1.2.12/src/llm_utils/vector_cache/types.py +0 -17
- speedy_utils-1.2.12/src/llm_utils/vector_cache/utils.py +0 -46
- speedy_utils-1.2.12/src/speedy_utils/__init__.py +0 -218
- speedy_utils-1.2.12/src/speedy_utils/multi_worker/__init__.py +0 -16
- speedy_utils-1.2.12/src/speedy_utils/multi_worker/_multi_process.py +0 -443
- speedy_utils-1.2.12/src/speedy_utils/multi_worker/_multi_process_ray.py +0 -309
- speedy_utils-1.2.12/src/speedy_utils/multi_worker/dataset_ray.py +0 -309
- speedy_utils-1.2.12/src/speedy_utils/multi_worker/parallel_gpu_pool.py +0 -185
- speedy_utils-1.2.12/src/speedy_utils/multi_worker/process.py +0 -128
- speedy_utils-1.2.12/src/speedy_utils/multi_worker/progress.py +0 -210
- speedy_utils-1.2.12/tests/integration_test.py +0 -51
- speedy_utils-1.2.12/tests/llm_utils/test_llm_mixins.py +0 -226
- speedy_utils-1.2.12/tests/test_process.py +0 -173
- speedy_utils-1.2.12/tests/test_process_update.py +0 -73
- speedy_utils-1.2.12/tests/test_pytorch_sharing.py +0 -274
- speedy_utils-1.2.12/tests/test_ray_optional_dependency.py +0 -46
- speedy_utils-1.2.12/tests/test_shared_kwargs.py +0 -208
- speedy_utils-1.2.12/tests/test_tokenization.py +0 -95
- speedy_utils-1.2.12/uv.lock +0 -10662
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.githooks/pre-commit +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.githooks/pre-push +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/prompts/improveParallelErrorHandling.prompt.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/caching-utilities/SKILL.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/caching-utilities/examples/caching_example.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/dataset-processing-multiprocessing/CONFIG_REFERENCE.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/dataset-processing-multiprocessing/SKILL.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/dataset-processing-multiprocessing/example_tokenize_pack.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/edit-llm-inference-style/SKILL.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/io-utilities/SKILL.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/io-utilities/examples/io_example.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/llm-integration/SKILL.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/llm-integration/examples/llm_example.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/multi-threading-processing/SKILL.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/ray-distributed-computing/SKILL.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/skill-creation/SKILL.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/vision-utilities/SKILL.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/skills/vision-utilities/examples/vision_example.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.github/workflows/publish.yml +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.gitignore +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/.pre-commit-config.yaml +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/AGENTS.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/bumpversion.sh +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/chainlit.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/debug/debug_generate_response.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/debug/debug_n_param.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/debug/debug_n_structure.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/debug/multi_thread.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/debug/test_decode_api.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/debug/test_endpoints.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/debug/test_generate.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/debug/test_generate_endpoint.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/examples/vision_utils_example.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/experiments/exp1/dockerfile +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/experiments/exp1/run_in_docker.sh +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/experiments/exp1/test.png +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/notebooks/test.ipynb +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/notebooks/test_multi_thread.ipynb +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/ruff.toml +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/scripts/debug_import_time.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/scripts/deploy.sh +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/scripts/imports.sh +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/scripts/install_hooks.sh +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/scripts/test_error_handling.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/scripts/test_import_time_vision.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/scripts/test_ray_mp.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/setup.cfg +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/datasets_utils/convert_to_arrow.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/chat_format/__init__.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/chat_format/transform.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/chat_format/utils.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/group_messages.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/llm_utils/lm/signature.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/clock.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/function_decorator.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/logger.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/notebook_utils.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/patcher.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/report_manager.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/utils_cache.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/utils_misc.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/common/utils_print.py +0 -0
- /speedy_utils-1.2.12/tests/test_mpython.py → /speedy_utils-2.0.2/src/speedy_utils/scripts/__init__.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/scripts/kill_mpython.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/scripts/openapi_client_codegen.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/speedy_utils/scripts/prefetch_large_model.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/src/vision_utils/README.md +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/import_all.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/sample_objects.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/test.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/test_logger.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/test_logger_format.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/test_memoize_typing.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/test_multithread_error_trace.py +0 -0
- {speedy_utils-1.2.12 → speedy_utils-2.0.2}/tests/test_prefetch_large_model.py +0 -0
|
File without changes
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
+
|
|
5
|
+
## Development Commands
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
# Install dependencies
|
|
9
|
+
uv sync
|
|
10
|
+
|
|
11
|
+
# Run tests
|
|
12
|
+
uv run pytest -n 32
|
|
13
|
+
|
|
14
|
+
# Run a single test file
|
|
15
|
+
uv run pytest tests/test_thread.py
|
|
16
|
+
|
|
17
|
+
# Run with verbose output
|
|
18
|
+
uv run pytest -v
|
|
19
|
+
|
|
20
|
+
# Check import time (must be < 0.4s)
|
|
21
|
+
uv run python -c "import time; start=time.perf_counter(); import speedy_utils; print(f'{time.perf_counter()-start:.3f}s')"
|
|
22
|
+
|
|
23
|
+
# Detailed import analysis
|
|
24
|
+
uv run python scripts/debug_import_time.py speedy_utils --min-sec 0.05 --no-stdlib
|
|
25
|
+
|
|
26
|
+
# Lint with ruff
|
|
27
|
+
uv run ruff check .
|
|
28
|
+
|
|
29
|
+
# Format with ruff
|
|
30
|
+
uv run ruff format .
|
|
31
|
+
|
|
32
|
+
# Bump version (runs tests first, then commits and pushes)
|
|
33
|
+
./bumpversion.sh patch # or minor, major
|
|
34
|
+
|
|
35
|
+
# Deploy to PyPI (requires PYPI_API_TOKEN)
|
|
36
|
+
./scripts/deploy.sh
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Performance Requirement: Import Time < 0.4s
|
|
40
|
+
|
|
41
|
+
**CRITICAL:** All modules must import in under 0.4 seconds. This is enforced by a git pre-commit hook.
|
|
42
|
+
|
|
43
|
+
### Lazy Import Strategy
|
|
44
|
+
|
|
45
|
+
Import heavy modules inside functions, NOT at module level:
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
# BAD - slow module-level import
|
|
49
|
+
import torch
|
|
50
|
+
def some_function():
|
|
51
|
+
return torch.tensor([1, 2, 3])
|
|
52
|
+
|
|
53
|
+
# GOOD - lazy import
|
|
54
|
+
def some_function():
|
|
55
|
+
import torch # Only imports when called
|
|
56
|
+
return torch.tensor([1, 2, 3])
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Heavy Modules to Lazy-Load
|
|
60
|
+
|
|
61
|
+
| Module | Import Time | Strategy |
|
|
62
|
+
|--------|-------------|----------|
|
|
63
|
+
| `torch` | ~5s | Lazy import inside functions |
|
|
64
|
+
| `matplotlib` | ~1.3s cumulative | Lazy via `__getattr__` in `__init__.py` |
|
|
65
|
+
| `pandas` | ~1.9s cumulative | Lazy via `__getattr__` in `__init__.py` |
|
|
66
|
+
| `IPython` | ~2.1s cumulative | Lazy via `__getattr__` in `__init__.py` |
|
|
67
|
+
|
|
68
|
+
### Rule: Never top-level import heavy modules in `__init__.py`
|
|
69
|
+
|
|
70
|
+
Only a short `__getattr__` block (keyed on module name) is allowed for heavy
|
|
71
|
+
modules. All internal `speedy_utils.*` imports must remain direct top-level
|
|
72
|
+
imports. External heavy modules (`pandas`, `matplotlib`, `IPython`, `torch`)
|
|
73
|
+
must **never** appear as top-level imports in any `__init__.py`.
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
# BAD — in __init__.py
|
|
77
|
+
import pandas as pd # adds ~1.3 s
|
|
78
|
+
|
|
79
|
+
# GOOD — in __init__.py
|
|
80
|
+
# pd is declared in _HEAVY dict, resolved by __getattr__ on first access
|
|
81
|
+
|
|
82
|
+
# GOOD — inside a function anywhere in the package
|
|
83
|
+
def load_dataframe(path):
|
|
84
|
+
import pandas as pd # only pays cost when called
|
|
85
|
+
return pd.read_csv(path)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### Rule: Heavy lazy names must NOT be in `__all__`
|
|
89
|
+
|
|
90
|
+
`from speedy_utils import *` resolves every name in `__all__`, which triggers
|
|
91
|
+
`__getattr__` for each heavy name and defeats the lazy-load strategy. Only
|
|
92
|
+
include names that are directly importable at module load time in `__all__`.
|
|
93
|
+
Heavy names (`np`, `pd`, `matplotlib`, `plt`, `get_ipython`, `HTML`, `display`,
|
|
94
|
+
`BaseModel`) must be kept out of `__all__`; they remain accessible via explicit
|
|
95
|
+
attribute access (`speedy_utils.pd`) or explicit import
|
|
96
|
+
(`from speedy_utils import pd`).
|
|
97
|
+
|
|
98
|
+
## Architecture
|
|
99
|
+
|
|
100
|
+
### Package Structure
|
|
101
|
+
|
|
102
|
+
Three main packages under `src/`:
|
|
103
|
+
|
|
104
|
+
- **`speedy_utils`**: Core utilities (caching, IO, parallel processing, timing)
|
|
105
|
+
- `common/`: Cache (`memoize`, `imemoize`), IO (`load_json_or_pickle`), logging, error handling
|
|
106
|
+
- `multi_worker/`: `multi_thread`, `multi_process`, dataset sharding helpers
|
|
107
|
+
- `__imports.py`: Lazy-loaded heavy dependencies (torch, pandas, matplotlib)
|
|
108
|
+
|
|
109
|
+
- **`llm_utils`**: LLM integration layer
|
|
110
|
+
- `lm/llm.py`: Main `LLM` class with OpenAI-compatible client support, structured outputs, caching
|
|
111
|
+
- `lm/openai_memoize.py`: `MOpenAI` - memoized OpenAI client
|
|
112
|
+
- `chat_format/`: Transform between ChatML, ShareGPT, text formats
|
|
113
|
+
|
|
114
|
+
- **`vision_utils`**: Image processing utilities
|
|
115
|
+
- `io_utils.py`: Image loading, video frame extraction
|
|
116
|
+
- `plot.py`: Matplotlib wrappers for visualization
|
|
117
|
+
|
|
118
|
+
### Key Patterns
|
|
119
|
+
|
|
120
|
+
**Lazy Exports via `__getattr__`**: Both `speedy_utils` and `llm_utils` use lazy attribute access to keep import time fast:
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
# In __init__.py
|
|
124
|
+
_LAZY_ATTRS = {"LLM": ("llm_utils.lm", "LLM")}
|
|
125
|
+
|
|
126
|
+
def __getattr__(name):
|
|
127
|
+
module_name, attr_name = _LAZY_ATTRS[name]
|
|
128
|
+
module = importlib.import_module(module_name)
|
|
129
|
+
return getattr(module, attr_name)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
**Memoization**: `@memoize` caches function results to disk; `@imemoize` caches in memory:
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
from speedy_utils import memoize, imemoize
|
|
136
|
+
|
|
137
|
+
@memoize # Persists to disk at ~/.cache/speedy_utils/
|
|
138
|
+
def expensive_call(x):
|
|
139
|
+
return x ** 2
|
|
140
|
+
|
|
141
|
+
@imemoize # In-memory only
|
|
142
|
+
def fast_cached_call(x):
|
|
143
|
+
return x * 2
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
**Parallel Processing with Error Handling**:
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
from speedy_utils import multi_thread, multi_process
|
|
150
|
+
|
|
151
|
+
# Three error handling modes
|
|
152
|
+
results = multi_thread(func, items, error_handler='raise') # Stop on error (default)
|
|
153
|
+
results = multi_thread(func, items, error_handler='ignore') # Continue, return None for errors
|
|
154
|
+
results = multi_thread(func, items, error_handler='log') # Log errors, continue
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### CLI Tools
|
|
158
|
+
|
|
159
|
+
The package provides these CLI commands:
|
|
160
|
+
|
|
161
|
+
- `mpython`: Run Python scripts across multiple tmux windows with GPU/CPU allocation
|
|
162
|
+
- `kill-mpython`: Kill all mpython sessions
|
|
163
|
+
- `sp_chat`: Interactive chat CLI
|
|
164
|
+
- `spu-prefetch-large-model`: Prefetch models to disk cache
|
|
165
|
+
|
|
166
|
+
## Code Style
|
|
167
|
+
|
|
168
|
+
- Line length: 88 characters (Black/Ruff default)
|
|
169
|
+
- Ruff handles linting and formatting (config in `ruff.toml` and `pyproject.toml`)
|
|
170
|
+
- Ignore rules `E402`, `F401`, `F403` in `__init__.py` files for lazy import patterns
|
|
171
|
+
- Quote style: double quotes
|
|
172
|
+
- Avoid hacky workarounds like sys.path.insert
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
## Version Management
|
|
176
|
+
|
|
177
|
+
Version is in `pyproject.toml`. Use `./bumpversion.sh` to bump version, which:
|
|
178
|
+
1. Runs pytest (requires 90% pass rate)
|
|
179
|
+
2. Bumps version using `uv version`
|
|
180
|
+
3. Commits and pushes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: speedy-utils
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.2
|
|
4
4
|
Summary: Fast and easy-to-use package for data science
|
|
5
5
|
Project-URL: Homepage, https://github.com/anhvth/speedy
|
|
6
6
|
Project-URL: Repository, https://github.com/anhvth/speedy
|
|
@@ -22,7 +22,9 @@ Requires-Dist: aiohttp
|
|
|
22
22
|
Requires-Dist: bump2version
|
|
23
23
|
Requires-Dist: cachetools
|
|
24
24
|
Requires-Dist: chainlit>=2.0
|
|
25
|
+
Requires-Dist: datasets
|
|
25
26
|
Requires-Dist: debugpy
|
|
27
|
+
Requires-Dist: dill
|
|
26
28
|
Requires-Dist: fastcore
|
|
27
29
|
Requires-Dist: fastprogress
|
|
28
30
|
Requires-Dist: freezegun
|
|
@@ -46,14 +48,6 @@ Requires-Dist: tabulate
|
|
|
46
48
|
Requires-Dist: tqdm
|
|
47
49
|
Requires-Dist: transformers>=4.57.6
|
|
48
50
|
Requires-Dist: xxhash
|
|
49
|
-
Provides-Extra: llm-utils
|
|
50
|
-
Requires-Dist: ray[data,llm]>=2.49.1; extra == 'llm-utils'
|
|
51
|
-
Requires-Dist: torch>=2.8.0; extra == 'llm-utils'
|
|
52
|
-
Requires-Dist: transformers>=4.57.6; extra == 'llm-utils'
|
|
53
|
-
Requires-Dist: vllm>=0.11.0; extra == 'llm-utils'
|
|
54
|
-
Provides-Extra: ray
|
|
55
|
-
Requires-Dist: ray[data,llm]>=2.40.0; extra == 'ray'
|
|
56
|
-
Requires-Dist: vllm>=0.6.3; extra == 'ray'
|
|
57
51
|
Description-Content-Type: text/markdown
|
|
58
52
|
|
|
59
53
|
# Speedy Utils
|
|
@@ -152,20 +146,6 @@ cd speedy
|
|
|
152
146
|
uv sync
|
|
153
147
|
```
|
|
154
148
|
|
|
155
|
-
### Extras
|
|
156
|
-
|
|
157
|
-
Optional dependencies can be installed via extras. For the `ray` backend
|
|
158
|
-
support (requires Python >= 3.9):
|
|
159
|
-
|
|
160
|
-
```bash
|
|
161
|
-
pip install 'speedy-utils[ray]'
|
|
162
|
-
# or
|
|
163
|
-
uv pip install 'speedy-utils[ray]'
|
|
164
|
-
|
|
165
|
-
# developing this repo
|
|
166
|
-
uv sync --extra ray
|
|
167
|
-
```
|
|
168
|
-
|
|
169
149
|
## Updating from previous versions
|
|
170
150
|
|
|
171
151
|
To update from previous versions or switch to v1.x, first uninstall any old
|
|
@@ -275,6 +255,65 @@ results = multi_process(
|
|
|
275
255
|
print(results) # [0, 1, 4, 9, 16, None, 36, 49, 64, 81, None, 121]
|
|
276
256
|
```
|
|
277
257
|
|
|
258
|
+
#### Multi-Process with Inner Thread Pools
|
|
259
|
+
|
|
260
|
+
For maximum parallelism, `multi_process` supports nested parallelism where each process has its own thread pool. This is ideal for CPU-bound tasks that also benefit from I/O parallelism.
|
|
261
|
+
|
|
262
|
+
```python
|
|
263
|
+
from speedy_utils import multi_process
|
|
264
|
+
|
|
265
|
+
def process_data(item):
|
|
266
|
+
"""Process a data item with potential I/O operations."""
|
|
267
|
+
# Simulate CPU work
|
|
268
|
+
result = item ** 2
|
|
269
|
+
# Simulate I/O (each thread can do I/O in parallel)
|
|
270
|
+
time.sleep(0.01)
|
|
271
|
+
return result
|
|
272
|
+
|
|
273
|
+
data = list(range(100))
|
|
274
|
+
|
|
275
|
+
# 4 processes, each with 4 threads = 16 concurrent workers
|
|
276
|
+
results = multi_process(
|
|
277
|
+
process_data,
|
|
278
|
+
data,
|
|
279
|
+
num_procs=4, # Number of processes
|
|
280
|
+
num_threads=4, # Threads per process
|
|
281
|
+
backend='mp',
|
|
282
|
+
progress=True,
|
|
283
|
+
)
|
|
284
|
+
```
|
|
285
|
+
|
|
286
|
+
**Backend Options:**
|
|
287
|
+
|
|
288
|
+
| Backend | Description | Use Case |
|
|
289
|
+
|---------|-------------|----------|
|
|
290
|
+
| `'mp'` | Multi-processing with optional inner threads | CPU-bound work, bypasses GIL |
|
|
291
|
+
| `'safe'` | In-process thread pool (for testing) | Debugging, unit tests |
|
|
292
|
+
| `'seq'` | Sequential execution | Debugging, reproducibility |
|
|
293
|
+
|
|
294
|
+
**When to use `num_procs` vs `num_threads`:**
|
|
295
|
+
|
|
296
|
+
- **CPU-bound tasks**: Use `num_procs` > 1, `num_threads` = 1 (processes bypass GIL)
|
|
297
|
+
- **I/O-bound tasks**: Use `num_procs` = 1, `num_threads` > 1 (threads are lighter)
|
|
298
|
+
- **Mixed workloads**: Use both (e.g., `num_procs=4, num_threads=4`)
|
|
299
|
+
|
|
300
|
+
```python
|
|
301
|
+
# Example: Web scraping with multi-process + multi-thread
|
|
302
|
+
def fetch_and_parse(url):
|
|
303
|
+
response = requests.get(url) # I/O bound
|
|
304
|
+
return parse_content(response.text) # CPU bound
|
|
305
|
+
|
|
306
|
+
# 4 processes for parsing, 8 threads per process for fetching
|
|
307
|
+
results = multi_process(
|
|
308
|
+
fetch_and_parse,
|
|
309
|
+
urls,
|
|
310
|
+
num_procs=4,
|
|
311
|
+
num_threads=8,
|
|
312
|
+
backend='mp',
|
|
313
|
+
error_handler='log', # Continue on failed URLs
|
|
314
|
+
)
|
|
315
|
+
```
|
|
316
|
+
|
|
278
317
|
#### mpython (CLI Tool)
|
|
279
318
|
|
|
280
319
|
`mpython` is a CLI tool for running Python scripts in multiple tmux windows with automatic GPU/CPU allocation for parallel processing.
|
|
@@ -498,7 +537,7 @@ clock.log()
|
|
|
498
537
|
|
|
499
538
|
### LLM
|
|
500
539
|
|
|
501
|
-
The `LLM` class provides a unified interface for language model interactions with structured input/output handling. It supports text completion, structured outputs, caching, streaming, and
|
|
540
|
+
The `LLM` class provides a unified interface for language model interactions with structured input/output handling. It supports text completion, structured outputs, caching, streaming, and OpenAI-compatible client integration.
|
|
502
541
|
|
|
503
542
|
#### Basic Text Completion
|
|
504
543
|
|
|
@@ -511,9 +550,8 @@ llm = LLM(
|
|
|
511
550
|
)
|
|
512
551
|
|
|
513
552
|
# Simple text completion
|
|
514
|
-
|
|
515
|
-
print(
|
|
516
|
-
print(results[0]["messages"]) # Full conversation history
|
|
553
|
+
message = llm.chat_completion("What is Python?")
|
|
554
|
+
print(message.content) # The text response
|
|
517
555
|
```
|
|
518
556
|
|
|
519
557
|
#### Structured Output with Pydantic
|
|
@@ -528,12 +566,13 @@ class Sentiment(BaseModel):
|
|
|
528
566
|
|
|
529
567
|
llm = LLM(
|
|
530
568
|
instruction="Analyze the sentiment of the input.",
|
|
531
|
-
output_model=Sentiment,
|
|
532
569
|
model="gpt-4o-mini"
|
|
533
570
|
)
|
|
534
571
|
|
|
535
|
-
|
|
536
|
-
|
|
572
|
+
parsed: Sentiment = llm.pydantic_parse(
|
|
573
|
+
"I love this product!",
|
|
574
|
+
response_model=Sentiment,
|
|
575
|
+
)
|
|
537
576
|
print(f"Sentiment: {parsed.sentiment}, Confidence: {parsed.confidence}")
|
|
538
577
|
```
|
|
539
578
|
|
|
@@ -564,33 +603,13 @@ from openai import OpenAI
|
|
|
564
603
|
custom_client = OpenAI(base_url="http://localhost:8000/v1", api_key="your-key")
|
|
565
604
|
llm = LLM(client=custom_client, model="llama-2-7b")
|
|
566
605
|
|
|
567
|
-
# Using a port number
|
|
606
|
+
# Using a port number
|
|
568
607
|
llm = LLM(client=8000, model="llama-2-7b")
|
|
569
608
|
|
|
570
609
|
# Using a base URL string
|
|
571
610
|
llm = LLM(client="http://localhost:8000/v1", model="llama-2-7b")
|
|
572
611
|
```
|
|
573
612
|
|
|
574
|
-
#### VLLM Integration
|
|
575
|
-
|
|
576
|
-
Start and manage VLLM servers automatically:
|
|
577
|
-
|
|
578
|
-
```python
|
|
579
|
-
from llm_utils import LLM
|
|
580
|
-
|
|
581
|
-
llm = LLM(
|
|
582
|
-
vllm_cmd="vllm serve meta-llama/Llama-2-7b-chat-hf --port 8000",
|
|
583
|
-
model="meta-llama/Llama-2-7b-chat-hf"
|
|
584
|
-
)
|
|
585
|
-
|
|
586
|
-
# The server starts automatically and is cleaned up on exit
|
|
587
|
-
results = llm("Hello!")
|
|
588
|
-
|
|
589
|
-
# Cleanup is automatic when using context manager
|
|
590
|
-
with LLM(vllm_cmd="vllm serve ...") as llm:
|
|
591
|
-
results = llm("Hello!")
|
|
592
|
-
```
|
|
593
|
-
|
|
594
613
|
#### Caching
|
|
595
614
|
|
|
596
615
|
Enable response caching to avoid redundant API calls:
|
|
@@ -619,18 +638,45 @@ Handle reasoning models that provide thinking traces:
|
|
|
619
638
|
from llm_utils import LLM
|
|
620
639
|
|
|
621
640
|
# For models like DeepSeek-R1 that output reasoning
|
|
622
|
-
llm = LLM(
|
|
623
|
-
model="deepseek-reasoner",
|
|
624
|
-
is_reasoning_model=True
|
|
625
|
-
)
|
|
641
|
+
llm = LLM(model="deepseek-reasoner")
|
|
626
642
|
|
|
627
|
-
|
|
643
|
+
message = llm.chat_completion("Solve this math problem: 15 * 23")
|
|
628
644
|
|
|
629
645
|
# Access the final answer
|
|
630
|
-
answer =
|
|
646
|
+
answer = message.content
|
|
647
|
+
|
|
648
|
+
# Access reasoning content when the response includes it
|
|
649
|
+
reasoning = getattr(message, "reasoning_content", None)
|
|
650
|
+
```
|
|
651
|
+
|
|
652
|
+
For custom staged generation with explicit tags, `Qwen3LLM` also exposes
|
|
653
|
+
`complete_until()`:
|
|
654
|
+
|
|
655
|
+
```python
|
|
656
|
+
from llm_utils import Qwen3LLM
|
|
631
657
|
|
|
632
|
-
|
|
633
|
-
|
|
658
|
+
llm = Qwen3LLM(model="Qwen/Qwen3-0.6B")
|
|
659
|
+
|
|
660
|
+
memory_state = llm.complete_until(
|
|
661
|
+
[{"role": "user", "content": "Solve this in stages"}],
|
|
662
|
+
"<memory>",
|
|
663
|
+
stop="</memory>",
|
|
664
|
+
max_tokens=256,
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
think_state = llm.complete_until(
|
|
668
|
+
[{"role": "user", "content": "Solve this in stages"}],
|
|
669
|
+
memory_state.assistant_prompt_prefix + "\n<think_efficient>",
|
|
670
|
+
stop="</think_efficient>",
|
|
671
|
+
max_tokens=512,
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
final_state = llm.complete_until(
|
|
675
|
+
[{"role": "user", "content": "Solve this in stages"}],
|
|
676
|
+
think_state.assistant_prompt_prefix,
|
|
677
|
+
stop="<|im_end|>",
|
|
678
|
+
max_tokens=512,
|
|
679
|
+
)
|
|
634
680
|
```
|
|
635
681
|
|
|
636
682
|
#### Conversation History
|
|
@@ -94,20 +94,6 @@ cd speedy
|
|
|
94
94
|
uv sync
|
|
95
95
|
```
|
|
96
96
|
|
|
97
|
-
### Extras
|
|
98
|
-
|
|
99
|
-
Optional dependencies can be installed via extras. For the `ray` backend
|
|
100
|
-
support (requires Python >= 3.9):
|
|
101
|
-
|
|
102
|
-
```bash
|
|
103
|
-
pip install 'speedy-utils[ray]'
|
|
104
|
-
# or
|
|
105
|
-
uv pip install 'speedy-utils[ray]'
|
|
106
|
-
|
|
107
|
-
# developing this repo
|
|
108
|
-
uv sync --extra ray
|
|
109
|
-
```
|
|
110
|
-
|
|
111
97
|
## Updating from previous versions
|
|
112
98
|
|
|
113
99
|
To update from previous versions or switch to v1.x, first uninstall any old
|
|
@@ -217,6 +203,65 @@ results = multi_process(
|
|
|
217
203
|
print(results) # [0, 1, 4, 9, 16, None, 36, 49, 64, 81, None, 121]
|
|
218
204
|
```
|
|
219
205
|
|
|
206
|
+
#### Multi-Process with Inner Thread Pools
|
|
207
|
+
|
|
208
|
+
For maximum parallelism, `multi_process` supports nested parallelism where each process has its own thread pool. This is ideal for CPU-bound tasks that also benefit from I/O parallelism.
|
|
209
|
+
|
|
210
|
+
```python
|
|
211
|
+
from speedy_utils import multi_process
|
|
212
|
+
|
|
213
|
+
def process_data(item):
|
|
214
|
+
"""Process a data item with potential I/O operations."""
|
|
215
|
+
# Simulate CPU work
|
|
216
|
+
result = item ** 2
|
|
217
|
+
# Simulate I/O (each thread can do I/O in parallel)
|
|
218
|
+
time.sleep(0.01)
|
|
219
|
+
return result
|
|
220
|
+
|
|
221
|
+
data = list(range(100))
|
|
222
|
+
|
|
223
|
+
# 4 processes, each with 4 threads = 16 concurrent workers
|
|
224
|
+
results = multi_process(
|
|
225
|
+
process_data,
|
|
226
|
+
data,
|
|
227
|
+
num_procs=4, # Number of processes
|
|
228
|
+
num_threads=4, # Threads per process
|
|
229
|
+
backend='mp',
|
|
230
|
+
progress=True,
|
|
231
|
+
)
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
**Backend Options:**
|
|
235
|
+
|
|
236
|
+
| Backend | Description | Use Case |
|
|
237
|
+
|---------|-------------|----------|
|
|
238
|
+
| `'mp'` | Multi-processing with optional inner threads | CPU-bound work, bypasses GIL |
|
|
239
|
+
| `'safe'` | In-process thread pool (for testing) | Debugging, unit tests |
|
|
240
|
+
| `'seq'` | Sequential execution | Debugging, reproducibility |
|
|
241
|
+
|
|
242
|
+
**When to use `num_procs` vs `num_threads`:**
|
|
243
|
+
|
|
244
|
+
- **CPU-bound tasks**: Use `num_procs` > 1, `num_threads` = 1 (processes bypass GIL)
|
|
245
|
+
- **I/O-bound tasks**: Use `num_procs` = 1, `num_threads` > 1 (threads are lighter)
|
|
246
|
+
- **Mixed workloads**: Use both (e.g., `num_procs=4, num_threads=4`)
|
|
247
|
+
|
|
248
|
+
```python
|
|
249
|
+
# Example: Web scraping with multi-process + multi-thread
|
|
250
|
+
def fetch_and_parse(url):
|
|
251
|
+
response = requests.get(url) # I/O bound
|
|
252
|
+
return parse_content(response.text) # CPU bound
|
|
253
|
+
|
|
254
|
+
# 4 processes for parsing, 8 threads per process for fetching
|
|
255
|
+
results = multi_process(
|
|
256
|
+
fetch_and_parse,
|
|
257
|
+
urls,
|
|
258
|
+
num_procs=4,
|
|
259
|
+
num_threads=8,
|
|
260
|
+
backend='mp',
|
|
261
|
+
error_handler='log', # Continue on failed URLs
|
|
262
|
+
)
|
|
263
|
+
```
|
|
264
|
+
|
|
220
265
|
#### mpython (CLI Tool)
|
|
221
266
|
|
|
222
267
|
`mpython` is a CLI tool for running Python scripts in multiple tmux windows with automatic GPU/CPU allocation for parallel processing.
|
|
@@ -440,7 +485,7 @@ clock.log()
|
|
|
440
485
|
|
|
441
486
|
### LLM
|
|
442
487
|
|
|
443
|
-
The `LLM` class provides a unified interface for language model interactions with structured input/output handling. It supports text completion, structured outputs, caching, streaming, and
|
|
488
|
+
The `LLM` class provides a unified interface for language model interactions with structured input/output handling. It supports text completion, structured outputs, caching, streaming, and OpenAI-compatible client integration.
|
|
444
489
|
|
|
445
490
|
#### Basic Text Completion
|
|
446
491
|
|
|
@@ -453,9 +498,8 @@ llm = LLM(
|
|
|
453
498
|
)
|
|
454
499
|
|
|
455
500
|
# Simple text completion
|
|
456
|
-
|
|
457
|
-
print(
|
|
458
|
-
print(results[0]["messages"]) # Full conversation history
|
|
501
|
+
message = llm.chat_completion("What is Python?")
|
|
502
|
+
print(message.content) # The text response
|
|
459
503
|
```
|
|
460
504
|
|
|
461
505
|
#### Structured Output with Pydantic
|
|
@@ -470,12 +514,13 @@ class Sentiment(BaseModel):
|
|
|
470
514
|
|
|
471
515
|
llm = LLM(
|
|
472
516
|
instruction="Analyze the sentiment of the input.",
|
|
473
|
-
output_model=Sentiment,
|
|
474
517
|
model="gpt-4o-mini"
|
|
475
518
|
)
|
|
476
519
|
|
|
477
|
-
|
|
478
|
-
|
|
520
|
+
parsed: Sentiment = llm.pydantic_parse(
|
|
521
|
+
"I love this product!",
|
|
522
|
+
response_model=Sentiment,
|
|
523
|
+
)
|
|
479
524
|
print(f"Sentiment: {parsed.sentiment}, Confidence: {parsed.confidence}")
|
|
480
525
|
```
|
|
481
526
|
|
|
@@ -506,33 +551,13 @@ from openai import OpenAI
|
|
|
506
551
|
custom_client = OpenAI(base_url="http://localhost:8000/v1", api_key="your-key")
|
|
507
552
|
llm = LLM(client=custom_client, model="llama-2-7b")
|
|
508
553
|
|
|
509
|
-
# Using a port number
|
|
554
|
+
# Using a port number
|
|
510
555
|
llm = LLM(client=8000, model="llama-2-7b")
|
|
511
556
|
|
|
512
557
|
# Using a base URL string
|
|
513
558
|
llm = LLM(client="http://localhost:8000/v1", model="llama-2-7b")
|
|
514
559
|
```
|
|
515
560
|
|
|
516
|
-
#### VLLM Integration
|
|
517
|
-
|
|
518
|
-
Start and manage VLLM servers automatically:
|
|
519
|
-
|
|
520
|
-
```python
|
|
521
|
-
from llm_utils import LLM
|
|
522
|
-
|
|
523
|
-
llm = LLM(
|
|
524
|
-
vllm_cmd="vllm serve meta-llama/Llama-2-7b-chat-hf --port 8000",
|
|
525
|
-
model="meta-llama/Llama-2-7b-chat-hf"
|
|
526
|
-
)
|
|
527
|
-
|
|
528
|
-
# The server starts automatically and is cleaned up on exit
|
|
529
|
-
results = llm("Hello!")
|
|
530
|
-
|
|
531
|
-
# Cleanup is automatic when using context manager
|
|
532
|
-
with LLM(vllm_cmd="vllm serve ...") as llm:
|
|
533
|
-
results = llm("Hello!")
|
|
534
|
-
```
|
|
535
|
-
|
|
536
561
|
#### Caching
|
|
537
562
|
|
|
538
563
|
Enable response caching to avoid redundant API calls:
|
|
@@ -561,18 +586,45 @@ Handle reasoning models that provide thinking traces:
|
|
|
561
586
|
from llm_utils import LLM
|
|
562
587
|
|
|
563
588
|
# For models like DeepSeek-R1 that output reasoning
|
|
564
|
-
llm = LLM(
|
|
565
|
-
model="deepseek-reasoner",
|
|
566
|
-
is_reasoning_model=True
|
|
567
|
-
)
|
|
589
|
+
llm = LLM(model="deepseek-reasoner")
|
|
568
590
|
|
|
569
|
-
|
|
591
|
+
message = llm.chat_completion("Solve this math problem: 15 * 23")
|
|
570
592
|
|
|
571
593
|
# Access the final answer
|
|
572
|
-
answer =
|
|
594
|
+
answer = message.content
|
|
595
|
+
|
|
596
|
+
# Access reasoning content when the response includes it
|
|
597
|
+
reasoning = getattr(message, "reasoning_content", None)
|
|
598
|
+
```
|
|
599
|
+
|
|
600
|
+
For custom staged generation with explicit tags, `Qwen3LLM` also exposes
|
|
601
|
+
`complete_until()`:
|
|
602
|
+
|
|
603
|
+
```python
|
|
604
|
+
from llm_utils import Qwen3LLM
|
|
573
605
|
|
|
574
|
-
|
|
575
|
-
|
|
606
|
+
llm = Qwen3LLM(model="Qwen/Qwen3-0.6B")
|
|
607
|
+
|
|
608
|
+
memory_state = llm.complete_until(
|
|
609
|
+
[{"role": "user", "content": "Solve this in stages"}],
|
|
610
|
+
"<memory>",
|
|
611
|
+
stop="</memory>",
|
|
612
|
+
max_tokens=256,
|
|
613
|
+
)
|
|
614
|
+
|
|
615
|
+
think_state = llm.complete_until(
|
|
616
|
+
[{"role": "user", "content": "Solve this in stages"}],
|
|
617
|
+
memory_state.assistant_prompt_prefix + "\n<think_efficient>",
|
|
618
|
+
stop="</think_efficient>",
|
|
619
|
+
max_tokens=512,
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
final_state = llm.complete_until(
|
|
623
|
+
[{"role": "user", "content": "Solve this in stages"}],
|
|
624
|
+
think_state.assistant_prompt_prefix,
|
|
625
|
+
stop="<|im_end|>",
|
|
626
|
+
max_tokens=512,
|
|
627
|
+
)
|
|
576
628
|
```
|
|
577
629
|
|
|
578
630
|
#### Conversation History
|
|
@@ -23,8 +23,8 @@ print()
|
|
|
23
23
|
print('✓ Test 2: Generate from text')
|
|
24
24
|
result = lm.generate('The answer is', max_tokens=10, temperature=0.5)
|
|
25
25
|
print(f' Input: "The answer is"')
|
|
26
|
-
print(f' Output: {result
|
|
27
|
-
assert
|
|
26
|
+
print(f' Output: {result.text[:50]}')
|
|
27
|
+
assert result.text is not None
|
|
28
28
|
print()
|
|
29
29
|
|
|
30
30
|
# Test 3: Generate from token IDs
|
|
@@ -32,10 +32,10 @@ print('✓ Test 3: Generate from token IDs')
|
|
|
32
32
|
input_ids = lm.encode('Python is')
|
|
33
33
|
result = lm.generate(input_ids, max_tokens=15, return_token_ids=True)
|
|
34
34
|
print(f' Input IDs: {input_ids}')
|
|
35
|
-
print(f' Output: {result
|
|
36
|
-
print(f' Output IDs: {result
|
|
37
|
-
assert
|
|
38
|
-
assert
|
|
35
|
+
print(f' Output: {result.text[:50]}')
|
|
36
|
+
print(f' Output IDs: {getattr(result, "token_ids", "N/A")}')
|
|
37
|
+
assert result.text is not None
|
|
38
|
+
assert hasattr(result, "token_ids") and result.token_ids is not None
|
|
39
39
|
print()
|
|
40
40
|
|
|
41
41
|
# Test 4: Multiple generations
|
|
@@ -43,7 +43,7 @@ print('✓ Test 4: Multiple generations (n=3)')
|
|
|
43
43
|
results = lm.generate('AI is', max_tokens=10, n=3, temperature=0.8)
|
|
44
44
|
print(f' Input: "AI is"')
|
|
45
45
|
for i, r in enumerate(results, 1):
|
|
46
|
-
print(f' {i}. {r
|
|
46
|
+
print(f' {i}. {r.text[:40]}')
|
|
47
47
|
# Note: Server may return fewer choices than requested
|
|
48
48
|
assert isinstance(results, list)
|
|
49
49
|
assert len(results) >= 1
|
|
@@ -53,8 +53,8 @@ print()
|
|
|
53
53
|
print('✓ Test 5: Temperature control')
|
|
54
54
|
low_temp = lm.generate('1 + 1 =', max_tokens=5, temperature=0.1, seed=42)
|
|
55
55
|
high_temp = lm.generate('1 + 1 =', max_tokens=5, temperature=1.5, seed=123)
|
|
56
|
-
print(f' Low temp (0.1): {low_temp
|
|
57
|
-
print(f' High temp (1.5): {high_temp
|
|
56
|
+
print(f' Low temp (0.1): {low_temp.text[:30]}')
|
|
57
|
+
print(f' High temp (1.5): {high_temp.text[:30]}')
|
|
58
58
|
print()
|
|
59
59
|
|
|
60
60
|
# Test 6: Stop sequences
|
|
@@ -66,16 +66,16 @@ result = lm.generate(
|
|
|
66
66
|
temperature=0.7,
|
|
67
67
|
)
|
|
68
68
|
print(f' Input: "Count: 1, 2, 3,"')
|
|
69
|
-
print(f' Output (stop at ", 6"): {result
|
|
69
|
+
print(f' Output (stop at ", 6"): {result.text[:50]}')
|
|
70
70
|
print()
|
|
71
71
|
|
|
72
72
|
# Test 7: Seed reproducibility
|
|
73
73
|
print('✓ Test 7: Seed reproducibility')
|
|
74
74
|
r1 = lm.generate('Test', max_tokens=10, seed=999, temperature=0.8)
|
|
75
75
|
r2 = lm.generate('Test', max_tokens=10, seed=999, temperature=0.8)
|
|
76
|
-
print(f' Run 1: {r1
|
|
77
|
-
print(f' Run 2: {r2
|
|
78
|
-
print(f' Match: {r1
|
|
76
|
+
print(f' Run 1: {r1.text[:40]}')
|
|
77
|
+
print(f' Run 2: {r2.text[:40]}')
|
|
78
|
+
print(f' Match: {r1.text == r2.text}')
|
|
79
79
|
print()
|
|
80
80
|
|
|
81
81
|
print('=' * 60)
|