speedy-utils 1.1.33__tar.gz → 1.1.34__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/caching-utilities/examples/caching_example.py +4 -2
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/io-utilities/examples/io_example.py +7 -5
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/llm-integration/examples/llm_example.py +1 -0
- speedy_utils-1.1.34/.github/skills/ray-distributed-computing/SKILL.md +704 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/vision-utilities/examples/vision_example.py +5 -2
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/PKG-INFO +1 -1
- speedy_utils-1.1.34/docs/IMPLEMENTATION.md +149 -0
- speedy_utils-1.1.34/docs/QUICKSTART.md +109 -0
- speedy_utils-1.1.34/docs/zero_copy_sharing.md +235 -0
- speedy_utils-1.1.34/examples/pytorch_large_model.py +285 -0
- speedy_utils-1.1.34/examples/shared_kwargs_example.py +200 -0
- speedy_utils-1.1.34/examples/test_share_ray.py +24 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/pyproject.toml +1 -1
- speedy_utils-1.1.34/src/datasets/convert_to_arrow.py +40 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/utils.py +5 -1
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/multi_worker/process.py +121 -25
- speedy_utils-1.1.34/src/vision_utils/__init__.py +11 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/vision_utils/io_utils.py +15 -14
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/vision_utils/plot.py +2 -2
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/import_time_report.py +2 -1
- speedy_utils-1.1.34/tests/integration_test.py +51 -0
- speedy_utils-1.1.34/tests/test_pytorch_sharing.py +269 -0
- speedy_utils-1.1.34/tests/test_shared_kwargs.py +197 -0
- speedy_utils-1.1.33/src/vision_utils/__init__.py +0 -4
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/copilot-instructions.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/caching-utilities/SKILL.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/io-utilities/SKILL.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/llm-integration/SKILL.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/multi-threading-processing/SKILL.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/skill-creation/SKILL.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/vision-utilities/SKILL.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/workflows/publish.yml +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.gitignore +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/.pre-commit-config.yaml +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/README.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/bumpversion.sh +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/examples/temperature_range_example.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/examples/vision_utils_example.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/experiments/exp1/dockerfile +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/experiments/exp1/run_in_docker.sh +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/experiments/exp1/test.png +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/experiments/test_read_image.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/notebooks/README.ipynb +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/notebooks/llm_utils/llm_as_a_judge.ipynb +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/notebooks/ray_tutorial.ipynb +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/notebooks/test_multi_thread.ipynb +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/ruff.toml +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/scripts/debug_import_time.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/scripts/deploy.sh +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/scripts/imports.sh +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/scripts/test_import_time_vision.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/setup.cfg +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/__init__.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/chat_format/__init__.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/chat_format/display.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/chat_format/transform.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/chat_format/utils.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/group_messages.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/__init__.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/async_lm/__init__.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/async_lm/_utils.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/async_lm/async_llm_task.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/async_lm/async_lm.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/async_lm/async_lm_base.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/async_lm/lm_specific.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/base_prompt_builder.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/llm.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/llm_signature.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/lm_base.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/mixins.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/openai_memoize.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/lm/signature.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/scripts/README.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/scripts/vllm_load_balancer.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/scripts/vllm_serve.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/vector_cache/__init__.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/vector_cache/cli.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/vector_cache/core.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/vector_cache/types.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/llm_utils/vector_cache/utils.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/__imports.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/__init__.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/__init__.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/clock.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/function_decorator.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/logger.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/notebook_utils.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/patcher.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/report_manager.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/utils_cache.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/utils_io.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/utils_misc.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/common/utils_print.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/multi_worker/__init__.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/multi_worker/thread.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/scripts/__init__.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/scripts/mpython.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/speedy_utils/scripts/openapi_client_codegen.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/src/vision_utils/README.md +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/import_all.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/llm_utils/test_llm_mixins.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/sample_objects.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/test.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/test_logger.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/test_logger_format.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/test_memoize_typing.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/test_mpython.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/test_multithread_error_trace.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/test_process.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/test_process_update.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/tests/test_thread.py +0 -0
- {speedy_utils-1.1.33 → speedy_utils-1.1.34}/uv.lock +0 -0
{speedy_utils-1.1.33 → speedy_utils-1.1.34}/.github/skills/io-utilities/examples/io_example.py
RENAMED
|
@@ -1,23 +1,25 @@
|
|
|
1
|
-
from speedy_utils import dump_json_or_pickle, load_by_ext, fast_load_jsonl
|
|
2
1
|
import os
|
|
3
2
|
|
|
3
|
+
from speedy_utils import dump_json_or_pickle, fast_load_jsonl, load_by_ext
|
|
4
|
+
|
|
5
|
+
|
|
4
6
|
def main():
|
|
5
7
|
# 1. Create some dummy data
|
|
6
8
|
data = [{"id": i, "value": f"item_{i}"} for i in range(100)]
|
|
7
|
-
|
|
9
|
+
|
|
8
10
|
# 2. Dump to JSONL
|
|
9
11
|
print("Dumping to data.jsonl...")
|
|
10
12
|
dump_json_or_pickle(data, 'data.jsonl')
|
|
11
|
-
|
|
13
|
+
|
|
12
14
|
# 3. Dump to Pickle
|
|
13
15
|
print("Dumping to data.pkl...")
|
|
14
16
|
dump_json_or_pickle(data, 'data.pkl')
|
|
15
|
-
|
|
17
|
+
|
|
16
18
|
# 4. Load using load_by_ext
|
|
17
19
|
print("Loading data.pkl...")
|
|
18
20
|
loaded_pkl = load_by_ext('data.pkl')
|
|
19
21
|
print(f"Loaded {len(loaded_pkl)} items from pickle.")
|
|
20
|
-
|
|
22
|
+
|
|
21
23
|
# 5. Stream using fast_load_jsonl
|
|
22
24
|
print("Streaming data.jsonl...")
|
|
23
25
|
count = 0
|