speedy-utils 2.0.6__tar.gz → 2.0.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/PKG-INFO +1 -15
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/pyproject.toml +7 -15
- speedy_utils-2.0.7/pyrightconfig.json +20 -0
- speedy_utils-2.0.7/src/datasets_utils/__init__.py +30 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/datasets_utils/convert_to_arrow.py +9 -2
- {speedy_utils-2.0.6/src/speedy_utils/common → speedy_utils-2.0.7/src/datasets_utils/pcat}/__init__.py +0 -0
- speedy_utils-2.0.7/src/datasets_utils/pcat/_shared.py +1108 -0
- speedy_utils-2.0.7/src/datasets_utils/pcat/cli.py +146 -0
- speedy_utils-2.0.7/src/datasets_utils/pcat/hf_dataset.py +11 -0
- speedy_utils-2.0.7/src/datasets_utils/pcat/jsonl.py +11 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/datasets_utils/viz_chat.py +153 -17
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/scripts/sp_chat.py +50 -21
- {speedy_utils-2.0.6/src/speedy_utils/scripts → speedy_utils-2.0.7/src/speedy_utils/common}/__init__.py +0 -0
- speedy_utils-2.0.7/src/speedy_utils/scripts/__init__.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/vision_utils/plot.py +1 -1
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_import_time.py +2 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_viz_chat.py +207 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/uv.lock +33 -2591
- speedy_utils-2.0.6/pyrightconfig.json +0 -8
- speedy_utils-2.0.6/src/datasets_utils/__init__.py +0 -25
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.codex +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.githooks/pre-commit +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.githooks/pre-push +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/prompts/improveParallelErrorHandling.prompt.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/caching-utilities/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/caching-utilities/examples/caching_example.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/dataset-processing-multiprocessing/CONFIG_REFERENCE.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/dataset-processing-multiprocessing/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/dataset-processing-multiprocessing/example_tokenize_pack.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/edit-llm-inference-style/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/io-utilities/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/io-utilities/examples/io_example.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/llm-integration/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/llm-integration/examples/llm_example.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/make-vscode-happy/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/make-vscode-happy/references/diagnostic-playbook.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/multi-threading-processing/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/ray-distributed-computing/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/skill-creation/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/vision-utilities/SKILL.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/vision-utilities/examples/vision_example.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/workflows/publish.yml +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.gitignore +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.pre-commit-config.yaml +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/AGENTS.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/CLAUDE.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/CODING_STYLE.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/README.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/bumpversion.sh +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/debug_generate_response.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/debug_n_param.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/debug_n_structure.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/integration_test.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/multi_thread.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_decode_api.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_endpoints.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_generate.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_generate_endpoint.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_qwen3_generate.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_qwen3_generate.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/GENERATE_QUICKREF.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/MIGRATION_THINKING.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/PR_LIBRARY_ERROR_HANDLING_REFACTOR_PLAN.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/TDD.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/TOKENIZATION.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/TOKENIZATION_IMPLEMENTATION.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/generate_example.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/llm_public_api_example.ipynb +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/model_specific_thinking_example.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/tokenization_example.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/vision_utils_example.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/experiments/exp1/dockerfile +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/experiments/exp1/run_in_docker.sh +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/notebooks/demo_multi_process.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/notebooks/llm_utils/llm_as_a_judge.ipynb +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/notebooks/test.ipynb +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/ruff.toml +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/bug_simple.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/debug_import_time.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/debug_multi_process.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/deploy.sh +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/imports.sh +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/install_hooks.sh +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test_error_handling.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test_import_time_vision.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test_locals.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test_ray_mp.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/setup.cfg +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/__init__.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/_traceback.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/chat_format/__init__.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/chat_format/display.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/chat_format/transform.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/chat_format/utils.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/__init__.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/base_prompt_builder.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/llm.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/llm_qwen3.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/llm_signature.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/lm_base.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/openai_memoize.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/signature.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/utils.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/utils.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/__imports.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/__init__.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/clock.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/exceptions.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/function_decorator.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/logger.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/notebook_utils.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/patcher.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/report_manager.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_cache.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_error.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_io.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_misc.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_print.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/__init__.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/_mp_backends.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/_mp_progress.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/_mp_spawn.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/_multi_process.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/common.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/dataset_sharding.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/process.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/thread.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/scripts/kill_mpython.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/scripts/mpython.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/scripts/openapi_client_codegen.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/scripts/prefetch_large_model.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/vision_utils/README.md +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/vision_utils/__init__.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/vision_utils/io_utils.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/import_all.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/import_time_report.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_llm_call_contract.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_llm_mixins.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_llm_signature.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_openai_memoize_completion_cache.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_openai_proxy_localhost.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_public_api.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_sp_chat.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/sample_objects.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_identify.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_load_jsonl.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_logger.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_logger_format.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_memoize.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_memoize_typing.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_mp_no_main_guard.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_mpython.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_multi_process.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_multi_thread.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_multithread_error_trace.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_prefetch_large_model.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_process.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_process_update.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_thread.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_tokenization.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_utils_io.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tools/check_syntax.py +0 -0
- {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tools/uv_test.sh +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: speedy-utils
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.7
|
|
4
4
|
Summary: Fast and easy-to-use package for data science
|
|
5
5
|
Project-URL: Homepage, https://github.com/anhvth/speedy
|
|
6
6
|
Project-URL: Repository, https://github.com/anhvth/speedy
|
|
@@ -18,32 +18,18 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.13
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.14
|
|
20
20
|
Requires-Python: >=3.9
|
|
21
|
-
Requires-Dist: aiohttp
|
|
22
|
-
Requires-Dist: bump2version
|
|
23
21
|
Requires-Dist: cachetools
|
|
24
22
|
Requires-Dist: chainlit>=2.0
|
|
25
23
|
Requires-Dist: datasets
|
|
26
|
-
Requires-Dist: debugpy
|
|
27
24
|
Requires-Dist: dill
|
|
28
25
|
Requires-Dist: fastcore
|
|
29
|
-
Requires-Dist: fastprogress
|
|
30
|
-
Requires-Dist: freezegun
|
|
31
|
-
Requires-Dist: ipdb
|
|
32
|
-
Requires-Dist: ipywidgets
|
|
33
|
-
Requires-Dist: json-repair
|
|
34
|
-
Requires-Dist: jupyterlab
|
|
35
26
|
Requires-Dist: loguru
|
|
36
27
|
Requires-Dist: matplotlib
|
|
37
28
|
Requires-Dist: numpy
|
|
38
29
|
Requires-Dist: openai
|
|
39
|
-
Requires-Dist: packaging
|
|
40
30
|
Requires-Dist: pandas
|
|
41
31
|
Requires-Dist: pydantic
|
|
42
|
-
Requires-Dist: pytest
|
|
43
|
-
Requires-Dist: requests
|
|
44
32
|
Requires-Dist: rich>=14.3.1
|
|
45
|
-
Requires-Dist: ruff
|
|
46
|
-
Requires-Dist: scikit-learn
|
|
47
33
|
Requires-Dist: tabulate
|
|
48
34
|
Requires-Dist: tqdm
|
|
49
35
|
Requires-Dist: xxhash
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "speedy-utils"
|
|
3
|
-
version = "2.0.
|
|
3
|
+
version = "2.0.7"
|
|
4
4
|
description = "Fast and easy-to-use package for data science"
|
|
5
5
|
authors = [{ name = "AnhVTH", email = "anhvth.226@gmail.com" }]
|
|
6
6
|
readme = "README.md"
|
|
@@ -8,15 +8,9 @@ license = { text = "MIT" }
|
|
|
8
8
|
requires-python = ">=3.9"
|
|
9
9
|
dependencies = [
|
|
10
10
|
"numpy",
|
|
11
|
-
"requests",
|
|
12
11
|
"xxhash",
|
|
13
12
|
"loguru",
|
|
14
13
|
"fastcore",
|
|
15
|
-
"debugpy",
|
|
16
|
-
"ipywidgets",
|
|
17
|
-
"jupyterlab",
|
|
18
|
-
"ipdb",
|
|
19
|
-
"scikit-learn",
|
|
20
14
|
"matplotlib",
|
|
21
15
|
"pandas",
|
|
22
16
|
"tabulate",
|
|
@@ -24,15 +18,7 @@ dependencies = [
|
|
|
24
18
|
"tqdm",
|
|
25
19
|
"cachetools",
|
|
26
20
|
"dill",
|
|
27
|
-
"ruff",
|
|
28
|
-
"bump2version",
|
|
29
|
-
"json-repair",
|
|
30
|
-
"fastprogress",
|
|
31
|
-
"freezegun",
|
|
32
|
-
"packaging",
|
|
33
21
|
"openai",
|
|
34
|
-
"aiohttp",
|
|
35
|
-
"pytest",
|
|
36
22
|
"rich>=14.3.1",
|
|
37
23
|
"chainlit>=2.0",
|
|
38
24
|
"datasets",
|
|
@@ -65,6 +51,9 @@ kill-mpython = "speedy_utils.scripts.kill_mpython:main"
|
|
|
65
51
|
spu-prefetch-large-model = "speedy_utils.scripts.prefetch_large_model:main"
|
|
66
52
|
sp_chat = "llm_utils.scripts.sp_chat:main"
|
|
67
53
|
viz_chat = "datasets_utils.viz_chat:main"
|
|
54
|
+
pcat = "datasets_utils.pcat.cli:main"
|
|
55
|
+
pcat-jsonl = "datasets_utils.pcat.jsonl:main"
|
|
56
|
+
pcat-hf-dataset = "datasets_utils.pcat.hf_dataset:main"
|
|
68
57
|
|
|
69
58
|
[build-system]
|
|
70
59
|
requires = ["hatchling"]
|
|
@@ -154,4 +143,7 @@ norecursedirs = ["debug", "examples", "experiments", "notebooks", "scripts", ".g
|
|
|
154
143
|
dev = [
|
|
155
144
|
"pyright>=1.1.408",
|
|
156
145
|
"pytest-xdist>=3.8.0",
|
|
146
|
+
"pytest",
|
|
147
|
+
"freezegun",
|
|
148
|
+
"ruff",
|
|
157
149
|
]
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"include": [
|
|
3
|
+
"src",
|
|
4
|
+
"tests"
|
|
5
|
+
],
|
|
6
|
+
"exclude": [
|
|
7
|
+
"debug",
|
|
8
|
+
"examples",
|
|
9
|
+
"scripts",
|
|
10
|
+
"**/node_modules",
|
|
11
|
+
"**/__pycache__",
|
|
12
|
+
"**/.*",
|
|
13
|
+
".venv"
|
|
14
|
+
],
|
|
15
|
+
"reportUnsupportedDunderAll": "none",
|
|
16
|
+
"reportMissingImports": "none",
|
|
17
|
+
"reportMissingModuleSource": "none",
|
|
18
|
+
"reportAttributeAccessIssue": "none",
|
|
19
|
+
"pythonVersion": "3.11"
|
|
20
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""datasets_utils - dataset inspection and conversion helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"convert_to_arrow",
|
|
8
|
+
"extract_tools",
|
|
9
|
+
"load_data",
|
|
10
|
+
"main",
|
|
11
|
+
"normalize_messages",
|
|
12
|
+
"parse_args",
|
|
13
|
+
"print_item",
|
|
14
|
+
"sample_items",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def __getattr__(name: str):
|
|
19
|
+
if name == "convert_to_arrow":
|
|
20
|
+
from datasets_utils.convert_to_arrow import convert_to_arrow
|
|
21
|
+
|
|
22
|
+
return convert_to_arrow
|
|
23
|
+
if name in __all__:
|
|
24
|
+
import sys as _sys
|
|
25
|
+
|
|
26
|
+
mod = _sys.modules["datasets_utils.viz_chat"] = __import__(
|
|
27
|
+
"datasets_utils.viz_chat", fromlist=[name]
|
|
28
|
+
)
|
|
29
|
+
return getattr(mod, name)
|
|
30
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
os.environ[
|
|
4
|
+
os.environ["HF_HOME"] = "/mnt/data/anhvth/tmp/"
|
|
5
5
|
import sys
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
|
|
@@ -25,7 +25,13 @@ def convert_to_arrow(path_to_dir):
|
|
|
25
25
|
# 2. Load and Save
|
|
26
26
|
# 'keep_in_memory=False' ensures we don't crash RAM if the files are huge
|
|
27
27
|
n_proc = os.cpu_count() if os.cpu_count() is not None else 1
|
|
28
|
-
dataset = load_dataset(
|
|
28
|
+
dataset = load_dataset(
|
|
29
|
+
"parquet",
|
|
30
|
+
data_files=parquet_files,
|
|
31
|
+
split="train",
|
|
32
|
+
keep_in_memory=True,
|
|
33
|
+
num_proc=n_proc,
|
|
34
|
+
)
|
|
29
35
|
|
|
30
36
|
# This creates the native Arrow format for fast loading
|
|
31
37
|
dataset.save_to_disk(output_dir)
|
|
@@ -33,6 +39,7 @@ def convert_to_arrow(path_to_dir):
|
|
|
33
39
|
print(f"✅ Success! Saved to: {output_dir}")
|
|
34
40
|
print("Now load it instantly with: load_from_disk('path')")
|
|
35
41
|
|
|
42
|
+
|
|
36
43
|
if __name__ == "__main__":
|
|
37
44
|
if len(sys.argv) < 2:
|
|
38
45
|
print("Usage: python convert_to_arrow.py <path_to_dir>")
|
|
File without changes
|