speedy-utils 2.0.6__tar.gz → 2.0.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/PKG-INFO +1 -15
  2. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/pyproject.toml +7 -15
  3. speedy_utils-2.0.7/pyrightconfig.json +20 -0
  4. speedy_utils-2.0.7/src/datasets_utils/__init__.py +30 -0
  5. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/datasets_utils/convert_to_arrow.py +9 -2
  6. {speedy_utils-2.0.6/src/speedy_utils/common → speedy_utils-2.0.7/src/datasets_utils/pcat}/__init__.py +0 -0
  7. speedy_utils-2.0.7/src/datasets_utils/pcat/_shared.py +1108 -0
  8. speedy_utils-2.0.7/src/datasets_utils/pcat/cli.py +146 -0
  9. speedy_utils-2.0.7/src/datasets_utils/pcat/hf_dataset.py +11 -0
  10. speedy_utils-2.0.7/src/datasets_utils/pcat/jsonl.py +11 -0
  11. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/datasets_utils/viz_chat.py +153 -17
  12. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/scripts/sp_chat.py +50 -21
  13. {speedy_utils-2.0.6/src/speedy_utils/scripts → speedy_utils-2.0.7/src/speedy_utils/common}/__init__.py +0 -0
  14. speedy_utils-2.0.7/src/speedy_utils/scripts/__init__.py +0 -0
  15. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/vision_utils/plot.py +1 -1
  16. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_import_time.py +2 -0
  17. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_viz_chat.py +207 -0
  18. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/uv.lock +33 -2591
  19. speedy_utils-2.0.6/pyrightconfig.json +0 -8
  20. speedy_utils-2.0.6/src/datasets_utils/__init__.py +0 -25
  21. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.codex +0 -0
  22. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.githooks/pre-commit +0 -0
  23. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.githooks/pre-push +0 -0
  24. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/prompts/improveParallelErrorHandling.prompt.md +0 -0
  25. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/caching-utilities/SKILL.md +0 -0
  26. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/caching-utilities/examples/caching_example.py +0 -0
  27. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/dataset-processing-multiprocessing/CONFIG_REFERENCE.md +0 -0
  28. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/dataset-processing-multiprocessing/SKILL.md +0 -0
  29. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/dataset-processing-multiprocessing/example_tokenize_pack.py +0 -0
  30. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/edit-llm-inference-style/SKILL.md +0 -0
  31. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/io-utilities/SKILL.md +0 -0
  32. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/io-utilities/examples/io_example.py +0 -0
  33. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/llm-integration/SKILL.md +0 -0
  34. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/llm-integration/examples/llm_example.py +0 -0
  35. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/make-vscode-happy/SKILL.md +0 -0
  36. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/make-vscode-happy/references/diagnostic-playbook.md +0 -0
  37. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/multi-threading-processing/SKILL.md +0 -0
  38. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/ray-distributed-computing/SKILL.md +0 -0
  39. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/skill-creation/SKILL.md +0 -0
  40. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/vision-utilities/SKILL.md +0 -0
  41. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/skills/vision-utilities/examples/vision_example.py +0 -0
  42. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.github/workflows/publish.yml +0 -0
  43. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.gitignore +0 -0
  44. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/.pre-commit-config.yaml +0 -0
  45. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/AGENTS.md +0 -0
  46. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/CLAUDE.md +0 -0
  47. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/CODING_STYLE.md +0 -0
  48. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/README.md +0 -0
  49. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/bumpversion.sh +0 -0
  50. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/debug_generate_response.py +0 -0
  51. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/debug_n_param.py +0 -0
  52. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/debug_n_structure.py +0 -0
  53. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/integration_test.py +0 -0
  54. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/multi_thread.py +0 -0
  55. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_decode_api.py +0 -0
  56. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_endpoints.py +0 -0
  57. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_generate.py +0 -0
  58. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_generate_endpoint.py +0 -0
  59. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_qwen3_generate.md +0 -0
  60. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/debug/test_qwen3_generate.py +0 -0
  61. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/GENERATE_QUICKREF.md +0 -0
  62. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/MIGRATION_THINKING.md +0 -0
  63. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/PR_LIBRARY_ERROR_HANDLING_REFACTOR_PLAN.md +0 -0
  64. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/TDD.md +0 -0
  65. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/TOKENIZATION.md +0 -0
  66. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/docs/TOKENIZATION_IMPLEMENTATION.md +0 -0
  67. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/generate_example.py +0 -0
  68. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/llm_public_api_example.ipynb +0 -0
  69. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/model_specific_thinking_example.py +0 -0
  70. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/tokenization_example.py +0 -0
  71. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/examples/vision_utils_example.py +0 -0
  72. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/experiments/exp1/dockerfile +0 -0
  73. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/experiments/exp1/run_in_docker.sh +0 -0
  74. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/notebooks/demo_multi_process.py +0 -0
  75. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/notebooks/llm_utils/llm_as_a_judge.ipynb +0 -0
  76. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/notebooks/test.ipynb +0 -0
  77. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/ruff.toml +0 -0
  78. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/bug_simple.py +0 -0
  79. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/debug_import_time.py +0 -0
  80. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/debug_multi_process.py +0 -0
  81. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/deploy.sh +0 -0
  82. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/imports.sh +0 -0
  83. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/install_hooks.sh +0 -0
  84. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test.py +0 -0
  85. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test_error_handling.py +0 -0
  86. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test_import_time_vision.py +0 -0
  87. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test_locals.py +0 -0
  88. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/scripts/test_ray_mp.py +0 -0
  89. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/setup.cfg +0 -0
  90. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/__init__.py +0 -0
  91. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/_traceback.py +0 -0
  92. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/chat_format/__init__.py +0 -0
  93. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/chat_format/display.py +0 -0
  94. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/chat_format/transform.py +0 -0
  95. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/chat_format/utils.py +0 -0
  96. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/__init__.py +0 -0
  97. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/base_prompt_builder.py +0 -0
  98. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/llm.py +0 -0
  99. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/llm_qwen3.py +0 -0
  100. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/llm_signature.py +0 -0
  101. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/lm_base.py +0 -0
  102. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/openai_memoize.py +0 -0
  103. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/signature.py +0 -0
  104. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/lm/utils.py +0 -0
  105. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/llm_utils/utils.py +0 -0
  106. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/__imports.py +0 -0
  107. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/__init__.py +0 -0
  108. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/clock.py +0 -0
  109. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/exceptions.py +0 -0
  110. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/function_decorator.py +0 -0
  111. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/logger.py +0 -0
  112. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/notebook_utils.py +0 -0
  113. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/patcher.py +0 -0
  114. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/report_manager.py +0 -0
  115. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_cache.py +0 -0
  116. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_error.py +0 -0
  117. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_io.py +0 -0
  118. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_misc.py +0 -0
  119. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/common/utils_print.py +0 -0
  120. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/__init__.py +0 -0
  121. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/_mp_backends.py +0 -0
  122. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/_mp_progress.py +0 -0
  123. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/_mp_spawn.py +0 -0
  124. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/_multi_process.py +0 -0
  125. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/common.py +0 -0
  126. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/dataset_sharding.py +0 -0
  127. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/process.py +0 -0
  128. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/multi_worker/thread.py +0 -0
  129. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/scripts/kill_mpython.py +0 -0
  130. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/scripts/mpython.py +0 -0
  131. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/scripts/openapi_client_codegen.py +0 -0
  132. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/speedy_utils/scripts/prefetch_large_model.py +0 -0
  133. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/vision_utils/README.md +0 -0
  134. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/vision_utils/__init__.py +0 -0
  135. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/src/vision_utils/io_utils.py +0 -0
  136. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/import_all.py +0 -0
  137. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/import_time_report.py +0 -0
  138. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_llm_call_contract.py +0 -0
  139. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_llm_mixins.py +0 -0
  140. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_llm_signature.py +0 -0
  141. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_openai_memoize_completion_cache.py +0 -0
  142. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_openai_proxy_localhost.py +0 -0
  143. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_public_api.py +0 -0
  144. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/llm_utils/test_sp_chat.py +0 -0
  145. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/sample_objects.py +0 -0
  146. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test.py +0 -0
  147. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_identify.py +0 -0
  148. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_load_jsonl.py +0 -0
  149. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_logger.py +0 -0
  150. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_logger_format.py +0 -0
  151. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_memoize.py +0 -0
  152. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_memoize_typing.py +0 -0
  153. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_mp_no_main_guard.py +0 -0
  154. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_mpython.py +0 -0
  155. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_multi_process.py +0 -0
  156. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_multi_thread.py +0 -0
  157. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_multithread_error_trace.py +0 -0
  158. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_prefetch_large_model.py +0 -0
  159. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_process.py +0 -0
  160. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_process_update.py +0 -0
  161. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_thread.py +0 -0
  162. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_tokenization.py +0 -0
  163. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tests/test_utils_io.py +0 -0
  164. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tools/check_syntax.py +0 -0
  165. {speedy_utils-2.0.6 → speedy_utils-2.0.7}/tools/uv_test.sh +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: speedy-utils
3
- Version: 2.0.6
3
+ Version: 2.0.7
4
4
  Summary: Fast and easy-to-use package for data science
5
5
  Project-URL: Homepage, https://github.com/anhvth/speedy
6
6
  Project-URL: Repository, https://github.com/anhvth/speedy
@@ -18,32 +18,18 @@ Classifier: Programming Language :: Python :: 3.12
18
18
  Classifier: Programming Language :: Python :: 3.13
19
19
  Classifier: Programming Language :: Python :: 3.14
20
20
  Requires-Python: >=3.9
21
- Requires-Dist: aiohttp
22
- Requires-Dist: bump2version
23
21
  Requires-Dist: cachetools
24
22
  Requires-Dist: chainlit>=2.0
25
23
  Requires-Dist: datasets
26
- Requires-Dist: debugpy
27
24
  Requires-Dist: dill
28
25
  Requires-Dist: fastcore
29
- Requires-Dist: fastprogress
30
- Requires-Dist: freezegun
31
- Requires-Dist: ipdb
32
- Requires-Dist: ipywidgets
33
- Requires-Dist: json-repair
34
- Requires-Dist: jupyterlab
35
26
  Requires-Dist: loguru
36
27
  Requires-Dist: matplotlib
37
28
  Requires-Dist: numpy
38
29
  Requires-Dist: openai
39
- Requires-Dist: packaging
40
30
  Requires-Dist: pandas
41
31
  Requires-Dist: pydantic
42
- Requires-Dist: pytest
43
- Requires-Dist: requests
44
32
  Requires-Dist: rich>=14.3.1
45
- Requires-Dist: ruff
46
- Requires-Dist: scikit-learn
47
33
  Requires-Dist: tabulate
48
34
  Requires-Dist: tqdm
49
35
  Requires-Dist: xxhash
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "speedy-utils"
3
- version = "2.0.6"
3
+ version = "2.0.7"
4
4
  description = "Fast and easy-to-use package for data science"
5
5
  authors = [{ name = "AnhVTH", email = "anhvth.226@gmail.com" }]
6
6
  readme = "README.md"
@@ -8,15 +8,9 @@ license = { text = "MIT" }
8
8
  requires-python = ">=3.9"
9
9
  dependencies = [
10
10
  "numpy",
11
- "requests",
12
11
  "xxhash",
13
12
  "loguru",
14
13
  "fastcore",
15
- "debugpy",
16
- "ipywidgets",
17
- "jupyterlab",
18
- "ipdb",
19
- "scikit-learn",
20
14
  "matplotlib",
21
15
  "pandas",
22
16
  "tabulate",
@@ -24,15 +18,7 @@ dependencies = [
24
18
  "tqdm",
25
19
  "cachetools",
26
20
  "dill",
27
- "ruff",
28
- "bump2version",
29
- "json-repair",
30
- "fastprogress",
31
- "freezegun",
32
- "packaging",
33
21
  "openai",
34
- "aiohttp",
35
- "pytest",
36
22
  "rich>=14.3.1",
37
23
  "chainlit>=2.0",
38
24
  "datasets",
@@ -65,6 +51,9 @@ kill-mpython = "speedy_utils.scripts.kill_mpython:main"
65
51
  spu-prefetch-large-model = "speedy_utils.scripts.prefetch_large_model:main"
66
52
  sp_chat = "llm_utils.scripts.sp_chat:main"
67
53
  viz_chat = "datasets_utils.viz_chat:main"
54
+ pcat = "datasets_utils.pcat.cli:main"
55
+ pcat-jsonl = "datasets_utils.pcat.jsonl:main"
56
+ pcat-hf-dataset = "datasets_utils.pcat.hf_dataset:main"
68
57
 
69
58
  [build-system]
70
59
  requires = ["hatchling"]
@@ -154,4 +143,7 @@ norecursedirs = ["debug", "examples", "experiments", "notebooks", "scripts", ".g
154
143
  dev = [
155
144
  "pyright>=1.1.408",
156
145
  "pytest-xdist>=3.8.0",
146
+ "pytest",
147
+ "freezegun",
148
+ "ruff",
157
149
  ]
@@ -0,0 +1,20 @@
1
+ {
2
+ "include": [
3
+ "src",
4
+ "tests"
5
+ ],
6
+ "exclude": [
7
+ "debug",
8
+ "examples",
9
+ "scripts",
10
+ "**/node_modules",
11
+ "**/__pycache__",
12
+ "**/.*",
13
+ ".venv"
14
+ ],
15
+ "reportUnsupportedDunderAll": "none",
16
+ "reportMissingImports": "none",
17
+ "reportMissingModuleSource": "none",
18
+ "reportAttributeAccessIssue": "none",
19
+ "pythonVersion": "3.11"
20
+ }
@@ -0,0 +1,30 @@
1
+ """datasets_utils - dataset inspection and conversion helpers."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ __all__ = [
7
+ "convert_to_arrow",
8
+ "extract_tools",
9
+ "load_data",
10
+ "main",
11
+ "normalize_messages",
12
+ "parse_args",
13
+ "print_item",
14
+ "sample_items",
15
+ ]
16
+
17
+
18
+ def __getattr__(name: str):
19
+ if name == "convert_to_arrow":
20
+ from datasets_utils.convert_to_arrow import convert_to_arrow
21
+
22
+ return convert_to_arrow
23
+ if name in __all__:
24
+ import sys as _sys
25
+
26
+ mod = _sys.modules["datasets_utils.viz_chat"] = __import__(
27
+ "datasets_utils.viz_chat", fromlist=[name]
28
+ )
29
+ return getattr(mod, name)
30
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -1,7 +1,7 @@
1
1
  import os
2
2
 
3
3
 
4
- os.environ['HF_HOME'] = '/mnt/data/anhvth/tmp/'
4
+ os.environ["HF_HOME"] = "/mnt/data/anhvth/tmp/"
5
5
  import sys
6
6
  from pathlib import Path
7
7
 
@@ -25,7 +25,13 @@ def convert_to_arrow(path_to_dir):
25
25
  # 2. Load and Save
26
26
  # 'keep_in_memory=False' ensures we don't crash RAM if the files are huge
27
27
  n_proc = os.cpu_count() if os.cpu_count() is not None else 1
28
- dataset = load_dataset("parquet", data_files=parquet_files, split="train", keep_in_memory=True, num_proc=n_proc)
28
+ dataset = load_dataset(
29
+ "parquet",
30
+ data_files=parquet_files,
31
+ split="train",
32
+ keep_in_memory=True,
33
+ num_proc=n_proc,
34
+ )
29
35
 
30
36
  # This creates the native Arrow format for fast loading
31
37
  dataset.save_to_disk(output_dir)
@@ -33,6 +39,7 @@ def convert_to_arrow(path_to_dir):
33
39
  print(f"✅ Success! Saved to: {output_dir}")
34
40
  print("Now load it instantly with: load_from_disk('path')")
35
41
 
42
+
36
43
  if __name__ == "__main__":
37
44
  if len(sys.argv) < 2:
38
45
  print("Usage: python convert_to_arrow.py <path_to_dir>")