speedy-utils 2.0.7__tar.gz → 2.0.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- speedy_utils-2.0.9/.agents/PRs/pcat-vscode-command-resolution.md +94 -0
- speedy_utils-2.0.9/.agents/plans/pcat-vscode-command-resolution.md +102 -0
- speedy_utils-2.0.9/.codex/skills/speedy-utils-core +1 -0
- speedy_utils-2.0.9/.github/skills/speedy-utils-core/SKILL.md +92 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/.gitignore +11 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/CLAUDE.md +2 -2
- speedy_utils-2.0.7/README.md → speedy_utils-2.0.9/PKG-INFO +94 -2
- speedy_utils-2.0.7/PKG-INFO → speedy_utils-2.0.9/README.md +50 -41
- speedy_utils-2.0.9/agents/PRs/parallel_llm_job.md +103 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/test_qwen3_generate.md +1 -1
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/docs/MIGRATION_THINKING.md +3 -3
- speedy_utils-2.0.9/docs/MULTI_THREAD_HOOKS.md +94 -0
- speedy_utils-2.0.9/docs/PARALLEL_LLM_JOB.md +119 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/examples/llm_public_api_example.ipynb +1 -1
- speedy_utils-2.0.9/examples/qwen3llm_usage_demo.ipynb +250 -0
- speedy_utils-2.0.9/install-tools.sh +87 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/pyproject.toml +6 -1
- speedy_utils-2.0.9/setup-skill.sh +6 -0
- speedy_utils-2.0.9/setup-speedy-utils.sh +202 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/datasets_utils/convert_to_arrow.py +3 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/datasets_utils/pcat/_shared.py +467 -47
- speedy_utils-2.0.9/src/datasets_utils/pcat/cli.py +280 -0
- speedy_utils-2.0.9/src/datasets_utils/pcat/serve.py +2006 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/datasets_utils/viz_chat.py +14 -1
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/__init__.py +2 -1
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/chat_format/display.py +192 -18
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/lm/base_prompt_builder.py +3 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/lm/llm.py +561 -165
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/lm/llm_qwen3.py +85 -2
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/lm/llm_signature.py +3 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/lm/lm_base.py +50 -45
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/lm/openai_memoize.py +4 -14
- speedy_utils-2.0.9/src/llm_utils/lm/ssh_tunnel.py +143 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/lm/utils.py +15 -2
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/scripts/sp_chat.py +260 -39
- speedy_utils-2.0.9/src/llm_utils/system_prompted_assistant/__init__.py +6 -0
- speedy_utils-2.0.9/src/llm_utils/system_prompted_assistant/wrong_student_conversation_to_answer.py +112 -0
- speedy_utils-2.0.9/src/llm_utils/utils.py +89 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/__init__.py +12 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/exceptions.py +7 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/logger.py +3 -0
- speedy_utils-2.0.9/src/speedy_utils/common/mirror_path.py +193 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/notebook_utils.py +3 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/report_manager.py +3 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/utils_cache.py +1 -1
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/utils_error.py +7 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/utils_io.py +2 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/multi_worker/_multi_process.py +17 -4
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/multi_worker/common.py +14 -10
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/multi_worker/thread.py +221 -40
- speedy_utils-2.0.9/src/speedy_utils/parallel_llm_job.py +675 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/vision_utils/plot.py +3 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/llm_utils/test_llm_call_contract.py +183 -0
- speedy_utils-2.0.9/tests/llm_utils/test_llm_helpers.py +78 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/llm_utils/test_llm_mixins.py +134 -17
- speedy_utils-2.0.9/tests/llm_utils/test_llm_wait_for_endpoint.py +183 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/llm_utils/test_public_api.py +45 -0
- speedy_utils-2.0.9/tests/llm_utils/test_sp_chat.py +389 -0
- speedy_utils-2.0.9/tests/llm_utils/test_ssh_tunnel.py +87 -0
- speedy_utils-2.0.9/tests/test_common_examples.py +39 -0
- speedy_utils-2.0.9/tests/test_mirror_path.py +33 -0
- speedy_utils-2.0.9/tests/test_parallel_llm_job.py +239 -0
- speedy_utils-2.0.9/tests/test_pcat.py +1268 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_process_update.py +38 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_thread.py +124 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/uv.lock +2101 -116
- speedy_utils-2.0.9/vscode-pcat/.gitignore +4 -0
- speedy_utils-2.0.9/vscode-pcat/.vscodeignore +5 -0
- speedy_utils-2.0.9/vscode-pcat/README.md +61 -0
- speedy_utils-2.0.9/vscode-pcat/package.json +144 -0
- speedy_utils-2.0.9/vscode-pcat/src/extension.ts +734 -0
- speedy_utils-2.0.9/vscode-pcat/tsconfig.json +26 -0
- speedy_utils-2.0.7/.codex +0 -0
- speedy_utils-2.0.7/.github/skills/caching-utilities/SKILL.md +0 -133
- speedy_utils-2.0.7/.github/skills/caching-utilities/examples/caching_example.py +0 -37
- speedy_utils-2.0.7/.github/skills/dataset-processing-multiprocessing/CONFIG_REFERENCE.md +0 -217
- speedy_utils-2.0.7/.github/skills/dataset-processing-multiprocessing/SKILL.md +0 -797
- speedy_utils-2.0.7/.github/skills/dataset-processing-multiprocessing/example_tokenize_pack.py +0 -323
- speedy_utils-2.0.7/.github/skills/edit-llm-inference-style/SKILL.md +0 -133
- speedy_utils-2.0.7/.github/skills/io-utilities/SKILL.md +0 -111
- speedy_utils-2.0.7/.github/skills/io-utilities/examples/io_example.py +0 -35
- speedy_utils-2.0.7/.github/skills/llm-integration/SKILL.md +0 -115
- speedy_utils-2.0.7/.github/skills/llm-integration/examples/llm_example.py +0 -27
- speedy_utils-2.0.7/.github/skills/make-vscode-happy/SKILL.md +0 -80
- speedy_utils-2.0.7/.github/skills/make-vscode-happy/references/diagnostic-playbook.md +0 -94
- speedy_utils-2.0.7/.github/skills/multi-threading-processing/SKILL.md +0 -486
- speedy_utils-2.0.7/.github/skills/ray-distributed-computing/SKILL.md +0 -704
- speedy_utils-2.0.7/.github/skills/skill-creation/SKILL.md +0 -516
- speedy_utils-2.0.7/.github/skills/vision-utilities/SKILL.md +0 -105
- speedy_utils-2.0.7/.github/skills/vision-utilities/examples/vision_example.py +0 -44
- speedy_utils-2.0.7/src/datasets_utils/pcat/cli.py +0 -146
- speedy_utils-2.0.7/src/llm_utils/utils.py +0 -42
- speedy_utils-2.0.7/tests/llm_utils/test_sp_chat.py +0 -198
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/.githooks/pre-commit +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/.githooks/pre-push +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/.github/prompts/improveParallelErrorHandling.prompt.md +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/.github/workflows/publish.yml +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/.pre-commit-config.yaml +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/AGENTS.md +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/CODING_STYLE.md +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/bumpversion.sh +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/debug_generate_response.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/debug_n_param.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/debug_n_structure.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/integration_test.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/multi_thread.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/test_decode_api.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/test_endpoints.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/test_generate.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/test_generate_endpoint.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/debug/test_qwen3_generate.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/docs/GENERATE_QUICKREF.md +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/docs/PR_LIBRARY_ERROR_HANDLING_REFACTOR_PLAN.md +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/docs/TDD.md +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/docs/TOKENIZATION.md +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/docs/TOKENIZATION_IMPLEMENTATION.md +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/examples/generate_example.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/examples/model_specific_thinking_example.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/examples/tokenization_example.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/examples/vision_utils_example.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/experiments/exp1/dockerfile +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/experiments/exp1/run_in_docker.sh +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/notebooks/demo_multi_process.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/notebooks/llm_utils/llm_as_a_judge.ipynb +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/notebooks/test.ipynb +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/pyrightconfig.json +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/ruff.toml +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/bug_simple.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/debug_import_time.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/debug_multi_process.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/deploy.sh +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/imports.sh +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/install_hooks.sh +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/test.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/test_error_handling.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/test_import_time_vision.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/test_locals.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/scripts/test_ray_mp.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/setup.cfg +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/datasets_utils/__init__.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/datasets_utils/pcat/__init__.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/datasets_utils/pcat/hf_dataset.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/datasets_utils/pcat/jsonl.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/_traceback.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/chat_format/__init__.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/chat_format/transform.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/chat_format/utils.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/lm/__init__.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/llm_utils/lm/signature.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/__imports.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/__init__.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/clock.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/function_decorator.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/patcher.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/utils_misc.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/common/utils_print.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/multi_worker/__init__.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/multi_worker/_mp_backends.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/multi_worker/_mp_progress.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/multi_worker/_mp_spawn.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/multi_worker/dataset_sharding.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/multi_worker/process.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/scripts/__init__.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/scripts/kill_mpython.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/scripts/mpython.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/scripts/openapi_client_codegen.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/speedy_utils/scripts/prefetch_large_model.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/vision_utils/README.md +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/vision_utils/__init__.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/src/vision_utils/io_utils.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/import_all.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/import_time_report.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/llm_utils/test_llm_signature.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/llm_utils/test_openai_memoize_completion_cache.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/llm_utils/test_openai_proxy_localhost.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/sample_objects.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_identify.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_import_time.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_load_jsonl.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_logger.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_logger_format.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_memoize.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_memoize_typing.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_mp_no_main_guard.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_mpython.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_multi_process.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_multi_thread.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_multithread_error_trace.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_prefetch_large_model.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_process.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_tokenization.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_utils_io.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tests/test_viz_chat.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tools/check_syntax.py +0 -0
- {speedy_utils-2.0.7 → speedy_utils-2.0.9}/tools/uv_test.sh +0 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# Problem Report: PCAT VS Code Command Resolution Fails Outside Interactive Shells
|
|
2
|
+
|
|
3
|
+
## Summary
|
|
4
|
+
|
|
5
|
+
The PCAT VS Code extension opens `.jsonl` files and can launch `pcat` in the
|
|
6
|
+
integrated terminal, but preview actions can fail with:
|
|
7
|
+
|
|
8
|
+
```text
|
|
9
|
+
PCAT preview failed: /bin/sh: 1: pcat: not found
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
This is a shipped behavior bug in the extension, not a user configuration
|
|
13
|
+
mistake.
|
|
14
|
+
|
|
15
|
+
## User-visible Symptom
|
|
16
|
+
|
|
17
|
+
- Opening a `.jsonl` file can show the PCAT custom editor as expected.
|
|
18
|
+
- Using `PCAT: Preview Plain Row` fails with `pcat: not found`.
|
|
19
|
+
- Interactive terminal launches may appear to work in the same workspace.
|
|
20
|
+
|
|
21
|
+
## Reproduction
|
|
22
|
+
|
|
23
|
+
1. Install the current `pcat-vscode` extension build.
|
|
24
|
+
2. Leave `pcat.command` at its default value of `pcat`.
|
|
25
|
+
3. Use a shell where `pcat` is defined only through interactive shell setup,
|
|
26
|
+
such as an alias in `.zshrc`.
|
|
27
|
+
4. Open a `.jsonl` file and run `PCAT: Preview Plain Row`.
|
|
28
|
+
5. Observe the failure notification.
|
|
29
|
+
|
|
30
|
+
## Confirmed Evidence
|
|
31
|
+
|
|
32
|
+
Interactive shell resolution succeeds:
|
|
33
|
+
|
|
34
|
+
```text
|
|
35
|
+
alias pcat='uvx --from /home/anhvth8/projects/speedy_utils pcat'
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The non-interactive shell used by the extension cannot resolve it:
|
|
39
|
+
|
|
40
|
+
```text
|
|
41
|
+
/bin/sh: 32: source: not found
|
|
42
|
+
__PCAT_MISSING_IN_SH__
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Root Cause
|
|
46
|
+
|
|
47
|
+
The extension currently has two different execution paths:
|
|
48
|
+
|
|
49
|
+
- Interactive open uses the VS Code integrated terminal.
|
|
50
|
+
- Plain preview uses `child_process.exec(...)`.
|
|
51
|
+
|
|
52
|
+
The preview path inherits Node's default shell execution behavior, which uses a
|
|
53
|
+
non-interactive `/bin/sh` environment. That environment does not load the same
|
|
54
|
+
interactive shell aliases, functions, or PATH mutations that make `pcat`
|
|
55
|
+
available in the user's zsh terminal.
|
|
56
|
+
|
|
57
|
+
As a result, the extension is inconsistent:
|
|
58
|
+
|
|
59
|
+
- `pcat` can appear available in the integrated terminal.
|
|
60
|
+
- the same command string fails in preview mode.
|
|
61
|
+
|
|
62
|
+
## Why This Is a Product Bug
|
|
63
|
+
|
|
64
|
+
- The default extension setting is `pcat.command = "pcat"`.
|
|
65
|
+
- In this environment, `pcat` is not a real executable on PATH for `/bin/sh`.
|
|
66
|
+
- The extension depends on shell-specific behavior without resolving or
|
|
67
|
+
validating the command.
|
|
68
|
+
- The extension does not fail early with actionable guidance.
|
|
69
|
+
|
|
70
|
+
## Impact
|
|
71
|
+
|
|
72
|
+
- `.jsonl` custom-editor flows are unreliable.
|
|
73
|
+
- preview is broken in common remote and shell-customized setups.
|
|
74
|
+
- the extension currently depends on undocumented shell state.
|
|
75
|
+
|
|
76
|
+
## Expected Behavior
|
|
77
|
+
|
|
78
|
+
The extension should either:
|
|
79
|
+
|
|
80
|
+
- resolve `pcat` in a shell-agnostic way before use, or
|
|
81
|
+
- require an explicit executable command and validate it clearly, or
|
|
82
|
+
- reuse one consistent execution strategy for both terminal and preview flows.
|
|
83
|
+
|
|
84
|
+
At minimum, preview and open actions must succeed or fail with the same command
|
|
85
|
+
resolution behavior.
|
|
86
|
+
|
|
87
|
+
## Acceptance Criteria
|
|
88
|
+
|
|
89
|
+
- `PCAT: Preview Plain Row` works when `pcat.command` is configured to a valid
|
|
90
|
+
executable command.
|
|
91
|
+
- Default behavior does not silently rely on zsh aliases.
|
|
92
|
+
- When command resolution fails, the user sees an actionable error describing
|
|
93
|
+
how to set `pcat.command`.
|
|
94
|
+
- Interactive open and preview use the same command-resolution rules.
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Plan: Fix and Validate PCAT VS Code Command Resolution
|
|
2
|
+
|
|
3
|
+
Related problem report:
|
|
4
|
+
|
|
5
|
+
- `.agents/PRs/pcat-vscode-command-resolution.md`
|
|
6
|
+
|
|
7
|
+
## Goal
|
|
8
|
+
|
|
9
|
+
Make the PCAT VS Code extension reliable in non-interactive environments,
|
|
10
|
+
especially for preview mode, and validate the fix against the problem report.
|
|
11
|
+
|
|
12
|
+
## Working Hypothesis
|
|
13
|
+
|
|
14
|
+
The bug exists because preview uses `child_process.exec(...)` in a non-
|
|
15
|
+
interactive `/bin/sh` environment while interactive open uses the integrated
|
|
16
|
+
terminal. The fix should remove that mismatch and stop relying on shell aliases.
|
|
17
|
+
|
|
18
|
+
## Fix Strategy
|
|
19
|
+
|
|
20
|
+
1. Introduce explicit command resolution.
|
|
21
|
+
|
|
22
|
+
- Read `pcat.command` as a user-provided command string.
|
|
23
|
+
- Validate it before execution.
|
|
24
|
+
- If it is the default `pcat`, resolve whether a real executable exists.
|
|
25
|
+
- If not resolvable, show a clear error telling the user to set
|
|
26
|
+
`pcat.command` to something explicit such as:
|
|
27
|
+
- `uv run pcat`
|
|
28
|
+
- `uvx --from /home/anhvth8/projects/speedy_utils pcat`
|
|
29
|
+
- an absolute executable path
|
|
30
|
+
|
|
31
|
+
2. Unify execution semantics.
|
|
32
|
+
|
|
33
|
+
- Ensure interactive open and plain preview use the same resolved command
|
|
34
|
+
model.
|
|
35
|
+
- Avoid depending on shell aliases, shell functions, or interactive rc files.
|
|
36
|
+
- Prefer spawning a command with explicit args over concatenating a shell
|
|
37
|
+
command string when practical.
|
|
38
|
+
|
|
39
|
+
3. Improve failure reporting.
|
|
40
|
+
|
|
41
|
+
- Detect missing command resolution before invoking preview.
|
|
42
|
+
- Report the exact setting key to change: `pcat.command`.
|
|
43
|
+
- Include one working example command in the error.
|
|
44
|
+
|
|
45
|
+
4. Keep the `.jsonl` custom editor behavior intact.
|
|
46
|
+
|
|
47
|
+
- Do not remove the current default custom-editor association.
|
|
48
|
+
- Do not regress `Open As Text` or integrated-terminal launch behavior.
|
|
49
|
+
|
|
50
|
+
## Implementation Notes
|
|
51
|
+
|
|
52
|
+
- Replace shell-string execution for preview with a spawn-style path where the
|
|
53
|
+
command and args are separated.
|
|
54
|
+
- If `pcat.command` remains a free-form string, parse it once and reuse the
|
|
55
|
+
parsed command for both preview and integrated terminal launch.
|
|
56
|
+
- Consider adding a helper that returns a normalized `{ command, args }`
|
|
57
|
+
structure plus a user-facing diagnostic on failure.
|
|
58
|
+
|
|
59
|
+
## Test Plan
|
|
60
|
+
|
|
61
|
+
### Reproduction Test Against The Problem Report
|
|
62
|
+
|
|
63
|
+
1. Install the fixed extension build.
|
|
64
|
+
2. Configure no alias-dependent workaround in the extension itself.
|
|
65
|
+
3. Use the same environment described in the problem report where `pcat` is
|
|
66
|
+
available only through interactive shell setup.
|
|
67
|
+
4. Attempt preview.
|
|
68
|
+
5. Verify one of these outcomes:
|
|
69
|
+
- preview succeeds with an explicit configured command, or
|
|
70
|
+
- the extension gives a clear configuration error instead of `/bin/sh: ... pcat: not found`.
|
|
71
|
+
|
|
72
|
+
### Manual Behavior Tests
|
|
73
|
+
|
|
74
|
+
1. Open a `.jsonl` file.
|
|
75
|
+
2. Confirm the PCAT custom editor still opens by default.
|
|
76
|
+
3. Confirm automatic terminal launch still works when enabled.
|
|
77
|
+
4. Run `PCAT: Preview Plain Row`.
|
|
78
|
+
5. Confirm the preview opens JSON content in a text editor tab.
|
|
79
|
+
6. Use `PCAT: Reopen As Text Editor` and confirm raw text opens normally.
|
|
80
|
+
|
|
81
|
+
### Negative Tests
|
|
82
|
+
|
|
83
|
+
1. Set `pcat.command` to an invalid executable name.
|
|
84
|
+
2. Run preview.
|
|
85
|
+
3. Verify the error message references `pcat.command` and suggests a concrete
|
|
86
|
+
valid command.
|
|
87
|
+
|
|
88
|
+
### Regression Tests
|
|
89
|
+
|
|
90
|
+
1. Verify row index prompting still works for preview.
|
|
91
|
+
2. Verify HF dataset split prompting still works where applicable.
|
|
92
|
+
3. Verify the package still builds:
|
|
93
|
+
- `npm run compile`
|
|
94
|
+
- `npm run package`
|
|
95
|
+
|
|
96
|
+
## Definition of Done
|
|
97
|
+
|
|
98
|
+
- The failure from the problem report is either fixed or converted into a clear,
|
|
99
|
+
actionable configuration error.
|
|
100
|
+
- Preview no longer depends on interactive shell aliases.
|
|
101
|
+
- Interactive open and preview follow the same command-resolution policy.
|
|
102
|
+
- The updated extension compiles, packages, and passes the manual checks above.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
/home/anhvth8/projects/speedy_utils/.github/skills/speedy-utils-core
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: "speedy-utils-core"
|
|
3
|
+
description: "Use when a simple for loop should be sped up with speedy_utils multi_thread or multi_process, or when the native OpenAI client should be wrapped with llm_utils.LLM or Qwen3LLM for a cleaner API. Focus only on these repo-specific surfaces."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Speedy Utils Core Skill
|
|
7
|
+
|
|
8
|
+
Use this skill only for changes involving:
|
|
9
|
+
|
|
10
|
+
- `speedy_utils.multi_worker.thread.multi_thread`
|
|
11
|
+
- `speedy_utils.multi_worker._multi_process.multi_process`
|
|
12
|
+
- `llm_utils.lm.llm.LLM`
|
|
13
|
+
- `llm_utils.lm.llm_qwen3.Qwen3LLM`
|
|
14
|
+
|
|
15
|
+
Reach for this skill in two common cases:
|
|
16
|
+
|
|
17
|
+
- a simple `for` loop should run faster with `multi_thread(...)` or `multi_process(...)`
|
|
18
|
+
- raw OpenAI client usage should be replaced with `LLM(...)` or `Qwen3LLM(...)` to make call sites cleaner and more consistent
|
|
19
|
+
|
|
20
|
+
Do not use this skill for caching, vision, dataset tooling, Ray, or generic repo cleanup outside those surfaces.
|
|
21
|
+
|
|
22
|
+
## Working rules
|
|
23
|
+
|
|
24
|
+
- Prefer public APIs and current behavior over historical helper patterns.
|
|
25
|
+
- Keep changes small, direct, and easy to read.
|
|
26
|
+
- Follow Red -> Green -> Refactor for bug fixes.
|
|
27
|
+
- Add or update focused regression tests for behavior changes.
|
|
28
|
+
- Keep imports lazy when touching LLM or Qwen3 code paths.
|
|
29
|
+
- Do not introduce heavy top-level imports in package `__init__.py` files.
|
|
30
|
+
|
|
31
|
+
## Repository-specific guidance
|
|
32
|
+
|
|
33
|
+
### `multi_thread`
|
|
34
|
+
|
|
35
|
+
- Primary file: `src/speedy_utils/multi_worker/thread.py`
|
|
36
|
+
- Use `multi_thread` for managed thread-pool execution over an input iterable.
|
|
37
|
+
- Preserve support for batching, ordered results, progress reporting, timeouts, and error handling.
|
|
38
|
+
- `error_handler` is the active control surface. Treat `stop_on_error` as compatibility behavior only.
|
|
39
|
+
- `n_proc > 1` is a fan-out path layered on top of `multi_thread`; avoid duplicating process orchestration logic elsewhere.
|
|
40
|
+
|
|
41
|
+
### `multi_process`
|
|
42
|
+
|
|
43
|
+
- Primary file: `src/speedy_utils/multi_worker/_multi_process.py`
|
|
44
|
+
- Public API returns a `list[Any]`.
|
|
45
|
+
- Respect the current normalization behavior around `num_procs`; do not assume auto-scaling unless the implementation explicitly does it.
|
|
46
|
+
- Keep fallback behavior across sequential, thread, and multiprocess execution paths coherent.
|
|
47
|
+
- Error handling and progress behavior should stay aligned with the thread utilities where practical.
|
|
48
|
+
|
|
49
|
+
### `LLM`
|
|
50
|
+
|
|
51
|
+
- Primary file: `src/llm_utils/lm/llm.py`
|
|
52
|
+
- `LLM()` defaults to the chat path.
|
|
53
|
+
- `LLM.generate()` expects a string prompt and supports only `n=1`.
|
|
54
|
+
- `LLM.pydantic_parse()` is the structured-output path.
|
|
55
|
+
- `LLM(..., return_dict=True)` behavior must stay compatible with the normalized dict contract used in this repo.
|
|
56
|
+
- Avoid constructor regressions: legacy constructor keys are intentionally rejected.
|
|
57
|
+
|
|
58
|
+
### `Qwen3LLM`
|
|
59
|
+
|
|
60
|
+
- Primary file: `src/llm_utils/lm/llm_qwen3.py`
|
|
61
|
+
- `Qwen3LLM.chat_completion()` returns an OpenAI-style chat message object.
|
|
62
|
+
- `complete_until()` returns continuation state, not a chat message.
|
|
63
|
+
- `complete_reasoning()` and `complete_content()` are staged helpers built on prefix completion.
|
|
64
|
+
- Keep tokenizer loading lazy and cached.
|
|
65
|
+
- Preserve reasoning/content split behavior and the assistant-prefix normalization flow.
|
|
66
|
+
|
|
67
|
+
## Clean-code bar
|
|
68
|
+
|
|
69
|
+
- Prefer one obvious implementation path.
|
|
70
|
+
- Remove dead branches only when you can prove they are obsolete.
|
|
71
|
+
- Keep naming aligned with existing public APIs.
|
|
72
|
+
- Add short comments only where the control flow is genuinely non-obvious.
|
|
73
|
+
- Avoid broad refactors unless they directly reduce risk in the touched logic.
|
|
74
|
+
|
|
75
|
+
## Validation
|
|
76
|
+
|
|
77
|
+
Run the smallest relevant checks after changes:
|
|
78
|
+
|
|
79
|
+
- `./tools/uv_test.sh <targeted_test>`
|
|
80
|
+
- `uv run python tools/check_syntax.py`
|
|
81
|
+
- `uv run ruff check .`
|
|
82
|
+
|
|
83
|
+
If import-time-sensitive files are touched, also run:
|
|
84
|
+
|
|
85
|
+
- `uv run python scripts/debug_import_time.py speedy_utils llm_utils vision_utils --max-total-sec 0.4 --top 12 --min-sec 0.01 --no-stdlib`
|
|
86
|
+
|
|
87
|
+
## Examples of when to use this skill
|
|
88
|
+
|
|
89
|
+
- Fix a regression in `multi_thread` batching or error handling.
|
|
90
|
+
- Adjust `multi_process` backend behavior without changing unrelated utilities.
|
|
91
|
+
- Update `LLM` request/response behavior while preserving current public contracts.
|
|
92
|
+
- Modify `Qwen3LLM` staged reasoning/content generation without breaking lazy tokenizer loading.
|
|
@@ -155,7 +155,7 @@ The wheel currently ships four packages from `src/`:
|
|
|
155
155
|
- `LLM.pydantic_parse()` is the structured-output API.
|
|
156
156
|
- `LLM(..., return_dict=True)` returns a normalized dict with `completion`, `message`, `messages`, and `parsed`.
|
|
157
157
|
- `LLMSignature` defaults structured outputs to the signature's output model.
|
|
158
|
-
- `Qwen3LLM.chat_completion()` returns an OpenAI-style `ChatCompletionMessage` with optional dynamic attrs such as `
|
|
158
|
+
- `Qwen3LLM.chat_completion()` returns an OpenAI-style `ChatCompletionMessage` with optional dynamic attrs such as `reasoning`, `usage`, and `call_count`.
|
|
159
159
|
- `Qwen3LLM.complete_until()` returns a continuation-state object, not a `ChatCompletionMessage`.
|
|
160
160
|
|
|
161
161
|
### `vision_utils`
|
|
@@ -201,7 +201,7 @@ Current notable suppressions:
|
|
|
201
201
|
| Rule | Why suppressed |
|
|
202
202
|
|------|----------------|
|
|
203
203
|
| `reportMissingImports` | Optional GPU and backend dependencies are not always installed |
|
|
204
|
-
| `reportAttributeAccessIssue` | OpenAI response models and runtime message objects gain dynamic attrs such as `usage`, `
|
|
204
|
+
| `reportAttributeAccessIssue` | OpenAI response models and runtime message objects gain dynamic attrs such as `usage`, `reasoning`, and `call_count` |
|
|
205
205
|
| `reportUnsupportedDunderAll` | Some modules rely on broad export patterns that are noisy for pyright |
|
|
206
206
|
|
|
207
207
|
### Common annotation patterns in this repo
|
|
@@ -1,3 +1,47 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: speedy-utils
|
|
3
|
+
Version: 2.0.9
|
|
4
|
+
Summary: Fast and easy-to-use package for data science
|
|
5
|
+
Project-URL: Homepage, https://github.com/anhvth/speedy
|
|
6
|
+
Project-URL: Repository, https://github.com/anhvth/speedy
|
|
7
|
+
Author-email: AnhVTH <anhvth.226@gmail.com>
|
|
8
|
+
License: MIT
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Requires-Python: >=3.9
|
|
21
|
+
Requires-Dist: cachetools
|
|
22
|
+
Requires-Dist: chainlit>=2.0
|
|
23
|
+
Requires-Dist: datasets
|
|
24
|
+
Requires-Dist: dill
|
|
25
|
+
Requires-Dist: fastcore
|
|
26
|
+
Requires-Dist: json-repair>=0.44.1
|
|
27
|
+
Requires-Dist: jupyter>=1.1.1
|
|
28
|
+
Requires-Dist: jupyterlab>=4.5.9
|
|
29
|
+
Requires-Dist: loguru
|
|
30
|
+
Requires-Dist: matplotlib
|
|
31
|
+
Requires-Dist: notebook>=7.5.7
|
|
32
|
+
Requires-Dist: numpy
|
|
33
|
+
Requires-Dist: openai
|
|
34
|
+
Requires-Dist: pandas
|
|
35
|
+
Requires-Dist: psutil
|
|
36
|
+
Requires-Dist: pydantic
|
|
37
|
+
Requires-Dist: rich>=14.3.1
|
|
38
|
+
Requires-Dist: tabulate
|
|
39
|
+
Requires-Dist: tqdm
|
|
40
|
+
Requires-Dist: xxhash
|
|
41
|
+
Provides-Extra: transformers
|
|
42
|
+
Requires-Dist: transformers>=4.57.6; extra == 'transformers'
|
|
43
|
+
Description-Content-Type: text/markdown
|
|
44
|
+
|
|
1
45
|
# Speedy Utils
|
|
2
46
|
|
|
3
47
|

|
|
@@ -44,6 +88,18 @@ cd speedy_utils
|
|
|
44
88
|
uv sync
|
|
45
89
|
```
|
|
46
90
|
|
|
91
|
+
Install the repo's AI skill definitions locally:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
./setup-skill.sh
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Or bootstrap them directly from GitHub:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
curl -fsSL https://raw.githubusercontent.com/anhvth/speedy_utils/master/setup-speedy-utils.sh | bash
|
|
101
|
+
```
|
|
102
|
+
|
|
47
103
|
Upgrading from older split packages:
|
|
48
104
|
|
|
49
105
|
```bash
|
|
@@ -256,6 +312,7 @@ The installed console scripts are:
|
|
|
256
312
|
| `sp_chat` | Launch a Chainlit chat UI for an OpenAI-compatible backend |
|
|
257
313
|
| `spu-prefetch-large-model` | Read large model files into the OS page cache |
|
|
258
314
|
| `viz_chat` | Inspect chat datasets from JSON, JSONL, folders, or HF saves |
|
|
315
|
+
| `pcat` | Interactive JSON/JSONL/HF-dataset viewer (TUI) |
|
|
259
316
|
| `openapi_client_codegen` | Generate a sync client from an OpenAPI JSON spec |
|
|
260
317
|
|
|
261
318
|
Examples:
|
|
@@ -407,8 +464,20 @@ llm = LLM(
|
|
|
407
464
|
llm = LLM(client=8000, model="llama-3")
|
|
408
465
|
llm = LLM(client="http://localhost:8000/v1", model="llama-3")
|
|
409
466
|
llm = LLM(client=[8000, 8001, 8002], model="llama-3")
|
|
467
|
+
llm = LLM(client="h1-31:8000", model="llama-3")
|
|
468
|
+
llm = LLM(client=["h1-31:8000", "h2-14:8100"], model="llama-3")
|
|
410
469
|
```
|
|
411
470
|
|
|
471
|
+
An integer means a local port. An HTTP(S) URL is used directly. A bare
|
|
472
|
+
`HOST:PORT` string is treated as an SSH host or alias plus its remote service
|
|
473
|
+
port; Speedy Utils creates a loopback-only SSH forward and reuses it for the
|
|
474
|
+
life of the Python process. Lists may mix all three forms.
|
|
475
|
+
|
|
476
|
+
For large checkpointed jobs, subclass `ParallelLLMJob` and implement only the
|
|
477
|
+
per-item domain logic. It provides persistent process/thread workers, lazy LLM
|
|
478
|
+
pools, exact-size JSONL output, and atomic resume state; see
|
|
479
|
+
[`docs/PARALLEL_LLM_JOB.md`](docs/PARALLEL_LLM_JOB.md).
|
|
480
|
+
|
|
412
481
|
### Caching and history inspection
|
|
413
482
|
|
|
414
483
|
```python
|
|
@@ -453,14 +522,14 @@ Standard chat path:
|
|
|
453
522
|
from llm_utils import Qwen3LLM
|
|
454
523
|
|
|
455
524
|
llm = Qwen3LLM(client=8000)
|
|
456
|
-
message = llm
|
|
525
|
+
message = llm(
|
|
457
526
|
[{"role": "user", "content": "Solve x^2 + 2x + 1 = 0"}],
|
|
458
527
|
thinking_max_tokens=32,
|
|
459
528
|
content_max_tokens=128,
|
|
460
529
|
)
|
|
461
530
|
|
|
462
531
|
print(message.content)
|
|
463
|
-
print(getattr(message, "
|
|
532
|
+
print(getattr(message, "reasoning", None))
|
|
464
533
|
print(getattr(message, "call_count", None))
|
|
465
534
|
```
|
|
466
535
|
|
|
@@ -519,6 +588,29 @@ viz_chat data/tokenized_dataset/ --tokenizer Qwen/Qwen3-8B
|
|
|
519
588
|
viz_chat data/with_tools.jsonl --show-tools
|
|
520
589
|
```
|
|
521
590
|
|
|
591
|
+
### `pcat` — Interactive JSON/JSONL/HF-dataset viewer
|
|
592
|
+
|
|
593
|
+
Install once:
|
|
594
|
+
|
|
595
|
+
```bash
|
|
596
|
+
uv tool install git+https://github.com/anhvth/speedy_utils
|
|
597
|
+
```
|
|
598
|
+
|
|
599
|
+
Then use the installed executable:
|
|
600
|
+
|
|
601
|
+
```bash
|
|
602
|
+
pcat data/file.jsonl
|
|
603
|
+
pcat data/tokenized-sdd --serve --no-browser
|
|
604
|
+
pcat data/dataset-dict --split validation --serve --no-browser
|
|
605
|
+
```
|
|
606
|
+
|
|
607
|
+
Hugging Face `Dataset` and `DatasetDict` directories saved with
|
|
608
|
+
`save_to_disk()` are loaded with `datasets.load_from_disk()`. Arrow files stay
|
|
609
|
+
memory-mapped and rows are read by index; `pcat` does not materialize the full
|
|
610
|
+
dataset. `DatasetDict` defaults to its `train` split and accepts `--split` for
|
|
611
|
+
another split. Tokenized SDD rows with `student_ids`, `teacher_ids`, and
|
|
612
|
+
`response_ids` use the existing tokenizer-aware SDD view in `--serve` mode.
|
|
613
|
+
|
|
522
614
|
## Vision Utils
|
|
523
615
|
|
|
524
616
|
`vision_utils` exports:
|
|
@@ -1,42 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: speedy-utils
|
|
3
|
-
Version: 2.0.7
|
|
4
|
-
Summary: Fast and easy-to-use package for data science
|
|
5
|
-
Project-URL: Homepage, https://github.com/anhvth/speedy
|
|
6
|
-
Project-URL: Repository, https://github.com/anhvth/speedy
|
|
7
|
-
Author-email: AnhVTH <anhvth.226@gmail.com>
|
|
8
|
-
License: MIT
|
|
9
|
-
Classifier: Development Status :: 4 - Beta
|
|
10
|
-
Classifier: Intended Audience :: Developers
|
|
11
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
-
Classifier: Programming Language :: Python :: 3
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
-
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
-
Requires-Python: >=3.9
|
|
21
|
-
Requires-Dist: cachetools
|
|
22
|
-
Requires-Dist: chainlit>=2.0
|
|
23
|
-
Requires-Dist: datasets
|
|
24
|
-
Requires-Dist: dill
|
|
25
|
-
Requires-Dist: fastcore
|
|
26
|
-
Requires-Dist: loguru
|
|
27
|
-
Requires-Dist: matplotlib
|
|
28
|
-
Requires-Dist: numpy
|
|
29
|
-
Requires-Dist: openai
|
|
30
|
-
Requires-Dist: pandas
|
|
31
|
-
Requires-Dist: pydantic
|
|
32
|
-
Requires-Dist: rich>=14.3.1
|
|
33
|
-
Requires-Dist: tabulate
|
|
34
|
-
Requires-Dist: tqdm
|
|
35
|
-
Requires-Dist: xxhash
|
|
36
|
-
Provides-Extra: transformers
|
|
37
|
-
Requires-Dist: transformers>=4.57.6; extra == 'transformers'
|
|
38
|
-
Description-Content-Type: text/markdown
|
|
39
|
-
|
|
40
1
|
# Speedy Utils
|
|
41
2
|
|
|
42
3
|

|
|
@@ -83,6 +44,18 @@ cd speedy_utils
|
|
|
83
44
|
uv sync
|
|
84
45
|
```
|
|
85
46
|
|
|
47
|
+
Install the repo's AI skill definitions locally:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
./setup-skill.sh
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Or bootstrap them directly from GitHub:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
curl -fsSL https://raw.githubusercontent.com/anhvth/speedy_utils/master/setup-speedy-utils.sh | bash
|
|
57
|
+
```
|
|
58
|
+
|
|
86
59
|
Upgrading from older split packages:
|
|
87
60
|
|
|
88
61
|
```bash
|
|
@@ -295,6 +268,7 @@ The installed console scripts are:
|
|
|
295
268
|
| `sp_chat` | Launch a Chainlit chat UI for an OpenAI-compatible backend |
|
|
296
269
|
| `spu-prefetch-large-model` | Read large model files into the OS page cache |
|
|
297
270
|
| `viz_chat` | Inspect chat datasets from JSON, JSONL, folders, or HF saves |
|
|
271
|
+
| `pcat` | Interactive JSON/JSONL/HF-dataset viewer (TUI) |
|
|
298
272
|
| `openapi_client_codegen` | Generate a sync client from an OpenAPI JSON spec |
|
|
299
273
|
|
|
300
274
|
Examples:
|
|
@@ -446,8 +420,20 @@ llm = LLM(
|
|
|
446
420
|
llm = LLM(client=8000, model="llama-3")
|
|
447
421
|
llm = LLM(client="http://localhost:8000/v1", model="llama-3")
|
|
448
422
|
llm = LLM(client=[8000, 8001, 8002], model="llama-3")
|
|
423
|
+
llm = LLM(client="h1-31:8000", model="llama-3")
|
|
424
|
+
llm = LLM(client=["h1-31:8000", "h2-14:8100"], model="llama-3")
|
|
449
425
|
```
|
|
450
426
|
|
|
427
|
+
An integer means a local port. An HTTP(S) URL is used directly. A bare
|
|
428
|
+
`HOST:PORT` string is treated as an SSH host or alias plus its remote service
|
|
429
|
+
port; Speedy Utils creates a loopback-only SSH forward and reuses it for the
|
|
430
|
+
life of the Python process. Lists may mix all three forms.
|
|
431
|
+
|
|
432
|
+
For large checkpointed jobs, subclass `ParallelLLMJob` and implement only the
|
|
433
|
+
per-item domain logic. It provides persistent process/thread workers, lazy LLM
|
|
434
|
+
pools, exact-size JSONL output, and atomic resume state; see
|
|
435
|
+
[`docs/PARALLEL_LLM_JOB.md`](docs/PARALLEL_LLM_JOB.md).
|
|
436
|
+
|
|
451
437
|
### Caching and history inspection
|
|
452
438
|
|
|
453
439
|
```python
|
|
@@ -492,14 +478,14 @@ Standard chat path:
|
|
|
492
478
|
from llm_utils import Qwen3LLM
|
|
493
479
|
|
|
494
480
|
llm = Qwen3LLM(client=8000)
|
|
495
|
-
message = llm
|
|
481
|
+
message = llm(
|
|
496
482
|
[{"role": "user", "content": "Solve x^2 + 2x + 1 = 0"}],
|
|
497
483
|
thinking_max_tokens=32,
|
|
498
484
|
content_max_tokens=128,
|
|
499
485
|
)
|
|
500
486
|
|
|
501
487
|
print(message.content)
|
|
502
|
-
print(getattr(message, "
|
|
488
|
+
print(getattr(message, "reasoning", None))
|
|
503
489
|
print(getattr(message, "call_count", None))
|
|
504
490
|
```
|
|
505
491
|
|
|
@@ -558,6 +544,29 @@ viz_chat data/tokenized_dataset/ --tokenizer Qwen/Qwen3-8B
|
|
|
558
544
|
viz_chat data/with_tools.jsonl --show-tools
|
|
559
545
|
```
|
|
560
546
|
|
|
547
|
+
### `pcat` — Interactive JSON/JSONL/HF-dataset viewer
|
|
548
|
+
|
|
549
|
+
Install once:
|
|
550
|
+
|
|
551
|
+
```bash
|
|
552
|
+
uv tool install git+https://github.com/anhvth/speedy_utils
|
|
553
|
+
```
|
|
554
|
+
|
|
555
|
+
Then use the installed executable:
|
|
556
|
+
|
|
557
|
+
```bash
|
|
558
|
+
pcat data/file.jsonl
|
|
559
|
+
pcat data/tokenized-sdd --serve --no-browser
|
|
560
|
+
pcat data/dataset-dict --split validation --serve --no-browser
|
|
561
|
+
```
|
|
562
|
+
|
|
563
|
+
Hugging Face `Dataset` and `DatasetDict` directories saved with
|
|
564
|
+
`save_to_disk()` are loaded with `datasets.load_from_disk()`. Arrow files stay
|
|
565
|
+
memory-mapped and rows are read by index; `pcat` does not materialize the full
|
|
566
|
+
dataset. `DatasetDict` defaults to its `train` split and accepts `--split` for
|
|
567
|
+
another split. Tokenized SDD rows with `student_ids`, `teacher_ids`, and
|
|
568
|
+
`response_ids` use the existing tokenizer-aware SDD view in `--serve` mode.
|
|
569
|
+
|
|
561
570
|
## Vision Utils
|
|
562
571
|
|
|
563
572
|
`vision_utils` exports:
|