azrvus-dev-toolkit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- azrvus_dev_toolkit-0.1.0/.github/workflows/ci.yml +41 -0
- azrvus_dev_toolkit-0.1.0/.github/workflows/release.yml +100 -0
- azrvus_dev_toolkit-0.1.0/.gitignore +19 -0
- azrvus_dev_toolkit-0.1.0/LICENSE +21 -0
- azrvus_dev_toolkit-0.1.0/PKG-INFO +113 -0
- azrvus_dev_toolkit-0.1.0/README.md +93 -0
- azrvus_dev_toolkit-0.1.0/RELEASE.md +75 -0
- azrvus_dev_toolkit-0.1.0/RELEASE_NOTES.md +23 -0
- azrvus_dev_toolkit-0.1.0/examples/eval-invalid.jsonl +2 -0
- azrvus_dev_toolkit-0.1.0/examples/eval-valid.jsonl +2 -0
- azrvus_dev_toolkit-0.1.0/pyproject.toml +45 -0
- azrvus_dev_toolkit-0.1.0/scripts/check_dist.py +118 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/__init__.py +163 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/_version.py +3 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/async_io.py +19 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/banner.py +37 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/cache.py +88 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/cli.py +60 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/cli_formatters.py +79 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/cli_spinner.py +87 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/color.py +34 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/concurrency.py +58 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/config.py +75 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/crypto_utils.py +21 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/decorators.py +47 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/dict_utils.py +77 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/encoding.py +19 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/env.py +23 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/env_validator.py +24 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/events.py +44 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/formatters.py +45 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/hash.py +37 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/http.py +22 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/io.py +40 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/jsonl.py +62 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/logger.py +36 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/masking.py +51 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/pagination.py +29 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/path_utils.py +84 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/process.py +65 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/random_utils.py +19 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/rate_limiter.py +77 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/retry.py +37 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/retry_utils.py +93 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/system.py +24 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/templating.py +28 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/text.py +45 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/timer.py +41 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/transforms.py +21 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/url_utils.py +35 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/validators.py +74 -0
- azrvus_dev_toolkit-0.1.0/src/dev_toolkit/watcher.py +66 -0
- azrvus_dev_toolkit-0.1.0/tests/__init__.py +1 -0
- azrvus_dev_toolkit-0.1.0/tests/test_async_io.py +19 -0
- azrvus_dev_toolkit-0.1.0/tests/test_banner.py +18 -0
- azrvus_dev_toolkit-0.1.0/tests/test_cache.py +58 -0
- azrvus_dev_toolkit-0.1.0/tests/test_cli.py +42 -0
- azrvus_dev_toolkit-0.1.0/tests/test_cli_formatters.py +32 -0
- azrvus_dev_toolkit-0.1.0/tests/test_cli_spinner.py +26 -0
- azrvus_dev_toolkit-0.1.0/tests/test_color.py +13 -0
- azrvus_dev_toolkit-0.1.0/tests/test_concurrency.py +31 -0
- azrvus_dev_toolkit-0.1.0/tests/test_config.py +30 -0
- azrvus_dev_toolkit-0.1.0/tests/test_crypto_utils.py +17 -0
- azrvus_dev_toolkit-0.1.0/tests/test_decorators.py +66 -0
- azrvus_dev_toolkit-0.1.0/tests/test_dict_utils.py +51 -0
- azrvus_dev_toolkit-0.1.0/tests/test_encoding.py +8 -0
- azrvus_dev_toolkit-0.1.0/tests/test_env.py +21 -0
- azrvus_dev_toolkit-0.1.0/tests/test_env_validator.py +28 -0
- azrvus_dev_toolkit-0.1.0/tests/test_events.py +34 -0
- azrvus_dev_toolkit-0.1.0/tests/test_formatters.py +34 -0
- azrvus_dev_toolkit-0.1.0/tests/test_hash.py +36 -0
- azrvus_dev_toolkit-0.1.0/tests/test_http.py +24 -0
- azrvus_dev_toolkit-0.1.0/tests/test_io.py +31 -0
- azrvus_dev_toolkit-0.1.0/tests/test_jsonl.py +63 -0
- azrvus_dev_toolkit-0.1.0/tests/test_logger.py +19 -0
- azrvus_dev_toolkit-0.1.0/tests/test_masking.py +26 -0
- azrvus_dev_toolkit-0.1.0/tests/test_pagination.py +24 -0
- azrvus_dev_toolkit-0.1.0/tests/test_path_utils.py +60 -0
- azrvus_dev_toolkit-0.1.0/tests/test_process.py +55 -0
- azrvus_dev_toolkit-0.1.0/tests/test_public_api.py +9 -0
- azrvus_dev_toolkit-0.1.0/tests/test_random_utils.py +11 -0
- azrvus_dev_toolkit-0.1.0/tests/test_rate_limiter.py +64 -0
- azrvus_dev_toolkit-0.1.0/tests/test_retry.py +35 -0
- azrvus_dev_toolkit-0.1.0/tests/test_retry_utils.py +74 -0
- azrvus_dev_toolkit-0.1.0/tests/test_system.py +22 -0
- azrvus_dev_toolkit-0.1.0/tests/test_templating.py +25 -0
- azrvus_dev_toolkit-0.1.0/tests/test_text.py +41 -0
- azrvus_dev_toolkit-0.1.0/tests/test_timer.py +33 -0
- azrvus_dev_toolkit-0.1.0/tests/test_transforms.py +14 -0
- azrvus_dev_toolkit-0.1.0/tests/test_url_utils.py +24 -0
- azrvus_dev_toolkit-0.1.0/tests/test_validators.py +47 -0
- azrvus_dev_toolkit-0.1.0/tests/test_watcher.py +21 -0
- azrvus_dev_toolkit-0.1.0/uv.lock +8 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ main ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ main ]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
test:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
strategy:
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.11", "3.12"]
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- name: Check out repository
|
|
18
|
+
uses: actions/checkout@v4
|
|
19
|
+
|
|
20
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
21
|
+
uses: actions/setup-python@v5
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python-version }}
|
|
24
|
+
|
|
25
|
+
- name: Install dependencies
|
|
26
|
+
run: |
|
|
27
|
+
python -m pip install --upgrade pip
|
|
28
|
+
pip install -e . pytest ruff uv twine
|
|
29
|
+
|
|
30
|
+
- name: Run Ruff Linter
|
|
31
|
+
run: |
|
|
32
|
+
ruff check .
|
|
33
|
+
|
|
34
|
+
- name: Run Pytest Suite
|
|
35
|
+
run: |
|
|
36
|
+
pytest
|
|
37
|
+
- name: Build and validate distributions
|
|
38
|
+
run: |
|
|
39
|
+
uv build
|
|
40
|
+
python -m twine check --strict dist/*
|
|
41
|
+
python scripts/check_dist.py
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
inputs:
|
|
8
|
+
destination:
|
|
9
|
+
description: Verify artifacts only, or rehearse on configured TestPyPI
|
|
10
|
+
required: true
|
|
11
|
+
default: verify
|
|
12
|
+
type: choice
|
|
13
|
+
options: [verify, testpypi]
|
|
14
|
+
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
18
|
+
concurrency:
|
|
19
|
+
group: release-${{ github.ref }}
|
|
20
|
+
cancel-in-progress: false
|
|
21
|
+
|
|
22
|
+
jobs:
|
|
23
|
+
build:
|
|
24
|
+
runs-on: ubuntu-latest
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
with:
|
|
28
|
+
fetch-depth: 0
|
|
29
|
+
persist-credentials: false
|
|
30
|
+
- uses: actions/setup-python@v5
|
|
31
|
+
with:
|
|
32
|
+
python-version: '3.11'
|
|
33
|
+
- name: Require a merged commit and matching release version
|
|
34
|
+
env:
|
|
35
|
+
EVENT_NAME: ${{ github.event_name }}
|
|
36
|
+
REF_NAME: ${{ github.ref_name }}
|
|
37
|
+
REF_TYPE: ${{ github.ref_type }}
|
|
38
|
+
run: |
|
|
39
|
+
git merge-base --is-ancestor HEAD origin/main
|
|
40
|
+
python - <<'PY'
|
|
41
|
+
import os
|
|
42
|
+
import runpy
|
|
43
|
+
version = runpy.run_path('src/dev_toolkit/_version.py')['__version__']
|
|
44
|
+
if os.environ['EVENT_NAME'] == 'release':
|
|
45
|
+
assert os.environ['REF_TYPE'] == 'tag'
|
|
46
|
+
assert os.environ['REF_NAME'] == f'v{version}', 'Tag/version mismatch'
|
|
47
|
+
else:
|
|
48
|
+
assert os.environ['REF_NAME'] == 'main', 'Dispatch from main only'
|
|
49
|
+
PY
|
|
50
|
+
- name: Test and build
|
|
51
|
+
run: |
|
|
52
|
+
python -m pip install -e . pytest ruff uv twine
|
|
53
|
+
ruff check .
|
|
54
|
+
pytest
|
|
55
|
+
uv build
|
|
56
|
+
python -m twine check --strict dist/*
|
|
57
|
+
python scripts/check_dist.py
|
|
58
|
+
- uses: actions/upload-artifact@v4
|
|
59
|
+
with:
|
|
60
|
+
name: distributions
|
|
61
|
+
path: |
|
|
62
|
+
dist/*.whl
|
|
63
|
+
dist/*.tar.gz
|
|
64
|
+
if-no-files-found: error
|
|
65
|
+
|
|
66
|
+
publish:
|
|
67
|
+
if: github.event_name == 'release' && !github.event.release.prerelease
|
|
68
|
+
needs: build
|
|
69
|
+
runs-on: ubuntu-latest
|
|
70
|
+
environment:
|
|
71
|
+
name: pypi
|
|
72
|
+
url: https://pypi.org/p/azrvus-dev-toolkit
|
|
73
|
+
permissions:
|
|
74
|
+
id-token: write
|
|
75
|
+
steps:
|
|
76
|
+
- uses: actions/download-artifact@v4
|
|
77
|
+
with:
|
|
78
|
+
name: distributions
|
|
79
|
+
path: dist
|
|
80
|
+
- name: Publish to PyPI using Trusted Publishing
|
|
81
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
82
|
+
|
|
83
|
+
testpypi:
|
|
84
|
+
if: github.event_name == 'workflow_dispatch' && inputs.destination == 'testpypi'
|
|
85
|
+
needs: build
|
|
86
|
+
runs-on: ubuntu-latest
|
|
87
|
+
environment:
|
|
88
|
+
name: testpypi
|
|
89
|
+
url: https://test.pypi.org/p/azrvus-dev-toolkit
|
|
90
|
+
permissions:
|
|
91
|
+
id-token: write
|
|
92
|
+
steps:
|
|
93
|
+
- uses: actions/download-artifact@v4
|
|
94
|
+
with:
|
|
95
|
+
name: distributions
|
|
96
|
+
path: dist
|
|
97
|
+
- name: Rehearse with TestPyPI Trusted Publishing
|
|
98
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
99
|
+
with:
|
|
100
|
+
repository-url: https://test.pypi.org/legacy/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 azrvus
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: azrvus-dev-toolkit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Dependency-free Python utilities and local JSONL validation for LLM evaluation datasets.
|
|
5
|
+
Project-URL: Homepage, https://github.com/azrvus/dev-toolkit
|
|
6
|
+
Project-URL: Repository, https://github.com/azrvus/dev-toolkit
|
|
7
|
+
Project-URL: Issues, https://github.com/azrvus/dev-toolkit/issues
|
|
8
|
+
Author: azrvus
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: developer-tools,evaluation,jsonl,llm,validation
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# Dev Toolkit
|
|
22
|
+
|
|
23
|
+
[](https://github.com/azrvus/dev-toolkit/actions)
|
|
24
|
+
[](LICENSE)
|
|
25
|
+
|
|
26
|
+
Published distribution name: `azrvus-dev-toolkit`; Python import: `dev_toolkit`;
|
|
27
|
+
CLI command: `dev-toolkit`.
|
|
28
|
+
|
|
29
|
+
Python 3.11+ utilities with no runtime dependencies. Includes JSON file I/O,
|
|
30
|
+
text helpers, validation, retries, caching, environment checks, and process tools.
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
Install from GitHub (verified in a fresh virtual environment):
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
python -m pip install 'git+https://github.com/azrvus/dev-toolkit.git'
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Public PyPI installation is pending release verification. Do not use
|
|
41
|
+
`pip install azrvus-dev-toolkit` or `uv add azrvus-dev-toolkit` yet.
|
|
42
|
+
|
|
43
|
+
## Python examples
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from dev_toolkit import is_email, slugify, read_json, write_json
|
|
47
|
+
|
|
48
|
+
assert slugify("Hello World") == "hello-world"
|
|
49
|
+
assert is_email("user@example.com")
|
|
50
|
+
write_json("example.json", {"status": "ok"})
|
|
51
|
+
assert read_json("example.json") == {"status": "ok"}
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`write_json` creates parent directories and refuses to overwrite existing files
|
|
55
|
+
unless `overwrite=True` is passed.
|
|
56
|
+
|
|
57
|
+
## Command line
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
dev-toolkit --help
|
|
61
|
+
dev-toolkit --version
|
|
62
|
+
dev-toolkit
|
|
63
|
+
dev-toolkit --verbose
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
With no subcommand, the CLI prints `Dev Toolkit CLI operational.`. With `--verbose`
|
|
67
|
+
(or `-v`), it also prints `Verbose mode enabled.`. Library utilities are used
|
|
68
|
+
through Python imports. The dataset validation subcommand is documented below.
|
|
69
|
+
|
|
70
|
+
## Development
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
uv venv
|
|
74
|
+
uv pip install -e . pytest ruff
|
|
75
|
+
uv run pytest
|
|
76
|
+
uv run ruff check .
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
CI runs tests and Ruff on Python 3.11 and 3.12. Test results are checked in CI;
|
|
80
|
+
no coverage percentage is claimed.
|
|
81
|
+
|
|
82
|
+
## License
|
|
83
|
+
|
|
84
|
+
[MIT](LICENSE).
|
|
85
|
+
|
|
86
|
+
## Validate LLM evaluation datasets
|
|
87
|
+
|
|
88
|
+
Check local UTF-8 JSONL without sending data anywhere. Each line must be a JSON
|
|
89
|
+
object. Blank lines, empty files, malformed JSON, nonfinite numbers, missing keys,
|
|
90
|
+
and unreadable input are errors. Required keys are top-level; null values count
|
|
91
|
+
as present. Errors include line numbers and never print complete records.
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
dev-toolkit validate-jsonl examples/eval-valid.jsonl --required-key prompt --required-key expected
|
|
95
|
+
# Valid JSONL. (exit 0)
|
|
96
|
+
dev-toolkit validate-jsonl examples/eval-invalid.jsonl --required-key prompt --required-key expected
|
|
97
|
+
# line 1: missing required keys: ['expected']
|
|
98
|
+
# line 2: expected a JSON object
|
|
99
|
+
# exit 1; diagnostics go to stderr
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
The command checks JSON structure and key presence, not semantic correctness or
|
|
103
|
+
model quality. File-level errors use `file:` instead of a line number.
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from dev_toolkit import validate_jsonl
|
|
107
|
+
|
|
108
|
+
for issue in validate_jsonl("examples/eval-valid.jsonl", required_keys=["prompt"]):
|
|
109
|
+
print(issue.line, issue.message)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The API yields issues lazily and reads one line at a time; consume the iterator
|
|
113
|
+
to perform validation. It retains no records and has no runtime dependencies.
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# Dev Toolkit
|
|
2
|
+
|
|
3
|
+
[](https://github.com/azrvus/dev-toolkit/actions)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
|
|
6
|
+
Published distribution name: `azrvus-dev-toolkit`; Python import: `dev_toolkit`;
|
|
7
|
+
CLI command: `dev-toolkit`.
|
|
8
|
+
|
|
9
|
+
Python 3.11+ utilities with no runtime dependencies. Includes JSON file I/O,
|
|
10
|
+
text helpers, validation, retries, caching, environment checks, and process tools.
|
|
11
|
+
|
|
12
|
+
## Installation
|
|
13
|
+
|
|
14
|
+
Install from GitHub (verified in a fresh virtual environment):
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
python -m pip install 'git+https://github.com/azrvus/dev-toolkit.git'
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Public PyPI installation is pending release verification. Do not use
|
|
21
|
+
`pip install azrvus-dev-toolkit` or `uv add azrvus-dev-toolkit` yet.
|
|
22
|
+
|
|
23
|
+
## Python examples
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
from dev_toolkit import is_email, slugify, read_json, write_json
|
|
27
|
+
|
|
28
|
+
assert slugify("Hello World") == "hello-world"
|
|
29
|
+
assert is_email("user@example.com")
|
|
30
|
+
write_json("example.json", {"status": "ok"})
|
|
31
|
+
assert read_json("example.json") == {"status": "ok"}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
`write_json` creates parent directories and refuses to overwrite existing files
|
|
35
|
+
unless `overwrite=True` is passed.
|
|
36
|
+
|
|
37
|
+
## Command line
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
dev-toolkit --help
|
|
41
|
+
dev-toolkit --version
|
|
42
|
+
dev-toolkit
|
|
43
|
+
dev-toolkit --verbose
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
With no subcommand, the CLI prints `Dev Toolkit CLI operational.`. With `--verbose`
|
|
47
|
+
(or `-v`), it also prints `Verbose mode enabled.`. Library utilities are used
|
|
48
|
+
through Python imports. The dataset validation subcommand is documented below.
|
|
49
|
+
|
|
50
|
+
## Development
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
uv venv
|
|
54
|
+
uv pip install -e . pytest ruff
|
|
55
|
+
uv run pytest
|
|
56
|
+
uv run ruff check .
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
CI runs tests and Ruff on Python 3.11 and 3.12. Test results are checked in CI;
|
|
60
|
+
no coverage percentage is claimed.
|
|
61
|
+
|
|
62
|
+
## License
|
|
63
|
+
|
|
64
|
+
[MIT](LICENSE).
|
|
65
|
+
|
|
66
|
+
## Validate LLM evaluation datasets
|
|
67
|
+
|
|
68
|
+
Check local UTF-8 JSONL without sending data anywhere. Each line must be a JSON
|
|
69
|
+
object. Blank lines, empty files, malformed JSON, nonfinite numbers, missing keys,
|
|
70
|
+
and unreadable input are errors. Required keys are top-level; null values count
|
|
71
|
+
as present. Errors include line numbers and never print complete records.
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
dev-toolkit validate-jsonl examples/eval-valid.jsonl --required-key prompt --required-key expected
|
|
75
|
+
# Valid JSONL. (exit 0)
|
|
76
|
+
dev-toolkit validate-jsonl examples/eval-invalid.jsonl --required-key prompt --required-key expected
|
|
77
|
+
# line 1: missing required keys: ['expected']
|
|
78
|
+
# line 2: expected a JSON object
|
|
79
|
+
# exit 1; diagnostics go to stderr
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The command checks JSON structure and key presence, not semantic correctness or
|
|
83
|
+
model quality. File-level errors use `file:` instead of a line number.
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from dev_toolkit import validate_jsonl
|
|
87
|
+
|
|
88
|
+
for issue in validate_jsonl("examples/eval-valid.jsonl", required_keys=["prompt"]):
|
|
89
|
+
print(issue.line, issue.message)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The API yields issues lazily and reads one line at a time; consume the iterator
|
|
93
|
+
to perform validation. It retains no records and has no runtime dependencies.
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Release procedure
|
|
2
|
+
|
|
3
|
+
The first public version is **0.1.0**. Publication is pending PyPI Trusted
|
|
4
|
+
Publisher setup and verification; building a distribution is not publication.
|
|
5
|
+
|
|
6
|
+
## One-time account setup
|
|
7
|
+
|
|
8
|
+
Register a pending GitHub Trusted Publisher at
|
|
9
|
+
<https://pypi.org/manage/account/publishing/> using these exact values:
|
|
10
|
+
|
|
11
|
+
| Field | Value |
|
|
12
|
+
| --- | --- |
|
|
13
|
+
| PyPI project name | `azrvus-dev-toolkit` |
|
|
14
|
+
| GitHub owner | `azrvus` |
|
|
15
|
+
| Repository | `dev-toolkit` |
|
|
16
|
+
| Workflow filename | `release.yml` |
|
|
17
|
+
| Environment | `pypi` |
|
|
18
|
+
|
|
19
|
+
No password or API token belongs in repository secrets or chat. If TestPyPI is
|
|
20
|
+
configured, use the same fields at
|
|
21
|
+
<https://test.pypi.org/manage/account/publishing/> with environment `testpypi`.
|
|
22
|
+
The two registries have separate accounts and publisher configurations.
|
|
23
|
+
|
|
24
|
+
See [PyPI's pending publisher guide](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/).
|
|
25
|
+
A 404 for the project does not reserve the name or guarantee PyPI will accept it.
|
|
26
|
+
Recheck <https://pypi.org/pypi/azrvus-dev-toolkit/json> immediately before publishing;
|
|
27
|
+
if someone else owns it, stop and select a new distribution name with the maintainer.
|
|
28
|
+
|
|
29
|
+
## Verify and optionally rehearse
|
|
30
|
+
|
|
31
|
+
After all code PRs are merged and main CI is green:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
gh workflow run release.yml -R azrvus/dev-toolkit --ref main -f destination=verify
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
This builds and checks wheel/sdist without publishing. When the TestPyPI account
|
|
38
|
+
and publisher are configured, use `-f destination=testpypi` to rehearse. Watch the
|
|
39
|
+
run and verify the TestPyPI version before continuing. TestPyPI versions cannot
|
|
40
|
+
be overwritten; do not dispatch the same version twice after a successful upload.
|
|
41
|
+
|
|
42
|
+
## Publish intentionally
|
|
43
|
+
|
|
44
|
+
1. Fetch main and record its verified commit. Confirm all checks are green and
|
|
45
|
+
`src/dev_toolkit/_version.py` agrees with the release tag `v0.1.0`.
|
|
46
|
+
2. Recheck the public project name and confirm Trusted Publisher configuration.
|
|
47
|
+
3. Publish a GitHub release against that exact merged commit:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
gh release create v0.1.0 -R azrvus/dev-toolkit --target VERIFIED_COMMIT \
|
|
51
|
+
--title 'Dev Toolkit 0.1.0' --notes-file RELEASE_NOTES.md
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
4. Watch the Release workflow. The build job repeats tests, Ruff, metadata checks,
|
|
55
|
+
and isolated artifact installs. Only the separate publishing job has OIDC
|
|
56
|
+
permission. It publishes the downloaded build artifacts without checking out
|
|
57
|
+
or executing package code. A prerelease does not publish to public PyPI.
|
|
58
|
+
5. Confirm <https://pypi.org/project/azrvus-dev-toolkit/0.1.0/> and test all three public
|
|
59
|
+
install paths in fresh environments outside the checkout:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
python -m pip install azrvus-dev-toolkit==0.1.0
|
|
63
|
+
uv add azrvus-dev-toolkit==0.1.0
|
|
64
|
+
uv tool install azrvus-dev-toolkit==0.1.0
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Isolate the uv tool directories using `UV_TOOL_DIR` and `UV_TOOL_BIN_DIR`.
|
|
68
|
+
Check imports, `dev-toolkit --version`, `--help`, `--verbose`, and valid/invalid
|
|
69
|
+
JSONL exit codes. These public install commands remain pending until verified.
|
|
70
|
+
6. Update README installation copy through a small PR after public verification.
|
|
71
|
+
|
|
72
|
+
If GitHub requires an environment approval, approve the named deployment in the
|
|
73
|
+
workflow run. If OIDC reports an invalid publisher, check all five fields above;
|
|
74
|
+
fix the account configuration and rerun the failed publishing job. Do not bypass
|
|
75
|
+
protections or substitute a long-lived token.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Dev Toolkit 0.1.0
|
|
2
|
+
|
|
3
|
+
First public release, pending publication and public install verification.
|
|
4
|
+
|
|
5
|
+
- Validate local UTF-8 JSONL evaluation datasets with `dev-toolkit validate-jsonl`.
|
|
6
|
+
Require top-level keys, find malformed JSON and invalid records by line number,
|
|
7
|
+
and return useful exit codes without printing dataset records.
|
|
8
|
+
- Use the streaming `validate_jsonl` Python API with no runtime dependencies.
|
|
9
|
+
- Use existing utilities for text, JSON I/O, environment checks, caching, retries,
|
|
10
|
+
processes, and more. Fix package exports and restore default IO test discovery.
|
|
11
|
+
- Publish as `azrvus-dev-toolkit` with the maintainer username prefix.
|
|
12
|
+
- Install the Python 3.11+ package under `dev_toolkit`; run the `dev-toolkit` CLI.
|
|
13
|
+
- Include the MIT license, accurate documentation, and one package/CLI version source.
|
|
14
|
+
- Verify wheel and source distributions in Python 3.11/3.12 CI, including isolated
|
|
15
|
+
installs, metadata checks, imports, CLI flags, and JSONL command behavior.
|
|
16
|
+
|
|
17
|
+
Validation at preparation: 134 tests pass. CI also runs Ruff and distribution
|
|
18
|
+
checks. The release workflow uses separate build/publish jobs and PyPI Trusted
|
|
19
|
+
Publishing, with an optional configured TestPyPI rehearsal.
|
|
20
|
+
|
|
21
|
+
The validator checks JSON structure and required key presence. It does not grade
|
|
22
|
+
model responses, enforce value types, or validate dataset semantics. Blank lines
|
|
23
|
+
and empty files are rejected; required keys may have null values.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "azrvus-dev-toolkit"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Dependency-free Python utilities and local JSONL validation for LLM evaluation datasets."
|
|
9
|
+
license = "MIT"
|
|
10
|
+
license-files = ["LICENSE"]
|
|
11
|
+
authors = [{name = "azrvus"}]
|
|
12
|
+
keywords = ["jsonl", "llm", "evaluation", "validation", "developer-tools"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
17
|
+
"Programming Language :: Python :: 3.11",
|
|
18
|
+
"Programming Language :: Python :: 3.12",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
]
|
|
21
|
+
readme = "README.md"
|
|
22
|
+
requires-python = ">=3.11"
|
|
23
|
+
dependencies = []
|
|
24
|
+
|
|
25
|
+
[project.scripts]
|
|
26
|
+
dev-toolkit = "dev_toolkit.cli:main"
|
|
27
|
+
|
|
28
|
+
[tool.pytest.ini_options]
|
|
29
|
+
testpaths = ["tests"]
|
|
30
|
+
pythonpath = ["src"]
|
|
31
|
+
|
|
32
|
+
[tool.ruff]
|
|
33
|
+
line-length = 88
|
|
34
|
+
target-version = "py311"
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://github.com/azrvus/dev-toolkit"
|
|
38
|
+
Repository = "https://github.com/azrvus/dev-toolkit"
|
|
39
|
+
Issues = "https://github.com/azrvus/dev-toolkit/issues"
|
|
40
|
+
|
|
41
|
+
[tool.hatch.version]
|
|
42
|
+
path = "src/dev_toolkit/_version.py"
|
|
43
|
+
|
|
44
|
+
[tool.hatch.build.targets.wheel]
|
|
45
|
+
packages = ["src/dev_toolkit"]
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Check distribution metadata and smoke-test both artifacts outside the checkout."""
|
|
2
|
+
|
|
3
|
+
import email
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
import tarfile
|
|
9
|
+
import tempfile
|
|
10
|
+
import zipfile
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
14
|
+
UV = shutil.which("uv")
|
|
15
|
+
if UV is None:
|
|
16
|
+
raise SystemExit("Install uv before checking distributions")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def run(*args, cwd, env):
|
|
20
|
+
return subprocess.run(
|
|
21
|
+
args, cwd=cwd, env=env, check=True, capture_output=True, text=True
|
|
22
|
+
).stdout.strip()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def main():
|
|
26
|
+
"""Verify metadata, then install and exercise wheel and source distribution."""
|
|
27
|
+
artifacts = sorted(
|
|
28
|
+
p for p in (ROOT / "dist").iterdir() if p.name.endswith((".whl", ".tar.gz"))
|
|
29
|
+
)
|
|
30
|
+
assert len(artifacts) == 2, "Expected exactly one wheel and one sdist"
|
|
31
|
+
for artifact in artifacts:
|
|
32
|
+
if artifact.suffix == ".whl":
|
|
33
|
+
with zipfile.ZipFile(artifact) as archive:
|
|
34
|
+
metadata_path = next(
|
|
35
|
+
name for name in archive.namelist() if name.endswith("/METADATA")
|
|
36
|
+
)
|
|
37
|
+
metadata = email.message_from_bytes(archive.read(metadata_path))
|
|
38
|
+
assert any(name.endswith("/LICENSE") for name in archive.namelist())
|
|
39
|
+
else:
|
|
40
|
+
assert artifact.name.endswith(".tar.gz")
|
|
41
|
+
with tarfile.open(artifact) as archive:
|
|
42
|
+
member = next(m for m in archive if m.name.endswith("/PKG-INFO"))
|
|
43
|
+
metadata = email.message_from_bytes(archive.extractfile(member).read())
|
|
44
|
+
assert any(m.name.endswith("/LICENSE") for m in archive)
|
|
45
|
+
assert metadata["Name"] == "azrvus-dev-toolkit"
|
|
46
|
+
assert metadata["Requires-Python"] == ">=3.11"
|
|
47
|
+
assert metadata["License-Expression"] == "MIT"
|
|
48
|
+
assert not metadata.get_all("Requires-Dist")
|
|
49
|
+
assert metadata["Project-URL"]
|
|
50
|
+
with tempfile.TemporaryDirectory() as directory:
|
|
51
|
+
work = Path(directory)
|
|
52
|
+
env = dict(os.environ)
|
|
53
|
+
env.pop("PYTHONPATH", None)
|
|
54
|
+
env.pop("VIRTUAL_ENV", None)
|
|
55
|
+
venv = work / "venv"
|
|
56
|
+
run(UV, "venv", "--python", sys.executable, str(venv), cwd=work, env=env)
|
|
57
|
+
scripts = venv / ("Scripts" if os.name == "nt" else "bin")
|
|
58
|
+
python = scripts / ("python.exe" if os.name == "nt" else "python")
|
|
59
|
+
cli = scripts / ("dev-toolkit.exe" if os.name == "nt" else "dev-toolkit")
|
|
60
|
+
run(
|
|
61
|
+
UV,
|
|
62
|
+
"pip",
|
|
63
|
+
"install",
|
|
64
|
+
"--python",
|
|
65
|
+
str(python),
|
|
66
|
+
str(artifact),
|
|
67
|
+
cwd=work,
|
|
68
|
+
env=env,
|
|
69
|
+
)
|
|
70
|
+
version = run(
|
|
71
|
+
str(python),
|
|
72
|
+
"-c",
|
|
73
|
+
(
|
|
74
|
+
"import dev_toolkit as d; from importlib.metadata import version; "
|
|
75
|
+
"assert all(hasattr(d, n) for n in d.__all__); "
|
|
76
|
+
"assert d.__version__ == version('azrvus-dev-toolkit'); "
|
|
77
|
+
"print(d.__version__)"
|
|
78
|
+
),
|
|
79
|
+
cwd=work,
|
|
80
|
+
env=env,
|
|
81
|
+
)
|
|
82
|
+
assert version == metadata["Version"]
|
|
83
|
+
assert run(str(cli), "--version", cwd=work, env=env) == (
|
|
84
|
+
f"dev-toolkit {version}"
|
|
85
|
+
)
|
|
86
|
+
assert "validate-jsonl" in run(str(cli), "--help", cwd=work, env=env)
|
|
87
|
+
assert run(str(cli), cwd=work, env=env) == "Dev Toolkit CLI operational."
|
|
88
|
+
assert "Verbose mode enabled." in run(str(cli), "-v", cwd=work, env=env)
|
|
89
|
+
data = work / "valid.jsonl"
|
|
90
|
+
data.write_text('{"prompt":"synthetic"}\n', encoding="utf-8")
|
|
91
|
+
assert (
|
|
92
|
+
run(
|
|
93
|
+
str(cli),
|
|
94
|
+
"validate-jsonl",
|
|
95
|
+
str(data),
|
|
96
|
+
"--required-key",
|
|
97
|
+
"prompt",
|
|
98
|
+
cwd=work,
|
|
99
|
+
env=env,
|
|
100
|
+
)
|
|
101
|
+
== "Valid JSONL."
|
|
102
|
+
)
|
|
103
|
+
invalid = subprocess.run(
|
|
104
|
+
[str(cli), "validate-jsonl", str(data), "--required-key", "expected"],
|
|
105
|
+
cwd=work,
|
|
106
|
+
env=env,
|
|
107
|
+
capture_output=True,
|
|
108
|
+
text=True,
|
|
109
|
+
check=False,
|
|
110
|
+
)
|
|
111
|
+
assert invalid.returncode == 1
|
|
112
|
+
assert "line 1: missing required keys" in invalid.stderr
|
|
113
|
+
assert "synthetic" not in invalid.stderr
|
|
114
|
+
print(f"Verified {artifact.name}")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
if __name__ == "__main__":
|
|
118
|
+
main()
|