localdesk 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. localdesk-0.3.0/.dockerignore +10 -0
  2. localdesk-0.3.0/.gitattributes +4 -0
  3. localdesk-0.3.0/.github/workflows/ci.yml +33 -0
  4. localdesk-0.3.0/.github/workflows/release.yml +80 -0
  5. localdesk-0.3.0/.gitignore +22 -0
  6. localdesk-0.3.0/CHANGELOG.md +50 -0
  7. localdesk-0.3.0/Dockerfile +10 -0
  8. localdesk-0.3.0/LICENSE +21 -0
  9. localdesk-0.3.0/NOTICE.md +43 -0
  10. localdesk-0.3.0/PKG-INFO +314 -0
  11. localdesk-0.3.0/README.md +274 -0
  12. localdesk-0.3.0/compose.yaml +18 -0
  13. localdesk-0.3.0/data/README.md +55 -0
  14. localdesk-0.3.0/docs/ARCHITECTURE.md +107 -0
  15. localdesk-0.3.0/docs/BENCHMARK.md +154 -0
  16. localdesk-0.3.0/docs/DATASETS.md +77 -0
  17. localdesk-0.3.0/docs/GPU_HTTP.md +89 -0
  18. localdesk-0.3.0/docs/RELEASING.md +29 -0
  19. localdesk-0.3.0/docs/RUNBOOK.md +92 -0
  20. localdesk-0.3.0/docs/TRAINING.md +111 -0
  21. localdesk-0.3.0/docs/TRAINING_PLAN.md +39 -0
  22. localdesk-0.3.0/docs/VALIDATION.md +109 -0
  23. localdesk-0.3.0/docs/demo.png +0 -0
  24. localdesk-0.3.0/docs/risk-coverage.svg +46 -0
  25. localdesk-0.3.0/docs/service-demo.png +0 -0
  26. localdesk-0.3.0/experiments/banking77.json +31 -0
  27. localdesk-0.3.0/experiments/clinc150.json +31 -0
  28. localdesk-0.3.0/experiments/it-support.json +34 -0
  29. localdesk-0.3.0/experiments/sms-filter.json +27 -0
  30. localdesk-0.3.0/notebooks/benchmark_qwen.ipynb +191 -0
  31. localdesk-0.3.0/notebooks/train_qwen.ipynb +215 -0
  32. localdesk-0.3.0/notebooks/train_qwen_drive.ipynb +272 -0
  33. localdesk-0.3.0/pyproject.toml +71 -0
  34. localdesk-0.3.0/reports/baseline.json +162 -0
  35. localdesk-0.3.0/reports/comparison-colab-it.json +1980 -0
  36. localdesk-0.3.0/reports/comparison-colab-it.md +27 -0
  37. localdesk-0.3.0/reports/comparison-it.json +1333 -0
  38. localdesk-0.3.0/reports/comparison-it.md +24 -0
  39. localdesk-0.3.0/reports/comparison-sms.json +751 -0
  40. localdesk-0.3.0/reports/comparison-sms.md +24 -0
  41. localdesk-0.3.0/reports/cpu-suite.json +1070 -0
  42. localdesk-0.3.0/reports/cpu-suite.md +32 -0
  43. localdesk-0.3.0/reports/data_manifest.json +3417 -0
  44. localdesk-0.3.0/reports/demo.json +309 -0
  45. localdesk-0.3.0/reports/gpu-http-it.json +383 -0
  46. localdesk-0.3.0/reports/observed-colab-it.json +30 -0
  47. localdesk-0.3.0/reports/qwen-serving-training.json +213 -0
  48. localdesk-0.3.0/reports/service-reuse.json +260 -0
  49. localdesk-0.3.0/requirements-gpu.txt +8 -0
  50. localdesk-0.3.0/scripts/build_benchmark_notebook.py +131 -0
  51. localdesk-0.3.0/scripts/build_drive_notebook.py +203 -0
  52. localdesk-0.3.0/scripts/fixtures/verify_gpu_http_20261012.py.txt +276 -0
  53. localdesk-0.3.0/scripts/prepare_banking77.py +76 -0
  54. localdesk-0.3.0/scripts/prepare_clinc150.py +51 -0
  55. localdesk-0.3.0/scripts/prepare_data.py +47 -0
  56. localdesk-0.3.0/scripts/prepare_sms.py +42 -0
  57. localdesk-0.3.0/scripts/publish_benchmark_report.py +92 -0
  58. localdesk-0.3.0/scripts/publish_cpu_suite.py +241 -0
  59. localdesk-0.3.0/scripts/train_baseline.py +141 -0
  60. localdesk-0.3.0/scripts/train_demo.py +77 -0
  61. localdesk-0.3.0/scripts/verify_it_qwen_http.py +11 -0
  62. localdesk-0.3.0/scripts/verify_service_reuse.py +89 -0
  63. localdesk-0.3.0/src/localdesk/__init__.py +3 -0
  64. localdesk-0.3.0/src/localdesk/analysis.py +86 -0
  65. localdesk-0.3.0/src/localdesk/api.py +375 -0
  66. localdesk-0.3.0/src/localdesk/assets/baseline.json.gz +0 -0
  67. localdesk-0.3.0/src/localdesk/assets/demo.json.gz +0 -0
  68. localdesk-0.3.0/src/localdesk/assets/index.html +50 -0
  69. localdesk-0.3.0/src/localdesk/assets/service.html +7 -0
  70. localdesk-0.3.0/src/localdesk/baseline.py +84 -0
  71. localdesk-0.3.0/src/localdesk/benchmark.py +618 -0
  72. localdesk-0.3.0/src/localdesk/benchmark_qwen.py +206 -0
  73. localdesk-0.3.0/src/localdesk/cli.py +151 -0
  74. localdesk-0.3.0/src/localdesk/config.py +284 -0
  75. localdesk-0.3.0/src/localdesk/core.py +80 -0
  76. localdesk-0.3.0/src/localdesk/data.py +207 -0
  77. localdesk-0.3.0/src/localdesk/dataset.py +251 -0
  78. localdesk-0.3.0/src/localdesk/metrics.py +107 -0
  79. localdesk-0.3.0/src/localdesk/model.py +103 -0
  80. localdesk-0.3.0/src/localdesk/pipeline.py +196 -0
  81. localdesk-0.3.0/src/localdesk/registry.py +229 -0
  82. localdesk-0.3.0/src/localdesk/schema.py +31 -0
  83. localdesk-0.3.0/src/localdesk/server.py +104 -0
  84. localdesk-0.3.0/src/localdesk/sources.py +96 -0
  85. localdesk-0.3.0/src/localdesk/tfidf.py +225 -0
  86. localdesk-0.3.0/src/localdesk/training.py +255 -0
  87. localdesk-0.3.0/tasks/banking77.json +86 -0
  88. localdesk-0.3.0/tasks/clinc150.json +159 -0
  89. localdesk-0.3.0/tasks/it-support.json +15 -0
  90. localdesk-0.3.0/tasks/sms-filter.json +11 -0
  91. localdesk-0.3.0/tests/test_api.py +201 -0
  92. localdesk-0.3.0/tests/test_baseline.py +63 -0
  93. localdesk-0.3.0/tests/test_benchmark.py +236 -0
  94. localdesk-0.3.0/tests/test_benchmark_qwen.py +116 -0
  95. localdesk-0.3.0/tests/test_configurable.py +233 -0
  96. localdesk-0.3.0/tests/test_core.py +99 -0
  97. localdesk-0.3.0/tests/test_data_metrics.py +157 -0
  98. localdesk-0.3.0/tests/test_demo.py +44 -0
  99. localdesk-0.3.0/tests/test_model.py +129 -0
  100. localdesk-0.3.0/tests/test_pipeline.py +246 -0
  101. localdesk-0.3.0/tests/test_server.py +62 -0
  102. localdesk-0.3.0/tests/test_tfidf.py +167 -0
  103. localdesk-0.3.0/tests/test_training.py +59 -0
  104. localdesk-0.3.0/uv.lock +629 -0
@@ -0,0 +1,10 @@
1
+ .git
2
+ .venv
3
+ __pycache__
4
+ data
5
+ artifacts
6
+ work
7
+ reports
8
+ tests
9
+ .env
10
+ *.zip
@@ -0,0 +1,4 @@
1
+ # Recipe/data hashes refer to LF bytes on every platform.
2
+ * text=auto eol=lf
3
+ *.gz binary
4
+ *.png binary
@@ -0,0 +1,33 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+
7
+ permissions:
8
+ contents: read
9
+
10
+ jobs:
11
+ checks:
12
+ strategy:
13
+ matrix:
14
+ os: [ubuntu-latest, windows-latest, macos-latest]
15
+ python: ['3.11', '3.12', '3.13']
16
+ runs-on: ${{ matrix.os }}
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - uses: actions/setup-python@v5
20
+ with:
21
+ python-version: ${{ matrix.python }}
22
+ - name: Install project and development tools
23
+ run: python -m pip install '.[dev,baseline,service]'
24
+ - name: Lint
25
+ run: ruff check .
26
+ - name: Format
27
+ run: ruff format --check .
28
+ - name: Test core, packaged model, notebook syntax and HTTP boundary
29
+ run: pytest -q
30
+ - name: Build wheel and sdist
31
+ run: python -m build
32
+ - name: CLI smoke
33
+ run: localdesk predict 'File share access request'
@@ -0,0 +1,80 @@
1
+ name: Release
2
+
3
+ # Publishes with PyPI Trusted Publishing (OIDC): no API tokens are stored in GitHub.
4
+ # A `v*` tag publishes to PyPI; a manual run publishes to TestPyPI for a dry run.
5
+ on:
6
+ push:
7
+ tags: ['v*']
8
+ workflow_dispatch:
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ jobs:
14
+ build:
15
+ runs-on: ubuntu-latest
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - uses: actions/setup-python@v5
19
+ with:
20
+ python-version: '3.13'
21
+ - name: Install build tools
22
+ run: python -m pip install build==1.3.0 twine==7.0.0
23
+ - name: Tag must match the package version
24
+ if: startsWith(github.ref, 'refs/tags/v')
25
+ run: |
26
+ python - <<'EOF'
27
+ import os, tomllib
28
+ version = tomllib.load(open("pyproject.toml", "rb"))["project"]["version"]
29
+ tag = os.environ["GITHUB_REF_NAME"].removeprefix("v")
30
+ if tag != version:
31
+ raise SystemExit(f"tag v{tag} does not match pyproject version {version}")
32
+ EOF
33
+ - name: Build wheel and sdist
34
+ run: python -m build
35
+ - name: Check package metadata and README rendering
36
+ run: twine check --strict dist/*
37
+ - name: Smoke test the wheel without optional dependencies
38
+ run: |
39
+ python -m venv /tmp/smoke
40
+ /tmp/smoke/bin/pip install dist/*.whl
41
+ /tmp/smoke/bin/localdesk predict 'File share access request'
42
+ - uses: actions/upload-artifact@v7
43
+ with:
44
+ name: dist
45
+ path: dist/
46
+ if-no-files-found: error
47
+
48
+ testpypi:
49
+ if: github.event_name == 'workflow_dispatch'
50
+ needs: build
51
+ runs-on: ubuntu-latest
52
+ environment:
53
+ name: testpypi
54
+ url: https://test.pypi.org/p/localdesk
55
+ permissions:
56
+ id-token: write
57
+ steps:
58
+ - uses: actions/download-artifact@v8
59
+ with:
60
+ name: dist
61
+ path: dist/
62
+ - uses: pypa/gh-action-pypi-publish@v1.14.2
63
+ with:
64
+ repository-url: https://test.pypi.org/legacy/
65
+
66
+ pypi:
67
+ if: startsWith(github.ref, 'refs/tags/v')
68
+ needs: build
69
+ runs-on: ubuntu-latest
70
+ environment:
71
+ name: pypi
72
+ url: https://pypi.org/p/localdesk
73
+ permissions:
74
+ id-token: write
75
+ steps:
76
+ - uses: actions/download-artifact@v8
77
+ with:
78
+ name: dist
79
+ path: dist/
80
+ - uses: pypa/gh-action-pypi-publish@v1.14.2
@@ -0,0 +1,22 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.py[cod]
4
+ .pytest_cache/
5
+ .ruff_cache/
6
+ *.egg-info/
7
+ dist/
8
+ build/
9
+ data/raw/
10
+ data/zenodo/
11
+ data/sms/
12
+ artifacts/
13
+ *.safetensors
14
+ *.pt
15
+ *.pth
16
+ *.pickle
17
+ *.h5
18
+ .env
19
+ .env.*
20
+ .DS_Store
21
+ data/banking77/
22
+ data/clinc150/
@@ -0,0 +1,50 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
5
+ [Semantic Versioning](https://semver.org/).
6
+
7
+ ## [0.3.0] - 2026-10-12
8
+
9
+ First release on PyPI. The distribution is now named `localdesk`
10
+ (previously `localdesk-decisions`, never published).
11
+
12
+ ### Added
13
+ - `tfidf` CPU backend: hashed word 1–2 grams and character 3–5 grams with
14
+ sublinear TF-IDF and balanced logistic regression. Inference needs only the
15
+ standard library; artifacts store hash buckets, not a training vocabulary.
16
+ Shipped experiments use it.
17
+ - Packaged TF-IDF demo model, now the default for `localdesk predict`, `serve`
18
+ and `evaluate` without `--model` (76.5% vs 69.5% accuracy on the IT holdout).
19
+ `--backend baseline` keeps the previous model.
20
+ - Risk-coverage curves in every training report (coverage at 90/95/99%
21
+ accuracy, AURC) and out-of-scope analysis (review share, AUROC).
22
+ - Banking77 and CLINC150 adapters with pinned sources and checksums, and
23
+ `scripts/publish_cpu_suite.py` for the four-dataset comparison.
24
+ - Release workflow using PyPI Trusted Publishing; Python 3.12 in CI.
25
+
26
+ ### Changed
27
+ - Concurrent API requests wait in a bounded queue (32 requests, 30 s) instead
28
+ of failing immediately with `MODEL_BUSY`.
29
+ - A failed training run moves its record to `.failed-<id>-…` and frees the
30
+ run ID for a retry.
31
+ - Optional dependencies use version ranges instead of exact pins; exact
32
+ tested versions stay in `uv.lock`. Tested at both ends of each range.
33
+ - README rewritten around a quick start and measured results.
34
+
35
+ ### Fixed
36
+ - A version that fails post-publish verification is withdrawn instead of
37
+ remaining under a completed run ID.
38
+ - `localdesk runs` lists damaged registry entries as invalid instead of
39
+ aborting.
40
+ - Dataset downloads read until EOF and retry truncated bodies.
41
+ - `/metrics` declares a type for every counter; JSONL errors name the line.
42
+
43
+ ## [0.2.0] - 2026-10-11
44
+
45
+ Configurable service: task/experiment JSON, immutable registry with quality
46
+ gates, FastAPI inference API, operator dashboard, Qwen/Unsloth recipe and the
47
+ comparative benchmark. Not published to PyPI.
48
+
49
+ [0.3.0]: https://github.com/denis-samatov/localdesk/compare/52a3f7c...v0.3.0
50
+ [0.2.0]: https://github.com/denis-samatov/localdesk/commit/52a3f7c
@@ -0,0 +1,10 @@
1
+ FROM python:3.13-slim
2
+ ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 LOCALDESK_REGISTRY=/models
3
+ WORKDIR /app
4
+ COPY pyproject.toml README.md LICENSE NOTICE.md ./
5
+ COPY src ./src
6
+ RUN python -m pip install --no-cache-dir '.[service]' && useradd --uid 10001 --create-home app
7
+ USER 10001
8
+ EXPOSE 8765
9
+ HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8765/health/ready', timeout=3)"
10
+ CMD ["uvicorn", "localdesk.api:application", "--factory", "--host", "0.0.0.0", "--port", "8765", "--no-access-log"]
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Denis Samatov
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,43 @@
1
+ The project code and original notebook are MIT licensed.
2
+
3
+ The training data are **Classification of IT Support Tickets**, Leonardo
4
+ Santiago Benitez Pereira, Federal Institute of Santa Catarina, 2022, v2.
5
+ Source: https://doi.org/10.5281/zenodo.7648117
6
+ License: https://creativecommons.org/licenses/by/4.0/
7
+
8
+ Changes made by this project: ID-based CSV joining; normalization for duplicate
9
+ detection (NFKC, case folding, whitespace collapse); conservative grouping by
10
+ PII-tag-stripped, tokenized and hashed baseline input features; duplicate and conflicting
11
+ label removal; a calibration split from source training data; source EOL templates
12
+ reserved as an unsupported-category challenge; conversion to Unsloth choice rows.
13
+ The published linear baseline was trained on this adapted data. Keep this attribution
14
+ with the baseline. Ticket texts are downloaded on demand and excluded from this Git repository.
15
+
16
+ The second configured domain uses **SMS Spam Collection**, Tiago Almeida and
17
+ Jos Hidalgo, UCI Machine Learning Repository, 2011.
18
+ Source: https://doi.org/10.24432/C5CC84 — CC BY 4.0.
19
+ Changes: verified archive extraction and conversion to id/text/label CSV;
20
+ conservative equivalent-input deduplication and deterministic grouped splits.
21
+ Original labels are retained. Raw SMS content is excluded from Git and serving logs.
22
+
23
+ The third and fourth domains are downloaded on demand and excluded from Git:
24
+
25
+ **Banking77**, Iñigo Casanueva, Tadas Temčinas, Daniela Gerz, Matthew Henderson
26
+ and Ivan Vulić, "Efficient Intent Detection with Dual Sentence Encoders", 2020.
27
+ Source: https://github.com/PolyAI-LDN/task-specific-datasets — CC BY 4.0.
28
+
29
+ **CLINC150**, Stefan Larson, Anish Mahendran, Joseph J. Peper, Christopher Clarke,
30
+ Andrew Lee, Parker Hill, Jonathan K. Kummerfeld, Kevin Leach, Michael A. Laurenzano,
31
+ Lingjia Tang and Jason Mars, "An Evaluation Dataset for Intent Classification and
32
+ Out-of-Scope Prediction", EMNLP 2019. Source: https://github.com/clinc/oos-eval —
33
+ CC BY 3.0 (https://creativecommons.org/licenses/by/3.0/).
34
+
35
+ Changes to both: verified extraction to id/text/label JSONL; collapsing of
36
+ model-equivalent duplicates so no group crosses splits; dropping of conflicting
37
+ labels; a deterministic calibration split from Banking77 training data; CLINC150
38
+ out-of-scope test queries kept in a separate file. Original labels are retained.
39
+
40
+ Optional Unsloth / Unsloth Zoo dependencies and the Qwen model retain their own
41
+ licenses. No Unsloth implementation code or notebook content is copied into this
42
+ repository. API integration does not relicense the upstream software. Consult the
43
+ upstream licenses before redistribution or service deployment.
@@ -0,0 +1,314 @@
1
+ Metadata-Version: 2.4
2
+ Name: localdesk
3
+ Version: 0.3.0
4
+ Summary: Calibrated text classification service that abstains when unsure: quality gates, immutable model registry, CPU TF-IDF and optional Qwen/Unsloth fine-tuning
5
+ Project-URL: Homepage, https://github.com/denis-samatov/localdesk
6
+ Project-URL: Documentation, https://github.com/denis-samatov/localdesk/tree/main/docs
7
+ Project-URL: Changelog, https://github.com/denis-samatov/localdesk/blob/main/CHANGELOG.md
8
+ Project-URL: Issues, https://github.com/denis-samatov/localdesk/issues
9
+ Author: Denis Samatov
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ License-File: NOTICE.md
13
+ Keywords: abstention,calibration,fastapi,intent-detection,mlops,model-registry,selective-classification,text-classification,ticket-routing,unsloth
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: Framework :: FastAPI
16
+ Classifier: Intended Audience :: Developers
17
+ Classifier: Intended Audience :: Science/Research
18
+ Classifier: Operating System :: OS Independent
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3 :: Only
21
+ Classifier: Programming Language :: Python :: 3.11
22
+ Classifier: Programming Language :: Python :: 3.12
23
+ Classifier: Programming Language :: Python :: 3.13
24
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
25
+ Classifier: Topic :: Text Processing :: Linguistic
26
+ Requires-Python: >=3.11
27
+ Provides-Extra: baseline
28
+ Requires-Dist: numpy<3,>=1.26; extra == 'baseline'
29
+ Requires-Dist: scikit-learn<2,>=1.6; extra == 'baseline'
30
+ Requires-Dist: scipy<2,>=1.13; extra == 'baseline'
31
+ Provides-Extra: dev
32
+ Requires-Dist: build==1.3.0; extra == 'dev'
33
+ Requires-Dist: httpx2==2.13.1; extra == 'dev'
34
+ Requires-Dist: pytest==8.4.2; extra == 'dev'
35
+ Requires-Dist: ruff==0.13.3; extra == 'dev'
36
+ Provides-Extra: service
37
+ Requires-Dist: fastapi<1,>=0.115; extra == 'service'
38
+ Requires-Dist: uvicorn<1,>=0.30; extra == 'service'
39
+ Description-Content-Type: text/markdown
40
+
41
+ # LocalDesk
42
+
43
+ **Train a text classifier that knows when it isn't sure, and ship it with receipts.**
44
+
45
+ [![CI](https://github.com/denis-samatov/localdesk/actions/workflows/ci.yml/badge.svg)](https://github.com/denis-samatov/localdesk/actions/workflows/ci.yml)
46
+ [![PyPI](https://img.shields.io/pypi/v/localdesk)](https://pypi.org/project/localdesk/)
47
+ ![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)
48
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](https://github.com/denis-samatov/localdesk/blob/main/LICENSE)
49
+ [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/denis-samatov/localdesk/blob/main/notebooks/benchmark_qwen.ipynb)
50
+
51
+ LocalDesk turns a CSV or JSONL file and a JSON list of labels into a versioned,
52
+ calibrated classification service that runs locally. It is built for routing:
53
+ support tickets, intents, spam, triage queues. In those settings a confident
54
+ wrong answer costs more than asking a human.
55
+
56
+ - **Abstains instead of guessing.** Every prediction is either `route` or `review`,
57
+ based on calibrated confidence and the margin between the top two labels.
58
+ - **Quality gates before deploy.** A version that misses your accuracy, macro-F1,
59
+ calibration or coverage thresholds cannot be promoted or served.
60
+ - **Every version is reproducible and tamper-evident.** Each version is immutable
61
+ and pins the task, the data fingerprints, the report and the training plan with
62
+ SHA-256 checksums.
63
+ - **CPU first, LLM when it pays off.** The default TF-IDF model trains in seconds,
64
+ reaches **91–99% accuracy on three of four public benchmarks**, and needs only the
65
+ Python standard library for inference. An optional Qwen + Unsloth LoRA recipe
66
+ gave a further **+5.7 pp accuracy** on the hardest one, real IT tickets.
67
+
68
+ ![LocalDesk operator dashboard](https://raw.githubusercontent.com/denis-samatov/localdesk/main/docs/service-demo.png)
69
+
70
+ ## Try it in 10 seconds
71
+
72
+ The package ships with the TF-IDF demo model trained on real IT tickets
73
+ (76.5% accuracy on the 524-ticket holdout, [report](https://github.com/denis-samatov/localdesk/blob/main/reports/demo.json)).
74
+ Running a prediction has no runtime dependencies:
75
+
76
+ ```bash
77
+ pip install localdesk
78
+ localdesk predict "File share access request"
79
+ localdesk predict "I can't open the shared finance folder"
80
+ ```
81
+
82
+ A clear request is routed. An ambiguous one goes to a human with the reason
83
+ attached (output abbreviated):
84
+
85
+ ```jsonc
86
+ {"category": "Fileservice", "action": "route", "probabilities": {"Fileservice": 0.999, ...}, "review_reasons": []}
87
+ {"category": "Fileservice", "action": "review", "probabilities": {"Fileservice": 0.43, "Support general": 0.24, ...},
88
+ "review_reasons": ["low_confidence", "small_margin"]}
89
+ ```
90
+
91
+ Run `localdesk serve` to open the same model in a local web page at `http://127.0.0.1:8765`.
92
+
93
+ ## Train on your own data (5 minutes)
94
+
95
+ **1. Describe the task.** The labels are yours; nothing in the core is IT-specific.
96
+
97
+ ```json
98
+ {
99
+ "id": "my-intents",
100
+ "version": "1",
101
+ "instructions": "Choose the intent of the message.",
102
+ "labels": {
103
+ "returns": "Requests to return a purchased item.",
104
+ "delivery": "Questions or problems with delivery.",
105
+ "other": "Other supported customer requests."
106
+ },
107
+ "min_confidence": 0.8,
108
+ "min_margin": 0.2
109
+ }
110
+ ```
111
+
112
+ **2. Point an experiment at your file and set the bar a model must clear.**
113
+
114
+ ```json
115
+ {
116
+ "task": "my-task.json",
117
+ "dataset": {
118
+ "path": "my-records.csv",
119
+ "format": "csv",
120
+ "text_column": "message",
121
+ "label_column": "intent",
122
+ "id_column": "id"
123
+ },
124
+ "registry": "my-registry",
125
+ "training": {"backend": "tfidf"},
126
+ "gates": {"min_accuracy": 0.7, "min_macro_f1": 0.6, "max_ece": 0.1}
127
+ }
128
+ ```
129
+
130
+ **3. Validate, train, promote, serve.**
131
+
132
+ ```bash
133
+ pip install 'localdesk[baseline,service]' # training + HTTP API
134
+
135
+ localdesk validate my-experiment.json # checks labels, leakage, class support
136
+ localdesk train my-experiment.json --run-id v1 # fit, calibrate, evaluate, gate
137
+ localdesk runs --registry my-registry # list versions and gate results
138
+ localdesk promote v1 --registry my-registry # refuses versions that failed gates
139
+ localdesk api --registry my-registry --anonymous-local
140
+ ```
141
+
142
+ Paths are relative to the experiment file. JSONL supports dotted fields such as
143
+ `gold.category`, and you can supply explicit train/calibration/test files.
144
+ Unknown labels, conflicting duplicates, groups that cross splits and typos in the
145
+ configuration fail early with a clear message.
146
+
147
+ ## Results on real public data
148
+
149
+ The same pipeline and commands on four public datasets. Only the configuration changes.
150
+ Default `tfidf` backend, test sets never used for fitting or calibration:
151
+
152
+ | Dataset | Labels | Test rows | Accuracy | Macro F1 | Auto-routed | Accuracy when routed |
153
+ | --- | ---: | ---: | ---: | ---: | ---: | ---: |
154
+ | [IT Support Tickets](https://zenodo.org/records/7648117) | 6 | 524 | 76.5% | 72.9% | 60.5% | **92.4%** |
155
+ | [UCI SMS Spam](https://archive.ics.uci.edu/dataset/228/sms+spam+collection) | 2 | 1,024 | 98.9% | 97.4% | 97.7% | **99.3%** |
156
+ | [Banking77](https://github.com/PolyAI-LDN/task-specific-datasets) | 77 | 3,071 | 91.2% | 91.3% | 86.9% | **96.8%** |
157
+ | [CLINC150](https://github.com/clinc/oos-eval) | 150 | 4,486 | 92.3% | 92.2% | 88.9% | **97.0%** |
158
+
159
+ "Auto-routed" is the share of requests the model handled without a human under
160
+ the task's fixed review policy, and "Accuracy when routed" is measured on those
161
+ requests only. Compared with the legacy hashed baseline, TF-IDF adds 1.5–7.1 pp
162
+ accuracy on every dataset and routes 2.6–16.4 pp more requests automatically
163
+ ([full comparison](https://github.com/denis-samatov/localdesk/blob/main/reports/cpu-suite.md)). Training takes 1–40 seconds on a laptop CPU.
164
+
165
+ ![Accuracy versus share of requests handled automatically](https://raw.githubusercontent.com/denis-samatov/localdesk/main/docs/risk-coverage.svg)
166
+
167
+ The fewer requests the model takes, the more accurate it gets. On Banking77,
168
+ TF-IDF keeps 95% accuracy on its most confident 93% of queries; the old baseline
169
+ holds that accuracy on only 84%. These curves are read off the test sets: use them
170
+ to choose a policy, then confirm it on your own data.
171
+
172
+ **Queries outside every label.** CLINC150 includes 1,000 test queries that match
173
+ none of its 150 intents and are never seen in training. Under the default policy,
174
+ **85% of them go to human review** instead of receiving a wrong intent.
175
+
176
+ To reproduce from a clone of this repository (the scripts download and checksum
177
+ the public data):
178
+
179
+ ```bash
180
+ pip install -e '.[baseline]'
181
+ python scripts/prepare_data.py && python scripts/prepare_sms.py
182
+ python scripts/prepare_banking77.py && python scripts/prepare_clinc150.py
183
+ python scripts/publish_cpu_suite.py # both CPU backends on all four datasets
184
+ ```
185
+
186
+ **Fine-tuned Qwen vs classic baselines** (same 524 IT test rows, free Colab T4):
187
+
188
+ | Method | Accuracy | Macro F1 | p50 latency |
189
+ | --- | ---: | ---: | ---: |
190
+ | Qwen 3.5 4B + LoRA (LocalDesk recipe) | **82.8%** | **80.6%** | 389 ms |
191
+ | TF-IDF + SVM (tuned) | 77.1% | 73.5% | 1.8 ms |
192
+ | TF-IDF + logistic regression | 77.7% | 74.0% | 2.1 ms |
193
+ | Qwen 3.5 4B zero-shot | 55.3% | 54.1% | 251 ms |
194
+
195
+ The accuracy gain over the SVM is +5.73 pp, with a paired-bootstrap 95% CI of
196
+ [+2.48, +9.16] pp. Macro-F1 gain is +7.06 pp, with a 95% CI of [+3.31, +11.10] pp.
197
+ Full protocol: [docs/BENCHMARK.md](https://github.com/denis-samatov/localdesk/blob/main/docs/BENCHMARK.md) · report:
198
+ [reports/comparison-colab-it.md](https://github.com/denis-samatov/localdesk/blob/main/reports/comparison-colab-it.md) ·
199
+ run it yourself: [Colab notebook](https://colab.research.google.com/github/denis-samatov/localdesk/blob/main/notebooks/benchmark_qwen.ipynb).
200
+
201
+ These are reused public holdouts, not a field trial. Measure on your own data
202
+ before relying on the numbers. [docs/VALIDATION.md](https://github.com/denis-samatov/localdesk/blob/main/docs/VALIDATION.md) lists
203
+ exactly what was and wasn't verified.
204
+
205
+ ## How it works
206
+
207
+ ```mermaid
208
+ flowchart LR
209
+ C[Task + experiment JSON] --> D[CSV / JSONL intake]
210
+ D --> V[Validate labels, group duplicates, isolate splits]
211
+ V --> T[Train: CPU TF-IDF or Qwen LoRA]
212
+ T --> K[Calibrate on a separate split]
213
+ K --> E[Evaluate on untouched test]
214
+ E --> G{Quality gates}
215
+ G -->|pass| R[Immutable version in registry]
216
+ G -->|fail| X[Kept for inspection, never served]
217
+ R --> P[promote / rollback]
218
+ P --> A[HTTP API + dashboard]
219
+ ```
220
+
221
+ - **No leakage by construction.** Records that share a conversation or produce
222
+ identical model features are grouped together, and a group never spans
223
+ train/calibration/test.
224
+ - **Training and serving are separate processes.** The HTTP API cannot upload
225
+ data, train or promote; it only answers predictions. Operators use the CLI.
226
+ - **Promotion is an atomic pointer swap.** Rollback is `localdesk promote <older-run>`.
227
+ Workers verify every checksum on startup.
228
+
229
+ More detail: [architecture and contracts](https://github.com/denis-samatov/localdesk/blob/main/docs/ARCHITECTURE.md) ·
230
+ [training plan](https://github.com/denis-samatov/localdesk/blob/main/docs/TRAINING_PLAN.md).
231
+
232
+ ## HTTP API
233
+
234
+ ```bash
235
+ export LOCALDESK_API_KEY=change-me
236
+ localdesk api --registry my-registry # or: docker compose up --build -d
237
+
238
+ curl http://127.0.0.1:8765/v1/predictions \
239
+ -H "X-API-Key: $LOCALDESK_API_KEY" -H 'Content-Type: application/json' \
240
+ -d '{"text": "Where is my parcel?"}'
241
+ ```
242
+
243
+ | Endpoint | Purpose |
244
+ | --- | --- |
245
+ | `POST /v1/predictions`, `POST /v1/predictions/batch` | Decision, probabilities, model version and task hash |
246
+ | `GET /v1/model`, `/v1/model/report`, `/v1/model/plan` | Active labels, measured quality, gates and training plan |
247
+ | `GET /health/live`, `/health/ready` | Liveness; readiness only when a qualified model is loaded |
248
+ | `GET /metrics` | Prometheus counters (no request bodies are logged) |
249
+
250
+ Inference requests are queued and run one at a time, so a single process can
251
+ safely hold a GPU model. The API key is compared in constant time, bodies are
252
+ size- and time-limited, and anonymous mode accepts only loopback hosts and
253
+ origins. The Docker setup runs as a non-root user with a read-only filesystem
254
+ and models mounted read-only. See the [runbook](https://github.com/denis-samatov/localdesk/blob/main/docs/RUNBOOK.md).
255
+
256
+ ## Fine-tune Qwen with Unsloth (optional, CUDA)
257
+
258
+ Swap the `training` block and keep the rest of the pipeline:
259
+
260
+ ```json
261
+ "training": {"backend": "unsloth", "epochs": 2, "rank": 16, "batch_size": 2,
262
+ "gradient_accumulation": 4, "learning_rate": 0.0002}
263
+ ```
264
+
265
+ The classification head is built from your label descriptions. Upstream
266
+ revisions are pinned in `requirements-gpu.txt`. Training stops on skipped or
267
+ truncated examples instead of silently degrading. Setup: [docs/TRAINING.md](https://github.com/denis-samatov/localdesk/blob/main/docs/TRAINING.md).
268
+ GPU serving details: [docs/GPU_HTTP.md](https://github.com/denis-samatov/localdesk/blob/main/docs/GPU_HTTP.md).
269
+
270
+ ## Status
271
+
272
+ LocalDesk is **v0.2, early but tested**. CI runs lint, formatting, the test suite,
273
+ package build and a CLI smoke test on Linux, macOS and Windows with Python 3.11,
274
+ 3.12 and 3.13.
275
+
276
+ Current scope is single-label text classification on a single node. Not included
277
+ yet: multi-label targets, a hosted Qwen checkpoint you can download, multi-tenant
278
+ isolation, and hot-swapping models without a restart.
279
+
280
+ ## Development
281
+
282
+ ```bash
283
+ git clone https://github.com/denis-samatov/localdesk.git && cd localdesk
284
+ uv sync --frozen --extra dev --extra baseline --extra service
285
+ uv run pytest -q
286
+ uv run ruff check . && uv run ruff format --check .
287
+ ```
288
+
289
+ Issues and pull requests are welcome. Please include a test for any change in behaviour.
290
+
291
+ ## Documentation
292
+
293
+ [Architecture](https://github.com/denis-samatov/localdesk/blob/main/docs/ARCHITECTURE.md) · [Runbook & API](https://github.com/denis-samatov/localdesk/blob/main/docs/RUNBOOK.md) ·
294
+ [Datasets & provenance](https://github.com/denis-samatov/localdesk/blob/main/docs/DATASETS.md) · [Training](https://github.com/denis-samatov/localdesk/blob/main/docs/TRAINING.md) ·
295
+ [Benchmark protocol](https://github.com/denis-samatov/localdesk/blob/main/docs/BENCHMARK.md) · [GPU HTTP check](https://github.com/denis-samatov/localdesk/blob/main/docs/GPU_HTTP.md) ·
296
+ [Validation evidence](https://github.com/denis-samatov/localdesk/blob/main/docs/VALIDATION.md)
297
+
298
+ ## По-русски
299
+
300
+ LocalDesk превращает CSV/JSONL и список меток в калиброванный сервис классификации
301
+ текста с версиями и порогами качества. Модель отвечает «маршрутизировать» или
302
+ «на ручной разбор», если не уверена. Ядро не привязано к домену: меняются только
303
+ JSON-описание задачи и датасет. Обучение на CPU занимает секунды. Дообучение
304
+ Qwen через Unsloth дало +5,7 п.п. точности к сильному TF-IDF baseline на реальных
305
+ IT-заявках. На четырёх публичных датасетах (IT-заявки, SMS, Banking77, CLINC150)
306
+ CPU-модель по умолчанию даёт от 76,5% до 98,9% точности, а 85% запросов вне всех
307
+ категорий отправляет на ручной разбор. Качество на других языках и в других организациях нужно измерять
308
+ отдельно.
309
+
310
+ ## License
311
+
312
+ Code and notebooks: [MIT](https://github.com/denis-samatov/localdesk/blob/main/LICENSE). The bundled demo model and benchmarks use
313
+ public datasets under CC BY 4.0; attribution is in [NOTICE.md](https://github.com/denis-samatov/localdesk/blob/main/NOTICE.md).
314
+ Optional dependencies and model weights keep their own licenses.