tokenmizer 0.3.2__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. tokenmizer-0.5.0/.gitignore +37 -0
  2. tokenmizer-0.5.0/CHANGELOG.md +1367 -0
  3. tokenmizer-0.5.0/CONTRIBUTING.md +447 -0
  4. tokenmizer-0.5.0/PKG-INFO +426 -0
  5. tokenmizer-0.5.0/README.md +345 -0
  6. tokenmizer-0.5.0/SECURITY.md +245 -0
  7. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/benchmarks/checkpoint_accuracy/runner.py +0 -3
  8. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/benchmarks/checkpoint_accuracy/runner_v2.py +4 -2
  9. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/benchmarks/checkpoint_accuracy/runner_v3.py +2 -3
  10. tokenmizer-0.5.0/benchmarks/eval/__main__.py +253 -0
  11. tokenmizer-0.5.0/benchmarks/eval/corpus/data_migration.json +72 -0
  12. tokenmizer-0.5.0/benchmarks/eval/corpus/fastapi_auth.json +115 -0
  13. tokenmizer-0.5.0/benchmarks/eval/corpus/flaky_ci.json +61 -0
  14. tokenmizer-0.5.0/benchmarks/eval/corpus/frontend_perf.json +60 -0
  15. tokenmizer-0.5.0/benchmarks/eval/corpus/go_microservice.json +77 -0
  16. tokenmizer-0.5.0/benchmarks/eval/corpus/ml_pipeline.json +71 -0
  17. tokenmizer-0.5.0/benchmarks/eval/corpus/react_dashboard.json +78 -0
  18. tokenmizer-0.5.0/benchmarks/eval/corpus/real_audit_concurrency.json +69 -0
  19. tokenmizer-0.5.0/benchmarks/eval/corpus/real_audit_corruption.json +70 -0
  20. tokenmizer-0.5.0/benchmarks/eval/corpus/real_audit_extraction.json +79 -0
  21. tokenmizer-0.5.0/benchmarks/eval/corpus/real_audit_persistence.json +76 -0
  22. tokenmizer-0.5.0/benchmarks/eval/corpus/real_audit_security.json +68 -0
  23. tokenmizer-0.5.0/benchmarks/eval/corpus/real_audit_tokenizer.json +59 -0
  24. tokenmizer-0.5.0/benchmarks/eval/corpus/rust_cli.json +63 -0
  25. tokenmizer-0.5.0/benchmarks/eval/corpus.py +231 -0
  26. tokenmizer-0.5.0/benchmarks/eval/metrics.py +195 -0
  27. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/benchmarks/graph_retrieval/runner.py +1 -1
  28. tokenmizer-0.5.0/benchmarks/persistence/runner.py +207 -0
  29. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/pyproject.toml +48 -8
  30. tokenmizer-0.5.0/tests/conftest.py +43 -0
  31. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/integration/test_api_endpoint.py +51 -9
  32. tokenmizer-0.5.0/tests/unit/test_analytics_accuracy.py +170 -0
  33. tokenmizer-0.5.0/tests/unit/test_audit_fixes.py +436 -0
  34. tokenmizer-0.5.0/tests/unit/test_cache.py +188 -0
  35. tokenmizer-0.5.0/tests/unit/test_cache_scoping.py +172 -0
  36. tokenmizer-0.5.0/tests/unit/test_checkpoint_retention.py +111 -0
  37. tokenmizer-0.5.0/tests/unit/test_cli_commands.py +349 -0
  38. tokenmizer-0.5.0/tests/unit/test_cli_serve.py +91 -0
  39. tokenmizer-0.5.0/tests/unit/test_concurrency.py +196 -0
  40. tokenmizer-0.5.0/tests/unit/test_contested_decisions.py +140 -0
  41. tokenmizer-0.5.0/tests/unit/test_context_injection.py +94 -0
  42. tokenmizer-0.5.0/tests/unit/test_context_tracking.py +138 -0
  43. tokenmizer-0.5.0/tests/unit/test_dashboard.py +20 -0
  44. tokenmizer-0.5.0/tests/unit/test_decay_idempotence.py +148 -0
  45. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_decision_cache_async.py +34 -15
  46. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_decision_tracker.py +43 -14
  47. tokenmizer-0.5.0/tests/unit/test_durability.py +364 -0
  48. tokenmizer-0.5.0/tests/unit/test_endpoint_schema_extraction.py +124 -0
  49. tokenmizer-0.5.0/tests/unit/test_extraction_quality.py +463 -0
  50. tokenmizer-0.5.0/tests/unit/test_extractor_data_integrity.py +122 -0
  51. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_file_intelligence.py +73 -0
  52. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_graph.py +94 -5
  53. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_graph_persistence.py +9 -6
  54. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_hybrid_extractor.py +49 -13
  55. tokenmizer-0.5.0/tests/unit/test_internal_error_leak.py +96 -0
  56. tokenmizer-0.5.0/tests/unit/test_invalidate_label_validation.py +121 -0
  57. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_mcp_server.py +49 -18
  58. tokenmizer-0.5.0/tests/unit/test_multiprocess.py +224 -0
  59. tokenmizer-0.5.0/tests/unit/test_negation_scope.py +144 -0
  60. tokenmizer-0.5.0/tests/unit/test_persist_retry.py +165 -0
  61. tokenmizer-0.5.0/tests/unit/test_prune_cap.py +186 -0
  62. tokenmizer-0.5.0/tests/unit/test_rate_limit_key.py +114 -0
  63. tokenmizer-0.5.0/tests/unit/test_rate_limiter.py +82 -0
  64. tokenmizer-0.5.0/tests/unit/test_reasoning.py +244 -0
  65. tokenmizer-0.5.0/tests/unit/test_release_workflow.py +219 -0
  66. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_security.py +11 -3
  67. tokenmizer-0.5.0/tests/unit/test_settings.py +160 -0
  68. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_tokenizer.py +34 -0
  69. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_validator.py +86 -7
  70. tokenmizer-0.5.0/tests/unit/test_version_consistency.py +378 -0
  71. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/__init__.py +1 -1
  72. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/analytics/engine.py +91 -18
  73. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/api/app.py +512 -389
  74. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/api/rate_limiter.py +19 -11
  75. tokenmizer-0.5.0/tokenmizer/api/routes_graph.py +507 -0
  76. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/checkpoints/manager.py +166 -28
  77. tokenmizer-0.5.0/tokenmizer/cli.py +357 -0
  78. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/compression/engine.py +16 -10
  79. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/compression/output_trimmer.py +5 -5
  80. tokenmizer-0.5.0/tokenmizer/config/settings.py +347 -0
  81. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/core/dto.py +10 -5
  82. tokenmizer-0.5.0/tokenmizer/core/tokenizer.py +162 -0
  83. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/dashboard/page.py +51 -2
  84. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/filters/file_intelligence.py +79 -16
  85. tokenmizer-0.5.0/tokenmizer/graph_memory/context_block.py +231 -0
  86. tokenmizer-0.5.0/tokenmizer/graph_memory/decision_tracker.py +566 -0
  87. tokenmizer-0.5.0/tokenmizer/graph_memory/filelock.py +254 -0
  88. tokenmizer-0.5.0/tokenmizer/graph_memory/graph.py +911 -0
  89. tokenmizer-0.5.0/tokenmizer/graph_memory/hybrid_extractor.py +1651 -0
  90. tokenmizer-0.5.0/tokenmizer/graph_memory/ontology.py +161 -0
  91. tokenmizer-0.5.0/tokenmizer/graph_memory/persistence.py +769 -0
  92. tokenmizer-0.5.0/tokenmizer/graph_memory/pruning.py +230 -0
  93. tokenmizer-0.5.0/tokenmizer/graph_memory/reasoning.py +255 -0
  94. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/graph_memory/types.py +46 -0
  95. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/graph_memory/validator.py +112 -24
  96. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/graph_memory/visualization.py +7 -13
  97. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/mcp/server.py +93 -25
  98. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/providers/providers.py +116 -26
  99. tokenmizer-0.5.0/tokenmizer/security/__init__.py +0 -0
  100. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/security/auth.py +46 -17
  101. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/security/middleware.py +9 -14
  102. tokenmizer-0.5.0/tokenmizer/security/ownership.py +177 -0
  103. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/security/redaction.py +9 -13
  104. tokenmizer-0.5.0/tokenmizer/semantic_cache/__init__.py +0 -0
  105. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/semantic_cache/cache.py +129 -42
  106. tokenmizer-0.5.0/tokenmizer/state/__init__.py +0 -0
  107. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/state/backend.py +17 -9
  108. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/storage/__init__.py +5 -6
  109. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer.yaml +34 -3
  110. tokenmizer-0.3.2/.claude-plugin/marketplace.json +0 -31
  111. tokenmizer-0.3.2/.claude-plugin/plugin.json +0 -39
  112. tokenmizer-0.3.2/.claude-plugin/skills/analyze/SKILL.md +0 -58
  113. tokenmizer-0.3.2/.claude-plugin/skills/checkpoint/SKILL.md +0 -56
  114. tokenmizer-0.3.2/.claude-plugin/skills/resume/SKILL.md +0 -49
  115. tokenmizer-0.3.2/.claude-plugin/skills/stats/SKILL.md +0 -57
  116. tokenmizer-0.3.2/.github/ISSUE_TEMPLATE/bug_report.md +0 -24
  117. tokenmizer-0.3.2/.github/ISSUE_TEMPLATE/extraction_miss.md +0 -19
  118. tokenmizer-0.3.2/.github/PULL_REQUEST_TEMPLATE.md +0 -20
  119. tokenmizer-0.3.2/.github/workflows/ci.yml +0 -111
  120. tokenmizer-0.3.2/.github/workflows/release.yml +0 -71
  121. tokenmizer-0.3.2/.gitignore +0 -15
  122. tokenmizer-0.3.2/.mcp.json +0 -12
  123. tokenmizer-0.3.2/CHANGELOG.md +0 -491
  124. tokenmizer-0.3.2/CONTRIBUTING.md +0 -159
  125. tokenmizer-0.3.2/Dockerfile +0 -51
  126. tokenmizer-0.3.2/PKG-INFO +0 -696
  127. tokenmizer-0.3.2/README.md +0 -621
  128. tokenmizer-0.3.2/SECURITY.md +0 -166
  129. tokenmizer-0.3.2/TESTING.md +0 -83
  130. tokenmizer-0.3.2/USAGE.md +0 -526
  131. tokenmizer-0.3.2/docker-compose.yml +0 -43
  132. tokenmizer-0.3.2/docs/DEMO_SCRIPT.md +0 -142
  133. tokenmizer-0.3.2/docs/assets/architecture.svg +0 -169
  134. tokenmizer-0.3.2/docs/assets/demo.gif +0 -0
  135. tokenmizer-0.3.2/docs/assets/logo.svg +0 -246
  136. tokenmizer-0.3.2/docs/superpowers/plans/2026-07-10-audit-fixes.md +0 -90
  137. tokenmizer-0.3.2/examples/basic_usage.py +0 -156
  138. tokenmizer-0.3.2/scripts/gen_demo_gif.py +0 -92
  139. tokenmizer-0.3.2/scripts/install.sh +0 -260
  140. tokenmizer-0.3.2/scripts/mcp_e2e_check.py +0 -158
  141. tokenmizer-0.3.2/scripts/run_stdlib_tests.py +0 -398
  142. tokenmizer-0.3.2/scripts/setup.sh +0 -89
  143. tokenmizer-0.3.2/scripts/static_audit.py +0 -126
  144. tokenmizer-0.3.2/server.json +0 -34
  145. tokenmizer-0.3.2/tests/conftest.py +0 -14
  146. tokenmizer-0.3.2/tests/unit/test_cache.py +0 -102
  147. tokenmizer-0.3.2/tests/unit/test_rate_limiter.py +0 -48
  148. tokenmizer-0.3.2/tests/unit/test_version_consistency.py +0 -68
  149. tokenmizer-0.3.2/tokenmizer/cli.py +0 -166
  150. tokenmizer-0.3.2/tokenmizer/config/settings.py +0 -170
  151. tokenmizer-0.3.2/tokenmizer/core/tokenizer.py +0 -96
  152. tokenmizer-0.3.2/tokenmizer/graph_memory/decision_tracker.py +0 -309
  153. tokenmizer-0.3.2/tokenmizer/graph_memory/graph.py +0 -1394
  154. tokenmizer-0.3.2/tokenmizer/graph_memory/hybrid_extractor.py +0 -746
  155. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/LICENSE +0 -0
  156. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/benchmarks/__init__.py +0 -0
  157. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/benchmarks/checkpoint_accuracy/__init__.py +0 -0
  158. {tokenmizer-0.3.2/benchmarks/graph_retrieval → tokenmizer-0.5.0/benchmarks/eval}/__init__.py +0 -0
  159. {tokenmizer-0.3.2/benchmarks/latency → tokenmizer-0.5.0/benchmarks/graph_retrieval}/__init__.py +0 -0
  160. {tokenmizer-0.3.2/benchmarks/resume_quality → tokenmizer-0.5.0/benchmarks/latency}/__init__.py +0 -0
  161. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/benchmarks/latency/runner.py +0 -0
  162. {tokenmizer-0.3.2/tests → tokenmizer-0.5.0/benchmarks/persistence}/__init__.py +0 -0
  163. {tokenmizer-0.3.2/tests/chaos → tokenmizer-0.5.0/benchmarks/resume_quality}/__init__.py +0 -0
  164. {tokenmizer-0.3.2/tests/integration → tokenmizer-0.5.0/tests}/__init__.py +0 -0
  165. {tokenmizer-0.3.2/tests/memory_accuracy → tokenmizer-0.5.0/tests/chaos}/__init__.py +0 -0
  166. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/chaos/test_recovery.py +0 -0
  167. {tokenmizer-0.3.2/tests/unit → tokenmizer-0.5.0/tests/integration}/__init__.py +0 -0
  168. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/integration/test_checkpoint.py +0 -0
  169. {tokenmizer-0.3.2/tokenmizer/agents → tokenmizer-0.5.0/tests/memory_accuracy}/__init__.py +0 -0
  170. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/memory_accuracy/test_retention.py +0 -0
  171. {tokenmizer-0.3.2/tokenmizer/analytics → tokenmizer-0.5.0/tests/unit}/__init__.py +0 -0
  172. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tests/unit/test_compression_correctness.py +0 -0
  173. {tokenmizer-0.3.2/tokenmizer/api → tokenmizer-0.5.0/tokenmizer/agents}/__init__.py +0 -0
  174. {tokenmizer-0.3.2/tokenmizer/checkpoints → tokenmizer-0.5.0/tokenmizer/analytics}/__init__.py +0 -0
  175. {tokenmizer-0.3.2/tokenmizer/compression → tokenmizer-0.5.0/tokenmizer/api}/__init__.py +0 -0
  176. {tokenmizer-0.3.2/tokenmizer/config → tokenmizer-0.5.0/tokenmizer/checkpoints}/__init__.py +0 -0
  177. {tokenmizer-0.3.2/tokenmizer/core → tokenmizer-0.5.0/tokenmizer/compression}/__init__.py +0 -0
  178. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/compression/window.py +0 -0
  179. {tokenmizer-0.3.2/tokenmizer/dashboard → tokenmizer-0.5.0/tokenmizer/config}/__init__.py +0 -0
  180. {tokenmizer-0.3.2/tokenmizer/filters → tokenmizer-0.5.0/tokenmizer/core}/__init__.py +0 -0
  181. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/core/errors.py +0 -0
  182. {tokenmizer-0.3.2/tokenmizer/graph_memory → tokenmizer-0.5.0/tokenmizer/dashboard}/__init__.py +0 -0
  183. {tokenmizer-0.3.2/tokenmizer/mcp → tokenmizer-0.5.0/tokenmizer/filters}/__init__.py +0 -0
  184. {tokenmizer-0.3.2/tokenmizer/providers → tokenmizer-0.5.0/tokenmizer/graph_memory}/__init__.py +0 -0
  185. {tokenmizer-0.3.2 → tokenmizer-0.5.0}/tokenmizer/graph_memory/helpers.py +0 -0
  186. {tokenmizer-0.3.2/tokenmizer/security → tokenmizer-0.5.0/tokenmizer/mcp}/__init__.py +0 -0
  187. {tokenmizer-0.3.2/tokenmizer/semantic_cache → tokenmizer-0.5.0/tokenmizer/providers}/__init__.py +0 -0
  188. /tokenmizer-0.3.2/tokenmizer/state/__init__.py → /tokenmizer-0.5.0/tokenmizer/py.typed +0 -0
tokenmizer-0.5.0/.gitignore ADDED
@@ -0,0 +1,37 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ dist/
5
+ build/
6
+ .venv/
7
+ venv/
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .coverage
11
+ coverage.xml
12
+ htmlcov/
13
+
14
+ # Benchmark output. The runners rewrite these on every invocation; the
15
+ # numbers that matter live in the README, sourced from a runner you can
16
+ # re-run yourself. A stale committed result is worse than none.
17
+ benchmark_results.json
18
+ benchmark_v*_results.json
19
+ benchmark_persistence_results.json
20
+
21
+ # Runtime storage: graph/checkpoint/ownership databases and the
22
+ # per-session lock files created beside them.
23
+ #
24
+ # The leading slash anchors this to the REPO ROOT, so it ignores the
25
+ # default `storage_dir` without touching the `tokenmizer/checkpoints/`
26
+ # source package. An earlier revision commented the rule out entirely
27
+ # because an unanchored `checkpoints/` matched both — which is how 135
28
+ # lock files and two result JSONs ended up committed.
29
+ /checkpoints/
30
+ *.db
31
+ *.db-wal
32
+ *.db-shm
33
+ *.db.corrupt-*
34
+
35
+ .DS_Store
36
+ .idea/
37
+ .vscode/