gigatok 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. gigatok-0.1.0/.cargo/config.toml +5 -0
  2. gigatok-0.1.0/.gitignore +252 -0
  3. gigatok-0.1.0/.python-version +1 -0
  4. gigatok-0.1.0/.vscode/settings.json +7 -0
  5. gigatok-0.1.0/Cargo.lock +5232 -0
  6. gigatok-0.1.0/Cargo.toml +102 -0
  7. gigatok-0.1.0/PKG-INFO +9 -0
  8. gigatok-0.1.0/README.md +6 -0
  9. gigatok-0.1.0/benches/encode.rs +43 -0
  10. gigatok-0.1.0/benches/encode_doc.rs +126 -0
  11. gigatok-0.1.0/benches/encode_st.rs +66 -0
  12. gigatok-0.1.0/benches/pretokenize.rs +151 -0
  13. gigatok-0.1.0/benches/pretokenize_profile.rs +83 -0
  14. gigatok-0.1.0/benches/simdutf_transcode.rs +82 -0
  15. gigatok-0.1.0/benches/unicode.rs +88 -0
  16. gigatok-0.1.0/benchmarks.md +89 -0
  17. gigatok-0.1.0/design_doc.md +109 -0
  18. gigatok-0.1.0/examples/bench_char_check.rs +118 -0
  19. gigatok-0.1.0/examples/bench_collision_check.rs +59 -0
  20. gigatok-0.1.0/examples/bench_word_count.rs +27 -0
  21. gigatok-0.1.0/examples/bpe_train_owt.py +14 -0
  22. gigatok-0.1.0/examples/olmo3_encode.py +121 -0
  23. gigatok-0.1.0/gigatok/__init__.py +20 -0
  24. gigatok-0.1.0/gigatok/_hf_compat.py +357 -0
  25. gigatok-0.1.0/gigatok/_load/__init__.py +1 -0
  26. gigatok-0.1.0/gigatok/_load/hf.py +56 -0
  27. gigatok-0.1.0/gigatok/_load/tiktoken.py +21 -0
  28. gigatok-0.1.0/gigatok/_tokenizer.py +64 -0
  29. gigatok-0.1.0/gigatok/gigatok_rs/__init__.pyi +95 -0
  30. gigatok-0.1.0/notebooks/bit_patterns.py +1977 -0
  31. gigatok-0.1.0/notebooks/bit_patterns2.py +240 -0
  32. gigatok-0.1.0/notebooks/data_view.py +155 -0
  33. gigatok-0.1.0/notebooks/dclm_sample.jsonl +3 -0
  34. gigatok-0.1.0/notebooks/dfa_states.py +817 -0
  35. gigatok-0.1.0/notebooks/gather_free.py +367 -0
  36. gigatok-0.1.0/notebooks/inspect_tokenizers.py +56 -0
  37. gigatok-0.1.0/notebooks/luke_ref.py +514 -0
  38. gigatok-0.1.0/notebooks/simd_unicode.py +590 -0
  39. gigatok-0.1.0/notebooks/unicode.py +103 -0
  40. gigatok-0.1.0/notebooks/unicode_classes.py +135 -0
  41. gigatok-0.1.0/notebooks/unicode_gather.py +17 -0
  42. gigatok-0.1.0/pretokenizer_optimization_log.md +306 -0
  43. gigatok-0.1.0/pyproject.toml +56 -0
  44. gigatok-0.1.0/radix_algorithm.md +241 -0
  45. gigatok-0.1.0/rust-toolchain.toml +2 -0
  46. gigatok-0.1.0/scripts/profile-cpu.fish +37 -0
  47. gigatok-0.1.0/src/bpe/mod.rs +402 -0
  48. gigatok-0.1.0/src/bpe/sentencepiece.rs +149 -0
  49. gigatok-0.1.0/src/bpe/tiktoken.rs +495 -0
  50. gigatok-0.1.0/src/bpe_train.rs +353 -0
  51. gigatok-0.1.0/src/encode/mod.rs +1 -0
  52. gigatok-0.1.0/src/encode/tiktoken_radix.rs +47 -0
  53. gigatok-0.1.0/src/input/decompress.rs +15 -0
  54. gigatok-0.1.0/src/input/file_source.rs +483 -0
  55. gigatok-0.1.0/src/input/jsonl.rs +94 -0
  56. gigatok-0.1.0/src/input/mod.rs +335 -0
  57. gigatok-0.1.0/src/lib.rs +1096 -0
  58. gigatok-0.1.0/src/load_tokenizer/hf.rs +513 -0
  59. gigatok-0.1.0/src/load_tokenizer/mod.rs +2 -0
  60. gigatok-0.1.0/src/load_tokenizer/tiktoken.rs +30 -0
  61. gigatok-0.1.0/src/main.rs +99 -0
  62. gigatok-0.1.0/src/output/mod.rs +1 -0
  63. gigatok-0.1.0/src/pretokenize/fast/cl100k.rs +417 -0
  64. gigatok-0.1.0/src/pretokenize/fast/deepseek_v3.rs +584 -0
  65. gigatok-0.1.0/src/pretokenize/fast/mod.rs +325 -0
  66. gigatok-0.1.0/src/pretokenize/fast/olmo3.rs +496 -0
  67. gigatok-0.1.0/src/pretokenize/fast/qwen2.rs +417 -0
  68. gigatok-0.1.0/src/pretokenize/fast/qwen3_5.rs +515 -0
  69. gigatok-0.1.0/src/pretokenize/fast/r50k.rs +461 -0
  70. gigatok-0.1.0/src/pretokenize/mod.rs +455 -0
  71. gigatok-0.1.0/src/pretokenize/options.rs +117 -0
  72. gigatok-0.1.0/src/pretokenize/pretoken.rs +42 -0
  73. gigatok-0.1.0/src/pretokenize/pretoken_avx512.rs +522 -0
  74. gigatok-0.1.0/src/pretokenize/pretoken_chunks.rs +2 -0
  75. gigatok-0.1.0/src/pretokenize/pretoken_combinator.rs +569 -0
  76. gigatok-0.1.0/src/pretokenize/pretoken_combinator_scalar.rs +217 -0
  77. gigatok-0.1.0/src/pretokenize/pretoken_simd.rs +852 -0
  78. gigatok-0.1.0/src/pretokenize/pretoken_state_machine.rs +384 -0
  79. gigatok-0.1.0/src/pretokenize/pretokenize_traits.rs +93 -0
  80. gigatok-0.1.0/src/pretokenize/unicode.rs +260 -0
  81. gigatok-0.1.0/src/simd/avx512/ascii.rs +144 -0
  82. gigatok-0.1.0/src/simd/avx512/mod.rs +3 -0
  83. gigatok-0.1.0/src/simd/avx512/table.rs +109 -0
  84. gigatok-0.1.0/src/simd/avx512/unicode.rs +71 -0
  85. gigatok-0.1.0/src/simd/m_series/ascii.rs +144 -0
  86. gigatok-0.1.0/src/simd/m_series/hashmap.rs +13 -0
  87. gigatok-0.1.0/src/simd/m_series/mod.rs +4 -0
  88. gigatok-0.1.0/src/simd/m_series/table.rs +224 -0
  89. gigatok-0.1.0/src/simd/m_series/unicode.rs +65 -0
  90. gigatok-0.1.0/src/simd/mod.rs +13 -0
  91. gigatok-0.1.0/src/token.rs +42 -0
  92. gigatok-0.1.0/src/unicode_tables.rs +222 -0
  93. gigatok-0.1.0/src/utils.rs +72 -0
  94. gigatok-0.1.0/strategy.md +33 -0
  95. gigatok-0.1.0/tests/bench_file_source.py +124 -0
  96. gigatok-0.1.0/tests/bench_train_encode.py +339 -0
  97. gigatok-0.1.0/tests/conftest.py +168 -0
  98. gigatok-0.1.0/tests/scripts/build_hf_tokenizer.py +63 -0
  99. gigatok-0.1.0/tests/scripts/build_toker_tokenizer.py +137 -0
  100. gigatok-0.1.0/tests/test_bpe_train.py +8 -0
  101. gigatok-0.1.0/tests/test_bpe_train_compare.py +249 -0
  102. gigatok-0.1.0/tests/test_encode.py +19 -0
  103. gigatok-0.1.0/tests/test_encode_dclm.py +67 -0
  104. gigatok-0.1.0/tests/test_encode_files.py +209 -0
  105. gigatok-0.1.0/tests/test_file_source.py +163 -0
  106. gigatok-0.1.0/tests/test_from_hf.py +138 -0
  107. gigatok-0.1.0/tests/test_from_tiktoken.py +165 -0
  108. gigatok-0.1.0/tests/test_hf_compat.py +334 -0
  109. gigatok-0.1.0/tests/test_load_hf.py +11 -0
  110. gigatok-0.1.0/tests/test_pretokenizer_iterator.py +62 -0
  111. gigatok-0.1.0/tests/test_tokenizer_usage.py +35 -0
  112. gigatok-0.1.0/tests/test_use_with_tokenizers.py +19 -0
  113. gigatok-0.1.0/tests/tokenizers/conftest.py +68 -0
  114. gigatok-0.1.0/tests/tokenizers/test_chunked_doc.py +75 -0
  115. gigatok-0.1.0/tests/tokenizers/test_hf_parity.py +266 -0
  116. gigatok-0.1.0/uv.lock +3159 -0
@@ -0,0 +1,5 @@
1
+ # Allow per-profile `rustflags` (nightly-only). This lets the `profiling`
2
+ # profile in Cargo.toml set `force-frame-pointers` without affecting normal
3
+ # release/dev builds.
4
+ [unstable]
5
+ profile-rustflags = true
@@ -0,0 +1,252 @@
1
+ # Generated by Cargo
2
+ # will have compiled files and executables
3
+ debug
4
+ target
5
+
6
+ # These are backup files generated by rustfmt
7
+ **/*.rs.bk
8
+
9
+ # MSVC Windows builds of rustc generate these, which store debugging information
10
+ *.pdb
11
+
12
+ # Generated by cargo mutants
13
+ # Contains mutation testing data
14
+ **/mutants.out*/
15
+
16
+ # RustRover
17
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
18
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
19
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
20
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
21
+ #.idea/
22
+
23
+
24
+ # Added by cargo
25
+
26
+ /target
27
+
28
+ # Byte-compiled / optimized / DLL files
29
+ __pycache__/
30
+ *.py[codz]
31
+ *$py.class
32
+
33
+ # C extensions
34
+ *.so
35
+
36
+ # Distribution / packaging
37
+ .Python
38
+ build/
39
+ develop-eggs/
40
+ dist/
41
+ downloads/
42
+ eggs/
43
+ .eggs/
44
+ lib/
45
+ lib64/
46
+ parts/
47
+ sdist/
48
+ var/
49
+ wheels/
50
+ share/python-wheels/
51
+ *.egg-info/
52
+ .installed.cfg
53
+ *.egg
54
+ MANIFEST
55
+
56
+ # PyInstaller
57
+ # Usually these files are written by a python script from a template
58
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
59
+ *.manifest
60
+ *.spec
61
+
62
+ # Installer logs
63
+ pip-log.txt
64
+ pip-delete-this-directory.txt
65
+
66
+ # Unit test / coverage reports
67
+ htmlcov/
68
+ .tox/
69
+ .nox/
70
+ .coverage
71
+ .coverage.*
72
+ .cache
73
+ nosetests.xml
74
+ coverage.xml
75
+ *.cover
76
+ *.py.cover
77
+ .hypothesis/
78
+ .pytest_cache/
79
+ cover/
80
+
81
+ # Translations
82
+ *.mo
83
+ *.pot
84
+
85
+ # Django stuff:
86
+ *.log
87
+ local_settings.py
88
+ db.sqlite3
89
+ db.sqlite3-journal
90
+
91
+ # Flask stuff:
92
+ instance/
93
+ .webassets-cache
94
+
95
+ # Scrapy stuff:
96
+ .scrapy
97
+
98
+ # Sphinx documentation
99
+ docs/_build/
100
+
101
+ # PyBuilder
102
+ .pybuilder/
103
+ target/
104
+
105
+ # Jupyter Notebook
106
+ .ipynb_checkpoints
107
+
108
+ # IPython
109
+ profile_default/
110
+ ipython_config.py
111
+
112
+ # pyenv
113
+ # For a library or package, you might want to ignore these files since the code is
114
+ # intended to run in multiple environments; otherwise, check them in:
115
+ # .python-version
116
+
117
+ # pipenv
118
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
119
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
120
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
121
+ # install all needed dependencies.
122
+ # Pipfile.lock
123
+
124
+ # UV
125
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
126
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
127
+ # commonly ignored for libraries.
128
+ # uv.lock
129
+
130
+ # poetry
131
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
132
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
133
+ # commonly ignored for libraries.
134
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
135
+ # poetry.lock
136
+ # poetry.toml
137
+
138
+ # pdm
139
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
140
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
141
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
142
+ # pdm.lock
143
+ # pdm.toml
144
+ .pdm-python
145
+ .pdm-build/
146
+
147
+ # pixi
148
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
149
+ # pixi.lock
150
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
151
+ # in the .venv directory. It is recommended not to include this directory in version control.
152
+ .pixi
153
+
154
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
155
+ __pypackages__/
156
+
157
+ # Celery stuff
158
+ celerybeat-schedule
159
+ celerybeat.pid
160
+
161
+ # Redis
162
+ *.rdb
163
+ *.aof
164
+ *.pid
165
+
166
+ # RabbitMQ
167
+ mnesia/
168
+ rabbitmq/
169
+ rabbitmq-data/
170
+
171
+ # ActiveMQ
172
+ activemq-data/
173
+
174
+ # SageMath parsed files
175
+ *.sage.py
176
+
177
+ # Environments
178
+ .env
179
+ .envrc
180
+ .venv
181
+ env/
182
+ venv/
183
+ ENV/
184
+ env.bak/
185
+ venv.bak/
186
+
187
+ # Spyder project settings
188
+ .spyderproject
189
+ .spyproject
190
+
191
+ # Rope project settings
192
+ .ropeproject
193
+
194
+ # mkdocs documentation
195
+ /site
196
+
197
+ # mypy
198
+ .mypy_cache/
199
+ .dmypy.json
200
+ dmypy.json
201
+
202
+ # Pyre type checker
203
+ .pyre/
204
+
205
+ # pytype static type analyzer
206
+ .pytype/
207
+
208
+ # Cython debug symbols
209
+ cython_debug/
210
+
211
+ # PyCharm
212
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
213
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
214
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
215
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
216
+ # .idea/
217
+
218
+ # Abstra
219
+ # Abstra is an AI-powered process automation framework.
220
+ # Ignore directories containing user credentials, local state, and settings.
221
+ # Learn more at https://abstra.io/docs
222
+ .abstra/
223
+
224
+ # Visual Studio Code
225
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
226
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
227
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
228
+ # you could uncomment the following to ignore the entire vscode folder
229
+ # .vscode/
230
+
231
+ # Ruff stuff:
232
+ .ruff_cache/
233
+
234
+ # PyPI configuration file
235
+ .pypirc
236
+
237
+ # Marimo
238
+ marimo/_static/
239
+ marimo/_lsp/
240
+ __marimo__/
241
+
242
+ # Streamlit
243
+ .streamlit/secrets.toml
244
+
245
+ .idea/
246
+
247
+ # Downloaded test data
248
+ /data/
249
+
250
+ .DS_Store
251
+
252
+ *.trace/
@@ -0,0 +1 @@
1
+ 3.13
@@ -0,0 +1,7 @@
1
+ {
2
+ "python.testing.pytestArgs": [
3
+ "tests"
4
+ ],
5
+ "python.testing.unittestEnabled": false,
6
+ "python.testing.pytestEnabled": true
7
+ }