wvc 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. wvc-0.1.0/.gitignore +37 -0
  2. wvc-0.1.0/CHANGELOG.md +12 -0
  3. wvc-0.1.0/CONTRIBUTING.md +47 -0
  4. wvc-0.1.0/LICENSE +202 -0
  5. wvc-0.1.0/PKG-INFO +216 -0
  6. wvc-0.1.0/README.md +191 -0
  7. wvc-0.1.0/SECURITY.md +32 -0
  8. wvc-0.1.0/pyproject.toml +112 -0
  9. wvc-0.1.0/spec/vectors/v0/README.md +37 -0
  10. wvc-0.1.0/spec/vectors/v0/a.manifest.json +35 -0
  11. wvc-0.1.0/spec/vectors/v0/a.safetensors +0 -0
  12. wvc-0.1.0/spec/vectors/v0/b.manifest.json +33 -0
  13. wvc-0.1.0/spec/vectors/v0/b.safetensors +0 -0
  14. wvc-0.1.0/spec/vectors/v0/diff.json +113 -0
  15. wvc-0.1.0/spec/vectors/v0/make_vectors.py +131 -0
  16. wvc-0.1.0/src/wvc/__init__.py +3 -0
  17. wvc-0.1.0/src/wvc/__main__.py +4 -0
  18. wvc-0.1.0/src/wvc/bisect.py +298 -0
  19. wvc-0.1.0/src/wvc/canonical.py +53 -0
  20. wvc-0.1.0/src/wvc/card.py +87 -0
  21. wvc-0.1.0/src/wvc/checkout.py +386 -0
  22. wvc-0.1.0/src/wvc/cli.py +656 -0
  23. wvc-0.1.0/src/wvc/cli_bisect.py +80 -0
  24. wvc-0.1.0/src/wvc/cli_common.py +63 -0
  25. wvc-0.1.0/src/wvc/cli_fsck.py +40 -0
  26. wvc-0.1.0/src/wvc/cli_peft.py +70 -0
  27. wvc-0.1.0/src/wvc/cli_remote.py +136 -0
  28. wvc-0.1.0/src/wvc/codec.py +282 -0
  29. wvc-0.1.0/src/wvc/diff.py +351 -0
  30. wvc-0.1.0/src/wvc/errors.py +27 -0
  31. wvc-0.1.0/src/wvc/extras.py +258 -0
  32. wvc-0.1.0/src/wvc/file_layout.py +212 -0
  33. wvc-0.1.0/src/wvc/fsck.py +197 -0
  34. wvc-0.1.0/src/wvc/fsio.py +361 -0
  35. wvc-0.1.0/src/wvc/hashing.py +149 -0
  36. wvc-0.1.0/src/wvc/hub.py +301 -0
  37. wvc-0.1.0/src/wvc/ids.py +33 -0
  38. wvc-0.1.0/src/wvc/lowrank.py +156 -0
  39. wvc-0.1.0/src/wvc/merge_adapter.py +167 -0
  40. wvc-0.1.0/src/wvc/output.py +43 -0
  41. wvc-0.1.0/src/wvc/peft.py +214 -0
  42. wvc-0.1.0/src/wvc/py.typed +0 -0
  43. wvc-0.1.0/src/wvc/quote.py +30 -0
  44. wvc-0.1.0/src/wvc/records.py +310 -0
  45. wvc-0.1.0/src/wvc/refs.py +194 -0
  46. wvc-0.1.0/src/wvc/remote.py +614 -0
  47. wvc-0.1.0/src/wvc/repo.py +732 -0
  48. wvc-0.1.0/src/wvc/reuse.py +184 -0
  49. wvc-0.1.0/src/wvc/safetensors_io.py +508 -0
  50. wvc-0.1.0/src/wvc/sources.py +99 -0
  51. wvc-0.1.0/src/wvc/store.py +672 -0
  52. wvc-0.1.0/src/wvc/store_config.py +109 -0
  53. wvc-0.1.0/src/wvc/store_objects.py +133 -0
  54. wvc-0.1.0/src/wvc/transport.py +355 -0
  55. wvc-0.1.0/src/wvc/verify.py +126 -0
  56. wvc-0.1.0/tests/conftest.py +63 -0
  57. wvc-0.1.0/tests/helpers.py +375 -0
  58. wvc-0.1.0/tests/test_bisect.py +434 -0
  59. wvc-0.1.0/tests/test_canonical.py +67 -0
  60. wvc-0.1.0/tests/test_checkpoint_lifecycle.py +178 -0
  61. wvc-0.1.0/tests/test_cli.py +257 -0
  62. wvc-0.1.0/tests/test_cli_repo.py +338 -0
  63. wvc-0.1.0/tests/test_codec.py +252 -0
  64. wvc-0.1.0/tests/test_diff.py +236 -0
  65. wvc-0.1.0/tests/test_extras.py +939 -0
  66. wvc-0.1.0/tests/test_fsck.py +169 -0
  67. wvc-0.1.0/tests/test_fsio.py +199 -0
  68. wvc-0.1.0/tests/test_gguf_hash.py +111 -0
  69. wvc-0.1.0/tests/test_hashing.py +56 -0
  70. wvc-0.1.0/tests/test_header_fuzz.py +409 -0
  71. wvc-0.1.0/tests/test_hub.py +479 -0
  72. wvc-0.1.0/tests/test_lowrank.py +469 -0
  73. wvc-0.1.0/tests/test_peft.py +676 -0
  74. wvc-0.1.0/tests/test_refs.py +135 -0
  75. wvc-0.1.0/tests/test_remote.py +944 -0
  76. wvc-0.1.0/tests/test_repo.py +541 -0
  77. wvc-0.1.0/tests/test_repo_integrity.py +656 -0
  78. wvc-0.1.0/tests/test_safetensors_io.py +321 -0
  79. wvc-0.1.0/tests/test_sources_card.py +221 -0
  80. wvc-0.1.0/tests/test_store.py +611 -0
  81. wvc-0.1.0/tests/test_store_trust.py +253 -0
  82. wvc-0.1.0/tests/test_tensor_limits.py +300 -0
  83. wvc-0.1.0/tests/test_transport.py +471 -0
  84. wvc-0.1.0/tests/test_vectors.py +58 -0
  85. wvc-0.1.0/tests/test_verify.py +423 -0
  86. wvc-0.1.0/tests/test_version.py +27 -0
wvc-0.1.0/.gitignore ADDED
@@ -0,0 +1,37 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.pyc
4
+ .pytest_cache/
5
+ .ruff_cache/
6
+ dist/
7
+ build/
8
+ *.egg-info/
9
+ # Weights and run artifacts stay out of Git.
10
+ *.safetensors
11
+ # The tiny spec test vectors are tracked.
12
+ !spec/vectors/**/*.safetensors
13
+ *.gguf
14
+ *.npz
15
+ models/
16
+ runs/
17
+ .coverage
18
+ htmlcov/
19
+ # E003 end-to-end repository and checkouts hold weights.
20
+ experiments/E003/repo/
21
+ experiments/E003/checkout-*/
22
+ # A wvc repository created inside this source tree holds weight payloads.
23
+ .wvc/
24
+ # E004 holds a training run, generations, and a wvc repository with weights.
25
+ experiments/E004/repo/
26
+ experiments/E004/raw/
27
+ experiments/E004/checkpoints/
28
+ experiments/E004/merged/
29
+ experiments/E004/work/
30
+
31
+ # macOS
32
+ .DS_Store
33
+ # E005 repository and checkouts hold weights.
34
+ experiments/E005/repo/
35
+ experiments/E005/checkout-*/
36
+ .hypothesis/
37
+ CLAUDE.local.md
wvc-0.1.0/CHANGELOG.md ADDED
@@ -0,0 +1,12 @@
1
+ # Changelog
2
+
3
+ User-visible changes, newest first. Releases are git tags `vX.Y.Z`: CI publishes a tag to PyPI after the checks pass, and a tag must name a section here.
4
+
5
+ ## 0.1.0
6
+ - Hardened against hostile input before this first release, following two rounds of code and security review: a checkpoint, store, or remote controlled by someone else can make wvc refuse, never write outside the directory you named, run code, or report bytes as verified that are not. What it refuses, and why: tensor records above `max_tensor_bytes` (16 GiB by default, 1 TiB at most) and frames that do not match their record, so a kilobyte store cannot claim gigabytes; shapes over 64 dimensions and headers with more brackets than tensors could need; non-regular input files and symbolic links out of a checkpoint; remote files larger than their records imply; a stored object that does not decode back to its tensor id; `--force` replacing a file no earlier checkout wrote unless named with `--overwrite`; and values longer than 40 characters in any message. Checkouts check free disk space first and move existing files aside so a failed rename restores them; every wvc error exits 2, a bug propagates as a traceback, and `WVC_DEBUG=1` re-raises. The documented limits are in `SECURITY.md`.
7
+ - First release: `hash`, `diff`, and `verify` on local and Hub checkpoints; repositories with `init`, `commit`, `log`, `show`, `checkout`, and `fsck`; `bisect`; `merge-adapter`; `card`; `remote`, `push`, `pull`, and `clone`; spec v0 with conformance vectors under `spec/vectors/v0`.
8
+ - `wvc --version` prints the version.
9
+ - A `v*` tag on a `main` commit publishes the sdist and wheel to PyPI through trusted publishing (`.github/workflows/release.yml`), after the tag is checked against the project version and this changelog, the checks pass on Ubuntu, and the sdist is checked to hold only the package, tests, conformance vectors, and top-level documents.
10
+ - `wvc diff --rank` estimates the rank of touched blocks up to 2^26 elements, so a 9B-class projection matrix gets a rank without `--rank-cap`. The block is decoded to float64 and copied twice, so a block at the cap peaks near 1.1 GiB of memory and can take about 20 s.
11
+ - `wvc hash`, `wvc diff`, and `wvc verify` accept a Hub repository as `hf:OWNER/NAME[@REVISION]` (install the `hf` extra); a diff of two Hub repositories fetches only the shards whose Hub hashes differ and reports the rest unchanged from their headers.
12
+ - `wvc hash --format gguf-hash` prints llama.cpp gguf-hash manifest lines for a safetensors checkpoint.
@@ -0,0 +1,47 @@
1
+ # Contributing to wvc
2
+
3
+ wvc has one maintainer. Bug reports, use cases, reviews, and small focused pull requests are welcome. Open an issue before starting anything large.
4
+
5
+ ## Scope and non-goals
6
+ - In scope (v0): `hash`, `diff`, `verify`, and a local object store with `init`, `commit`, `log`, `show`, and `checkout`; remotes as plain directories first, then the Hugging Face Hub and OCI registries. The formats are specified in [docs/spec-v0.md](docs/spec-v0.md).
7
+ - Non-goals: hosting, git filters, a UI, lossy-only storage (a checkout must reproduce bytes exactly), and runtime dependencies beyond numpy and zstandard (no PyTorch).
8
+ - Scope, identity, and licence are maintainer decisions; the chosen formats are in [docs/spec-v0.md](docs/spec-v0.md), and the reasoning behind each one is kept in the maintainer's research notes outside this repository. Work outside them needs an issue and a recorded decision first.
9
+
10
+ ## Running the checks
11
+ CI runs these commands on Ubuntu, macOS, and Windows with Python 3.11 and 3.13.
12
+
13
+ ```bash
14
+ uv sync
15
+ uv run ruff check .
16
+ uv run ruff format --check .
17
+ uvx ty@0.0.95 check src
18
+ uv run pytest -p randomly --cov=wvc --cov-report=term-missing --cov-fail-under=95
19
+ ```
20
+
21
+ - The test run is in random order (pytest-randomly prints its seed in the header; `--randomly-seed=N` repeats an order), treats every warning as an error (`filterwarnings = ["error"]` in `pyproject.toml`), and fails below 95% line coverage. If a third-party warning appears on one platform, ignore that one message in `filterwarnings` rather than dropping the setting.
22
+ - `tests/test_header_fuzz.py` fuzzes both safetensors header parsers with Hypothesis (200 examples per property, a few seconds in all).
23
+ - On Ubuntu only, CI also runs `uvx codespell .` with the skip list in `[tool.codespell]` in `pyproject.toml`; words that are correct here go in its `ignore-words-list`.
24
+ - A `package` job runs `uv build`, `uvx twine check dist/*`, and `validate-pyproject`, and fails if the sdist holds anything beyond the package, its tests, the conformance vectors, and the top-level documents.
25
+ - An `audit` job runs `pip-audit` on the locked dependencies (`uv export` then `uvx pip-audit -r requirements.txt --disable-pip --no-deps`) on every push and every Monday at 06:00 UTC; it does not block the test matrix.
26
+ - CI installs with `uv sync --locked`, so commit `uv.lock` together with any dependency change. CI pins the `ty` version it runs; bump the pin in `.github/workflows/ci.yml` deliberately.
27
+ - Tests use small synthetic safetensors fixtures and must not download weights or need the network.
28
+ - Never commit weights or run artifacts (`*.safetensors`, `*.gguf`, `*.npz`, `models/`, `runs/`).
29
+
30
+ ## Experiments and receipts
31
+ - Every experiment gets an ID (E001, E002, ...) and a receipt under [experiments/](experiments/): commands, input revisions and hashes, tool versions, the code revision, and outputs.
32
+ - Numbers are never restated in prose (README, docs, issues, pull requests) without a link to the receipt they come from. Receipts outrank summaries.
33
+ - Negative and failed results get receipts too.
34
+
35
+ ## Commit messages
36
+ - Use `type: description`, where type is one of `feat`, `fix`, `docs`, `test`, `chore`, `refactor`, `perf`, or `ci`; for example `docs: record the code revision for E001`.
37
+ - No attribution trailers (no `Co-Authored-By:` or "Generated with" lines).
38
+
39
+ ## Code style
40
+ - numpy and zstandard are the only runtime dependencies.
41
+ - Keep functions small and single-purpose.
42
+ - Validate explicitly at boundaries (file headers, manifests, CLI arguments) and raise clear errors.
43
+ - No `no `print` outside the CLI modules (`cli*.py`)
44
+ - Line length is 100; ruff enforces formatting and the lint rules in `pyproject.toml`.
45
+
46
+ ## Licence
47
+ wvc is licensed under the [Apache License 2.0](LICENSE). Contributions are accepted under the same licence.
wvc-0.1.0/LICENSE ADDED
@@ -0,0 +1,202 @@
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
wvc-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,216 @@
1
+ Metadata-Version: 2.5
2
+ Name: wvc
3
+ Version: 0.1.0
4
+ Summary: Version control primitives for model weights: tensor-level hashing and diffs of safetensors checkpoints, without PyTorch.
5
+ Project-URL: Homepage, https://github.com/evnsnclr/wvc
6
+ Project-URL: Repository, https://github.com/evnsnclr/wvc
7
+ Project-URL: Issues, https://github.com/evnsnclr/wvc/issues
8
+ Project-URL: Changelog, https://github.com/evnsnclr/wvc/blob/main/CHANGELOG.md
9
+ Project-URL: Security, https://github.com/evnsnclr/wvc/blob/main/SECURITY.md
10
+ Author: Evan Smith
11
+ License-Expression: Apache-2.0
12
+ License-File: LICENSE
13
+ Keywords: checkpoints,model weights,provenance,safetensors,version control
14
+ Classifier: Development Status :: 3 - Alpha
15
+ Classifier: Intended Audience :: Science/Research
16
+ Classifier: License :: OSI Approved :: Apache Software License
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Requires-Python: >=3.11
20
+ Requires-Dist: numpy>=1.26
21
+ Requires-Dist: zstandard>=0.22
22
+ Provides-Extra: hf
23
+ Requires-Dist: huggingface-hub>=1; extra == 'hf'
24
+ Description-Content-Type: text/markdown
25
+
26
+ <p align="center">
27
+ <img src="docs/assets/hero.svg" width="100%" alt="wvc: version control for open model weights. Animation of a real diff: two Qwen3.5-9B checkpoints on the Hugging Face Hub are compared shard by shard. Three of the four shards have the same SHA-256 on both sides and are skipped, the fourth is fetched, and one of its 53 tensors, layer 17's mlp.down_proj.weight, lights up as changed, with rank 1.">
28
+ </p>
29
+
30
+ <p align="center">
31
+ <a href="#quick-start">Quick start</a> ·
32
+ <a href="#what-it-found">What it found</a> ·
33
+ <a href="#how-it-works">How it works</a> ·
34
+ <a href="#commands">Commands</a> ·
35
+ <a href="#status">Status</a> ·
36
+ <a href="docs/spec-v0.md">Spec</a> ·
37
+ <a href="experiments/">Receipts</a>
38
+ </p>
39
+
40
+ <p align="center">
41
+ <img alt="Release: none yet" src="https://img.shields.io/badge/release-none%20yet-8b949e">
42
+ <img alt="Python 3.11+" src="https://img.shields.io/badge/python-3.11%2B-3776ab">
43
+ <a href="LICENSE"><img alt="Licence: Apache-2.0" src="https://img.shields.io/badge/licence-Apache--2.0-2ea44f"></a>
44
+ <a href="https://github.com/evnsnclr/wvc/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/evnsnclr/wvc/actions/workflows/ci.yml/badge.svg"></a>
45
+ </p>
46
+
47
+ **wvc is an experimental tool that treats a model checkpoint as a set of tensors, not one opaque file.** It gives every tensor an id (the SHA-256 of its bytes), shows exactly which tensors two checkpoints disagree on and how (elements, rows, columns, rank), and stores a new revision as deltas against its parent that check out byte for byte. It reads safetensors with numpy alone, needs no PyTorch or GPU, and can diff two Hugging Face Hub repositories while downloading only the shards that differ.
48
+
49
+ Open-weight models change through fine-tunes, merges, adapters, abliteration, and, [since November 2026](https://rikiyat.github.io/blog/weight-space-programming/), direct edits written by LLM agents. The Hub records each change as new files and a model card that names a parent with [one of four words](https://huggingface.co/docs/hub/model-cards). A card that says `finetune` doesn't say which weights moved, and safety, interpretability, and supply-chain work all need to know.
50
+
51
+ <p align="center">
52
+ <img src="docs/assets/demo.svg" width="100%" alt="A terminal runs wvc diff --rank hf:Qwen/Qwen3.5-9B hf:bowmanslayer/Qwen3.5-9B-Evil-BF16. Output: 4 shards per side, 3 byte-identical on the Hub and skipped, 1 fetched from each side; 775 tensors, 1 changed; 46,281,237 of 9,653,104,368 elements changed (0.4794%); the changed tensor is model.language_model.layers.17.mlp.down_proj.weight, BF16 [4096,12288], every row and column touched, rank 1, top-1 energy 0.9997.">
53
+ </p>
54
+ <p align="center"><sub>The command and output of a real run, repeated on 2026-11-03 with output byte-identical to its <a href="experiments/README.md">receipt</a>. The animation compresses time: from the Hugging Face cache the run takes 15 to 40 s on an M4 Max; the first run also downloads the two shards that differ (10.67 GB).</sub></p>
55
+
56
+ ## What it found
57
+
58
+ Two public weight changes and what `wvc diff` reports for them. The figure is drawn from the bytes of both checkpoints by [`make_deltas.py`](docs/assets/make_deltas.py), which first checks every count against the receipts.
59
+
60
+ <picture>
61
+ <source media="(prefers-color-scheme: dark)" srcset="docs/assets/deltas-dark.png">
62
+ <img src="docs/assets/deltas-light.png" width="100%" alt="Heat maps of |edited − base| for every changed tensor. Left: two 1024 × 3584 down-projection matrices of Qwen3.5-0.8B, in which only 64 thin columns changed. Right: one 4096 × 12288 down-projection matrix of Qwen3.5-9B, changed almost everywhere in a striped, rank-1 pattern.">
63
+ </picture>
64
+
65
+ | | An edit written by an LLM agent | An upload that names no tensor |
66
+ |---|---|---|
67
+ | **Pair** | `Qwen/Qwen3.5-0.8B` → `RikiyaT/Qwen3.5-0.8B-weight-space-programming` | `Qwen/Qwen3.5-9B` → `bowmanslayer/Qwen3.5-9B-Evil-BF16` |
68
+ | **The card says** | the two edited tensors and their hashes ([Weight-Space Programming](https://rikiyat.github.io/blog/weight-space-programming/)) | `finetune`, with no method and no tensor named |
69
+ | **wvc finds** | 2 of 488 tensors: 64 of 3,584 columns of `mlp.down_proj` in layers 15 and 19, rank 1 each | 1 of 775 tensors: layer 17's `mlp.down_proj`, 92% of its weights, rank 1 (top-1 energy 0.9997) |
70
+ | **Check** | changed counts and hashes equal the authors' own manifest | none beyond the bytes: no method was published |
71
+ | **Read** | one 1.7 GB file per side; the diff took 9.8 s on an M4 Max | 1 of 4 shards per side: 10.67 of the pair's 38.61 GB |
72
+
73
+ A diff shows where a model changed and how, not what the change does.
74
+
75
+ ## Quick start
76
+
77
+ wvc is not on PyPI yet. Install it from a clone with [uv](https://docs.astral.sh/uv/) (Python 3.11 or newer):
78
+
79
+ ```bash
80
+ git clone https://github.com/evnsnclr/wvc && cd wvc
81
+ uv sync --extra hf # the hf extra reads Hub repositories; plain `uv sync` is enough for local files
82
+ ```
83
+
84
+ Diff the two sub-kilobyte checkpoints that ship with the [spec's conformance vectors](spec/vectors/v0/README.md); nothing is downloaded:
85
+
86
+ ```bash
87
+ uv run wvc diff --rank spec/vectors/v0/a.safetensors spec/vectors/v0/b.safetensors
88
+ ```
89
+
90
+ ```text
91
+ tensors: 4 (1 changed, 1 unchanged, 1 added, 1 removed, 0 reshaped)
92
+ elements changed: 4 of 32 (12.5000%)
93
+
94
+ status tensor dtype shape changed rows cols max|d| |d|/|a| rank top1
95
+ removed h F16 [2,2,2] - - - - - - -
96
+ changed w F32 [4,6] 4/24 4 1 1 0.418 1 1.0000
97
+ added x None->I8 [3] - - - - - - -
98
+ ```
99
+
100
+ Then a real pair from the Hub (the first run downloads 1.7 GB per side into the Hugging Face cache):
101
+
102
+ ```bash
103
+ uv run wvc diff --rank hf:Qwen/Qwen3.5-0.8B hf:RikiyaT/Qwen3.5-0.8B-weight-space-programming
104
+ ```
105
+
106
+ Keep revisions in a repository and get the exact files back:
107
+
108
+ ```bash
109
+ export WVC_AUTHOR="Your Name" # commits never record an author silently
110
+ uv run wvc init demo
111
+ uv run wvc commit spec/vectors/v0/a.safetensors -m "base" --op pretrain-checkpoint --repo demo
112
+ uv run wvc commit spec/vectors/v0/b.safetensors -m "add 1.0 to column 2 of w" --op direct-edit --repo demo
113
+ uv run wvc log --repo demo
114
+ uv run wvc checkout HEAD --out restored --repo demo
115
+ ```
116
+
117
+ <details>
118
+ <summary>What that prints</summary>
119
+
120
+ Commit ids include the time and the author, so yours will differ.
121
+
122
+ ```text
123
+ initialised an empty wvc repository in /path/to/wvc/demo/.wvc
124
+ commit b44ff44ddbce97c02cad0786047c939e1d54f70c25ca5d0320d5dd16d5cc0956 (main, no parent)
125
+ tensors: 3 (0 reused by hash, 3 stored full, 0 stored as delta)
126
+ payload bytes written: 448 in 0.01 s
127
+ commit 8461fce5701f28ae37aafd431ec8d2cdfa372be949f78105139078f715f68216 (main, parent b44ff44ddbce)
128
+ tensors: 3 (1 reused by hash, 1 stored full, 1 stored as delta: 1 delta-xor-planes)
129
+ payload bytes written: 364 in 0.01 s
130
+ 8461fce5701f 2026-11-03T17:43:19Z direct-edit 3 tensors add 1.0 to column 2 of w
131
+ b44ff44ddbce 2026-11-03T17:43:19Z pretrain-checkpoint 3 tensors base
132
+ checked out 8461fce5701f into restored: 1 file, 355 bytes in 0.00 s
133
+ b.safetensors 355 bytes sha256 ok
134
+ manifest root 226839773eb209266cbb929481b4e106d483eec17361879692c0bd87789b15cc verified against the files
135
+ ```
136
+
137
+ </details>
138
+
139
+ `diff` exits 1 when the checkpoints differ, like `diff(1)`; `verify` exits 0 only when every tensor and the root match; every wvc error exits 2. Add `--json` for a machine-readable record.
140
+
141
+ ## How it works
142
+
143
+ <picture>
144
+ <source media="(prefers-color-scheme: dark)" srcset="docs/assets/how-dark.svg">
145
+ <img src="docs/assets/how-light.svg" width="100%" alt="Three panels with real values from the receipts. wvc hash: each tensor's bytes get a SHA-256, and the sorted manifest lines get a root. wvc diff: of four layer-15 tensors only mlp.down_proj has a different id, so only it is decoded: 65,260 weights in 64 of 3,584 columns, rank 1. wvc commit and checkout: the edited revision reuses 486 tensors by id and stores 2 deltas of 513 KB; checkout writes a file whose SHA-256 equals the Hub's.">
146
+ </picture>
147
+
148
+ 1. **Identity.** A tensor's id is the SHA-256 of its stored bytes; a manifest lists every tensor's name, dtype, shape, and id, and its root is a SHA-256 over those lines. Sharding, header order, and file names don't change the root ([spec §1](docs/spec-v0.md#1-tensor-identity-and-manifests-implemented)).
149
+ 2. **Diff.** Tensors with equal ids are never decoded. The rest are decoded exactly (BF16, FP16, FP8, and the rest; a dtype that can't be decoded gets a byte-level count) and compared element by element: changed count, rows and columns touched, largest change, relative norm, and an SVD rank estimate. Across two Hub repositories, a shard whose name, size, and SHA-256 agree is skipped; its tensors are reported from its header, read by range request ([spec §5.5](docs/spec-v0.md#55-hub-checkpoints-implemented-2026-11-03)).
150
+ 3. **Store.** `commit` keeps each tensor once by id and stores a changed tensor as an XOR-plane or sparse delta against its parent when that is smaller. It also records every file's header bytes and the non-tensor files a loader needs (config, tokenizer, chat template), so `checkout` rebuilds the original files and checks each one's SHA-256 before it appears ([spec §3](docs/spec-v0.md#3-repository-objects-and-deltas-implemented-2026-11-02)).
151
+
152
+ ## Storage
153
+
154
+ Bytes needed to store a revision given its parent, as a share of its raw tensor bytes (smaller is better), on four public pairs:
155
+
156
+ | Revision pair | Compress each tensor (zstd) | Byte-chunk dedup, best case | wvc delta |
157
+ |---|---|---|---|
158
+ | Qwen3.5-0.8B → weight-space edit (sparse rank-1 edit, 1.75 GB) | 78.2% | 0.66% | 0.030% |
159
+ | Qwen2.5-0.5B + rank-8 merge (synthetic adapter merge, 988 MB) | 77.8% | 3.51% | 1.67% |
160
+ | pythia-160m, step 142k → 143k (consecutive pretraining checkpoints, 649 MB) | 58.0% | 58.0% | 33.6% |
161
+ | Qwen2.5-0.5B → Instruct (dense post-training fine-tune, 988 MB) | 77.9% | 77.9% | 56.3% |
162
+
163
+ Byte-level chunk dedup, fixed or content-defined, found nothing beyond whole-tensor identity on any of four public revision pairs: a fine-tune rewrites bytes in place, and a 64 KiB chunk can't see one changed element. Numeric deltas can. On the sparse edit a delta needs 22× fewer bytes than tensor dedup alone (0.53 MB against 11.5 MB), on the synthetic rank-8 merge 2.1× fewer, and on dense updates it saves a modest 21% to 27% beyond byte-grouped compression. In a wvc repository the LLM-written edit costs 513 KB on top of a 1.245 GB base, and both revisions check out hash-identical to the Hub's files.
164
+
165
+ ## Commands
166
+
167
+ | Command | What it does |
168
+ |---|---|
169
+ | `wvc hash CKPT` | Print a manifest: every tensor's SHA-256 and a root. `--format gguf-hash` prints llama.cpp's `gguf-hash` lines instead. |
170
+ | `wvc diff A B` | Compare two checkpoints tensor by tensor. `--rank` estimates the rank of each change, `--all` lists unchanged tensors too. |
171
+ | `wvc verify CKPT MANIFEST` | Check a checkpoint against a manifest saved from `wvc hash`. |
172
+ | `wvc init`, `commit`, `log`, `show`, `checkout` | A local repository: tensors stored once by id, changes as deltas, files rebuilt byte for byte. |
173
+ | `wvc fsck` | Re-verify every payload, object, manifest, commit, and ref in a repository. |
174
+ | `wvc bisect GOOD BAD --test CMD` | Find the first commit where a behavioural test fails (`--sweep` tests every commit instead). |
175
+ | `wvc merge-adapter BASE ADAPTER` | Merge a PEFT LoRA adapter deterministically and store its targets as low-rank deltas. |
176
+ | `wvc remote`, `push`, `pull`, `clone` | Move commits between repositories: plain directories, or Hub repositories with the `hf` extra. |
177
+ | `wvc card REF` | Print model-card metadata that carries the manifest root. |
178
+
179
+ `CKPT` is a `.safetensors` file or a directory of shards; `hash`, `diff`, and `verify` also take a Hub repository as `hf:OWNER/NAME[@REVISION]`. Without `--repo`, repository commands use the nearest `.wvc/` above the current directory.
180
+
181
+ ## Status
182
+
183
+ - **Run on real models:** `hash`, `diff`, and `verify`, on local files and on the Hub; `commit`, `checkout`, and `fsck`, with byte-identical files and a checkout that loads in transformers with identical logits; `bisect` on a real fine-tuning run, where the behaviour had no single onset and bisect said so. Each run has a [receipt](experiments/README.md).
184
+ - **Unit-tested, not yet run on real models:** `merge-adapter`, remotes (directory and Hub), `card`.
185
+ - **Not built:** reading GGUF (wvc only writes `gguf-hash` lines), signing and attestation, garbage collection, a release.
186
+ - **Not planned:** hosting, a UI, git filters, lossy storage, PyTorch at runtime.
187
+
188
+ CI runs the tests, ruff, and a type check on Ubuntu, macOS, and Windows with Python 3.11 and 3.13, with warnings as errors, a coverage floor, a dependency audit, and header-parser fuzzing. Every measurement so far comes from one machine (an Apple M4 Max), safetensors files, and the model families named in the receipts. [SECURITY.md](SECURITY.md) describes how wvc handles untrusted input and lists its known limits.
189
+
190
+ ## Receipts
191
+
192
+ Each experiment has a record under [experiments/](experiments/README.md): the question, predictions written before the run, exact commands, input revisions and hashes, tool versions, raw outputs, and which predictions failed. Every number in this README comes from one of them; the index lists each run, including the negative result.
193
+
194
+ ## Contributing
195
+
196
+ wvc has one maintainer, and the most useful contributions right now are real checkpoint pairs that break it, use cases, and reviews. Read [CONTRIBUTING.md](CONTRIBUTING.md) first. Security reports go through GitHub's private vulnerability reporting; see [SECURITY.md](SECURITY.md).
197
+
198
+ The figures here are drawn from the receipts by [`docs/assets/make_figures.py`](docs/assets/make_figures.py) and, from the weights themselves, by [`docs/assets/make_deltas.py`](docs/assets/make_deltas.py).
199
+
200
+ ## Citing
201
+
202
+ If wvc is useful in your work, please cite the repository; GitHub's "Cite this repository" button reads [CITATION.cff](CITATION.cff).
203
+
204
+ ```bibtex
205
+ @software{smith2026wvc,
206
+ author = {Smith, Evan},
207
+ title = {{wvc}: version control for open model weights},
208
+ year = {2026},
209
+ url = {https://github.com/evnsnclr/wvc},
210
+ license = {Apache-2.0}
211
+ }
212
+ ```
213
+
214
+ ## Licence
215
+
216
+ Apache-2.0; see [LICENSE](LICENSE). The models named here belong to their publishers; this repository includes none of their files.