tensorfs 0.3.7__tar.gz → 0.3.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. {tensorfs-0.3.7 → tensorfs-0.3.8}/Cargo.lock +2 -2
  2. {tensorfs-0.3.7 → tensorfs-0.3.8}/Cargo.toml +1 -1
  3. {tensorfs-0.3.7 → tensorfs-0.3.8}/PKG-INFO +1 -1
  4. tensorfs-0.3.8/crates/tensorfs-core/src/derived/roots.rs +196 -0
  5. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/derived.rs +341 -17
  6. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/durability.rs +71 -0
  7. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/gc.rs +21 -4
  8. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/journal.rs +33 -0
  9. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/source.rs +52 -1
  10. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-py/src/store.rs +18 -1
  11. {tensorfs-0.3.7 → tensorfs-0.3.8}/pyproject.toml +1 -1
  12. {tensorfs-0.3.7 → tensorfs-0.3.8}/python/tensorfs/_ext.pyi +1 -0
  13. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/Cargo.toml +0 -0
  14. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/b64.rs +0 -0
  15. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/bin/tfs/ckpt.rs +0 -0
  16. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/bin/tfs/conform.rs +0 -0
  17. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/bin/tfs/fetch.rs +0 -0
  18. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/bin/tfs/ingest.rs +0 -0
  19. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/bin/tfs/main.rs +0 -0
  20. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/bin/tfs/rd.rs +0 -0
  21. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/bin/tfs/repo.rs +0 -0
  22. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/bin/tfs/snap.rs +0 -0
  23. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/bin/tfs/watch.rs +0 -0
  24. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/canon.rs +0 -0
  25. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/capability.rs +0 -0
  26. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/catalog.rs +0 -0
  27. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/cbor.rs +0 -0
  28. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/checkpoint.rs +0 -0
  29. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/corpus.rs +0 -0
  30. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/dtype.rs +0 -0
  31. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/err.rs +0 -0
  32. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/fetch.rs +0 -0
  33. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/fill.rs +0 -0
  34. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/fit.rs +0 -0
  35. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/header.rs +0 -0
  36. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/home.rs +0 -0
  37. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ids.rs +0 -0
  38. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/carrier.rs +0 -0
  39. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/convert.rs +0 -0
  40. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/fingerprint.rs +0 -0
  41. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/gguf.rs +0 -0
  42. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/mod.rs +0 -0
  43. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/preflight.rs +0 -0
  44. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/stamp.rs +0 -0
  45. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/ingest/transaction.rs +0 -0
  46. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/jcs.rs +0 -0
  47. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/lib.rs +0 -0
  48. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/limits.rs +0 -0
  49. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/machine.rs +0 -0
  50. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/manifest.rs +0 -0
  51. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/meta.rs +0 -0
  52. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/project.rs +0 -0
  53. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/providers.rs +0 -0
  54. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/read.rs +0 -0
  55. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/receipt.rs +0 -0
  56. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/reclaim.rs +0 -0
  57. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/registry.rs +0 -0
  58. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/relation.rs +0 -0
  59. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/repo_cache.rs +0 -0
  60. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/repository.rs +0 -0
  61. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/sha256.rs +0 -0
  62. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/spec.rs +0 -0
  63. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/staging.rs +0 -0
  64. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/storage.rs +0 -0
  65. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/store.rs +0 -0
  66. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/api.rs +0 -0
  67. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/http.rs +0 -0
  68. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/hub.rs +0 -0
  69. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/ledger.rs +0 -0
  70. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/mod.rs +0 -0
  71. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/policy.rs +0 -0
  72. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/pull.rs +0 -0
  73. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/push.rs +0 -0
  74. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/ranged.rs +0 -0
  75. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/transport/tests.rs +0 -0
  76. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/src/vectors.rs +0 -0
  77. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/fetch_url_progress.rs +0 -0
  78. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/fit.rs +0 -0
  79. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/fixtures/repo_apply/blobs.jsonl +0 -0
  80. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_list.jsonl +0 -0
  81. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_repository.json +0 -0
  82. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/fixtures/repo_apply/expected_rows.jsonl +0 -0
  83. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/fixtures/repo_apply/header.cbor +0 -0
  84. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/fixtures/repo_apply/manifest.json +0 -0
  85. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/fixtures/repo_apply/mutation.json +0 -0
  86. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/home.rs +0 -0
  87. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/liveness.rs +0 -0
  88. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/reclaim.rs +0 -0
  89. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/reclaim_run.rs +0 -0
  90. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/repo_cache.rs +0 -0
  91. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/repo_store.rs +0 -0
  92. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/sha256_backends.rs +0 -0
  93. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/sharded_cas.rs +0 -0
  94. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/source_plan.rs +0 -0
  95. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/source_preflight.rs +0 -0
  96. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/staging.rs +0 -0
  97. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/store_boundaries.rs +0 -0
  98. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/tensor_schema.rs +0 -0
  99. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/version.rs +0 -0
  100. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-core/tests/write_through.rs +0 -0
  101. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-py/Cargo.toml +0 -0
  102. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-py/src/docs.rs +0 -0
  103. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-py/src/errors.rs +0 -0
  104. {tensorfs-0.3.7 → tensorfs-0.3.8}/crates/tensorfs-py/src/lib.rs +0 -0
  105. {tensorfs-0.3.7 → tensorfs-0.3.8}/python/tensorfs/__init__.py +0 -0
  106. {tensorfs-0.3.7 → tensorfs-0.3.8}/python/tensorfs/errors.py +0 -0
  107. {tensorfs-0.3.7 → tensorfs-0.3.8}/python/tensorfs/py.typed +0 -0
@@ -392,7 +392,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
392
392
 
393
393
  [[package]]
394
394
  name = "tensorfs-core"
395
- version = "0.3.7"
395
+ version = "0.3.8"
396
396
  dependencies = [
397
397
  "fs2",
398
398
  "minicbor",
@@ -404,7 +404,7 @@ dependencies = [
404
404
 
405
405
  [[package]]
406
406
  name = "tensorfs-py"
407
- version = "0.3.7"
407
+ version = "0.3.8"
408
408
  dependencies = [
409
409
  "pyo3",
410
410
  "tensorfs-core",
@@ -3,5 +3,5 @@ resolver = "2"
3
3
  members = ["crates/tensorfs-core", "crates/tensorfs-py"]
4
4
 
5
5
  [workspace.package]
6
- version = "0.3.7"
6
+ version = "0.3.8"
7
7
  edition = "2021"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tensorfs
3
- Version: 0.3.7
3
+ Version: 0.3.8
4
4
  Classifier: Programming Language :: Python :: 3
5
5
  Classifier: Programming Language :: Python :: Implementation :: CPython
6
6
  Classifier: Programming Language :: Rust
@@ -0,0 +1,196 @@
1
+ //! Private derived roots are filesystem facts, just like repository roots. Transaction
2
+ //! metadata journals lifecycle; GC never needs that database to discover retained bytes.
3
+
4
+ use std::fs::{self, File, OpenOptions};
5
+ use std::io::{Read, Write};
6
+ use std::path::PathBuf;
7
+
8
+ use crate::canon::{Fields, Value};
9
+ use crate::err::{refuse, Code, Refusal, Result};
10
+ use crate::ids::{ObjectRef, Plain};
11
+ use crate::store::Store;
12
+
13
+ #[derive(Clone, Debug, PartialEq, Eq)]
14
+ pub(super) struct Root {
15
+ pub transaction: String,
16
+ pub manifest: ObjectRef,
17
+ pub name: Option<String>,
18
+ }
19
+
20
+ impl Plain for Root {
21
+ const MAX_BYTES: usize = 16 * 1024;
22
+ fn from_value(value: &Value) -> Result<Self> {
23
+ let mut fields = Fields::new("PrivateDerivedRoot", value)?;
24
+ let transaction = super::transaction_id(fields.req_str("transaction_id")?)?;
25
+ let manifest = ObjectRef::from_value("private root manifest", fields.req("manifest")?)?;
26
+ let name = fields
27
+ .opt("scratch_root_id")
28
+ .map(|value| {
29
+ let name = crate::canon::as_str("PrivateDerivedRoot", "scratch_root_id", value)?;
30
+ super::name("private scratch root", name)?;
31
+ Ok(name.to_string())
32
+ })
33
+ .transpose()?;
34
+ fields.done()?;
35
+ Ok(Self {
36
+ transaction,
37
+ manifest,
38
+ name,
39
+ })
40
+ }
41
+ fn to_value(&self) -> Value {
42
+ let mut fields = vec![
43
+ ("transaction_id", Value::str(self.transaction.clone())),
44
+ ("manifest", self.manifest.to_value()),
45
+ ];
46
+ if let Some(name) = &self.name {
47
+ fields.push(("scratch_root_id", Value::str(name.clone())));
48
+ }
49
+ Value::obj(fields)
50
+ }
51
+ }
52
+
53
+ fn io(error: std::io::Error) -> Refusal {
54
+ Refusal {
55
+ code: Code::IO_FAILED,
56
+ detail: format!("private derived root: {error}"),
57
+ }
58
+ }
59
+ fn directory(store: &Store) -> PathBuf {
60
+ store.root().join("roots/derived")
61
+ }
62
+ fn path(store: &Store, transaction: &str) -> Result<PathBuf> {
63
+ let transaction = super::transaction_id(transaction)?;
64
+ Ok(directory(store).join(format!(
65
+ "{}.json",
66
+ transaction.trim_start_matches("sha256:")
67
+ )))
68
+ }
69
+
70
+ pub(super) fn read(store: &Store, transaction: &str) -> Result<Option<Root>> {
71
+ let path = path(store, transaction)?;
72
+ let file = match File::open(&path) {
73
+ Ok(file) => file,
74
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
75
+ Err(error) => return Err(io(error)),
76
+ };
77
+ let mut bytes = Vec::new();
78
+ file.take(Root::MAX_BYTES as u64 + 1)
79
+ .read_to_end(&mut bytes)
80
+ .map_err(io)?;
81
+ let root = Root::parse(&bytes)?;
82
+ if root.transaction != transaction {
83
+ return refuse(
84
+ Code::CROSS_SUBJECT_REPLAY,
85
+ "private root path names another transaction",
86
+ );
87
+ }
88
+ Ok(Some(root))
89
+ }
90
+
91
+ pub(super) fn all(store: &Store) -> Result<Vec<Root>> {
92
+ let entries = match fs::read_dir(directory(store)) {
93
+ Ok(entries) => entries,
94
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
95
+ Err(error) => return Err(io(error)),
96
+ };
97
+ let mut roots = Vec::new();
98
+ for entry in entries {
99
+ let entry = entry.map_err(io)?;
100
+ let name = entry.file_name();
101
+ let name = name.to_string_lossy();
102
+ let Some(hex) = name.strip_suffix(".json") else {
103
+ continue;
104
+ };
105
+ let transaction = format!("sha256:{hex}");
106
+ if let Some(root) = read(store, &transaction)? {
107
+ roots.push(root);
108
+ }
109
+ }
110
+ Ok(roots)
111
+ }
112
+
113
+ pub(super) fn write(store: &Store, root: &Root) -> Result<()> {
114
+ let path = path(store, &root.transaction)?;
115
+ fs::create_dir_all(directory(store)).map_err(io)?;
116
+ let temporary = path.with_extension(format!("tmp-{}", crate::meta::now_nanos_unique()));
117
+ let mut file = OpenOptions::new()
118
+ .write(true)
119
+ .create_new(true)
120
+ .open(&temporary)
121
+ .map_err(io)?;
122
+ file.write_all(&root.canonical_bytes())
123
+ .and_then(|()| file.sync_all())
124
+ .map_err(io)?;
125
+ fs::rename(&temporary, &path).map_err(io)?;
126
+ for directory in [
127
+ directory(store),
128
+ store.root().join("roots"),
129
+ store.root().to_path_buf(),
130
+ ] {
131
+ File::open(directory)
132
+ .and_then(|file| file.sync_all())
133
+ .map_err(io)?;
134
+ }
135
+ Ok(())
136
+ }
137
+
138
+ pub(super) fn pending(store: &Store, transaction: &str, manifest: &ObjectRef) -> Result<()> {
139
+ if read(store, transaction)?.is_some_and(|root| root.name.is_some()) {
140
+ return refuse(
141
+ Code::DISPOSITION_CONFLICT,
142
+ "a named private root cannot become a pending candidate",
143
+ );
144
+ }
145
+ write(
146
+ store,
147
+ &Root {
148
+ transaction: transaction.into(),
149
+ manifest: manifest.clone(),
150
+ name: None,
151
+ },
152
+ )
153
+ }
154
+
155
+ pub(super) fn adopt(
156
+ store: &Store,
157
+ transaction: &str,
158
+ manifest: &ObjectRef,
159
+ name: &str,
160
+ ) -> Result<()> {
161
+ if let Some(root) = read(store, transaction)? {
162
+ if root.manifest != *manifest || root.name.as_ref().is_some_and(|old| old != name) {
163
+ return refuse(
164
+ Code::DISPOSITION_CONFLICT,
165
+ "private root already binds different output or name",
166
+ );
167
+ }
168
+ }
169
+ if all(store)?
170
+ .iter()
171
+ .any(|root| root.transaction != transaction && root.name.as_deref() == Some(name))
172
+ {
173
+ return refuse(
174
+ Code::DISPOSITION_CONFLICT,
175
+ "private scratch root already names another transaction",
176
+ );
177
+ }
178
+ write(
179
+ store,
180
+ &Root {
181
+ transaction: transaction.into(),
182
+ manifest: manifest.clone(),
183
+ name: Some(name.into()),
184
+ },
185
+ )
186
+ }
187
+
188
+ pub(super) fn remove(store: &Store, transaction: &str) -> Result<()> {
189
+ match fs::remove_file(path(store, transaction)?) {
190
+ Ok(()) => File::open(directory(store))
191
+ .and_then(|file| file.sync_all())
192
+ .map_err(io),
193
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
194
+ Err(error) => Err(io(error)),
195
+ }
196
+ }
@@ -6,6 +6,8 @@
6
6
  //! additions through the ordinary object writer. There is no filename inference, model
7
7
  //! vocabulary, Torch, schedule, publication, or serving decision here.
8
8
 
9
+ mod roots;
10
+
9
11
  use std::collections::HashSet;
10
12
  use std::io::Read;
11
13
 
@@ -820,9 +822,23 @@ impl TransactionState {
820
822
  #[derive(Debug, Clone, PartialEq, Eq)]
821
823
  pub(crate) enum Disposition {
822
824
  Pending,
825
+ Adopted(String),
823
826
  Released,
824
827
  }
825
828
 
829
+ impl Disposition {
830
+ fn to_value(&self) -> Value {
831
+ match self {
832
+ Self::Pending => Value::obj(vec![("kind", Value::str("pending"))]),
833
+ Self::Released => Value::obj(vec![("kind", Value::str("released"))]),
834
+ Self::Adopted(root) => Value::obj(vec![
835
+ ("kind", Value::str("adopted")),
836
+ ("scratch_root_id", Value::str(root.clone())),
837
+ ]),
838
+ }
839
+ }
840
+ }
841
+
826
842
  #[derive(Debug, Clone, PartialEq, Eq)]
827
843
  pub(crate) struct TransactionRow {
828
844
  pub(crate) id: String,
@@ -998,13 +1014,18 @@ impl TransactionRow {
998
1014
  None => None,
999
1015
  Some(value) => {
1000
1016
  let mut df = Fields::new("DerivedTransaction.disposition", value)?;
1001
- let kind = df.req_str("kind")?;
1002
- df.done()?;
1003
- Some(match kind {
1017
+ let disposition = match df.req_str("kind")? {
1004
1018
  "pending" => Disposition::Pending,
1005
1019
  "released" => Disposition::Released,
1020
+ "adopted" => {
1021
+ let root = df.req_str("scratch_root_id")?;
1022
+ name("private scratch root", root)?;
1023
+ Disposition::Adopted(root.into())
1024
+ }
1006
1025
  _ => return refuse(Code::UNKNOWN_FIELD, "invalid derived root disposition"),
1007
- })
1026
+ };
1027
+ df.done()?;
1028
+ Some(disposition)
1008
1029
  }
1009
1030
  };
1010
1031
  f.done()?;
@@ -1249,10 +1270,7 @@ impl TransactionRow {
1249
1270
  fields.push(("manifest".to_string(), manifest.to_value()));
1250
1271
  }
1251
1272
  if let TransactionState::Committed(disposition) = &self.state {
1252
- let value = match disposition {
1253
- Disposition::Pending => Value::obj(vec![("kind", Value::str("pending"))]),
1254
- Disposition::Released => Value::obj(vec![("kind", Value::str("released"))]),
1255
- };
1273
+ let value = disposition.to_value();
1256
1274
  fields.push(("disposition".to_string(), value));
1257
1275
  }
1258
1276
  Value::map(fields)
@@ -2343,6 +2361,7 @@ pub fn commit(
2343
2361
  "the addition set changed while commit constructed the target; retry commit",
2344
2362
  );
2345
2363
  }
2364
+ roots::pending(store, &transaction, &manifest_ref)?;
2346
2365
  row.state = TransactionState::Committed(Disposition::Pending);
2347
2366
  row.current_session = None;
2348
2367
  row.header = Some(header_ref);
@@ -2358,7 +2377,7 @@ pub fn commit(
2358
2377
  pub enum Lookup {
2359
2378
  Absent,
2360
2379
  Open { writer_session: Option<u64> },
2361
- Committed(Box<ReceiptFacts>),
2380
+ Committed(Box<DispositionResult>),
2362
2381
  Abandoned,
2363
2382
  }
2364
2383
 
@@ -2372,8 +2391,12 @@ pub fn lookup(store: &Store, meta: &Meta, transaction: &str) -> Result<Lookup> {
2372
2391
  TransactionState::Open => Ok(Lookup::Open {
2373
2392
  writer_session: row.current_session,
2374
2393
  }),
2375
- TransactionState::Committed(_) => {
2376
- Ok(Lookup::Committed(Box::new(receipt_from(store, row)?)))
2394
+ TransactionState::Committed(ref recorded) => {
2395
+ let disposition = effective_disposition(store, row, recorded)?;
2396
+ Ok(Lookup::Committed(Box::new(DispositionResult {
2397
+ receipt: receipt_from(store, row)?,
2398
+ disposition,
2399
+ })))
2377
2400
  }
2378
2401
  TransactionState::Abandoned => Ok(Lookup::Abandoned),
2379
2402
  }
@@ -2406,7 +2429,7 @@ pub fn abandon(store: &Store, meta: &Meta, transaction: &str) -> Result<Option<R
2406
2429
  let result = match txn.rows.iter_mut().find(|row| row.id == transaction) {
2407
2430
  None => {
2408
2431
  txn.rows.push(TransactionRow {
2409
- id: transaction,
2432
+ id: transaction.clone(),
2410
2433
  declaration: None,
2411
2434
  state: TransactionState::Abandoned,
2412
2435
  highest_session: 0,
@@ -2446,6 +2469,7 @@ pub fn abandon(store: &Store, meta: &Meta, transaction: &str) -> Result<Option<R
2446
2469
  return Ok(Some(receipt_from(store, &row)?));
2447
2470
  }
2448
2471
  txn.commit()?;
2472
+ roots::remove(store, &transaction)?;
2449
2473
  Ok(None)
2450
2474
  }
2451
2475
 
@@ -2457,21 +2481,129 @@ pub struct DispositionResult {
2457
2481
 
2458
2482
  impl DispositionResult {
2459
2483
  pub fn to_value(&self) -> Value {
2460
- let disposition = match &self.disposition {
2461
- Disposition::Pending => Value::obj(vec![("kind", Value::str("pending"))]),
2462
- Disposition::Released => Value::obj(vec![("kind", Value::str("released"))]),
2463
- };
2484
+ let disposition = self.disposition.to_value();
2464
2485
  Value::obj(vec![
2465
2486
  ("disposition", disposition),
2466
2487
  ("receipt", self.receipt.to_value()),
2467
2488
  ])
2468
2489
  }
2469
2490
 
2491
+ pub fn disposition_value(&self) -> Value {
2492
+ self.disposition.to_value()
2493
+ }
2494
+
2470
2495
  pub fn receipt(&self) -> &ReceiptFacts {
2471
2496
  &self.receipt
2472
2497
  }
2473
2498
  }
2474
2499
 
2500
+ /// Retain a committed result under an opaque private root. The name is supplied by the
2501
+ /// RecordOwner; TensorFS neither derives that name nor fabricates a model repository.
2502
+ pub fn adopt(
2503
+ store: &Store,
2504
+ meta: &Meta,
2505
+ transaction: &str,
2506
+ scratch_root_id: &str,
2507
+ ) -> Result<DispositionResult> {
2508
+ let transaction = transaction_id(transaction)?;
2509
+ name("private scratch root", scratch_root_id)?;
2510
+ let mut txn = meta.txn()?;
2511
+ let index = txn
2512
+ .rows
2513
+ .iter()
2514
+ .position(|row| row.id == transaction)
2515
+ .ok_or_else(|| Refusal {
2516
+ code: Code::ROOT_ABSENT,
2517
+ detail: format!("no derived transaction {transaction}"),
2518
+ })?;
2519
+ let desired = Disposition::Adopted(scratch_root_id.into());
2520
+ match &txn.rows[index].state {
2521
+ TransactionState::Committed(Disposition::Pending) => {}
2522
+ TransactionState::Committed(existing) if existing == &desired => {
2523
+ let row = txn.rows[index].clone();
2524
+ roots::adopt(
2525
+ store,
2526
+ &transaction,
2527
+ row.manifest.as_ref().expect("committed manifest"),
2528
+ scratch_root_id,
2529
+ )?;
2530
+ drop(txn);
2531
+ return Ok(DispositionResult {
2532
+ receipt: receipt_from(store, &row)?,
2533
+ disposition: desired,
2534
+ });
2535
+ }
2536
+ TransactionState::Committed(_) => {
2537
+ return refuse(
2538
+ Code::DISPOSITION_CONFLICT,
2539
+ "derived result was adopted under another name or already released",
2540
+ )
2541
+ }
2542
+ _ => {
2543
+ return refuse(
2544
+ Code::TRANSACTION_CLOSED,
2545
+ "only a committed result can be adopted",
2546
+ )
2547
+ }
2548
+ }
2549
+ if txn.rows.iter().any(|row| {
2550
+ row.id != transaction && row.state == TransactionState::Committed(desired.clone())
2551
+ }) {
2552
+ return refuse(
2553
+ Code::DISPOSITION_CONFLICT,
2554
+ "private scratch root already names another transaction",
2555
+ );
2556
+ }
2557
+ roots::adopt(
2558
+ store,
2559
+ &transaction,
2560
+ txn.rows[index].manifest.as_ref().ok_or_else(|| Refusal {
2561
+ code: Code::ARTIFACT_INCOMPLETE,
2562
+ detail: "committed transaction has no manifest".into(),
2563
+ })?,
2564
+ scratch_root_id,
2565
+ )?;
2566
+ txn.rows[index].state = TransactionState::Committed(desired.clone());
2567
+ let row = txn.rows[index].clone();
2568
+ txn.commit()?;
2569
+ Ok(DispositionResult {
2570
+ receipt: receipt_from(store, &row)?,
2571
+ disposition: desired,
2572
+ })
2573
+ }
2574
+
2575
+ /// Committed output is retained while awaiting the owner and after private adoption.
2576
+ /// Once disposed, ordinary repository roots alone decide whether its bytes remain live.
2577
+ pub(crate) fn retained_manifests(store: &Store) -> Result<Vec<ObjectRef>> {
2578
+ Ok(roots::all(store)?
2579
+ .into_iter()
2580
+ .map(|root| root.manifest)
2581
+ .collect())
2582
+ }
2583
+
2584
+ fn effective_disposition(
2585
+ store: &Store,
2586
+ row: &TransactionRow,
2587
+ recorded: &Disposition,
2588
+ ) -> Result<Disposition> {
2589
+ if recorded == &Disposition::Released {
2590
+ return Ok(Disposition::Released);
2591
+ }
2592
+ match roots::read(store, &row.id)? {
2593
+ Some(root) => {
2594
+ if row.manifest.as_ref() != Some(&root.manifest) {
2595
+ return refuse(
2596
+ Code::DISPOSITION_CONFLICT,
2597
+ "private root differs from its committed receipt",
2598
+ );
2599
+ }
2600
+ Ok(root.name.map_or(Disposition::Pending, Disposition::Adopted))
2601
+ }
2602
+ None if recorded == &Disposition::Pending => Ok(Disposition::Pending),
2603
+ None => refuse(Code::DURABILITY_UNPROVEN, "adopted private root is absent"),
2604
+ }
2605
+ }
2606
+
2475
2607
  pub fn dispose(store: &Store, meta: &Meta, transaction: &str) -> Result<DispositionResult> {
2476
2608
  let transaction = transaction_id(transaction)?;
2477
2609
  let mut txn = meta.txn()?;
@@ -2489,10 +2621,11 @@ pub fn dispose(store: &Store, meta: &Meta, transaction: &str) -> Result<Disposit
2489
2621
  })?;
2490
2622
  let desired = Disposition::Released;
2491
2623
  match &txn.rows[index].state {
2492
- TransactionState::Committed(Disposition::Pending) => {}
2624
+ TransactionState::Committed(Disposition::Pending | Disposition::Adopted(_)) => {}
2493
2625
  TransactionState::Committed(existing) if existing == &desired => {
2494
2626
  let row = txn.rows[index].clone();
2495
2627
  drop(txn);
2628
+ roots::remove(store, &transaction)?;
2496
2629
  return Ok(DispositionResult {
2497
2630
  receipt: receipt_from(store, &row)?,
2498
2631
  disposition: desired,
@@ -2515,6 +2648,7 @@ pub fn dispose(store: &Store, meta: &Meta, transaction: &str) -> Result<Disposit
2515
2648
  txn.rows[index].state = TransactionState::Committed(desired.clone());
2516
2649
  let row = txn.rows[index].clone();
2517
2650
  txn.commit()?;
2651
+ roots::remove(store, &transaction)?;
2518
2652
  Ok(DispositionResult {
2519
2653
  receipt: receipt_from(store, &row)?,
2520
2654
  disposition: desired,
@@ -2526,6 +2660,196 @@ mod tests {
2526
2660
  use super::*;
2527
2661
  use crate::header::{Closure, Part, Tensor};
2528
2662
  use crate::ids::Doc;
2663
+
2664
+ fn created_declaration() -> Declaration {
2665
+ let encoding = registry::seeds()
2666
+ .into_iter()
2667
+ .find(|seed| seed.alias == "plain/1")
2668
+ .unwrap()
2669
+ .spec
2670
+ .object_id();
2671
+ Declaration {
2672
+ sources: Vec::new(),
2673
+ components: vec![ComponentDeclaration {
2674
+ target: "model".into(),
2675
+ source: None,
2676
+ source_component: None,
2677
+ drop: Vec::new(),
2678
+ add: vec![TensorDeclaration {
2679
+ key: "weight".into(),
2680
+ dtype: Dtype::F32,
2681
+ shape: vec![512],
2682
+ encoding,
2683
+ parts: vec![PartDeclaration {
2684
+ role: "value".into(),
2685
+ dtype: Dtype::F32,
2686
+ shape: vec![512],
2687
+ }],
2688
+ }],
2689
+ }],
2690
+ configs: Vec::new(),
2691
+ order: vec![("model".into(), "weight".into())],
2692
+ max_new_bytes: 2048,
2693
+ }
2694
+ }
2695
+
2696
+ #[test]
2697
+ fn private_adoption_retains_real_payload_until_dispose_without_a_repository() {
2698
+ let root = std::env::temp_dir().join(format!(
2699
+ "tensorfs-private-adopt-{}",
2700
+ crate::meta::now_nanos_unique()
2701
+ ));
2702
+ let store = Store::init(&root).unwrap();
2703
+ let meta = Meta::open(&store).unwrap();
2704
+ let id = format!("sha256:{}", "55".repeat(32));
2705
+ let writer = begin(&store, &meta, &id, 1, created_declaration()).unwrap();
2706
+ let bytes = vec![0x11; 2048];
2707
+ let written = add_part(
2708
+ &store,
2709
+ &meta,
2710
+ &id,
2711
+ 1,
2712
+ ("model", "weight", "value"),
2713
+ &mut bytes.as_slice(),
2714
+ )
2715
+ .unwrap();
2716
+ let receipt = commit(&store, &meta, &id, 1).unwrap();
2717
+ writer.writer_hold.release(&meta).unwrap();
2718
+ let payload = &written.part.segments()[0];
2719
+
2720
+ // The owner has not answered yet: even this pending result is rooted after its
2721
+ // writer is gone, closing the commit-to-finalize GC window.
2722
+ crate::gc::collect(&root, false).unwrap();
2723
+ assert!(store.contains(&payload.sha256));
2724
+ store.read_manifest(&receipt.manifest).unwrap();
2725
+ // Crash after the private name reached disk but before its SQL lifecycle update.
2726
+ roots::adopt(&store, &id, &receipt.manifest, "private-root").unwrap();
2727
+ let Lookup::Committed(observed) = lookup(&store, &meta, &id).unwrap() else {
2728
+ panic!("committed output missing")
2729
+ };
2730
+ assert_eq!(
2731
+ observed.disposition_value(),
2732
+ Disposition::Adopted("private-root".into()).to_value()
2733
+ );
2734
+ assert_eq!(
2735
+ adopt(&store, &meta, &id, "different-root")
2736
+ .unwrap_err()
2737
+ .code,
2738
+ Code::DISPOSITION_CONFLICT
2739
+ );
2740
+ let adopted = adopt(&store, &meta, &id, "private-root").unwrap();
2741
+ assert_eq!(adopted.receipt().to_value(), receipt.to_value());
2742
+ let reopened = Meta::open(&store).unwrap();
2743
+ assert_eq!(
2744
+ adopt(&store, &reopened, &id, "private-root")
2745
+ .unwrap()
2746
+ .to_value(),
2747
+ adopted.to_value()
2748
+ );
2749
+ assert_eq!(
2750
+ adopt(&store, &reopened, &id, "different-root")
2751
+ .unwrap_err()
2752
+ .code,
2753
+ Code::DISPOSITION_CONFLICT
2754
+ );
2755
+ crate::gc::collect(&root, false).unwrap();
2756
+ assert!(store.contains(&payload.sha256));
2757
+ store.read_manifest(&receipt.manifest).unwrap();
2758
+ // Liveness is discoverable even when the transaction journal cannot be decoded.
2759
+ // A GC hook that consults SQLite rather than private root files fails this arm.
2760
+ let database = crate::catalog::Catalog::path(&root);
2761
+ let connection = rusqlite::Connection::open(&database).unwrap();
2762
+ let saved: Vec<u8> = connection
2763
+ .query_row(
2764
+ "SELECT bytes FROM tensorfs_derived_transactions WHERE id=?1",
2765
+ [&id],
2766
+ |row| row.get(0),
2767
+ )
2768
+ .unwrap();
2769
+ connection
2770
+ .execute(
2771
+ "UPDATE tensorfs_derived_transactions SET bytes=?1 WHERE id=?2",
2772
+ rusqlite::params![b"not a transaction".as_slice(), &id],
2773
+ )
2774
+ .unwrap();
2775
+ drop(connection);
2776
+ crate::gc::collect(&root, false).unwrap();
2777
+ assert!(store.contains(&payload.sha256));
2778
+ let connection = rusqlite::Connection::open(&database).unwrap();
2779
+ connection
2780
+ .execute(
2781
+ "UPDATE tensorfs_derived_transactions SET bytes=?1 WHERE id=?2",
2782
+ rusqlite::params![saved, &id],
2783
+ )
2784
+ .unwrap();
2785
+ drop(connection);
2786
+ assert!(std::fs::read_dir(root.join("repos"))
2787
+ .map(|rows| rows.count() == 0)
2788
+ .unwrap_or(true));
2789
+
2790
+ let saved_root = roots::read(&store, &id).unwrap().unwrap();
2791
+ let released = dispose(&store, &reopened, &id).unwrap();
2792
+ assert_eq!(released.receipt().to_value(), receipt.to_value());
2793
+ // Crash after the terminal journal update but before the root was removed.
2794
+ roots::write(&store, &saved_root).unwrap();
2795
+ let Lookup::Committed(observed) = lookup(&store, &reopened, &id).unwrap() else {
2796
+ panic!("committed receipt missing")
2797
+ };
2798
+ assert_eq!(
2799
+ observed.disposition_value(),
2800
+ Disposition::Released.to_value()
2801
+ );
2802
+ assert_eq!(
2803
+ dispose(&store, &reopened, &id).unwrap().to_value(),
2804
+ released.to_value()
2805
+ );
2806
+ assert!(roots::read(&store, &id).unwrap().is_none());
2807
+ assert_eq!(
2808
+ adopt(&store, &reopened, &id, "private-root")
2809
+ .unwrap_err()
2810
+ .code,
2811
+ Code::DISPOSITION_CONFLICT
2812
+ );
2813
+ crate::gc::collect(&root, false).unwrap();
2814
+ assert!(!store.contains(&payload.sha256));
2815
+ assert!(!store.manifest_path(&receipt.manifest.sha256).exists());
2816
+ assert_eq!(
2817
+ dispose(&store, &reopened, &id).unwrap().to_value(),
2818
+ released.to_value()
2819
+ );
2820
+ let _ = std::fs::remove_dir_all(root);
2821
+ }
2822
+
2823
+ #[test]
2824
+ fn absent_abandonment_is_a_durable_fence_against_a_late_begin() {
2825
+ let root = std::env::temp_dir().join(format!(
2826
+ "tensorfs-absent-abandon-{}",
2827
+ crate::meta::now_nanos_unique()
2828
+ ));
2829
+ let store = Store::init(&root).unwrap();
2830
+ let meta = Meta::open(&store).unwrap();
2831
+ let id = format!("sha256:{}", "66".repeat(32));
2832
+ assert!(abandon(&store, &meta, &id).unwrap().is_none());
2833
+ let reopened = Meta::open(&store).unwrap();
2834
+ assert!(matches!(
2835
+ lookup(&store, &reopened, &id).unwrap(),
2836
+ Lookup::Abandoned
2837
+ ));
2838
+ assert_eq!(
2839
+ begin(&store, &reopened, &id, 1, created_declaration())
2840
+ .err()
2841
+ .unwrap()
2842
+ .code,
2843
+ Code::TRANSACTION_CLOSED
2844
+ );
2845
+ assert_eq!(
2846
+ adopt(&store, &reopened, &id, "private-root")
2847
+ .unwrap_err()
2848
+ .code,
2849
+ Code::TRANSACTION_CLOSED
2850
+ );
2851
+ let _ = std::fs::remove_dir_all(root);
2852
+ }
2529
2853
  use crate::manifest::{Draft, Entry};
2530
2854
  use crate::repository::{Mutation, ReleaseLane, RepositoryName};
2531
2855
  use crate::store::Fault;