tensorfs 0.3.59__tar.gz → 0.3.61__tar.gz

{tensorfs-0.3.59 → tensorfs-0.3.61}/Cargo.lock

@@ -439,7 +439,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
 
 [[package]]
 name = "tensorfs-core"
-version = "0.3.59"
+version = "0.3.61"
 dependencies = [
  "fs2",
  "libc",
@@ -454,7 +454,7 @@ dependencies = [
 
 [[package]]
 name = "tensorfs-py"
-version = "0.3.59"
+version = "0.3.61"
 dependencies = [
  "pyo3",
  "tensorfs-core",

{tensorfs-0.3.59 → tensorfs-0.3.61}/Cargo.toml

@@ -3,5 +3,5 @@ resolver = "2"
 members = ["crates/tensorfs-core", "crates/tensorfs-py"]
 
 [workspace.package]
-version = "0.3.59"
+version = "0.3.61"
 edition = "2021"

{tensorfs-0.3.59 → tensorfs-0.3.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tensorfs
-Version: 0.3.59
+Version: 0.3.61
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust

{tensorfs-0.3.59 → tensorfs-0.3.61}/crates/tensorfs-core/src/bin/tfs/fetch.rs

@@ -273,7 +273,8 @@ fn pull_with_budget(
     let mut request = PullRequest::new(&store, base, refspec, &credential, &policy);
     request.lane = flag(flags, "lane").unwrap_or("");
     request.session = flag(flags, "session").unwrap_or("");
-    request.streams = crate::flag_num(flags, "streams", transport::STREAMS);
+    request.streams = crate::flag_num(flags, "streams", transport::PULL_STREAMS);
+    request.streams_start = crate::flag_num(flags, "streams-start", transport::STREAMS);
     request.deadline = deadline_of(flags);
     request.download_budget = budget;
     // THE WALK REPORTS WHILE IT WALKS.
@@ -300,7 +301,11 @@ fn pull_with_budget(
     let observe = |_object: &ObjectRef, moved: u64, door: ObjectSource| progress.saw(moved, door);
     request.on_plan = Some(&planned);
     request.on_object = Some(&observe);
+    let before = tensorfs_core::stats::snapshot();
+    let started = std::time::Instant::now();
     let report = ok!(transport::pull(&request));
+    let seconds = started.elapsed().as_secs_f64();
+    let after = tensorfs_core::stats::snapshot();
     progress.settle();
     if !tensorfs_core::repo_cache::finish_optional_backfills() {
         eprintln!("repo cache replication incomplete; local fetch remains complete");
@@ -313,6 +318,32 @@ fn pull_with_budget(
         quoted(&report.model), quoted(&report.manifest), quoted(&report.scope),
         report.cache_writes.bytes(), report.bytes_cached, report.bytes_total,
     );
+    // What each stream level bought, so a slow pull says whether more streams helped
+    // (one GET was the limit) or not (the link, disk or origin was).
+    let levels: Vec<String> = report
+        .levels
+        .iter()
+        .map(|level| {
+            format!(
+                "[{},{}]",
+                level.streams,
+                level.bytes_per_second.round() as u64
+            )
+        })
+        .collect();
+    // And what admitting them cost: a pull whose streams stop growing while its batches
+    // take most of the wall time is held by the disk, not the link.
+    eprintln!(
+        "{{\"event\":\"pull.streams\",\"model\":{},\"seconds\":{:.3},\"bytes_moved\":{},\"streams\":{},\"levels\":[{}],\"admission\":{{\"objects\":{},\"batches\":{},\"seconds\":{:.3}}}}}",
+        quoted(&report.model),
+        seconds,
+        report.bytes_moved,
+        report.streams,
+        levels.join(","),
+        after.admitted_objects - before.admitted_objects,
+        after.admission_batches - before.admission_batches,
+        (after.admission_nanos - before.admission_nanos) as f64 / 1e9,
+    );
     println!(
         "model        {}@{} lane {}",
         report.model, report.release, report.lane

{tensorfs-0.3.59 → tensorfs-0.3.61}/crates/tensorfs-core/src/bin/tfs/main.rs

@@ -42,7 +42,9 @@
 //! The ONE transport (tfs-048/tfs-049/tfs-050). The credential is `--credential-file` or
 //! TFS_CREDENTIAL (`bearer <token>`, or empty for anonymous), never bare argv; a
 //! URL of `-` arrives on stdin. Flags: --hub <base> --lane --session --streams <n>
-//! --timeout <s> --allow-hosts a,b,.c --allow-local --max-redirects <n>.
+//! --streams-start <n> --timeout <s> --allow-hosts a,b,.c --allow-local --max-redirects <n>.
+//! `--streams` is the most objects in flight (default 128, memory-bounded); the walk starts
+//! at `--streams-start` (default 16) and doubles while measured goodput follows.
 //! `tfs fetch` reads and backfills the Store's bound repo cache between the local Store
 //! and the origin. It is bound by `tfs store ensure --repo-cache` and never by a flag here.
 //!
@@ -505,7 +507,7 @@ fn cmd_repo_cache_restore(store: &Path, object: &str, length: &str, flags: &Flag
 /// The chain head names links that live on the cache; each is read at its exact digest, and
 /// the chain is DISCARDED WHOLE unless every link was written under the plan `--plan` names
 /// — the plan a resuming run re-derives from the source headers alone, for zero tensor
-/// bytes. That is what makes a stale chain harmless rather than dangerous.
+/// bytes. A discarded chain restores nothing and succeeds: the run converts afresh.
 ///
 /// The objects come back exactly as a checkpoint's closure does. The conversion journal the
 /// newest link points at is installed beside them under `--session`, so the next
@@ -548,7 +550,15 @@ fn cmd_repo_cache_resume(store: &Path, object: &str, length: &str, flags: &Flags
     };
     let set = match durability::fold(&chain, plan) {
         Ok(set) => set,
-        Err(error) => return bail(error),
+        Err(error) if error.code == tensorfs_core::err::Code::IO_FAILED => return bail(error),
+        Err(error) => {
+            println!(
+                "chain        DISCARDED {}: {} — nothing restored; the conversion starts a new chain",
+                error.code.as_str(),
+                error.detail
+            );
+            return ExitCode::SUCCESS;
+        }
     };
     println!(
         "chain        {} link(s) over operation {} — {} B declared durable",

{tensorfs-0.3.59 → tensorfs-0.3.61}/crates/tensorfs-core/src/catalog.rs

@@ -903,8 +903,9 @@ impl Catalog {
         hold_on(&self.connection()?, operation, key, length)
     }
 
-    /// The object's existing hold and verification become durable together. Filesystem
-    /// publication still precedes its verification record, with FULL synchronization.
+    /// A HELD admission: the hold is written before the object is published, so an
+    /// operation that cannot hold it publishes nothing. Unheld admissions, which is every
+    /// pull, take `record_grouped` instead.
     pub(crate) fn admit_blob(
         &self,
         operation: Option<&str>,
@@ -914,7 +915,7 @@ impl Catalog {
         // One admission at a time per catalog in this process, holding the write lock from
         // before publication: streams landing objects together queue here instead of in
         // SQLite's busy timeout, where a lost race refused bytes already verified.
-        let turn = admissions(&self.path);
+        let turn = held_admissions(&self.path);
         let _turn = turn.lock().unwrap_or_else(|error| error.into_inner());
         let mut connection = self.connection()?;
         let transaction = connection
@@ -941,6 +942,117 @@ impl Catalog {
         Ok(verified.is_some())
     }
 
+    /// The object's existing hold and verification become durable together. Filesystem
+    /// publication has already happened, on the caller's thread, and precedes this record,
+    /// which is written with FULL synchronization.
+    ///
+    /// Records queue per catalog in this process, and whichever arrives while no batch is
+    /// committing leads the next one: one immediate transaction and one commit make every
+    /// waiting record durable. One object alone costs what it always did. Before this,
+    /// directory creation, link, directory fsync and commit ran for one object at a time
+    /// under one lock, several fsyncs each, and on a disk busy flushing those same objects'
+    /// bodies that queue, not the network, set a pull's rate.
+    pub(crate) fn record_grouped(&self, recording: Recording) -> Result<()> {
+        let group = admission_group(&self.path);
+        let mut state = group
+            .state
+            .lock()
+            .unwrap_or_else(|error| error.into_inner());
+        let ticket = state.next;
+        state.next += 1;
+        state.queue.push((ticket, recording));
+        loop {
+            if let Some(result) = state.done.remove(&ticket) {
+                return result;
+            }
+            if state.leading {
+                state = group
+                    .turned
+                    .wait(state)
+                    .unwrap_or_else(|error| error.into_inner());
+                continue;
+            }
+            state.leading = true;
+            let batch = std::mem::take(&mut state.queue);
+            let mut lead = Lead {
+                group: &group,
+                tickets: batch.iter().map(|(ticket, _)| *ticket).collect(),
+                results: Vec::new(),
+            };
+            drop(state);
+            let started = std::time::Instant::now();
+            let objects = batch.len();
+            lead.results = self.record_batch(batch);
+            crate::stats::admission_batch(objects, started.elapsed());
+            drop(lead);
+            state = group
+                .state
+                .lock()
+                .unwrap_or_else(|error| error.into_inner());
+        }
+    }
+
+    fn record_batch(&self, batch: Vec<(u64, Recording)>) -> Vec<(u64, Result<()>)> {
+        let fail_all = |batch: Vec<(u64, Recording)>, error: Refusal| {
+            batch
+                .into_iter()
+                .map(|(ticket, _)| (ticket, Err(error.clone())))
+                .collect::<Vec<_>>()
+        };
+        let mut connection = match self.connection() {
+            Ok(connection) => connection,
+            Err(error) => return fail_all(batch, error),
+        };
+        let mut transaction =
+            match connection.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) {
+                Ok(transaction) => transaction,
+                Err(error) => return fail_all(batch, sql("begin blob admission", error)),
+            };
+        let mut results = Vec::with_capacity(batch.len());
+        let mut written = Vec::with_capacity(batch.len());
+        // Each object's row sits behind its own savepoint: one refused row refuses that
+        // object, as it did when every object had a transaction of its own.
+        for (ticket, recording) in batch {
+            let outcome = transaction
+                .savepoint()
+                .map_err(|error| sql("begin object admission", error))
+                .and_then(|savepoint| {
+                    if let Some(row) = &recording.record {
+                        write_verification_on(&savepoint, row)?;
+                    }
+                    savepoint
+                        .commit()
+                        .map_err(|error| sql("record object admission", error))
+                });
+            match outcome {
+                Ok(()) => written.push((ticket, recording.record)),
+                Err(error) => results.push((ticket, Err(error))),
+            }
+        }
+        match transaction
+            .commit()
+            .map_err(|error| sql("commit blob admission", error))
+        {
+            Ok(()) => {
+                let trust = Trust::live(&self.path);
+                for (ticket, record) in written {
+                    if let (Some(row), Some(trust)) = (&record, &trust) {
+                        trust.remember(row);
+                    }
+                    results.push((ticket, Ok(())));
+                }
+            }
+            Err(error) => {
+                results.extend(
+                    written
+                        .into_iter()
+                        .map(|(ticket, _)| (ticket, Err(error.clone()))),
+                );
+            }
+        }
+        results
+    }
+
     pub fn held_keys(&self) -> Result<Vec<crate::storage::HeldKey>> {
         let connection = self.connection()?;
         let mut statement = connection
@@ -1084,7 +1196,7 @@ impl Catalog {
     }
 }
 
-fn admissions(path: &Path) -> Arc<Mutex<()>> {
+fn held_admissions(path: &Path) -> Arc<Mutex<()>> {
     static TURNS: OnceLock<Mutex<HashMap<PathBuf, Arc<Mutex<()>>>>> = OnceLock::new();
     TURNS
         .get_or_init(Default::default)
@@ -1095,6 +1207,73 @@ fn admissions(path: &Path) -> Arc<Mutex<()>> {
         .clone()
 }
 
+/// One published object's catalog rows, handed to the admission group.
+pub(crate) struct Recording {
+    /// The verification row, when this call installed the object.
+    pub record: Option<crate::store::VRecord>,
+}
+
+#[derive(Default)]
+struct AdmissionGroup {
+    state: Mutex<GroupState>,
+    turned: std::sync::Condvar,
+}
+
+#[derive(Default)]
+struct GroupState {
+    next: u64,
+    leading: bool,
+    queue: Vec<(u64, Recording)>,
+    done: HashMap<u64, Result<()>>,
+}
+
+fn admission_group(path: &Path) -> Arc<AdmissionGroup> {
+    static GROUPS: OnceLock<Mutex<HashMap<PathBuf, Arc<AdmissionGroup>>>> = OnceLock::new();
+    GROUPS
+        .get_or_init(Default::default)
+        .lock()
+        .unwrap_or_else(|error| error.into_inner())
+        .entry(path.to_path_buf())
+        .or_default()
+        .clone()
+}
+
+/// A leader hands its batch's results back however it leaves, so a panic inside one batch
+/// cannot strand the admissions queued behind it.
+struct Lead<'a> {
+    group: &'a AdmissionGroup,
+    tickets: Vec<u64>,
+    results: Vec<(u64, Result<()>)>,
+}
+
+impl Drop for Lead<'_> {
+    fn drop(&mut self) {
+        let mut state = self
+            .group
+            .state
+            .lock()
+            .unwrap_or_else(|error| error.into_inner());
+        state.leading = false;
+        let mut answered = std::collections::HashSet::new();
+        for (ticket, result) in self.results.drain(..) {
+            answered.insert(ticket);
+            state.done.insert(ticket, result);
+        }
+        for ticket in &self.tickets {
+            if !answered.contains(ticket) {
+                state.done.insert(
+                    *ticket,
+                    Err(Refusal {
+                        code: Code::IO_FAILED,
+                        detail: "the admission batch holding this object did not finish".into(),
+                    }),
+                );
+            }
+        }
+        self.group.turned.notify_all();
+    }
+}
+
 fn hold_on(connection: &Connection, operation: &str, key: &str, length: u64) -> Result<()> {
     let kind = if key.starts_with("blobs/") {
         "blob"

{tensorfs-0.3.59 → tensorfs-0.3.61}/crates/tensorfs-core/src/checkpoint.rs

@@ -418,6 +418,28 @@ impl Walk {
         }
         Ok(())
     }
+
+    /// Every reached object present, or held by a live source operation's publication
+    /// custodian. A complete checkpoint for publication, never a readable one.
+    pub fn require_held(
+        &self,
+        store: &Store,
+        custodied: &std::collections::BTreeSet<String>,
+    ) -> Result<()> {
+        for r in self.distinct() {
+            if !store.contains(&r.sha256) && !custodied.contains(&r.sha256) {
+                return refuse(
+                    Code::OBJECT_ABSENT,
+                    format!(
+                        "{} is committed by this manifest but neither resident nor held by \
+                         a live publication custodian",
+                        r.id()
+                    ),
+                );
+            }
+        }
+        Ok(())
+    }
 }
 
 /// Manifest → CozyTensorsHeader → contained tensor rules → storage-part object runs.

{tensorfs-0.3.59 → tensorfs-0.3.61}/crates/tensorfs-core/src/derived/resume_tests.rs

@@ -351,7 +351,7 @@ fn missing_corrupt_and_changed_work_cannot_be_reused() {
 }
 
 #[test]
-fn recovery_refuses_old_writer_authority_and_malformed_part_geometry() {
+fn recovery_discards_old_writer_authority_and_malformed_part_geometry() {
     let (store, meta, id, declaration) = fixture("authority");
     let writer = begin(&store, &meta, &id, 9, declaration.clone(), None).unwrap();
     data(&store, &meta, &id, 9, 0x31);
@@ -359,7 +359,8 @@ fn recovery_refuses_old_writer_authority_and_malformed_part_geometry() {
     let plan = progress_plan(&row);
     let mut geometry = progress_row(&row);
     geometry.added_parts[0].part.shape = vec![16, 31];
-    for snapshot in [row, geometry] {
+    let mut writers = vec![writer];
+    for (epoch, snapshot) in [(10, row), (11, geometry)] {
         let bytes = crate::canon::write(&snapshot.to_value());
         let head = crate::durability::checkpoint_bytes(
             &store,
@@ -378,11 +379,20 @@ fn recovery_refuses_old_writer_authority_and_malformed_part_geometry() {
         .head
         .head
         .unwrap();
-        assert!(begin(&store, &meta, &id, 10, declaration.clone(), Some(&head)).is_err());
-        check_writer(&meta, &id, 9).unwrap();
+        // The checkpoint cannot be continued: it is discarded, nothing of it is imported,
+        // and the new writer proceeds from the work this Store already holds.
+        writers.push(begin(&store, &meta, &id, epoch, declaration.clone(), Some(&head)).unwrap());
+        check_writer(&meta, &id, epoch).unwrap();
+        let current = meta.derived_rows().unwrap().remove(0);
+        assert!(current
+            .added_parts
+            .iter()
+            .all(|added| added.part.shape != vec![16, 31]));
+    }
+    fence(&meta, &id, 11).unwrap();
+    for writer in writers {
+        release(&meta, writer);
     }
-    fence(&meta, &id, 9).unwrap();
-    release(&meta, writer);
     abandon(&store, &meta, &id).unwrap();
     std::fs::remove_dir_all(store.root()).unwrap();
 }
@@ -489,7 +499,7 @@ fn source_identity_is_locked_and_source_closure_stays_rooted_after_writer_death(
 }
 
 #[test]
-fn wrong_document_lengths_refuse_before_parsing_or_importing_work() {
+fn wrong_document_lengths_refuse_to_validate_and_are_discarded_on_resume() {
     let (store, meta, id, declaration) = fixture("lengths");
     let writer = begin(&store, &meta, &id, 1, declaration.clone(), None).unwrap();
     data(&store, &meta, &id, 1, 0x31);
@@ -528,16 +538,13 @@ fn wrong_document_lengths_refuse_before_parsing_or_importing_work() {
             .code,
         Code::LENGTH_MISMATCH
     );
-    assert_eq!(
-        begin(&store, &meta, &id, 2, declaration, Some(&forged))
-            .err()
-            .unwrap()
-            .code,
-        Code::LENGTH_MISMATCH
-    );
-    check_writer(&meta, &id, 1).unwrap();
-    fence(&meta, &id, 1).unwrap();
+    // A forged checkpoint is discarded, not imported; this begin then meets the live writer.
+    // Resuming from it discards it instead of refusing: the next writer proceeds.
+    let next = begin(&store, &meta, &id, 2, declaration, Some(&forged)).unwrap();
+    check_writer(&meta, &id, 2).unwrap();
+    fence(&meta, &id, 2).unwrap();
     release(&meta, writer);
+    release(&meta, next);
     abandon(&store, &meta, &id).unwrap();
     std::fs::remove_dir_all(store.root()).unwrap();
 }

{tensorfs-0.3.59 → tensorfs-0.3.61}/crates/tensorfs-core/src/derived.rs

@@ -1601,6 +1601,9 @@ fn require_source_roots(store: &Store, declaration: &Declaration) -> Result<()>
 }
 
 fn load_sources(store: &Store, declaration: &Declaration) -> Result<Vec<SourceData>> {
+    // A prepared model whose output a live source operation's publication custodian holds
+    // derives by reference without its bytes; the result is publishable, never readable.
+    let custodied = crate::ingest::custody::live(store.root())?;
     let mut out = Vec::new();
     for declared in &declaration.sources {
         let manifest = checkpoint::load_manifest(store, &declared.manifest)?;
@@ -1625,11 +1628,13 @@ fn load_sources(store: &Store, declaration: &Declaration) -> Result<Vec<SourceDa
         let closure = checkpoint::load_closure(store, &header)?;
         header.validate(&closure)?;
         let walk = checkpoint::walk_cozytensors(store, &manifest)?;
-        walk.require_resident(store)?;
+        walk.require_held(store, &custodied)?;
         // This is the zero-read/zero-hash inherit gate: the verification record proves the
         // exact installed inode/length/generation without opening or hashing payload bytes.
         for reached in &walk.objects {
-            if reached.kind == "part" || reached.kind == "config" {
+            if (reached.kind == "part" || reached.kind == "config")
+                && !custodied.contains(&reached.obj.sha256)
+            {
                 require_payload_record(store, &reached.obj, reached.kind)?;
             }
         }
@@ -2308,9 +2313,35 @@ pub fn begin(
         release_guards(meta, Some(writer_hold), Vec::new());
         return Err(error);
     }
+    // Pin every resident source object. A custodied object has no local bytes to pin: a
+    // derivation over it carries it by reference and never reads it.
+    let custodied = match crate::ingest::custody::live(store.root()) {
+        Ok(custodied) => custodied,
+        Err(error) => {
+            release_guards(meta, Some(writer_hold), Vec::new());
+            return Err(error);
+        }
+    };
     let mut source_leases = Vec::new();
     for source in &sources {
-        match read::acquire_cozytensors(store, meta, &source.manifest) {
+        let pinned = if custodied.is_empty() {
+            read::acquire_cozytensors(store, meta, &source.manifest)
+        } else {
+            checkpoint::walk_cozytensors(store, &source.manifest).and_then(|walk| {
+                let id = source.manifest.manifest_id();
+                read::acquire(
+                    store,
+                    meta,
+                    id.strip_prefix("sha256:").unwrap_or(&id),
+                    walk.distinct()
+                        .into_iter()
+                        .filter(|object| !custodied.contains(&object.sha256))
+                        .cloned()
+                        .collect(),
+                )
+            })
+        };
+        match pinned {
             Ok((lease, receipts)) if receipts.0.is_empty() => {
                 source_leases.push((source.fact.alias.clone(), lease));
             }
@@ -2342,15 +2373,22 @@ pub fn begin(
         })
         .collect();
 
+    // A checkpoint this work cannot continue (another plan or work context, or no longer
+    // whole) is discarded: the work re-derives what it lost instead of refusing.
+    let mut checkpoint = checkpoint;
     let incoming = match checkpoint
         .map(|head| read_progress(store, head, &transaction, &declaration_text, &source_facts))
-        .transpose()
     {
-        Ok(incoming) => incoming,
-        Err(error) => {
+        None => None,
+        Some(Ok(row)) => Some(row),
+        Some(Err(error)) if error.code == Code::IO_FAILED => {
             release_guards(meta, Some(writer_hold), source_leases);
             return Err(error);
         }
+        Some(Err(_)) => {
+            checkpoint = None;
+            None
+        }
     };
     // Byte verification can update the verification catalog. Keep it outside the SQL
     // writer transaction, then compare the observed row before assigning this epoch.
@@ -3269,7 +3307,7 @@ pub fn commit(
     let manifest_ref = store.put_manifest(&manifest)?.obj;
     store.read_manifest(&manifest_ref)?;
     let walk = checkpoint::walk(store, &manifest)?;
-    walk.require_resident(store)?;
+    walk.require_held(store, &crate::ingest::custody::live(store.root())?)?;
     let (inherited_payload_objects, inherited_payload_bytes) =
         inherit_observation(&header, &row_before)?;
 

{tensorfs-0.3.59 → tensorfs-0.3.61}/crates/tensorfs-core/src/durability.rs

@@ -289,6 +289,23 @@ pub fn local_chain(store: &Store, head: &ObjectRef, plan: &str) -> Result<Set> {
     fold(&links, plan)
 }
 
+/// A supplied chain this conversion can continue, or `None` when it was written under another
+/// plan or slot or is no longer whole in this Store. Such a chain is discarded and the work
+/// reconverts into a new chain: losing a checkpoint costs time, never a refusal.
+pub fn continuable(
+    store: &Store,
+    head: &ObjectRef,
+    plan: &str,
+    chain: &str,
+) -> Result<Option<Set>> {
+    match local_chain(store, head, plan) {
+        Ok(set) if set.operation == chain => Ok(Some(set)),
+        Ok(_) => Ok(None),
+        Err(error) if error.code == Code::IO_FAILED => Err(error),
+        Err(_) => Ok(None),
+    }
+}
+
 /// Export locally complete progress. This does not assert remote durability: the RecordOwner
 /// publishes the bounded chain and advances its durable pointer only after upload acceptance.
 /// NFS is not consulted and cannot delay this checkpoint.

{tensorfs-0.3.59 → tensorfs-0.3.61}/crates/tensorfs-core/src/gc.rs

@@ -108,8 +108,12 @@ pub fn session_holds(store: &Store) -> Result<(Vec<HeldKey>, Vec<String>)> {
     let mut sessions = Vec::new();
     for session in tx::sessions(store) {
         let root = tx::read_session(store.root(), &session)?;
-        let journalled =
+        // An object the operation's publication custodian already holds is not in flight:
+        // the journal stops naming it, so the local copy is ordinary unreferenced bytes.
+        let custody = crate::ingest::custody::read_session(store.root(), &session)?;
+        let mut journalled =
             crate::ingest::journal::session_objects(&tx::candidate_dir(store.root(), &session))?;
+        journalled.retain(|object| !custody.contains(object));
         if root.candidates.is_empty() && journalled.is_empty() {
             continue;
         }