p2p: make doc-sync's client a plain future with a progress callback

sync_from_peer ran the exchange in a detached tokio::spawn and handed back a
channel stream whose sends ignored a closed receiver, so dropping the stream
did not cancel anything despite the doc saying it did (audit B3). The
scheduler's PEER_CATALOG_TIMEOUT, its cancel break, and asset_sync's timeouts
all returned while the sync kept running, and the next tick could start a
second concurrent sync with the same peer.

It is now an ordinary async fn returning a SyncOutcome, reporting through an
FnMut(SyncProgress) -> ControlFlow callback. Dropping or timing out the future
abandons the exchange; the callback answering Break stops it at the next
chunk with DocSyncError::Canceled, which gives the scheduler's catalog pass
and refresh_document_catalog a per-chunk cancel instead of between-peers.
Without the spawn, sync_from_peer and its callers (scheduler passes,
asset_sync's refresh/recursive/push) take &Store instead of Arc<Store>.

Behavior change on cancel: a canceled catalog sync is now an error (recorded
as Canceled on the job) rather than Ok with zero counts, so it no longer
advances the peer's last_synced_at watermark over a partial pull, and
scheduler::sync_with_peer no longer goes on to trigger the remote and run
content passes after the user canceled.

Adds a test that an abandoned staging session leaves the next sync free to
complete without its records.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Greg ShuflinandClaude Opus 5.5 committed 2026-09-28 06:47:43 -07:00
1 parent 96f3e36f70
commit a97423a6b8
14 files changed
+173 -221

No files matched your search

+1 -5
View File
@@ -376,11 +376,7 @@ since it isn't worth taking before it shows up in a profile.
### Keeping track of long-running jobs
Model and dedup design: `docs/adr/0011-jobs-first-class-model.md`; product design:
`docs/sync-management.md`'s "Jobs" section.
- **Cancellation is still coarse on the catalog plane, and for an eviction.** The content
plane now stops within a chunk, but `doc_sync::sync_from_peer` yields a stream with no
cancel hook of its own, so both the scheduler's catalog pass and
`asset_sync::refresh_document_catalog` can only stop between peers — up to
`PEER_CATALOG_REFRESH_TIMEOUT` late. An eviction is local disk work and stops not at all.
- **An eviction does not stop on cancel.** It is local disk work with no cancel check at all.
- **The TUI has no import affordance at all.** When it gains one it should start a job from
the outset rather than being retrofitted the way the GUI's library screens were.
- **A blob's plaintext is read whole before any of it is sealed.** `BlobStore::store` takes
+3 -3
View File
@@ -553,9 +553,9 @@ be invisible to peers on their next incremental sync.
Implemented in `lib/src/p2p/doc_sync.rs`:
- `DocSyncHandler` — server-side protocol handler, registered with the iroh
`Router` alongside `PingHandler`
- `sync_from_peer(endpoint, target_node_id, request, db_path, local_node)`
— client-side entry point, returns a `Stream<Item = SyncProgress>` the caller
drives/displays
- `sync_from_peer(endpoint, peer, request, store, local_node, progress)`
— client-side entry point, a plain future that reports `SyncProgress` through
the `progress` callback (which can also cancel) and returns a `SyncOutcome`
- `SyncStagingStore` — staging table management (`lib/src/stores/sync_staging.rs`)
The existing `DocumentStore`, `NodeDocumentStore`, `ArtifactStore`, and
@@ -490,7 +490,7 @@ impl DocumentsView {
let peer_count = peer_ids.len();
let res = sync_document_assets_recursive(
&endpoint,
store,
&store,
&doc_id,
&peer_ids,
local_node,
@@ -741,7 +741,7 @@ impl DocumentsView {
let endpoint = endpoint.ok_or_else(|| "P2P not running".to_string())?;
let res = push_document_to_peer(
&endpoint,
store,
&store,
&doc_id,
node_pubkey,
local_node,
+1 -1
View File
@@ -68,7 +68,7 @@ pub fn spawn(
Some(ep) => {
let summary = scheduler::run_all_passes(
&ep,
Arc::clone(&store),
&store,
&job_runner,
local_node,
JobOrigin::Scheduler,
+1 -1
View File
@@ -489,7 +489,7 @@ impl EnsembleView {
.and_then(|s| s.timezone);
let summary = scheduler::sync_with_peer(
&ep,
Arc::clone(&store),
&store,
&job_runner,
local_node,
peer,
+1 -1
View File
@@ -405,7 +405,7 @@ impl SettingsView {
let ep = endpoint.ok_or_else(|| "P2P not running".to_string())?;
let summary = scheduler::run_all_passes(
&ep,
store,
&store,
&job_runner,
local_node,
JobOrigin::User,
+3 -5
View File
@@ -548,7 +548,7 @@ impl EnsembleFfi {
let summary = rt_handle
.block_on(scheduler::sync_with_peer(
&endpoint,
store,
&store,
&self.core.job_runner,
local_node,
peer,
@@ -604,7 +604,6 @@ impl EnsembleFfi {
let peer_ids = self.ensemble_peer_ids()?;
let local_node = self.core.local_node_pubkey_required()?;
let (endpoint, rt_handle, store) = self.p2p_parts()?;
let sync_store = Arc::clone(&store);
// Started on the shared `job_runner`, same as `sync_documents` and the
// import methods above — matches desktop's `document_detail.rs` pull,
@@ -627,7 +626,7 @@ impl EnsembleFfi {
rt_handle
.block_on(crate::p2p::asset_sync::sync_document_assets_recursive(
&endpoint,
sync_store,
&store,
&doc_id,
&peer_ids,
local_node,
@@ -740,7 +739,6 @@ impl EnsembleFfi {
(p2p.node.endpoint(), p2p.runtime.handle().clone(), Arc::clone(p2p.node.store()))
};
let push_store = Arc::clone(&store);
let res = run_as_job_with_progress(
&self.core.job_runner,
JobKind::Push,
@@ -750,7 +748,7 @@ impl EnsembleFfi {
rt_handle
.block_on(crate::p2p::asset_sync::push_document_to_peer(
&endpoint,
push_store,
&store,
&doc_id,
peer,
local_node,
+1 -1
View File
@@ -121,7 +121,7 @@ impl SettingsFfi {
let (endpoint, rt_handle, store) = self.core.require_p2p()?;
let summary = rt_handle.block_on(scheduler::run_all_passes(
&endpoint,
store,
&store,
&self.core.job_runner,
local_node,
JobOrigin::Scheduler,
+26 -47
View File
@@ -23,12 +23,11 @@
//! the missing/stubbed set yields the most up-to-date version the ensemble
//! knows about.
use std::sync::Arc;
use std::ops::ControlFlow;
use std::time::Duration;
use crate::p2p::admission::EnsembleEndpoint;
use log::{info, warn};
use tokio_stream::StreamExt;
use crate::documents::{ArtifactRef, BlobHash, DocumentID};
use crate::jobs::ProgressReporter;
@@ -38,7 +37,7 @@ use crate::stores::artifact::ArtifactStoreError;
use crate::stores::document::DocumentStoreError;
use crate::stores::node_content_status::ContentKey;
use super::doc_sync::{SyncProgress, SyncRequest};
use super::doc_sync::{DocSyncError, SyncRequest};
use super::{artifact_sync, blob_sync, doc_sync};
type Result<T> = std::result::Result<T, AssetSyncError>;
@@ -311,7 +310,7 @@ const PEER_CATALOG_REFRESH_TIMEOUT: Duration = Duration::from_secs(30);
/// hold a newer version than the first).
pub async fn refresh_document_catalog(
endpoint: &EnsembleEndpoint,
store: Arc<Store>,
store: &Store,
doc_id: &DocumentID,
peers: &[String],
local_node: NodePubkey,
@@ -324,10 +323,6 @@ pub async fn refresh_document_catalog(
CatalogRefreshResult { peers_available: peers.len() as u32, ..Default::default() };
for target in peers {
// Between peers only. A doc-sync exchange is a stream of chunks with
// no cancel hook of its own, so this phase's granularity is one peer's
// subtree — the same limitation the scheduler's catalog pass carries,
// and bounded by PEER_CATALOG_REFRESH_TIMEOUT either way.
if stop_requested(reporter) {
result.canceled = true;
break;
@@ -355,27 +350,23 @@ pub async fn refresh_document_catalog(
expand_subtree: true,
};
let exchange = async {
let mut stream =
doc_sync::sync_from_peer(endpoint, peer, request, Arc::clone(&store), local_node)
.await?;
let mut received = 0;
while let Some(item) = stream.next().await {
match item {
Ok(SyncProgress::Complete { total_documents_received, .. }) => {
received = total_documents_received;
}
Ok(_) => {}
Err(e) => return Err(e),
}
// The exchange stops at the next chunk once the user cancels.
let exchange = doc_sync::sync_from_peer(endpoint, peer, request, store, local_node, |_| {
if stop_requested(reporter) {
ControlFlow::Break(())
} else {
ControlFlow::Continue(())
}
Ok::<u32, super::doc_sync::DocSyncError>(received)
};
});
match tokio::time::timeout(PEER_CATALOG_REFRESH_TIMEOUT, exchange).await {
Ok(Ok(received)) => {
Ok(Ok(outcome)) => {
result.peers_refreshed += 1;
result.documents_received += received;
result.documents_received += outcome.received;
}
Ok(Err(DocSyncError::Canceled)) => {
result.canceled = true;
break;
}
Ok(Err(e)) => warn!("catalog-refresh: doc-sync with {short} failed: {e}"),
Err(_) => warn!("catalog-refresh: doc-sync with {short} timed out"),
@@ -435,15 +426,14 @@ pub struct CatalogRefreshResult {
/// is exactly the pre-refresh behavior.
pub async fn sync_document_assets_recursive(
endpoint: &EnsembleEndpoint,
store: Arc<Store>,
store: &Store,
doc_id: &DocumentID,
peers: &[String],
local_node: NodePubkey,
reporter: Option<&dyn ProgressReporter>,
) -> Result<DocAssetSyncResult> {
let refresh =
refresh_document_catalog(endpoint, Arc::clone(&store), doc_id, peers, local_node, reporter)
.await?;
refresh_document_catalog(endpoint, store, doc_id, peers, local_node, reporter).await?;
// A cancel during the catalog phase ends the pass rather than falling into
// the content phase: the user stopped the pull, and "we stopped asking who
@@ -464,7 +454,7 @@ pub async fn sync_document_assets_recursive(
// keep the byte counters.
let quiet = reporter.map(CancelOnly);
let quiet = quiet.as_ref().map(|q| q as &dyn ProgressReporter);
let mut result = pull_content_recursive(endpoint, &store, doc_id, peers, quiet).await?;
let mut result = pull_content_recursive(endpoint, store, doc_id, peers, quiet).await?;
result.catalog_peers_available = refresh.peers_available;
result.catalog_peers_refreshed = refresh.peers_refreshed;
result.catalog_documents_received = refresh.documents_received;
@@ -741,7 +731,7 @@ pub struct DocPushResult {
/// [`sync_document_assets`].
pub async fn push_document_to_peer(
endpoint: &EnsembleEndpoint,
store: Arc<Store>,
store: &Store,
doc_id: &DocumentID,
peer: NodePubkey,
local_node: NodePubkey,
@@ -765,23 +755,12 @@ pub async fn push_document_to_peer(
let mut result = DocPushResult::default();
let metadata_phase = async {
let mut stream =
doc_sync::sync_from_peer(endpoint, peer, request, Arc::clone(&store), local_node)
.await?;
while let Some(item) = stream.next().await {
match item {
Ok(SyncProgress::Complete { total_documents_sent, .. }) => {
result.documents_pushed = total_documents_sent;
}
Ok(_) => {}
Err(e) => warn!("push: doc-sync with {target} failed: {e}"),
}
}
Ok::<(), super::doc_sync::DocSyncError>(())
};
let metadata_phase =
doc_sync::sync_from_peer(endpoint, peer, request, store, local_node, |_| {
ControlFlow::Continue(())
});
match tokio::time::timeout(PEER_PUSH_METADATA_TIMEOUT, metadata_phase).await {
Ok(Ok(())) => {}
Ok(Ok(outcome)) => result.documents_pushed = outcome.sent,
Ok(Err(e)) => warn!("push: doc-sync with {target} failed: {e}"),
Err(_) => warn!("push: doc-sync with {target} timed out"),
}
@@ -846,7 +825,7 @@ pub async fn push_document_to_peer(
if blob_hashes.is_empty() {
return;
}
let push = blob_sync::push_blobs_to_peer(endpoint, peer, &blob_hashes, &store, reporter);
let push = blob_sync::push_blobs_to_peer(endpoint, peer, &blob_hashes, store, reporter);
match tokio::time::timeout(PEER_BLOB_TIMEOUT, push).await {
Ok(Ok(r)) => result.blobs_pushed = r.received,
Ok(Err(blob_sync::BlobSyncError::Canceled)) => blobs_canceled = true,
+78 -71
View File
@@ -18,6 +18,7 @@
//! ```
use std::collections::HashMap;
use std::ops::ControlFlow;
use std::sync::Arc;
use iroh::{EndpointAddr, EndpointId, KeyParsingError, endpoint::Connection};
@@ -131,6 +132,8 @@ pub enum DocSyncError {
NodeDocStatus(#[from] NodeDocumentStatusError),
#[error(transparent)]
NodeBlobStatus(#[from] crate::stores::node_content_status::NodeContentStatusError),
#[error("canceled")]
Canceled,
}
// ── Wire message types ────────────────────────────────────────────────────────
@@ -573,23 +576,24 @@ async fn read_chunk(recv: &mut iroh::endpoint::RecvStream) -> Result<(DocumentCh
// ── Progress (client-facing) ──────────────────────────────────────────────────
/// Events emitted by [`sync_from_peer`] as the sync progresses.
/// Events reported by [`sync_from_peer`] as the pull phase progresses.
#[derive(Debug)]
pub enum SyncProgress {
/// Received the header: total work estimate now available.
Started { total_documents: u32, total_blob_bytes: u64, total_artifact_bytes: u64 },
/// One chunk has been received and applied to the local stores.
/// One chunk has been received and staged.
ChunkApplied { documents_received_so_far: u32, cursor: Vec<u8> },
/// Sync completed successfully (both pull and push phases).
Complete {
total_documents_received: u32,
total_documents_sent: u32,
/// Documents dropped during the pull phase because this build could
/// not decode them — see [`read_chunk`]. Nonzero here means the peer
/// is on a newer sync schema; the same records will be picked up
/// once this node upgrades.
records_skipped: u32,
},
}
/// What one completed [`sync_from_peer`] exchange moved, in both directions.
#[derive(Debug, Clone, Copy, Default)]
pub struct SyncOutcome {
pub received: u32,
pub sent: u32,
/// Documents dropped during the pull phase because this build could not
/// decode them — see [`read_chunk`]. Nonzero means the peer is on a newer
/// sync schema; the same records will be picked up once this node upgrades.
pub skipped: u32,
}
// ── Server ────────────────────────────────────────────────────────────────────
@@ -1330,72 +1334,58 @@ async fn serve_pull_response(
// ── Client ────────────────────────────────────────────────────────────────────
/// Connect to `peer` and pull document updates, applying them to local stores.
/// Connect to `peer`, pull its document updates into the local stores, then
/// serve its reverse pull.
///
/// Returns a stream of [`SyncProgress`] events backed by a tokio channel.
/// Drive it to completion with `StreamExt::collect` or a loop; dropping it
/// early cancels the background sync task.
/// A plain future: dropping it — a timeout, a `select!` — abandons the
/// exchange where it stands, and so does `progress` answering
/// [`ControlFlow::Break`], which returns [`DocSyncError::Canceled`]. Either way
/// the half-received staging session is left incomplete and never promoted,
/// and the next sync opens a fresh one.
pub async fn sync_from_peer(
endpoint: &EnsembleEndpoint,
peer: NodePubkey,
request: SyncRequest,
store: Arc<Store>,
store: &Store,
local_node: NodePubkey,
) -> Result<tokio_stream::wrappers::ReceiverStream<Result<SyncProgress>>> {
mut progress: impl FnMut(SyncProgress) -> ControlFlow<()>,
) -> Result<SyncOutcome> {
let endpoint_id: EndpointId = peer.try_into()?;
let conn = endpoint
.connect(EndpointAddr::from(endpoint_id), DOC_SYNC_ALPN)
.await
.map_err(WireError::from)?;
let (tx, rx) = tokio::sync::mpsc::channel(16);
tokio::spawn(async move {
if let Err(e) = run_client_sync(conn, request, store, local_node, peer, &tx).await {
let _ = tx.send(Err(e)).await;
}
});
Ok(tokio_stream::wrappers::ReceiverStream::new(rx))
}
async fn run_client_sync(
conn: Connection,
request: SyncRequest,
store: Arc<Store>,
local_node: NodePubkey,
peer_pubkey: NodePubkey,
tx: &tokio::sync::mpsc::Sender<Result<SyncProgress>>,
) -> Result<()> {
let (mut send, mut recv) = conn.open_bi().await.map_err(WireError::from)?;
write_msg(&mut send, &request).await?;
let header: SyncHeader = read_msg(&mut recv).await?;
log_if_schema_version_differs(SYNC_PROTOCOL_VERSION, &header, peer_pubkey.to_hex());
log_if_schema_version_differs(SYNC_PROTOCOL_VERSION, &header, peer.to_hex());
info!(
"doc-sync header: {} docs, {} blob bytes, {} artifact bytes",
header.total_documents, header.total_blob_bytes, header.total_artifact_bytes
);
tx.send(Ok(SyncProgress::Started {
if progress(SyncProgress::Started {
total_documents: header.total_documents,
total_blob_bytes: header.total_blob_bytes,
total_artifact_bytes: header.total_artifact_bytes,
}))
.await
.ok();
})
.is_break()
{
return Err(DocSyncError::Canceled);
}
let revocation_batch: MembershipBatch = read_msg(&mut recv).await?;
apply_membership_batch(&revocation_batch, &store, peer_pubkey)?;
apply_membership_batch(&revocation_batch, store, peer)?;
let tombstone_batch: DocumentTombstoneBatch = read_msg(&mut recv).await?;
apply_document_tombstone_batch(
&tombstone_batch,
&store.doc_store,
Some(&store.peer_health_store),
peer_pubkey,
peer,
)?;
let release_batch: ContentReleaseBatch = read_msg(&mut recv).await?;
apply_content_release_batch(&release_batch, &store.node_content_status_store, peer_pubkey)?;
apply_content_release_batch(&release_batch, &store.node_content_status_store, peer)?;
let doc_store = &store.doc_store;
let artifact_store = &store.artifact_store;
@@ -1410,14 +1400,13 @@ async fn run_client_sync(
.as_deref()
.and_then(|c| {
staging_store
.batch(|db_tx| staging_store.find_resumable_session(db_tx, peer_pubkey, c))
.batch(|db_tx| staging_store.find_resumable_session(db_tx, peer, c))
.ok()
.flatten()
})
.map(Ok)
.unwrap_or_else(|| {
staging_store
.batch(|db_tx| staging_store.create_session(db_tx, peer_pubkey, local_node))
staging_store.batch(|db_tx| staging_store.create_session(db_tx, peer, local_node))
})?;
let mut total_staged: u32 = 0;
@@ -1431,26 +1420,21 @@ async fn run_client_sync(
stage_chunk(&chunk.documents, staging_store, &session_id, &chunk.cursor)?;
total_staged += n;
tx.send(Ok(SyncProgress::ChunkApplied {
let step = progress(SyncProgress::ChunkApplied {
documents_received_so_far: total_staged,
cursor: chunk.cursor,
}))
.await
.ok();
});
if step.is_break() {
return Err(DocSyncError::Canceled);
}
if chunk.is_final {
break;
}
}
record_schema_gap(
&store,
&peer_pubkey,
header.sync_protocol_version,
total_skipped,
peer_pubkey.to_hex(),
);
record_schema_gap(store, &peer, header.sync_protocol_version, total_skipped, peer.to_hex());
let total_received = promote_session(
let received = promote_session(
&session_id,
staging_store,
doc_store,
@@ -1458,7 +1442,7 @@ async fn run_client_sync(
node_doc_store,
node_content_store,
Some(&store.peer_health_store),
peer_pubkey,
peer,
local_node,
)?;
@@ -1467,21 +1451,14 @@ async fn run_client_sync(
// Phase 2: serve local docs to the server (reverse pull initiated by server).
let (mut send2, mut recv2) = wire::accept(&conn).await?;
let reverse_req: SyncRequest = read_msg(&mut recv2).await?;
let total_sent = serve_pull_response(&mut send2, &reverse_req, &store).await?;
let sent = serve_pull_response(&mut send2, &reverse_req, store).await?;
// Wait for the server to close the connection after it finishes processing.
// Without this, dropping conn here would send CONNECTION_CLOSE before the
// server has read all chunks, potentially losing data mid-stream.
conn.closed().await;
tx.send(Ok(SyncProgress::Complete {
total_documents_received: total_received,
total_documents_sent: total_sent,
records_skipped: total_skipped,
}))
.await
.ok();
Ok(())
Ok(SyncOutcome { received, sent, skipped: total_skipped })
}
// ── Staging helpers ───────────────────────────────────────────────────────────
@@ -2544,6 +2521,36 @@ mod tests {
assert!(get_document(&doc_store, &doc_id).is_none());
}
/// A sync abandoned mid-pull — its future dropped by a timeout, or
/// canceled — leaves its staging session unpromoted. The next sync must
/// still land, and must not drag the abandoned session's records in.
#[rstest]
fn an_abandoned_sync_leaves_the_next_one_free_to_complete(store: TestStore) {
let (peer, local) = (node(7), node(8));
let staging = &store.sync_staging_store;
let session = || staging.batch(|tx| staging.create_session(tx, peer, local)).unwrap();
let (abandoned, next) = ([1u8; 16], [2u8; 16]);
stage_chunk(&[make_record(abandoned, None)], staging, &session(), b"c").unwrap();
let second = session();
stage_chunk(&[make_record(next, None)], staging, &second, b"c").unwrap();
let promoted = promote_session(
&second,
staging,
&store.doc_store,
&store.artifact_store,
&store.node_doc_status_store,
&store.node_content_status_store,
None,
peer,
local,
);
assert_eq!(promoted.unwrap(), 1);
assert!(get_document(&store.doc_store, &DocumentID::from_uuid_bytes(next)).is_some());
assert!(get_document(&store.doc_store, &DocumentID::from_uuid_bytes(abandoned)).is_none());
}
// ── Edges, and property removal ──────────────────────────────────────────
/// A store quartet plus the node key, for the record-application tests below.
+1 -1
View File
@@ -262,7 +262,7 @@ impl NodeAdminHandler {
// via the shared `BackgroundSync` job either way.
let summary = scheduler::run_all_passes(
&endpoint,
store,
&store,
&job_runner,
local_node,
JobOrigin::Scheduler,
+52 -80
View File
@@ -33,12 +33,11 @@
//! invasive than a timer, so every driver waits for its next tick instead.
use std::collections::HashMap;
use std::sync::Arc;
use std::ops::ControlFlow;
use std::time::{Duration, Instant};
use crate::p2p::admission::EnsembleEndpoint;
use log::{info, warn};
use tokio_stream::StreamExt;
use crate::content_ledger::content_ledger;
use crate::documents::{ArtifactKind, BlobHash, DocumentID};
@@ -81,21 +80,20 @@ pub struct SchedulerPassResult {
/// the same peers and stores.
pub async fn run_all_passes(
endpoint: &EnsembleEndpoint,
store: Arc<Store>,
store: &Store,
job_runner: &JobRunner,
local_node: NodePubkey,
origin: JobOrigin,
) -> Option<SchedulerPassResult> {
let handle =
job_runner.start(JobKind::BackgroundSync, JobSubject::None, "Background sync", origin)?;
let peers = ensemble_peers_freshest_first(&store);
let peers = ensemble_peers_freshest_first(store);
let started = Instant::now();
let catalog =
run_catalog_pass(endpoint, Arc::clone(&store), job_runner, local_node, &peers).await;
let catalog = run_catalog_pass(endpoint, store, job_runner, local_node, &peers).await;
let after_catalog = started.elapsed();
let catalog_fetch = run_catalog_fetch_pass(endpoint, &store, &peers).await;
let catalog_fetch = run_catalog_fetch_pass(endpoint, store, &peers).await;
let after_fetch = started.elapsed();
let content = run_content_pass(endpoint, Arc::clone(&store), job_runner, &peers).await;
let content = run_content_pass(endpoint, store, job_runner, &peers).await;
info!(
"scheduler: pass over {} peer(s) took {:.1}s (catalog {:.1}s, \
catalog-fetch {:.1}s, content {:.1}s)",
@@ -124,15 +122,15 @@ pub async fn run_all_passes(
/// back to reporting only the catalog pass it already ran.
pub async fn run_content_passes_for_peer(
endpoint: &EnsembleEndpoint,
store: Arc<Store>,
store: &Store,
job_runner: &JobRunner,
peer: NodePubkey,
) -> Option<(CatalogFetchPassResult, ContentPassResult)> {
let peer_node =
ensemble_peers_freshest_first(&store).into_iter().find(|p| p.node_pubkey == peer.0)?;
ensemble_peers_freshest_first(store).into_iter().find(|p| p.node_pubkey == peer.0)?;
let peers = [peer_node];
let catalog_fetch = run_catalog_fetch_pass(endpoint, &store, &peers).await;
let content = run_content_pass(endpoint, Arc::clone(&store), job_runner, &peers).await;
let catalog_fetch = run_catalog_fetch_pass(endpoint, store, &peers).await;
let content = run_content_pass(endpoint, store, job_runner, &peers).await;
Some((catalog_fetch, content))
}
@@ -251,7 +249,7 @@ impl RemoteReconcile {
/// instruction, and the content passes stay in one place.
pub async fn sync_with_peer(
endpoint: &EnsembleEndpoint,
store: Arc<Store>,
store: &Store,
job_runner: &JobRunner,
local_node: NodePubkey,
peer: NodePubkey,
@@ -265,7 +263,7 @@ pub async fn sync_with_peer(
return Err("that is this node — pick another ensemble member".to_string());
}
let Some(peer_node) =
ensemble_peers_freshest_first(&store).into_iter().find(|p| p.node_pubkey == peer.0)
ensemble_peers_freshest_first(store).into_iter().find(|p| p.node_pubkey == peer.0)
else {
return Err(format!("{} is not a member of this node's ensemble", peer.to_hex()));
};
@@ -311,7 +309,7 @@ pub async fn sync_with_peer(
let started = Instant::now();
let outcome = tokio::time::timeout(
PEER_CATALOG_TIMEOUT,
drive_catalog_sync(endpoint, peer, request, Arc::clone(&store), local_node, &handle),
drive_catalog_sync(endpoint, peer, request, store, local_node, &handle),
)
.await;
let catalog_took = started.elapsed();
@@ -498,7 +496,7 @@ pub struct CatalogPassResult {
/// starting a second concurrent sync.
pub async fn run_catalog_pass(
endpoint: &EnsembleEndpoint,
store: Arc<Store>,
store: &Store,
job_runner: &JobRunner,
local_node: NodePubkey,
peers: &[PeerNode],
@@ -534,14 +532,7 @@ pub async fn run_catalog_pass(
let peer_started = Instant::now();
let outcome = tokio::time::timeout(
PEER_CATALOG_TIMEOUT,
drive_catalog_sync(
endpoint,
node_pubkey,
request,
Arc::clone(&store),
local_node,
&handle,
),
drive_catalog_sync(endpoint, node_pubkey, request, store, local_node, &handle),
)
.await;
@@ -587,68 +578,49 @@ pub async fn run_catalog_pass(
}
/// Drives one catalog exchange to completion, reporting progress onto
/// `handle`. Returns `(documents received + sent, records skipped)` — the
/// scheduler passes ignore both and only care that it succeeded, while
/// [`sync_with_peer`] reports them back to whoever pressed the button.
/// `handle` and stopping at the next chunk once it is canceled. Returns
/// `(documents received + sent, records skipped)` — the scheduler passes
/// ignore both and only care that it succeeded, while [`sync_with_peer`]
/// reports them back to whoever pressed the button.
async fn drive_catalog_sync(
endpoint: &EnsembleEndpoint,
peer: NodePubkey,
request: SyncRequest,
store: Arc<Store>,
store: &Store,
local_node: NodePubkey,
handle: &JobHandle,
) -> Result<(u32, u32), String> {
let mut stream = sync_from_peer(endpoint, peer, request, store, local_node)
let progress = |event| {
handle.report_progress(match event {
SyncProgress::Started { total_documents, total_blob_bytes, .. } => JobProgress {
units_done: 0,
units_total: Some(total_documents),
bytes_done: 0,
bytes_total: Some(total_blob_bytes),
message: Some(format!("0/{total_documents} documents")),
},
SyncProgress::ChunkApplied { documents_received_so_far, .. } => JobProgress {
units_done: documents_received_so_far,
units_total: None,
bytes_done: 0,
bytes_total: None,
message: Some(format!("{documents_received_so_far} documents")),
},
});
if handle.is_canceled() { ControlFlow::Break(()) } else { ControlFlow::Continue(()) }
};
let outcome = sync_from_peer(endpoint, peer, request, store, local_node, progress)
.await
.map_err(|e| e.to_string())?;
let (mut documents, mut skipped) = (0u32, 0u32);
while let Some(item) = stream.next().await {
match item.map_err(|e| e.to_string())? {
SyncProgress::Started { total_documents, total_blob_bytes, .. } => {
handle.report_progress(JobProgress {
units_done: 0,
units_total: Some(total_documents),
bytes_done: 0,
bytes_total: Some(total_blob_bytes),
message: Some(format!("0/{total_documents} documents")),
});
}
SyncProgress::ChunkApplied { documents_received_so_far, .. } => {
handle.report_progress(JobProgress {
units_done: documents_received_so_far,
units_total: None,
bytes_done: 0,
bytes_total: None,
message: Some(format!("{documents_received_so_far} documents")),
});
}
SyncProgress::Complete {
total_documents_received,
total_documents_sent,
records_skipped,
} => {
documents = total_documents_received + total_documents_sent;
skipped = records_skipped;
if records_skipped > 0 {
warn!(
"scheduler: catalog sync with {}: {records_skipped} document(s) \
skipped (peer likely on a newer build) — upgrade to pick them up",
peer.to_hex()
);
}
}
}
// Best-effort: sync_from_peer has no per-chunk cancel hook, so this
// can only stop between yielded stream items, not mid-chunk. The
// *content* plane no longer has that limitation — a blob transfer
// reads the token per chunk — so the catalog plane is now the coarse
// half, and `asset_sync::refresh_document_catalog` carries the same
// note for the on-demand version of this exchange.
if handle.is_canceled() {
break;
}
if outcome.skipped > 0 {
warn!(
"scheduler: catalog sync with {}: {} document(s) skipped (peer likely on a newer \
build) — upgrade to pick them up",
peer.to_hex(),
outcome.skipped
);
}
Ok((documents, skipped))
Ok((outcome.received + outcome.sent, outcome.skipped))
}
// ── Pass 2: catalog-plane fetch ────────────────────────────────────────────
@@ -778,13 +750,13 @@ pub struct ContentPassResult {
/// the document-detail screen).
pub async fn run_content_pass(
endpoint: &EnsembleEndpoint,
store: Arc<Store>,
store: &Store,
job_runner: &JobRunner,
peers: &[PeerNode],
) -> ContentPassResult {
let mut result = ContentPassResult::default();
let wants = match content_ledger(&store) {
let wants = match content_ledger(store) {
Ok(ledger) => ledger.wants,
Err(e) => {
warn!("scheduler: failed to compute the content ledger: {e}");
@@ -812,7 +784,7 @@ pub async fn run_content_pass(
continue;
};
let targets = peer_order_by_blob_presence(&store, peers, &hashes);
let targets = peer_order_by_blob_presence(store, peers, &hashes);
let mut missing = hashes;
let mut received = 0u32;
@@ -824,7 +796,7 @@ pub async fn run_content_pass(
// here, so a cancel stops the download that is running rather than
// waiting for it to finish and declining to start the next one.
let fetch =
blob_sync::fetch_blobs_from_peer(endpoint, target, &missing, &store, Some(&handle));
blob_sync::fetch_blobs_from_peer(endpoint, target, &missing, store, Some(&handle));
match tokio::time::timeout(PEER_BLOB_TIMEOUT, fetch).await {
Ok(Ok(r)) => received += r.received,
Ok(Err(blob_sync::BlobSyncError::Canceled)) => break,
+2 -2
View File
@@ -1805,7 +1805,7 @@ pub async fn dispatch_ensemble_sync(
let summary = scheduler::sync_with_peer(
&p2p.endpoint(),
Arc::clone(store),
store,
job_runner,
local_node,
peer,
@@ -1998,7 +1998,7 @@ pub async fn dispatch_sync_assets(
|reporter| async move {
synchronicity::p2p::asset_sync::sync_document_assets_recursive(
&p2p.endpoint(),
std::sync::Arc::clone(p2p.store()),
p2p.store(),
id,
&peer_ids,
local_node,
+1 -1
View File
@@ -51,7 +51,7 @@ pub fn spawn(runtime: &NodeRuntime, job_runner: Arc<JobRunner>) {
Ok(Some(local_node)) => {
let summary = scheduler::run_all_passes(
&endpoint,
Arc::clone(&store),
&store,
&job_runner,
local_node,
JobOrigin::Scheduler,