p2p: make doc-sync's client a plain future with a progress callback
sync_from_peer ran the exchange in a detached tokio::spawn and handed back a channel stream whose sends ignored a closed receiver, so dropping the stream did not cancel anything despite the doc saying it did (audit B3). The scheduler's PEER_CATALOG_TIMEOUT, its cancel break, and asset_sync's timeouts all returned while the sync kept running, and the next tick could start a second concurrent sync with the same peer. It is now an ordinary async fn returning a SyncOutcome, reporting through an FnMut(SyncProgress) -> ControlFlow callback. Dropping or timing out the future abandons the exchange; the callback answering Break stops it at the next chunk with DocSyncError::Canceled, which gives the scheduler's catalog pass and refresh_document_catalog a per-chunk cancel instead of between-peers. Without the spawn, sync_from_peer and its callers (scheduler passes, asset_sync's refresh/recursive/push) take &Store instead of Arc<Store>. Behavior change on cancel: a canceled catalog sync is now an error (recorded as Canceled on the job) rather than Ok with zero counts, so it no longer advances the peer's last_synced_at watermark over a partial pull, and scheduler::sync_with_peer no longer goes on to trigger the remote and run content passes after the user canceled. Adds a test that an abandoned staging session leaves the next sync free to complete without its records. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
96f3e36f70
commit
a97423a6b8
14 files changed
+173
-221
No files matched your search
@@ -376,11 +376,7 @@ since it isn't worth taking before it shows up in a profile.
|
||||
### Keeping track of long-running jobs
|
||||
Model and dedup design: `docs/adr/0011-jobs-first-class-model.md`; product design:
|
||||
`docs/sync-management.md`'s "Jobs" section.
|
||||
- **Cancellation is still coarse on the catalog plane, and for an eviction.** The content
|
||||
plane now stops within a chunk, but `doc_sync::sync_from_peer` yields a stream with no
|
||||
cancel hook of its own, so both the scheduler's catalog pass and
|
||||
`asset_sync::refresh_document_catalog` can only stop between peers — up to
|
||||
`PEER_CATALOG_REFRESH_TIMEOUT` late. An eviction is local disk work and stops not at all.
|
||||
- **An eviction does not stop on cancel.** It is local disk work with no cancel check at all.
|
||||
- **The TUI has no import affordance at all.** When it gains one it should start a job from
|
||||
the outset rather than being retrofitted the way the GUI's library screens were.
|
||||
- **A blob's plaintext is read whole before any of it is sealed.** `BlobStore::store` takes
|
||||
|
||||
@@ -553,9 +553,9 @@ be invisible to peers on their next incremental sync.
|
||||
Implemented in `lib/src/p2p/doc_sync.rs`:
|
||||
- `DocSyncHandler` — server-side protocol handler, registered with the iroh
|
||||
`Router` alongside `PingHandler`
|
||||
- `sync_from_peer(endpoint, target_node_id, request, db_path, local_node)`
|
||||
— client-side entry point, returns a `Stream<Item = SyncProgress>` the caller
|
||||
drives/displays
|
||||
- `sync_from_peer(endpoint, peer, request, store, local_node, progress)`
|
||||
— client-side entry point, a plain future that reports `SyncProgress` through
|
||||
the `progress` callback (which can also cancel) and returns a `SyncOutcome`
|
||||
- `SyncStagingStore` — staging table management (`lib/src/stores/sync_staging.rs`)
|
||||
|
||||
The existing `DocumentStore`, `NodeDocumentStore`, `ArtifactStore`, and
|
||||
|
||||
@@ -490,7 +490,7 @@ impl DocumentsView {
|
||||
let peer_count = peer_ids.len();
|
||||
let res = sync_document_assets_recursive(
|
||||
&endpoint,
|
||||
store,
|
||||
&store,
|
||||
&doc_id,
|
||||
&peer_ids,
|
||||
local_node,
|
||||
@@ -741,7 +741,7 @@ impl DocumentsView {
|
||||
let endpoint = endpoint.ok_or_else(|| "P2P not running".to_string())?;
|
||||
let res = push_document_to_peer(
|
||||
&endpoint,
|
||||
store,
|
||||
&store,
|
||||
&doc_id,
|
||||
node_pubkey,
|
||||
local_node,
|
||||
|
||||
@@ -68,7 +68,7 @@ pub fn spawn(
|
||||
Some(ep) => {
|
||||
let summary = scheduler::run_all_passes(
|
||||
&ep,
|
||||
Arc::clone(&store),
|
||||
&store,
|
||||
&job_runner,
|
||||
local_node,
|
||||
JobOrigin::Scheduler,
|
||||
|
||||
@@ -489,7 +489,7 @@ impl EnsembleView {
|
||||
.and_then(|s| s.timezone);
|
||||
let summary = scheduler::sync_with_peer(
|
||||
&ep,
|
||||
Arc::clone(&store),
|
||||
&store,
|
||||
&job_runner,
|
||||
local_node,
|
||||
peer,
|
||||
|
||||
@@ -405,7 +405,7 @@ impl SettingsView {
|
||||
let ep = endpoint.ok_or_else(|| "P2P not running".to_string())?;
|
||||
let summary = scheduler::run_all_passes(
|
||||
&ep,
|
||||
store,
|
||||
&store,
|
||||
&job_runner,
|
||||
local_node,
|
||||
JobOrigin::User,
|
||||
|
||||
@@ -548,7 +548,7 @@ impl EnsembleFfi {
|
||||
let summary = rt_handle
|
||||
.block_on(scheduler::sync_with_peer(
|
||||
&endpoint,
|
||||
store,
|
||||
&store,
|
||||
&self.core.job_runner,
|
||||
local_node,
|
||||
peer,
|
||||
@@ -604,7 +604,6 @@ impl EnsembleFfi {
|
||||
let peer_ids = self.ensemble_peer_ids()?;
|
||||
let local_node = self.core.local_node_pubkey_required()?;
|
||||
let (endpoint, rt_handle, store) = self.p2p_parts()?;
|
||||
let sync_store = Arc::clone(&store);
|
||||
|
||||
// Started on the shared `job_runner`, same as `sync_documents` and the
|
||||
// import methods above — matches desktop's `document_detail.rs` pull,
|
||||
@@ -627,7 +626,7 @@ impl EnsembleFfi {
|
||||
rt_handle
|
||||
.block_on(crate::p2p::asset_sync::sync_document_assets_recursive(
|
||||
&endpoint,
|
||||
sync_store,
|
||||
&store,
|
||||
&doc_id,
|
||||
&peer_ids,
|
||||
local_node,
|
||||
@@ -740,7 +739,6 @@ impl EnsembleFfi {
|
||||
(p2p.node.endpoint(), p2p.runtime.handle().clone(), Arc::clone(p2p.node.store()))
|
||||
};
|
||||
|
||||
let push_store = Arc::clone(&store);
|
||||
let res = run_as_job_with_progress(
|
||||
&self.core.job_runner,
|
||||
JobKind::Push,
|
||||
@@ -750,7 +748,7 @@ impl EnsembleFfi {
|
||||
rt_handle
|
||||
.block_on(crate::p2p::asset_sync::push_document_to_peer(
|
||||
&endpoint,
|
||||
push_store,
|
||||
&store,
|
||||
&doc_id,
|
||||
peer,
|
||||
local_node,
|
||||
|
||||
@@ -121,7 +121,7 @@ impl SettingsFfi {
|
||||
let (endpoint, rt_handle, store) = self.core.require_p2p()?;
|
||||
let summary = rt_handle.block_on(scheduler::run_all_passes(
|
||||
&endpoint,
|
||||
store,
|
||||
&store,
|
||||
&self.core.job_runner,
|
||||
local_node,
|
||||
JobOrigin::Scheduler,
|
||||
|
||||
@@ -23,12 +23,11 @@
|
||||
//! the missing/stubbed set yields the most up-to-date version the ensemble
|
||||
//! knows about.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::ops::ControlFlow;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::p2p::admission::EnsembleEndpoint;
|
||||
use log::{info, warn};
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
use crate::documents::{ArtifactRef, BlobHash, DocumentID};
|
||||
use crate::jobs::ProgressReporter;
|
||||
@@ -38,7 +37,7 @@ use crate::stores::artifact::ArtifactStoreError;
|
||||
use crate::stores::document::DocumentStoreError;
|
||||
use crate::stores::node_content_status::ContentKey;
|
||||
|
||||
use super::doc_sync::{SyncProgress, SyncRequest};
|
||||
use super::doc_sync::{DocSyncError, SyncRequest};
|
||||
use super::{artifact_sync, blob_sync, doc_sync};
|
||||
|
||||
type Result<T> = std::result::Result<T, AssetSyncError>;
|
||||
@@ -311,7 +310,7 @@ const PEER_CATALOG_REFRESH_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
/// hold a newer version than the first).
|
||||
pub async fn refresh_document_catalog(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
doc_id: &DocumentID,
|
||||
peers: &[String],
|
||||
local_node: NodePubkey,
|
||||
@@ -324,10 +323,6 @@ pub async fn refresh_document_catalog(
|
||||
CatalogRefreshResult { peers_available: peers.len() as u32, ..Default::default() };
|
||||
|
||||
for target in peers {
|
||||
// Between peers only. A doc-sync exchange is a stream of chunks with
|
||||
// no cancel hook of its own, so this phase's granularity is one peer's
|
||||
// subtree — the same limitation the scheduler's catalog pass carries,
|
||||
// and bounded by PEER_CATALOG_REFRESH_TIMEOUT either way.
|
||||
if stop_requested(reporter) {
|
||||
result.canceled = true;
|
||||
break;
|
||||
@@ -355,27 +350,23 @@ pub async fn refresh_document_catalog(
|
||||
expand_subtree: true,
|
||||
};
|
||||
|
||||
let exchange = async {
|
||||
let mut stream =
|
||||
doc_sync::sync_from_peer(endpoint, peer, request, Arc::clone(&store), local_node)
|
||||
.await?;
|
||||
let mut received = 0;
|
||||
while let Some(item) = stream.next().await {
|
||||
match item {
|
||||
Ok(SyncProgress::Complete { total_documents_received, .. }) => {
|
||||
received = total_documents_received;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
// The exchange stops at the next chunk once the user cancels.
|
||||
let exchange = doc_sync::sync_from_peer(endpoint, peer, request, store, local_node, |_| {
|
||||
if stop_requested(reporter) {
|
||||
ControlFlow::Break(())
|
||||
} else {
|
||||
ControlFlow::Continue(())
|
||||
}
|
||||
Ok::<u32, super::doc_sync::DocSyncError>(received)
|
||||
};
|
||||
});
|
||||
|
||||
match tokio::time::timeout(PEER_CATALOG_REFRESH_TIMEOUT, exchange).await {
|
||||
Ok(Ok(received)) => {
|
||||
Ok(Ok(outcome)) => {
|
||||
result.peers_refreshed += 1;
|
||||
result.documents_received += received;
|
||||
result.documents_received += outcome.received;
|
||||
}
|
||||
Ok(Err(DocSyncError::Canceled)) => {
|
||||
result.canceled = true;
|
||||
break;
|
||||
}
|
||||
Ok(Err(e)) => warn!("catalog-refresh: doc-sync with {short} failed: {e}"),
|
||||
Err(_) => warn!("catalog-refresh: doc-sync with {short} timed out"),
|
||||
@@ -435,15 +426,14 @@ pub struct CatalogRefreshResult {
|
||||
/// is exactly the pre-refresh behavior.
|
||||
pub async fn sync_document_assets_recursive(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
doc_id: &DocumentID,
|
||||
peers: &[String],
|
||||
local_node: NodePubkey,
|
||||
reporter: Option<&dyn ProgressReporter>,
|
||||
) -> Result<DocAssetSyncResult> {
|
||||
let refresh =
|
||||
refresh_document_catalog(endpoint, Arc::clone(&store), doc_id, peers, local_node, reporter)
|
||||
.await?;
|
||||
refresh_document_catalog(endpoint, store, doc_id, peers, local_node, reporter).await?;
|
||||
|
||||
// A cancel during the catalog phase ends the pass rather than falling into
|
||||
// the content phase: the user stopped the pull, and "we stopped asking who
|
||||
@@ -464,7 +454,7 @@ pub async fn sync_document_assets_recursive(
|
||||
// keep the byte counters.
|
||||
let quiet = reporter.map(CancelOnly);
|
||||
let quiet = quiet.as_ref().map(|q| q as &dyn ProgressReporter);
|
||||
let mut result = pull_content_recursive(endpoint, &store, doc_id, peers, quiet).await?;
|
||||
let mut result = pull_content_recursive(endpoint, store, doc_id, peers, quiet).await?;
|
||||
result.catalog_peers_available = refresh.peers_available;
|
||||
result.catalog_peers_refreshed = refresh.peers_refreshed;
|
||||
result.catalog_documents_received = refresh.documents_received;
|
||||
@@ -741,7 +731,7 @@ pub struct DocPushResult {
|
||||
/// [`sync_document_assets`].
|
||||
pub async fn push_document_to_peer(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
doc_id: &DocumentID,
|
||||
peer: NodePubkey,
|
||||
local_node: NodePubkey,
|
||||
@@ -765,23 +755,12 @@ pub async fn push_document_to_peer(
|
||||
|
||||
let mut result = DocPushResult::default();
|
||||
|
||||
let metadata_phase = async {
|
||||
let mut stream =
|
||||
doc_sync::sync_from_peer(endpoint, peer, request, Arc::clone(&store), local_node)
|
||||
.await?;
|
||||
while let Some(item) = stream.next().await {
|
||||
match item {
|
||||
Ok(SyncProgress::Complete { total_documents_sent, .. }) => {
|
||||
result.documents_pushed = total_documents_sent;
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => warn!("push: doc-sync with {target} failed: {e}"),
|
||||
}
|
||||
}
|
||||
Ok::<(), super::doc_sync::DocSyncError>(())
|
||||
};
|
||||
let metadata_phase =
|
||||
doc_sync::sync_from_peer(endpoint, peer, request, store, local_node, |_| {
|
||||
ControlFlow::Continue(())
|
||||
});
|
||||
match tokio::time::timeout(PEER_PUSH_METADATA_TIMEOUT, metadata_phase).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Ok(outcome)) => result.documents_pushed = outcome.sent,
|
||||
Ok(Err(e)) => warn!("push: doc-sync with {target} failed: {e}"),
|
||||
Err(_) => warn!("push: doc-sync with {target} timed out"),
|
||||
}
|
||||
@@ -846,7 +825,7 @@ pub async fn push_document_to_peer(
|
||||
if blob_hashes.is_empty() {
|
||||
return;
|
||||
}
|
||||
let push = blob_sync::push_blobs_to_peer(endpoint, peer, &blob_hashes, &store, reporter);
|
||||
let push = blob_sync::push_blobs_to_peer(endpoint, peer, &blob_hashes, store, reporter);
|
||||
match tokio::time::timeout(PEER_BLOB_TIMEOUT, push).await {
|
||||
Ok(Ok(r)) => result.blobs_pushed = r.received,
|
||||
Ok(Err(blob_sync::BlobSyncError::Canceled)) => blobs_canceled = true,
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
//! ```
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::ops::ControlFlow;
|
||||
use std::sync::Arc;
|
||||
|
||||
use iroh::{EndpointAddr, EndpointId, KeyParsingError, endpoint::Connection};
|
||||
@@ -131,6 +132,8 @@ pub enum DocSyncError {
|
||||
NodeDocStatus(#[from] NodeDocumentStatusError),
|
||||
#[error(transparent)]
|
||||
NodeBlobStatus(#[from] crate::stores::node_content_status::NodeContentStatusError),
|
||||
#[error("canceled")]
|
||||
Canceled,
|
||||
}
|
||||
|
||||
// ── Wire message types ────────────────────────────────────────────────────────
|
||||
@@ -573,23 +576,24 @@ async fn read_chunk(recv: &mut iroh::endpoint::RecvStream) -> Result<(DocumentCh
|
||||
|
||||
// ── Progress (client-facing) ──────────────────────────────────────────────────
|
||||
|
||||
/// Events emitted by [`sync_from_peer`] as the sync progresses.
|
||||
/// Events reported by [`sync_from_peer`] as the pull phase progresses.
|
||||
#[derive(Debug)]
|
||||
pub enum SyncProgress {
|
||||
/// Received the header: total work estimate now available.
|
||||
Started { total_documents: u32, total_blob_bytes: u64, total_artifact_bytes: u64 },
|
||||
/// One chunk has been received and applied to the local stores.
|
||||
/// One chunk has been received and staged.
|
||||
ChunkApplied { documents_received_so_far: u32, cursor: Vec<u8> },
|
||||
/// Sync completed successfully (both pull and push phases).
|
||||
Complete {
|
||||
total_documents_received: u32,
|
||||
total_documents_sent: u32,
|
||||
/// Documents dropped during the pull phase because this build could
|
||||
/// not decode them — see [`read_chunk`]. Nonzero here means the peer
|
||||
/// is on a newer sync schema; the same records will be picked up
|
||||
/// once this node upgrades.
|
||||
records_skipped: u32,
|
||||
},
|
||||
}
|
||||
|
||||
/// What one completed [`sync_from_peer`] exchange moved, in both directions.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct SyncOutcome {
|
||||
pub received: u32,
|
||||
pub sent: u32,
|
||||
/// Documents dropped during the pull phase because this build could not
|
||||
/// decode them — see [`read_chunk`]. Nonzero means the peer is on a newer
|
||||
/// sync schema; the same records will be picked up once this node upgrades.
|
||||
pub skipped: u32,
|
||||
}
|
||||
|
||||
// ── Server ────────────────────────────────────────────────────────────────────
|
||||
@@ -1330,72 +1334,58 @@ async fn serve_pull_response(
|
||||
|
||||
// ── Client ────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Connect to `peer` and pull document updates, applying them to local stores.
|
||||
/// Connect to `peer`, pull its document updates into the local stores, then
|
||||
/// serve its reverse pull.
|
||||
///
|
||||
/// Returns a stream of [`SyncProgress`] events backed by a tokio channel.
|
||||
/// Drive it to completion with `StreamExt::collect` or a loop; dropping it
|
||||
/// early cancels the background sync task.
|
||||
/// A plain future: dropping it — a timeout, a `select!` — abandons the
|
||||
/// exchange where it stands, and so does `progress` answering
|
||||
/// [`ControlFlow::Break`], which returns [`DocSyncError::Canceled`]. Either way
|
||||
/// the half-received staging session is left incomplete and never promoted,
|
||||
/// and the next sync opens a fresh one.
|
||||
pub async fn sync_from_peer(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
peer: NodePubkey,
|
||||
request: SyncRequest,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
local_node: NodePubkey,
|
||||
) -> Result<tokio_stream::wrappers::ReceiverStream<Result<SyncProgress>>> {
|
||||
mut progress: impl FnMut(SyncProgress) -> ControlFlow<()>,
|
||||
) -> Result<SyncOutcome> {
|
||||
let endpoint_id: EndpointId = peer.try_into()?;
|
||||
let conn = endpoint
|
||||
.connect(EndpointAddr::from(endpoint_id), DOC_SYNC_ALPN)
|
||||
.await
|
||||
.map_err(WireError::from)?;
|
||||
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(16);
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = run_client_sync(conn, request, store, local_node, peer, &tx).await {
|
||||
let _ = tx.send(Err(e)).await;
|
||||
}
|
||||
});
|
||||
|
||||
Ok(tokio_stream::wrappers::ReceiverStream::new(rx))
|
||||
}
|
||||
|
||||
async fn run_client_sync(
|
||||
conn: Connection,
|
||||
request: SyncRequest,
|
||||
store: Arc<Store>,
|
||||
local_node: NodePubkey,
|
||||
peer_pubkey: NodePubkey,
|
||||
tx: &tokio::sync::mpsc::Sender<Result<SyncProgress>>,
|
||||
) -> Result<()> {
|
||||
let (mut send, mut recv) = conn.open_bi().await.map_err(WireError::from)?;
|
||||
|
||||
write_msg(&mut send, &request).await?;
|
||||
|
||||
let header: SyncHeader = read_msg(&mut recv).await?;
|
||||
log_if_schema_version_differs(SYNC_PROTOCOL_VERSION, &header, peer_pubkey.to_hex());
|
||||
log_if_schema_version_differs(SYNC_PROTOCOL_VERSION, &header, peer.to_hex());
|
||||
info!(
|
||||
"doc-sync header: {} docs, {} blob bytes, {} artifact bytes",
|
||||
header.total_documents, header.total_blob_bytes, header.total_artifact_bytes
|
||||
);
|
||||
tx.send(Ok(SyncProgress::Started {
|
||||
if progress(SyncProgress::Started {
|
||||
total_documents: header.total_documents,
|
||||
total_blob_bytes: header.total_blob_bytes,
|
||||
total_artifact_bytes: header.total_artifact_bytes,
|
||||
}))
|
||||
.await
|
||||
.ok();
|
||||
})
|
||||
.is_break()
|
||||
{
|
||||
return Err(DocSyncError::Canceled);
|
||||
}
|
||||
|
||||
let revocation_batch: MembershipBatch = read_msg(&mut recv).await?;
|
||||
apply_membership_batch(&revocation_batch, &store, peer_pubkey)?;
|
||||
apply_membership_batch(&revocation_batch, store, peer)?;
|
||||
let tombstone_batch: DocumentTombstoneBatch = read_msg(&mut recv).await?;
|
||||
apply_document_tombstone_batch(
|
||||
&tombstone_batch,
|
||||
&store.doc_store,
|
||||
Some(&store.peer_health_store),
|
||||
peer_pubkey,
|
||||
peer,
|
||||
)?;
|
||||
let release_batch: ContentReleaseBatch = read_msg(&mut recv).await?;
|
||||
apply_content_release_batch(&release_batch, &store.node_content_status_store, peer_pubkey)?;
|
||||
apply_content_release_batch(&release_batch, &store.node_content_status_store, peer)?;
|
||||
|
||||
let doc_store = &store.doc_store;
|
||||
let artifact_store = &store.artifact_store;
|
||||
@@ -1410,14 +1400,13 @@ async fn run_client_sync(
|
||||
.as_deref()
|
||||
.and_then(|c| {
|
||||
staging_store
|
||||
.batch(|db_tx| staging_store.find_resumable_session(db_tx, peer_pubkey, c))
|
||||
.batch(|db_tx| staging_store.find_resumable_session(db_tx, peer, c))
|
||||
.ok()
|
||||
.flatten()
|
||||
})
|
||||
.map(Ok)
|
||||
.unwrap_or_else(|| {
|
||||
staging_store
|
||||
.batch(|db_tx| staging_store.create_session(db_tx, peer_pubkey, local_node))
|
||||
staging_store.batch(|db_tx| staging_store.create_session(db_tx, peer, local_node))
|
||||
})?;
|
||||
|
||||
let mut total_staged: u32 = 0;
|
||||
@@ -1431,26 +1420,21 @@ async fn run_client_sync(
|
||||
stage_chunk(&chunk.documents, staging_store, &session_id, &chunk.cursor)?;
|
||||
|
||||
total_staged += n;
|
||||
tx.send(Ok(SyncProgress::ChunkApplied {
|
||||
let step = progress(SyncProgress::ChunkApplied {
|
||||
documents_received_so_far: total_staged,
|
||||
cursor: chunk.cursor,
|
||||
}))
|
||||
.await
|
||||
.ok();
|
||||
});
|
||||
if step.is_break() {
|
||||
return Err(DocSyncError::Canceled);
|
||||
}
|
||||
|
||||
if chunk.is_final {
|
||||
break;
|
||||
}
|
||||
}
|
||||
record_schema_gap(
|
||||
&store,
|
||||
&peer_pubkey,
|
||||
header.sync_protocol_version,
|
||||
total_skipped,
|
||||
peer_pubkey.to_hex(),
|
||||
);
|
||||
record_schema_gap(store, &peer, header.sync_protocol_version, total_skipped, peer.to_hex());
|
||||
|
||||
let total_received = promote_session(
|
||||
let received = promote_session(
|
||||
&session_id,
|
||||
staging_store,
|
||||
doc_store,
|
||||
@@ -1458,7 +1442,7 @@ async fn run_client_sync(
|
||||
node_doc_store,
|
||||
node_content_store,
|
||||
Some(&store.peer_health_store),
|
||||
peer_pubkey,
|
||||
peer,
|
||||
local_node,
|
||||
)?;
|
||||
|
||||
@@ -1467,21 +1451,14 @@ async fn run_client_sync(
|
||||
// Phase 2: serve local docs to the server (reverse pull initiated by server).
|
||||
let (mut send2, mut recv2) = wire::accept(&conn).await?;
|
||||
let reverse_req: SyncRequest = read_msg(&mut recv2).await?;
|
||||
let total_sent = serve_pull_response(&mut send2, &reverse_req, &store).await?;
|
||||
let sent = serve_pull_response(&mut send2, &reverse_req, store).await?;
|
||||
|
||||
// Wait for the server to close the connection after it finishes processing.
|
||||
// Without this, dropping conn here would send CONNECTION_CLOSE before the
|
||||
// server has read all chunks, potentially losing data mid-stream.
|
||||
conn.closed().await;
|
||||
|
||||
tx.send(Ok(SyncProgress::Complete {
|
||||
total_documents_received: total_received,
|
||||
total_documents_sent: total_sent,
|
||||
records_skipped: total_skipped,
|
||||
}))
|
||||
.await
|
||||
.ok();
|
||||
Ok(())
|
||||
Ok(SyncOutcome { received, sent, skipped: total_skipped })
|
||||
}
|
||||
|
||||
// ── Staging helpers ───────────────────────────────────────────────────────────
|
||||
@@ -2544,6 +2521,36 @@ mod tests {
|
||||
assert!(get_document(&doc_store, &doc_id).is_none());
|
||||
}
|
||||
|
||||
/// A sync abandoned mid-pull — its future dropped by a timeout, or
|
||||
/// canceled — leaves its staging session unpromoted. The next sync must
|
||||
/// still land, and must not drag the abandoned session's records in.
|
||||
#[rstest]
|
||||
fn an_abandoned_sync_leaves_the_next_one_free_to_complete(store: TestStore) {
|
||||
let (peer, local) = (node(7), node(8));
|
||||
let staging = &store.sync_staging_store;
|
||||
let session = || staging.batch(|tx| staging.create_session(tx, peer, local)).unwrap();
|
||||
let (abandoned, next) = ([1u8; 16], [2u8; 16]);
|
||||
|
||||
stage_chunk(&[make_record(abandoned, None)], staging, &session(), b"c").unwrap();
|
||||
let second = session();
|
||||
stage_chunk(&[make_record(next, None)], staging, &second, b"c").unwrap();
|
||||
let promoted = promote_session(
|
||||
&second,
|
||||
staging,
|
||||
&store.doc_store,
|
||||
&store.artifact_store,
|
||||
&store.node_doc_status_store,
|
||||
&store.node_content_status_store,
|
||||
None,
|
||||
peer,
|
||||
local,
|
||||
);
|
||||
|
||||
assert_eq!(promoted.unwrap(), 1);
|
||||
assert!(get_document(&store.doc_store, &DocumentID::from_uuid_bytes(next)).is_some());
|
||||
assert!(get_document(&store.doc_store, &DocumentID::from_uuid_bytes(abandoned)).is_none());
|
||||
}
|
||||
|
||||
// ── Edges, and property removal ──────────────────────────────────────────
|
||||
|
||||
/// A store quartet plus the node key, for the record-application tests below.
|
||||
|
||||
@@ -262,7 +262,7 @@ impl NodeAdminHandler {
|
||||
// via the shared `BackgroundSync` job either way.
|
||||
let summary = scheduler::run_all_passes(
|
||||
&endpoint,
|
||||
store,
|
||||
&store,
|
||||
&job_runner,
|
||||
local_node,
|
||||
JobOrigin::Scheduler,
|
||||
|
||||
@@ -33,12 +33,11 @@
|
||||
//! invasive than a timer, so every driver waits for its next tick instead.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::ops::ControlFlow;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::p2p::admission::EnsembleEndpoint;
|
||||
use log::{info, warn};
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
use crate::content_ledger::content_ledger;
|
||||
use crate::documents::{ArtifactKind, BlobHash, DocumentID};
|
||||
@@ -81,21 +80,20 @@ pub struct SchedulerPassResult {
|
||||
/// the same peers and stores.
|
||||
pub async fn run_all_passes(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
job_runner: &JobRunner,
|
||||
local_node: NodePubkey,
|
||||
origin: JobOrigin,
|
||||
) -> Option<SchedulerPassResult> {
|
||||
let handle =
|
||||
job_runner.start(JobKind::BackgroundSync, JobSubject::None, "Background sync", origin)?;
|
||||
let peers = ensemble_peers_freshest_first(&store);
|
||||
let peers = ensemble_peers_freshest_first(store);
|
||||
let started = Instant::now();
|
||||
let catalog =
|
||||
run_catalog_pass(endpoint, Arc::clone(&store), job_runner, local_node, &peers).await;
|
||||
let catalog = run_catalog_pass(endpoint, store, job_runner, local_node, &peers).await;
|
||||
let after_catalog = started.elapsed();
|
||||
let catalog_fetch = run_catalog_fetch_pass(endpoint, &store, &peers).await;
|
||||
let catalog_fetch = run_catalog_fetch_pass(endpoint, store, &peers).await;
|
||||
let after_fetch = started.elapsed();
|
||||
let content = run_content_pass(endpoint, Arc::clone(&store), job_runner, &peers).await;
|
||||
let content = run_content_pass(endpoint, store, job_runner, &peers).await;
|
||||
info!(
|
||||
"scheduler: pass over {} peer(s) took {:.1}s (catalog {:.1}s, \
|
||||
catalog-fetch {:.1}s, content {:.1}s)",
|
||||
@@ -124,15 +122,15 @@ pub async fn run_all_passes(
|
||||
/// back to reporting only the catalog pass it already ran.
|
||||
pub async fn run_content_passes_for_peer(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
job_runner: &JobRunner,
|
||||
peer: NodePubkey,
|
||||
) -> Option<(CatalogFetchPassResult, ContentPassResult)> {
|
||||
let peer_node =
|
||||
ensemble_peers_freshest_first(&store).into_iter().find(|p| p.node_pubkey == peer.0)?;
|
||||
ensemble_peers_freshest_first(store).into_iter().find(|p| p.node_pubkey == peer.0)?;
|
||||
let peers = [peer_node];
|
||||
let catalog_fetch = run_catalog_fetch_pass(endpoint, &store, &peers).await;
|
||||
let content = run_content_pass(endpoint, Arc::clone(&store), job_runner, &peers).await;
|
||||
let catalog_fetch = run_catalog_fetch_pass(endpoint, store, &peers).await;
|
||||
let content = run_content_pass(endpoint, store, job_runner, &peers).await;
|
||||
Some((catalog_fetch, content))
|
||||
}
|
||||
|
||||
@@ -251,7 +249,7 @@ impl RemoteReconcile {
|
||||
/// instruction, and the content passes stay in one place.
|
||||
pub async fn sync_with_peer(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
job_runner: &JobRunner,
|
||||
local_node: NodePubkey,
|
||||
peer: NodePubkey,
|
||||
@@ -265,7 +263,7 @@ pub async fn sync_with_peer(
|
||||
return Err("that is this node — pick another ensemble member".to_string());
|
||||
}
|
||||
let Some(peer_node) =
|
||||
ensemble_peers_freshest_first(&store).into_iter().find(|p| p.node_pubkey == peer.0)
|
||||
ensemble_peers_freshest_first(store).into_iter().find(|p| p.node_pubkey == peer.0)
|
||||
else {
|
||||
return Err(format!("{} is not a member of this node's ensemble", peer.to_hex()));
|
||||
};
|
||||
@@ -311,7 +309,7 @@ pub async fn sync_with_peer(
|
||||
let started = Instant::now();
|
||||
let outcome = tokio::time::timeout(
|
||||
PEER_CATALOG_TIMEOUT,
|
||||
drive_catalog_sync(endpoint, peer, request, Arc::clone(&store), local_node, &handle),
|
||||
drive_catalog_sync(endpoint, peer, request, store, local_node, &handle),
|
||||
)
|
||||
.await;
|
||||
let catalog_took = started.elapsed();
|
||||
@@ -498,7 +496,7 @@ pub struct CatalogPassResult {
|
||||
/// starting a second concurrent sync.
|
||||
pub async fn run_catalog_pass(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
job_runner: &JobRunner,
|
||||
local_node: NodePubkey,
|
||||
peers: &[PeerNode],
|
||||
@@ -534,14 +532,7 @@ pub async fn run_catalog_pass(
|
||||
let peer_started = Instant::now();
|
||||
let outcome = tokio::time::timeout(
|
||||
PEER_CATALOG_TIMEOUT,
|
||||
drive_catalog_sync(
|
||||
endpoint,
|
||||
node_pubkey,
|
||||
request,
|
||||
Arc::clone(&store),
|
||||
local_node,
|
||||
&handle,
|
||||
),
|
||||
drive_catalog_sync(endpoint, node_pubkey, request, store, local_node, &handle),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -587,68 +578,49 @@ pub async fn run_catalog_pass(
|
||||
}
|
||||
|
||||
/// Drives one catalog exchange to completion, reporting progress onto
|
||||
/// `handle`. Returns `(documents received + sent, records skipped)` — the
|
||||
/// scheduler passes ignore both and only care that it succeeded, while
|
||||
/// [`sync_with_peer`] reports them back to whoever pressed the button.
|
||||
/// `handle` and stopping at the next chunk once it is canceled. Returns
|
||||
/// `(documents received + sent, records skipped)` — the scheduler passes
|
||||
/// ignore both and only care that it succeeded, while [`sync_with_peer`]
|
||||
/// reports them back to whoever pressed the button.
|
||||
async fn drive_catalog_sync(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
peer: NodePubkey,
|
||||
request: SyncRequest,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
local_node: NodePubkey,
|
||||
handle: &JobHandle,
|
||||
) -> Result<(u32, u32), String> {
|
||||
let mut stream = sync_from_peer(endpoint, peer, request, store, local_node)
|
||||
let progress = |event| {
|
||||
handle.report_progress(match event {
|
||||
SyncProgress::Started { total_documents, total_blob_bytes, .. } => JobProgress {
|
||||
units_done: 0,
|
||||
units_total: Some(total_documents),
|
||||
bytes_done: 0,
|
||||
bytes_total: Some(total_blob_bytes),
|
||||
message: Some(format!("0/{total_documents} documents")),
|
||||
},
|
||||
SyncProgress::ChunkApplied { documents_received_so_far, .. } => JobProgress {
|
||||
units_done: documents_received_so_far,
|
||||
units_total: None,
|
||||
bytes_done: 0,
|
||||
bytes_total: None,
|
||||
message: Some(format!("{documents_received_so_far} documents")),
|
||||
},
|
||||
});
|
||||
if handle.is_canceled() { ControlFlow::Break(()) } else { ControlFlow::Continue(()) }
|
||||
};
|
||||
let outcome = sync_from_peer(endpoint, peer, request, store, local_node, progress)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let (mut documents, mut skipped) = (0u32, 0u32);
|
||||
while let Some(item) = stream.next().await {
|
||||
match item.map_err(|e| e.to_string())? {
|
||||
SyncProgress::Started { total_documents, total_blob_bytes, .. } => {
|
||||
handle.report_progress(JobProgress {
|
||||
units_done: 0,
|
||||
units_total: Some(total_documents),
|
||||
bytes_done: 0,
|
||||
bytes_total: Some(total_blob_bytes),
|
||||
message: Some(format!("0/{total_documents} documents")),
|
||||
});
|
||||
}
|
||||
SyncProgress::ChunkApplied { documents_received_so_far, .. } => {
|
||||
handle.report_progress(JobProgress {
|
||||
units_done: documents_received_so_far,
|
||||
units_total: None,
|
||||
bytes_done: 0,
|
||||
bytes_total: None,
|
||||
message: Some(format!("{documents_received_so_far} documents")),
|
||||
});
|
||||
}
|
||||
SyncProgress::Complete {
|
||||
total_documents_received,
|
||||
total_documents_sent,
|
||||
records_skipped,
|
||||
} => {
|
||||
documents = total_documents_received + total_documents_sent;
|
||||
skipped = records_skipped;
|
||||
if records_skipped > 0 {
|
||||
warn!(
|
||||
"scheduler: catalog sync with {}: {records_skipped} document(s) \
|
||||
skipped (peer likely on a newer build) — upgrade to pick them up",
|
||||
peer.to_hex()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Best-effort: sync_from_peer has no per-chunk cancel hook, so this
|
||||
// can only stop between yielded stream items, not mid-chunk. The
|
||||
// *content* plane no longer has that limitation — a blob transfer
|
||||
// reads the token per chunk — so the catalog plane is now the coarse
|
||||
// half, and `asset_sync::refresh_document_catalog` carries the same
|
||||
// note for the on-demand version of this exchange.
|
||||
if handle.is_canceled() {
|
||||
break;
|
||||
}
|
||||
if outcome.skipped > 0 {
|
||||
warn!(
|
||||
"scheduler: catalog sync with {}: {} document(s) skipped (peer likely on a newer \
|
||||
build) — upgrade to pick them up",
|
||||
peer.to_hex(),
|
||||
outcome.skipped
|
||||
);
|
||||
}
|
||||
Ok((documents, skipped))
|
||||
Ok((outcome.received + outcome.sent, outcome.skipped))
|
||||
}
|
||||
|
||||
// ── Pass 2: catalog-plane fetch ────────────────────────────────────────────
|
||||
@@ -778,13 +750,13 @@ pub struct ContentPassResult {
|
||||
/// the document-detail screen).
|
||||
pub async fn run_content_pass(
|
||||
endpoint: &EnsembleEndpoint,
|
||||
store: Arc<Store>,
|
||||
store: &Store,
|
||||
job_runner: &JobRunner,
|
||||
peers: &[PeerNode],
|
||||
) -> ContentPassResult {
|
||||
let mut result = ContentPassResult::default();
|
||||
|
||||
let wants = match content_ledger(&store) {
|
||||
let wants = match content_ledger(store) {
|
||||
Ok(ledger) => ledger.wants,
|
||||
Err(e) => {
|
||||
warn!("scheduler: failed to compute the content ledger: {e}");
|
||||
@@ -812,7 +784,7 @@ pub async fn run_content_pass(
|
||||
continue;
|
||||
};
|
||||
|
||||
let targets = peer_order_by_blob_presence(&store, peers, &hashes);
|
||||
let targets = peer_order_by_blob_presence(store, peers, &hashes);
|
||||
let mut missing = hashes;
|
||||
let mut received = 0u32;
|
||||
|
||||
@@ -824,7 +796,7 @@ pub async fn run_content_pass(
|
||||
// here, so a cancel stops the download that is running rather than
|
||||
// waiting for it to finish and declining to start the next one.
|
||||
let fetch =
|
||||
blob_sync::fetch_blobs_from_peer(endpoint, target, &missing, &store, Some(&handle));
|
||||
blob_sync::fetch_blobs_from_peer(endpoint, target, &missing, store, Some(&handle));
|
||||
match tokio::time::timeout(PEER_BLOB_TIMEOUT, fetch).await {
|
||||
Ok(Ok(r)) => received += r.received,
|
||||
Ok(Err(blob_sync::BlobSyncError::Canceled)) => break,
|
||||
|
||||
@@ -1805,7 +1805,7 @@ pub async fn dispatch_ensemble_sync(
|
||||
|
||||
let summary = scheduler::sync_with_peer(
|
||||
&p2p.endpoint(),
|
||||
Arc::clone(store),
|
||||
store,
|
||||
job_runner,
|
||||
local_node,
|
||||
peer,
|
||||
@@ -1998,7 +1998,7 @@ pub async fn dispatch_sync_assets(
|
||||
|reporter| async move {
|
||||
synchronicity::p2p::asset_sync::sync_document_assets_recursive(
|
||||
&p2p.endpoint(),
|
||||
std::sync::Arc::clone(p2p.store()),
|
||||
p2p.store(),
|
||||
id,
|
||||
&peer_ids,
|
||||
local_node,
|
||||
|
||||
@@ -51,7 +51,7 @@ pub fn spawn(runtime: &NodeRuntime, job_runner: Arc<JobRunner>) {
|
||||
Ok(Some(local_node)) => {
|
||||
let summary = scheduler::run_all_passes(
|
||||
&endpoint,
|
||||
Arc::clone(&store),
|
||||
&store,
|
||||
&job_runner,
|
||||
local_node,
|
||||
JobOrigin::Scheduler,
|
||||
|
||||
Reference in new issue
Block a user