Add resilience tests

src/origins/network/watcher.rs

@@ -17,7 +17,8 @@ use crate::origins::{FileWatcher, WatcherHandle};
 use crate::proto::ManifestEntry as ProtoManifestEntry;
 use tracing::{info, warn};
 
-const POLL_FALLBACK_INTERVAL: Duration = Duration::from_secs(30);
+const INITIAL_RETRY_DELAY: Duration = Duration::from_secs(1);
+const MAX_RETRY_DELAY: Duration = Duration::from_secs(30);
 
 pub struct NetworkOriginFileWatcher {
     transport: NetworkTransport,
@@ -84,6 +85,7 @@ struct WatcherState {
 
 impl WatcherState {
     async fn run_loop(self, shutdown: Arc<Notify>) {
+        let mut retry_delay = INITIAL_RETRY_DELAY;
         loop {
             let subscribed = tokio::select! {
                 biased;
@@ -93,6 +95,7 @@ impl WatcherState {
             match subscribed {
                 Ok(mut stream) => {
                     info!("network watcher: subscribed to /events");
+                    retry_delay = INITIAL_RETRY_DELAY;
                     loop {
                         let item = tokio::select! {
                             biased;
@@ -120,8 +123,9 @@ impl WatcherState {
             tokio::select! {
                 biased;
                 _ = shutdown.notified() => return,
-                _ = tokio::time::sleep(POLL_FALLBACK_INTERVAL) => {}
+                _ = tokio::time::sleep(retry_delay) => {}
             }
+            retry_delay = (retry_delay * 2).min(MAX_RETRY_DELAY);
             if let Err(e) = self.reconcile_once().await {
                 warn!(error = %e, "network watcher: poll reconcile failed");
             }