Add crash to tests.

kvraft/src/testing_utils/config.rs

@@ -128,6 +128,41 @@ impl Config {
         Self::set_connect(&mut network, &all, &all, true);
     }
 
+    fn crash_server(&self, index: usize) {
+        {
+            let all: Vec<usize> = (0..self.server_count).collect();
+
+            let mut network = self.network.lock();
+            Self::set_connect(&mut network, &all, &[index], false);
+            Self::set_connect(&mut network, &[index], &all, false);
+
+            network.remove_server(Self::server_name(index));
+            network.remove_server(Self::kv_server_name(index));
+        }
+
+        let data = self.storage.lock().at(index).read();
+
+        let persister = self.storage.lock().replace(index);
+        persister.restore(data);
+
+        if let Some(kv_server) = self.state.lock().kv_servers[index].take() {
+            kv_server.kill();
+        }
+    }
+
+    pub fn crash_all(&self) {
+        for i in 0..self.server_count {
+            self.crash_server(i);
+        }
+    }
+
+    pub fn restart_all(&self) {
+        for index in 0..self.server_count {
+            self.start_server(index)
+                .expect("Start server should never fail");
+        }
+    }
+
     fn set_clerk_connect(
         network: &mut Network,
         clerk_index: usize,

kvraft/src/testing_utils/memory_persister.rs

@@ -2,6 +2,7 @@ use std::sync::Arc;
 
 use parking_lot::Mutex;
 
+#[derive(Clone)]
 pub struct State {
     bytes: bytes::Bytes,
     snapshot: Vec<u8>,
@@ -42,6 +43,16 @@ impl ruaft::Persister for MemoryPersister {
     }
 }
 
+impl MemoryPersister {
+    pub fn read(&self) -> State {
+        self.state.lock().clone()
+    }
+
+    pub fn restore(&self, state: State) {
+        *self.state.lock() = state;
+    }
+}
+
 #[derive(Default)]
 pub struct MemoryStorage {
     state_vec: Vec<Arc<MemoryPersister>>,
@@ -57,4 +68,10 @@ impl MemoryStorage {
     pub fn at(&self, index: usize) -> Arc<MemoryPersister> {
         self.state_vec[index].clone()
     }
+
+    pub fn replace(&mut self, index: usize) -> Arc<MemoryPersister> {
+        let persister = Arc::new(MemoryPersister::new());
+        self.state_vec[index] = persister.clone();
+        persister
+    }
 }

kvraft/tests/service_test.rs

@@ -91,6 +91,7 @@ struct GenericTestParams {
     clients: usize,
     unreliable: bool,
     partition: bool,
+    crash: bool,
     maxraftstate: Option<usize>,
 }
 
@@ -99,6 +100,7 @@ fn generic_test(test_params: GenericTestParams) {
         clients,
         unreliable,
         partition,
+        crash,
         maxraftstate,
     } = test_params;
     let maxraftstate = maxraftstate.unwrap_or(usize::MAX);
@@ -134,6 +136,12 @@ fn generic_test(test_params: GenericTestParams) {
             None
         };
 
+        if crash {
+            cfg.crash_all();
+            sleep_election_timeouts(1);
+            cfg.restart_all();
+        }
+
         std::thread::sleep(Duration::from_secs(5));
 
         // Stop partitions.
@@ -350,3 +358,12 @@ fn many_partitions_many_client() {
         ..Default::default()
     });
 }
+
+#[test]
+fn persist_one_client() {
+    generic_test(GenericTestParams {
+        clients: 1,
+        crash: true,
+        ..Default::default()
+    });
+}