Add a partition test.

kvraft/src/testing_utils/config.rs

@@ -76,7 +76,7 @@ impl Config {
         eprintln!("{}", msg);
     }
 
-    fn make_partition(&self) -> (Vec<usize>, Vec<usize>) {
+    pub fn make_partition(&self) -> (Vec<usize>, Vec<usize>) {
         let state = self.state.lock();
         let mut indexes: Vec<usize> = (0..state.kv_servers.len()).collect();
         indexes.shuffle(&mut thread_rng());
@@ -116,6 +116,12 @@ impl Config {
         Self::set_connect(&mut network, part_two, part_two, true);
     }
 
+    pub fn connect_all(&self) {
+        let all: Vec<usize> = (0..self.state.lock().kv_servers.len()).collect();
+        let mut network = self.network.lock();
+        Self::set_connect(&mut network, &all, &all, true);
+    }
+
     fn set_clerk_connect(
         network: &mut Network,
         clerk_index: usize,
@@ -162,6 +168,14 @@ impl Config {
         self.make_limited_clerk(&(0..self.server_count).collect::<Vec<usize>>())
     }
 
+    pub fn connect_all_clerks(&self) {
+        let mut network = self.network.lock();
+        let all = &(0..self.server_count).collect::<Vec<usize>>();
+        for clerk_index in 0..self.state.lock().next_clerk {
+            Self::set_clerk_connect(&mut network, clerk_index + 1, all, true);
+        }
+    }
+
     pub fn end(&self) {}
 
     pub fn clean_up(&self) {
@@ -222,3 +236,12 @@ pub fn make_config(
 
     cfg
 }
+
+pub fn sleep_millis(mills: u64) {
+    std::thread::sleep(std::time::Duration::from_millis(mills))
+}
+
+pub const LONG_ELECTION_TIMEOUT_MILLIS: u64 = 1000;
+pub fn sleep_election_timeouts(count: u64) {
+    sleep_millis(LONG_ELECTION_TIMEOUT_MILLIS * count)
+}

kvraft/tests/service_test.rs

@@ -5,7 +5,7 @@ extern crate rand;
 #[macro_use]
 extern crate scopeguard;
 
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::thread::JoinHandle;
 use std::time::Duration;
@@ -13,7 +13,9 @@ use std::time::Duration;
 use rand::{thread_rng, Rng};
 
 use anyhow::Context;
-use kvraft::testing_utils::config::{make_config, Config};
+use kvraft::testing_utils::config::{
+    make_config, sleep_election_timeouts, sleep_millis, Config,
+};
 use kvraft::Clerk;
 
 fn spawn_clients<T, Func>(
@@ -207,3 +209,67 @@ fn unreliable_one_key_many_clients() -> anyhow::Result<()> {
 
     check_concurrent_results(value, CLIENTS, vec![ATTEMPTS; CLIENTS])
 }
+
+#[test]
+fn one_partition() -> anyhow::Result<()> {
+    const SERVERS: usize = 5;
+    let cfg = Arc::new(make_config(SERVERS, false, 0));
+
+    cfg.begin("Test: progress in majority (3A)");
+
+    const KEY: &str = "1";
+    let mut clerk = cfg.make_clerk();
+    clerk.put(KEY, "13");
+
+    let (majority, minority) = cfg.make_partition();
+
+    assert!(minority.len() < majority.len());
+    assert_eq!(minority.len() + majority.len(), SERVERS);
+    cfg.partition(&majority, &minority);
+
+    let mut clerk_majority = cfg.make_limited_clerk(&majority);
+    let mut clerk_minority1 = cfg.make_limited_clerk(&minority);
+    let mut clerk_minority2 = cfg.make_limited_clerk(&minority);
+
+    clerk_majority.put(KEY, "14");
+    assert_eq!(clerk_majority.get(KEY), Some("14".to_owned()));
+
+    cfg.begin("Test: no progress in minority (3A)");
+    let counter = Arc::new(AtomicUsize::new(0));
+    let counter1 = counter.clone();
+    std::thread::spawn(move || {
+        clerk_minority1.put(KEY, "15");
+        counter1.fetch_or(1, Ordering::SeqCst);
+    });
+    let counter2 = counter.clone();
+    std::thread::spawn(move || {
+        clerk_minority2.get(KEY);
+        counter2.fetch_or(2, Ordering::SeqCst);
+    });
+
+    sleep_millis(1000);
+
+    assert_eq!(counter.load(Ordering::SeqCst), 0);
+
+    assert_eq!(clerk_majority.get(KEY), Some("14".to_owned()));
+    clerk_majority.put(KEY, "16");
+    assert_eq!(clerk_majority.get(KEY), Some("16".to_owned()));
+
+    cfg.begin("Test: completion after heal (3A)");
+
+    cfg.connect_all();
+    cfg.connect_all_clerks();
+
+    sleep_election_timeouts(1);
+    for _ in 0..100 {
+        sleep_millis(60);
+        if counter.load(Ordering::SeqCst) == 3 {
+            break;
+        }
+    }
+
+    assert_eq!(counter.load(Ordering::SeqCst), 3);
+    assert_eq!(clerk.get(KEY), Some("15".to_owned()));
+
+    Ok(())
+}