|
|
@@ -6,6 +6,7 @@ use parking_lot::Mutex;
|
|
|
use crate::index_term::IndexTerm;
|
|
|
use crate::{Peer, RaftState, State, Term};
|
|
|
|
|
|
+/// A convenient macro to record errors.
|
|
|
#[macro_export]
|
|
|
macro_rules! check_or_record {
|
|
|
($condition:expr, $error_kind:expr, $message:expr, $rf:expr) => {
|
|
|
@@ -20,6 +21,12 @@ macro_rules! check_or_record {
|
|
|
};
|
|
|
}
|
|
|
|
|
|
+/// Environment for daemons.
|
|
|
+///
|
|
|
+/// Each daemon thread should hold a copy of this struct, either directly or
|
|
|
+/// through a copy of [`crate::Raft`]. It can be used for logging unexpected
|
|
|
+/// errors to a central location, which cause a failure at shutdown. It also
|
|
|
+/// checks daemon thread panics and collect information if they do.
|
|
|
#[derive(Clone, Debug)]
|
|
|
pub(crate) struct DaemonEnv {
|
|
|
data: Arc<Mutex<DaemonEnvData>>,
|
|
|
@@ -42,12 +49,21 @@ pub(crate) struct Error {
|
|
|
|
|
|
#[derive(Debug)]
|
|
|
pub(crate) enum ErrorKind {
|
|
|
+ /// The leader sent log entries that do not match a committed log entry.
|
|
|
+ /// It could be caused by a misbehaving leader, or that the leader should
|
|
|
+ /// never have been elected, or that this Raft instance incorrectly moved
|
|
|
+ /// its commit index.
|
|
|
RollbackCommitted(usize),
|
|
|
+ /// Similar to [`Self::RollbackCommitted`], but the leader sent a snapshot
|
|
|
+ /// that is inconsistent with a committed log entry.
|
|
|
SnapshotBeforeCommitted(usize, Term),
|
|
|
+ /// The application sent a snapshot that contains items beyond the log end.
|
|
|
SnapshotAfterLogEnd(usize),
|
|
|
}
|
|
|
|
|
|
impl DaemonEnv {
|
|
|
+ /// Record an error, with a stripped version of the state of this instance.
|
|
|
+ /// Use macro `check_or_record` to auto populate the environment.
|
|
|
pub fn record_error<T, S: AsRef<str>>(
|
|
|
&self,
|
|
|
error_kind: ErrorKind,
|
|
|
@@ -63,10 +79,14 @@ impl DaemonEnv {
|
|
|
})
|
|
|
}
|
|
|
|
|
|
+ /// Register a daemon thread to make sure it is correctly shutdown when the
|
|
|
+ /// Raft instance is killed.
|
|
|
pub fn watch_daemon(&self, thread: std::thread::JoinHandle<()>) {
|
|
|
self.data.lock().daemons.push(thread);
|
|
|
}
|
|
|
|
|
|
+ /// Makes sure that all daemons have been shutdown, no more errors can be
|
|
|
+ /// added, checks if any error has been added, or if any daemon panicked.
|
|
|
pub fn shutdown(self) {
|
|
|
let data = Arc::try_unwrap(self.data)
|
|
|
.unwrap_or_else(|_| {
|
|
|
@@ -138,6 +158,10 @@ struct StrippedRaftState {
|
|
|
}
|
|
|
|
|
|
impl DaemonEnv {
|
|
|
+ /// Creates a daemon environment. Each Raft instance should share the same
|
|
|
+ /// environment. It should be added to any thread that executes Raft code.
|
|
|
+ /// Use [`DaemonEnv::for_thread`] or [`DaemonEnv::for_scope`] to register
|
|
|
+ /// the environment.
|
|
|
pub(crate) fn create() -> Self {
|
|
|
let data = Default::default();
|
|
|
// Pre-create a template thread_env, so that we can clone the weak
|
|
|
@@ -148,16 +172,25 @@ impl DaemonEnv {
|
|
|
Self { data, thread_env }
|
|
|
}
|
|
|
|
|
|
+ /// Creates a [`ThreadEnv`] that could be attached to a thread. Any code
|
|
|
+ /// running in the thread can use this `DaemonEnv` to log errors. The thread
|
|
|
+ /// must be stopped before `DaemonEnv::shutdown()` is called, otherwise it
|
|
|
+ /// will panic when logging an error.
|
|
|
pub(crate) fn for_thread(&self) -> ThreadEnv {
|
|
|
self.thread_env.clone()
|
|
|
}
|
|
|
|
|
|
+ /// Creates a [`ThreadEnvGuard`] that registers this `DaemonEnv` in the
|
|
|
+ /// current scope, which also remove it from the scope when dropped.
|
|
|
pub(crate) fn for_scope(&self) -> ThreadEnvGuard {
|
|
|
self.for_thread().attach();
|
|
|
ThreadEnvGuard {}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+/// A weak reference to a [`DaemonEnv`] that is attached to a thread.
|
|
|
+/// Use [`ThreadEnv::attach`] to consume this instance and attach to a thread,
|
|
|
+/// and [`ThreadEnv::detach`] to undo that.
|
|
|
#[derive(Clone, Debug, Default)]
|
|
|
pub(crate) struct ThreadEnv {
|
|
|
data: Weak<Mutex<DaemonEnvData>>,
|
|
|
@@ -166,6 +199,7 @@ pub(crate) struct ThreadEnv {
|
|
|
impl ThreadEnv {
|
|
|
thread_local! {static ENV: RefCell<ThreadEnv> = Default::default()}
|
|
|
|
|
|
+ /// Upgrade to the referenced [`DaemonEvn`].
|
|
|
// The dance between Arc<> and Weak<> is complex, but useful:
|
|
|
// 1) We do not have to worry about slow RPC threads causing
|
|
|
// DaemonEnv::shutdown() to fail. They only hold a Weak<> pointer after all;
|
|
|
@@ -190,15 +224,19 @@ impl ThreadEnv {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /// Attach this instance to the current thread.
|
|
|
pub fn attach(self) {
|
|
|
Self::ENV.with(|env| env.replace(self));
|
|
|
}
|
|
|
|
|
|
+ /// Detach the instance stored in the current thread.
|
|
|
pub fn detach() {
|
|
|
Self::ENV.with(|env| env.replace(Default::default()));
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+/// A guard that automatically cleans up the [`ThreadEnv`] attached to the
|
|
|
+/// current thread when dropped. It does not restore the previous value.
|
|
|
pub(crate) struct ThreadEnvGuard {}
|
|
|
|
|
|
impl Drop for ThreadEnvGuard {
|