diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index aa0df78082..e8459d8caa 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1819,12 +1819,16 @@ impl Vmm { mem_send, postponed_lifecycle_event, return_if_cancelled_cb, - )?; + ); + let downtime_begin = Instant::now(); // End throttle thread - info!("stopping vcpu thread"); - vm.stop_vcpu_throttling(); - info!("stopped vcpu thread"); + info!("stopping vcpu throttling"); + vm.reset_vcpu_throttle_thread(); + info!("stopped vcpu throttling"); + + let remaining = remaining?; + info!("pausing VM"); vm.pause()?; info!("paused VM"); diff --git a/vmm/src/vcpu_throttling.rs b/vmm/src/vcpu_throttling.rs index e8fd0d3b12..840fa06e67 100644 --- a/vmm/src/vcpu_throttling.rs +++ b/vmm/src/vcpu_throttling.rs @@ -30,28 +30,31 @@ use std::cell::Cell; use std::cmp::min; -use std::sync::mpsc::RecvTimeoutError; +use std::sync::mpsc::{RecvTimeoutError, SyncSender}; use std::sync::{Arc, Mutex, mpsc}; use std::thread; use std::thread::JoinHandle; use std::time::{Duration, Instant}; -use log::{debug, warn}; +use log::{debug, error, info, warn}; use vm_migration::Pausable; use crate::cpu::CpuManager; /// The possible command of the thread, i.e., the current state. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug)] enum ThrottleCommand { - /// Waiting for next event. - Waiting, - /// Ongoing vCPU throttling. + /// Exit the throttle loop and wait for next event. + Wait, + /// Throttle the vCPUs with the given throttle percentage in the range `1..=99` per time slice. + Throttle(u8 /* `1..=99` */), + /// Gracefully shutdown the vCPU throttling thread. + Exit, + /// Exit the throttle loop then rendezvous with the receiver before proceeding to wait for the next command. /// - /// The inner value shows the current throttling percentage in range `1..=99`. - Throttling(u8 /* `1..=99` */), - /// Thread is shutting down gracefully. - Exiting, + /// In other words the `report` is used to synchronize the throttle thread reset event with the thread that + /// sent this command. + Reset { report: SyncSender<()> }, } /// Helper to adapt the throttling timeslice as we go, depending on the time it @@ -252,14 +255,15 @@ impl ThrottleWorker { ); match maybe_task { None => None, - Some(ThrottleCommand::Throttling(next)) => { + Some(ThrottleCommand::Throttle(next)) => { // A new throttle value is only applied at the end of a full // throttling cycle. This is fine and negligible in a series of // (tens of) thousands of cycles. *current_throttle = next as u64; None } - Some(cmd @ (ThrottleCommand::Exiting | ThrottleCommand::Waiting)) => Some(cmd), + Some(cmd @ (ThrottleCommand::Exit | ThrottleCommand::Wait)) => Some(cmd), + Some(ThrottleCommand::Reset { report }) => Some(ThrottleCommand::Reset { report }), } } @@ -334,23 +338,44 @@ impl ThrottleWorker { 'control: loop { let thread_task = receiver.recv().expect("channel should not be closed"); match thread_task { - ThrottleCommand::Exiting => { + ThrottleCommand::Exit => { break 'control; } - ThrottleCommand::Waiting => { + ThrottleCommand::Wait => { continue 'control; } - ThrottleCommand::Throttling(initial_throttle) => { + ThrottleCommand::Throttle(initial_throttle) => { let next_task = Self::throttle_loop( &receiver, initial_throttle, &callback_pause_vcpus, &callback_resume_vcpus, ); - if next_task == ThrottleCommand::Exiting { - break 'control; + match next_task { + ThrottleCommand::Exit => { + break 'control; + } + // else: thread needs to go into waiting state + ThrottleCommand::Reset { report } => { + // Inform sender that we are back in the waiting state: Since `report` has capacity 0 + // this call will block until the command sender has received our message. + if let Err(e) = report.send(()) { + error!( + "Unable to synchronize throttle thread reset event: error = {e:#?}" + ); + } + } + _ => { + continue 'control; + } + } + } + ThrottleCommand::Reset { report } => { + if let Err(e) = report.send(()) { + error!( + "Unable to synchronize throttle thread reset event: error = {e:#?}" + ); } - // else: thread is in Waiting state } } } @@ -483,11 +508,11 @@ impl ThrottleThreadHandle { if percent_new == 0 { self.state_sender - .send(ThrottleCommand::Waiting) + .send(ThrottleCommand::Wait) .expect("channel should not be closed"); } else { self.state_sender - .send(ThrottleCommand::Throttling(percent_new)) + .send(ThrottleCommand::Throttle(percent_new)) .expect("channel should not be closed"); } @@ -513,7 +538,7 @@ impl ThrottleThreadHandle { // drop thread; ensure that the channel is still alive when it is dropped if let Some(worker) = self.throttle_thread.take() { self.state_sender - .send(ThrottleCommand::Exiting) + .send(ThrottleCommand::Exit) .expect("channel should not be closed"); // Ensure the sender is still living when this is dropped. @@ -529,6 +554,19 @@ impl ThrottleThreadHandle { ); } } + + /// Stops throttling and returns the throttle thread to the waiting state. + /// + /// This blocks until the throttling thread has exited the throttling loop. + pub fn reset(&self) { + let (report, recv) = mpsc::sync_channel(0); + self.state_sender + .send(ThrottleCommand::Reset { report }) + .expect("channel should not be closed"); + self.current_throttle.set(0); + info!("Waiting for throttle thread to acknowledge reset"); + recv.recv().expect("The throttle thread should acknowledge the reset event before dropping rendezvous channel"); + } } impl Drop for ThrottleThreadHandle { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 2f73adaccc..ced2bc0c2e 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -1387,11 +1387,11 @@ impl Vm { self.vcpu_throttler.throttle_percent() } - /// Stops and terminates the thread gracefully. + /// Sets the vCPU throttling thread back to its initial waiting state. /// - /// Waits for the thread to finish. - pub fn stop_vcpu_throttling(&mut self) { - self.vcpu_throttler.shutdown(); + /// Blocks until the throttling thread acknowledges the reset event. + pub fn reset_vcpu_throttle_thread(&self) { + self.vcpu_throttler.reset(); } pub fn set_post_migration_lifecycle_event(