Fix auto-converge on live migration retry

vmm/src/lib.rs

@@ -1819,12 +1819,16 @@ impl Vmm {
             mem_send,
             postponed_lifecycle_event,
             return_if_cancelled_cb,
-        )?;
+        );
+
         let downtime_begin = Instant::now();
         // End throttle thread
-        info!("stopping vcpu thread");
-        vm.stop_vcpu_throttling();
-        info!("stopped vcpu thread");
+        info!("stopping vcpu throttling");
+        vm.reset_vcpu_throttle_thread();
+        info!("stopped vcpu throttling");
+
+        let remaining = remaining?;
+
         info!("pausing VM");
         vm.pause()?;
         info!("paused VM");

vmm/src/vcpu_throttling.rs

@@ -30,28 +30,31 @@
 
 use std::cell::Cell;
 use std::cmp::min;
-use std::sync::mpsc::RecvTimeoutError;
+use std::sync::mpsc::{RecvTimeoutError, SyncSender};
 use std::sync::{Arc, Mutex, mpsc};
 use std::thread;
 use std::thread::JoinHandle;
 use std::time::{Duration, Instant};
 
-use log::{debug, warn};
+use log::{debug, error, info, warn};
 use vm_migration::Pausable;
 
 use crate::cpu::CpuManager;
 
 /// The possible command of the thread, i.e., the current state.
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[derive(Debug)]
 enum ThrottleCommand {
-    /// Waiting for next event.
-    Waiting,
-    /// Ongoing vCPU throttling.
+    /// Exit the throttle loop and wait for next event.
+    Wait,
+    /// Throttle the vCPUs with the given throttle percentage in the range `1..=99` per time slice.
+    Throttle(u8 /* `1..=99` */),
+    /// Gracefully shutdown the vCPU throttling thread.
+    Exit,
+    /// Exit the throttle loop then rendezvous with the receiver before proceeding to wait for the next command.
     ///
-    /// The inner value shows the current throttling percentage in range `1..=99`.
-    Throttling(u8 /* `1..=99` */),
-    /// Thread is shutting down gracefully.
-    Exiting,
+    /// In other words the `report` is used to synchronize the throttle thread reset event with the thread that
+    /// sent this command.
+    Reset { report: SyncSender<()> },
 }
 
 /// Helper to adapt the throttling timeslice as we go, depending on the time it
@@ -252,14 +255,15 @@ impl ThrottleWorker {
         );
         match maybe_task {
             None => None,
-            Some(ThrottleCommand::Throttling(next)) => {
+            Some(ThrottleCommand::Throttle(next)) => {
                 // A new throttle value is only applied at the end of a full
                 // throttling cycle. This is fine and negligible in a series of
                 // (tens of) thousands of cycles.
                 *current_throttle = next as u64;
                 None
             }
-            Some(cmd @ (ThrottleCommand::Exiting | ThrottleCommand::Waiting)) => Some(cmd),
+            Some(cmd @ (ThrottleCommand::Exit | ThrottleCommand::Wait)) => Some(cmd),
+            Some(ThrottleCommand::Reset { report }) => Some(ThrottleCommand::Reset { report }),
         }
     }
 
@@ -334,23 +338,44 @@ impl ThrottleWorker {
             'control: loop {
                 let thread_task = receiver.recv().expect("channel should not be closed");
                 match thread_task {
-                    ThrottleCommand::Exiting => {
+                    ThrottleCommand::Exit => {
                         break 'control;
                     }
-                    ThrottleCommand::Waiting => {
+                    ThrottleCommand::Wait => {
                         continue 'control;
                     }
-                    ThrottleCommand::Throttling(initial_throttle) => {
+                    ThrottleCommand::Throttle(initial_throttle) => {
                         let next_task = Self::throttle_loop(
                             &receiver,
                             initial_throttle,
                             &callback_pause_vcpus,
                             &callback_resume_vcpus,
                         );
-                        if next_task == ThrottleCommand::Exiting {
-                            break 'control;
+                        match next_task {
+                            ThrottleCommand::Exit => {
+                                break 'control;
+                            }
+                            // else: thread needs to go into waiting state
+                            ThrottleCommand::Reset { report } => {
+                                // Inform sender that we are back in the waiting state: Since `report` has capacity 0
+                                // this call will block until the command sender has received our message.
+                                if let Err(e) = report.send(()) {
+                                    error!(
+                                        "Unable to synchronize throttle thread reset event: error = {e:#?}"
+                                    );
+                                }
+                            }
+                            _ => {
+                                continue 'control;
+                            }
+                        }
+                    }
+                    ThrottleCommand::Reset { report } => {
+                        if let Err(e) = report.send(()) {
+                            error!(
+                                "Unable to synchronize throttle thread reset event: error = {e:#?}"
+                            );
                         }
-                        // else: thread is in Waiting state
                     }
                 }
             }
@@ -483,11 +508,11 @@ impl ThrottleThreadHandle {
 
         if percent_new == 0 {
             self.state_sender
-                .send(ThrottleCommand::Waiting)
+                .send(ThrottleCommand::Wait)
                 .expect("channel should not be closed");
         } else {
             self.state_sender
-                .send(ThrottleCommand::Throttling(percent_new))
+                .send(ThrottleCommand::Throttle(percent_new))
                 .expect("channel should not be closed");
         }
 
@@ -513,7 +538,7 @@ impl ThrottleThreadHandle {
             // drop thread; ensure that the channel is still alive when it is dropped
             if let Some(worker) = self.throttle_thread.take() {
                 self.state_sender
-                    .send(ThrottleCommand::Exiting)
+                    .send(ThrottleCommand::Exit)
                     .expect("channel should not be closed");
 
                 // Ensure the sender is still living when this is dropped.
@@ -529,6 +554,19 @@ impl ThrottleThreadHandle {
             );
         }
     }
+
+    /// Stops throttling and returns the throttle thread to the waiting state.
+    ///
+    /// This blocks until the throttling thread has exited the throttling loop.
+    pub fn reset(&self) {
+        let (report, recv) = mpsc::sync_channel(0);
+        self.state_sender
+            .send(ThrottleCommand::Reset { report })
+            .expect("channel should not be closed");
+        self.current_throttle.set(0);
+        info!("Waiting for throttle thread to acknowledge reset");
+        recv.recv().expect("The throttle thread should acknowledge the reset event before dropping rendezvous channel");
+    }
 }
 
 impl Drop for ThrottleThreadHandle {

vmm/src/vm.rs

@@ -1387,11 +1387,11 @@ impl Vm {
         self.vcpu_throttler.throttle_percent()
     }
 
-    /// Stops and terminates the thread gracefully.
+    /// Sets the vCPU throttling thread back to its initial waiting state.
     ///
-    /// Waits for the thread to finish.
-    pub fn stop_vcpu_throttling(&mut self) {
-        self.vcpu_throttler.shutdown();
+    /// Blocks until the throttling thread acknowledges the reset event.
+    pub fn reset_vcpu_throttle_thread(&self) {
+        self.vcpu_throttler.reset();
     }
 
     pub fn set_post_migration_lifecycle_event(