From 0df7bf2fa2103d3f39695c1d1fbe744a50b03e18 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Mon, 3 Aug 2026 10:06:52 +0200 Subject: [PATCH 1/5] vmm: Remove Copy and Clone Derive from ThrottleCommand The ThrottleCommand does not need to be Clone (nor Copy) as of now and we will need to introduce a variant wrapping a type that is not Copy in a follow up commit. It is also debatable whether a message type should be Clone, as they are typically intended to be sent only once (and not used for other purposes). Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com --- vmm/src/vcpu_throttling.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmm/src/vcpu_throttling.rs b/vmm/src/vcpu_throttling.rs index e8fd0d3b12..d2965eae1d 100644 --- a/vmm/src/vcpu_throttling.rs +++ b/vmm/src/vcpu_throttling.rs @@ -42,7 +42,7 @@ use vm_migration::Pausable; use crate::cpu::CpuManager; /// The possible command of the thread, i.e., the current state. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] enum ThrottleCommand { /// Waiting for next event. Waiting, From 5dba74f4b32e4dafbd5262718b0188468f6012e3 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Mon, 3 Aug 2026 10:57:57 +0200 Subject: [PATCH 2/5] vmm: Remove PartialEq and Eq derive from ThrottleCommand In order to add a command for resetting the throttle thread with confirmation we have to add an additional variant that wraps a type that does not implement the PartialEq and Eq traits. We thus need to choose between manually implementing these traits for ThrottleCommand, or avoiding them. We choose the latter because that requires much less code. Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com --- vmm/src/vcpu_throttling.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vmm/src/vcpu_throttling.rs b/vmm/src/vcpu_throttling.rs index d2965eae1d..6eaf9478ad 100644 --- a/vmm/src/vcpu_throttling.rs +++ b/vmm/src/vcpu_throttling.rs @@ -42,7 +42,7 @@ use vm_migration::Pausable; use crate::cpu::CpuManager; /// The possible command of the thread, i.e., the current state. -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug)] enum ThrottleCommand { /// Waiting for next event. Waiting, @@ -347,7 +347,7 @@ impl ThrottleWorker { &callback_pause_vcpus, &callback_resume_vcpus, ); - if next_task == ThrottleCommand::Exiting { + if matches!(next_task, ThrottleCommand::Exiting) { break 'control; } // else: thread is in Waiting state From b1e3cb5ce84e71e6a1581c5d04bce224b71b2fd6 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Mon, 3 Aug 2026 11:19:03 +0200 Subject: [PATCH 3/5] vmm: Switch from gerund to imperative form in ThrottleCommand variants It makes more sense for a command to use the imperative form and the enum does not need to be used to track the throttling thread's current state. This enables us to introduce a reset variant to the enum that asks the throttle thread to stop its throttle loop and inform us when it is back to waiting for the next incoming command. Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com --- vmm/src/vcpu_throttling.rs | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/vmm/src/vcpu_throttling.rs b/vmm/src/vcpu_throttling.rs index 6eaf9478ad..464728b9e6 100644 --- a/vmm/src/vcpu_throttling.rs +++ b/vmm/src/vcpu_throttling.rs @@ -44,14 +44,12 @@ use crate::cpu::CpuManager; /// The possible command of the thread, i.e., the current state. #[derive(Debug)] enum ThrottleCommand { - /// Waiting for next event. - Waiting, - /// Ongoing vCPU throttling. - /// - /// The inner value shows the current throttling percentage in range `1..=99`. - Throttling(u8 /* `1..=99` */), - /// Thread is shutting down gracefully. - Exiting, + /// Exit the throttle loop and wait for next event. + Wait, + /// Throttle the vCPUs with the given throttle percentage in the range `1..=99` per time slice. + Throttle(u8 /* `1..=99` */), + /// Gracefully shutdown the vCPU throttling thread. + Exit, } /// Helper to adapt the throttling timeslice as we go, depending on the time it @@ -252,14 +250,14 @@ impl ThrottleWorker { ); match maybe_task { None => None, - Some(ThrottleCommand::Throttling(next)) => { + Some(ThrottleCommand::Throttle(next)) => { // A new throttle value is only applied at the end of a full // throttling cycle. This is fine and negligible in a series of // (tens of) thousands of cycles. *current_throttle = next as u64; None } - Some(cmd @ (ThrottleCommand::Exiting | ThrottleCommand::Waiting)) => Some(cmd), + Some(cmd @ (ThrottleCommand::Exit | ThrottleCommand::Wait)) => Some(cmd), } } @@ -334,20 +332,20 @@ impl ThrottleWorker { 'control: loop { let thread_task = receiver.recv().expect("channel should not be closed"); match thread_task { - ThrottleCommand::Exiting => { + ThrottleCommand::Exit => { break 'control; } - ThrottleCommand::Waiting => { + ThrottleCommand::Wait => { continue 'control; } - ThrottleCommand::Throttling(initial_throttle) => { + ThrottleCommand::Throttle(initial_throttle) => { let next_task = Self::throttle_loop( &receiver, initial_throttle, &callback_pause_vcpus, &callback_resume_vcpus, ); - if matches!(next_task, ThrottleCommand::Exiting) { + if matches!(next_task, ThrottleCommand::Exit) { break 'control; } // else: thread is in Waiting state @@ -483,11 +481,11 @@ impl ThrottleThreadHandle { if percent_new == 0 { self.state_sender - .send(ThrottleCommand::Waiting) + .send(ThrottleCommand::Wait) .expect("channel should not be closed"); } else { self.state_sender - .send(ThrottleCommand::Throttling(percent_new)) + .send(ThrottleCommand::Throttle(percent_new)) .expect("channel should not be closed"); } @@ -513,7 +511,7 @@ impl ThrottleThreadHandle { // drop thread; ensure that the channel is still alive when it is dropped if let Some(worker) = self.throttle_thread.take() { self.state_sender - .send(ThrottleCommand::Exiting) + .send(ThrottleCommand::Exit) .expect("channel should not be closed"); // Ensure the sender is still living when this is dropped. From f5b7145a490a314f4fce9ba424459b4f65612372 Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Mon, 3 Aug 2026 14:15:12 +0200 Subject: [PATCH 4/5] vmm: Do not stop vCPU throttle thread at the end of a live migration The current behavior of joining the vCPU throttling thread towards the end of a live migration is problematic when the live migration fails because then auto-converge is no longer possible on a second attempt. We fix this by instead resetting the throttling thread to its initial state. The throttling thread is now instead gracefully stopped by the ThrottleThreadHandle's destructor which runs whenever the Vm instance goes out of scope. Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com --- vmm/src/lib.rs | 6 ++--- vmm/src/vcpu_throttling.rs | 50 ++++++++++++++++++++++++++++++++++---- vmm/src/vm.rs | 8 +++--- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index aa0df78082..891bf4081f 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1822,9 +1822,9 @@ impl Vmm { )?; let downtime_begin = Instant::now(); // End throttle thread - info!("stopping vcpu thread"); - vm.stop_vcpu_throttling(); - info!("stopped vcpu thread"); + info!("stopping vcpu throttling"); + vm.reset_vcpu_throttle_thread(); + info!("stopped vcpu throttling"); info!("pausing VM"); vm.pause()?; info!("paused VM"); diff --git a/vmm/src/vcpu_throttling.rs b/vmm/src/vcpu_throttling.rs index 464728b9e6..840fa06e67 100644 --- a/vmm/src/vcpu_throttling.rs +++ b/vmm/src/vcpu_throttling.rs @@ -30,13 +30,13 @@ use std::cell::Cell; use std::cmp::min; -use std::sync::mpsc::RecvTimeoutError; +use std::sync::mpsc::{RecvTimeoutError, SyncSender}; use std::sync::{Arc, Mutex, mpsc}; use std::thread; use std::thread::JoinHandle; use std::time::{Duration, Instant}; -use log::{debug, warn}; +use log::{debug, error, info, warn}; use vm_migration::Pausable; use crate::cpu::CpuManager; @@ -50,6 +50,11 @@ enum ThrottleCommand { Throttle(u8 /* `1..=99` */), /// Gracefully shutdown the vCPU throttling thread. Exit, + /// Exit the throttle loop then rendezvous with the receiver before proceeding to wait for the next command. + /// + /// In other words the `report` is used to synchronize the throttle thread reset event with the thread that + /// sent this command. + Reset { report: SyncSender<()> }, } /// Helper to adapt the throttling timeslice as we go, depending on the time it @@ -258,6 +263,7 @@ impl ThrottleWorker { None } Some(cmd @ (ThrottleCommand::Exit | ThrottleCommand::Wait)) => Some(cmd), + Some(ThrottleCommand::Reset { report }) => Some(ThrottleCommand::Reset { report }), } } @@ -345,10 +351,31 @@ impl ThrottleWorker { &callback_pause_vcpus, &callback_resume_vcpus, ); - if matches!(next_task, ThrottleCommand::Exit) { - break 'control; + match next_task { + ThrottleCommand::Exit => { + break 'control; + } + // else: thread needs to go into waiting state + ThrottleCommand::Reset { report } => { + // Inform sender that we are back in the waiting state: Since `report` has capacity 0 + // this call will block until the command sender has received our message. + if let Err(e) = report.send(()) { + error!( + "Unable to synchronize throttle thread reset event: error = {e:#?}" + ); + } + } + _ => { + continue 'control; + } + } + } + ThrottleCommand::Reset { report } => { + if let Err(e) = report.send(()) { + error!( + "Unable to synchronize throttle thread reset event: error = {e:#?}" + ); } - // else: thread is in Waiting state } } } @@ -527,6 +554,19 @@ impl ThrottleThreadHandle { ); } } + + /// Stops throttling and returns the throttle thread to the waiting state. + /// + /// This blocks until the throttling thread has exited the throttling loop. + pub fn reset(&self) { + let (report, recv) = mpsc::sync_channel(0); + self.state_sender + .send(ThrottleCommand::Reset { report }) + .expect("channel should not be closed"); + self.current_throttle.set(0); + info!("Waiting for throttle thread to acknowledge reset"); + recv.recv().expect("The throttle thread should acknowledge the reset event before dropping rendezvous channel"); + } } impl Drop for ThrottleThreadHandle { diff --git a/vmm/src/vm.rs b/vmm/src/vm.rs index 2f73adaccc..ced2bc0c2e 100644 --- a/vmm/src/vm.rs +++ b/vmm/src/vm.rs @@ -1387,11 +1387,11 @@ impl Vm { self.vcpu_throttler.throttle_percent() } - /// Stops and terminates the thread gracefully. + /// Sets the vCPU throttling thread back to its initial waiting state. /// - /// Waits for the thread to finish. - pub fn stop_vcpu_throttling(&mut self) { - self.vcpu_throttler.shutdown(); + /// Blocks until the throttling thread acknowledges the reset event. + pub fn reset_vcpu_throttle_thread(&self) { + self.vcpu_throttler.reset(); } pub fn set_post_migration_lifecycle_event( From e01ae7e6a1fc8b34984491e79ff8a35e3034b7ed Mon Sep 17 00:00:00 2001 From: Oliver Anderson Date: Thu, 6 Aug 2026 14:53:08 +0200 Subject: [PATCH 5/5] vmm: Reset throttle thread when do_memory_iteration fails When a live migration fails before all memory iterations have been sent we also need to reset the throttling thread, otherwise the VM continues existing on the migration source with throttled vCPUs. Signed-off-by: Oliver Anderson On-behalf-of: SAP oliver.anderson@sap.com --- vmm/src/lib.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 891bf4081f..e8459d8caa 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -1819,12 +1819,16 @@ impl Vmm { mem_send, postponed_lifecycle_event, return_if_cancelled_cb, - )?; + ); + let downtime_begin = Instant::now(); // End throttle thread info!("stopping vcpu throttling"); vm.reset_vcpu_throttle_thread(); info!("stopped vcpu throttling"); + + let remaining = remaining?; + info!("pausing VM"); vm.pause()?; info!("paused VM");