Created
January 24, 2026 17:26
-
-
Save ecto/2749b99a6f899cc9f7caf3d36d7e8f0c to your computer and use it in GitHub Desktop.
Reproduction attempt for rerun + mutex deadlock (rayon#592)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [package] | |
| name = "rerun-deadlock-repro" | |
| version = "0.1.0" | |
| edition = "2021" | |
| [dependencies] | |
| rerun = "0.22" | |
| tokio = { version = "1", features = ["full"] } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| //! Reproduction of rerun + tokio + std::sync::Mutex deadlock | |
| //! | |
| //! This models the actual bvrd pattern more accurately: | |
| //! - Main runs in tokio runtime | |
| //! - Control loop runs on dedicated std::thread | |
| //! - Multiple tokio tasks also access the shared mutex | |
| //! - Rerun is called within the control loop | |
| use std::sync::{Arc, Mutex}; | |
| use std::time::{Duration, Instant}; | |
| use std::sync::atomic::{AtomicU64, Ordering}; | |
| #[tokio::main(flavor = "multi_thread", worker_threads = 8)] | |
| async fn main() { | |
| println!("Rerun + Tokio + Mutex deadlock reproduction"); | |
| println!("============================================"); | |
| println!("This models bvrd: tokio runtime + dedicated control thread\n"); | |
| // Create rerun recording stream | |
| let rec = Arc::new( | |
| rerun::RecordingStreamBuilder::new("deadlock-repro-tokio") | |
| .save("/tmp/deadlock-repro-tokio.rrd") | |
| .expect("Failed to create rerun stream") | |
| ); | |
| // Shared state protected by std::sync::Mutex (not tokio::sync::Mutex) | |
| let shared: Arc<Mutex<u64>> = Arc::new(Mutex::new(0)); | |
| let iteration_counter = Arc::new(AtomicU64::new(0)); | |
| // Spawn tokio tasks that also use the mutex | |
| // (simulates WebRTC, telemetry push, metrics, etc.) | |
| for i in 0..4 { | |
| let shared_clone = shared.clone(); | |
| tokio::spawn(async move { | |
| loop { | |
| { | |
| let guard = shared_clone.lock().unwrap(); | |
| std::hint::black_box(*guard); | |
| } | |
| tokio::time::sleep(Duration::from_millis(1 + i)).await; | |
| } | |
| }); | |
| } | |
| // Simulate WebRTC-like task that does more work | |
| let shared_webrtc = shared.clone(); | |
| tokio::spawn(async move { | |
| loop { | |
| // Simulate building telemetry packet | |
| { | |
| let guard = shared_webrtc.lock().unwrap(); | |
| std::hint::black_box(*guard); | |
| // Simulate serialization work | |
| std::thread::sleep(Duration::from_micros(100)); | |
| } | |
| tokio::time::sleep(Duration::from_millis(10)).await; | |
| } | |
| }); | |
| // Heartbeat thread to detect deadlock | |
| let counter_clone = iteration_counter.clone(); | |
| std::thread::spawn(move || { | |
| let mut last = 0; | |
| loop { | |
| std::thread::sleep(Duration::from_secs(5)); | |
| let current = counter_clone.load(Ordering::Relaxed); | |
| if current == last && current > 0 { | |
| eprintln!("\n*** DEADLOCK DETECTED! Stuck at iteration {} ***\n", current); | |
| } else { | |
| eprintln!("[heartbeat] iteration {} (+{})", current, current - last); | |
| } | |
| last = current; | |
| } | |
| }); | |
| // Control loop on dedicated std::thread (like bvrd) | |
| let shared_control = shared.clone(); | |
| let rec_control = rec.clone(); | |
| let counter_control = iteration_counter.clone(); | |
| let control_handle = std::thread::spawn(move || { | |
| let start = Instant::now(); | |
| let mut iteration = 0u64; | |
| let mut last_report = Instant::now(); | |
| println!("Control loop started on dedicated thread...\n"); | |
| loop { | |
| iteration += 1; | |
| counter_control.store(iteration, Ordering::Relaxed); | |
| // Simulate 100Hz timing | |
| std::thread::sleep(Duration::from_millis(10)); | |
| // PHASE 1: First mutex acquisition (motor commands) | |
| { | |
| let mut guard = shared_control.lock().unwrap(); | |
| *guard = iteration; | |
| } | |
| // PHASE 2: LiDAR processing with rerun log | |
| // Generate point cloud | |
| let num_points = 5000; | |
| let points: Vec<[f32; 3]> = (0..num_points) | |
| .map(|i| { | |
| let angle = (i as f32) * 0.01; | |
| [angle.cos() * 5.0, angle.sin() * 5.0, (i as f32) * 0.001] | |
| }) | |
| .collect(); | |
| // This is where bvrd deadlocked - calling rerun with mutex pressure | |
| rec_control.log("lidar/points", &rerun::Points3D::new(points)).ok(); | |
| // PHASE 3: Second mutex acquisition (telemetry) | |
| { | |
| let guard = shared_control.lock().unwrap(); | |
| std::hint::black_box(*guard); | |
| } | |
| // Progress report | |
| if last_report.elapsed() > Duration::from_secs(1) { | |
| let elapsed = start.elapsed().as_secs(); | |
| println!( | |
| "[{:3}s] iteration={} ({}Hz)", | |
| elapsed, iteration, iteration / elapsed.max(1) | |
| ); | |
| last_report = Instant::now(); | |
| } | |
| if start.elapsed() > Duration::from_secs(120) { | |
| println!("\nTest completed 120s without deadlock!"); | |
| break; | |
| } | |
| } | |
| }); | |
| // Wait for control loop (or deadlock) | |
| control_handle.join().unwrap(); | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment