diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -290,6 +290,7 @@ dependencies = [ "objc2 0.6.5", "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", + "pollster 1.0.1", "rfd", "ron", "serde", diff --git a/apps/artcraft-launcher/Cargo.toml b/apps/artcraft-launcher/Cargo.toml --- a/apps/artcraft-launcher/Cargo.toml +++ b/apps/artcraft-launcher/Cargo.toml @@ -66,3 +66,7 @@ rfd = { version = "=0.17.2", default-features = false, features = ["xdg-portal", [build-dependencies] # Icon + VERSIONINFO for artcraft-launcher.exe; build.rs only uses it when targeting Windows. winresource = "0.1" + +[dev-dependencies] +# gpu_health's device-loss test blocks on wgpu's adapter request. Already in Cargo.lock through wgpu. +pollster = "1" diff --git a/apps/artcraft-launcher/src/gpu_health.rs b/apps/artcraft-launcher/src/gpu_health.rs new file mode 100644 --- /dev/null +++ b/apps/artcraft-launcher/src/gpu_health.rs @@ -0,0 +1,104 @@ +//! The window's graphics device (#274). A driver reset (for example after the PC slept) loses the +//! wgpu device the window draws with, and egui-wgpu can't draw another frame on it: one with shapes +//! panics creating its staging buffers ("Failed to create staging buffer for index data"), and +//! wgpu's default handler panics on the errors that follow. [`WindowGpu`] records the loss +//! instead, the shell then draws nothing, closes the window and opens a new one with a fresh +//! device, at most [`ReopenBudget::MAX`] times in [`ReopenBudget::WINDOW`]. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use eframe::wgpu; + +/// The window's device and whether wgpu reported it lost. +pub struct WindowGpu { + device: wgpu::Device, + lost: Arc, +} + +impl WindowGpu { + /// Watch `device`: its loss sets a flag rather than panicking, and an uncaptured error is + /// logged rather than ending the launcher (wgpu's default handler panics). + pub fn watch(device: &wgpu::Device) -> Self { + let lost = Arc::new(AtomicBool::new(false)); + let flag = lost.clone(); + // Called by wgpu with its own locks held: only record, never call back into wgpu. + device.set_device_lost_callback(move |reason, message| { + log::error!("graphics device lost ({reason:?}): {message}"); + flag.store(true, Ordering::Release); + }); + device.on_uncaptured_error(Arc::new(|e: wgpu::Error| log::error!("graphics error: {e}"))); + Self { device: device.clone(), lost } + } + + /// Whether the device is lost. Polls it (non-blocking) so a loss while the window sat idle is + /// noticed before the next frame is drawn with it; `Device::poll` panics on a lost device, so + /// a panic counts as a loss too, logged as handled rather than as an internal error. + pub fn lost(&self) -> bool { + if self.lost.load(Ordering::Acquire) { + return true; + } + if crate::logging::catch_expected(|| self.device.poll(wgpu::PollType::Poll)).is_err() { + self.lost.store(true, Ordering::Release); + } + self.lost.load(Ordering::Acquire) + } +} + +/// How often a lost window may be replaced: a GPU that keeps failing must not open windows in a +/// loop. Past the budget the launcher stays in the tray (or quits without one), as when the +/// window is closed. +#[derive(Default)] +pub struct ReopenBudget { + recent: Vec, +} + +impl ReopenBudget { + pub const MAX: usize = 3; + pub const WINDOW: Duration = Duration::from_secs(10 * 60); + + /// Whether a new window may open at `now`, counting it if so. + pub fn allow(&mut self, now: Instant) -> bool { + self.recent.retain(|t| now.saturating_duration_since(*t) < Self::WINDOW); + if self.recent.len() >= Self::MAX { + return false; + } + self.recent.push(now); + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reopening_is_limited_per_ten_minutes() { + let mut budget = ReopenBudget::default(); + let t = Instant::now(); + assert!((0..ReopenBudget::MAX).all(|i| budget.allow(t + Duration::from_secs(i as u64)))); + assert!(!budget.allow(t + Duration::from_secs(60)), "a GPU that keeps failing"); + assert!(budget.allow(t + ReopenBudget::WINDOW + Duration::from_secs(1)), "an old loss no longer counts"); + } + + /// A destroyed device reads as lost once wgpu notices (after its submitted work), and the + /// errors that follow don't panic. Skips without an adapter. + #[test] + fn a_destroyed_device_reads_as_lost_without_panicking() { + let instance = wgpu::Instance::default(); + let Ok(adapter) = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())) else { + eprintln!("skipping: no graphics adapter"); + return; + }; + let Ok((device, _queue)) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())) else { return }; + let gpu = WindowGpu::watch(&device); + assert!(!gpu.lost()); + device.destroy(); + let _ = device.poll(wgpu::PollType::Wait { submission_index: None, timeout: Some(Duration::from_secs(30)) }); + assert!(gpu.lost()); + // An error on the lost device is logged, not a panic from wgpu's default handler. + let _ = device.create_buffer(&wgpu::BufferDescriptor { label: None, size: 4, usage: wgpu::BufferUsages::UNIFORM, mapped_at_creation: false }); + assert!(gpu.lost()); + } +} diff --git a/apps/artcraft-launcher/src/logging.rs b/apps/artcraft-launcher/src/logging.rs --- a/apps/artcraft-launcher/src/logging.rs +++ b/apps/artcraft-launcher/src/logging.rs @@ -176,10 +176,29 @@ pub fn install() -> Option { Some(Logs { sink }) } +thread_local! { + /// Set while [`catch_expected`] runs: a panic then is one its caller handles. + static EXPECTED_PANIC: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Run `f`, which may panic in a way the caller handles (`gpu_health`: `Device::poll` panics on a +/// lost device). The panic hook logs such a panic as handled, not as an internal error; every +/// other panic is still reported. +pub fn catch_expected(f: impl FnOnce() -> R) -> std::thread::Result { + EXPECTED_PANIC.with(|e| e.set(true)); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)); + EXPECTED_PANIC.with(|e| e.set(false)); + result +} + /// Report a panic through the logger (standard error and the log file), with a backtrace when /// `RUST_BACKTRACE` asks for one; standard error alone when `RUST_LOG` turned errors off. pub fn install_panic_hook() { std::panic::set_hook(Box::new(|info| { + if EXPECTED_PANIC.with(std::cell::Cell::get) { + log::info!("handled: {info}"); + return; + } let trace = std::backtrace::Backtrace::capture(); let report = if trace.status() == std::backtrace::BacktraceStatus::Captured { format!("internal error: {info}\n{trace}") @@ -197,6 +216,16 @@ pub fn install_panic_hook() { #[cfg(test)] mod tests { use super::*; + + /// An expected panic comes back as an error, and only while `catch_expected` runs is a panic + /// treated as expected (#612 review). + #[test] + fn expected_panics_are_caught_and_the_mark_is_cleared() { + assert!(catch_expected(|| panic!("expected")).is_err()); + assert!(!EXPECTED_PANIC.with(std::cell::Cell::get)); + assert_eq!(catch_expected(|| 7).ok(), Some(7)); + assert!(!EXPECTED_PANIC.with(std::cell::Cell::get)); + } use log::Log; use std::sync::atomic::{AtomicUsize, Ordering}; diff --git a/apps/artcraft-launcher/src/main.rs b/apps/artcraft-launcher/src/main.rs --- a/apps/artcraft-launcher/src/main.rs +++ b/apps/artcraft-launcher/src/main.rs @@ -5,6 +5,7 @@ #![cfg_attr(windows, windows_subsystem = "windows")] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod gpu_health; mod logging; mod motion; mod settings_file; @@ -100,7 +101,9 @@ fn main() -> ExitCode { window = launcher.tray.is_none(); } // One pass per window: where a window can't hide, closing to the tray ends its pass and the - // launcher carries on without one until the window is asked for again. + // launcher carries on without one until the window is asked for again. A window whose + // graphics device was lost ends its pass too, and the next one starts at once (#274). + let mut reopens = gpu_health::ReopenBudget::default(); loop { if window { let result = { @@ -119,6 +122,14 @@ fn main() -> ExitCode { log::error!("{e}"); return ExitCode::FAILURE; } + if let Some(hidden_again) = launcher.reopen_window.take() { + if reopens.allow(std::time::Instant::now()) { + hidden = hidden_again; + continue; + } + log::error!("the graphics device keeps being lost; carrying on without a window"); + launcher.closed_to_tray = launcher.tray.is_some(); + } if !std::mem::take(&mut launcher.closed_to_tray) { return ExitCode::SUCCESS; } @@ -474,6 +485,9 @@ struct Launcher { started: bool, /// The window closed to carry on in the tray, not to quit. closed_to_tray: bool, + /// The window's graphics device was lost: open a new window (hidden if this is `true`) with a + /// fresh device instead of carrying on without one (#274). + reopen_window: Option, /// eframe's settings file, which the launcher writes itself while there is no window. settings_path: Option, waker: Option>, @@ -497,6 +511,7 @@ impl Launcher { run_again, started: false, closed_to_tray: false, + reopen_window: None, settings_path, waker: None, } @@ -633,6 +648,8 @@ struct Shell<'a> { /// The "show on the current desktop" setting last applied to the window (macOS). #[cfg_attr(not(target_os = "macos"), allow(dead_code))] applied_follow: Option, + /// The graphics device the window draws with (none in tests). + gpu: Option, } impl<'a> Shell<'a> { @@ -640,7 +657,8 @@ impl<'a> Shell<'a> { let window_hides = window_can_hide(cc.display_handle().ok().map(|d| d.as_raw())); // Started hidden at login but there's no tray to come back from: show the window. let visible = !hidden || launcher.tray.is_none(); - let mut shell = Self { launcher, window_hides, visible: true, quitting: false, applied_follow: None }; + let gpu = cc.wgpu_render_state.as_ref().map(|rs| gpu_health::WindowGpu::watch(&rs.device)); + let mut shell = Self { launcher, window_hides, visible: true, quitting: false, applied_follow: None, gpu }; if !visible { shell.put_away(&cc.egui_ctx); } else if hidden { @@ -706,7 +724,7 @@ impl eframe::App for Shell<'_> { // The close button puts the window away in the tray unless the user chose to quit. That is // the window manager's button or the header's own (no decorations), which asks to close // with `ViewportCommand::Close` and arrives here the same way. - if ctx.input(|i| i.viewport().close_requested()) && !self.quitting && !self.launcher.closed_to_tray { + if ctx.input(|i| i.viewport().close_requested()) && !self.quitting && !self.launcher.closed_to_tray && self.launcher.reopen_window.is_none() { match on_close(self.launcher.tray.is_some(), self.launcher.ui.settings.close_to_tray, self.window_hides) { OnClose::Hide => { ctx.send_viewport_cmd(egui::ViewportCommand::CancelClose); @@ -733,6 +751,16 @@ impl eframe::App for Shell<'_> { true }); }); + // Last, after everything drawn this frame: egui-wgpu can't draw on a lost device, and a + // frame with shapes panics inside it. Draw nothing and swap in a new window (#274). + if self.gpu.as_ref().is_some_and(gpu_health::WindowGpu::lost) { + ui.ctx().graphics_mut(|g| *g = Default::default()); + if self.launcher.reopen_window.is_none() { + log::error!("opening a new window with a fresh graphics device"); + self.launcher.reopen_window = Some(!self.visible); + ui.ctx().send_viewport_cmd(egui::ViewportCommand::Close); + } + } } fn raw_input_hook(&mut self, ctx: &egui::Context, raw_input: &mut egui::RawInput) { @@ -986,7 +1014,7 @@ mod tests { fn quitting_overrides_a_close_to_the_tray() { let (mut launcher, _show, _) = launcher(); let ctx = launcher.windowless.clone(); - let mut shell = Shell { launcher: &mut launcher, window_hides: false, visible: true, quitting: false, applied_follow: None }; + let mut shell = Shell { launcher: &mut launcher, window_hides: false, visible: true, quitting: false, applied_follow: None, gpu: None }; shell.put_away(&ctx); assert!(shell.launcher.closed_to_tray); shell.quit(&ctx); @@ -997,7 +1025,7 @@ mod tests { fn a_window_that_hides_is_not_closed_to_the_tray() { let (mut launcher, _show, _) = launcher(); let ctx = launcher.windowless.clone(); - let mut shell = Shell { launcher: &mut launcher, window_hides: true, visible: true, quitting: false, applied_follow: None }; + let mut shell = Shell { launcher: &mut launcher, window_hides: true, visible: true, quitting: false, applied_follow: None, gpu: None }; shell.put_away(&ctx); assert!(!shell.visible); assert!(!shell.launcher.closed_to_tray);