diff --git a/apps/lightcraft/src/gpu_crash.rs b/apps/lightcraft/src/gpu_crash.rs --- a/apps/lightcraft/src/gpu_crash.rs +++ b/apps/lightcraft/src/gpu_crash.rs @@ -18,7 +18,9 @@ pub(crate) fn window_files(in_memory: bool) -> window_gpu::Files { } /// Written while the GPU compute device is created, removed once that returned -/// (`lightcraft_gpu::backend::set_init_marker`). Not with `LIGHTCRAFT_NO_PREFS` (tests, scripts). +/// (`lightcraft_gpu::backend::set_init_marker`); still there after a quit while a creation that +/// passed its time limit was stuck in the driver (#3068). Not with `LIGHTCRAFT_NO_PREFS` (tests, +/// scripts). fn gpu_marker_path() -> Option { if std::env::var_os("LIGHTCRAFT_NO_PREFS").is_some() { return None; diff --git a/crates/gpu/src/backend.rs b/crates/gpu/src/backend.rs --- a/crates/gpu/src/backend.rs +++ b/crates/gpu/src/backend.rs @@ -179,7 +179,8 @@ static ACTIVE: Mutex = Mutex::new(0); /// Write this file before the compute device is created or any shader / pipeline is compiled /// (the kernels, the denoise network's), and remove it once that returned (successfully or not). /// A file still there at the next launch means the process died inside the driver — e.g. Apple's -/// Metal compiler (`MTLCompilerService`) aborting on a kernel, issue #250: see +/// Metal compiler (`MTLCompilerService`) aborting on a kernel, issue #250 — or ended while a device +/// creation that passed its time limit was still stuck there (issue #3068): see /// [`take_init_marker`]. `None` (the default) writes nothing. pub fn set_init_marker(path: Option) { *MARKER.lock().unwrap_or_else(|e| e.into_inner()) = path; @@ -225,6 +226,7 @@ impl Drop for Armed<'_> { /// Run driver work `f` (`what`: "device creation", "kernel compilation"…) between writing and /// removing the init marker (if one is set). +#[cfg_attr(not(feature = "denoise"), allow(dead_code))] // the denoise network's kernels pub(crate) fn with_marker(what: &str, f: impl FnOnce() -> T) -> T { let marker = MARKER.lock().unwrap_or_else(|e| e.into_inner()).clone(); with_marker_on(marker, &ACTIVE, what, f) @@ -240,9 +242,32 @@ fn with_marker_on(marker: Option, active: &Mutex, what: &str, f f() } -/// Run device creation (and the kernel build that follows it) `f` under the init marker. -pub(crate) fn with_init_marker(backends: Backends, f: impl FnOnce() -> T) -> T { - with_marker(&format!("device and kernel creation (backends {backends:?})"), f) +/// The init marker (if one is set) and its scope counter, taken on the thread that asks for a device +/// and carried to the thread that creates it (`init.rs`, issue #3068): the marker stays while the +/// driver work really runs, also past the caller's time limit. +pub(crate) struct InitMarker { + path: Option, + active: &'static Mutex, +} + +impl InitMarker { + /// The process-wide marker ([`set_init_marker`]). + pub(crate) fn current() -> InitMarker { + InitMarker { path: MARKER.lock().unwrap_or_else(|e| e.into_inner()).clone(), active: &ACTIVE } + } + + /// A marker at `path` on its own scope counter (tests: other tests create real devices through + /// the process-wide one). + #[cfg(test)] + pub(crate) fn at(path: PathBuf, active: &'static Mutex) -> InitMarker { + InitMarker { path: Some(path), active } + } + + /// Run device creation (and the kernel build that follows it) `f` on `backends` between writing + /// and removing the marker. + pub(crate) fn run(self, backends: Backends, f: impl FnOnce() -> T) -> T { + with_marker_on(self.path, self.active, &format!("device and kernel creation (backends {backends:?})"), f) + } } /// If the init marker `path` is present — the last process died while creating the GPU device — @@ -443,12 +468,12 @@ mod tests { } #[test] - fn with_init_marker_writes_the_process_wide_marker() { + fn the_current_init_marker_is_the_process_wide_one() { let dir = std::env::temp_dir().join(format!("lc-gpu-global-{}", std::process::id())); let m = dir.join("gpu-init.marker"); let _ = std::fs::remove_dir_all(&dir); set_init_marker(Some(m.clone())); - let seen = with_init_marker(Backends::DX12, || m.exists()); + let seen = InitMarker::current().run(Backends::DX12, || m.exists()); set_init_marker(None); // not checked for removal: device-creating tests running in parallel may still hold the file // (the test above checks removal on its own scope counter) diff --git a/crates/gpu/src/exit.rs b/crates/gpu/src/exit.rs --- a/crates/gpu/src/exit.rs +++ b/crates/gpu/src/exit.rs @@ -35,6 +35,11 @@ pub(crate) fn enter() -> Option> { GATE.enter() } +/// The process's gate (device creation carries a [`Work`] of it to the thread that creates it). +pub(crate) fn gate() -> &'static Gate { + &GATE +} + pub(crate) fn begin_shutdown() { GATE.close(); } diff --git a/crates/gpu/src/faces.rs b/crates/gpu/src/faces.rs --- a/crates/gpu/src/faces.rs +++ b/crates/gpu/src/faces.rs @@ -87,9 +87,8 @@ fn dev() -> Result<&'static Dev, String> { fn create_device() -> Result { let Some(backends) = crate::backend::compute_backends() else { return Err("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()) }; - crate::backend::with_init_marker(backends, || { - std::panic::catch_unwind(|| make_device(backends)).unwrap_or_else(|_| Err("device creation panicked".into())) - }) + // on a thread of its own, under the crash sentinel, with a time limit (#3068, `init.rs`) + crate::init::create(backends, move || make_device(backends)) } fn entry(binding: u32, ty: wgpu::BufferBindingType) -> wgpu::BindGroupLayoutEntry { diff --git a/crates/gpu/src/init.rs b/crates/gpu/src/init.rs new file mode 100644 --- /dev/null +++ b/crates/gpu/src/init.rs @@ -0,0 +1,388 @@ +//! A time limit on creating a GPU device (issue #3068). +//! +//! Creating the wgpu instance, adapter and device, and compiling the kernels after it, are calls +//! into the graphics driver, and on Linux the driver may talk to the display server first. With an +//! unresponsive one (a crash-looping WSLg compositor, a socket that accepts and never answers) +//! those calls never return, and wgpu's native calls are synchronous (`pollster::block_on` gets an +//! already-finished future), so there is nothing to time out from the calling thread. +//! +//! So every device (rendering, denoise, faces) is made by [`create`] on a thread of its own, and +//! the caller waits at most [`limit`] (`LIGHTCRAFT_GPU_INIT_TIMEOUT`). Past it the caller gets an +//! `Err` that it caches like any other creation failure — the GPU is unavailable for the process — +//! and the blocked thread is left behind. That thread keeps what has to last as long as the driver +//! work does: its share of the GPU work in flight (`exit.rs`: a quit then reports the GPU busy +//! instead of tearing the driver down under it) and the crash sentinel (`backend.rs`: a quit +//! leaves the marker, so the next launch starts on the CPU). While one is left behind, later creations (another kind +//! of device) fail at once with the same reason instead of leaving another thread behind; should +//! the driver come back, the late device is leaked (never dropped into a driver that just hung) +//! and creations are tried again. +//! +//! The error scopes popped after creation (`scope.pop()`) need no limit: on native wgpu they +//! resolve at once, and they are thread-local, so they could not move to another thread anyway. + +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError, mpsc}; +use std::time::Duration; + +use crate::backend::InitMarker; +use crate::exit::Gate; + +/// The environment variable that changes [`DEFAULT_LIMIT`] (whole seconds). +pub(crate) const LIMIT_ENV: &str = "LIGHTCRAFT_GPU_INIT_TIMEOUT"; +/// How long device creation may take by default. Normally well under a second, but a first run +/// that compiles every kernel with a slow driver compiler (DX12's FXC) takes seconds: generous. +pub(crate) const DEFAULT_LIMIT: Duration = Duration::from_secs(30); +/// The longest limit [`LIMIT_ENV`] can ask for (an hour). +const MAX_LIMIT_SECS: u64 = 3600; + +/// How long a device creation may take: [`LIMIT_ENV`], else [`DEFAULT_LIMIT`]. Read once. +pub(crate) fn limit() -> Duration { + static LIMIT: OnceLock = OnceLock::new(); + *LIMIT.get_or_init(|| { + let v = std::env::var(LIMIT_ENV).ok(); + parse_limit(v.as_deref()).unwrap_or_else(|| { + log::warn!( + "gpu: {LIMIT_ENV}={:?} is not a whole number of seconds from 1 to {MAX_LIMIT_SECS}; using {} s", + v.unwrap_or_default(), + DEFAULT_LIMIT.as_secs() + ); + DEFAULT_LIMIT + }) + }) +} + +/// The limit `v` (whole seconds, at most [`MAX_LIMIT_SECS`]) asks for; unset or blank is the +/// default; `None` when it is not a number of seconds ≥ 1. +fn parse_limit(v: Option<&str>) -> Option { + let Some(v) = v.map(str::trim).filter(|v| !v.is_empty()) else { return Some(DEFAULT_LIMIT) }; + match v.parse::() { + Ok(secs) if secs >= 1 => Some(Duration::from_secs(secs.min(MAX_LIMIT_SECS))), + // a number too big for u64 is still "wait as long as allowed" + Err(e) if *e.kind() == std::num::IntErrorKind::PosOverflow => Some(Duration::from_secs(MAX_LIMIT_SECS)), + _ => None, + } +} + +/// `d` for a message: "30 s", or "250 ms" under a second. +fn show(d: Duration) -> String { + if d >= Duration::from_secs(1) { format!("{} s", d.as_secs()) } else { format!("{} ms", d.as_millis()) } +} + +/// Device creations that are still inside the driver past their limit, and why the first was +/// given up on. +pub(crate) struct Stuck(Mutex); + +struct StuckState { + threads: usize, + reason: Option, +} + +impl Stuck { + pub(crate) const fn new() -> Stuck { + Stuck(Mutex::new(StuckState { threads: 0, reason: None })) + } + + fn lock(&self) -> MutexGuard<'_, StuckState> { + self.0.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Why device creation is not tried now (one is still stuck), or `None`. + fn reason(&self) -> Option { + let s = self.lock(); + if s.threads > 0 { s.reason.clone() } else { None } + } +} + +/// The process's stuck device creations (every kind of device: they share the driver). +static STUCK: Stuck = Stuck::new(); + +/// An attempt's state, changed only under the [`Stuck`] lock. +const WAITING: u8 = 0; +/// The worker finished in time; its result is (being) sent. +const DONE: u8 = 1; +/// The caller gave up waiting; the worker leaks its result. +const ABANDONED: u8 = 2; + +/// Why no device is created once the process is ending ([`crate::begin_shutdown`]). +const CLOSING: &str = "LightCraft is closing"; + +/// What a creation thread carries besides the work: the stuck-state it reports to, the shutdown +/// gate it counts as GPU work on, and the crash sentinel it runs under. +struct Scope { + stuck: &'static Stuck, + gate: &'static Gate, + marker: InitMarker, +} + +/// Create a device on `backends` with `make` on a thread of its own and wait at most [`limit`] for +/// it. A panic in it, or a deadline that passes, is an `Err` saying so. +/// +/// The thread, not the caller, holds what must last as long as the driver work does: it counts as +/// GPU work in flight ([`crate::wait_idle`] is not idle until it ends, issue #620) and runs under +/// the crash sentinel ([`crate::backend::set_init_marker`]). So a creation left behind past its +/// limit keeps both: a quit meanwhile reports the GPU busy, and leaves the marker for the next +/// launch, which then starts on the CPU. +pub(crate) fn create(backends: wgpu::Backends, make: impl FnOnce() -> Result + Send + 'static) -> Result { + let scope = Scope { stuck: &STUCK, gate: crate::exit::gate(), marker: InitMarker::current() }; + create_on(scope, limit(), backends, make) +} + +/// [`create`] with an explicit limit and scope (the tests use their own, so that a test's stuck +/// creation cannot stop the real devices, or hold up the shutdown, of other tests in the process). +fn create_on( + scope: Scope, + limit: Duration, + backends: wgpu::Backends, + make: impl FnOnce() -> Result + Send + 'static, +) -> Result { + const WHAT: &str = "device creation"; + let Scope { stuck, gate, marker } = scope; + if let Some(why) = stuck.reason() { + return Err(why); + } + // the thread's own share of "GPU work in flight", released when the driver work really ends + let work = gate.enter().ok_or(CLOSING)?; + let phase = Arc::new(AtomicU8::new(WAITING)); + let (tx, rx) = mpsc::sync_channel(1); + let worker_phase = Arc::clone(&phase); + std::thread::Builder::new() + .name("lc-gpu-device".into()) + .spawn(move || { + let _work = work; + let r = marker + .run(backends, || std::panic::catch_unwind(std::panic::AssertUnwindSafe(make)).unwrap_or_else(|_| Err(format!("{WHAT} panicked")))); + { + let mut s = stuck.lock(); + if worker_phase.load(Ordering::Relaxed) == ABANDONED { + // the caller gave up: never drop a device into the driver that just hung + s.threads = s.threads.saturating_sub(1); + if s.threads == 0 { + s.reason = None; + } + drop(s); + log::warn!("gpu: a device creation that was given up on has finished after all"); + std::mem::forget(r); + return; + } + worker_phase.store(DONE, Ordering::Relaxed); + } + let _ = tx.send(r); + }) + .map_err(|e| format!("{WHAT}: could not start its thread ({e})"))?; + match rx.recv_timeout(limit) { + Ok(r) => r, + Err(mpsc::RecvTimeoutError::Disconnected) => Err(format!("{WHAT} panicked")), + Err(mpsc::RecvTimeoutError::Timeout) => { + let mut s = stuck.lock(); + if phase.load(Ordering::Relaxed) == DONE { + // it finished just now: the result is on its way + drop(s); + return rx.recv().unwrap_or_else(|_| Err(format!("{WHAT} panicked"))); + } + phase.store(ABANDONED, Ordering::Relaxed); + let why = format!("GPU {WHAT} did not finish in {} ({LIMIT_ENV}): the graphics driver or display server is not responding", show(limit)); + log::warn!("gpu: {why}; the GPU is not used"); + s.threads = s.threads.saturating_add(1); + Err(s.reason.get_or_insert(why).clone()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use std::sync::atomic::AtomicBool; + use std::time::Instant; + + const B: wgpu::Backends = wgpu::Backends::VULKAN; + + /// A test's own stuck-state, gate and marker (a file in a fresh folder of its own). + struct Local { + stuck: &'static Stuck, + gate: &'static Gate, + active: &'static Mutex, + dir: PathBuf, + } + + impl Local { + fn new(name: &str, stuck: &'static Stuck, gate: &'static Gate, active: &'static Mutex) -> Local { + let dir = std::env::temp_dir().join(format!("lc-gpu-init-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + Local { stuck, gate, active, dir } + } + + fn marker(&self) -> PathBuf { + self.dir.join("gpu-init.marker") + } + + fn scope(&self) -> Scope { + Scope { stuck: self.stuck, gate: self.gate, marker: InitMarker::at(self.marker(), self.active) } + } + } + + impl Drop for Local { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } + } + + /// Runs `f` on a thread and fails the test if it has not returned after 20 s (instead of + /// hanging the test binary, which is what a creation without a limit did). + fn within_20_s(f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(20)).expect("device creation still blocked after 20 s: it has no time limit") + } + + /// Waits up to 20 s for `done`. + fn eventually(what: &str, done: impl Fn() -> bool) { + let t0 = Instant::now(); + while !done() { + assert!(t0.elapsed() < Duration::from_secs(20), "{what}"); + std::thread::sleep(Duration::from_millis(10)); + } + } + + // Given a driver call that never returns (an unresponsive display server), device creation + // fails after the limit with a reason; a second creation then fails at once without starting + // another blocked thread; once the driver comes back, creation works again + #[test] + fn a_device_creation_that_never_returns_fails_after_the_limit() { + static STUCK: Stuck = Stuck::new(); + static GATE: Gate = Gate::new(); + static ACTIVE: Mutex = Mutex::new(0); + let local = Local::new("limit", &STUCK, &GATE, &ACTIVE); + let (release, blocked) = mpsc::channel::<()>(); + let t0 = Instant::now(); + let scope = local.scope(); + let r = within_20_s(move || { + create_on(scope, Duration::from_millis(200), B, move || -> Result { + let _ = blocked.recv(); // "inside the driver" until the test lets go + Ok(1) + }) + }); + let took = t0.elapsed(); + let why = r.expect_err("a creation past its limit is an error"); + assert!(why.contains("did not finish in 200 ms"), "{why}"); + assert!(why.contains(LIMIT_ENV), "the message names the setting: {why}"); + assert!(took >= Duration::from_millis(200), "waited the limit: {took:?}"); + + // another creation does not start a second blocked thread + let started = Arc::new(AtomicBool::new(false)); + let s2 = Arc::clone(&started); + let t1 = Instant::now(); + let scope = local.scope(); + let again = within_20_s(move || { + create_on(scope, Duration::from_secs(30), B, move || { + s2.store(true, Ordering::Relaxed); + Ok(2) + }) + }); + assert_eq!(again, Err(why.clone()), "the same reason, at once"); + assert!(t1.elapsed() < Duration::from_secs(10)); + assert!(!started.load(Ordering::Relaxed), "nothing new went into the driver"); + + // the driver answers after all: the late result is let go and creation is tried again + drop(release); + eventually("the released thread never finished", || STUCK.reason().is_none()); + let scope = local.scope(); + assert_eq!(within_20_s(move || create_on(scope, Duration::from_secs(30), B, || Ok(3))), Ok(3)); + } + + // Given a creation left behind past its limit, the process is not idle for a quit (issue #620: + // no driver teardown under it) and the crash sentinel stays, until the driver work really + // ends; the quit's own wait stays bounded meanwhile + #[test] + fn a_creation_left_behind_keeps_the_gpu_busy_and_the_marker_until_it_ends() { + static STUCK: Stuck = Stuck::new(); + static GATE: Gate = Gate::new(); + static ACTIVE: Mutex = Mutex::new(0); + let local = Local::new("busy", &STUCK, &GATE, &ACTIVE); + let (release, blocked) = mpsc::channel::<()>(); + let scope = local.scope(); + let r = within_20_s(move || { + create_on(scope, Duration::from_millis(100), B, move || -> Result { + let _ = blocked.recv(); + Ok(1) + }) + }); + assert!(r.is_err(), "given up on"); + let marker = local.marker(); + assert!(marker.exists(), "the sentinel stays while the driver work runs"); + let text = std::fs::read_to_string(&marker).unwrap_or_default(); + assert!(text.contains("device and kernel creation (backends") && text.contains("VULKAN"), "{text}"); + + // a quit now: nothing new starts, and the wait for what is in flight ends at its deadline + GATE.close(); + let t0 = Instant::now(); + assert!(!GATE.wait_idle(Duration::from_millis(100)), "the creation left behind is still in flight"); + assert!(t0.elapsed() < Duration::from_secs(10), "the quit's wait stays bounded"); + assert!(marker.exists(), "a quit now leaves the marker for the next launch"); + + // and a creation asked for once the gate is closed does not start + let scope = local.scope(); + assert_eq!(create_on::(scope, Duration::from_secs(30), B, || Ok(2)), Err(STUCK.reason().unwrap_or_default())); + + drop(release); + assert!(GATE.wait_idle(Duration::from_secs(20)), "idle once the driver work ended"); + eventually("the marker is removed once the driver work ended", || !marker.exists()); + } + + // Given a creation that returns in time, its result (device or reason) comes back unchanged, + // and neither the work in flight nor the marker is left + #[test] + fn a_creation_in_time_returns_its_result() { + static STUCK: Stuck = Stuck::new(); + static GATE: Gate = Gate::new(); + static ACTIVE: Mutex = Mutex::new(0); + let local = Local::new("in-time", &STUCK, &GATE, &ACTIVE); + let marker = local.marker(); + let m = marker.clone(); + assert_eq!(create_on(local.scope(), Duration::from_secs(30), B, move || Ok(m.exists())), Ok(true), "under the marker"); + assert!(!marker.exists()); + let no = create_on::(local.scope(), Duration::from_secs(30), B, || Err("no GPU adapter".into())); + assert_eq!(no, Err("no GPU adapter".to_string())); + assert!(STUCK.reason().is_none()); + assert!(GATE.wait_idle(Duration::from_secs(20))); + assert!(!marker.exists()); + // once the process is ending, nothing is created + GATE.close(); + assert_eq!(create_on::(local.scope(), Duration::from_secs(30), B, || Ok(1)), Err(CLOSING.to_string())); + } + + // Given a creation that panics (inside the driver or ours), the caller gets an error, not the + // panic, and the marker is removed (the process survived) + #[test] + fn a_creation_that_panics_is_an_error() { + static STUCK: Stuck = Stuck::new(); + static GATE: Gate = Gate::new(); + static ACTIVE: Mutex = Mutex::new(0); + let local = Local::new("panic", &STUCK, &GATE, &ACTIVE); + #[allow(clippy::panic)] + let r = create_on::(local.scope(), Duration::from_secs(30), B, || panic!("driver bug (expected in this test)")); + assert_eq!(r, Err("device creation panicked".to_string())); + assert!(STUCK.reason().is_none()); + assert!(GATE.wait_idle(Duration::from_secs(20))); + assert!(!local.marker().exists()); + } + + // Given LIGHTCRAFT_GPU_INIT_TIMEOUT, whole seconds from 1 up to an hour are used; anything + // else is refused (and the default used) + #[test] + fn the_limit_comes_from_whole_seconds() { + assert_eq!(parse_limit(None), Some(DEFAULT_LIMIT)); + assert_eq!(parse_limit(Some(" ")), Some(DEFAULT_LIMIT)); + assert_eq!(parse_limit(Some("5")), Some(Duration::from_secs(5))); + assert_eq!(parse_limit(Some(" 120 ")), Some(Duration::from_secs(120))); + assert_eq!(parse_limit(Some("999999")), Some(Duration::from_secs(MAX_LIMIT_SECS))); + assert_eq!(parse_limit(Some("99999999999999999999999")), Some(Duration::from_secs(MAX_LIMIT_SECS))); + for bad in ["0", "-3", "1.5", "abc", "NaN", "inf", "30s"] { + assert_eq!(parse_limit(Some(bad)), None, "{bad}"); + } + assert_eq!(show(Duration::from_secs(30)), "30 s"); + assert_eq!(show(Duration::from_millis(250)), "250 ms"); + } +} diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -7,7 +7,8 @@ //! CPU inside the same render (per-stage hybrid); see `docs/gpu-pipeline.md`. //! //! Which backends wgpu may load (DX12 only on Windows; `LIGHTCRAFT_GPU_BACKEND`, `WGPU_BACKEND`) and -//! the crash sentinel around device creation: [`backend`] (issue #136). +//! the crash sentinel around device creation: [`backend`] (issue #136). Device creation has a time +//! limit (`LIGHTCRAFT_GPU_INIT_TIMEOUT`, issue #3068, see `init.rs`). //! //! Use [`render`]: it returns `None` when the GPU is unavailable, disabled (`LIGHTCRAFT_GPU=0`, //! `LIGHTCRAFT_GPU_BACKEND=off` or [`set_enabled`]), the render does not fit the device, or the device reported an error, ran out @@ -37,6 +38,8 @@ mod ctx; mod exit; #[cfg(not(target_arch = "wasm32"))] pub mod faces; +#[cfg(not(target_arch = "wasm32"))] +mod init; #[cfg(all(feature = "denoise", not(target_arch = "wasm32")))] pub mod nn; #[cfg(not(target_arch = "wasm32"))] @@ -288,9 +291,10 @@ pub(crate) fn device() -> Option<&'static ctx::Gpu> { let _work = exit::enter()?; GPU.get_or_init(|| { let Some(backends) = backend::compute_backends() else { return Err("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()) }; - let r = backend::with_init_marker(backends, || { - std::panic::catch_unwind(|| ctx::Gpu::new(backends)).unwrap_or_else(|_| Err("device creation panicked".into())) - }); + // on a thread of its own (under the crash sentinel, counted as GPU work), waited for at most + // `init::limit()`: a driver or display server that never answers leaves the GPU unavailable + // instead of every render waiting forever (#3068) + let r = init::create(backends, move || ctx::Gpu::new(backends)); if let Err(why) = &r { log::warn!("gpu: rendering on the CPU: {why}"); } diff --git a/crates/gpu/src/nn.rs b/crates/gpu/src/nn.rs --- a/crates/gpu/src/nn.rs +++ b/crates/gpu/src/nn.rs @@ -76,9 +76,8 @@ const CLOSING: &str = "LightCraft is closing"; fn create_device() -> Result { let Some(backends) = crate::backend::compute_backends() else { return Err("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()) }; - crate::backend::with_init_marker(backends, || { - std::panic::catch_unwind(|| make_device(backends)).unwrap_or_else(|_| Err("device creation panicked".into())) - }) + // on a thread of its own, under the crash sentinel, with a time limit (#3068, `init.rs`) + crate::init::create(backends, move || make_device(backends)) } fn make_device(backends: wgpu::Backends) -> Result { diff --git a/docs/gpu-pipeline.md b/docs/gpu-pipeline.md --- a/docs/gpu-pipeline.md +++ b/docs/gpu-pipeline.md @@ -49,6 +49,9 @@ Overrides (read once at launch): - `LIGHTCRAFT_GPU=0`: GPU rendering off for the process (the window is unaffected). - Both `LIGHTCRAFT_GPU=0` and `LIGHTCRAFT_GPU_BACKEND=off` also keep AI Denoise, the face models and the AI mask model (SAM 3 on Metal) on the CPU. +- `LIGHTCRAFT_GPU_INIT_TIMEOUT=` (whole seconds, 1–3600; default 30): how long creating a + compute device (rendering, AI Denoise, faces) and compiling its kernels may take. See *Device + creation time limit* below. **DX12 shader compiler (issue #471).** Every instance (window and compute) compiles DX12 shaders with FXC (`d3dcompiler_47.dll`, part of Windows; `lightcraft_gpu::backend::backend_options`). wgpu's @@ -73,6 +76,24 @@ up (`gpu::warm_up` from the first frame's settings), and never while GPU renderi Settings ▸ Performance ▸ *Use the GPU for rendering* unchecked (applied before the window opens), `LIGHTCRAFT_GPU=0` or `LIGHTCRAFT_GPU_BACKEND=off`, no GPU driver is loaded for rendering at all. +**Device creation time limit (issue #3068).** Creating the wgpu instance, adapter and device calls +into the driver, which on Linux may first talk to the display server; with one that accepts the +connection and never answers (a crash-looping WSLg compositor, a stale `WAYLAND_DISPLAY` socket) those +calls never returned, and every render waiting for the device, and every test that creates one, hung +for good. wgpu's native calls are synchronous, so each compute device (rendering, denoise, faces) is +now created, kernels included, on a thread of its own (`lc-gpu-device`, `crates/gpu/src/init.rs`), and +the caller waits at most `LIGHTCRAFT_GPU_INIT_TIMEOUT` seconds (default 30: a first run that compiles +every kernel with FXC takes seconds). Past it that device is unavailable for the process, with the +reason `GPU device creation did not finish in 30 s (LIGHTCRAFT_GPU_INIT_TIMEOUT): …` reported like any +other creation failure (`perf.gpuReason`, the no-GPU notice, the log), and the blocked thread is left +behind. That thread, not the caller, holds what must last as long as the driver work does: its share +of the GPU work in flight, so a quit meanwhile finds the GPU busy (`gpu::wait_idle` returns `false` at +its own deadline, *Quitting* below), and the crash sentinel, so a quit meanwhile leaves the marker and +the next launch starts on the CPU (*Crash sentinel* below). While it is left behind, creating another +kind of device fails at once with the same reason rather than leave a second thread behind; if the +driver ever answers, the marker is removed, its late device is leaked, not dropped, and later +creations are tried again. The desktop window's own device is not covered (#410). + **Quitting.** The devices are never dropped, and renders run on worker threads: a render still inside the driver while the process exits crashes there (a SIGSEGV in the NVIDIA Vulkan driver on Linux, issue #620), which no panic hook catches. So every entry into a device counts as work in flight @@ -93,7 +114,9 @@ Mac). A kernel the driver refuses without crashing is caught by wgpu error scope (`gpu: rendering on the CPU: …`) and renders stay on the CPU. `LIGHTCRAFT_GPU=0` (or Settings ▸ uncheck *Use the GPU for rendering*) forces the CPU path from the start. If the marker is still there at the next launch, the process died inside the driver: LightCraft starts with GPU rendering off (the preference is saved unchecked), removes the -marker and says so in a notice. Checking *Use the GPU for rendering* again tries the GPU once more +marker and says so in a notice. The same happens after a quit while a device creation that passed its +time limit was still inside the driver (above): the driver hung rather than crashed, but it would +likely hang again, so the next launch starts on the CPU too. Checking *Use the GPU for rendering* again tries the GPU once more (and re-arms the sentinel). Not with `LIGHTCRAFT_NO_PREFS` (tests, scripts), and not in a `--memory` session, which writes nothing: it neither arms the sentinel nor removes a marker it finds (it still starts with GPU rendering off when one is there, and the next ordinary launch reports and clears it). Killing the app during