diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -3620,9 +3620,11 @@ dependencies = [ "objc2-app-kit 0.3.2", "rfd", "serde_json", + "soundcraft-clap-host", "soundcraft-engine", "soundcraft-playback", "soundcraft-ui-egui", + "soundcraft-vst3-host", "winresource", ] @@ -3666,6 +3668,7 @@ dependencies = [ "libloading 0.9.0", "log", "serde", + "serde_json", "soundcraft-dsp", "thiserror 2.0.21", "windows-sys 0.61.2", @@ -3829,6 +3832,7 @@ dependencies = [ "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", "serde", + "serde_json", "soundcraft-dsp", "thiserror 2.0.21", "vst3", diff --git a/apps/soundcraft/Cargo.toml b/apps/soundcraft/Cargo.toml --- a/apps/soundcraft/Cargo.toml +++ b/apps/soundcraft/Cargo.toml @@ -18,6 +18,8 @@ asio = ["soundcraft-playback/asio"] soundcraft-engine.workspace = true soundcraft-ui-egui.workspace = true soundcraft-playback.workspace = true +soundcraft-vst3-host.workspace = true +soundcraft-clap-host.workspace = true eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11", "links"] } egui.workspace = true serde_json.workspace = true @@ -34,4 +36,4 @@ winresource = "0.1" [target.'cfg(target_os = "macos")'.dependencies] muda = { version = "0.21.2", default-features = false } objc2 = "0.6" -objc2-app-kit = { version = "0.3", default-features = false, features = ["NSApplication", "NSResponder"] } +objc2-app-kit = { version = "0.3", default-features = false, features = ["NSApplication", "NSResponder", "NSRunningApplication"] } diff --git a/apps/soundcraft/src/main.rs b/apps/soundcraft/src/main.rs --- a/apps/soundcraft/src/main.rs +++ b/apps/soundcraft/src/main.rs @@ -205,7 +205,34 @@ fn app_icon() -> Option { eframe::icon_data::from_png_bytes(png).map_err(|e| log::warn!("app icon: {e}")).ok() } +/// When started as a plugin-scanning child (` `, see +/// `soundcraft_vst3_host::set_scanner`), scans that bundle and returns the exit code. +#[cfg(not(target_arch = "wasm32"))] +fn plugin_scan_child() -> Option { + let mut args = std::env::args_os().skip(1); + let (flag, path) = (args.next()?, std::path::PathBuf::from(args.next()?)); + let scan: fn(&std::path::Path) -> i32 = if flag == soundcraft_vst3_host::SCAN_FLAG { + soundcraft_vst3_host::scan_child_main + } else if flag == soundcraft_clap_host::SCAN_FLAG { + soundcraft_clap_host::scan_child_main + } else { + return None; + }; + // No Dock icon or menu bar for the child, even if a plugin touches AppKit while loading. + #[cfg(target_os = "macos")] + if let Some(mtm) = objc2::MainThreadMarker::new() { + let app = objc2_app_kit::NSApplication::sharedApplication(mtm); + app.setActivationPolicy(objc2_app_kit::NSApplicationActivationPolicy::Prohibited); + } + Some(scan(&path)) +} + fn main() -> eframe::Result { + // A plugin-scanning child opens no log file, settings or window. + #[cfg(not(target_arch = "wasm32"))] + if let Some(code) = plugin_scan_child() { + std::process::exit(code); + } // First, so every start-up record (and the engine's panic hook, installed with the first // Engine) is captured; see `logging`. let logger = logging::install(); @@ -247,6 +274,16 @@ fn main() -> eframe::Result { } } + // Plugin scans load each bundle in a child process of this executable (see `plugin_scan_child`). + #[cfg(not(target_arch = "wasm32"))] + match std::env::current_exe() { + Ok(exe) => { + soundcraft_vst3_host::set_scanner(Some(exe.clone())); + soundcraft_clap_host::set_scanner(Some(exe)); + } + Err(e) => log::warn!("plugin scans run in-process: {e}"), + } + let engine = if demo { soundcraft_engine::demo::demo_engine() } else { Engine::default() }; let prefs = load_prefs().unwrap_or_default(); let mut options = eframe::NativeOptions { diff --git a/crates/clap-host/Cargo.toml b/crates/clap-host/Cargo.toml --- a/crates/clap-host/Cargo.toml +++ b/crates/clap-host/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true [dependencies] soundcraft-dsp.workspace = true serde.workspace = true +serde_json.workspace = true thiserror.workspace = true log.workspace = true diff --git a/crates/clap-host/src/isolate.rs b/crates/clap-host/src/isolate.rs new file mode 100644 --- /dev/null +++ b/crates/clap-host/src/isolate.rs @@ -0,0 +1,233 @@ +//! Scanning in helper processes: each `.clap` is loaded by a short-lived child (the app's own +//! executable started with [`SCAN_FLAG`]) that prints its descriptors and exits. +//! +//! Scanning in-process loads every installed binary for the life of the process, with whatever +//! its static initialisers take (thread-specific data keys among them, of which a process has +//! only `PTHREAD_KEYS_MAX`). See `soundcraft-vst3-host`'s `isolate` module, which does the same +//! for VST3: out of process, the app only loads the plugins it uses, and a plugin that crashes +//! or hangs while being scanned is skipped instead of taking the app down. + +use crate::{ClapDescriptor, ClapError, parse_id, registry}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Mutex, OnceLock, PoisonError, mpsc}; +use std::time::{Duration, Instant}; + +/// The argument that makes an executable scan one bundle (the path follows) and exit. +pub const SCAN_FLAG: &str = "--scan-clap-bundle"; +/// The line a child prints after its result; anything else is a crash. +const END: &str = "soundcraft-clap-scan-end"; +/// Longest a child may take to load one bundle. +const TIMEOUT: Duration = Duration::from_secs(60); +/// Longest a child may take to exit once it has printed its result (module exits). +const EXIT_GRACE: Duration = Duration::from_secs(5); +/// Upper bound on what is read from one child. +const MAX_OUTPUT: u64 = 8 << 20; +/// Most children running at once. +const MAX_WORKERS: usize = 8; + +fn scanner() -> &'static Mutex> { + static S: OnceLock>> = OnceLock::new(); + S.get_or_init(|| Mutex::new(None)) +} + +/// Sets (or clears) the executable that scans bundles in child processes. +pub fn set_scanner(exe: Option) { + *scanner().lock().unwrap_or_else(PoisonError::into_inner) = exe; +} + +/// The executable that scans bundles, when one is set. +pub fn scanner_exe() -> Option { + scanner().lock().unwrap_or_else(PoisonError::into_inner).clone() +} + +/// The child side: loads `path` in this process and prints the result. Returns the process exit +/// code. +pub fn child_main(path: &Path) -> i32 { + let result = registry::load_bundle(path).map_err(|e| e.to_string()); + let code = i32::from(result.is_err()); + let json = serde_json::to_string(&result).unwrap_or_else(|e| format!("{{\"Err\":\"cannot encode the scan: {e}\"}}")); + let mut out = std::io::stdout().lock(); + let _ = writeln!(out, "{json}\n{END}"); + let _ = out.flush(); + code +} + +/// Decodes a child's output for `path`: descriptors with valid ids, all pointing at `path`. +pub(crate) fn decode(path: &Path, json: &str) -> Result, ClapError> { + let err = |m: String| ClapError::Load(path.display().to_string(), m); + let result: Result, String> = serde_json::from_str(json).map_err(|e| err(format!("unreadable scan result: {e}")))?; + let mut ds = result.map_err(|m| err(m.strip_prefix(&format!("cannot load CLAP plugin {}: ", path.display())).unwrap_or(&m).to_string()))?; + ds.retain(|d| parse_id(&d.id) == Some(d.plugin_id.as_str())); + for d in &mut ds { + d.path = path.to_path_buf(); + } + Ok(ds) +} + +/// Kills `child` unless it exits within `grace`, and reaps it. +fn reap(mut child: Child, grace: Duration) -> Option { + let deadline = Instant::now() + grace; + loop { + match child.try_wait() { + Ok(Some(status)) => return Some(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(10)), + _ => { + let _ = child.kill(); + return child.wait().ok(); + } + } + } +} + +/// Scans one bundle in a child process. +pub(crate) fn scan_one(exe: &Path, path: &Path) -> Result, ClapError> { + let err = |m: String| ClapError::Load(path.display().to_string(), m); + let mut child = Command::new(exe) + .arg(SCAN_FLAG) + .arg(path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| err(format!("cannot start the plugin scanner: {e}")))?; + let Some(stdout) = child.stdout.take() else { + reap(child, Duration::ZERO); + return Err(err("the plugin scanner has no output".into())); + }; + // Read on a thread so a child that never writes (or leaves a helper process holding the + // pipe open) cannot block past the timeout. + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let mut json = None; + for line in BufReader::new(stdout.take(MAX_OUTPUT)).lines() { + let Ok(line) = line else { break }; + if line == END { + break; + } + json = Some(line); + } + let _ = tx.send(json); + }); + match rx.recv_timeout(TIMEOUT) { + Ok(Some(json)) => { + reap(child, EXIT_GRACE); + decode(path, &json) + } + Ok(None) | Err(mpsc::RecvTimeoutError::Disconnected) => { + let status = reap(child, EXIT_GRACE); + Err(err(format!("the plugin crashed while being scanned ({})", status.map_or_else(|| "no status".to_string(), |s| s.to_string())))) + } + Err(mpsc::RecvTimeoutError::Timeout) => { + reap(child, Duration::ZERO); + Err(err(format!("the plugin did not load within {} s", TIMEOUT.as_secs()))) + } + } +} + +/// Scans `bundles` in child processes, several at a time. Results are in the order of `bundles`. +pub(crate) fn scan_all(exe: &Path, bundles: &[PathBuf]) -> Vec, ClapError>> { + let workers = std::thread::available_parallelism().map_or(2, |n| n.get()).clamp(1, MAX_WORKERS).min(bundles.len()); + let next = AtomicUsize::new(0); + let done: Mutex, ClapError>)>> = Mutex::new(Vec::with_capacity(bundles.len())); + std::thread::scope(|s| { + for _ in 0..workers { + s.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(path) = bundles.get(i) else { break }; + let r = scan_one(exe, path); + done.lock().unwrap_or_else(PoisonError::into_inner).push((i, r)); + } + }); + } + }); + let mut done = done.into_inner().unwrap_or_else(PoisonError::into_inner); + done.sort_by_key(|(i, _)| *i); + done.into_iter().map(|(_, r)| r).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use soundcraft_dsp::Category; + + fn desc(plugin_id: &str) -> ClapDescriptor { + ClapDescriptor { + id: format!("{}{plugin_id}", crate::ID_PREFIX), + plugin_id: plugin_id.to_string(), + name: "Comp".into(), + vendor: "V".into(), + version: "1".into(), + description: String::new(), + features: vec!["audio-effect".into(), "compressor".into()], + category: Category::Dynamics, + is_instrument: false, + path: PathBuf::from("/elsewhere.clap"), + } + } + + #[test] + fn child_output_decodes_and_is_not_trusted() { + let path = Path::new("/p/Comp.clap"); + let good = desc("com.v.comp"); + let mut bad = desc("com.v.comp"); + bad.id = "clap:com.v.other".into(); + let json = serde_json::to_string(&Ok::<_, String>(vec![good.clone(), bad])).unwrap(); + let ds = decode(path, &json).unwrap(); + assert_eq!(ds.len(), 1, "invalid ids are dropped"); + assert_eq!(ds[0].path, path, "the path is the one the parent asked for"); + assert_eq!(ds[0].name, good.name); + + let json = + serde_json::to_string(&Err::, _>(format!("cannot load CLAP plugin {}: no loadable binary", path.display()))).unwrap(); + assert_eq!(decode(path, &json).unwrap_err().to_string(), format!("cannot load CLAP plugin {}: no loadable binary", path.display())); + assert!(decode(path, "not json").is_err()); + assert!(decode(path, "").is_err()); + } + + #[test] + fn a_scanner_that_cannot_start_is_an_error() { + let r = scan_one(Path::new("/no/such/scanner"), Path::new("/p/x.clap")); + assert!(r.unwrap_err().to_string().contains("cannot start the plugin scanner")); + } + + // Unix only: uses `/bin/sh` scripts as stand-in scanners. + #[cfg(unix)] + #[test] + fn crashing_and_silent_scanners_are_errors_and_results_keep_their_order() { + use std::os::unix::fs::PermissionsExt; + let d = std::env::temp_dir().join(format!("soundcraft-clap-isolate-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + let script = |name: &str, body: &str| { + let p = d.join(name); + std::fs::write(&p, format!("#!/bin/sh\n{body}\n")).unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); + p + }; + let crash = script("crash.sh", "kill -ABRT $$"); + let r = scan_one(&crash, Path::new("/p/x.clap")); + assert!(r.unwrap_err().to_string().contains("crashed")); + + let json = serde_json::to_string(&Ok::<_, String>(vec![desc("com.v.comp")])).unwrap(); + // Prints a result for bundles named `ok*`, crashes on the others. + let mixed = script("mixed.sh", &format!("case \"$2\" in */ok*) printf '%s\\n{END}\\n' '{json}';; *) exit 3;; esac")); + let bundles: Vec = + (0..12).map(|i| PathBuf::from(if i % 3 == 0 { format!("/p/bad{i}.clap") } else { format!("/p/ok{i}.clap") })).collect(); + let results = scan_all(&mixed, &bundles); + assert_eq!(results.len(), bundles.len()); + for (b, r) in bundles.iter().zip(&results) { + match r { + Ok(ds) => { + assert!(b.to_string_lossy().contains("/ok")); + assert_eq!(ds[0].path, *b); + } + Err(_) => assert!(b.to_string_lossy().contains("/bad")), + } + } + let _ = std::fs::remove_dir_all(&d); + } +} diff --git a/crates/clap-host/src/lib.rs b/crates/clap-host/src/lib.rs --- a/crates/clap-host/src/lib.rs +++ b/crates/clap-host/src/lib.rs @@ -1,9 +1,10 @@ //! SoundCraft's CLAP plugin host. //! -//! Scans the standard CLAP folders (plus `CLAP_PATH`), loads `.clap` binaries in-process and -//! exposes each plugin as a [`soundcraft_dsp::Plugin`], so the mix engine treats third-party -//! plugins exactly like the built-in ones. Plugin ids are `clap:`; parameter ids -//! are the CLAP param ids as decimal strings, with their human names in [`PluginInfo::params`]. +//! Scans the standard CLAP folders (plus `CLAP_PATH`; in child processes when [`set_scanner`] is +//! set), loads `.clap` binaries in-process and exposes each plugin as a +//! [`soundcraft_dsp::Plugin`], so the mix engine treats third-party plugins exactly like the +//! built-in ones. Plugin ids are `clap:`; parameter ids are the CLAP param ids as +//! decimal strings, with their human names in [`PluginInfo::params`]. //! //! This is SoundCraft's single isolated `unsafe` crate (craftrules never-crash): `unsafe` is //! denied crate-wide and allowed only in the private `ffi` module. The public API is safe, returns @@ -35,10 +36,14 @@ pub mod scan; #[allow(unsafe_code)] mod ffi; #[cfg(not(target_arch = "wasm32"))] +mod isolate; +#[cfg(not(target_arch = "wasm32"))] mod plugin; #[cfg(not(target_arch = "wasm32"))] mod registry; +#[cfg(not(target_arch = "wasm32"))] +pub use isolate::SCAN_FLAG; #[cfg(not(target_arch = "wasm32"))] pub use plugin::ClapPlugin; @@ -54,7 +59,7 @@ pub fn parse_id(id: &str) -> Option<&str> { } /// One plugin found by a scan. -#[derive(Debug, Clone, PartialEq, serde::Serialize)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct ClapDescriptor { /// SoundCraft id: `clap:`. pub id: String, @@ -164,6 +169,22 @@ pub fn add_search_dir(dir: &Path) -> usize { api::add_search_dir(dir) } +/// Makes scans load each `.clap` in a child process: `exe` started with [`SCAN_FLAG`] and the +/// path, which must call [`scan_child_main`] (`None`: scan in this process, the default). The app +/// sets it so that a scan neither keeps every installed binary loaded nor dies with a plugin that +/// crashes while loading (see the `isolate` module). +#[cfg(not(target_arch = "wasm32"))] +pub fn set_scanner(exe: Option) { + isolate::set_scanner(exe) +} + +/// The child side of [`set_scanner`]: loads one `.clap` and prints what it holds for the parent. +/// Returns the process exit code. +#[cfg(not(target_arch = "wasm32"))] +pub fn scan_child_main(path: &Path) -> i32 { + isolate::child_main(path) +} + /// Scans the given folders without touching the cache. pub fn scan_paths(dirs: &[PathBuf]) -> Vec { api::scan_paths(dirs) diff --git a/crates/clap-host/src/registry.rs b/crates/clap-host/src/registry.rs --- a/crates/clap-host/src/registry.rs +++ b/crates/clap-host/src/registry.rs @@ -49,12 +49,17 @@ pub fn load_bundle(path: &Path) -> Result, ClapError> { .collect()) } -/// Scans `dirs` (uncached). Unloadable files are logged and skipped; the first plugin with a -/// given id wins. +/// Scans `dirs` (uncached), in child processes when a scanner is set. Unloadable files are logged +/// and skipped; the first plugin with a given id wins. pub fn scan_paths(dirs: &[PathBuf]) -> Vec { + let bundles = paths::find_bundles(dirs); + let results = match crate::isolate::scanner_exe() { + Some(exe) => crate::isolate::scan_all(&exe, &bundles), + None => bundles.iter().map(|p| load_bundle(p)).collect(), + }; let mut out: Vec = Vec::new(); - for path in paths::find_bundles(dirs) { - match load_bundle(&path) { + for result in results { + match result { Ok(ds) => { for d in ds { if !out.iter().any(|o| o.id == d.id) { diff --git a/crates/vst3-host/Cargo.toml b/crates/vst3-host/Cargo.toml --- a/crates/vst3-host/Cargo.toml +++ b/crates/vst3-host/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true [dependencies] soundcraft-dsp.workspace = true serde.workspace = true +serde_json.workspace = true thiserror.workspace = true log.workspace = true diff --git a/crates/vst3-host/src/isolate.rs b/crates/vst3-host/src/isolate.rs new file mode 100644 --- /dev/null +++ b/crates/vst3-host/src/isolate.rs @@ -0,0 +1,238 @@ +//! Scanning in helper processes: each bundle is loaded by a short-lived child (the app's own +//! executable started with [`SCAN_FLAG`]) that prints its descriptors and exits. +//! +//! Scanning in-process loads every installed binary for the life of the process. Each one runs +//! its static initialisers and keeps whatever they took, notably thread-specific data keys, of +//! which a process has only `PTHREAD_KEYS_MAX` (512 on macOS). With a few hundred plugins a scan +//! alone uses most of them; the next plugin that starts a `std::thread` then aborts in libc++ +//! (`__thread_specific_ptr construction failed`), which took SoundCraft down when opening a Waves +//! plugin after a scan of ~190 bundles. Out of process, the app only loads the plugins it uses, +//! and a plugin that crashes or hangs while being scanned is skipped instead of taking the app +//! down. + +use crate::{ID_PREFIX, Vst3Descriptor, Vst3Error, parse_id, registry}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Mutex, OnceLock, PoisonError, mpsc}; +use std::time::{Duration, Instant}; + +/// The argument that makes an executable scan one bundle (the path follows) and exit. +pub const SCAN_FLAG: &str = "--scan-vst3-bundle"; +/// The line a child prints after its result; anything else is a crash. +const END: &str = "soundcraft-vst3-scan-end"; +/// Longest a child may take to load one bundle. +const TIMEOUT: Duration = Duration::from_secs(60); +/// Longest a child may take to exit once it has printed its result (module exits). +const EXIT_GRACE: Duration = Duration::from_secs(5); +/// Upper bound on what is read from one child. +const MAX_OUTPUT: u64 = 8 << 20; +/// Most children running at once. +const MAX_WORKERS: usize = 8; + +fn scanner() -> &'static Mutex> { + static S: OnceLock>> = OnceLock::new(); + S.get_or_init(|| Mutex::new(None)) +} + +/// Sets (or clears) the executable that scans bundles in child processes. +pub fn set_scanner(exe: Option) { + *scanner().lock().unwrap_or_else(PoisonError::into_inner) = exe; +} + +/// The executable that scans bundles, when one is set. +pub fn scanner_exe() -> Option { + scanner().lock().unwrap_or_else(PoisonError::into_inner).clone() +} + +/// The child side: loads `path` in this process, prints the result and runs the module exits. +/// Returns the process exit code. +pub fn child_main(path: &Path) -> i32 { + let result = registry::load_bundle(path).map_err(|e| e.to_string()); + let code = i32::from(result.is_err()); + let json = serde_json::to_string(&result).unwrap_or_else(|e| format!("{{\"Err\":\"cannot encode the scan: {e}\"}}")); + let mut out = std::io::stdout().lock(); + let _ = writeln!(out, "{json}\n{END}"); + let _ = out.flush(); + drop(out); + crate::shutdown(); + code +} + +/// Decodes a child's output for `path`: descriptors with valid ids, all pointing at `path`. +pub(crate) fn decode(path: &Path, json: &str) -> Result, Vst3Error> { + let err = |m: String| Vst3Error::Load(path.display().to_string(), m); + let result: Result, String> = serde_json::from_str(json).map_err(|e| err(format!("unreadable scan result: {e}")))?; + let mut ds = result.map_err(|m| err(m.strip_prefix(&format!("cannot load VST3 plugin {}: ", path.display())).unwrap_or(&m).to_string()))?; + ds.retain(|d| parse_id(&d.id).is_some() && d.id.strip_prefix(ID_PREFIX) == Some(d.class_id.as_str())); + for d in &mut ds { + d.path = path.to_path_buf(); + } + Ok(ds) +} + +/// Kills `child` unless it exits within `grace`, and reaps it. +fn reap(mut child: Child, grace: Duration) -> Option { + let deadline = Instant::now() + grace; + loop { + match child.try_wait() { + Ok(Some(status)) => return Some(status), + Ok(None) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(10)), + _ => { + let _ = child.kill(); + return child.wait().ok(); + } + } + } +} + +/// Scans one bundle in a child process. +pub(crate) fn scan_one(exe: &Path, path: &Path) -> Result, Vst3Error> { + let err = |m: String| Vst3Error::Load(path.display().to_string(), m); + let mut child = Command::new(exe) + .arg(SCAN_FLAG) + .arg(path) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|e| err(format!("cannot start the plugin scanner: {e}")))?; + let Some(stdout) = child.stdout.take() else { + reap(child, Duration::ZERO); + return Err(err("the plugin scanner has no output".into())); + }; + // Read on a thread so a child that never writes (or leaves a helper process holding the + // pipe open) cannot block past the timeout. + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let mut json = None; + for line in BufReader::new(stdout.take(MAX_OUTPUT)).lines() { + let Ok(line) = line else { break }; + if line == END { + break; + } + json = Some(line); + } + let _ = tx.send(json); + }); + match rx.recv_timeout(TIMEOUT) { + Ok(Some(json)) => { + reap(child, EXIT_GRACE); + decode(path, &json) + } + Ok(None) | Err(mpsc::RecvTimeoutError::Disconnected) => { + let status = reap(child, EXIT_GRACE); + Err(err(format!("the plugin crashed while being scanned ({})", status.map_or_else(|| "no status".to_string(), |s| s.to_string())))) + } + Err(mpsc::RecvTimeoutError::Timeout) => { + reap(child, Duration::ZERO); + Err(err(format!("the plugin did not load within {} s", TIMEOUT.as_secs()))) + } + } +} + +/// Scans `bundles` in child processes, several at a time. Results are in the order of `bundles`. +pub(crate) fn scan_all(exe: &Path, bundles: &[PathBuf]) -> Vec, Vst3Error>> { + let workers = std::thread::available_parallelism().map_or(2, |n| n.get()).clamp(1, MAX_WORKERS).min(bundles.len()); + let next = AtomicUsize::new(0); + let done: Mutex, Vst3Error>)>> = Mutex::new(Vec::with_capacity(bundles.len())); + std::thread::scope(|s| { + for _ in 0..workers { + s.spawn(|| { + loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(path) = bundles.get(i) else { break }; + let r = scan_one(exe, path); + done.lock().unwrap_or_else(PoisonError::into_inner).push((i, r)); + } + }); + } + }); + let mut done = done.into_inner().unwrap_or_else(PoisonError::into_inner); + done.sort_by_key(|(i, _)| *i); + done.into_iter().map(|(_, r)| r).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use soundcraft_dsp::Category; + + fn desc(class_id: &str) -> Vst3Descriptor { + Vst3Descriptor { + id: format!("{ID_PREFIX}{class_id}"), + class_id: class_id.to_string(), + name: "Comp".into(), + vendor: "V".into(), + version: "1".into(), + sdk_version: "VST 3.7".into(), + sub_categories: vec!["Fx".into(), "Dynamics".into()], + category: Category::Dynamics, + is_instrument: false, + path: PathBuf::from("/elsewhere.vst3"), + } + } + + #[test] + fn child_output_decodes_and_is_not_trusted() { + let path = Path::new("/p/Comp.vst3"); + let good = desc("0123456789ABCDEF0123456789ABCDEF"); + let mut bad = desc("0123456789ABCDEF0123456789ABCDEF"); + bad.id = "vst3:nonsense".into(); + let json = serde_json::to_string(&Ok::<_, String>(vec![good.clone(), bad])).unwrap(); + let ds = decode(path, &json).unwrap(); + assert_eq!(ds.len(), 1, "invalid ids are dropped"); + assert_eq!(ds[0].path, path, "the path is the one the parent asked for"); + assert_eq!(ds[0].name, good.name); + + let json = + serde_json::to_string(&Err::, _>(format!("cannot load VST3 plugin {}: no loadable binary", path.display()))).unwrap(); + assert_eq!(decode(path, &json).unwrap_err().to_string(), format!("cannot load VST3 plugin {}: no loadable binary", path.display())); + assert!(decode(path, "not json").is_err()); + assert!(decode(path, "").is_err()); + } + + #[test] + fn a_scanner_that_cannot_start_is_an_error() { + let r = scan_one(Path::new("/no/such/scanner"), Path::new("/p/x.vst3")); + assert!(r.unwrap_err().to_string().contains("cannot start the plugin scanner")); + } + + // Unix only: uses `/bin/sh` scripts as stand-in scanners. + #[cfg(unix)] + #[test] + fn crashing_and_silent_scanners_are_errors_and_results_keep_their_order() { + use std::os::unix::fs::PermissionsExt; + let d = std::env::temp_dir().join(format!("soundcraft-vst3-isolate-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + let script = |name: &str, body: &str| { + let p = d.join(name); + std::fs::write(&p, format!("#!/bin/sh\n{body}\n")).unwrap(); + std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap(); + p + }; + let crash = script("crash.sh", "kill -ABRT $$"); + let r = scan_one(&crash, Path::new("/p/x.vst3")); + assert!(r.unwrap_err().to_string().contains("crashed")); + + let json = serde_json::to_string(&Ok::<_, String>(vec![desc("0123456789ABCDEF0123456789ABCDEF")])).unwrap(); + // Prints a result for bundles named `ok*`, crashes on the others. + let mixed = script("mixed.sh", &format!("case \"$2\" in */ok*) printf '%s\\n{END}\\n' '{json}';; *) exit 3;; esac")); + let bundles: Vec = + (0..12).map(|i| PathBuf::from(if i % 3 == 0 { format!("/p/bad{i}.vst3") } else { format!("/p/ok{i}.vst3") })).collect(); + let results = scan_all(&mixed, &bundles); + assert_eq!(results.len(), bundles.len()); + for (b, r) in bundles.iter().zip(&results) { + match r { + Ok(ds) => { + assert!(b.to_string_lossy().contains("/ok")); + assert_eq!(ds[0].path, *b); + } + Err(_) => assert!(b.to_string_lossy().contains("/bad")), + } + } + let _ = std::fs::remove_dir_all(&d); + } +} diff --git a/crates/vst3-host/src/lib.rs b/crates/vst3-host/src/lib.rs --- a/crates/vst3-host/src/lib.rs +++ b/crates/vst3-host/src/lib.rs @@ -1,8 +1,9 @@ //! SoundCraft's VST3 plugin host. //! -//! Scans the standard VST3 folders (plus `VST3_PATH`), loads `.vst3` bundles in-process and -//! exposes each "Audio Module Class" as a [`soundcraft_dsp::Plugin`], so the mix engine treats -//! third-party plugins exactly like the built-in ones. Plugin ids are `vst3:` (32 +//! Scans the standard VST3 folders (plus `VST3_PATH`; in child processes when [`set_scanner`] is +//! set), loads `.vst3` bundles in-process and exposes each "Audio Module Class" as a +//! [`soundcraft_dsp::Plugin`], so the mix engine treats third-party plugins exactly like the +//! built-in ones. Plugin ids are `vst3:` (32 //! upper-case hex digits of the component's class id); parameter ids are VST3 `ParamID`s as //! decimal strings, with titles, units and plain ranges from the edit controller in //! [`PluginInfo::params`]. Values are plain (display) values; the host converts to and from the @@ -47,10 +48,14 @@ pub mod scan; #[allow(unsafe_code)] mod ffi; #[cfg(not(target_arch = "wasm32"))] +mod isolate; +#[cfg(not(target_arch = "wasm32"))] mod plugin; #[cfg(not(target_arch = "wasm32"))] mod registry; +#[cfg(not(target_arch = "wasm32"))] +pub use isolate::SCAN_FLAG; #[cfg(not(target_arch = "wasm32"))] pub use plugin::Vst3Plugin; @@ -78,7 +83,7 @@ pub fn parse_id(id: &str) -> Option<[u8; 16]> { } /// One plugin found by a scan. -#[derive(Debug, Clone, PartialEq, serde::Serialize)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct Vst3Descriptor { /// SoundCraft id: `vst3:`. pub id: String, @@ -194,6 +199,22 @@ pub fn add_search_dir(dir: &Path) -> usize { api::add_search_dir(dir) } +/// Makes scans load each bundle in a child process: `exe` started with [`SCAN_FLAG`] and the +/// bundle path, which must call [`scan_child_main`] (`None`: scan in this process, the default). +/// The app sets it so that a scan neither keeps every installed binary loaded nor dies with a +/// plugin that crashes while loading (see the `isolate` module). +#[cfg(not(target_arch = "wasm32"))] +pub fn set_scanner(exe: Option) { + isolate::set_scanner(exe) +} + +/// The child side of [`set_scanner`]: loads one bundle, prints what it holds for the parent and +/// runs the module exits. Returns the process exit code. +#[cfg(not(target_arch = "wasm32"))] +pub fn scan_child_main(path: &Path) -> i32 { + isolate::child_main(path) +} + /// Scans the given folders without touching the cache. pub fn scan_paths(dirs: &[PathBuf]) -> Vec { api::scan_paths(dirs) diff --git a/crates/vst3-host/src/registry.rs b/crates/vst3-host/src/registry.rs --- a/crates/vst3-host/src/registry.rs +++ b/crates/vst3-host/src/registry.rs @@ -53,12 +53,17 @@ pub fn load_bundle(path: &Path) -> Result, Vst3Error> { .collect()) } -/// Scans `dirs` (uncached). Unloadable files are logged and skipped; the first plugin with a -/// given id wins. +/// Scans `dirs` (uncached), in child processes when a scanner is set. Unloadable files are logged +/// and skipped; the first plugin with a given id wins. pub fn scan_paths(dirs: &[PathBuf]) -> Vec { + let bundles = paths::find_bundles(dirs); + let results = match crate::isolate::scanner_exe() { + Some(exe) => crate::isolate::scan_all(&exe, &bundles), + None => bundles.iter().map(|p| load_bundle(p)).collect(), + }; let mut out: Vec = Vec::new(); - for path in paths::find_bundles(dirs) { - match load_bundle(&path) { + for result in results { + match result { Ok(ds) => { for d in ds { if !out.iter().any(|o| o.id == d.id) { diff --git a/docs/architecture.md b/docs/architecture.md --- a/docs/architecture.md +++ b/docs/architecture.md @@ -61,6 +61,13 @@ and recordings are captured in memory until stop. `soundcraft-video` holds our own H.264, ProRes and Motion JPEG decoders and the ISO-BMFF demuxer for the video track. Hosted plugins (`clap-host`, `vst3-host`, `au-host`) are scanned, created and destroyed off the audio thread; only `process` runs on it. +The desktop app scans CLAP and VST3 bundles out of process: each bundle is loaded by a short-lived +child of the app's own executable (`--scan-clap-bundle` / `--scan-vst3-bundle `, several at a +time) that prints its descriptors and exits, so the app itself loads only the plugins a session +uses. A scan that loaded every installed binary in-process used up most of the process's +thread-specific data keys (`PTHREAD_KEYS_MAX`), and the next plugin to start a thread aborted +(Waves). A plugin that crashes or hangs while being scanned is skipped. Without a scanner set +(the CLI, tests) scans run in-process. The cpal callbacks mark their thread (`soundcraft_playback::mark_audio_thread`). Code that may run there can ask `on_audio_thread()` before doing anything that blocks; the desktop app's logger does,