diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -3585,6 +3585,7 @@ dependencies = [ "soundcraft-engine", "soundcraft-playback", "soundcraft-ui-egui", + "soundcraft-vst3-host", "winresource", ] @@ -3789,6 +3790,7 @@ dependencies = [ "objc2-app-kit 0.3.2", "objc2-foundation 0.3.2", "serde", + "serde_json", "soundcraft-dsp", "thiserror 2.0.21", "vst3", diff --git a/apps/soundcraft/Cargo.toml b/apps/soundcraft/Cargo.toml --- a/apps/soundcraft/Cargo.toml +++ b/apps/soundcraft/Cargo.toml @@ -18,6 +18,7 @@ asio = ["soundcraft-playback/asio"] soundcraft-engine.workspace = true soundcraft-ui-egui.workspace = true soundcraft-playback.workspace = true +soundcraft-vst3-host.workspace = true eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11", "links"] } egui.workspace = true serde_json.workspace = true diff --git a/apps/soundcraft/src/main.rs b/apps/soundcraft/src/main.rs --- a/apps/soundcraft/src/main.rs +++ b/apps/soundcraft/src/main.rs @@ -245,6 +245,13 @@ fn main() -> eframe::Result { } } + // Unchanged plugins are then listed from disk instead of loading every binary on each launch. + if prefs_enabled() + && let Some(dir) = config_dir() + { + soundcraft_vst3_host::set_cache_file(&dir.join("vst3-cache.json")); + } + let engine = if demo { soundcraft_engine::demo::demo_engine() } else { Engine::default() }; let prefs = load_prefs().unwrap_or_default(); let mut options = eframe::NativeOptions { diff --git a/crates/ui-egui/src/mix_window.rs b/crates/ui-egui/src/mix_window.rs --- a/crates/ui-egui/src/mix_window.rs +++ b/crates/ui-egui/src/mix_window.rs @@ -759,12 +759,15 @@ fn plugin_search(ui: &mut Ui, search: &MenuSearch, instruments: bool) -> Option< } if clap.is_none() || vst3.is_none() { ui.separator(); + if vst3.is_none() && soundcraft_vst3_host::scan_in_progress() { + vst3_scan_status(ui); + } ui.horizontal(|ui| { if clap.is_none() && ui.small_button("Scan CLAP").clicked() { soundcraft_clap_host::scan(); } - if vst3.is_none() && ui.small_button("Scan VST3").clicked() { - soundcraft_vst3_host::scan(); + if vst3.is_none() && !soundcraft_vst3_host::scan_in_progress() && ui.small_button("Scan VST3").clicked() { + vst3_scan_status(ui); } }); } @@ -851,6 +854,25 @@ struct MenuPlugin { vendor: String, } +/// UI time given to the VST3 scan per frame. Bundles the scan cache knows cost a file stat; a +/// new or changed one is loaded (at most one per frame), which can take a second or more. +const VST3_SCAN_BUDGET: std::time::Duration = std::time::Duration::from_millis(30); + +/// Advances the VST3 scan a step (starting it if needed) and, while it runs, shows its progress +/// and keeps frames coming. True once the list is ready. The scan stays on the UI thread: plugin +/// modules adopt the thread that loads them, and their editors must open on this one. +fn vst3_scan_status(ui: &mut Ui) -> bool { + match soundcraft_vst3_host::scan_step(VST3_SCAN_BUDGET) { + soundcraft_vst3_host::ScanProgress::Done => true, + soundcraft_vst3_host::ScanProgress::Scanning { done, total, next } => { + ui.label(format!("Scanning VST3 plugins… {done} of {total}")); + ui.weak(next); + ui.ctx().request_repaint(); + false + } + } +} + /// The CLAP, VST3 and (on macOS) Audio Units submenus, of effects or of instruments; returns the /// id picked. Each format is scanned the first time its submenu opens, then cached by its host. fn hosted_menus(ui: &mut Ui, instruments: bool) -> Option { @@ -862,7 +884,10 @@ fn hosted_menus(ui: &mut Ui, instruments: bool) -> Option { } }); ui.menu_button("VST3", |ui| { - let list = soundcraft_vst3_host::scan().into_iter().filter(|d| d.is_instrument == instruments); + if !vst3_scan_status(ui) { + return; + } + let list = soundcraft_vst3_host::cached().unwrap_or_default().into_iter().filter(|d| d.is_instrument == instruments); if let Some(p) = vendor_menus(ui, list.map(|d| MenuPlugin { id: d.id, name: d.name, vendor: d.vendor }).collect(), "No VST3 plugins found") { picked = Some(p); } diff --git a/crates/vst3-host/Cargo.toml b/crates/vst3-host/Cargo.toml --- a/crates/vst3-host/Cargo.toml +++ b/crates/vst3-host/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true [dependencies] soundcraft-dsp.workspace = true serde.workspace = true +serde_json.workspace = true thiserror.workspace = true log.workspace = true diff --git a/crates/vst3-host/src/cache.rs b/crates/vst3-host/src/cache.rs new file mode 100644 --- /dev/null +++ b/crates/vst3-host/src/cache.rs @@ -0,0 +1,137 @@ +//! The on-disk scan cache: what each `.vst3` held the last time it was loaded, keyed by the +//! bundle path plus its binary's size and modification time. A bundle whose binary is unchanged +//! is listed from the cache without loading it, so a launch with a cache loads no plugin code +//! until a plugin is actually used. Bundles that failed to load are remembered too (not retried +//! until their binary changes or the user rescans). Safe, pure `std::fs` + serde. + +use crate::Vst3Descriptor; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::time::UNIX_EPOCH; + +/// Bumped whenever the entry layout or the meaning of a descriptor changes; other versions are ignored. +const VERSION: u32 = 1; +/// Largest cache file read (a hostile or corrupt file must not exhaust memory). +const MAX_BYTES: u64 = 16 * 1024 * 1024; + +/// The size and modification time of a bundle's binary (what decides whether it changed). +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Stamp { + pub size: u64, + pub modified_ns: u64, +} + +impl Stamp { + /// The stamp of `bundle`'s binary; `None` when it has none or cannot be read. + pub fn of(bundle: &Path) -> Option { + let exe = crate::scan::bundle_binary(bundle)?; + let md = std::fs::metadata(exe).ok()?; + let modified = md.modified().ok()?.duration_since(UNIX_EPOCH).ok()?; + Some(Stamp { size: md.len(), modified_ns: u64::try_from(modified.as_nanos()).unwrap_or(u64::MAX) }) + } +} + +/// One bundle as it was last loaded: its plugin classes, or why it would not load. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Entry { + pub path: PathBuf, + pub stamp: Stamp, + pub descriptors: Vec, + pub error: Option, +} + +#[derive(serde::Serialize, serde::Deserialize)] +struct File { + version: u32, + entries: Vec, +} + +/// Reads a cache file; empty when it is missing, too big, unreadable or another version. +pub fn read(path: &Path) -> HashMap { + let ok = std::fs::metadata(path).is_ok_and(|m| m.is_file() && m.len() <= MAX_BYTES); + let parsed = ok.then(|| std::fs::read(path).ok()).flatten().and_then(|b| serde_json::from_slice::(&b).ok()); + match parsed { + Some(f) if f.version == VERSION => f.entries.into_iter().map(|e| (e.path.clone(), e)).collect(), + _ => HashMap::new(), + } +} + +/// Writes the cache atomically (a temporary file renamed over the old one). +pub fn write(path: &Path, entries: Vec) -> std::io::Result<()> { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + let json = serde_json::to_vec_pretty(&File { version: VERSION, entries }).map_err(std::io::Error::other)?; + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, json)?; + std::fs::rename(&tmp, path) +} + +#[cfg(test)] +mod tests { + use super::*; + use soundcraft_dsp::Category; + + fn tmp(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("sc-vst3-cache-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + fn entry(path: &Path) -> Entry { + Entry { + path: path.to_path_buf(), + stamp: Stamp { size: 3, modified_ns: 7 }, + descriptors: vec![Vst3Descriptor { + id: "vst3:00".into(), + class_id: "00".into(), + name: "Bass".into(), + vendor: "V".into(), + version: "1".into(), + sdk_version: "VST 3.7".into(), + sub_categories: vec!["Instrument".into()], + category: Category::Instrument, + is_instrument: true, + path: path.to_path_buf(), + }], + error: None, + } + } + + #[test] + fn round_trips() { + let d = tmp("rt"); + let f = d.join("sub").join("vst3-cache.json"); + let e = entry(&d.join("Bass.vst3")); + write(&f, vec![e.clone()]).unwrap(); + let back = read(&f); + assert_eq!(back.get(&e.path), Some(&e)); + let _ = std::fs::remove_dir_all(&d); + } + + #[test] + fn junk_missing_and_other_versions_read_empty() { + let d = tmp("junk"); + assert!(read(&d.join("missing.json")).is_empty()); + std::fs::write(d.join("junk.json"), b"{not json").unwrap(); + assert!(read(&d.join("junk.json")).is_empty()); + std::fs::write(d.join("v0.json"), br#"{"version":0,"entries":[]}"#).unwrap(); + assert!(read(&d.join("v0.json")).is_empty()); + assert!(read(&d).is_empty(), "a directory is not a cache file"); + let _ = std::fs::remove_dir_all(&d); + } + + #[test] + fn stamp_follows_the_binary() { + let d = tmp("stamp"); + let b = d.join("one.vst3"); + assert_eq!(Stamp::of(&b), None); + std::fs::write(&b, b"abc").unwrap(); + let s = Stamp::of(&b).unwrap(); + assert_eq!(s.size, 3); + std::fs::write(&b, b"abcdef").unwrap(); + assert_ne!(Stamp::of(&b), Some(s)); + let _ = std::fs::remove_dir_all(&d); + } +} diff --git a/crates/vst3-host/src/lib.rs b/crates/vst3-host/src/lib.rs --- a/crates/vst3-host/src/lib.rs +++ b/crates/vst3-host/src/lib.rs @@ -43,6 +43,8 @@ pub mod scan; +#[cfg(not(target_arch = "wasm32"))] +mod cache; #[cfg(not(target_arch = "wasm32"))] #[allow(unsafe_code)] mod ffi; @@ -56,6 +58,7 @@ pub use plugin::Vst3Plugin; use soundcraft_dsp::{Category, Plugin, PluginInfo}; use std::path::{Path, PathBuf}; +use std::time::Duration; /// Prefix of every hosted-VST3 plugin id. pub const ID_PREFIX: &str = "vst3:"; @@ -78,7 +81,7 @@ pub fn parse_id(id: &str) -> Option<[u8; 16]> { } /// One plugin found by a scan. -#[derive(Debug, Clone, PartialEq, serde::Serialize)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct Vst3Descriptor { /// SoundCraft id: `vst3:`. pub id: String, @@ -112,6 +115,15 @@ pub enum Vst3Error { Unsupported, } +/// Where a scan advanced with [`scan_step`] has got to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ScanProgress { + /// The list is ready ([`cached`] returns it). + Done, + /// `done` of `total` bundles listed; `next` is the one being listed next. + Scanning { done: usize, total: usize, next: String }, +} + #[cfg(not(target_arch = "wasm32"))] mod api { use super::*; @@ -125,6 +137,15 @@ mod api { pub fn rescan() -> Vec { registry::rescan() } + pub fn scan_step(budget: Duration) -> ScanProgress { + registry::scan_step(budget) + } + pub fn scan_in_progress() -> bool { + registry::scan_in_progress() + } + pub fn set_cache_file(path: &Path) { + registry::set_cache_file(path) + } pub fn add_search_dir(dir: &Path) -> usize { registry::add_search_dir(dir) } @@ -158,6 +179,13 @@ mod api { pub fn rescan() -> Vec { Vec::new() } + pub fn scan_step(_budget: Duration) -> ScanProgress { + ScanProgress::Done + } + pub fn scan_in_progress() -> bool { + false + } + pub fn set_cache_file(_path: &Path) {} pub fn add_search_dir(_dir: &Path) -> usize { 0 } @@ -177,17 +205,38 @@ mod api { } /// Every plugin in the standard VST3 folders, `VST3_PATH` and folders added with -/// [`add_search_dir`]. Scanned on first call (loads each binary and reads its factory, but -/// creates no plugin instances), then cached in memory. +/// [`add_search_dir`]. Scanned on first call (loads each new or changed binary and reads its +/// factory, but creates no plugin instances; see [`set_cache_file`]), then cached in memory. pub fn scan() -> Vec { api::scan() } -/// Forgets the cached scan (and remembered failures) and scans again. +/// Forgets the cached scan (and remembered failures) and scans again, loading every bundle +/// (the disk cache is rewritten, not consulted). pub fn rescan() -> Vec { api::rescan() } +/// Advances the scan without blocking for long, for a UI that must keep drawing: bundles the +/// disk cache knows (unchanged binary) are listed without loading them; at most one bundle that +/// has to be loaded is loaded per call once any work is done. Starts the scan if needed. Call it +/// on the main thread: modules adopt the thread that loads them (JUCE ties its message thread to +/// it), so loading on a worker would break their editors later. +pub fn scan_step(budget: Duration) -> ScanProgress { + api::scan_step(budget) +} + +/// Whether a scan started by [`scan_step`] is still under way. +pub fn scan_in_progress() -> bool { + api::scan_in_progress() +} + +/// Caches scans in `path` (JSON): later scans list unchanged bundles without loading them, and +/// skip bundles that failed to load until they change. Call once at startup. +pub fn set_cache_file(path: &Path) { + api::set_cache_file(path) +} + /// Adds a folder to the search path and merges its plugins into the cache. Returns how many /// plugins the folder holds. pub fn add_search_dir(dir: &Path) -> usize { diff --git a/crates/vst3-host/src/registry.rs b/crates/vst3-host/src/registry.rs --- a/crates/vst3-host/src/registry.rs +++ b/crates/vst3-host/src/registry.rs @@ -1,13 +1,15 @@ //! The process-wide VST3 registry: scan cache, leaked `PluginInfo`s, instantiation. +use crate::cache::{self, Entry, Stamp}; use crate::ffi::{Bundle, Instance, PARAM_BYPASS, PARAM_HIDDEN, PARAM_READ_ONLY, RawParam}; use crate::plugin::Vst3Plugin; use crate::scan as paths; -use crate::{AUDIO_MODULE_CLASS, Vst3Descriptor, Vst3Error, parse_id}; +use crate::{AUDIO_MODULE_CLASS, ScanProgress, Vst3Descriptor, Vst3Error, parse_id}; use soundcraft_dsp::{ParamInfo, Plugin, PluginInfo, Taper, Unit}; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock, PoisonError}; +use std::time::{Duration, Instant}; /// Most labels read for a stepped parameter (more steps are shown as a plain range). const MAX_CHOICES: i32 = 128; @@ -21,6 +23,81 @@ struct Registry { infos: HashMap, /// Ids that failed to instantiate (not retried every UI frame). failed: HashSet, + /// Where the scan cache lives (`None`: scans are not cached on disk). + cache_file: Option, + /// The scan under way, advanced a bundle at a time by [`scan_step`]. + job: Option, +} + +/// A scan in progress: the bundles still to list and what the finished ones held. +struct ScanJob { + pending: VecDeque, + total: usize, + /// The cache read at the start (empty when rescanning: every bundle is loaded again). + known: HashMap, + found: Vec, + entries: Vec, +} + +impl ScanJob { + fn new(dirs: &[PathBuf], known: HashMap) -> ScanJob { + let pending: VecDeque = paths::find_bundles(dirs).into(); + ScanJob { total: pending.len(), pending, known, found: Vec::new(), entries: Vec::new() } + } + + /// Adds `bundles` the scan has not listed or queued yet (a folder added while it runs). + fn include(&mut self, bundles: Vec) { + for b in bundles { + if !self.pending.contains(&b) && !self.entries.iter().any(|e| e.path == b) { + self.pending.push_back(b); + self.total += 1; + } + } + } + + fn progress(&self) -> ScanProgress { + let next = self.pending.front().and_then(|p| p.file_stem()).map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(); + ScanProgress::Scanning { done: self.total.saturating_sub(self.pending.len()), total: self.total, next } + } + + /// Lists bundles until `budget` is spent; a bundle that must be loaded (new, changed or never + /// cached) is the slow part, so at most one is loaded per call once any work has been done. + fn step(&mut self, budget: Duration) { + let start = Instant::now(); + let mut loaded = false; + while let Some(path) = self.pending.front().cloned() { + let stamp = Stamp::of(&path); + let cached = stamp.and_then(|st| self.known.get(&path).filter(|e| e.stamp == st)).cloned(); + if cached.is_none() && loaded { + break; + } + self.pending.pop_front(); + let entry = match (cached, stamp) { + (Some(e), _) => e, + (None, stamp) => { + loaded = true; + let (descriptors, error) = match load_bundle(&path) { + Ok(ds) => (ds, None), + Err(e) => { + log::warn!("{e}"); + (Vec::new(), Some(e.to_string())) + } + }; + let stamp = stamp.unwrap_or(Stamp { size: 0, modified_ns: 0 }); + Entry { path: path.clone(), stamp, descriptors, error } + } + }; + for d in &entry.descriptors { + if !self.found.iter().any(|o| o.id == d.id) { + self.found.push(d.clone()); + } + } + self.entries.push(entry); + if start.elapsed() >= budget { + break; + } + } + } } fn registry() -> &'static Mutex { @@ -73,11 +150,46 @@ pub fn scan_paths(dirs: &[PathBuf]) -> Vec { out } +fn search_dirs(r: &Registry) -> Vec { + let mut dirs = paths::default_search_paths(); + dirs.extend(r.extra_dirs.iter().cloned()); + dirs +} + +/// Starts a scan (unless one is under way): from the disk cache, or loading every bundle when `fresh`. +fn start_job(r: &mut Registry, fresh: bool) { + if r.job.is_none() { + let known = match (&r.cache_file, fresh) { + (Some(f), false) => cache::read(f), + _ => HashMap::new(), + }; + r.job = Some(ScanJob::new(&search_dirs(r), known)); + } +} + +/// Advances the scan; when it has finished, publishes the list and saves the cache. +fn advance(r: &mut Registry, budget: Duration) -> ScanProgress { + let Some(job) = r.job.as_mut() else { return ScanProgress::Done }; + job.step(budget); + if !job.pending.is_empty() { + return job.progress(); + } + let Some(job) = r.job.take() else { return ScanProgress::Done }; + let mut found = job.found; + found.sort_by_key(|d| d.name.to_lowercase()); + r.scanned = Some(found); + if let Some(f) = &r.cache_file + && let Err(e) = cache::write(f, job.entries) + { + log::warn!("vst3: cannot save the scan cache {}: {e}", f.display()); + } + ScanProgress::Done +} + fn ensure_scanned(r: &mut Registry) -> &Vec { if r.scanned.is_none() { - let mut dirs = paths::default_search_paths(); - dirs.extend(r.extra_dirs.iter().cloned()); - r.scanned = Some(scan_paths(&dirs)); + start_job(r, false); + while advance(r, Duration::MAX) != ScanProgress::Done {} } r.scanned.get_or_insert_with(Vec::new) } @@ -94,10 +206,29 @@ pub fn cached() -> Option> { pub fn rescan() -> Vec { let mut r = registry().lock().unwrap_or_else(PoisonError::into_inner); r.scanned = None; + r.job = None; r.failed.clear(); + start_job(&mut r, true); ensure_scanned(&mut r).clone() } +pub fn scan_step(budget: Duration) -> ScanProgress { + let mut r = registry().lock().unwrap_or_else(PoisonError::into_inner); + if r.scanned.is_some() { + return ScanProgress::Done; + } + start_job(&mut r, false); + advance(&mut r, budget) +} + +pub fn scan_in_progress() -> bool { + registry().lock().unwrap_or_else(PoisonError::into_inner).job.is_some() +} + +pub fn set_cache_file(path: &Path) { + registry().lock().unwrap_or_else(PoisonError::into_inner).cache_file = Some(path.to_path_buf()); +} + pub fn add_search_dir(dir: &Path) -> usize { let mut r = registry().lock().unwrap_or_else(PoisonError::into_inner); if !r.extra_dirs.iter().any(|d| d == dir) { @@ -105,6 +236,14 @@ pub fn add_search_dir(dir: &Path) -> usize { } let found = scan_paths(&[dir.to_path_buf()]); let n = found.len(); + // A scan under way lists the folder too (its bundles are loaded now, so that is cheap): + // a list of only this folder would end it with the standard folders' plugins missing. + if r.scanned.is_none() + && let Some(job) = r.job.as_mut() + { + job.include(paths::find_bundles(&[dir.to_path_buf()])); + return n; + } let list = r.scanned.get_or_insert_with(Vec::new); for d in found { if !list.iter().any(|o| o.id == d.id) { @@ -269,4 +408,39 @@ mod tests { assert_eq!(unit_from_label("st"), Unit::Semitones); assert_eq!(unit_from_label("furlongs"), Unit::None); } + + fn folder_with(name: &str, bundles: &[&str]) -> PathBuf { + let d = std::env::temp_dir().join(format!("sc-vst3-job-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + for b in bundles { + // Not a loadable module: the scan records why and moves on. + std::fs::write(d.join(b), b"not a plugin").unwrap(); + } + d + } + + #[test] + fn a_folder_added_during_a_scan_joins_it_and_unchanged_bundles_come_from_the_cache() { + let (a, b) = (folder_with("a", &["One.vst3"]), folder_with("b", &["Two.vst3", "Three.vst3"])); + let mut job = ScanJob::new(std::slice::from_ref(&a), HashMap::new()); + assert_eq!(job.total, 1); + job.include(paths::find_bundles(std::slice::from_ref(&b))); + job.include(paths::find_bundles(std::slice::from_ref(&b))); + assert_eq!(job.total, 3, "an added folder's bundles are queued once"); + // At most one bundle is loaded per step: three steps for three uncached bundles. + for _ in 0..3 { + job.step(Duration::MAX); + } + assert!(job.pending.is_empty()); + assert_eq!(job.entries.len(), 3); + assert!(job.entries.iter().all(|e| e.error.is_some() && e.descriptors.is_empty())); + // A later scan lists every unchanged bundle from the cache in one step, loading none. + let known: HashMap = job.entries.iter().map(|e| (e.path.clone(), e.clone())).collect(); + let mut again = ScanJob::new(&[a.clone(), b.clone()], known); + again.step(Duration::MAX); + assert!(again.pending.is_empty(), "nothing had to be loaded"); + assert_eq!(again.entries, job.entries, "the same entries, in the same (path) order"); + let _ = (std::fs::remove_dir_all(&a), std::fs::remove_dir_all(&b)); + } }