collective
Browse / github.com/storytold/lightcraft / patch
#366: SAM 3 mirrors accept a local folder; tokenizer files pinned
#366: SAM 3 mirrors accept a local folder; tokenizer files pinned **Problem:** people who download the `facebook/sam3` checkpoint by hand (gated on Hugging Face, see #366) had no way to point the app at it except through an HTTP mirror. **Change** - `lightcraft-fetch`: a mirror may be a `file://` URL or an absolute path to a folder. Files are copied from it with the same size cap, size and SHA-256 checks and cancel handling as downloads. A missing folder or file gives a clear error. - `lightcraft-segment`: `vocab.json` and `merges.txt` are pinned (exact size + SHA-256 of the official checkpoint), which resolves the TODO. Both crates are Apache-2.0; the cha
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- emmertex (imported from pull request #3010, open)
- Base
387bed199fca44d76af5aa1c24dd7cc3de4c8db2- Size
- +149 −13 in 3 files
- Review
- pending
- Shared
- 2026-10-28
- Id
1e8afc95986b45e9d6db729f
How it went
No other operator has reported using it yet.
Diff
raw · crates/fetch/src/lib.rs crates/fetch/src/tests.rs crates/segment/src/fetch/mod.rs
@@ -50,12 +50,17 @@ const MAX_REDIRECTS: usize = 8; /// The mirrors to try, in order: `env`'s (base URLs separated by commas, spaces or newlines), then /// the mirrors file's (one base URL per line, `#` comments), then `defaults`; duplicates and-/// unusable URLs dropped, at most 16.+/// unusable URLs dropped, at most 16. A mirror can be `http(s)://`, `file://<absolute path>` or an+/// absolute folder path: files from a local folder are copied (then checked like a download). pub fn mirrors(env: Option<&str>, file: Option<&Path>, defaults: &[&str]) -> Vec<String> { let mut out: Vec<String> = Vec::new(); let mut add = |s: &str| {- let s = s.trim().trim_end_matches('/');- if !s.is_empty() && Url::parse(s).is_ok() && !out.iter().any(|m| m == s) && out.len() < MAX_MIRRORS {+ let s = s.trim();+ // a local folder (an absolute path) is the same as its `file://` URL+ let s = if Path::new(s).is_absolute() { format!("file://{}", s.replace('%', "%25")) } else { s.to_string() };+ let s = s.trim_end_matches('/');+ let usable = local_path(s).is_some() || Url::parse(s).is_ok();+ if !s.is_empty() && usable && !out.iter().any(|m| m == s) && out.len() < MAX_MIRRORS { out.push(s.to_string()); } };@@ -202,7 +207,7 @@ fn download_source( return Err(DownloadError::Cancelled); } let mut report = |n: u64, announced: u64| {- let mirror = Url::parse(m).map(|u| u.display()).unwrap_or_default();+ let mirror = short(m); progress(&Progress { file: f.name.to_string(), done: before.saturating_add(n),@@ -282,6 +287,9 @@ fn bearer<'a>(token: Option<&'a str>, url: &Url, origin: &str) -> Option<&'a str /// A mirror for messages (no query string: it may hold a token). fn short(m: &str) -> String {+ if local_path(m).is_some() {+ return m.to_string();+ } Url::parse(m).map(|u| u.display()).unwrap_or_else(|_| "mirror".into()) } @@ -343,8 +351,11 @@ fn fetch_file(f: &FileSpec<'_>, url: &str, dir: &Path, opts: &Options, cancel: & } Err(_) => 0, };- let fetched = f.size != Some(have) || have == 0;- if fetched {+ let local = local_path(url);+ let fetched = local.is_some() || f.size != Some(have) || have == 0;+ if let Some(src) = local {+ copy_local(&src, &part, limit, f.size, cancel, report)?;+ } else if fetched { let limits = Limits { connect: opts.connect_timeout, stall: opts.stall_timeout, cancel }; let mut url = Url::parse(url).map_err(|e| Fail::NextMirror(e.to_string()))?; let origin = url.host.clone();@@ -453,6 +464,78 @@ fn fetch_file(f: &FileSpec<'_>, url: &str, dir: &Path, opts: &Options, cancel: & Ok(()) } +/// The local path of a `file://` URL (absolute only; `file:///C:/x` → `C:/x` on Windows).+fn local_path(url: &str) -> Option<PathBuf> {+ let rest = url.strip_prefix("file://")?;+ let rest = if rest.get(2..3) == Some(":") { rest.strip_prefix('/').unwrap_or(rest) } else { rest };+ let rest = percent_decode(rest)?;+ let p = PathBuf::from(&rest);+ if p.is_absolute() && !rest.chars().any(char::is_control) { Some(p) } else { None }+}++/// Decode `%XX` escapes (a `file://` URL's path); `None` for a broken escape or non-UTF-8 result.+fn percent_decode(s: &str) -> Option<String> {+ let b = s.as_bytes();+ let mut out = Vec::with_capacity(b.len());+ let mut i = 0;+ while let Some(&c) = b.get(i) {+ if c == b'%' {+ let hex = std::str::from_utf8(b.get(i + 1..i + 3)?).ok()?;+ out.push(u8::from_str_radix(hex, 16).ok()?);+ i += 3;+ } else {+ out.push(c);+ i += 1;+ }+ }+ String::from_utf8(out).ok()+}++/// Copy a file from a local mirror into `part` (from the start; it is checked afterwards like a download).+fn copy_local(src: &Path, part: &Path, limit: u64, size: Option<u64>, cancel: &AtomicBool, report: &mut dyn FnMut(u64, u64)) -> Result<(), Fail> {+ let disk = |e: std::io::Error| Fail::Disk(format!("{}: {e}", part.display()));+ let meta = std::fs::metadata(src).map_err(|e| Fail::NextMirror(format!("{}: {e}", src.display())))?;+ if !meta.is_file() {+ return Err(Fail::NextMirror(format!("{} is not a file", src.display())));+ }+ if meta.len() > limit || size.is_some_and(|s| s != meta.len()) {+ return Err(Fail::NextMirror(format!("{} has the wrong size ({} bytes)", src.display(), meta.len())));+ }+ let mut input = File::open(src).map_err(|e| Fail::NextMirror(format!("{}: {e}", src.display())))?;+ let mut out = File::create(part).map_err(disk)?;+ let announced = size.unwrap_or(meta.len());+ let mut have = 0u64;+ report(0, announced);+ let mut buf = vec![0u8; 1 << 20];+ loop {+ if cancel.load(Ordering::Relaxed) {+ drop(out);+ let _ = std::fs::remove_file(part);+ return Err(Fail::Cancelled);+ }+ let n = match input.read(&mut buf) {+ Ok(n) => n,+ Err(e) => {+ drop(out);+ let _ = std::fs::remove_file(part);+ return Err(Fail::NextMirror(format!("{}: {e}", src.display())));+ }+ };+ if n == 0 {+ break;+ }+ if have.saturating_add(n as u64) > limit {+ drop(out);+ let _ = std::fs::remove_file(part);+ return Err(Fail::NextMirror(format!("{} grew past the expected {limit} bytes", src.display())));+ }+ out.write_all(buf.get(..n).unwrap_or_default()).map_err(disk)?;+ have = have.saturating_add(n as u64);+ report(have, announced);+ }+ out.sync_all().map_err(disk)+}+ fn http_fail(e: HttpError) -> Fail { match e { HttpError::Cancelled => Fail::Cancelled,@@ -408,3 +408,49 @@ fn hostile_file_specs_are_rejected_before_touching_disk() { assert!(!dir.exists()); } }++#[test]+fn copies_from_a_local_folder_mirror_and_checks_the_hash() {+ let src = tmp("local-src");+ let dst = tmp("local-dst");+ std::fs::create_dir_all(&src).unwrap();+ std::fs::write(src.join("model.bin"), payload()).unwrap();+ // a bare absolute path and a file:// URL are both mirrors+ let m = mirrors(Some(src.to_str().unwrap()), None, &[]);+ assert_eq!(m, vec![format!("file://{}", src.display())]);+ let (r, prog) = run(&[pinned()], &m, &dst, &AtomicBool::new(false));+ r.unwrap();+ assert_eq!(std::fs::read(dst.join("model.bin")).unwrap(), payload());+ assert!(prog.last().is_some_and(|p| p.done == p.total));++ // a damaged local copy is rejected and never moved into place+ let bad = tmp("local-bad");+ std::fs::create_dir_all(&bad).unwrap();+ let mut damaged = payload();+ damaged[7] ^= 1;+ std::fs::write(bad.join("model.bin"), damaged).unwrap();+ let dst2 = tmp("local-dst2");+ let (r, _) = run(&[pinned()], &[format!("file://{}", bad.display())], &dst2, &AtomicBool::new(false));+ assert!(matches!(r, Err(DownloadError::AllMirrorsFailed { .. })), "{r:?}");+ assert!(!dst2.join("model.bin").exists() && !dst2.join("model.bin.part").exists());++ // a missing folder, a relative path and a directory in place of a file are errors, not panics+ let (r, _) = run(&[pinned()], &["file:///nonexistent/model-mirror".to_string()], &dst2, &AtomicBool::new(false));+ assert!(r.is_err());+ assert!(mirrors(Some("file://relative/dir"), None, &[]).is_empty());+ std::fs::create_dir_all(src.join("dir.bin")).unwrap();+ let dir_spec = FileSpec { name: "dir.bin", size: None, sha256: None, max: 10 };+ let (r, _) = run(&[dir_spec], &[format!("file://{}", src.display())], &dst2, &AtomicBool::new(false));+ assert!(r.is_err());+}++#[test]+fn file_urls_are_percent_decoded_but_bare_paths_keep_their_percent_signs() {+ let bare = mirrors(Some("/models/50%20off"), None, &[]);+ assert_eq!(bare.len(), 1);+ assert_eq!(local_path(&bare[0]), Some(std::path::PathBuf::from("/models/50%20off")));+ assert_eq!(local_path("file:///models/my%20sam3"), Some(std::path::PathBuf::from("/models/my sam3")));+ for hostile in ["file:///x%", "file:///x%2", "file:///x%zz", "file:///x%ff", "file:///x%0a"] {+ assert_eq!(local_path(hostile), None, "{hostile}");+ }+}@@ -11,14 +11,21 @@ use std::path::Path; /// The `facebook/sam3` checkpoint files [`crate::Sam3::load`] needs. ///-/// `model.safetensors` is pinned to the official checkpoint (Hugging Face LFS SHA-256). The two-/// tokenizer files are small text files without a pinned hash yet: they are only parsed by the-/// tokenizer (never executed) and capped in size.-// TODO(maintainer): pin `vocab.json` and `merges.txt` (size + SHA-256) when the files are-// uploaded to LightCraft's CDN.+/// All three are pinned (exact size + SHA-256) to the official `facebook/sam3` checkpoint files,+/// checked against a copy of them on 2026-10-13. pub const SAM3_FILES: &[FileSpec] = &[- FileSpec { name: "vocab.json", size: None, sha256: None, max: 16 << 20 },- FileSpec { name: "merges.txt", size: None, sha256: None, max: 16 << 20 },+ FileSpec {+ name: "vocab.json",+ size: Some(862_328),+ sha256: Some("5047b556ce86ccaf6aa22b3ffccfc52d391ea4accdab9c2f2407da5b742d4363"),+ max: 16 << 20,+ },+ FileSpec {+ name: "merges.txt",+ size: Some(524_619),+ sha256: Some("9fd691f7c8039210e0fced15865466c65820d09b63988b0174bfe25de299051a"),+ max: 16 << 20,+ }, FileSpec { name: crate::WEIGHTS_FILE, size: Some(SAM3_WEIGHTS_SIZE),