collective
Browse / github.com/storytold/craft-repo-status-app / patch
Skip refetching a due repo when one probe shows nothing changed
Skip refetching a due repo when one probe shows nothing changed ## Why Since #10 a repo is fetched in full once its data is `poll_minutes` old. A fetch costs about six GraphQL requests and one REST request, whether or not anything changed on GitHub. ## What changes **Probe** - When repos fall due, `step()` first sends one GraphQL request (`GitHub::probe`, up to 50 repos per request). For each due repo that has data from a successful fetch, it reads the head of the default branch, the issue and PR counts, when the most recently updated issue and PR were updated, and the newest release. A probe of three repos cost 1 point. - A repo whose values match the
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- dotCooCoo (imported from pull request #17, open)
- Base
9eaf2b22d3630c5ca0731009a2a3e835f49ee6d0- Size
- +1139 −44 in 10 files
- Review
- pending
- Shared
- 2026-10-28
- Id
bebc245544239dab0b84e36b
How it went
No other operator has reported using it yet.
Diff
raw · README.md craft_status_config.template.toml crates/craft-core/src/model.rs crates/craft-github/src/bin/craft-fetch.rs crates/craft-github/src/lib.rs crates/craft-status-app/src/lib.rs ui/src/app.test.ts ui/src/app.ts ui/src/model.test.ts ui/src/model.ts
@@ -61,10 +61,11 @@ sort by a column again. - **Fix first** (`⌘2`) ranks open issues across every repo. Each issue shows the reasons behind its score. Click a row in the Repos table to filter the list to that repo. Click an issue to open it on GitHub.-- **Freshness**: the header shows when the oldest data on screen was fetched,- and a pill that reads `FRESH`, `REFRESHING` or **`STALE`**. Data is stale- when any repo has gone `stale_minutes` (default 15) without a successful- fetch. A stale row is dimmed and gets an amber dot; a failed row gets a red+- **Freshness**: the header shows when the oldest data on screen was fetched+ or last checked, and a pill that reads `FRESH`, `REFRESHING` or+ **`STALE`**. Data is stale when any repo has gone `stale_minutes` (default+ 60) without a successful fetch or a check that found nothing changed. A+ stale row is dimmed and gets an amber dot; a failed row gets a red dot, and its error appears in the banner. The tray tooltip also says whether the data is stale. @@ -100,6 +101,21 @@ scored. request. It never uses search, which GitHub limits far more tightly (to about 30 a minute); PR and issue counts per window come from walking the recent PRs and issues.+- When repos fall due, the app first reads a few values for all of them in+ one GraphQL request: the commit at the head of the default branch, the+ issue and PR counts, when the most recently updated issue and PR were+ updated, and the newest release. When a repo's values match the ones read+ before its last fetch, the app does not fetch it. It computes the repo's+ numbers again from that fetch at the time of the check: the activity+ windows move to that time, and issues are ranked by their age then. The+ Fetched column still shows the age of the last full fetch; its tooltip adds+ the time of the last check that found nothing changed. The app fetches the+ repo anyway when the six-hourly fetch from scratch is due, when its last+ fetch failed, when the commits kept in `raw/` do not match the commit+ counts GitHub reported for each window, and when you refresh its row from+ the Fetched column. Reactions, download counts and edits to older releases+ do not change these values, so those numbers can be up to six hours plus+ one `poll_minutes` interval old. - Each repo's icon is `assets/app-icon/hicolor/64x64/apps/*.png` on its default branch. It is downloaded once and again only when the file changes; it's kept in the cache. Repos without one (libraries) get a lettered tile.@@ -111,12 +127,15 @@ scored. - Timing follows the wall clock, so a laptop that wakes from sleep polls within about 15 s. - `⌘R` or the ⟳ button makes every repo due at once; they are still fetched- one at a time, stalest first, and a rate-limit pause still holds.+ one at a time, stalest first, and a rate-limit pause still holds. Repos+ that the check above finds unchanged are not fetched. - Two caches in `~/Library/Caches/craft-status/` survive restarts: - `raw/<owner>@<repo>.json`: each repo's commit authors, PRs and issues from the last week (or since the latest build). Each poll fetches only what's new since the previous one, and refetches from scratch every six- hours to drop deleted issues and pick up late-dated commits.+ hours to drop deleted issues and pick up late-dated commits. The file+ also holds the last fetch's other answers (counts, open issues, releases,+ the contributor list) with the values the check read before it. - `snapshot.json`: what the window shows, saved after each repo. On launch the window shows it at once, marked `STALE` if it's old. Repos whose data is still fresh aren't refetched at launch.@@ -23,7 +23,8 @@ repos = [ # How old a repo's data gets before it's refetched (minutes, at least 1). # Repos are fetched one at a time, missing data first, then the stalest, 20 s-# apart; each costs roughly 6 GraphQL requests + 1 REST request.+# apart; each costs roughly 6 GraphQL requests + 1 REST request. A due repo+# that one small probe request finds unchanged is not refetched. poll_minutes = 30.0 # Data older than this is flagged STALE in the window (never less than poll_minutes).@@ -110,6 +110,10 @@ pub struct RepoEntry { pub stats: Option<RepoStats>, /// When `stats` was fetched. Kept across failed polls so stale data stays visible. pub fetched_at: Option<Time>,+ /// When a probe last found nothing changed on GitHub since `fetched_at`,+ /// so `stats` was measured again at this time without a fetch. A+ /// successful fetch clears it.+ pub checked_at: Option<Time>, pub last_attempt_at: Option<Time>, /// Error from the most recent attempt, cleared on success. pub error: Option<String>,@@ -149,11 +153,18 @@ impl RepoEntry { }) } + /// When `stats` was last known to be current: the fetch, or a later probe+ /// that found nothing changed.+ pub fn current_at(&self) -> Option<Time> {+ self.fetched_at.max(self.checked_at)+ }+ /// When this repo should next be fetched: at once if it's a hole or older /// than a manual refresh, else once its data is `refresh_secs` old; never- /// sooner than `retry_secs` after a failed attempt.+ /// sooner than `retry_secs` after a failed attempt. The data's age counts+ /// from [`RepoEntry::current_at`]. pub fn due_at(&self, schedule: &Schedule) -> Time {- let due = match self.fetched_at {+ let due = match self.current_at() { Some(t) if !self.is_hole() && schedule.forced_since.is_none_or(|f| t >= f) => { t + chrono::Duration::seconds(schedule.refresh_secs) }@@ -166,7 +177,7 @@ impl RepoEntry { } pub fn is_stale(&self, now: Time, stale_after_secs: i64) -> bool {- match self.fetched_at {+ match self.current_at() { Some(t) => (now - t).num_seconds() > stale_after_secs, None => true, }@@ -224,7 +235,7 @@ impl Snapshot { pub fn oldest_data_at(&self) -> Option<Time> { self.repos .iter()- .map(|r| r.fetched_at)+ .map(|r| r.current_at()) .try_fold(None::<Time>, |acc, t| { let t = t?; Some(Some(acc.map_or(t, |a| a.min(t))))@@ -232,13 +243,27 @@ impl Snapshot { .flatten() } + /// The repos due at `now` that a probe may find unchanged: each holds+ /// stats from a successful fetch and isn't waiting for a refresh by hand.+ /// Holes, failed and inaccessible repos are always fetched.+ pub fn skip_candidates(&self, now: Time, schedule: &Schedule) -> Vec<String> {+ let asked = |repo: &str| self.requested.iter().any(|r| r.eq_ignore_ascii_case(repo));+ self.repos+ .iter()+ .filter(|e| e.due_at(schedule) <= now && !e.is_hole())+ .filter(|e| e.stats.is_some() && e.error.is_none() && !e.inaccessible)+ .filter(|e| !asked(&e.repo))+ .map(|e| e.repo.clone())+ .collect()+ }+ /// The repo to fetch next at `now`: holes first, in config order, then /// whichever due repo has the oldest data. `None` when nothing is due. pub fn next_due(&self, now: Time, schedule: &Schedule) -> Option<&RepoEntry> { self.repos .iter() .filter(|e| e.due_at(schedule) <= now)- .min_by_key(|e| (!e.is_hole(), e.fetched_at))+ .min_by_key(|e| (!e.is_hole(), e.current_at())) } /// Queue `repo` to be fetched on its own, ahead of the schedule. False if@@ -400,6 +425,94 @@ mod tests { assert!(Snapshot::default().next_due_at(&sched).is_none()); } + /// Stats with every window, so the entry isn't a hole.+ fn complete_stats() -> RepoStats {+ let mut stats = RepoStats::default();+ for id in WINDOWS.iter().map(|(id, _)| *id).chain([ALL_TIME]) {+ stats.recent.insert(id.to_string(), Activity::default());+ }+ stats+ }++ #[test]+ fn a_probe_that_found_nothing_new_keeps_the_data_current() {+ let now = Utc::now();+ let sched = Schedule {+ refresh_secs: 30 * 60,+ retry_secs: 10 * 60,+ forced_since: None,+ };+ let mut s = Snapshot {+ stale_after_secs: 60 * 60,+ ..Default::default()+ };+ s.sync_repos(&["o/checked".into(), "o/fetched".into()]);+ for e in &mut s.repos {+ e.stats = Some(complete_stats());+ }+ s.repos[0].fetched_at = at(90, now);+ s.repos[0].checked_at = at(10, now);+ s.repos[1].fetched_at = at(40, now);+ let checked = &s.repos[0];+ assert_eq!(checked.current_at(), at(10, now));+ assert_eq!(checked.due_at(&sched), now + Duration::minutes(20));+ assert!(!checked.is_stale(now, s.stale_after_secs));+ assert_eq!(s.oldest_data_at(), at(40, now));+ assert!(!s.is_stale(now));+ assert_eq!(+ s.next_due(now, &sched).map(|e| e.repo.as_str()),+ Some("o/fetched"),+ "fetched 90 minutes ago, but checked since"+ );+ // A check older than the fetch does not change `current_at`.+ s.repos[1].checked_at = at(50, now);+ assert_eq!(s.repos[1].current_at(), at(40, now));+ let forced = Schedule {+ forced_since: at(5, now),+ ..sched+ };+ assert_eq!(+ s.next_due(now, &forced).map(|e| e.repo.as_str()),+ Some("o/fetched"),+ "a manual refresh is due first where the data is oldest"+ );+ }++ #[test]+ fn skip_candidates_are_due_repos_with_good_data() {+ let now = Utc::now();+ let sched = Schedule {+ refresh_secs: 30 * 60,+ retry_secs: 10 * 60,+ forced_since: None,+ };+ let names = [+ "o/due", "o/fresh", "o/hole", "o/failed", "o/gone", "o/asked",+ ];+ let mut s = Snapshot::default();+ s.sync_repos(&names.map(String::from));+ for e in &mut s.repos {+ e.stats = Some(complete_stats());+ e.fetched_at = at(45, now);+ }+ s.repos[1].fetched_at = at(5, now);+ s.repos[2].stats = Some(RepoStats::default());+ s.repos[3].error = Some("network".into());+ s.repos[3].last_attempt_at = at(20, now);+ s.repos[4].stats = None;+ s.repos[4].inaccessible = true;+ assert!(s.request("o/asked"));+ assert_eq!(s.skip_candidates(now, &sched), vec!["o/due".to_string()]);+ let forced = Schedule {+ forced_since: Some(now),+ ..sched+ };+ assert_eq!(+ s.skip_candidates(now, &forced),+ vec!["o/due".to_string(), "o/fresh".to_string()]+ );+ }+ #[test] fn rows_refreshed_by_hand_queue_once_each() { let mut s = Snapshot::default();@@ -47,7 +47,7 @@ fn main() { .as_deref() .map(|d| RawActivity::load(d, repo)) .unwrap_or_default();- match gh.fetch_repo(repo, now, &mut raw) {+ match gh.fetch_repo(repo, now, &mut raw, None) { Ok((s, budget)) => { if let Some(dir) = &raw_dir { if let Err(e) = raw.save(dir, repo) {@@ -14,13 +14,18 @@ //! one (usually) REST call for the contributor list, which GraphQL lacks. //! The icon itself is a REST blob fetch, made only when its blob id changes //! (see [`GitHub::icon`]). Blocking I/O: callers run this on worker threads.+//!+//! [`GitHub::probe`] asks about many repositories in one GraphQL query. When+//! a repo's [`Probe`] matches the one taken before its last fetch,+//! [`can_skip`] allows [`restat`] to measure that fetch again at a later time+//! without a request. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Duration; -use chrono::{DateTime, Utc};+use chrono::{DateTime, SubsecRound, Utc}; use craft_core::{ rank_issues, summarize_releases, Activity, Asset, BuildLag, IssueInput, LagBasis, Release, RepoStats, ALL_TIME, SINCE_RELEASE, WINDOWS,@@ -47,6 +52,8 @@ const FULL_REFRESH_HOURS: i64 = 6; /// Pages of 100 each history walk may take; a busier stretch undercounts its /// oldest windows (photocraft opens ~2000 PRs a week). const MAX_PAGES: usize = 30;+/// Repositories per [`probe_query`] request.+const PROBE_CHUNK: usize = 50; #[derive(Debug, thiserror::Error)] pub enum Error {@@ -219,14 +226,39 @@ impl GitHub { Ok(v["data"].clone()) } + /// A [`Probe`] of each of `repos`, in order, from one request per+ /// [`PROBE_CHUNK`] repos. An error in any answer fails the whole probe,+ /// as does a repository GitHub can't find or show.+ pub fn probe(&self, repos: &[String]) -> Result<(Vec<Option<Probe>>, Budget)> {+ let mut out = Vec::with_capacity(repos.len());+ let mut budget = Budget::default();+ for chunk in repos.chunks(PROBE_CHUNK) {+ let mut vars = serde_json::Map::new();+ for (i, repo) in chunk.iter().enumerate() {+ let (owner, name) = repo.split_once('/').unwrap_or((repo.as_str(), ""));+ vars.insert(format!("o{i}"), json!(owner));+ vars.insert(format!("n{i}"), json!(name));+ }+ let data = self.graphql(&probe_query(chunk.len()), Value::Object(vars))?;+ out.extend(parse_probe(&data, chunk.len()));+ if let Some(left) = data["rateLimit"]["remaining"].as_u64() {+ budget.graphql_remaining =+ Some(budget.graphql_remaining.map_or(left, |r| r.min(left)));+ }+ }+ Ok((out, budget))+ }+ /// Everything the UI shows for `owner/name`, measured at `now`. `raw` is /// the activity kept from the previous fetch; it's brought up to date- /// only when the whole fetch succeeds.+ /// only when the whole fetch succeeds, and then also holds this fetch+ /// ([`RawActivity::last`]) with `probe`, the [`Probe`] taken before it. pub fn fetch_repo( &self, repo: &str, now: DateTime<Utc>, raw: &mut RawActivity,+ probe: Option<Probe>, ) -> Result<(RepoStats, Budget)> { let (owner, name) = repo .split_once('/')@@ -255,19 +287,27 @@ impl GitHub { let authors: Vec<CommitAuthor> = fresh.commits.values().cloned().collect(); let events = fresh.events(); let contributors = self.contributors(repo)?;- *raw = fresh; let budget = Budget { graphql_remaining: main["rateLimit"]["remaining"].as_u64(), }; let stats = build_stats( &main, &releases,- since_build,+ since_build.clone(), &contributors, &authors, &events, now, );+ fresh.last = Some(LastFetch {+ at: now,+ probe,+ main,+ releases,+ since_build,+ contributors,+ });+ *raw = fresh; Ok((stats, budget)) } @@ -613,6 +653,48 @@ pub struct RawActivity { pub prs_opened: BTreeMap<u64, DateTime<Utc>>, pub prs_merged: BTreeMap<u64, DateTime<Utc>>, pub issues_opened: BTreeMap<u64, DateTime<Utc>>,+ /// The last successful fetch, which [`restat`] measures again at a later+ /// time. Files written before it existed load with `None`.+ pub last: Option<LastFetch>,+}++/// What [`build_stats`] read on the last successful fetch, apart from the+/// activity [`RawActivity`] already holds.+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct LastFetch {+ /// When the fetch started: the `now` its windows were measured at.+ pub at: DateTime<Utc>,+ /// The probe taken before the fetch, if there was one.+ pub probe: Option<Probe>,+ /// The answer to [`main_query`].+ pub main: Value,+ pub releases: Vec<Release>,+ pub since_build: Option<BuildLag>,+ pub contributors: Vec<String>,+}++/// The values one [`probe_query`] alias reads for a repository. They change+/// with every push to the default branch, every new, edited, commented,+/// labeled, closed or deleted issue or PR, every merge and every new or+/// edited latest release. Reactions and download counts do not change them.+#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]+#[serde(default)]+pub struct Probe {+ /// The commit the default branch points at.+ pub head_oid: Option<String>,+ /// Issues in every state.+ pub issues: u64,+ /// When the most recently updated issue was updated.+ pub issue_updated_at: Option<DateTime<Utc>>,+ /// PRs in every state.+ pub prs: u64,+ pub pr_updated_at: Option<DateTime<Utc>>,+ pub releases: u64,+ /// The newest release by creation date.+ pub release_tag: Option<String>,+ pub release_draft: bool,+ pub release_published_at: Option<DateTime<Utc>>,+ pub release_updated_at: Option<DateTime<Utc>>, } impl RawActivity {@@ -677,6 +759,136 @@ fn activity_start(now: DateTime<Utc>, release_at: Option<DateTime<Utc>>) -> Date (now - chrono::Duration::minutes(longest)).min(release_at.unwrap_or(now)) } +/// Whether `raw`'s last fetch can be measured again at `now` ([`restat`])+/// instead of fetching: `probe` matches the probe taken before that fetch,+/// the fetch from scratch every [`FULL_REFRESH_HOURS`] is not due, and the+/// commits `raw` holds match the fetch's per-window counts (see+/// [`commits_match_counts`]) and include the branch head (see+/// [`holds_head`]). The probe does not see reaction and download counts, so+/// a repo measured again keeps the counts of its last fetch until the next+/// one, which can be [`FULL_REFRESH_HOURS`] plus one poll interval later.+pub fn can_skip(raw: &RawActivity, probe: Option<&Probe>, now: DateTime<Utc>) -> bool {+ let (Some(probe), Some(last)) = (probe, raw.last.as_ref()) else {+ return false;+ };+ if last.probe.as_ref() != Some(probe) {+ return false;+ }+ let release_at = summarize_releases(&last.releases).latest_at;+ !raw.fetch_from(activity_start(now, release_at), now).1+ && commits_match_counts(&last.main, &raw.commits, last.at)+ && holds_head(last, &raw.commits, activity_start(last.at, release_at))+}++/// Whether `commits` holds the default branch's head that the probe before+/// `last` saw, when the head's date in `last.main` is `since` or later.+/// After a rewrite of the branch, an incremental fetch walks only the new+/// commits dated after it started, so a head dated earlier is missing, and+/// the commits held no longer match the branch.+fn holds_head(+ last: &LastFetch,+ commits: &BTreeMap<String, CommitAuthor>,+ since: DateTime<Utc>,+) -> bool {+ let head = last.probe.as_ref().and_then(|p| p.head_oid.as_deref());+ let target = &last.main["repository"]["defaultBranchRef"]["target"];+ match (head, first_time(&target["all"], "committedDate")) {+ (Some(head), Some(at)) if at >= since => commits.contains_key(head),+ _ => true,+ }+}++/// Whether `commits` holds exactly as many commits in each [`WINDOWS`]+/// window at `then` as GitHub counted in `main`. [`aged_window_counts`]+/// needs that. It fails when a history walk stopped at [`MAX_PAGES`], when+/// an incremental fetch missed a commit dated before it started, and when a+/// rewritten default branch left commits that are no longer on it.+fn commits_match_counts(+ main: &Value,+ commits: &BTreeMap<String, CommitAuthor>,+ then: DateTime<Utc>,+) -> bool {+ let head = &main["repository"]["defaultBranchRef"]["target"];+ windows(then, None).all(|(_, key, start)| {+ let Some(n) = head[format!("c{key}")]["totalCount"].as_u64() else {+ return true;+ };+ let start = start.trunc_subsecs(0);+ commits.values().filter(|(at, _)| *at >= start).count() as u64 == n+ })+}++/// The stats of `raw`'s last fetch measured at `now`, without a request:+/// the windows move to `now`, and issues are ranked by their age at `now`.+/// Valid only while nothing changed on GitHub (see [`can_skip`]). `raw` is+/// left as it is, so the next fetch is incremental from the last real one+/// and a later `restat` still has the commits it needs.+pub fn restat(raw: &RawActivity, now: DateTime<Utc>) -> Option<RepoStats> {+ let last = raw.last.as_ref()?;+ let release_at = summarize_releases(&last.releases).latest_at;+ let mut main = last.main.clone();+ aged_window_counts(&mut main, &raw.commits, last.at, now, release_at);+ // What a fetch at `now` would hold after pruning.+ let since = activity_start(now, release_at);+ let authors: Vec<CommitAuthor> = raw+ .commits+ .values()+ .filter(|(at, _)| *at >= since)+ .cloned()+ .collect();+ let mut events = raw.events();+ for times in [+ &mut events.prs_opened,+ &mut events.prs_merged,+ &mut events.issues_opened,+ ] {+ times.retain(|at| *at >= since);+ }+ Some(build_stats(+ &main,+ &last.releases,+ last.since_build.clone(),+ &last.contributors,+ &authors,+ &events,+ now,+ ))+}++/// Moves the per-window commit counts in `main`, measured at `then`, to+/// `now`: each count loses the `commits` whose dates its window has left.+/// Exact as long as no commit reached the default branch in between and+/// `commits` holds every commit since the start of the longest window at+/// `then`. The window starts are truncated to whole seconds, as+/// [`main_query`] sends them.+pub fn aged_window_counts(+ main: &mut Value,+ commits: &BTreeMap<String, CommitAuthor>,+ then: DateTime<Utc>,+ now: DateTime<Utc>,+ release_at: Option<DateTime<Utc>>,+) {+ // Indexing a missing key mutably would insert it, so look first.+ if !main["repository"]["defaultBranchRef"]["target"].is_object() {+ return;+ }+ let head = &mut main["repository"]["defaultBranchRef"]["target"];+ for ((_, key, start_then), (_, _, start_now)) in+ windows(then, release_at).zip(windows(now, release_at))+ {+ let (start_then, start_now) = (start_then.trunc_subsecs(0), start_now.trunc_subsecs(0));+ let alias = format!("c{key}");+ let Some(n) = head[&alias]["totalCount"].as_u64() else {+ continue;+ };+ let gone = commits+ .values()+ .filter(|(at, _)| start_then <= *at && *at < start_now)+ .count() as u64;+ head[&alias]["totalCount"] = json!(n.saturating_sub(gone));+ }+}+ /// `(number, field time)` for the nodes of one page of `conn` whose `field` /// is at or after `since`; the nodes are sorted newest first by `order`. /// `true` once the page has reached nodes older than `since`, so later pages@@ -790,6 +1002,48 @@ pub fn main_query(now: DateTime<Utc>, release_at: Option<DateTime<Utc>>) -> Stri MAIN_QUERY.replace("{HISTORY}", &history) } +/// [`PROBE_QUERY`] for `n` repositories, aliased `r0`, `r1`… and named by+/// the variables `$o0`/`$n0`, `$o1`/`$n1`…+pub fn probe_query(n: usize) -> String {+ let mut vars = Vec::new();+ let mut repos = String::new();+ for i in 0..n {+ vars.push(format!("$o{i}: String!, $n{i}: String!"));+ repos += &format!(" r{i}: repository(owner: $o{i}, name: $n{i}) {{ ...P }}\n");+ }+ PROBE_QUERY+ .replace("{VARS}", &vars.join(", "))+ .replace("{REPOS}", &repos)+}++/// One [`Probe`] per alias of [`probe_query`]`(n)`; `None` for an alias+/// GitHub answered with null (a missing or hidden repository).+pub fn parse_probe(data: &Value, n: usize) -> Vec<Option<Probe>> {+ (0..n)+ .map(|i| {+ let r = &data[format!("r{i}")];+ if r.is_null() {+ return None;+ }+ let release = &r["releases"]["nodes"][0];+ Some(Probe {+ head_oid: r["defaultBranchRef"]["target"]["oid"]+ .as_str()+ .map(str::to_string),+ issues: count(&r["issues"]),+ issue_updated_at: first_time(&r["lastIssue"], "updatedAt"),+ prs: count(&r["pullRequests"]),+ pr_updated_at: first_time(&r["lastPr"], "updatedAt"),+ releases: count(&r["releases"]),+ release_tag: release["tagName"].as_str().map(str::to_string),+ release_draft: release["isDraft"].as_bool().unwrap_or(false),+ release_published_at: time(&release["publishedAt"]),+ release_updated_at: time(&release["updatedAt"]),+ })+ })+ .collect()+}+ /// Pure: turn the GraphQL payloads into the model (unit-tested below). pub fn build_stats( main: &Value,@@ -875,6 +1129,25 @@ fragment I on Issue { } "#; +/// What can change a repository's stats, for many repositories at once (see+/// [`Probe`]).+const PROBE_QUERY: &str = r#"+query({VARS}) {+{REPOS} rateLimit { cost remaining resetAt }+}+fragment P on Repository {+ defaultBranchRef { target { oid } }+ issues { totalCount }+ lastIssue: issues(first: 1, orderBy: {field: UPDATED_AT, direction: DESC}) { nodes { updatedAt } }+ pullRequests { totalCount }+ lastPr: pullRequests(first: 1, orderBy: {field: UPDATED_AT, direction: DESC}) { nodes { updatedAt } }+ releases(first: 1, orderBy: {field: CREATED_AT, direction: DESC}) {+ totalCount+ nodes { tagName publishedAt updatedAt isDraft }+ }+}+"#;+ const RELEASES_QUERY: &str = r#" query($owner: String!, $name: String!, $after: String) { repository(owner: $owner, name: $name) {@@ -983,8 +1256,19 @@ mod tests { assert!(env_or_config_token("", env(&[])).is_none()); } - #[test]- fn builds_stats_from_payloads() {+ /// The payloads of one fetch, shared by [`builds_stats_from_payloads`]+ /// and [`restat_at_fetch_time_matches_build_stats`].+ struct Payloads {+ main: Value,+ events: Events,+ rel: Vec<Release>,+ now: DateTime<Utc>,+ lag: BuildLag,+ commits: Vec<(String, CommitAuthor)>,+ people: Vec<String>,+ }++ fn payloads() -> Payloads { let main = json!({ "repository": { "openPrs": {"totalCount": 3}, "allPrs": {"totalCount": 40}, "mergedPrs": {"totalCount": 33},@@ -1028,17 +1312,39 @@ mod tests { basis: LagBasis::Exact, from_sha: Some("abc".into()), };- let authors = parse_authors(&json!({"nodes": [+ let commits = parse_authors(&json!({"nodes": [ {"committedDate": "2026-10-08T03:55:00Z", "author": {"email": "A@x", "name": "A", "user": {"login": "ann"}}}, {"committedDate": "2026-10-08T03:00:00Z", "author": {"email": "a@x", "name": "A", "user": {"login": "ann"}}}, {"committedDate": "2026-10-08T02:00:00Z", "author": {"email": "Bo@Y", "name": "Bo", "user": null}}, {"committedDate": "2026-10-03T00:00:00Z", "author": {"email": "", "name": "Cy", "user": null}} ]}));- let authors: Vec<CommitAuthor> = authors.into_iter().map(|(_, a)| a).collect();- assert_eq!(authors[2].1, "bo@y"); let mut people = parse_contributors(&json!([{"login": "ann"}, {"email": "Old@Z", "type": "Anonymous"}])); people.extend((0..10).map(|i| format!("p{i}")));+ Payloads {+ main,+ events,+ rel,+ now,+ lag,+ commits,+ people,+ }+ }++ #[test]+ fn builds_stats_from_payloads() {+ let Payloads {+ main,+ events,+ rel,+ now,+ lag,+ commits,+ people,+ } = payloads();+ let authors: Vec<CommitAuthor> = commits.into_iter().map(|(_, a)| a).collect();+ assert_eq!(authors[2].1, "bo@y"); assert_eq!(people[1], "old@z"); let s = build_stats( &main,@@ -1234,4 +1540,337 @@ mod tests { s.recent.clear(); assert_eq!(s, RepoStats::default()); }++ #[test]+ fn probe_query_aliases_every_repo() {+ let q = probe_query(3);+ for i in 0..3 {+ assert!(+ q.contains(&format!("$o{i}: String!, $n{i}: String!")),+ "{q}"+ );+ let alias = format!("r{i}: repository(owner: $o{i}, name: $n{i}) {{ ...P }}");+ assert!(q.contains(&alias), "{q}");+ }+ assert!(!q.contains("r3:") && !q.contains("{VARS}") && !q.contains("{REPOS}"));+ assert!(q.contains("defaultBranchRef { target { oid } }"));+ assert!(q.contains("issues(first: 1, orderBy: {field: UPDATED_AT, direction: DESC})"));+ assert!(q.contains("pullRequests(first: 1, orderBy: {field: UPDATED_AT, direction: DESC})"));+ assert!(q.contains("releases(first: 1, orderBy: {field: CREATED_AT, direction: DESC})"));+ assert!(q.contains("rateLimit { cost remaining resetAt }"));+ assert!(!q.contains("search(") && !q.contains("pushedAt"));+ }++ #[test]+ fn parse_probe_reads_values_and_skips_missing() {+ let t = |s: &str| Some(s.parse::<DateTime<Utc>>().unwrap());+ let data = json!({+ "r0": {+ "defaultBranchRef": {"target": {"oid": "h1"}},+ "issues": {"totalCount": 90},+ "lastIssue": {"nodes": [{"updatedAt": "2026-10-08T02:00:00Z"}]},+ "pullRequests": {"totalCount": 40},+ "lastPr": {"nodes": [{"updatedAt": "2026-10-08T03:00:00Z"}]},+ "releases": {"totalCount": 2, "nodes": [+ {"tagName": "v1.1.0", "isDraft": true, "publishedAt": null, "updatedAt": "2026-10-08T00:30:00Z"}+ ]}+ },+ "r1": null,+ "r2": {+ "defaultBranchRef": null, "issues": {"totalCount": 0}, "lastIssue": {"nodes": []},+ "pullRequests": {"totalCount": 0}, "lastPr": {"nodes": []},+ "releases": {"totalCount": 0, "nodes": []}+ },+ "rateLimit": {"cost": 1, "remaining": 4999, "resetAt": "2026-10-08T05:00:00Z"}+ });+ let probes = parse_probe(&data, 3);+ assert_eq!(+ probes[0],+ Some(Probe {+ head_oid: Some("h1".into()),+ issues: 90,+ issue_updated_at: t("2026-10-08T02:00:00Z"),+ prs: 40,+ pr_updated_at: t("2026-10-08T03:00:00Z"),+ releases: 2,+ release_tag: Some("v1.1.0".into()),+ release_draft: true,+ release_published_at: None,+ release_updated_at: t("2026-10-08T00:30:00Z"),+ })+ );+ assert_eq!(probes[1], None);+ // An empty repository: no branch, no issues, no releases.+ assert_eq!(probes[2], Some(Probe::default()));+ assert_eq!(probes.len(), 3);+ }++ #[test]+ fn skip_needs_a_matching_probe() {+ let t = |s: &str| s.parse::<DateTime<Utc>>().unwrap();+ let now = t("2026-10-08T04:00:00Z");+ let probe = Probe {+ head_oid: Some("h1".into()),+ issues: 3,+ ..Default::default()+ };+ let raw = RawActivity {+ covers_from: Some(t("2026-10-01T03:25:00Z")),+ fetched_at: Some(t("2026-10-08T03:25:00Z")),+ full_at: Some(t("2026-10-08T01:00:00Z")),+ last: Some(LastFetch {+ at: t("2026-10-08T03:25:00Z"),+ probe: Some(probe.clone()),+ main: Value::Null,+ releases: vec![],+ since_build: None,+ contributors: vec![],+ }),+ ..Default::default()+ };+ assert!(can_skip(&raw, Some(&probe), now));+ let no_last = RawActivity {+ last: None,+ ..raw.clone()+ };+ assert!(!can_skip(&no_last, Some(&probe), now));+ assert!(!can_skip(&raw, None, now));+ let pushed = Probe {+ head_oid: Some("h2".into()),+ ..probe.clone()+ };+ assert!(!can_skip(&raw, Some(&pushed), now));+ // The last fetch went without a probe: a row refresh, or a failed probe.+ let unprobed = RawActivity {+ last: raw.last.clone().map(|l| LastFetch { probe: None, ..l }),+ ..raw.clone()+ };+ assert!(!can_skip(&unprobed, Some(&probe), now));+ // The fetch from scratch is due six hours after the last one.+ assert!(can_skip(&raw, Some(&probe), t("2026-10-08T06:59:00Z")));+ assert!(!can_skip(&raw, Some(&probe), t("2026-10-08T07:00:00Z")));+ // GitHub counted one commit in the last 10 minutes at 03:25.+ let mut counted = raw.clone();+ if let Some(last) = counted.last.as_mut() {+ last.main =+ json!({"repository": {"defaultBranchRef": {"target": {"c0": {"totalCount": 1}}}}});+ }+ // A history walk that stopped early doesn't hold it.+ assert!(!can_skip(&counted, Some(&probe), now));+ counted+ .commits+ .insert("a".into(), (t("2026-10-08T03:20:00Z"), "ann".into()));+ assert!(can_skip(&counted, Some(&probe), now));+ // The cache holds a commit that a force-push removed from the branch.+ counted+ .commits+ .insert("x".into(), (t("2026-10-08T03:22:00Z"), "bo".into()));+ assert!(!can_skip(&counted, Some(&probe), now));+ // The head h1 is dated inside the week, and an incremental fetch after+ // a rewrite did not walk it: the commits held miss it.+ let head_dated = |date: &str| {+ let mut r = raw.clone();+ if let Some(last) = r.last.as_mut() {+ last.main = json!({"repository": {"defaultBranchRef": {"target": {+ "all": {"totalCount": 9, "nodes": [{"committedDate": date}]}}}}});+ }+ r+ };+ let mut rewritten = head_dated("2026-10-06T00:00:00Z");+ assert!(!can_skip(&rewritten, Some(&probe), now));+ rewritten+ .commits+ .insert("h1".into(), (t("2026-10-06T00:00:00Z"), "ann".into()));+ assert!(can_skip(&rewritten, Some(&probe), now));+ // A head older than the week isn't held, and needn't be.+ assert!(can_skip(+ &head_dated("2026-09-01T00:00:00Z"),+ Some(&probe),+ now+ ));+ }++ #[test]+ fn restat_at_fetch_time_matches_build_stats() {+ let p = payloads();+ let mut raw = RawActivity {+ commits: p.commits.iter().cloned().collect(),+ ..Default::default()+ };+ for (held, times) in [+ (&mut raw.prs_opened, &p.events.prs_opened),+ (&mut raw.prs_merged, &p.events.prs_merged),+ (&mut raw.issues_opened, &p.events.issues_opened),+ ] {+ held.extend(times.iter().enumerate().map(|(i, t)| (i as u64, *t)));+ }+ assert_eq!(restat(&raw, p.now), None, "no fetch to measure again");+ raw.last = Some(LastFetch {+ at: p.now,+ probe: None,+ main: p.main.clone(),+ releases: p.rel.clone(),+ since_build: Some(p.lag.clone()),+ contributors: p.people.clone(),+ });+ let authors: Vec<CommitAuthor> = p.commits.into_iter().map(|(_, a)| a).collect();+ let fetched = build_stats(+ &p.main,+ &p.rel,+ Some(p.lag),+ &p.people,+ &authors,+ &p.events,+ p.now,+ );+ assert_eq!(restat(&raw, p.now), Some(fetched));+ }++ #[test]+ fn restat_moves_windows_to_now() {+ let t = |s: &str| s.parse::<DateTime<Utc>>().unwrap();+ let then = t("2026-10-08T04:00:00Z");+ let main = json!({"repository": {+ "openPrs": {"totalCount": 1}, "openIssues": {"totalCount": 1},+ "recentOpen": {"nodes": [+ {"number": 5, "title": "Export loses layers", "url": "u5", "createdAt": "2026-10-07T04:10:00Z",+ "updatedAt": "2026-10-07T04:10:00Z", "authorAssociation": "NONE",+ "comments": {"totalCount": 0}, "reactions": {"totalCount": 0}, "labels": {"nodes": [{"name": "bug"}]}}+ ]},+ "hotOpen": {"nodes": []},+ "defaultBranchRef": {"target": {+ "all": {"totalCount": 500, "nodes": [{"committedDate": "2026-10-08T03:58:00Z"}]},+ "c0": {"totalCount": 2}, "c1": {"totalCount": 3}, "c2": {"totalCount": 3}, "c3": {"totalCount": 3},+ "c4": {"totalCount": 3}, "c5": {"totalCount": 3}, "c6": {"totalCount": 4}}}+ }});+ let mut raw = RawActivity {+ covers_from: Some(t("2026-10-01T04:00:00Z")),+ fetched_at: Some(then),+ full_at: Some(then),+ last: Some(LastFetch {+ at: then,+ probe: None,+ main,+ releases: vec![],+ since_build: None,+ contributors: vec![],+ }),+ ..Default::default()+ };+ for (oid, at, who) in [+ ("a", "2026-10-08T03:55:00Z", "ann"),+ ("b", "2026-10-08T03:40:00Z", "bo"),+ ("c", "2026-10-01T04:10:00Z", "cy"),+ ("d", "2026-10-08T03:58:00Z", "dee"),+ ] {+ raw.commits.insert(oid.into(), (t(at), who.into()));+ }+ // Opened inside the last 4 h at `then`; merged inside the last week.+ raw.prs_opened.insert(1, t("2026-10-08T00:10:00Z"));+ raw.prs_merged.insert(2, t("2026-10-01T04:10:00Z"));+ let held = raw.clone();+ let before = restat(&raw, then).unwrap();+ let w = |s: &RepoStats, id: &str| s.recent[id];+ assert_eq!(+ ["10m", "30m", "7d"].map(|id| w(&before, id).commits),+ [2, 3, 4]+ );+ assert_eq!(w(&before, "4h").prs_opened, 1);+ assert!(before.urgent[0].reasons.contains(&"new".to_string()));++ let now = t("2026-10-08T04:20:00Z");+ let after = restat(&raw, now).unwrap();+ // a and d left the 10-minute window, b the 30-minute one, c the week.+ assert_eq!(w(&after, "10m").commits, 0);+ assert_eq!(w(&after, "30m").commits, 2);+ assert_eq!(w(&after, "1h").commits, 3);+ assert_eq!(w(&after, "7d").commits, 3);+ assert_eq!(w(&after, "4h").prs_opened, 0);+ assert_eq!(w(&after, "7d").prs_merged, 0);+ // cy's only commit is older than the week, so a fetch would drop it.+ assert_eq!(+ (w(&before, ALL_TIME).people, w(&after, ALL_TIME).people),+ (4, 3)+ );+ // Issue #5 is now more than 24 hours old.+ assert!(!after.urgent[0].reasons.contains(&"new".to_string()));+ assert_eq!(after.urgent[0].score, before.urgent[0].score - 4.0);+ assert!(after.urgency < before.urgency);+ // Unchanged: a later restat from the same fetch still has every commit.+ assert_eq!(raw, held);+ let later = restat(&raw, t("2026-10-08T04:25:00Z")).unwrap();+ assert_eq!(w(&later, "7d").commits, 3);+ raw.last = None;+ assert_eq!(restat(&raw, now), None);+ }++ #[test]+ fn aged_counts_start_windows_on_whole_seconds() {+ let t = |s: &str| s.parse::<DateTime<Utc>>().unwrap();+ // The query sent "since: 03:50:00Z", so GitHub counted this commit.+ let commits = BTreeMap::from([(+ "a".to_string(),+ (t("2026-10-08T03:50:00Z"), "ann".to_string()),+ )]);+ let mut main =+ json!({"repository": {"defaultBranchRef": {"target": {"c0": {"totalCount": 1}}}}});+ aged_window_counts(+ &mut main,+ &commits,+ t("2026-10-08T04:00:00.700Z"),+ t("2026-10-08T04:20:00.700Z"),+ None,+ );+ assert_eq!(+ main["repository"]["defaultBranchRef"]["target"]["c0"]["totalCount"],+ 0+ );+ let mut empty = json!({"repository": {"defaultBranchRef": null}});+ aged_window_counts(+ &mut empty,+ &commits,+ t("2026-10-08T04:00:00Z"),+ t("2026-10-08T05:00:00Z"),+ None,+ );+ assert_eq!(empty, json!({"repository": {"defaultBranchRef": null}}));+ }++ #[test]+ fn raw_without_last_loads() {+ let dir = std::env::temp_dir().join(format!("craft-raw-last-{}", std::process::id()));+ std::fs::create_dir_all(dir.join("raw")).unwrap();+ // A file written before the last fetch was kept.+ std::fs::write(+ RawActivity::path(&dir, "o/old"),+ r#"{"fetched_at":"2026-10-08T03:55:00Z","prs_opened":{"7":"2026-10-08T03:00:00Z"}}"#,+ )+ .unwrap();+ let old = RawActivity::load(&dir, "o/old");+ assert_eq!(old.last, None);+ assert_eq!(+ old.fetched_at,+ Some("2026-10-08T03:55:00Z".parse().unwrap())+ );+ assert_eq!(old.prs_opened.len(), 1);+ let p = payloads();+ let raw = RawActivity {+ last: Some(LastFetch {+ at: p.now,+ probe: Some(Probe {+ head_oid: Some("h1".into()),+ release_published_at: Some(p.now),+ ..Default::default()+ }),+ main: p.main,+ releases: p.rel,+ since_build: Some(p.lag),+ contributors: p.people,+ }),+ ..Default::default()+ };+ raw.save(&dir, "o/new").unwrap();+ assert_eq!(RawActivity::load(&dir, "o/new"), raw);+ std::fs::remove_dir_all(&dir).ok();+ } }@@ -3,9 +3,12 @@ //! //! One background thread owns polling. Every `poll_minutes` (and on demand) //! it fetches all repositories one at a time, publishing the snapshot to the-//! UI and the cache after each repository lands.+//! UI and the cache after each repository lands. Before a fetch, one probe+//! request finds the due repositories that have not changed since their last+//! fetch; those are measured again from that fetch instead. //! Failed repositories keep their previous numbers; staleness is derived from-//! each repository's `fetched_at`, so old data is always visibly old.+//! when each repository's data was last known current+//! (`RepoEntry::current_at`), so old data is always visibly old. use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{mpsc, Mutex};@@ -13,7 +16,7 @@ use std::time::{Duration, SystemTime}; use chrono::{DateTime, Local, Timelike, Utc}; use craft_core::{Cache, Config, Schedule, Snapshot, TokenStatus, WindowState};-use craft_github::{resolve_token, Error as GhError, GitHub, RawActivity};+use craft_github::{can_skip, resolve_token, restat, Error as GhError, GitHub, Probe, RawActivity}; use serde::Serialize; use tauri::menu::{CheckMenuItem, Menu, MenuItem, PredefinedMenuItem}; use tauri::tray::{MouseButton, MouseButtonState, TrayIconBuilder, TrayIconEvent};@@ -238,11 +241,20 @@ struct Pacing { strikes: u32, /// The last manual refresh, while repos older than it remain. forced_since: Option<DateTime<Utc>>,+ /// Probes that found a repo changed, each kept until that repo's fetch+ /// succeeds. Those repos aren't probed again in the meantime.+ probes: Vec<(String, Probe)>, } -/// Fetch `repo` into the snapshot. Errors about this repo land on its row;-/// a missing or rejected token doesn't, since it isn't the repo's fault.-fn fetch_into_snapshot(app: &AppHandle, cfg: &Config, repo: &str) -> Result<(), GhError> {+/// Fetch `repo` into the snapshot, keeping `probe` (taken before the fetch)+/// in its raw cache. Errors about this repo land on its row; a missing or+/// rejected token doesn't, since it isn't the repo's fault.+fn fetch_into_snapshot(+ app: &AppHandle,+ cfg: &Config,+ repo: &str,+ probe: Option<Probe>,+) -> Result<(), GhError> { let state = app.state::<AppState>(); let token = resolve_token(&cfg.github_token); state.snapshot.lock().unwrap().token = Some(TokenStatus {@@ -257,7 +269,7 @@ fn fetch_into_snapshot(app: &AppHandle, cfg: &Config, repo: &str) -> Result<(), .as_deref() .map(|d| RawActivity::load(d, repo)) .unwrap_or_default();- let result = gh.fetch_repo(repo, now, &mut raw);+ let result = gh.fetch_repo(repo, now, &mut raw, probe); if let (Ok(_), Some(dir)) = (&result, &raw_dir) { if let Err(e) = raw.save(dir, repo) { eprintln!("{repo}: could not save raw cache: {e}");@@ -302,6 +314,7 @@ fn fetch_into_snapshot(app: &AppHandle, cfg: &Config, repo: &str) -> Result<(), } entry.stats = Some(stats); entry.fetched_at = Some(now);+ entry.checked_at = None; entry.error = None; entry.inaccessible = false; Ok(budget.graphql_remaining)@@ -311,6 +324,7 @@ fn fetch_into_snapshot(app: &AppHandle, cfg: &Config, repo: &str) -> Result<(), entry.icon = None; entry.icon_oid = None; entry.fetched_at = Some(now);+ entry.checked_at = None; entry.error = None; entry.inaccessible = true; Ok(None)@@ -339,6 +353,120 @@ fn fetch_into_snapshot(app: &AppHandle, cfg: &Config, repo: &str) -> Result<(), } } +/// Probes the due repos that [`Snapshot::skip_candidates`] lists and that+/// have no probe in `pending`, in one request, and measures again each one+/// that has not changed since its last fetch (see [`restat_unchanged`]).+/// Returns how many it measured again, and the probes of the others for+/// their fetches to keep. Without a token, a raw cache or the probe's+/// answer, it measures nothing again and the due repos are fetched. When+/// GitHub rate-limits the probe, it returns that error.+fn skip_unchanged(+ app: &AppHandle,+ cfg: &Config,+ schedule: &Schedule,+ now: DateTime<Utc>,+ pending: &[(String, Probe)],+) -> Result<(usize, Vec<(String, Probe)>), GhError> {+ let state = app.state::<AppState>();+ let candidates = state+ .snapshot+ .lock()+ .unwrap()+ .skip_candidates(now, schedule);+ let repos = unprobed(candidates, pending);+ let Some(dir) = state.cache.as_ref().map(|c| c.dir().to_path_buf()) else {+ return Ok((0, Vec::new()));+ };+ if repos.is_empty() {+ return Ok((0, Vec::new()));+ }+ // Show the token in use, as a fetch does. Only a fetch marks it rejected.+ let token = resolve_token(&cfg.github_token);+ state.snapshot.lock().unwrap().token = Some(TokenStatus {+ source: token.as_ref().ok().map(|t| t.source.to_string()),+ rejected: false,+ });+ let Ok(gh) = token.and_then(|t| GitHub::new(t.value)) else {+ return Ok((0, Vec::new()));+ };+ let probed_at = Utc::now();+ let probes = match gh.probe(&repos) {+ Ok((probes, budget)) => {+ if budget.graphql_remaining.is_some() {+ state.snapshot.lock().unwrap().rate_limit_remaining = budget.graphql_remaining;+ }+ probes+ }+ Err(e @ GhError::RateLimited(_)) => return Err(e),+ Err(e) => {+ eprintln!("probe failed, fetching instead: {e}");+ return Ok((0, Vec::new()));+ }+ };+ let checks = repos+ .into_iter()+ .zip(probes)+ .map(|(repo, probe)| {+ let raw = RawActivity::load(&dir, &repo);+ (repo, raw, probe)+ })+ .collect();+ let (restated, left) = {+ let mut snap = state.snapshot.lock().unwrap();+ let (restated, left) = restat_unchanged(&mut snap, checks, probed_at);+ if let (false, Some(cache)) = (restated.is_empty(), &state.cache) {+ if let Err(e) = cache.save(&snap) {+ eprintln!("could not save cache: {e}");+ }+ }+ (restated, left)+ };+ if !restated.is_empty() {+ emit_snapshot(app);+ }+ Ok((restated.len(), left))+}++/// The `candidates` that have no probe in `pending`.+fn unprobed(candidates: Vec<String>, pending: &[(String, Probe)]) -> Vec<String> {+ candidates+ .into_iter()+ .filter(|repo| !pending.iter().any(|(r, _)| r == repo))+ .collect()+}++/// Measures again, at `probed_at` (when the probe was sent), each of+/// `checks` (a repo, its raw cache and its probe) whose probe shows that+/// nothing changed since its last fetch. Such a repo keeps its `fetched_at`,+/// and its `checked_at` becomes `probed_at`. Returns the repos measured+/// again, and the probes of the others.+fn restat_unchanged(+ snap: &mut Snapshot,+ checks: Vec<(String, RawActivity, Option<Probe>)>,+ probed_at: DateTime<Utc>,+) -> (Vec<String>, Vec<(String, Probe)>) {+ let mut restated = Vec::new();+ let mut left = Vec::new();+ for (repo, raw, probe) in checks {+ let stats = can_skip(&raw, probe.as_ref(), probed_at)+ .then(|| restat(&raw, probed_at))+ .flatten();+ match (stats, snap.repos.iter_mut().find(|e| e.repo == repo)) {+ (Some(stats), Some(entry)) => {+ entry.stats = Some(stats);+ entry.checked_at = Some(probed_at);+ restated.push(repo);+ }+ _ => {+ if let Some(probe) = probe {+ left.push((repo, probe));+ }+ }+ }+ }+ (restated, left)+}+ /// When a rate limit GitHub reported (`RateLimited`'s text) lifts. fn rate_limit_end(until: &str, now: DateTime<Utc>) -> Option<DateTime<Utc>> { if let Ok(t) = DateTime::parse_from_rfc3339(until) {@@ -358,7 +486,8 @@ fn clock(t: DateTime<Utc>) -> String { } /// Fetch the most overdue repository, if one is due and pacing allows, and-/// decide how long to wait before the next.+/// decide how long to wait before the next. Due repositories that a probe+/// finds unchanged are measured again first, without a fetch. fn step(app: &AppHandle, pacing: &mut Pacing) { let state = app.state::<AppState>(); let cfg = state.config.lock().unwrap().clone();@@ -368,14 +497,35 @@ fn step(app: &AppHandle, pacing: &mut Pacing) { forced_since: pacing.forced_since, }; let now = Utc::now();- let due = {+ let paused = pacing.not_before.is_some_and(|t| now < t);+ { let mut snap = state.snapshot.lock().unwrap(); snap.sync_repos(&cfg.repos); snap.poll_secs = cfg.poll_secs(); snap.stale_after_secs = cfg.stale_secs();- let paused = pacing.not_before.is_some_and(|t| now < t);- // Rows refreshed by hand go first.- let due = (!paused)+ pacing+ .probes+ .retain(|(r, _)| snap.repos.iter().any(|e| e.repo == *r));+ }+ // Due repos that one probe finds unchanged are measured again instead of+ // fetched. After a step that only probes, the pacing below runs as after+ // a fetch.+ let probed = if paused {+ Ok((0, Vec::new()))+ } else {+ skip_unchanged(app, &cfg, &schedule(pacing), now, &pacing.probes)+ };+ let (restated, limited) = match probed {+ Ok((restated, changed)) => {+ pacing.probes.extend(changed);+ (restated, None)+ }+ Err(e) => (0, Some(e)),+ };+ let due = {+ let mut snap = state.snapshot.lock().unwrap();+ // Rows refreshed by hand go first, and are always fetched in full.+ let due = (!paused && limited.is_none()) .then(|| { snap.take_requested().or_else(|| { snap.next_due(now, &schedule(pacing))@@ -387,9 +537,24 @@ fn step(app: &AppHandle, pacing: &mut Pacing) { snap.fetching = due.clone(); due };- if let Some(repo) = &due {- emit_snapshot(app);- let result = fetch_into_snapshot(app, &cfg, repo);+ let result = match (&due, limited) {+ // A rate-limited probe pauses polling as a rate-limited fetch does.+ (_, Some(e)) => Some(Err(e)),+ (Some(repo), None) => {+ emit_snapshot(app);+ let probe = pacing.probes.iter().find(|(r, _)| r == repo);+ let result = fetch_into_snapshot(app, &cfg, repo, probe.map(|(_, p)| p.clone()));+ // A failed fetch keeps the probe for its retry.+ if result.is_ok() {+ pacing.probes.retain(|(r, _)| r != repo);+ }+ Some(result)+ }+ // Only the probe was sent: the gap and the low-budget pause follow it.+ (None, None) => (restated > 0).then_some(Ok(())),+ };+ let acted = result.is_some();+ if let Some(result) = result { let now = Utc::now(); let mut snap = state.snapshot.lock().unwrap(); let (pause, error) = match result {@@ -458,7 +623,7 @@ fn step(app: &AppHandle, pacing: &mut Pacing) { .next_due_at(&schedule(pacing)) .map(|t| t.max(Utc::now())) .map(|t| pacing.not_before.map_or(t, |p| t.max(p)));- if snap.next_refresh_at == next && due.is_none() {+ if snap.next_refresh_at == next && !acted { return; } snap.next_refresh_at = next;@@ -854,6 +1019,115 @@ mod tests { s } + /// A raw cache whose last fetch, at `fetched_at`, was taken after `probe`.+ fn raw_fetched(fetched_at: DateTime<Utc>, probe: &Probe) -> RawActivity {+ RawActivity {+ covers_from: Some(fetched_at - CDuration::days(7)),+ fetched_at: Some(fetched_at),+ full_at: Some(fetched_at),+ last: Some(craft_github::LastFetch {+ at: fetched_at,+ probe: Some(probe.clone()),+ main: serde_json::json!({"repository": {+ "openPrs": {"totalCount": 4}, "defaultBranchRef": null}}),+ releases: vec![],+ since_build: None,+ contributors: vec![],+ }),+ ..Default::default()+ }+ }++ fn probe(head: &str) -> Probe {+ Probe {+ head_oid: Some(head.into()),+ ..Default::default()+ }+ }++ #[test]+ fn three_due_repos_with_one_changed_cost_one_probe_and_one_fetch() {+ let now = Utc::now();+ let schedule = Schedule {+ refresh_secs: 30 * 60,+ retry_secs: REPO_RETRY,+ forced_since: None,+ };+ let mut snap = snap_with(&[40, 50, 60], now);+ for e in &mut snap.repos {+ e.stats = Some(craft_github::restat(&raw_fetched(now, &probe("h")), now).unwrap());+ }+ let due = snap.skip_candidates(now, &schedule);+ assert_eq!(due, ["o/r0", "o/r1", "o/r2"]);+ // One probe query names all three.+ let query = craft_github::probe_query(due.len());+ assert!(query.contains("r2: repository(") && !query.contains("r3:"));+ let fetched_at = |m| now - CDuration::minutes(m);+ let probed_at = now + CDuration::seconds(1);+ // A repo fetched `mins` ago at head `before`, and now at head `head`.+ let check = |repo: &str, mins, before: &str, head: &str| {+ let raw = raw_fetched(fetched_at(mins), &probe(before));+ (repo.to_string(), raw, Some(probe(head)))+ };+ let checks = vec![+ check("o/r0", 40, "a", "a"),+ // r1 has a new head since its last fetch.+ check("o/r1", 50, "b", "b2"),+ check("o/r2", 60, "c", "c"),+ ];+ let (restated, left) = restat_unchanged(&mut snap, checks, probed_at);+ assert_eq!(restated, ["o/r0", "o/r2"]);+ assert_eq!(+ left,+ [("o/r1".to_string(), probe("b2"))],+ "the fetch keeps the changed repo's probe"+ );+ for (i, mins) in [(0, 40), (2, 60)] {+ let e = &snap.repos[i];+ assert_eq!(+ e.fetched_at,+ Some(fetched_at(mins)),+ "a check is not a fetch"+ );+ assert_eq!(e.checked_at, Some(probed_at));+ assert_eq!(e.stats.as_ref().map(|s| s.open_prs), Some(4));+ }+ assert_eq!(snap.repos[1].checked_at, None);+ // Only the changed repo is left to fetch, and its probe waits for+ // that fetch: the next step doesn't probe it again.+ let left_due = snap.skip_candidates(probed_at, &schedule);+ assert_eq!(left_due, ["o/r1"]);+ assert!(unprobed(left_due, &left).is_empty());+ assert_eq!(+ snap.next_due(probed_at, &schedule).map(|e| e.repo.as_str()),+ Some("o/r1")+ );+ }++ #[test]+ fn unchanged_repos_are_measured_at_the_probe_time() {+ let now = Utc::now();+ let mut snap = snap_with(&[40], now);+ let fetched_at = now - CDuration::minutes(40);+ let mut raw = raw_fetched(fetched_at, &probe("a"));+ // Opened 3 h 55 min before the probe: inside the 4-hour window then,+ // outside it ten minutes later.+ raw.prs_opened+ .insert(7, now - CDuration::minutes(3 * 60 + 55));+ let checks = vec![("o/r0".into(), raw.clone(), Some(probe("a")))];+ let (restated, _) = restat_unchanged(&mut snap, checks, now);+ assert_eq!(restated, ["o/r0"]);+ let stats = snap.repos[0].stats.clone().unwrap();+ assert_eq!(stats.recent["4h"].prs_opened, 1);+ assert_eq!(Some(stats), craft_github::restat(&raw, now));+ let later = craft_github::restat(&raw, now + CDuration::minutes(10)).unwrap();+ assert_eq!(later.recent["4h"].prs_opened, 0);+ // Without a probe there is nothing to compare: the repo is fetched.+ let checks = vec![("o/r0".into(), raw, None)];+ let (restated, left) = restat_unchanged(&mut snap, checks, now);+ assert!(restated.is_empty() && left.is_empty());+ }+ #[test] fn rate_limits_end_when_github_says() { let now = "2026-10-18T04:20:00Z".parse::<DateTime<Utc>>().unwrap();@@ -246,6 +246,26 @@ test('the Fetched column shows each repo’s data age and sorts by it, freshest assert.deepEqual(h.rows(), ['beta', 'alpha', 'delta', 'gamma']); }); +test('a check that found nothing changed shows in the Fetched tooltip and keeps the row fresh', async () => {+ const ago = (s: number) => new Date(Date.now() - s * 1000).toISOString();+ const h = await setup({ snap: { ...snapshot([+ repo('alpha', {}, { fetched_at: ago(90 * 60), checked_at: ago(5 * 60) }),+ repo('beta', {}, { fetched_at: ago(90 * 60) }),+ repo('gamma', {}, { fetched_at: ago(5 * 60), checked_at: ago(60 * 60) }),+ ]), stale_after_secs: 3600 } });+ const age = (name: string) => h.row(name).cells[1]!.querySelector('span[title]')!;+ assert.equal(age('alpha').textContent, '1h', 'the age of the last full fetch');+ const lines = age('alpha').getAttribute('title')!.split('\n');+ assert.equal(lines.length, 2);+ assert.match(lines[0]!, /^Fetched /);+ assert.match(lines[1]!, /^Checked .+; nothing had changed since the fetch$/);+ assert.doesNotMatch(age('beta').getAttribute('title')!, /Checked/);+ assert.doesNotMatch(age('gamma').getAttribute('title')!, /Checked/, 'a check older than the fetch');+ assert.ok(!h.row('alpha').classList.contains('stale'), 'checked five minutes ago');+ assert.ok(h.row('beta').classList.contains('stale'));+ assert.ok(!h.row('gamma').classList.contains('stale'));+});+ test('a row’s refresh button refetches only that repo, and spins while it waits', async () => { const h = await setup(); const button = () => h.row('gamma').querySelector<HTMLButtonElement>('.row-refresh')!;@@ -3,7 +3,7 @@ import { COLUMNS, RECENT, WINDOWS, MANUAL, windowById, value, orderedEntries, nextSort, dropIndex,- freshness, ago, totals, urgentAcross, tokenStatus, isRefreshing,+ freshness, ago, totals, urgentAcross, tokenStatus, isRefreshing, checkedAfterFetch, isStale, } from './model.ts'; import type { Column, Meta, RepoEntry, Snapshot, Sort } from './model.ts'; @@ -119,7 +119,9 @@ export function start(api: Api): () => void { return el('span', { class: 'repo', title: tip }, el('span', { class: 'dot' }), appIcon(entry), link(String(v), gh(entry.repo))); } if (col.key === 'fetched') {- const age = v ? el('span', { title: `Fetched ${localTime(String(v))}` }, ago(String(v))) : el('span', { class: 'muted', title: 'Not fetched yet' }, '—');+ const checked = checkedAfterFetch(entry);+ const tip = [`Fetched ${localTime(String(v))}`, checked && `Checked ${localTime(checked)}; nothing had changed since the fetch`].filter(Boolean).join('\n');+ const age = v ? el('span', { title: tip }, ago(String(v))) : el('span', { class: 'muted', title: 'Not fetched yet' }, '—'); return el('span', { class: 'fetched' }, age, rowRefresh(entry)); } if (!s) return entry.inaccessible ? document.createTextNode('') : el('span', { class: 'muted' }, '—');@@ -238,7 +240,7 @@ export function start(api: Api): () => void { const rows = orderedEntries(snap.repos, state.sort, state.order, state.window); const now = Date.now(); $('tbody').replaceChildren(...rows.map((e) => {- const stale = !e.fetched_at || now - Date.parse(e.fetched_at) > snap.stale_after_secs * 1000;+ const stale = isStale(e, snap.stale_after_secs, now); return el('tr', { class: classes(stale && 'stale', e.error && 'error'), dataset: { repo: e.repo },@@ -140,6 +140,19 @@ test('a repo is stale once its data is older than the limit, or never fetched', assert.equal(isStale(entry('a', null, null), 1e9, now), true); }); +test('a check that found nothing changed keeps old data current', () => {+ const now = Date.parse('2026-10-08T13:00:00Z');+ const checked = { ...entry('a', {}), checked_at: '2026-10-08T12:55:00Z' };+ assert.equal(isStale(entry('a', {}), 600, now), true, 'fetched an hour ago');+ assert.equal(isStale(checked, 600, now), false, 'checked five minutes ago');+ const snap = { stale_after_secs: 600, repos: [checked, entry('b', {}, '2026-10-08T12:52:00Z')] };+ assert.deepEqual(freshness(snap, now), { oldest: Date.parse('2026-10-08T12:52:00Z'), stale: false });+ // A check from before the fetch does not make the data current.+ const early = { ...entry('a', {}), checked_at: '2026-10-08T11:00:00Z' };+ assert.equal(isStale(early, 600, now), true);+ assert.equal(freshness({ stale_after_secs: 600, repos: [early] }, now).oldest, Date.parse('2026-10-08T12:00:00Z'));+});+ test('formatting', () => { const now = Date.parse('2026-10-08T12:00:00Z'); assert.equal(ago('2026-10-08T11:59:55Z', now), 'now');@@ -60,6 +60,8 @@ export interface RepoEntry { repo: string; stats: RepoStats | null; fetched_at: Time | null;+ /** When a probe last found nothing changed since `fetched_at` (absent from older builds). */+ checked_at?: Time | null; last_attempt_at?: Time | null; error?: string | null; inaccessible?: boolean;@@ -283,14 +285,26 @@ export function dropIndex(mids: readonly number[], y: number): number { // ───────────────────────────── freshness ───────────────────────────── +/** The check time when a probe found nothing changed after the fetch, else null. As `checked_at` in model.rs. */+export function checkedAfterFetch(entry: RepoEntry): Time | null {+ const { fetched_at: fetched, checked_at: checked } = entry;+ return checked && (!fetched || Date.parse(checked) > Date.parse(fetched)) ? checked : null;+}++/** When the entry's data was last known current: the fetch, or a later check. As `RepoEntry::current_at` in model.rs. */+export function currentAt(entry: RepoEntry): Time | null {+ return checkedAfterFetch(entry) ?? entry.fetched_at;+}+ export function isStale(entry: RepoEntry, staleAfterSecs: number, now: number = Date.now()): boolean {- if (!entry.fetched_at) return true;- return now - Date.parse(entry.fetched_at) > staleAfterSecs * 1000;+ const at = currentAt(entry);+ if (!at) return true;+ return now - Date.parse(at) > staleAfterSecs * 1000; } /** `{ oldest, stale }` for the whole snapshot: the age of the oldest data on screen. */ export function freshness(snap: Pick<Snapshot, 'repos' | 'stale_after_secs'>, now: number = Date.now()): { oldest: number | null; stale: boolean } {- const times = snap.repos.map((r) => (r.fetched_at ? Date.parse(r.fetched_at) : null));+ const times = snap.repos.map((r) => { const at = currentAt(r); return at ? Date.parse(at) : null; }); const oldest = times.length && times.every((t) => t !== null) ? Math.min(...(times as number[])) : null; const stale = !snap.repos.length || snap.repos.some((r) => isStale(r, snap.stale_after_secs, now)); return { oldest, stale };