collective
Browse / github.com/storytold/artcraft / patch
Fix Grok index scrape for CDN chunk URLs and verification meta
Fix Grok index scrape for CDN chunk URLs and verification meta ## Summary - Update Grok index script parsing for absolute `cdn.grok.com` chunk URLs - Accept unicode-dash `grok-site―verification` meta name - Note remaining XSID marker drift (`anonPrivateKey` / `880932`) Fixes part of #1033 (consumer scrape layer). ## Test plan - [ ] `SQLX_OFFLINE=true cargo check -p grok_consumer_client` - [ ] With valid Grok cookies, confirm index scripts/verification token parse (XSID may still fail)
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- giorgioabbruscato (imported from pull request #2284, open)
- Base
d4e4ac69bbd93d237eaf5d0260bc6a5ff3a5e373- Size
- +87 −12 in 5 files
- Review
- pending
- Shared
- 2026-10-28
- Id
acf0c3d80c319db4c908a67d
How it went
No other operator has reported using it yet.
Diff
raw · crates/api_clients/grok_consumer_client/src/requests/index_page/get_numbers.rs crates/api_clients/grok_consumer_client/src/requests/index_page/get_xsid_script.rs crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_script_list.rs crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/index/parse_index_verification_token.rs crates/api_clients/grok_consumer_client/src/requests/index_page/parsers/script/parse_script_actions_and_xsid_script_path.rs
@@ -24,7 +24,8 @@ pub async fn get_numbers(args: GetNumbersArgs<'_>) -> Result<XsidNumbers, GrokEr let on_demand = on_demand.unwrap_or_else(|| "".to_string()); format!("https://abs.twimg.com/responsive-web/client-web/ondemand.s.{}a.js", on_demand) },- _ => format!("https://grok.com/_next/{}", args.xsid_script_id),+ // 2026-03+: chunks are served from cdn.grok.com rather than grok.com.+ _ => format!("https://cdn.grok.com/_next/{}", args.xsid_script_id), }; let xsid_script_body = get_index_page_script_with_client(GetIndexPageScriptArgs {@@ -27,7 +27,8 @@ pub async fn get_xsid_script(args: GetXsidScriptArgs<'_>) -> Result<XsidScript, let on_demand = on_demand.unwrap_or_else(|| "".to_string()); format!("https://abs.twimg.com/responsive-web/client-web/ondemand.s.{}a.js", on_demand) },- _ => format!("https://grok.com/_next/{}", args.xsid_script_id),+ // 2026-03+: chunks are served from cdn.grok.com rather than grok.com.+ _ => format!("https://cdn.grok.com/_next/{}", args.xsid_script_id), }; let xsid_script_body = get_index_page_script_with_client(GetIndexPageScriptArgs {@@ -1,7 +1,9 @@ use once_cell::sync::Lazy; use scraper::{Html, Selector}; -// scripts: list = [s['src'] for s in BeautifulSoup(load_site.text, 'html.parser').find_all('script', src=True) if s['src'].startswith('/_next/static/chunks/')]+// Historical (relative): /_next/static/chunks/...+// 2026-03+: absolute CDN: https://cdn.grok.com/_next/static/chunks/...+const NEXT_STATIC_CHUNKS_MARKER: &str = "/_next/static/chunks/"; static SCRIPT_SELECTOR: Lazy<Selector> = Lazy::new(|| { Selector::parse("script[src]")@@ -14,18 +16,49 @@ pub fn parse_index_svg_paths(html: &str) -> Vec<String> { let selected = document.select(&SCRIPT_SELECTOR); selected .filter_map(|node| node.attr("src"))- .filter(|src| src.starts_with("/_next/static/chunks/"))+ .filter(|src| is_next_static_chunk_script(src)) .map(|s| s.to_string()) .collect::<Vec<_>>() } +fn is_next_static_chunk_script(src: &str) -> bool {+ src.starts_with(NEXT_STATIC_CHUNKS_MARKER) || src.contains(NEXT_STATIC_CHUNKS_MARKER)+}+ #[cfg(test)] mod tests { use crate::requests::index_page::get_index_page::{get_index, GetIndexPageArgs};- use crate::requests::index_page::parsers::index::parse_index_script_list::parse_index_svg_paths;+ use crate::requests::index_page::parsers::index::parse_index_script_list::{+ is_next_static_chunk_script, parse_index_svg_paths,+ }; use crate::test_utils::get_test_cookies::get_test_cookies; use errors::AnyhowResult; + #[test]+ fn accepts_relative_and_cdn_chunk_urls() {+ assert!(is_next_static_chunk_script("/_next/static/chunks/abc.js"));+ assert!(is_next_static_chunk_script(+ "https://cdn.grok.com/_next/static/chunks/abc.js"+ ));+ assert!(!is_next_static_chunk_script("https://cdn.grok.com/_next/static/css/abc.css"));+ }++ #[test]+ fn parses_cdn_script_tags_from_index_html() {+ let html = r#"+ <html><head>+ <script src="https://cdn.grok.com/_next/static/chunks/one.js"></script>+ <script src="/_next/static/chunks/two.js"></script>+ <script src="https://example.com/other.js"></script>+ </head></html>+ "#;++ let scripts = parse_index_svg_paths(html);+ assert_eq!(scripts.len(), 2);+ assert!(scripts.iter().any(|s| s.contains("one.js")));+ assert!(scripts.iter().any(|s| s.contains("two.js")));+ }+ #[tokio::test] #[ignore] // Manual test invocation async fn test() -> AnyhowResult<()> {@@ -3,23 +3,28 @@ use once_cell::sync::Lazy; use regex::Regex; use scraper::{Html, Selector}; +/// Historical ASCII: grok-site-verification+/// 2026-03+: grok-site―verification (U+2015 HORIZONTAL BAR)+const VERIFICATION_META_NAME_ASCII: &str = "grok-site-verification";+const VERIFICATION_META_NAME_UNICODE: &str = "grok-site\u{2015}verification";+ /// Find the verification token in the meta tags using HTML parsing. static META_SELECTOR : Lazy<Selector> = Lazy::new(|| {- Selector::parse("meta[name=grok-site-verification]")+ Selector::parse("meta[name]") .expect("HTML selector should parse") }); /// Find the verification token in the meta tags using regex // eg. <meta name="grok-site-verification" content="iFlTOEJNfQZP1B6YMRf/zuj3eYFrKWi6eNUg5XvvpllTOw5TS82coZkeUBdiHxYr"/> static META_TAG_REGEX : Lazy<Regex> = Lazy::new(|| {- Regex::new(r#""grok-site-verification"\s*content="([A-Za-z0-9/=+-]{5,})""#)+ Regex::new(r#""grok-site[\-\x{2015}]verification"\s*content="([A-Za-z0-9/=+-]{5,})""#) .expect("Regex should parse") }); /// Find the verification token in the script tags using regex /// eg. {\"name\":\"grok-site-verification\",\"content\":\"iFlTOEJNfQZP1B6YMRf/zuj3eYFrKWi6eNUg5XvvpllTOw5TS82coZkeUBdiHxYr\"}] static JSON_REGEX: Lazy<Regex> = Lazy::new(|| {- Regex::new(r#""grok-site-verification\\?",\s*\\?"content\\?":\\?"([A-Za-z0-9/=+-]{5,})\\?""#)+ Regex::new(r#""grok-site[\-\x{2015}]verification\\?",\s*\\?"content\\?":\\?"([A-Za-z0-9/=+-]{5,})\\?""#) .expect("Regex should parse") }); @@ -31,25 +36,42 @@ pub fn parse_index_verification_token(html: &str) -> Option<VerificationToken> { return Some(VerificationToken(meta)) } - let maybe_script = scrape_script_via_regex(html);+ let maybe_meta_regex = scrape_meta_tag_via_regex(html);++ if let Some(meta) = maybe_meta_regex {+ return Some(VerificationToken(meta))+ }++ let maybe_json = scrape_json_via_regex(html); - if let Some(script) = maybe_script {- return Some(VerificationToken(script))+ if let Some(token) = maybe_json {+ return Some(VerificationToken(token)) } None } -fn scrape_script_via_regex(html: &str) -> Option<String> {+fn is_verification_meta_name(name: &str) -> bool {+ name == VERIFICATION_META_NAME_ASCII || name == VERIFICATION_META_NAME_UNICODE+}++fn scrape_meta_tag_via_regex(html: &str) -> Option<String> { let captures = META_TAG_REGEX.captures(html)?; let capture = captures.get(1)?; Some(capture.as_str().to_string()) } +fn scrape_json_via_regex(html: &str) -> Option<String> {+ let captures = JSON_REGEX.captures(html)?;+ let capture = captures.get(1)?;+ Some(capture.as_str().to_string())+}+ fn scrape_meta_tag_via_parsing(html: &str) -> Option<String> { let document = Html::parse_document(html); let selected = document.select(&META_SELECTOR); let mut values = selected+ .filter(|node| node.attr("name").map(is_verification_meta_name).unwrap_or(false)) .filter_map(|node| node.attr("content")) .map(|s| s.to_string()) .collect::<Vec<_>>();@@ -63,6 +85,21 @@ mod tests { use crate::test_utils::get_test_cookies::get_test_cookies; use errors::AnyhowResult; + #[test]+ fn parses_ascii_hyphen_meta_name() {+ let html = r#"<meta name="grok-site-verification" content="iFlTOEJNfQZP1B6YMRf/zuj3eYFrKWi6eNUg5XvvpllTOw5TS82coZkeUBdiHxYr"/>"#;+ let token = parse_index_verification_token(html).expect("token");+ assert_eq!(token.0, "iFlTOEJNfQZP1B6YMRf/zuj3eYFrKWi6eNUg5XvvpllTOw5TS82coZkeUBdiHxYr");+ }++ #[test]+ fn parses_unicode_horizontal_bar_meta_name() {+ // 2026-03+: grok.com emits U+2015 instead of ASCII '-'+ let html = "<meta name=\"grok-site\u{2015}verification\" content=\"uRjaEmD3mkuTK6n0N+ZxvGdx2Bq1e8j2AYfyCaFmO1eHKULyZUaSB8EkYgNapjwS\"/>";+ let token = parse_index_verification_token(html).expect("token");+ assert_eq!(token.0, "uRjaEmD3mkuTK6n0N+ZxvGdx2Bq1e8j2AYfyCaFmO1eHKULyZUaSB8EkYgNapjwS");+ }+ #[tokio::test] #[ignore] // Manual test invocation async fn test() -> AnyhowResult<()> {@@ -11,6 +11,9 @@ static ACTIONS_REGEX : Lazy<Regex> = Lazy::new(|| { /// 2025-10-01: "(static/chunks/[^"]+\.js)"[^}]*?a\(880932\)" /// 2025-12-01: "(static/chunks/[^"]+\.js)"[^}]*?\(880932\)"+/// 2026-03: public chunks no longer embed `880932` / `anonPrivateKey` markers in the+/// same form; CDN script discovery was fixed separately. Update these markers when+/// a new stable fingerprint is identified. static XSID_SCRIPT_REGEX : Lazy<Regex> = Lazy::new(|| { Regex::new(r#""(static/chunks/[^"]+\.js)"[^}]*?\(880932\)"#) .expect("Regex should parse")