diff --git a/_docs/sora_latency_investigation_1069.md b/_docs/sora_latency_investigation_1069.md new file mode 100644 --- /dev/null +++ b/_docs/sora_latency_investigation_1069.md @@ -0,0 +1,67 @@ +# Sora latency investigation (#1069) + +Investigation-only instrumentation for +[storytold/artcraft#1069](https://github.com/storytold/artcraft/issues/1069): +users reporting longer Sora times after upgrading to Basic or Pro. + +**No product behavior change.** Sora models remain `is_disabled` until a +separate fix PR is approved after reviewing these metrics. + +## Metrics (Datadog) + +| Metric | Type | When | +|--------------------------------|--------------|-------------------------------------------| +| `artcraft.sora.enqueue` | count | Successful Fal/DB enqueue for Sora jobs | +| `artcraft.sora.job_duration_ms`| distribution | Fal webhook success/failure for Sora jobs | + +### Tags (low cardinality) + +| Tag | Values | +|---------------|---------------------------------------------------------------------| +| `model` | `sora_2`, `sora_2_pro` | +| `plan` | `free`, `artcraft_basic`, `artcraft_pro`, `artcraft_max`, `unknown` | +| `resolution` | `720p`, `1080p`, `other` | +| `duration_s` | `4`, `8`, `12`, `other` | +| `outcome` | `success`, `failure` (duration metric only) | + +`plan` comes from the user's Artcraft subscription row (`free` when none). +Completion events may use `resolution:other` / `duration_s:other` when those +fields are not cheaply available on the Fal job row. + +## Suggested Datadog queries + +Compare paid vs free wall-clock: + +```text +p50:artcraft.sora.job_duration_ms{outcome:success} by {plan,model} +p90:artcraft.sora.job_duration_ms{outcome:success} by {plan,model} +``` + +Enqueue volume after upgrades (are paid users submitting more?): + +```text +sum:artcraft.sora.enqueue{*}.as_count() by {plan,model} +``` + +Filter to a single tier: + +```text +p90:artcraft.sora.job_duration_ms{plan:artcraft_basic,outcome:success} by {model,resolution,duration_s} +p90:artcraft.sora.job_duration_ms{plan:free,outcome:success} by {model,resolution,duration_s} +``` + +## Code touchpoints + +- Tag helpers / recorders: + `crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/sora_latency_metrics.rs` +- Omni enqueue: `shared_video_generation.rs` (Fal branch) +- Legacy multi-function enqueue: `sora_2_*_multi_function_video_gen_handler.rs` +- Completion: Fal webhook success/failure handlers + +## Follow-up + +1. Deploy with `ENABLE_METRICS=true` and a Datadog API key. +2. Collect enough Sora samples (may require temporary staging enablement while + catalog `is_disabled` remains true in production). +3. Summarize p50/p90 by `plan` for an admin. +4. Open a **separate** fix PR only after admin approval. diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen_handler.rs --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen_handler.rs @@ -382,6 +382,29 @@ pub async fn sora_2_multi_function_video_gen_handler( return Err(CommonWebError::from_error(err)); } }; + + // Issue #1069 investigation: legacy multi-function Sora enqueue (no behavior change). + { + use crate::http_server::endpoints::omni_gen::shared_utils::sora_latency_metrics::{ + duration_tag, record_sora_enqueue, resolution_tag, resolve_plan_tag_for_user_in_transaction, + }; + use enums::common::generation::common_resolution::CommonResolution; + + let plan = resolve_plan_tag_for_user_in_transaction(user_token, &mut transaction).await; + let duration_seconds = request.duration.as_ref().map(|d| match d { + Sora2MultiFunctionVideoGenDuration::FourSeconds => 4u16, + Sora2MultiFunctionVideoGenDuration::EightSeconds => 8, + Sora2MultiFunctionVideoGenDuration::TwelveSeconds => 12, + }); + let resolution = Some(CommonResolution::SevenTwentyP); + record_sora_enqueue( + &server_state.metrics, + "sora_2", + plan, + resolution_tag(resolution), + duration_tag(duration_seconds), + ); + } let _r = transaction .commit() diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen_handler.rs --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen_handler.rs @@ -384,6 +384,34 @@ pub async fn sora_2_pro_multi_function_video_gen_handler( return Err(CommonWebError::from_error(err)); } }; + + // Issue #1069 investigation: legacy multi-function Sora Pro enqueue (no behavior change). + { + use crate::http_server::endpoints::omni_gen::shared_utils::sora_latency_metrics::{ + duration_tag, record_sora_enqueue, resolution_tag, resolve_plan_tag_for_user_in_transaction, + }; + use enums::common::generation::common_resolution::CommonResolution; + + let plan = resolve_plan_tag_for_user_in_transaction(user_token, &mut transaction).await; + let duration_seconds = request.duration.as_ref().map(|d| match d { + Sora2ProMultiFunctionVideoGenDuration::FourSeconds => 4u16, + Sora2ProMultiFunctionVideoGenDuration::EightSeconds => 8, + Sora2ProMultiFunctionVideoGenDuration::TwelveSeconds => 12, + }); + let resolution = match request.resolution { + Some(Sora2ProMultiFunctionVideoGenResolution::TenEightyP) => Some(CommonResolution::TenEightyP), + Some(Sora2ProMultiFunctionVideoGenResolution::SevenTwentyP) + | Some(Sora2ProMultiFunctionVideoGenResolution::Auto) + | None => Some(CommonResolution::SevenTwentyP), + }; + record_sora_enqueue( + &server_state.metrics, + "sora_2_pro", + plan, + resolution_tag(resolution), + duration_tag(duration_seconds), + ); + } let _r = transaction .commit() diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/shared_video_generation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/shared_video_generation.rs --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/shared_video_generation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/shared_video_generation.rs @@ -58,6 +58,10 @@ use crate::http_server::endpoints::omni_gen::generate::video::insert_db_job::ins use crate::http_server::endpoints::omni_gen::generate::video::insert_db_job::shared_job_args::SharedJobArgs; use crate::http_server::endpoints::omni_gen::generate::video::pipeline_v2::run_pipeline_v2::{run_pipeline_v2, RunPipelineV2Args}; use crate::http_server::endpoints::omni_gen::shared_utils::kinovi_account::KinoviAccount; +use crate::http_server::endpoints::omni_gen::shared_utils::sora_latency_metrics::{ + duration_tag, model_tag_from_video_model, record_sora_enqueue, resolution_tag, + resolve_plan_tag_for_user_in_transaction, +}; use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; use crate::state::server_state::ServerState; use crate::util::lookup::lookup_media_files_as_cdn_url_list_and_map::lookup_media_files_as_cdn_url_list_and_map; @@ -394,6 +398,18 @@ server_state.maybe_media_cdn_override_url.as_deref(), }, }).await?; + // Issue #1069 investigation: Sora enqueue volume by plan (no behavior change). + if let Some(model) = request.model.and_then(model_tag_from_video_model) { + let plan = resolve_plan_tag_for_user_in_transaction(user_token, &mut transaction).await; + record_sora_enqueue( + &server_state.metrics, + model, + plan, + resolution_tag(request.resolution), + duration_tag(request.duration_seconds), + ); + } + ( token.clone(), vec![token], diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/support.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/support.rs --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/support.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/generate/video/tests/support.rs @@ -735,6 +735,7 @@ pub(crate) fn build_test_server_state(pool: MySqlPool, media_cdn_override_url: S daily_generations_id: None, }, }, + metrics: metrics::collector::MetricsCollector::noop(), } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/mod.rs --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/mod.rs @@ -3,5 +3,6 @@ pub mod kinovi_account; pub mod map_router_cost_error; pub mod map_kinovi_web_router_error; pub mod mesh; +pub mod sora_latency_metrics; pub mod splat; pub mod video; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/sora_latency_metrics.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/sora_latency_metrics.rs new file mode 100644 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_gen/shared_utils/sora_latency_metrics.rs @@ -0,0 +1,283 @@ +//! Investigation metrics for GitHub issue #1069 (Sora latency by plan). +//! +//! Emits Datadog counters/distributions only — no behavior change. +//! See `_docs/sora_latency_investigation_1069.md`. + +use chrono::{DateTime, Utc}; +use enums::common::artcraft_subscription_slug::ArtcraftSubscriptionSlug; +use enums::common::generation::common_model_type::CommonModelType; +use enums::common::generation::common_resolution::CommonResolution; +use enums::common::generation::common_video_model::CommonVideoModel; +use enums::common::payments_namespace::PaymentsNamespace; +use log::warn; +use metrics::collector::MetricsCollector; +use mysql_queries::queries::users::user_subscriptions::find_subscription_for_owner_user::{ + find_subscription_for_owner_user_using_connection, + find_subscription_for_owner_user_using_transaction, +}; +use sqlx::pool::PoolConnection; +use sqlx::MySql; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::users::UserToken; + +pub const METRIC_ENQUEUE: &str = "artcraft.sora.enqueue"; +pub const METRIC_JOB_DURATION_MS: &str = "artcraft.sora.job_duration_ms"; + +/// Bucketed plan tag for Datadog (low cardinality). +pub fn plan_tag_from_product_slug(maybe_slug: Option<&str>) -> &'static str { + match maybe_slug { + None => "free", + Some(slug) => match ArtcraftSubscriptionSlug::from_str(slug) { + Ok(ArtcraftSubscriptionSlug::ArtcraftBasic) => "artcraft_basic", + Ok(ArtcraftSubscriptionSlug::ArtcraftPro) => "artcraft_pro", + Ok(ArtcraftSubscriptionSlug::ArtcraftMax) => "artcraft_max", + Err(_) => "unknown", + }, + } +} + +pub fn model_tag_from_video_model(model: CommonVideoModel) -> Option<&'static str> { + match model { + CommonVideoModel::Sora2 => Some("sora_2"), + CommonVideoModel::Sora2Pro => Some("sora_2_pro"), + _ => None, + } +} + +pub fn model_tag_from_model_type_str(maybe_model_type: Option<&str>) -> Option<&'static str> { + match maybe_model_type { + Some(s) => match CommonModelType::from_str(s) { + Ok(CommonModelType::Sora2) => Some("sora_2"), + Ok(CommonModelType::Sora2Pro) => Some("sora_2_pro"), + _ => None, + }, + None => None, + } +} + +pub fn resolution_tag(maybe_resolution: Option) -> &'static str { + match maybe_resolution { + Some(CommonResolution::SevenTwentyP) => "720p", + Some(CommonResolution::TenEightyP) => "1080p", + Some(_) | None => "other", + } +} + +pub fn duration_tag(maybe_duration_seconds: Option) -> &'static str { + match maybe_duration_seconds { + Some(4) => "4", + Some(8) => "8", + Some(12) => "12", + _ => "other", + } +} + +fn build_tags( + model: &str, + plan: &str, + resolution: &str, + duration_s: &str, + maybe_outcome: Option<&str>, +) -> Vec { + let mut tags = vec![ + format!("model:{}", model), + format!("plan:{}", plan), + format!("resolution:{}", resolution), + format!("duration_s:{}", duration_s), + ]; + if let Some(outcome) = maybe_outcome { + tags.push(format!("outcome:{}", outcome)); + } + tags +} + +pub fn record_sora_enqueue( + collector: &MetricsCollector, + model: &str, + plan: &str, + resolution: &str, + duration_s: &str, +) { + collector.increment( + METRIC_ENQUEUE, + build_tags(model, plan, resolution, duration_s, None), + ); +} + +pub fn record_sora_job_duration_ms( + collector: &MetricsCollector, + model: &str, + plan: &str, + resolution: &str, + duration_s: &str, + outcome: &str, + duration_ms: f64, +) { + collector.observe( + METRIC_JOB_DURATION_MS, + duration_ms, + build_tags(model, plan, resolution, duration_s, Some(outcome)), + ); +} + +fn plan_tag_from_lookup_result( + user_token: &UserToken, + result: Result< + Option, + mysql_queries::errors::select_optional_record_error::SelectOptionalRecordError, + >, +) -> &'static str { + match result { + Ok(Some(sub)) => plan_tag_from_product_slug(Some(sub.subscription_product_slug.as_str())), + Ok(None) => "free", + Err(err) => { + warn!( + "Sora latency metrics: failed to look up subscription for {}: {:?}", + user_token.as_str(), + err + ); + "unknown" + } + } +} + +/// Resolve Artcraft plan slug for metrics (`free` when no active subscription). +pub async fn resolve_plan_tag_for_user( + user_token: &UserToken, + mysql_connection: &mut PoolConnection, +) -> &'static str { + plan_tag_from_lookup_result( + user_token, + find_subscription_for_owner_user_using_connection( + user_token, + PaymentsNamespace::Artcraft, + mysql_connection, + ) + .await, + ) +} + +/// Same as [`resolve_plan_tag_for_user`] but on an open transaction (enqueue path). +pub async fn resolve_plan_tag_for_user_in_transaction( + user_token: &UserToken, + transaction: &mut sqlx::Transaction<'_, MySql>, +) -> &'static str { + plan_tag_from_lookup_result( + user_token, + find_subscription_for_owner_user_using_transaction( + user_token, + PaymentsNamespace::Artcraft, + transaction, + ) + .await, + ) +} + +/// Runtime (non-macro) lookup so we do not need a new SQLx prepare entry. +pub async fn fetch_maybe_model_type_for_job( + job_token: &InferenceJobToken, + mysql_connection: &mut PoolConnection, +) -> Option { + let result: Result, sqlx::Error> = sqlx::query_scalar( + "SELECT maybe_model_type FROM generic_inference_jobs WHERE token = ?", + ) + .bind(job_token.as_str()) + .fetch_optional(&mut **mysql_connection) + .await; + + match result { + Ok(value) => value, + Err(err) => { + warn!( + "Sora latency metrics: failed to load maybe_model_type for {}: {:?}", + job_token.as_str(), + err + ); + None + } + } +} + +pub fn job_duration_ms_since(created_at: DateTime) -> f64 { + let now = Utc::now(); + let millis = (now - created_at).num_milliseconds(); + if millis < 0 { + 0.0 + } else { + millis as f64 + } +} + +/// Best-effort completion metric for Fal webhooks on Sora jobs. +pub async fn maybe_record_sora_fal_job_completion( + collector: &MetricsCollector, + job_token: &InferenceJobToken, + maybe_creator_user_token: Option<&UserToken>, + created_at: DateTime, + outcome: &str, + mysql_connection: &mut PoolConnection, +) { + let maybe_model_type = fetch_maybe_model_type_for_job(job_token, mysql_connection).await; + let Some(model) = model_tag_from_model_type_str(maybe_model_type.as_deref()) else { + return; + }; + + let plan = match maybe_creator_user_token { + Some(user_token) => resolve_plan_tag_for_user(user_token, mysql_connection).await, + None => "unknown", + }; + + record_sora_job_duration_ms( + collector, + model, + plan, + "other", + "other", + outcome, + job_duration_ms_since(created_at), + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plan_tag_maps_known_slugs_and_defaults() { + assert_eq!(plan_tag_from_product_slug(None), "free"); + assert_eq!(plan_tag_from_product_slug(Some("artcraft_basic")), "artcraft_basic"); + assert_eq!(plan_tag_from_product_slug(Some("artcraft_pro")), "artcraft_pro"); + assert_eq!(plan_tag_from_product_slug(Some("artcraft_max")), "artcraft_max"); + assert_eq!(plan_tag_from_product_slug(Some("fakeyou_plus")), "unknown"); + } + + #[test] + fn model_tag_only_for_sora() { + assert_eq!(model_tag_from_video_model(CommonVideoModel::Sora2), Some("sora_2")); + assert_eq!(model_tag_from_video_model(CommonVideoModel::Sora2Pro), Some("sora_2_pro")); + assert_eq!(model_tag_from_video_model(CommonVideoModel::Kling2p6Pro), None); + assert_eq!(model_tag_from_model_type_str(Some("sora_2")), Some("sora_2")); + assert_eq!(model_tag_from_model_type_str(Some("sora_2_pro")), Some("sora_2_pro")); + assert_eq!(model_tag_from_model_type_str(Some("kling_2p6_pro")), None); + assert_eq!(model_tag_from_model_type_str(None), None); + } + + #[test] + fn resolution_and_duration_buckets() { + assert_eq!(resolution_tag(Some(CommonResolution::SevenTwentyP)), "720p"); + assert_eq!(resolution_tag(Some(CommonResolution::TenEightyP)), "1080p"); + assert_eq!(resolution_tag(Some(CommonResolution::FourEightyP)), "other"); + assert_eq!(resolution_tag(None), "other"); + assert_eq!(duration_tag(Some(4)), "4"); + assert_eq!(duration_tag(Some(8)), "8"); + assert_eq!(duration_tag(Some(12)), "12"); + assert_eq!(duration_tag(Some(5)), "other"); + assert_eq!(duration_tag(None), "other"); + } + + #[test] + fn job_duration_non_negative() { + let past = Utc::now() - chrono::Duration::seconds(3); + assert!(job_duration_ms_since(past) >= 0.0); + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_failure/handle_failed_fal_webhook.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_failure/handle_failed_fal_webhook.rs --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_failure/handle_failed_fal_webhook.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_failure/handle_failed_fal_webhook.rs @@ -1,4 +1,5 @@ use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::endpoints::omni_gen::shared_utils::sora_latency_metrics::maybe_record_sora_fal_job_completion; use crate::state::server_state::ServerState; use actix_web::web::Json; use enums::by_table::debug_logs::debug_log_level::DebugLogLevel; @@ -116,6 +117,16 @@ pub async fn handle_failed_fal_webhook( request_id, ); + // Issue #1069 investigation: Sora wall-clock by plan (no behavior change). + maybe_record_sora_fal_job_completion( + &server_state.metrics, + &job.job_token, + job.maybe_creator_user_token.as_ref(), + job.created_at, + "failure", + mysql_connection, + ).await; + Ok(SimpleGenericJsonSuccess::wrapped(true)) } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/handle_successful_fal_webhook.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/handle_successful_fal_webhook.rs --- a/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/handle_successful_fal_webhook.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/handle_successful_fal_webhook.rs @@ -11,6 +11,7 @@ use pager::client::pager::Pager; use sqlx::pool::PoolConnection; use sqlx::MySql; use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::endpoints::omni_gen::shared_utils::sora_latency_metrics::maybe_record_sora_fal_job_completion; use crate::state::server_state::ServerState; use super::process_audio_payload::process_audio_payload; @@ -177,5 +178,15 @@ pub async fn handle_successful_fal_webhook( warn!("No media token found in payload for request_id {} / job {:?}", request_id, job.job_token); } + // Issue #1069 investigation: Sora wall-clock by plan (no behavior change). + maybe_record_sora_fal_job_completion( + &server_state.metrics, + &job.job_token, + job.maybe_creator_user_token.as_ref(), + job.created_at, + "success", + mysql_connection, + ).await; + Ok(SimpleGenericJsonSuccess::wrapped(true)) } diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -53,7 +53,6 @@ use crate::http_server::routes::add_routes::add_routes; use crate::http_server::web_utils::handle_multipart_error::handle_multipart_error; use crate::startup::build_dependencies::setup_dependencies; use crate::startup::setup_disabled_endpoints::read_disabled_endpoints; -use crate::startup::setup_metrics::build_metrics; use crate::state::server_state::ServerState; use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; use crate::threads::expire_login_challenges::expire_login_challenges; @@ -109,6 +108,7 @@ async fn main() -> AnyhowResult<()> { let server_state = setup.server_state; let health_check_interval = setup.health_check_interval; let pager_worker = setup.pager_worker; + let maybe_metrics_worker = setup.maybe_metrics_worker; // ==================== Background threads ==================== // @@ -168,8 +168,7 @@ async fn main() -> AnyhowResult<()> { // ==================== Metrics worker ==================== // - let (metrics_collector, maybe_metrics_worker) = - build_metrics(server_state.server_environment, &server_state.hostname); + let metrics_collector = server_state.metrics.clone(); let maybe_metrics_shutdown = maybe_metrics_worker.map(|worker| { info!("Spawning metrics worker thread."); diff --git a/crates/service/web/storyteller_web/src/startup/build_dependencies.rs b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs --- a/crates/service/web/storyteller_web/src/startup/build_dependencies.rs +++ b/crates/service/web/storyteller_web/src/startup/build_dependencies.rs @@ -15,6 +15,7 @@ use memory_caching::single_item_ttl_cache::SingleItemTtlCache; use mysql_queries::mediators::badge_granter::BadgeGranter; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use crate::http_server::web_utils::web_opaque_cursor_encoder_v2::WebOpaqueCursorEncoderV2; +use metrics::worker::MetricsWorker; use pager::worker::pager_worker::PagerWorker; use redis::Client; use redis_caching::redis_ttl_cache::RedisTtlCache; @@ -36,6 +37,7 @@ use crate::startup::setup_bans::{ use crate::startup::setup_inference_providers::setup_inference_providers; use crate::startup::setup_internal_api_keys::setup_internal_api_keys; use crate::startup::setup_dashboards::setup_dashboards; +use crate::startup::setup_metrics::build_metrics; use crate::startup::setup_seedance_video_bucket::setup_seedance_video_bucket; use crate::startup::setup_static_feature_flags::setup_static_feature_flags; use crate::startup::setup_stripe_artcraft::setup_stripe_artcraft; @@ -65,6 +67,8 @@ pub struct SetupResult { pub server_state: ServerState, pub pager_worker: PagerWorker, pub health_check_interval: Duration, + /// Present when Datadog metrics are enabled; spawn and drain on shutdown. + pub maybe_metrics_worker: Option, } pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult { @@ -199,6 +203,9 @@ pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult AnyhowResult