Production Debugging

use tracing::{info, instrument};
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
 
pub fn init_tracing() {
    tracing_subscriber::registry()
        .with(EnvFilter::from_default_env())
        .with(tracing_subscriber::fmt::layer().json())
        .init();
}
 
#[instrument(skip(pool), fields(trace_id, tenant_id))]
async fn handle_order(
    trace_id: String,
    tenant_id: String,
    pool: &sqlx::PgPool,
) -> Result<(), sqlx::Error> {
    tracing::Span::current().record("trace_id", &trace_id);
    tracing::Span::current().record("tenant_id", &tenant_id);
    info!("order_created", order_id = "ord_123");
    sqlx::query("SELECT 1").execute(pool).await?;
    Ok(())
}

# Live async task inspection (enable tokio_unstable + console subscriber in staging)
RUSTFLAGS="--cfg tokio_unstable" cargo run --features console
 
# Attach debugger to running binary (needs debug symbols in image)
lldb -p $(pgrep orders-api)
(gdb) thread apply all bt

//! prod_debug_helpers.rs - minimal toolkit for correlating prod evidence.
use std::time::Instant;
 
use serde::Serialize;
use tracing::{info, instrument};
 
#[derive(Debug, Serialize)]
pub struct DeployContext {
    pub git_sha: String,
    pub rust_version: String,
    pub service: String,
}
 
pub fn deploy_context() -> DeployContext {
    DeployContext {
        git_sha: std::env::var("GIT_SHA").unwrap_or_else(|_| "unknown".into()),
        rust_version: env!("RUSTC_VERSION").to_string(),
        service: std::env::var("SERVICE_NAME").unwrap_or_else(|_| "api".into()),
    }
}
 
#[instrument(skip(f), fields(request_id, trace_id))]
pub async fn with_request_scope<F, T>(
    request_id: &str,
    trace_id: &str,
    f: F,
) -> T
where
    F: std::future::Future<Output = T>,
{
    let span = tracing::Span::current();
    span.record("request_id", request_id);
    span.record("trace_id", trace_id);
    let ctx = deploy_context();
    info!(?ctx, "request_started");
    let start = Instant::now();
    let result = f.await;
    info!(duration_ms = start.elapsed().as_millis() as u64, "request_finished");
    result
}

# Filter JSON logs by trace_id in prod
kubectl logs deploy/orders-api | jq 'select(.fields.trace_id=="abc-123")'
 
# Sample stacks without restart (Linux, needs perf permissions)
perf record -p $(pgrep orders-api) -g -- sleep 30
perf script

// Cargo.toml dependencies (typical Axum service)
// tracing, tracing-subscriber, tracing-opentelemetry
// tower-http with TraceLayer for HTTP request IDs