Integration Patterns

Learn advanced patterns for HTTP propagation, gRPC tracing, and distributed tracing across services

Overview

This guide covers real-world integration patterns for distributed tracing, including HTTP context propagation, gRPC instrumentation, and baggage handling.

HTTP Context Propagation

Context propagation ensures that trace context is passed between services, maintaining the parent-child relationship across network boundaries.

Client-Side: Injecting Context

1. Set Up Propagators

   Configure the propagators that will serialize trace context into HTTP headers.

   use opentelemetry::global;
   use opentelemetry_sdk::propagation::TraceContextPropagator;
   
   fn init_tracer() -> SdkTracerProvider {
       global::set_text_map_propagator(TraceContextPropagator::new());
       
       let provider = SdkTracerProvider::builder()
           .with_simple_exporter(SpanExporter::default())
           .build();
   
       global::set_tracer_provider(provider.clone());
       provider
   }
   
2. Inject Context into Request Headers

   Before sending an HTTP request, inject the current trace context into headers.

   use opentelemetry::{
       global,
       trace::{SpanKind, TraceContextExt, Tracer},
       Context,
   };
   use opentelemetry_http::{Bytes, HeaderInjector};
   use hyper_util::{client::legacy::Client, rt::TokioExecutor};
   use http_body_util::Full;
   
   async fn send_request(
       url: &str,
       body_content: &str,
       span_name: &str,
   ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
       let client = Client::builder(TokioExecutor::new()).build_http();
       let tracer = global::tracer("example/client");
       
       // Create a client span
       let span = tracer
           .span_builder(String::from(span_name))
           .with_kind(SpanKind::Client)
           .start(&tracer);
       let cx = Context::current_with_span(span);
   
       // Build request and inject context
       let mut req = hyper::Request::builder().uri(url);
       global::get_text_map_propagator(|propagator| {
           propagator.inject_context(&cx, &mut HeaderInjector(req.headers_mut().unwrap()))
       });
       
       let res = client
           .request(req.body(Full::new(Bytes::from(body_content.to_string())))?)
           .await?;
   
       Ok(())
   }
   

Server-Side: Extracting Context

1. Extract Context from Incoming Headers

   On the server side, extract trace context from incoming request headers.

   use opentelemetry::{
       global,
       propagation::TextMapCompositePropagator,
       trace::{SpanKind, TraceContextExt, Tracer},
       Context,
   };
   use opentelemetry_http::HeaderExtractor;
   use opentelemetry_sdk::propagation::{BaggagePropagator, TraceContextPropagator};
   use hyper::{body::Incoming, Request};
   
   // Utility function to extract the context from the incoming request headers
   fn extract_context_from_request(req: &Request<Incoming>) -> Context {
       global::get_text_map_propagator(|propagator| {
           propagator.extract(&HeaderExtractor(req.headers()))
       })
   }
   
2. Create Server Span with Parent Context

   Use the extracted context to create a server span that continues the trace.

   use opentelemetry::KeyValue;
   use opentelemetry_semantic_conventions::trace;
   
   async fn router(req: Request<Incoming>) -> Result<Response<BoxBody<Bytes, hyper::Error>>, Infallible> {
       // Extract the context from incoming headers
       let parent_cx = extract_context_from_request(&req);
       
       // Create a span parenting the remote client span
       let tracer = global::tracer("example/server");
       let span = tracer
           .span_builder("router")
           .with_kind(SpanKind::Server)
           .start_with_context(&tracer, &parent_cx);
   
       let cx = parent_cx.with_span(span);
       
       // Handle request within context
       match (req.method(), req.uri().path()) {
           (&hyper::Method::GET, "/health") => handle_health_check(req).with_context(cx).await,
           _ => {
               cx.span().set_attribute(KeyValue::new(trace::HTTP_RESPONSE_STATUS_CODE, 404));
               let mut not_found = Response::new(BoxBody::default());
               *not_found.status_mut() = StatusCode::NOT_FOUND;
               Ok(not_found)
           }
       }
   }
   

Baggage Propagation

Baggage allows you to propagate key-value pairs across service boundaries, useful for cross-cutting concerns like user IDs or request IDs.

Setting Up Composite Propagator

use opentelemetry::propagation::TextMapCompositePropagator;
use opentelemetry_sdk::propagation::{BaggagePropagator, TraceContextPropagator};

fn init_tracer_with_baggage() -> SdkTracerProvider {
    let baggage_propagator = BaggagePropagator::new();
    let trace_context_propagator = TraceContextPropagator::new();
    let composite_propagator = TextMapCompositePropagator::new(vec![
        Box::new(baggage_propagator),
        Box::new(trace_context_propagator),
    ]);

    global::set_text_map_propagator(composite_propagator);

    let provider = SdkTracerProvider::builder()
        .with_span_processor(EnrichWithBaggageSpanProcessor)
        .with_simple_exporter(SpanExporter::default())
        .build();

    global::set_tracer_provider(provider.clone());
    provider
}

Enriching Spans with Baggage

Create a custom span processor to automatically add baggage attributes to spans:

use opentelemetry::baggage::BaggageExt;
use opentelemetry_sdk::{
    error::OTelSdkResult,
    trace::{SpanProcessor, SpanData},
};
use std::time::Duration;

#[derive(Debug)]
struct EnrichWithBaggageSpanProcessor;

impl SpanProcessor for EnrichWithBaggageSpanProcessor {
    fn on_start(&self, span: &mut opentelemetry_sdk::trace::Span, cx: &Context) {
        for (key, value) in cx.baggage().iter() {
            span.set_attribute(KeyValue::new(key.clone(), value.0.clone()));
        }
    }

    fn on_end(&self, _span: SpanData) {}

    fn force_flush(&self) -> OTelSdkResult {
        Ok(())
    }

    fn shutdown_with_timeout(&self, _timeout: Duration) -> OTelSdkResult {
        Ok(())
    }
}

Sending Baggage in HTTP Requests

// Client side: Add baggage to request
req.headers_mut()
    .unwrap()
    .insert("baggage", "user_id=12345,is_synthetic=true".parse().unwrap());

gRPC Distributed Tracing

Instrument gRPC services with OpenTelemetry for complete distributed tracing.

gRPC Server Implementation

1. Define gRPC Service

   Create a proto file for your service.

   proto/helloworld.proto

   syntax = "proto3";
   package helloworld;
   
   service Greeter {
       rpc SayHello (HelloRequest) returns (HelloReply);
   }
   
   message HelloRequest {
       string name = 1;
   }
   
   message HelloReply {
       string message = 1;
   }
   
2. Create Metadata Extractor

   Implement an extractor for gRPC metadata to extract trace context.

   use opentelemetry::propagation::Extractor;
   use tonic::metadata::MetadataMap;
   
   struct MetadataExtractor<'a>(&'a MetadataMap);
   
   impl Extractor for MetadataExtractor<'_> {
       fn get(&self, key: &str) -> Option<&str> {
           self.0.get(key).and_then(|metadata| metadata.to_str().ok())
       }
   
       fn keys(&self) -> Vec<&str> {
           self.0
               .keys()
               .map(|key| match key {
                   tonic::metadata::KeyRef::Ascii(v) => v.as_str(),
                   tonic::metadata::KeyRef::Binary(v) => v.as_str(),
               })
               .collect::<Vec<_>>()
       }
   }
   
3. Implement Service with Tracing

   Extract context and create server spans in your gRPC handlers.

   use tonic::{Request, Response, Status};
   
   #[derive(Debug, Default)]
   pub struct MyGreeter {}
   
   #[tonic::async_trait]
   impl Greeter for MyGreeter {
       async fn say_hello(
           &self,
           request: Request<HelloRequest>,
       ) -> Result<Response<HelloReply>, Status> {
           // Extract parent context from metadata
           let parent_cx = global::get_text_map_propagator(|prop| {
               prop.extract(&MetadataExtractor(request.metadata()))
           });
           
           let tracer = global::tracer("example/server");
           let mut span = tracer
               .span_builder("Greeter/server")
               .with_kind(SpanKind::Server)
               .with_attributes([
                   KeyValue::new("rpc.system", "grpc"),
                   KeyValue::new("server.port", 50052),
                   KeyValue::new("rpc.method", "say_hello"),
               ])
               .start_with_context(&tracer, &parent_cx);
   
           let name = request.into_inner().name;
           span.add_event(format!("Got name: {}", name), vec![]);
   
           let reply = HelloReply {
               message: format!("Hello {}!", name),
           };
   
           Ok(Response::new(reply))
       }
   }
   

gRPC Client Implementation

1. Create Metadata Injector

   Implement an injector to add trace context to gRPC metadata.

   use opentelemetry::propagation::Injector;
   
   struct MetadataInjector<'a>(&'a mut MetadataMap);
   
   impl Injector for MetadataInjector<'_> {
       fn set(&mut self, key: &str, value: String) {
           if let Ok(key) = tonic::metadata::MetadataKey::from_bytes(key.as_bytes()) {
               if let Ok(val) = tonic::metadata::MetadataValue::try_from(&value) {
                   self.0.insert(key, val);
               }
           }
       }
   }
   
2. Make Traced gRPC Calls

   Create a client span and inject context into the request metadata.

   use hello_world::greeter_client::GreeterClient;
   use hello_world::HelloRequest;
   
   async fn greet() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
       let tracer = global::tracer("example/client");
       let span = tracer
           .span_builder("Greeter/client")
           .with_kind(SpanKind::Client)
           .with_attributes([
               KeyValue::new("rpc.system", "grpc"),
               KeyValue::new("server.port", 50052),
               KeyValue::new("rpc.method", "say_hello"),
           ])
           .start(&tracer);
       
       let cx = Context::current_with_span(span);
       let mut client = GreeterClient::connect("http://[::1]:50052").await?;
   
       let mut request = tonic::Request::new(HelloRequest {
           name: "Tonic".into(),
       });
   
       // Inject trace context into metadata
       global::get_text_map_propagator(|propagator| {
           propagator.inject_context(&cx, &mut MetadataInjector(request.metadata_mut()))
       });
   
       let response = client.say_hello(request).await;
       
       // Add response status to span
       let span = cx.span();
       match response {
           Ok(_) => span.set_attribute(KeyValue::new("response", "OK")),
           Err(status) => {
               span.set_attribute(KeyValue::new("response_code_desc", status.code().description()));
           }
       }
   
       Ok(())
   }
   

Complete HTTP Propagation Example

Here’s a complete working example of an HTTP server and client with context propagation:

Server

use hyper::{body::Incoming, service::service_fn, Request, Response};
use hyper_util::rt::{TokioExecutor, TokioIo};
use tokio::net::TcpListener;
use std::net::SocketAddr;

#[tokio::main]
async fn main() {
    use hyper_util::server::conn::auto::Builder;

    let provider = init_tracer();
    let logger_provider = init_logs();
    let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
    let listener = TcpListener::bind(addr).await.unwrap();

    while let Ok((stream, _addr)) = listener.accept().await {
        if let Err(err) = Builder::new(TokioExecutor::new())
            .serve_connection(TokioIo::new(stream), service_fn(router))
            .await
        {
            eprintln!("{err}");
        }
    }

    provider.shutdown().expect("Shutdown provider failed");
    logger_provider.shutdown().expect("Shutdown provider failed");
}

Dependencies for Full Examples

Cargo.toml

[dependencies]
http-body-util = "0.1"
hyper = { version = "1.0", features = ["full"] }
hyper-util = { version = "0.1", features = ["full"] }
tokio = { version = "1", features = ["full"] }
opentelemetry = "*"
opentelemetry_sdk = "*"
opentelemetry-http = "*"
opentelemetry-stdout = { version = "*", features = ["trace", "logs"] }
opentelemetry-semantic-conventions = "*"
opentelemetry-appender-tracing = "*"
tracing = { version = "0.1", features = ["std"] }
tracing-subscriber = { version = "0.3", features = ["env-filter", "registry", "std", "fmt"] }

# For gRPC examples
tonic = { version = "0.12", features = ["server", "codegen", "channel", "router"] }
prost = "0.13"

[build-dependencies]
tonic-prost-build = "0.4"  # For compiling .proto files

These integration patterns are production-ready and based on the official OpenTelemetry Rust examples. Adapt them to your specific framework and use case.

Always propagate context across async boundaries using with_context() from the FutureExt trait to maintain proper parent-child relationships in your traces.