Merge branch 'otel'

This migrates AWS SDK instrumentation from the deprecated X-Ray SDK to
OpenTelemetry. OTel is better about not crashing the program when
there's no parent span or the collected traces aren't going anywhere,
so in general the instrumentation is always enabled but X-Ray exports
are only active if set in the environment.

In theory, this really opens up what we can do on the randomizer-server
side, but I'm not sure how many options are out there to export traces.

Author: featherbread

CloudFormation.yaml

@@ -73,7 +73,7 @@ Resources:
           DYNAMODB_TABLE: !Ref GroupsTable
           SLACK_TOKEN_SSM_NAME: !Sub '/${SlackTokenSSMName}'
           SLACK_TOKEN_SSM_TTL: !Ref SlackTokenSSMTTL
-          AWS_CLIENT_XRAY_TRACING: !If [HasXRayTracingEnabled, '1', !Ref AWS::NoValue]
+          AWS_XRAY_TRACING_ENABLED: !If [HasXRayTracingEnabled, '1', !Ref AWS::NoValue]
           AWS_CLIENT_EMBEDDED_TLS_ROOTS: !If [HasAWSClientEmbeddedTLSRoots, '1', !Ref AWS::NoValue]
       FunctionUrlConfig:
         AuthType: NONE

SERVERLESS.md

@@ -132,10 +132,10 @@ Run `./hfc help` to learn more about additional commands that might be useful.
   isn't eligible for the AWS Free Tier. See the [Read/Write Capacity
   Mode][capacity mode] documentation for details.
 - The default configuration enables [AWS X-Ray][x-ray] tracing for the function
-  and its requests to DynamoDB. X-Ray is free for up to 100,000 traces per month
-  for every AWS account, and it's useful to see where each request is spending
-  time. However, you can turn it off by passing `XRayTracingEnabled=false` to
-  the deployment script.
+  and its AWS SDK requests. Every AWS account can collect up to 100,000 traces
+  per month for free, which is useful to see where requests are spending time.
+  However, you can turn this off by passing `XRayTracingEnabled=false` to the
+  deployment script.
 - My co-workers and I collectively make a little over 500 requests to the
   randomizer per month, and at that small of a volume it's essentially free to
   run on AWS even without the 12 month free tier. My _rough_ estimate is that

cmd/randomizer-lambda/main.go

@@ -18,14 +18,25 @@ import (
 	"log/slog"
 	"os"
 
+	"github.com/aws-observability/aws-otel-go/exporters/xrayudp"
 	"github.com/aws/aws-lambda-go/lambda"
 	"github.com/awslabs/aws-lambda-go-api-proxy/httpadapter"
+	lambdadetector "go.opentelemetry.io/contrib/detectors/aws/lambda"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig"
+	xraypropagator "go.opentelemetry.io/contrib/propagators/aws/xray"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.37.0"
 
 	"github.com/featherbread/randomizer/internal/slack"
 	"github.com/featherbread/randomizer/internal/store/dynamodb"
 )
 
 func main() {
+	ctx := context.Background()
 	logger := slog.New(slog.NewJSONHandler(os.Stderr, nil))
 
 	tokenProvider, err := slack.TokenProviderFromEnv()
@@ -34,16 +45,76 @@ func main() {
 		os.Exit(2)
 	}
 
-	storeFactory, err := dynamodb.FactoryFromEnv(context.Background())
+	storeFactory, err := dynamodb.FactoryFromEnv(ctx)
 	if err != nil {
 		logger.Error("Failed to create DynamoDB store", "err", err)
 		os.Exit(2)
 	}
 
+	// OpenTelemetry is always active, but traces are only exported to AWS X-Ray
+	// (and charged for usage) if enabled in the environment.
+	tp := initTracerProvider(ctx, logger)
+	otel.SetTracerProvider(tp)
+	if xrayTracingEnabled {
+		otel.SetTextMapPropagator(xraypropagator.Propagator{})
+	}
+	defer func() {
+		err := tp.Shutdown(ctx)
+		if err != nil {
+			logger.Warn("Failed to shut down tracer provider", "err", err)
+		}
+	}()
+
 	app := slack.App{
 		TokenProvider: tokenProvider,
 		StoreFactory:  storeFactory,
 		Logger:        logger,
 	}
-	lambda.Start(httpadapter.NewV2(app).ProxyWithContext)
+	appHandler := httpadapter.NewV2(app).ProxyWithContext
+	handler := otellambda.InstrumentHandler(appHandler, xrayconfig.WithRecommendedOptions(tp)...)
+	lambda.Start(handler)
+}
+
+var xrayTracingEnabled = os.Getenv("AWS_XRAY_TRACING_ENABLED") == "1"
+
+func initTracerProvider(ctx context.Context, logger *slog.Logger) *trace.TracerProvider {
+	traceResource := initTraceResource(ctx, logger)
+	tp := trace.NewTracerProvider(trace.WithResource(traceResource))
+
+	if !xrayTracingEnabled {
+		return tp
+	}
+
+	exporter, err := xrayudp.NewSpanExporter(ctx)
+	if err != nil {
+		logger.Warn("Failed to initialize X-Ray span exporter", "err", err)
+		return tp
+	}
+
+	tp.RegisterSpanProcessor(trace.NewSimpleSpanProcessor(exporter))
+	return tp
+}
+
+func initTraceResource(ctx context.Context, logger *slog.Logger) *resource.Resource {
+	baseResource := resource.NewWithAttributes(semconv.SchemaURL, attribute.KeyValue{
+		Key:   semconv.ServiceNameKey,
+		Value: attribute.StringValue(os.Getenv("AWS_LAMBDA_FUNCTION_NAME"))})
+
+	if !xrayTracingEnabled {
+		return baseResource
+	}
+
+	lambdaResource, err := lambdadetector.NewResourceDetector().Detect(ctx)
+	if err != nil {
+		logger.Warn("Skipping Lambda resources in traces", "err", err, "step", "detect")
+		return baseResource
+	}
+
+	mergedResource, err := resource.Merge(lambdaResource, baseResource)
+	if err != nil {
+		logger.Warn("Skipping Lambda resources in traces", "err", err, "step", "merge")
+		return baseResource
+	}
+
+	return mergedResource
 }

go.mod

@@ -4,18 +4,25 @@ go 1.25.0
 
 require (
 	cloud.google.com/go/firestore v1.18.0
-	github.com/aws/aws-lambda-go v1.49.0
-	github.com/aws/aws-sdk-go-v2 v1.38.3
+	github.com/aws-observability/aws-otel-go/exporters/xrayudp v1.0.0
+	github.com/aws/aws-lambda-go v1.51.1
+	github.com/aws/aws-sdk-go-v2 v1.40.1
 	github.com/aws/aws-sdk-go-v2/config v1.31.6
 	github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.8.9
-	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.50.1
+	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.53.3
 	github.com/aws/aws-sdk-go-v2/service/ssm v1.64.2
-	github.com/aws/aws-xray-sdk-go/v2 v2.0.0
 	github.com/awslabs/aws-lambda-go-api-proxy v0.16.2
 	github.com/googleapis/gax-go/v2 v2.15.0
 	github.com/spf13/cobra v1.10.1
 	go.etcd.io/bbolt v1.4.3
-	google.golang.org/grpc v1.75.0
+	go.opentelemetry.io/contrib/detectors/aws/lambda v0.64.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda v0.64.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda/xrayconfig v0.64.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws v0.64.0
+	go.opentelemetry.io/contrib/propagators/aws v1.39.0
+	go.opentelemetry.io/otel v1.39.0
+	go.opentelemetry.io/otel/sdk v1.39.0
+	google.golang.org/grpc v1.78.0
 )
 
 require (
@@ -24,48 +31,53 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	cloud.google.com/go/longrunning v0.6.7 // indirect
-	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.18.10 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.20.9 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.30.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.39.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.42.18 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 // indirect
-	github.com/aws/smithy-go v1.23.0 // indirect
+	github.com/aws/smithy-go v1.24.0 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/fasthttp v1.65.0 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
-	go.opentelemetry.io/otel v1.38.0 // indirect
-	go.opentelemetry.io/otel/metric v1.38.0 // indirect
-	go.opentelemetry.io/otel/trace v1.38.0 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
-	golang.org/x/net v0.47.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.64.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 // indirect
+	go.opentelemetry.io/otel/metric v1.39.0 // indirect
+	go.opentelemetry.io/otel/trace v1.39.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	golang.org/x/crypto v0.46.0 // indirect
+	golang.org/x/net v0.48.0 // indirect
 	golang.org/x/oauth2 v0.33.0 // indirect
-	golang.org/x/sync v0.18.0 // indirect
-	golang.org/x/sys v0.38.0 // indirect
-	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sys v0.39.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	google.golang.org/api v0.248.0 // indirect
 	google.golang.org/genproto v0.0.0-20250826171959-ef028d996bc1 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 // indirect
-	google.golang.org/protobuf v1.36.8 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 )