[Bug]: Monitor tab no longer can query prometheus #8066
Description
What happened?
When I redeployed the jaeger image "latest", the monitor tab no longer gives me information like it did before. In further testing, v2.14.1 worked and v2.15.0 onwards doesn't work.
Steps to reproduce
v2.14.1 container running correctly shows the monitor tab with the data coming from prometheus
v2.15.0 container running doesn't show the monitor tab with data coming from prometheus
Expected behavior
Monitor tab continues to work and show the data that it used to.
Relevant log output
v2.14.1 working logs:
2026/02/17 21:58:20 application version: git-commit=9173e343bfabb4ce8ff04d5b60c4d34058e5744a, git-version=v2.14.1, build-date=2026-01-02T18:34:55Z
2026-02-17T21:58:20.106Z debug builders/builders.go:24 Beta component. May change in the future. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "prometheus", "otelcol.component.kind": "exporter", "otelcol.signal": "metrics"}
2026-02-17T21:58:20.106Z debug builders/builders.go:24 Alpha component. May change in the future. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "spanmetrics", "otelcol.component.kind": "connector", "otelcol.signal": "traces", "otelcol.signal.output": "metrics"}
2026-02-17T21:58:20.106Z info spanmetricsconnector@v0.142.0/connector.go:120 Building spanmetrics connector {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "spanmetrics", "otelcol.component.kind": "connector", "otelcol.signal": "traces", "otelcol.signal.output": "metrics"}
2026-02-17T21:58:20.106Z info builders/builders.go:26 Development component. May change in the future. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage_exporter", "otelcol.component.kind": "exporter", "otelcol.signal": "traces"}
2026-02-17T21:58:20.106Z debug builders/builders.go:24 Stable component. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "otlp", "otelcol.component.kind": "receiver", "otelcol.signal": "traces"}
2026-02-17T21:58:20.106Z debug otlpreceiver@v0.142.0/otlp.go:57 created signal-agnostic logger {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "otlp", "otelcol.component.kind": "receiver"}
2026-02-17T21:58:20.106Z debug builders/extension.go:48 Beta component. May change in the future. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T21:58:20.106Z debug builders/extension.go:48 Beta component. May change in the future. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2026-02-17T21:58:20.108Z info service@v0.142.0/service.go:250 Starting jaeger... {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "Version": "v2.14.1", "NumCPU": 24}
2026-02-17T21:58:20.108Z info extensions/extensions.go:40 Starting extensions... {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}}
2026-02-17T21:58:20.108Z info extensions/extensions.go:44 Extension is starting... {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T21:58:20.108Z info storageconfig/factory.go:39 Initializing storage 'es_cluster_backend' {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T21:58:20.116Z info config/config.go:319 Elasticsearch detected {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "version": 8}
2026-02-17T21:58:20.131Z info jaegerstorage/extension.go:175 Initializing metrics storage 'prometheus_backend' {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T21:58:20.132Z info extensions/extensions.go:61 Extension started. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T21:58:20.132Z info extensions/extensions.go:44 Extension is starting... {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2026-02-17T21:58:20.132Z info grpc@v1.78.0/clientconn.go:214 [core] original dial target is: "localhost:4317" {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:20.132Z info grpc@v1.78.0/clientconn.go:514 [core] [Channel #1] Channel created for target "localhost:4317" {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:20.132Z info grpc@v1.78.0/clientconn.go:245 [core] [Channel #1] parsed dial target is: resolver.Target{URL:url.URL{Scheme:"dns", Opaque:"", User:(*url.Userinfo)(nil), Host:"", Path:"/localhost:4317", RawPath:"", OmitHost:false, ForceQuery:false, RawQuery:"", Fragment:"", RawFragment:""}} {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:20.132Z info grpc@v1.78.0/clientconn.go:246 [core] [Channel #1] Channel authority set to "localhost:4317" {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:20.132Z info jaegerquery/server.go:151 Archive storage not configured {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2026-02-17T21:58:20.132Z info metricstore/reader.go:126 Prometheus reader initialized {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "addr": "http://prometheus.smcron-monitoring.svc.cluster.local:9090"}
2026-02-17T21:58:20.132Z info grpc@v1.78.0/server.go:717 [core] [Server #2] Server created {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:20.132Z info internal/static_handler.go:92 Using UI configuration {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "path": ""}
2026-02-17T21:58:20.133Z info internal/server.go:266 Query server started {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "http_addr": "[::]:16686", "grpc_addr": "[::]:16685"}
2026-02-17T21:58:20.133Z info extensions/extensions.go:61 Extension started. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2026-02-17T21:58:20.133Z info internal/server.go:294 Starting HTTP server {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "port": 16686, "addr": ":16686"}
2026-02-17T21:58:20.133Z info internal/server.go:308 Starting GRPC server {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "port": 16685, "addr": ":16685"}
2026-02-17T21:58:20.133Z info spanmetricsconnector@v0.142.0/connector.go:216 Starting spanmetrics connector {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "spanmetrics", "otelcol.component.kind": "connector", "otelcol.signal": "traces", "otelcol.signal.output": "metrics"}
2026-02-17T21:58:20.133Z info grpc@v1.78.0/server.go:913 [core] [Server #2 ListenSocket #3] ListenSocket created {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:20.133Z info grpc@v1.78.0/server.go:717 [core] [Server #4] Server created {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:20.133Z info otlpreceiver@v0.142.0/otlp.go:120 Starting GRPC server {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "otlp", "otelcol.component.kind": "receiver", "endpoint": "[::]:4317"}
2026-02-17T21:58:20.133Z info otlpreceiver@v0.142.0/otlp.go:178 Starting HTTP server {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "otlp", "otelcol.component.kind": "receiver", "endpoint": "[::]:4318"}
2026-02-17T21:58:20.133Z info service@v0.142.0/service.go:273 Everything is ready. Begin running and processing data. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}}
2026-02-17T21:58:20.133Z info grpc@v1.78.0/server.go:913 [core] [Server #4 ListenSocket #5] ListenSocket created {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
********
2026-02-17T21:58:25.132Z info grpc@v1.78.0/clientconn.go:618 [core] [Channel #1] Channel Connectivity change to CONNECTING {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.132Z info grpc@v1.78.0/clientconn.go:419 [core] [Channel #1] Channel exiting idle mode {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.133Z info grpc@v1.78.0/resolver_wrapper.go:211 [core] [Channel #1] Resolver state updated: {
"Addresses": [
{
"Addr": "127.0.0.1:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
},
{
"Addr": "[::1]:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Endpoints": [
{
"Addresses": [
{
"Addr": "127.0.0.1:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Attributes": null
},
{
"Addresses": [
{
"Addr": "[::1]:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Attributes": null
}
],
"ServiceConfig": null,
"Attributes": null
} (resolver returned new addresses) {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.133Z info grpc@v1.78.0/balancer_wrapper.go:122 [core] [Channel #1] Channel switches to new LB policy "pick_first" {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.133Z info gracefulswitch/gracefulswitch.go:200 [pick-first-leaf-lb] [pick-first-leaf-lb 0xc0005faea0] Received new config {
"shuffleAddressList": false
}, resolver state {
"Addresses": [
{
"Addr": "127.0.0.1:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
},
{
"Addr": "[::1]:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Endpoints": [
{
"Addresses": [
{
"Addr": "127.0.0.1:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Attributes": null
},
{
"Addresses": [
{
"Addr": "[::1]:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Attributes": null
}
],
"ServiceConfig": null,
"Attributes": null
} {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.133Z info grpc@v1.78.0/balancer_wrapper.go:195 [core] [Channel #1 SubChannel #6] Subchannel created {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.133Z info grpc@v1.78.0/clientconn.go:1296 [core] [Channel #1 SubChannel #6] Subchannel Connectivity change to CONNECTING {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.133Z info grpc@v1.78.0/clientconn.go:1464 [core] [Channel #1 SubChannel #6] Subchannel picks a new address "127.0.0.1:4317" to connect {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.134Z info grpc@v1.78.0/clientconn.go:1296 [core] [Channel #1 SubChannel #6] Subchannel Connectivity change to READY {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.134Z info pickfirst/pickfirst.go:170 [pick-first-leaf-lb] [pick-first-leaf-lb 0xc0005faea0] SubConn 0xc00073f090 reported connectivity state READY and the health listener is disabled. Transitioning SubConn to READY. {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.134Z info grpc@v1.78.0/clientconn.go:618 [core] [Channel #1] Channel Connectivity change to READY {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "grpc_log": true}
2026-02-17T21:58:25.800Z debug prometheusexporter@v0.142.0/collector.go:604 collect called {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "prometheus", "otelcol.component.kind": "exporter", "otelcol.signal": "metrics"}
2026-02-17T21:58:25.800Z debug prometheusexporter@v0.142.0/accumulator.go:376 Accumulator collect called {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "prometheus", "otelcol.component.kind": "exporter", "otelcol.signal": "metrics"}
2026-02-17T21:58:30.800Z debug prometheusexporter@v0.142.0/collector.go:604 collect called {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "prometheus", "otelcol.component.kind": "exporter", "otelcol.signal": "metrics"}
2026-02-17T21:58:30.800Z debug prometheusexporter@v0.142.0/accumulator.go:376 Accumulator collect called {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "prometheus", "otelcol.component.kind": "exporter", "otelcol.signal": "metrics"}
2026-02-17T21:58:30.878Z debug spanstore/reader.go:189 Reading from ES indices {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "index": ["jaeger-service-2026-02-17", "jaeger-service-2026-02-16", "jaeger-service-2026-02-15", "jaeger-service-2026-02-14"]}
2026-02-17T21:58:30.976Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "", "query": "sum(rate(traces_span_metrics_calls_total{service_name =~ \"aws\", status_code = \"STATUS_CODE_ERROR\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name) / sum(rate(traces_span_metrics_calls_total{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name)", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
2026-02-17T21:58:30.976Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "", "query": "sum(rate(traces_span_metrics_calls_total{service_name =~ \"aws\", status_code = \"STATUS_CODE_ERROR\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name,span_name) / sum(rate(traces_span_metrics_calls_total{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name,span_name)", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
2026-02-17T21:58:30.976Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "{service_name=\"aws\", span_name=\"heartbeat\"} =>\n0.3277310924369748 @[1771361910.883]\n0.3277310924369748 @[1771361970.883]\n0.3277310924369748 @[1771362030.883]\n0.32941176470588235 @[1771362090.883]\n0.3277310924369748 @[1771362150.883]\n0.3277310924369748 @[1771362210.883]\n0.3277310924369748 @[1771362270.883]\n0.3277310924369748 @[1771362330.883]\n0.3277310924369748 @[1771362390.883]\n0.3277310924369748 @[1771362450.883]\n0.3277310924369748 @[1771362510.883]\n0.3277310924369748 @[1771362570.883]\n0.3277310924369748 @[1771362630.883]\n0.3277310924369748 @[1771362690.883]\n0.3277310924369748 @[1771362750.883]\n0.3277310924369748 @[1771362810.883]\n0.3277310924369748 @[1771362870.883]\n0.3277310924369748 @[1771362930.883]\n0.3277310924369748 @[1771362990.883]\n0.3277310924369748 @[1771363050.883]\n0.3277310924369748 @[1771363110.883]\n0.3277310924369748 @[1771363170.883]\n0.3277310924369748 @[1771363230.883]\n0.3277310924369748 @[1771363290.883]\n0.3277310924369748 @[1771363350.883]\n0.3277310924369748 @[1771363410.883]\n0.3277310924369748 @[1771363470.883]\n0.3277310924369748 @[1771363530.883]\n0.3277310924369748 @[1771363590.883]\n0.3277310924369748 @[1771363650.883]\n0.3277310924369748 @[1771363710.883]\n0.3277310924369748 @[1771363770.883]\n0.3277310924369748 @[1771363830.883]\n0.3277310924369748 @[1771363890.883]\n0.3277310924369748 @[1771363950.883]\n0.3277310924369748 @[1771364010.883]\n0.3277310924369748 @[1771364070.883]\n0.3135593220338983 @[1771364130.883]\n0.2941176470588235 @[1771364190.883]\n0.2941176470588235 @[1771364250.883]\n0.2941176470588235 @[1771364310.883]\n0.2941176470588235 @[1771364370.883]\n0.2941176470588235 @[1771364430.883]\n0.2941176470588235 @[1771364490.883]\n0.2941176470588235 @[1771364550.883]\n0.2941176470588235 @[1771364610.883]\n0.2941176470588235 @[1771364670.883]\n0.3167126780626781 @[1771364730.883]\n0.3277310924369748 @[1771364790.883]\n0.3277310924369748 @[1771364850.883]\n0.3277310924369748 @[1771364910.883]\n0.3277310924369748 @[1771364970.883]\n0.3277310924369748 @[1771365030.883]\n0.3277310924369748 @[1771365090.883]\n0.3277310924369748 @[1771365150.883]\n0.3277310924369748 @[1771365210.883]\n0.3277310924369748 @[1771365270.883]\n0.3277310924369748 @[1771365330.883]\n0.3277310924369748 @[1771365390.883]\n0.3361344537815126 @[1771365450.883]\n0.32911944444444435 @[1771365510.883]", "query": "sum(rate(traces_span_metrics_calls_total{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name,span_name)", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
2026-02-17T21:58:30.977Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "{service_name=\"aws\"} =>\n0.3277310924369748 @[1771361910.883]\n0.3277310924369748 @[1771361970.883]\n0.3277310924369748 @[1771362030.883]\n0.32941176470588235 @[1771362090.883]\n0.3277310924369748 @[1771362150.883]\n0.3277310924369748 @[1771362210.883]\n0.3277310924369748 @[1771362270.883]\n0.3277310924369748 @[1771362330.883]\n0.3277310924369748 @[1771362390.883]\n0.3277310924369748 @[1771362450.883]\n0.3277310924369748 @[1771362510.883]\n0.3277310924369748 @[1771362570.883]\n0.3277310924369748 @[1771362630.883]\n0.3277310924369748 @[1771362690.883]\n0.3277310924369748 @[1771362750.883]\n0.3277310924369748 @[1771362810.883]\n0.3277310924369748 @[1771362870.883]\n0.3277310924369748 @[1771362930.883]\n0.3277310924369748 @[1771362990.883]\n0.3277310924369748 @[1771363050.883]\n0.3277310924369748 @[1771363110.883]\n0.3277310924369748 @[1771363170.883]\n0.3277310924369748 @[1771363230.883]\n0.3277310924369748 @[1771363290.883]\n0.3277310924369748 @[1771363350.883]\n0.3277310924369748 @[1771363410.883]\n0.3277310924369748 @[1771363470.883]\n0.3277310924369748 @[1771363530.883]\n0.3277310924369748 @[1771363590.883]\n0.3277310924369748 @[1771363650.883]\n0.3277310924369748 @[1771363710.883]\n0.3277310924369748 @[1771363770.883]\n0.3277310924369748 @[1771363830.883]\n0.3277310924369748 @[1771363890.883]\n0.3277310924369748 @[1771363950.883]\n0.3277310924369748 @[1771364010.883]\n0.3277310924369748 @[1771364070.883]\n0.3135593220338983 @[1771364130.883]\n0.2941176470588235 @[1771364190.883]\n0.2941176470588235 @[1771364250.883]\n0.2941176470588235 @[1771364310.883]\n0.2941176470588235 @[1771364370.883]\n0.2941176470588235 @[1771364430.883]\n0.2941176470588235 @[1771364490.883]\n0.2941176470588235 @[1771364550.883]\n0.2941176470588235 @[1771364610.883]\n0.2941176470588235 @[1771364670.883]\n0.3167126780626781 @[1771364730.883]\n0.3277310924369748 @[1771364790.883]\n0.3277310924369748 @[1771364850.883]\n0.3277310924369748 @[1771364910.883]\n0.3277310924369748 @[1771364970.883]\n0.3277310924369748 @[1771365030.883]\n0.3277310924369748 @[1771365090.883]\n0.3277310924369748 @[1771365150.883]\n0.3277310924369748 @[1771365210.883]\n0.3277310924369748 @[1771365270.883]\n0.3277310924369748 @[1771365330.883]\n0.3277310924369748 @[1771365390.883]\n0.3361344537815126 @[1771365450.883]\n0.32911944444444435 @[1771365510.883]", "query": "sum(rate(traces_span_metrics_calls_total{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name)", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
2026-02-17T21:58:30.977Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "{service_name=\"aws\"} =>\n0.3277310924369748 @[1771361910.883]\n0.3277310924369748 @[1771361970.883]\n0.3277310924369748 @[1771362030.883]\n0.32941176470588235 @[1771362090.883]\n0.3277310924369748 @[1771362150.883]\n0.3277310924369748 @[1771362210.883]\n0.3277310924369748 @[1771362270.883]\n0.3277310924369748 @[1771362330.883]\n0.3277310924369748 @[1771362390.883]\n0.3277310924369748 @[1771362450.883]\n0.3277310924369748 @[1771362510.883]\n0.3277310924369748 @[1771362570.883]\n0.3277310924369748 @[1771362630.883]\n0.3277310924369748 @[1771362690.883]\n0.3277310924369748 @[1771362750.883]\n0.3277310924369748 @[1771362810.883]\n0.3277310924369748 @[1771362870.883]\n0.3277310924369748 @[1771362930.883]\n0.3277310924369748 @[1771362990.883]\n0.3277310924369748 @[1771363050.883]\n0.3277310924369748 @[1771363110.883]\n0.3277310924369748 @[1771363170.883]\n0.3277310924369748 @[1771363230.883]\n0.3277310924369748 @[1771363290.883]\n0.3277310924369748 @[1771363350.883]\n0.3277310924369748 @[1771363410.883]\n0.3277310924369748 @[1771363470.883]\n0.3277310924369748 @[1771363530.883]\n0.3277310924369748 @[1771363590.883]\n0.3277310924369748 @[1771363650.883]\n0.3277310924369748 @[1771363710.883]\n0.3277310924369748 @[1771363770.883]\n0.3277310924369748 @[1771363830.883]\n0.3277310924369748 @[1771363890.883]\n0.3277310924369748 @[1771363950.883]\n0.3277310924369748 @[1771364010.883]\n0.3277310924369748 @[1771364070.883]\n0.3135593220338983 @[1771364130.883]\n0.2941176470588235 @[1771364190.883]\n0.2941176470588235 @[1771364250.883]\n0.2941176470588235 @[1771364310.883]\n0.2941176470588235 @[1771364370.883]\n0.2941176470588235 @[1771364430.883]\n0.2941176470588235 @[1771364490.883]\n0.2941176470588235 @[1771364550.883]\n0.2941176470588235 @[1771364610.883]\n0.2941176470588235 @[1771364670.883]\n0.3167126780626781 @[1771364730.883]\n0.3277310924369748 @[1771364790.883]\n0.3277310924369748 @[1771364850.883]\n0.3277310924369748 @[1771364910.883]\n0.3277310924369748 @[1771364970.883]\n0.3277310924369748 @[1771365030.883]\n0.3277310924369748 @[1771365090.883]\n0.3277310924369748 @[1771365150.883]\n0.3277310924369748 @[1771365210.883]\n0.3277310924369748 @[1771365270.883]\n0.3277310924369748 @[1771365330.883]\n0.3277310924369748 @[1771365390.883]\n0.3361344537815126 @[1771365450.883]\n0.32911944444444435 @[1771365510.883]", "query": "sum(rate(traces_span_metrics_calls_total{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name)", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
2026-02-17T21:58:30.977Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "{service_name=\"aws\", span_name=\"heartbeat\"} =>\n0.3277310924369748 @[1771361910.883]\n0.3277310924369748 @[1771361970.883]\n0.3277310924369748 @[1771362030.883]\n0.32941176470588235 @[1771362090.883]\n0.3277310924369748 @[1771362150.883]\n0.3277310924369748 @[1771362210.883]\n0.3277310924369748 @[1771362270.883]\n0.3277310924369748 @[1771362330.883]\n0.3277310924369748 @[1771362390.883]\n0.3277310924369748 @[1771362450.883]\n0.3277310924369748 @[1771362510.883]\n0.3277310924369748 @[1771362570.883]\n0.3277310924369748 @[1771362630.883]\n0.3277310924369748 @[1771362690.883]\n0.3277310924369748 @[1771362750.883]\n0.3277310924369748 @[1771362810.883]\n0.3277310924369748 @[1771362870.883]\n0.3277310924369748 @[1771362930.883]\n0.3277310924369748 @[1771362990.883]\n0.3277310924369748 @[1771363050.883]\n0.3277310924369748 @[1771363110.883]\n0.3277310924369748 @[1771363170.883]\n0.3277310924369748 @[1771363230.883]\n0.3277310924369748 @[1771363290.883]\n0.3277310924369748 @[1771363350.883]\n0.3277310924369748 @[1771363410.883]\n0.3277310924369748 @[1771363470.883]\n0.3277310924369748 @[1771363530.883]\n0.3277310924369748 @[1771363590.883]\n0.3277310924369748 @[1771363650.883]\n0.3277310924369748 @[1771363710.883]\n0.3277310924369748 @[1771363770.883]\n0.3277310924369748 @[1771363830.883]\n0.3277310924369748 @[1771363890.883]\n0.3277310924369748 @[1771363950.883]\n0.3277310924369748 @[1771364010.883]\n0.3277310924369748 @[1771364070.883]\n0.3135593220338983 @[1771364130.883]\n0.2941176470588235 @[1771364190.883]\n0.2941176470588235 @[1771364250.883]\n0.2941176470588235 @[1771364310.883]\n0.2941176470588235 @[1771364370.883]\n0.2941176470588235 @[1771364430.883]\n0.2941176470588235 @[1771364490.883]\n0.2941176470588235 @[1771364550.883]\n0.2941176470588235 @[1771364610.883]\n0.2941176470588235 @[1771364670.883]\n0.3167126780626781 @[1771364730.883]\n0.3277310924369748 @[1771364790.883]\n0.3277310924369748 @[1771364850.883]\n0.3277310924369748 @[1771364910.883]\n0.3277310924369748 @[1771364970.883]\n0.3277310924369748 @[1771365030.883]\n0.3277310924369748 @[1771365090.883]\n0.3277310924369748 @[1771365150.883]\n0.3277310924369748 @[1771365210.883]\n0.3277310924369748 @[1771365270.883]\n0.3277310924369748 @[1771365330.883]\n0.3277310924369748 @[1771365390.883]\n0.3361344537815126 @[1771365450.883]\n0.32911944444444435 @[1771365510.883]", "query": "sum(rate(traces_span_metrics_calls_total{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name,span_name)", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
2026-02-17T21:58:30.979Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "{service_name=\"aws\"} =>\n1.9 @[1771361910.883]\n1.9 @[1771361970.883]\n1.9 @[1771362030.883]\n1.9 @[1771362090.883]\n1.9 @[1771362150.883]\n1.9 @[1771362210.883]\n1.9 @[1771362270.883]\n1.9 @[1771362330.883]\n1.9 @[1771362390.883]\n1.9 @[1771362450.883]\n1.9 @[1771362510.883]\n1.9 @[1771362570.883]\n1.9 @[1771362630.883]\n1.9 @[1771362690.883]\n1.9 @[1771362750.883]\n1.9 @[1771362810.883]\n1.9 @[1771362870.883]\n1.9 @[1771362930.883]\n1.9 @[1771362990.883]\n1.9 @[1771363050.883]\n1.9 @[1771363110.883]\n1.9 @[1771363170.883]\n1.9 @[1771363230.883]\n1.9 @[1771363290.883]\n1.9 @[1771363350.883]\n1.9 @[1771363410.883]\n1.9 @[1771363470.883]\n1.9 @[1771363530.883]\n1.9 @[1771363590.883]\n1.9 @[1771363650.883]\n1.9 @[1771363710.883]\n1.9 @[1771363770.883]\n1.9 @[1771363830.883]\n1.9 @[1771363890.883]\n1.9 @[1771363950.883]\n1.9 @[1771364010.883]\n1.9 @[1771364070.883]\n1.9 @[1771364130.883]\n1.9 @[1771364190.883]\n1.9 @[1771364250.883]\n1.9 @[1771364310.883]\n1.9 @[1771364370.883]\n1.9 @[1771364430.883]\n1.9 @[1771364490.883]\n1.9 @[1771364550.883]\n1.9 @[1771364610.883]\n1.9 @[1771364670.883]\n1.9 @[1771364730.883]\n1.9 @[1771364790.883]\n1.9 @[1771364850.883]\n1.9 @[1771364910.883]\n1.9 @[1771364970.883]\n1.9 @[1771365030.883]\n1.9 @[1771365090.883]\n1.9 @[1771365150.883]\n1.9 @[1771365210.883]\n1.9 @[1771365270.883]\n1.9 @[1771365330.883]\n1.9 @[1771365390.883]\n1.9 @[1771365450.883]\n1.9 @[1771365510.883]", "query": "histogram_quantile(0.95, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name,le))", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
2026-02-17T21:58:30.979Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "{service_name=\"aws\"} =>\n1 @[1771361910.883]\n1 @[1771361970.883]\n1 @[1771362030.883]\n1 @[1771362090.883]\n1 @[1771362150.883]\n1 @[1771362210.883]\n1 @[1771362270.883]\n1 @[1771362330.883]\n1 @[1771362390.883]\n1 @[1771362450.883]\n1 @[1771362510.883]\n1 @[1771362570.883]\n1 @[1771362630.883]\n1 @[1771362690.883]\n1 @[1771362750.883]\n1 @[1771362810.883]\n1 @[1771362870.883]\n1 @[1771362930.883]\n1 @[1771362990.883]\n1 @[1771363050.883]\n1 @[1771363110.883]\n1 @[1771363170.883]\n1 @[1771363230.883]\n1 @[1771363290.883]\n1 @[1771363350.883]\n1 @[1771363410.883]\n1 @[1771363470.883]\n1 @[1771363530.883]\n1 @[1771363590.883]\n1 @[1771363650.883]\n1 @[1771363710.883]\n1 @[1771363770.883]\n1 @[1771363830.883]\n1 @[1771363890.883]\n1 @[1771363950.883]\n1 @[1771364010.883]\n1 @[1771364070.883]\n1 @[1771364130.883]\n1 @[1771364190.883]\n1 @[1771364250.883]\n1 @[1771364310.883]\n1 @[1771364370.883]\n1 @[1771364430.883]\n1 @[1771364490.883]\n1 @[1771364550.883]\n1 @[1771364610.883]\n1 @[1771364670.883]\n1 @[1771364730.883]\n1 @[1771364790.883]\n1 @[1771364850.883]\n1 @[1771364910.883]\n1 @[1771364970.883]\n1 @[1771365030.883]\n1 @[1771365090.883]\n1 @[1771365150.883]\n1 @[1771365210.883]\n1 @[1771365270.883]\n1 @[1771365330.883]\n1 @[1771365390.883]\n1 @[1771365450.883]\n1 @[1771365510.883]", "query": "histogram_quantile(0.50, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name,le))", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
2026-02-17T21:58:30.979Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "{service_name=\"aws\"} =>\n1.5 @[1771361910.883]\n1.5 @[1771361970.883]\n1.5 @[1771362030.883]\n1.5 @[1771362090.883]\n1.5 @[1771362150.883]\n1.5 @[1771362210.883]\n1.5 @[1771362270.883]\n1.5 @[1771362330.883]\n1.5 @[1771362390.883]\n1.5 @[1771362450.883]\n1.5 @[1771362510.883]\n1.5 @[1771362570.883]\n1.5 @[1771362630.883]\n1.5 @[1771362690.883]\n1.5 @[1771362750.883]\n1.5 @[1771362810.883]\n1.5 @[1771362870.883]\n1.5 @[1771362930.883]\n1.5 @[1771362990.883]\n1.5 @[1771363050.883]\n1.5 @[1771363110.883]\n1.5 @[1771363170.883]\n1.5 @[1771363230.883]\n1.5 @[1771363290.883]\n1.5 @[1771363350.883]\n1.5 @[1771363410.883]\n1.5 @[1771363470.883]\n1.5 @[1771363530.883]\n1.5 @[1771363590.883]\n1.5 @[1771363650.883]\n1.5 @[1771363710.883]\n1.5 @[1771363770.883]\n1.5 @[1771363830.883]\n1.5 @[1771363890.883]\n1.5 @[1771363950.883]\n1.5 @[1771364010.883]\n1.5 @[1771364070.883]\n1.5 @[1771364130.883]\n1.5 @[1771364190.883]\n1.5 @[1771364250.883]\n1.5 @[1771364310.883]\n1.5 @[1771364370.883]\n1.5 @[1771364430.883]\n1.5 @[1771364490.883]\n1.5 @[1771364550.883]\n1.5 @[1771364610.883]\n1.5 @[1771364670.883]\n1.5 @[1771364730.883]\n1.5 @[1771364790.883]\n1.5 @[1771364850.883]\n1.5 @[1771364910.883]\n1.5 @[1771364970.883]\n1.5 @[1771365030.883]\n1.5 @[1771365090.883]\n1.5 @[1771365150.883]\n1.5 @[1771365210.883]\n1.5 @[1771365270.883]\n1.5 @[1771365330.883]\n1.5 @[1771365390.883]\n1.5 @[1771365450.883]\n1.5 @[1771365510.883]", "query": "histogram_quantile(0.75, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name,le))", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
2026-02-17T21:58:30.980Z debug metricstore/reader.go:289 Prometheus query results {"resource": {"service.instance.id": "955d7a6a-41cb-46eb-bf58-9b2ca212b7a7", "service.name": "jaeger", "service.version": "v2.14.1"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "results": "{service_name=\"aws\", span_name=\"heartbeat\"} =>\n1.9 @[1771361910.883]\n1.9 @[1771361970.883]\n1.9 @[1771362030.883]\n1.9 @[1771362090.883]\n1.9 @[1771362150.883]\n1.9 @[1771362210.883]\n1.9 @[1771362270.883]\n1.9 @[1771362330.883]\n1.9 @[1771362390.883]\n1.9 @[1771362450.883]\n1.9 @[1771362510.883]\n1.9 @[1771362570.883]\n1.9 @[1771362630.883]\n1.9 @[1771362690.883]\n1.9 @[1771362750.883]\n1.9 @[1771362810.883]\n1.9 @[1771362870.883]\n1.9 @[1771362930.883]\n1.9 @[1771362990.883]\n1.9 @[1771363050.883]\n1.9 @[1771363110.883]\n1.9 @[1771363170.883]\n1.9 @[1771363230.883]\n1.9 @[1771363290.883]\n1.9 @[1771363350.883]\n1.9 @[1771363410.883]\n1.9 @[1771363470.883]\n1.9 @[1771363530.883]\n1.9 @[1771363590.883]\n1.9 @[1771363650.883]\n1.9 @[1771363710.883]\n1.9 @[1771363770.883]\n1.9 @[1771363830.883]\n1.9 @[1771363890.883]\n1.9 @[1771363950.883]\n1.9 @[1771364010.883]\n1.9 @[1771364070.883]\n1.9 @[1771364130.883]\n1.9 @[1771364190.883]\n1.9 @[1771364250.883]\n1.9 @[1771364310.883]\n1.9 @[1771364370.883]\n1.9 @[1771364430.883]\n1.9 @[1771364490.883]\n1.9 @[1771364550.883]\n1.9 @[1771364610.883]\n1.9 @[1771364670.883]\n1.9 @[1771364730.883]\n1.9 @[1771364790.883]\n1.9 @[1771364850.883]\n1.9 @[1771364910.883]\n1.9 @[1771364970.883]\n1.9 @[1771365030.883]\n1.9 @[1771365090.883]\n1.9 @[1771365150.883]\n1.9 @[1771365210.883]\n1.9 @[1771365270.883]\n1.9 @[1771365330.883]\n1.9 @[1771365390.883]\n1.9 @[1771365450.883]\n1.9 @[1771365510.883]", "query": "histogram_quantile(0.95, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name =~ \"aws\", span_kind =~ \"SPAN_KIND_INTERNAL\"}[10m])) by (service_name,span_name,le))", "range": {"Start":"2026-02-17T20:58:30.883Z","End":"2026-02-17T21:58:30.883Z","Step":60000000000}}
-----------------------------------------------------------------------
New 2.15.0 logs where it isn't working:
2026/02/17 22:02:06 application version: git-commit=1f242236a79be6961bd604e18bd6e9b9697ba066, git-version=v2.15.0, build-date=2026-02-06T18:52:45Z
2026-02-17T22:02:06.727Z info builders/builders.go:28 Development component. May change in the future. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_storage_exporter", "otelcol.component.kind": "exporter", "otelcol.signal": "traces"}
2026-02-17T22:02:06.727Z debug builders/builders.go:26 Beta component. May change in the future. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "prometheus", "otelcol.component.kind": "exporter", "otelcol.signal": "metrics"}
2026-02-17T22:02:06.727Z debug builders/builders.go:26 Alpha component. May change in the future. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "spanmetrics", "otelcol.component.kind": "connector", "otelcol.signal": "traces", "otelcol.signal.output": "metrics"}
2026-02-17T22:02:06.727Z info spanmetricsconnector@v0.142.0/connector.go:120 Building spanmetrics connector {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "spanmetrics", "otelcol.component.kind": "connector", "otelcol.signal": "traces", "otelcol.signal.output": "metrics"}
2026-02-17T22:02:06.727Z debug builders/builders.go:26 Stable component. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "otlp", "otelcol.component.kind": "receiver", "otelcol.signal": "traces"}
2026-02-17T22:02:06.727Z debug otlpreceiver@v0.145.0/otlp.go:57 created signal-agnostic logger {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "otlp", "otelcol.component.kind": "receiver"}
2026-02-17T22:02:06.727Z debug builders/builders.go:26 Beta component. May change in the future. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T22:02:06.727Z debug builders/builders.go:26 Beta component. May change in the future. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2026-02-17T22:02:06.729Z info service@v0.145.0/service.go:241 Starting jaeger... {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "Version": "v2.15.0", "NumCPU": 24}
2026-02-17T22:02:06.729Z info extensions/extensions.go:40 Starting extensions... {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}}
2026-02-17T22:02:06.729Z info extensions/extensions.go:44 Extension is starting... {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T22:02:06.729Z info extensions/extensions.go:61 Extension started. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T22:02:06.729Z info extensions/extensions.go:44 Extension is starting... {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2026-02-17T22:02:06.729Z info grpc@v1.78.0/clientconn.go:214 [core] original dial target is: "localhost:4317" {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:06.729Z info grpc@v1.78.0/clientconn.go:514 [core] [Channel #1] Channel created for target "localhost:4317" {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:06.729Z info grpc@v1.78.0/clientconn.go:245 [core] [Channel #1] parsed dial target is: resolver.Target{URL:url.URL{Scheme:"dns", Opaque:"", User:(*url.Userinfo)(nil), Host:"", Path:"/localhost:4317", RawPath:"", OmitHost:false, ForceQuery:false, RawQuery:"", Fragment:"", RawFragment:""}} {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:06.729Z info grpc@v1.78.0/clientconn.go:246 [core] [Channel #1] Channel authority set to "localhost:4317" {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:06.729Z info storageconfig/factory.go:39 Initializing storage 'es_cluster_backend' {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T22:02:06.739Z info config/config.go:319 Elasticsearch detected {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "version": 8}
2026-02-17T22:02:06.756Z info jaegerquery/server.go:146 Archive storage not configured {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2026-02-17T22:02:06.756Z info jaegerstorage/extension.go:241 Initializing metrics storage 'prometheus_backend' {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension"}
2026-02-17T22:02:06.756Z info metricstore/reader.go:126 Prometheus reader initialized {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "addr": "http://prometheus.smcron-monitoring.svc.cluster.local:9090"}
2026-02-17T22:02:06.756Z info grpc@v1.78.0/server.go:717 [core] [Server #2] Server created {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:06.756Z info internal/static_handler.go:92 Using UI configuration {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "path": ""}
2026-02-17T22:02:06.756Z info internal/server.go:259 Query server started {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "http_addr": "[::]:16686", "grpc_addr": "[::]:16685"}
2026-02-17T22:02:06.756Z info extensions/extensions.go:61 Extension started. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension"}
2026-02-17T22:02:06.756Z info internal/server.go:287 Starting HTTP server {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "port": 16686, "addr": ":16686"}
2026-02-17T22:02:06.756Z info internal/server.go:301 Starting GRPC server {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_query", "otelcol.component.kind": "extension", "port": 16685, "addr": ":16685"}
2026-02-17T22:02:06.757Z info grpc@v1.78.0/server.go:913 [core] [Server #2 ListenSocket #3] ListenSocket created {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:06.757Z info spanmetricsconnector@v0.142.0/connector.go:216 Starting spanmetrics connector {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "spanmetrics", "otelcol.component.kind": "connector", "otelcol.signal": "traces", "otelcol.signal.output": "metrics"}
2026-02-17T22:02:06.757Z info grpc@v1.78.0/server.go:717 [core] [Server #4] Server created {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:06.757Z info otlpreceiver@v0.145.0/otlp.go:120 Starting GRPC server {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "otlp", "otelcol.component.kind": "receiver", "endpoint": "[::]:4317"}
2026-02-17T22:02:06.757Z info otlpreceiver@v0.145.0/otlp.go:178 Starting HTTP server {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "otlp", "otelcol.component.kind": "receiver", "endpoint": "[::]:4318"}
2026-02-17T22:02:06.757Z info service@v0.145.0/service.go:264 Everything is ready. Begin running and processing data. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}}
2026-02-17T22:02:06.757Z info grpc@v1.78.0/server.go:913 [core] [Server #4 ListenSocket #5] ListenSocket created {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
*************************
2026-02-17T22:02:11.729Z info grpc@v1.78.0/clientconn.go:618 [core] [Channel #1] Channel Connectivity change to CONNECTING {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.730Z info grpc@v1.78.0/clientconn.go:419 [core] [Channel #1] Channel exiting idle mode {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.731Z info grpc@v1.78.0/resolver_wrapper.go:211 [core] [Channel #1] Resolver state updated: {
"Addresses": [
{
"Addr": "127.0.0.1:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
},
{
"Addr": "[::1]:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Endpoints": [
{
"Addresses": [
{
"Addr": "127.0.0.1:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Attributes": null
},
{
"Addresses": [
{
"Addr": "[::1]:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Attributes": null
}
],
"ServiceConfig": null,
"Attributes": null
} (resolver returned new addresses) {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.731Z info grpc@v1.78.0/balancer_wrapper.go:122 [core] [Channel #1] Channel switches to new LB policy "pick_first" {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.731Z info gracefulswitch/gracefulswitch.go:200 [pick-first-leaf-lb] [pick-first-leaf-lb 0xc00016a240] Received new config {
"shuffleAddressList": false
}, resolver state {
"Addresses": [
{
"Addr": "127.0.0.1:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
},
{
"Addr": "[::1]:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Endpoints": [
{
"Addresses": [
{
"Addr": "127.0.0.1:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Attributes": null
},
{
"Addresses": [
{
"Addr": "[::1]:4317",
"ServerName": "",
"Attributes": null,
"BalancerAttributes": null,
"Metadata": null
}
],
"Attributes": null
}
],
"ServiceConfig": null,
"Attributes": null
} {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.731Z info grpc@v1.78.0/balancer_wrapper.go:195 [core] [Channel #1 SubChannel #6] Subchannel created {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.731Z info grpc@v1.78.0/clientconn.go:1296 [core] [Channel #1 SubChannel #6] Subchannel Connectivity change to CONNECTING {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.731Z info grpc@v1.78.0/clientconn.go:1464 [core] [Channel #1 SubChannel #6] Subchannel picks a new address "127.0.0.1:4317" to connect {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.731Z info grpc@v1.78.0/clientconn.go:1296 [core] [Channel #1 SubChannel #6] Subchannel Connectivity change to READY {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.731Z info pickfirst/pickfirst.go:170 [pick-first-leaf-lb] [pick-first-leaf-lb 0xc00016a240] SubConn 0xc0008a2550 reported connectivity state READY and the health listener is disabled. Transitioning SubConn to READY. {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
2026-02-17T22:02:11.731Z info grpc@v1.78.0/clientconn.go:618 [core] [Channel #1] Channel Connectivity change to READY {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "grpc_log": true}
**************************
(The only log line I get when refreshing the monitor page)
2026-02-17T22:07:09.662Z debug spanstore/reader.go:189 Reading from ES indices {"resource": {"service.instance.id": "cb0ccd2e-1003-425d-aef1-a448ec72c7c2", "service.name": "jaeger", "service.version": "v2.15.0"}, "otelcol.component.id": "jaeger_storage", "otelcol.component.kind": "extension", "index": ["jaeger-service-2026-02-17", "jaeger-service-2026-02-16", "jaeger-service-2026-02-15", "jaeger-service-2026-02-14"]}Screenshot
Old version 2.14.1 that works:
new version 2.15.0 that doesn't work:
Additional context
No response
Jaeger backend version
v2.14.1 / v2.15.*
SDK
Python OTEL SDK
Pipeline
OTEL SDK -> Jaeger Collector -> spanmetrics -> prometheus
Stogage backend
Elasticsearch for traces, prometheus for metrics
Operating system
Linux
Deployment model
Kubernetes helm deployments
Deployment configs
{{ if .Values.utils.jaeger }}
apiVersion: v1
kind: ConfigMap
metadata:
name: jaeger-conf
data:
collector.yaml: |
service:
extensions:
- jaeger_storage
- jaeger_query
telemetry:
logs:
level: "debug"
pipelines:
traces:
receivers:
- otlp
exporters:
- jaeger_storage_exporter
- spanmetrics
metrics:
receivers:
- spanmetrics
exporters:
- prometheus
extensions:
jaeger_query:
storage:
metrics: prometheus_backend
traces: es_cluster_backend
jaeger_storage:
backends:
es_cluster_backend:
elasticsearch:
indices:
dependencies:
date_layout: "2006-01-02"
replicas: 0
rollover_frequency: day
shards: 1
index_prefix: ""
sampling:
date_layout: "2006-01-02"
replicas: 0
rollover_frequency: day
shards: 1
services:
date_layout: "2006-01-02"
replicas: 0
rollover_frequency: day
shards: 1
spans:
date_layout: "2006-01-02"
replicas: 0
rollover_frequency: day
shards: 1
server_urls:
- https://elasticsearch:9443
tls:
insecure: true
insecure_skip_verify: true
metric_backends:
prometheus_backend:
prometheus:
endpoint: {{ .Values.infra.jaeger.prometheus_url }}
normalize_calls: true
normalize_duration: true
connectors:
spanmetrics:
dimensions:
- name: http.method
default: UNKNOWN
- name: http.status_code
default: UNKNOWN
- name: http.url
default: UNKNOWN
metrics_flush_interval: 15s
resource_metrics_key_attributes:
- service.name
- telemetry.sdk.language
- telemetry.sdk.name
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
exporters:
jaeger_storage_exporter:
trace_storage: es_cluster_backend
prometheus:
endpoint: "0.0.0.0:8889"
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-jaeger
annotations:
traefik.ingress.kubernetes.io/router.entrypoints: websecure
spec:
rules:
- host: "jaeger-{{ .Release.Namespace }}.{{ .Values.infra.site.private_domain }}"
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: jaeger
port:
number: 16686
- path: /metrics
pathType: Prefix
backend:
service:
name: jaeger
port:
number: 8889
---
apiVersion: v1
kind: Service
metadata:
name: jaeger # dns name that the apps call this service
spec:
selector:
app.kubernetes.io/name: jaeger
{{- if kindIs "float64" .Values.jaeger.nodePort }}
type: NodePort
{{- end }}
ports:
- protocol: TCP
name: jaeger-tcp
port: 16686
{{- if kindIs "float64" .Values.jaeger.nodePort }}
nodePort: {{ .Values.jaeger.nodePort }}
{{- end }}
- protocol: TCP
name: jaeger-grpc-q
port: 16685
- protocol: TCP
name: jaeger-grpc
port: 4317
- protocol: TCP
name: jaeger-http
port: 4318
- name: services-info
port: 8889
protocol: TCP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: jaeger
labels:
app.kubernetes.io/name: jaeger
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: jaeger
template:
metadata:
labels:
app.kubernetes.io/name: jaeger
spec:
volumes:
- configMap:
defaultMode: 420
items:
- key: collector.yaml
path: collector.yaml
name: jaeger-conf
name: otc-internal
containers:
- name: jaeger
image: "{{ .Values.registry.dockerhub }}/{{ .Values.jaeger.repository }}:{{ .Values.jaeger.tag }}"
imagePullPolicy: IfNotPresent
args:
- --config=/conf/collector.yaml
volumeMounts:
- mountPath: /conf
name: otc-internal
livenessProbe:
failureThreshold: 5
httpGet:
path: /search
port: 16686
scheme: HTTP
initialDelaySeconds: 5
periodSeconds: 15
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /search
port: 16686
scheme: HTTP
initialDelaySeconds: 1
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
ports:
- containerPort: 16686
name: jaeger
protocol: TCP
- containerPort: 8889
name: services-info
protocol: TCP
- containerPort: 4317
name: otlp-grpc
protocol: TCP
- containerPort: 4318
name: otlp-http
protocol: TCP
env:
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
resources:
limits:
cpu: "1"
memory: "1G"
{{- end }}