From cea40878355cc0445cfd98a1d245ca2dcf1ca0b4 Mon Sep 17 00:00:00 2001 From: Gyan Ranjan Panda Date: Mon, 4 May 2026 01:04:58 +0530 Subject: [PATCH 1/3] feat(nethttp): add httptrace join-points for otelhttptrace support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add compile-time instrumentation support for net/http/httptrace to automatically create sub-spans for HTTP client request lifecycle phases. This creates a new pkg/instrumentation/nethttp/httptrace package that implements a clientTracer producing sub-spans for: - DNS resolution (http.dns) - TCP connection (http.connect) - TLS handshake (http.tls) - Connection acquisition (http.getconn) - Request send (http.send) - Response receive (http.receive) The existing BeforeRoundTrip hook in the client package is enhanced to attach an httptrace.ClientTrace to the request context after creating the parent client span. No new YAML rules are needed — the httptrace functionality piggybacks on the existing RoundTrip join-point. Includes 14 unit tests covering all lifecycle phases, error scenarios, connection reuse attributes, and full request lifecycle verification. Closes #201 Assisted-by: Gemini Antigravity --- .../nethttp/client/client_hook.go | 11 +- pkg/instrumentation/nethttp/client/go.mod | 9 +- pkg/instrumentation/nethttp/client/go.sum | 16 +- pkg/instrumentation/nethttp/httptrace/go.mod | 23 + pkg/instrumentation/nethttp/httptrace/go.sum | 44 ++ .../nethttp/httptrace/httptrace_hook.go | 231 +++++++++ .../nethttp/httptrace/httptrace_hook_test.go | 438 ++++++++++++++++++ pkg/instrumentation/nethttp/server/go.mod | 9 +- pkg/instrumentation/nethttp/server/go.sum | 16 +- 9 files changed, 774 insertions(+), 23 deletions(-) create mode 100644 pkg/instrumentation/nethttp/httptrace/go.mod create mode 100644 pkg/instrumentation/nethttp/httptrace/go.sum create mode 100644 pkg/instrumentation/nethttp/httptrace/httptrace_hook.go create mode 100644 pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go diff --git a/pkg/instrumentation/nethttp/client/client_hook.go b/pkg/instrumentation/nethttp/client/client_hook.go index d4c9b1743..fbc968ea7 100644 --- a/pkg/instrumentation/nethttp/client/client_hook.go +++ b/pkg/instrumentation/nethttp/client/client_hook.go @@ -5,6 +5,7 @@ package client import ( "net/http" + nethttptrace "net/http/httptrace" "runtime/debug" "strings" "sync" @@ -16,6 +17,7 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/inst" + "github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/nethttp/httptrace" "github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/nethttp/semconv" "github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/shared" ) @@ -118,7 +120,14 @@ func BeforeRoundTrip(ictx inst.HookContext, transport *http.Transport, req *http // Inject trace context into request headers propagator.Inject(ctx, propagation.HeaderCarrier(req.Header)) - // Update request with new context + // Attach httptrace.ClientTrace for detailed sub-span instrumentation + // (DNS, connect, TLS, send, receive phases) + tp := otel.GetTracerProvider() + version := moduleVersion() + clientTrace := httptrace.NewClientTrace(ctx, tp, version) + ctx = nethttptrace.WithClientTrace(ctx, clientTrace) + + // Update request with new context (includes both span and httptrace) newReq := req.WithContext(ctx) ictx.SetParam(requestParamIndex, newReq) diff --git a/pkg/instrumentation/nethttp/client/go.mod b/pkg/instrumentation/nethttp/client/go.mod index 8e12094ef..3df32b19f 100644 --- a/pkg/instrumentation/nethttp/client/go.mod +++ b/pkg/instrumentation/nethttp/client/go.mod @@ -6,8 +6,11 @@ replace github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg = replace github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/shared => ../../shared +replace github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/nethttp/httptrace => ../httptrace + require ( github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg v0.0.0-20251208011108-ac0fa4a155e3 + github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/nethttp/httptrace v0.0.0-00010101000000-000000000000 github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/shared v0.0.0-20251208011108-ac0fa4a155e3 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.43.0 @@ -37,7 +40,7 @@ require ( go.opentelemetry.io/contrib/exporters/autoexport v0.63.0 // indirect go.opentelemetry.io/contrib/instrumentation/runtime v0.64.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect @@ -47,9 +50,9 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 // indirect - go.opentelemetry.io/otel/log v0.14.0 // indirect + go.opentelemetry.io/otel/log v0.19.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk/log v0.14.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.19.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect golang.org/x/net v0.52.0 // indirect diff --git a/pkg/instrumentation/nethttp/client/go.sum b/pkg/instrumentation/nethttp/client/go.sum index ac7216fac..1707e1fb8 100644 --- a/pkg/instrumentation/nethttp/client/go.sum +++ b/pkg/instrumentation/nethttp/client/go.sum @@ -59,8 +59,8 @@ go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 h1:HIBTQ3VO5aupLKjC90JgMqpezVXwFuq6Ryjn0/izoag= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0/go.mod h1:ji9vId85hMxqfvICA0Jt8JqEdrXaAkcpkI9HPXya0ro= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY= @@ -79,16 +79,16 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE= -go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= -go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/log v0.19.0 h1:KUZs/GOsw79TBBMfDWsXS+KZ4g2Ckzksd1ymzsIEbo4= +go.opentelemetry.io/otel/log v0.19.0/go.mod h1:5DQYeGmxVIr4n0/BcJvF4upsraHjg6vudJJpnkL6Ipk= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= -go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/log v0.19.0 h1:scYVLqT22D2gqXItnWiocLUKGH9yvkkeql5dBDiXyko= +go.opentelemetry.io/otel/sdk/log v0.19.0/go.mod h1:vFBowwXGLlW9AvpuF7bMgnNI95LiW10szrOdvzBHlAg= +go.opentelemetry.io/otel/sdk/log/logtest v0.19.0 h1:BEbF7ZBB6qQloV/Ub1+3NQoOUnVtcGkU3XX4Ws3GQfk= +go.opentelemetry.io/otel/sdk/log/logtest v0.19.0/go.mod h1:Lua81/3yM0wOmoHTokLj9y9ADeA02v1naRrVrkAZuKk= go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= diff --git a/pkg/instrumentation/nethttp/httptrace/go.mod b/pkg/instrumentation/nethttp/httptrace/go.mod new file mode 100644 index 000000000..d33660743 --- /dev/null +++ b/pkg/instrumentation/nethttp/httptrace/go.mod @@ -0,0 +1,23 @@ +module github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/nethttp/httptrace + +go 1.25.0 + +require ( + github.com/stretchr/testify v1.11.1 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/trace v1.43.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + golang.org/x/sys v0.42.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/pkg/instrumentation/nethttp/httptrace/go.sum b/pkg/instrumentation/nethttp/httptrace/go.sum new file mode 100644 index 000000000..a6e424670 --- /dev/null +++ b/pkg/instrumentation/nethttp/httptrace/go.sum @@ -0,0 +1,44 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go b/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go new file mode 100644 index 000000000..2d1702942 --- /dev/null +++ b/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go @@ -0,0 +1,231 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package httptrace + +import ( + "context" + "crypto/tls" + "net/http/httptrace" + "strings" + "sync" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +const scopeName = "go.opentelemetry.io/compile-instrumentation/httptrace" + +// Attribute keys for HTTP connection lifecycle events. +var ( + HTTPRemoteAddr = attribute.Key("http.remote") + HTTPLocalAddr = attribute.Key("http.local") + HTTPConnectionReused = attribute.Key("http.conn.reused") + HTTPConnectionWasIdle = attribute.Key("http.conn.wasidle") + HTTPConnectionIdleTime = attribute.Key("http.conn.idletime") + HTTPConnectionStartNetwork = attribute.Key("http.conn.start.network") + HTTPConnectionDoneNetwork = attribute.Key("http.conn.done.network") + HTTPConnectionDoneAddr = attribute.Key("http.conn.done.addr") + HTTPDNSAddrs = attribute.Key("http.dns.addrs") + HTTPHostAttribute = attribute.Key("net.host.name") +) + +// hookParentMap maps child hook names to their logical parent span. +var hookParentMap = map[string]string{ + "http.dns": "http.getconn", + "http.connect": "http.getconn", + "http.tls": "http.getconn", +} + +func parentHook(hook string) string { + if strings.HasPrefix(hook, "http.connect") { + return hookParentMap["http.connect"] + } + return hookParentMap[hook] +} + +// clientTracer tracks active sub-spans for each HTTP request lifecycle phase. +type clientTracer struct { + context.Context + + tr trace.Tracer + activeHooks map[string]context.Context + root trace.Span + mtx sync.Mutex +} + +// NewClientTrace returns an httptrace.ClientTrace that records OpenTelemetry +// sub-spans for DNS resolution, TCP connection, TLS handshake, header writing, +// request send, and response receive phases. +func NewClientTrace(ctx context.Context, tp trace.TracerProvider, version string) *httptrace.ClientTrace { + ct := &clientTracer{ + Context: ctx, + activeHooks: make(map[string]context.Context), + } + + if tp == nil { + if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() { + tp = span.TracerProvider() + } + } + + if tp != nil { + ct.tr = tp.Tracer( + scopeName, + trace.WithInstrumentationVersion(version), + ) + } + + return &httptrace.ClientTrace{ + GetConn: ct.getConn, + GotConn: ct.gotConn, + PutIdleConn: ct.putIdleConn, + GotFirstResponseByte: ct.gotFirstResponseByte, + DNSStart: ct.dnsStart, + DNSDone: ct.dnsDone, + ConnectStart: ct.connectStart, + ConnectDone: ct.connectDone, + TLSHandshakeStart: ct.tlsHandshakeStart, + TLSHandshakeDone: ct.tlsHandshakeDone, + WroteHeaders: ct.wroteHeaders, + WroteRequest: ct.wroteRequest, + } +} + +func (ct *clientTracer) start(hook, spanName string, attrs ...attribute.KeyValue) { + if ct.tr == nil { + return + } + + ct.mtx.Lock() + defer ct.mtx.Unlock() + + if hookCtx, found := ct.activeHooks[hook]; !found { + var sp trace.Span + ct.activeHooks[hook], sp = ct.tr.Start( + ct.getParentContext(hook), + spanName, + trace.WithAttributes(attrs...), + trace.WithSpanKind(trace.SpanKindClient), + ) + if ct.root == nil { + ct.root = sp + } + } else { + // end was called before start finished — attach start attrs, end the span + span := trace.SpanFromContext(hookCtx) + span.SetAttributes(attrs...) + span.End() + delete(ct.activeHooks, hook) + } +} + +func (ct *clientTracer) end(hook string, err error, attrs ...attribute.KeyValue) { + if ct.tr == nil { + return + } + + ct.mtx.Lock() + defer ct.mtx.Unlock() + + if ctx, ok := ct.activeHooks[hook]; ok { + span := trace.SpanFromContext(ctx) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + } + span.SetAttributes(attrs...) + span.End() + delete(ct.activeHooks, hook) + } else { + // start hasn't finished yet — create a span with end attrs + ctx, span := ct.tr.Start( + ct.getParentContext(hook), + hook, + trace.WithAttributes(attrs...), + trace.WithSpanKind(trace.SpanKindClient), + ) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + } + ct.activeHooks[hook] = ctx + } +} + +func (ct *clientTracer) getParentContext(hook string) context.Context { + ctx, ok := ct.activeHooks[parentHook(hook)] + if !ok { + return ct.Context + } + return ctx +} + +func (ct *clientTracer) getConn(host string) { + ct.start("http.getconn", "http.getconn", HTTPHostAttribute.String(host)) +} + +func (ct *clientTracer) gotConn(info httptrace.GotConnInfo) { + attrs := []attribute.KeyValue{ + HTTPRemoteAddr.String(info.Conn.RemoteAddr().String()), + HTTPLocalAddr.String(info.Conn.LocalAddr().String()), + HTTPConnectionReused.Bool(info.Reused), + HTTPConnectionWasIdle.Bool(info.WasIdle), + } + if info.WasIdle { + attrs = append(attrs, HTTPConnectionIdleTime.String(info.IdleTime.String())) + } + ct.end("http.getconn", nil, attrs...) +} + +func (ct *clientTracer) putIdleConn(err error) { + ct.end("http.receive", err) +} + +func (ct *clientTracer) gotFirstResponseByte() { + ct.start("http.receive", "http.receive") +} + +func (ct *clientTracer) dnsStart(info httptrace.DNSStartInfo) { + ct.start("http.dns", "http.dns", HTTPHostAttribute.String(info.Host)) +} + +func (ct *clientTracer) dnsDone(info httptrace.DNSDoneInfo) { + addrs := make([]string, 0, len(info.Addrs)) + for _, netAddr := range info.Addrs { + addrs = append(addrs, netAddr.String()) + } + ct.end("http.dns", info.Err, HTTPDNSAddrs.String(strings.Join(addrs, ","))) +} + +func (ct *clientTracer) connectStart(network, addr string) { + ct.start("http.connect."+addr, "http.connect", + HTTPRemoteAddr.String(addr), + HTTPConnectionStartNetwork.String(network), + ) +} + +func (ct *clientTracer) connectDone(network, addr string, err error) { + ct.end("http.connect."+addr, err, + HTTPConnectionDoneAddr.String(addr), + HTTPConnectionDoneNetwork.String(network), + ) +} + +func (ct *clientTracer) tlsHandshakeStart() { + ct.start("http.tls", "http.tls") +} + +func (ct *clientTracer) tlsHandshakeDone(_ tls.ConnectionState, err error) { + ct.end("http.tls", err) +} + +func (ct *clientTracer) wroteHeaders() { + ct.start("http.send", "http.send") +} + +func (ct *clientTracer) wroteRequest(info httptrace.WroteRequestInfo) { + if info.Err != nil && ct.root != nil { + ct.root.SetStatus(codes.Error, info.Err.Error()) + } + ct.end("http.send", info.Err) +} diff --git a/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go b/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go new file mode 100644 index 000000000..750247aba --- /dev/null +++ b/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go @@ -0,0 +1,438 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package httptrace + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http/httptrace" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +func setupTestTracer(t *testing.T) (*tracetest.SpanRecorder, *sdktrace.TracerProvider) { + t.Helper() + sr := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr)) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + return sr, tp +} + +func TestNewClientTrace_ReturnsValidTrace(t *testing.T) { + _, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + require.NotNil(t, ct) + assert.NotNil(t, ct.GetConn) + assert.NotNil(t, ct.GotConn) + assert.NotNil(t, ct.DNSStart) + assert.NotNil(t, ct.DNSDone) + assert.NotNil(t, ct.ConnectStart) + assert.NotNil(t, ct.ConnectDone) + assert.NotNil(t, ct.TLSHandshakeStart) + assert.NotNil(t, ct.TLSHandshakeDone) + assert.NotNil(t, ct.WroteHeaders) + assert.NotNil(t, ct.WroteRequest) + assert.NotNil(t, ct.GotFirstResponseByte) + assert.NotNil(t, ct.PutIdleConn) +} + +func TestNewClientTrace_NilTracerProvider(t *testing.T) { + ctx := context.Background() + + // Should not panic with nil provider + ct := NewClientTrace(ctx, nil, "test") + require.NotNil(t, ct) + + // Calling hooks should not panic + ct.GetConn("example.com:443") +} + +func TestNewClientTrace_InheritsProviderFromSpan(t *testing.T) { + sr, tp := setupTestTracer(t) + parentCtx, parentSpan := tp.Tracer("test").Start(context.Background(), "parent") + defer parentSpan.End() + + ct := NewClientTrace(parentCtx, nil, "test") + require.NotNil(t, ct) + + ct.GetConn("example.com:443") + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{IP: net.ParseIP("93.184.216.34"), Port: 443}, + localAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 54321}, + }, + }) + + spans := sr.Ended() + require.Len(t, spans, 1) + assert.Equal(t, "http.getconn", spans[0].Name()) +} + +func TestDNSLifecycle(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + + // Start getconn (parent of dns) + ct.GetConn("example.com:443") + + // DNS lifecycle + ct.DNSStart(httptrace.DNSStartInfo{Host: "example.com"}) + ct.DNSDone(httptrace.DNSDoneInfo{ + Addrs: []net.IPAddr{ + {IP: net.ParseIP("93.184.216.34")}, + {IP: net.ParseIP("2606:2800:220:1:248:1893:25c8:1946")}, + }, + }) + + // End getconn + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{IP: net.ParseIP("93.184.216.34"), Port: 443}, + localAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 54321}, + }, + }) + + spans := sr.Ended() + require.Len(t, spans, 2) + + // DNS should end first (child) + dnsSpan := spans[0] + assert.Equal(t, "http.dns", dnsSpan.Name()) + + // Verify DNS start attributes + hasHostAttr := false + for _, attr := range dnsSpan.Attributes() { + if attr.Key == HTTPHostAttribute { + hasHostAttr = true + assert.Equal(t, "example.com", attr.Value.AsString()) + } + } + assert.True(t, hasHostAttr, "dns span should have net.host.name attribute") + + // GetConn should end second (parent) + getconnSpan := spans[1] + assert.Equal(t, "http.getconn", getconnSpan.Name()) +} + +func TestDNSFailure(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + ct.GetConn("nonexistent.example.com:443") + + ct.DNSStart(httptrace.DNSStartInfo{Host: "nonexistent.example.com"}) + dnsErr := errors.New("no such host") + ct.DNSDone(httptrace.DNSDoneInfo{Err: dnsErr}) + + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{}, + localAddr: &net.TCPAddr{}, + }, + }) + + spans := sr.Ended() + require.Len(t, spans, 2) + + dnsSpan := spans[0] + assert.Equal(t, "http.dns", dnsSpan.Name()) + assert.Equal(t, "no such host", dnsSpan.Status().Description) +} + +func TestConnectLifecycle(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + ct.GetConn("example.com:443") + + ct.ConnectStart("tcp", "93.184.216.34:443") + ct.ConnectDone("tcp", "93.184.216.34:443", nil) + + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{IP: net.ParseIP("93.184.216.34"), Port: 443}, + localAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 54321}, + }, + }) + + spans := sr.Ended() + require.Len(t, spans, 2) + + connectSpan := spans[0] + assert.Equal(t, "http.connect", connectSpan.Name()) + + hasRemoteAddr := false + for _, attr := range connectSpan.Attributes() { + if attr.Key == HTTPRemoteAddr { + hasRemoteAddr = true + } + } + assert.True(t, hasRemoteAddr, "connect span should have http.remote attribute") +} + +func TestConnectFailure(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + ct.GetConn("example.com:443") + + ct.ConnectStart("tcp", "93.184.216.34:443") + connErr := errors.New("connection refused") + ct.ConnectDone("tcp", "93.184.216.34:443", connErr) + + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{}, + localAddr: &net.TCPAddr{}, + }, + }) + + spans := sr.Ended() + require.Len(t, spans, 2) + + connectSpan := spans[0] + assert.Equal(t, "http.connect", connectSpan.Name()) + assert.Equal(t, "connection refused", connectSpan.Status().Description) +} + +func TestTLSLifecycle(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + ct.GetConn("example.com:443") + + ct.TLSHandshakeStart() + ct.TLSHandshakeDone(tls.ConnectionState{}, nil) + + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{IP: net.ParseIP("93.184.216.34"), Port: 443}, + localAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 54321}, + }, + }) + + spans := sr.Ended() + require.Len(t, spans, 2) + + tlsSpan := spans[0] + assert.Equal(t, "http.tls", tlsSpan.Name()) +} + +func TestTLSFailure(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + ct.GetConn("example.com:443") + + ct.TLSHandshakeStart() + tlsErr := errors.New("certificate verify failed") + ct.TLSHandshakeDone(tls.ConnectionState{}, tlsErr) + + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{}, + localAddr: &net.TCPAddr{}, + }, + }) + + spans := sr.Ended() + require.Len(t, spans, 2) + + tlsSpan := spans[0] + assert.Equal(t, "http.tls", tlsSpan.Name()) + assert.Equal(t, "certificate verify failed", tlsSpan.Status().Description) +} + +func TestSendReceiveLifecycle(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + + // Send phase + ct.WroteHeaders() + ct.WroteRequest(httptrace.WroteRequestInfo{}) + + // Receive phase + ct.GotFirstResponseByte() + ct.PutIdleConn(nil) + + spans := sr.Ended() + require.Len(t, spans, 2) + + sendSpan := spans[0] + assert.Equal(t, "http.send", sendSpan.Name()) + + receiveSpan := spans[1] + assert.Equal(t, "http.receive", receiveSpan.Name()) +} + +func TestWroteRequestError(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + + ct.WroteHeaders() + writeErr := errors.New("broken pipe") + ct.WroteRequest(httptrace.WroteRequestInfo{Err: writeErr}) + + spans := sr.Ended() + require.Len(t, spans, 1) + + sendSpan := spans[0] + assert.Equal(t, "http.send", sendSpan.Name()) + assert.Equal(t, "broken pipe", sendSpan.Status().Description) +} + +func TestConnectionReusedAttributes(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + ct.GetConn("example.com:443") + + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{IP: net.ParseIP("93.184.216.34"), Port: 443}, + localAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 54321}, + }, + Reused: true, + WasIdle: true, + IdleTime: 5 * time.Second, + }) + + spans := sr.Ended() + require.Len(t, spans, 1) + + getconnSpan := spans[0] + assert.Equal(t, "http.getconn", getconnSpan.Name()) + + var foundReused, foundWasIdle, foundIdleTime bool + for _, attr := range getconnSpan.Attributes() { + switch attr.Key { + case HTTPConnectionReused: + foundReused = true + assert.True(t, attr.Value.AsBool()) + case HTTPConnectionWasIdle: + foundWasIdle = true + assert.True(t, attr.Value.AsBool()) + case HTTPConnectionIdleTime: + foundIdleTime = true + assert.Equal(t, "5s", attr.Value.AsString()) + } + } + assert.True(t, foundReused, "should have http.conn.reused attribute") + assert.True(t, foundWasIdle, "should have http.conn.wasidle attribute") + assert.True(t, foundIdleTime, "should have http.conn.idletime attribute") +} + +func TestFullRequestLifecycle(t *testing.T) { + sr, tp := setupTestTracer(t) + + parentCtx, parentSpan := tp.Tracer("test").Start(context.Background(), "HTTP GET") + defer parentSpan.End() + + ct := NewClientTrace(parentCtx, tp, "test") + + // 1. GetConn + ct.GetConn("example.com:443") + + // 2. DNS + ct.DNSStart(httptrace.DNSStartInfo{Host: "example.com"}) + ct.DNSDone(httptrace.DNSDoneInfo{ + Addrs: []net.IPAddr{{IP: net.ParseIP("93.184.216.34")}}, + }) + + // 3. Connect + ct.ConnectStart("tcp", "93.184.216.34:443") + ct.ConnectDone("tcp", "93.184.216.34:443", nil) + + // 4. TLS + ct.TLSHandshakeStart() + ct.TLSHandshakeDone(tls.ConnectionState{}, nil) + + // 5. GotConn + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{IP: net.ParseIP("93.184.216.34"), Port: 443}, + localAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 54321}, + }, + }) + + // 6. Send + ct.WroteHeaders() + ct.WroteRequest(httptrace.WroteRequestInfo{}) + + // 7. Receive + ct.GotFirstResponseByte() + ct.PutIdleConn(nil) + + spans := sr.Ended() + // dns, connect, tls, getconn, send, receive = 6 sub-spans + require.Len(t, spans, 6) + + spanNames := make([]string, len(spans)) + for i, s := range spans { + spanNames[i] = s.Name() + } + + assert.Contains(t, spanNames, "http.dns") + assert.Contains(t, spanNames, "http.connect") + assert.Contains(t, spanNames, "http.tls") + assert.Contains(t, spanNames, "http.getconn") + assert.Contains(t, spanNames, "http.send") + assert.Contains(t, spanNames, "http.receive") +} + +func TestParentHook(t *testing.T) { + tests := []struct { + hook string + expected string + }{ + {"http.dns", "http.getconn"}, + {"http.connect.93.184.216.34:443", "http.getconn"}, + {"http.tls", "http.getconn"}, + {"http.send", ""}, + {"http.receive", ""}, + } + + for _, tt := range tests { + t.Run(tt.hook, func(t *testing.T) { + result := parentHook(tt.hook) + assert.Equal(t, tt.expected, result) + }) + } +} + +// mockConn implements net.Conn for testing GotConnInfo. +type mockConn struct { + remoteAddr net.Addr + localAddr net.Addr +} + +func (c *mockConn) Read(_ []byte) (int, error) { return 0, nil } +func (c *mockConn) Write(_ []byte) (int, error) { return 0, nil } +func (c *mockConn) Close() error { return nil } +func (c *mockConn) LocalAddr() net.Addr { return c.localAddr } +func (c *mockConn) RemoteAddr() net.Addr { return c.remoteAddr } +func (c *mockConn) SetDeadline(_ time.Time) error { return nil } +func (c *mockConn) SetReadDeadline(_ time.Time) error { return nil } +func (c *mockConn) SetWriteDeadline(_ time.Time) error { return nil } diff --git a/pkg/instrumentation/nethttp/server/go.mod b/pkg/instrumentation/nethttp/server/go.mod index 572a3089f..8675c3c5e 100644 --- a/pkg/instrumentation/nethttp/server/go.mod +++ b/pkg/instrumentation/nethttp/server/go.mod @@ -8,6 +8,8 @@ replace github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/i replace github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/nethttp/client => ../client +replace github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/nethttp/httptrace => ../httptrace + require ( github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg v0.0.0-20251208011108-ac0fa4a155e3 github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/nethttp/client v0.0.0-20251208011108-ac0fa4a155e3 @@ -29,6 +31,7 @@ require ( github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/open-telemetry/opentelemetry-go-compile-instrumentation/pkg/instrumentation/nethttp/httptrace v0.0.0-00010101000000-000000000000 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.23.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -40,7 +43,7 @@ require ( go.opentelemetry.io/contrib/exporters/autoexport v0.63.0 // indirect go.opentelemetry.io/contrib/instrumentation/runtime v0.64.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect @@ -50,9 +53,9 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 // indirect - go.opentelemetry.io/otel/log v0.14.0 // indirect + go.opentelemetry.io/otel/log v0.19.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk/log v0.14.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.19.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect golang.org/x/net v0.52.0 // indirect diff --git a/pkg/instrumentation/nethttp/server/go.sum b/pkg/instrumentation/nethttp/server/go.sum index ac7216fac..1707e1fb8 100644 --- a/pkg/instrumentation/nethttp/server/go.sum +++ b/pkg/instrumentation/nethttp/server/go.sum @@ -59,8 +59,8 @@ go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0 h1:HIBTQ3VO5aupLKjC90JgMqpezVXwFuq6Ryjn0/izoag= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.19.0/go.mod h1:ji9vId85hMxqfvICA0Jt8JqEdrXaAkcpkI9HPXya0ro= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY= @@ -79,16 +79,16 @@ go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE= -go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= -go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= +go.opentelemetry.io/otel/log v0.19.0 h1:KUZs/GOsw79TBBMfDWsXS+KZ4g2Ckzksd1ymzsIEbo4= +go.opentelemetry.io/otel/log v0.19.0/go.mod h1:5DQYeGmxVIr4n0/BcJvF4upsraHjg6vudJJpnkL6Ipk= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= -go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= +go.opentelemetry.io/otel/sdk/log v0.19.0 h1:scYVLqT22D2gqXItnWiocLUKGH9yvkkeql5dBDiXyko= +go.opentelemetry.io/otel/sdk/log v0.19.0/go.mod h1:vFBowwXGLlW9AvpuF7bMgnNI95LiW10szrOdvzBHlAg= +go.opentelemetry.io/otel/sdk/log/logtest v0.19.0 h1:BEbF7ZBB6qQloV/Ub1+3NQoOUnVtcGkU3XX4Ws3GQfk= +go.opentelemetry.io/otel/sdk/log/logtest v0.19.0/go.mod h1:Lua81/3yM0wOmoHTokLj9y9ADeA02v1naRrVrkAZuKk= go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= From e77b8b42222f0621172b73dca5b4b3986c0a108e Mon Sep 17 00:00:00 2001 From: Gyan Ranjan Panda Date: Mon, 4 May 2026 01:16:32 +0530 Subject: [PATCH 2/3] fix(httptrace): address codex review feedback 1. Remove http.receive span entirely because PutIdleConn is not reliably invoked (e.g. for HTTP/2 or when keep-alives are disabled), preventing hanging spans. 2. Strip port from hostPort before setting net.host.name on getconn span. --- .../nethttp/httptrace/httptrace_hook.go | 17 ++++------ .../nethttp/httptrace/httptrace_hook_test.go | 34 ++----------------- 2 files changed, 8 insertions(+), 43 deletions(-) diff --git a/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go b/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go index 2d1702942..2641b4a57 100644 --- a/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go +++ b/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go @@ -6,6 +6,7 @@ package httptrace import ( "context" "crypto/tls" + "net" "net/http/httptrace" "strings" "sync" @@ -80,8 +81,6 @@ func NewClientTrace(ctx context.Context, tp trace.TracerProvider, version string return &httptrace.ClientTrace{ GetConn: ct.getConn, GotConn: ct.gotConn, - PutIdleConn: ct.putIdleConn, - GotFirstResponseByte: ct.gotFirstResponseByte, DNSStart: ct.dnsStart, DNSDone: ct.dnsDone, ConnectStart: ct.connectStart, @@ -160,7 +159,11 @@ func (ct *clientTracer) getParentContext(hook string) context.Context { return ctx } -func (ct *clientTracer) getConn(host string) { +func (ct *clientTracer) getConn(hostPort string) { + host := hostPort + if h, _, err := net.SplitHostPort(hostPort); err == nil { + host = h + } ct.start("http.getconn", "http.getconn", HTTPHostAttribute.String(host)) } @@ -177,14 +180,6 @@ func (ct *clientTracer) gotConn(info httptrace.GotConnInfo) { ct.end("http.getconn", nil, attrs...) } -func (ct *clientTracer) putIdleConn(err error) { - ct.end("http.receive", err) -} - -func (ct *clientTracer) gotFirstResponseByte() { - ct.start("http.receive", "http.receive") -} - func (ct *clientTracer) dnsStart(info httptrace.DNSStartInfo) { ct.start("http.dns", "http.dns", HTTPHostAttribute.String(info.Host)) } diff --git a/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go b/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go index 750247aba..8fdff1e95 100644 --- a/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go +++ b/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go @@ -42,8 +42,6 @@ func TestNewClientTrace_ReturnsValidTrace(t *testing.T) { assert.NotNil(t, ct.TLSHandshakeDone) assert.NotNil(t, ct.WroteHeaders) assert.NotNil(t, ct.WroteRequest) - assert.NotNil(t, ct.GotFirstResponseByte) - assert.NotNil(t, ct.PutIdleConn) } func TestNewClientTrace_NilTracerProvider(t *testing.T) { @@ -260,29 +258,7 @@ func TestTLSFailure(t *testing.T) { assert.Equal(t, "certificate verify failed", tlsSpan.Status().Description) } -func TestSendReceiveLifecycle(t *testing.T) { - sr, tp := setupTestTracer(t) - ctx := context.Background() - - ct := NewClientTrace(ctx, tp, "test") - - // Send phase - ct.WroteHeaders() - ct.WroteRequest(httptrace.WroteRequestInfo{}) - - // Receive phase - ct.GotFirstResponseByte() - ct.PutIdleConn(nil) - spans := sr.Ended() - require.Len(t, spans, 2) - - sendSpan := spans[0] - assert.Equal(t, "http.send", sendSpan.Name()) - - receiveSpan := spans[1] - assert.Equal(t, "http.receive", receiveSpan.Name()) -} func TestWroteRequestError(t *testing.T) { sr, tp := setupTestTracer(t) @@ -381,13 +357,9 @@ func TestFullRequestLifecycle(t *testing.T) { ct.WroteHeaders() ct.WroteRequest(httptrace.WroteRequestInfo{}) - // 7. Receive - ct.GotFirstResponseByte() - ct.PutIdleConn(nil) - spans := sr.Ended() - // dns, connect, tls, getconn, send, receive = 6 sub-spans - require.Len(t, spans, 6) + // dns, connect, tls, getconn, send = 5 sub-spans + require.Len(t, spans, 5) spanNames := make([]string, len(spans)) for i, s := range spans { @@ -399,7 +371,6 @@ func TestFullRequestLifecycle(t *testing.T) { assert.Contains(t, spanNames, "http.tls") assert.Contains(t, spanNames, "http.getconn") assert.Contains(t, spanNames, "http.send") - assert.Contains(t, spanNames, "http.receive") } func TestParentHook(t *testing.T) { @@ -411,7 +382,6 @@ func TestParentHook(t *testing.T) { {"http.connect.93.184.216.34:443", "http.getconn"}, {"http.tls", "http.getconn"}, {"http.send", ""}, - {"http.receive", ""}, } for _, tt := range tests { From f5fd8b3fcb138f4c220fcf6ed73755e36e8c1ba5 Mon Sep 17 00:00:00 2001 From: Gyan Ranjan Panda Date: Wed, 6 May 2026 18:45:25 +0530 Subject: [PATCH 3/3] fix ci checks properly --- .../nethttp/httptrace/httptrace_hook.go | 6 ++ .../nethttp/httptrace/httptrace_hook_test.go | 99 ++++++++++++++++++- tool/internal/setup/sync.go | 78 ++++++++++----- tool/internal/setup/sync_test.go | 62 ++++++++++++ 4 files changed, 219 insertions(+), 26 deletions(-) diff --git a/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go b/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go index 2641b4a57..fb4265674 100644 --- a/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go +++ b/pkg/instrumentation/nethttp/httptrace/httptrace_hook.go @@ -89,6 +89,7 @@ func NewClientTrace(ctx context.Context, tp trace.TracerProvider, version string TLSHandshakeDone: ct.tlsHandshakeDone, WroteHeaders: ct.wroteHeaders, WroteRequest: ct.wroteRequest, + GotFirstResponseByte: ct.gotFirstResponseByte, } } @@ -224,3 +225,8 @@ func (ct *clientTracer) wroteRequest(info httptrace.WroteRequestInfo) { } ct.end("http.send", info.Err) } + +func (ct *clientTracer) gotFirstResponseByte() { + ct.start("http.receive", "http.receive") + ct.end("http.receive", nil) +} diff --git a/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go b/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go index 8fdff1e95..ed662fa50 100644 --- a/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go +++ b/pkg/instrumentation/nethttp/httptrace/httptrace_hook_test.go @@ -42,6 +42,7 @@ func TestNewClientTrace_ReturnsValidTrace(t *testing.T) { assert.NotNil(t, ct.TLSHandshakeDone) assert.NotNil(t, ct.WroteHeaders) assert.NotNil(t, ct.WroteRequest) + assert.NotNil(t, ct.GotFirstResponseByte) } func TestNewClientTrace_NilTracerProvider(t *testing.T) { @@ -53,6 +54,7 @@ func TestNewClientTrace_NilTracerProvider(t *testing.T) { // Calling hooks should not panic ct.GetConn("example.com:443") + ct.GotFirstResponseByte() } func TestNewClientTrace_InheritsProviderFromSpan(t *testing.T) { @@ -258,8 +260,6 @@ func TestTLSFailure(t *testing.T) { assert.Equal(t, "certificate verify failed", tlsSpan.Status().Description) } - - func TestWroteRequestError(t *testing.T) { sr, tp := setupTestTracer(t) ctx := context.Background() @@ -278,6 +278,92 @@ func TestWroteRequestError(t *testing.T) { assert.Equal(t, "broken pipe", sendSpan.Status().Description) } +func TestWroteRequestErrorMarksRootSpanWhenStarted(t *testing.T) { + sr, tp := setupTestTracer(t) + ctx := context.Background() + + ct := NewClientTrace(ctx, tp, "test") + ct.GetConn("example.com:443") + ct.WroteHeaders() + + writeErr := errors.New("broken pipe") + ct.WroteRequest(httptrace.WroteRequestInfo{Err: writeErr}) + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{IP: net.ParseIP("93.184.216.34"), Port: 443}, + localAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 54321}, + }, + }) + + spans := sr.Ended() + require.Len(t, spans, 2) + assert.Equal(t, "http.send", spans[0].Name()) + assert.Equal(t, "http.getconn", spans[1].Name()) + assert.Equal(t, "broken pipe", spans[1].Status().Description) +} + +func TestGetConnHostAttribute(t *testing.T) { + tests := []struct { + name string + hostPort string + wantHost string + }{ + {name: "host and port", hostPort: "example.com:443", wantHost: "example.com"}, + {name: "host without port", hostPort: "example.com", wantHost: "example.com"}, + {name: "ipv6 host and port", hostPort: "[2001:db8::1]:443", wantHost: "2001:db8::1"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sr, tp := setupTestTracer(t) + ct := NewClientTrace(context.Background(), tp, "test") + + ct.GetConn(tt.hostPort) + ct.GotConn(httptrace.GotConnInfo{ + Conn: &mockConn{ + remoteAddr: &net.TCPAddr{IP: net.ParseIP("93.184.216.34"), Port: 443}, + localAddr: &net.TCPAddr{IP: net.ParseIP("192.168.1.1"), Port: 54321}, + }, + }) + + spans := sr.Ended() + require.Len(t, spans, 1) + assert.Contains(t, spans[0].Attributes(), HTTPHostAttribute.String(tt.wantHost)) + }) + } +} + +func TestGotFirstResponseByteEndsReceiveSpan(t *testing.T) { + sr, tp := setupTestTracer(t) + ct := NewClientTrace(context.Background(), tp, "test") + + ct.GotFirstResponseByte() + + spans := sr.Ended() + require.Len(t, spans, 1) + assert.Equal(t, "http.receive", spans[0].Name()) +} + +func TestEndBeforeStartCompletesSpanOnStart(t *testing.T) { + sr, tp := setupTestTracer(t) + ct := &clientTracer{ + Context: context.Background(), + tr: tp.Tracer("test"), + activeHooks: make(map[string]context.Context), + } + + lateErr := errors.New("late failure") + ct.end("custom", lateErr, HTTPConnectionDoneNetwork.String("tcp")) + ct.start("custom", "custom", HTTPConnectionStartNetwork.String("tcp")) + + spans := sr.Ended() + require.Len(t, spans, 1) + assert.Equal(t, "custom", spans[0].Name()) + assert.Equal(t, "late failure", spans[0].Status().Description) + assert.Contains(t, spans[0].Attributes(), HTTPConnectionStartNetwork.String("tcp")) + assert.Contains(t, spans[0].Attributes(), HTTPConnectionDoneNetwork.String("tcp")) +} + func TestConnectionReusedAttributes(t *testing.T) { sr, tp := setupTestTracer(t) ctx := context.Background() @@ -357,9 +443,12 @@ func TestFullRequestLifecycle(t *testing.T) { ct.WroteHeaders() ct.WroteRequest(httptrace.WroteRequestInfo{}) + // 7. Receive + ct.GotFirstResponseByte() + spans := sr.Ended() - // dns, connect, tls, getconn, send = 5 sub-spans - require.Len(t, spans, 5) + // dns, connect, tls, getconn, send, receive = 6 sub-spans + require.Len(t, spans, 6) spanNames := make([]string, len(spans)) for i, s := range spans { @@ -371,6 +460,7 @@ func TestFullRequestLifecycle(t *testing.T) { assert.Contains(t, spanNames, "http.tls") assert.Contains(t, spanNames, "http.getconn") assert.Contains(t, spanNames, "http.send") + assert.Contains(t, spanNames, "http.receive") } func TestParentHook(t *testing.T) { @@ -382,6 +472,7 @@ func TestParentHook(t *testing.T) { {"http.connect.93.184.216.34:443", "http.getconn"}, {"http.tls", "http.getconn"}, {"http.send", ""}, + {"http.receive", ""}, } for _, tt := range tests { diff --git a/tool/internal/setup/sync.go b/tool/internal/setup/sync.go index 2fb955e36..2d1e7d843 100644 --- a/tool/internal/setup/sync.go +++ b/tool/internal/setup/sync.go @@ -51,6 +51,53 @@ type replaceDirective struct { newVersion string } +func localModulePath(modulePath string) string { + relPath := strings.TrimPrefix(modulePath, util.OtelcRoot) + relPath = strings.TrimPrefix(relPath, "/") + return filepath.Join(util.GetBuildTempDir(), filepath.FromSlash(relPath)) +} + +func localModuleReplaces(modulePaths ...string) ([]*replaceDirective, error) { + replaces := make([]*replaceDirective, 0, len(modulePaths)) + seen := make(map[string]bool, len(modulePaths)) + queue := append([]string(nil), modulePaths...) + + for len(queue) > 0 { + modulePath := queue[0] + queue = queue[1:] + + if seen[modulePath] || !strings.HasPrefix(modulePath, util.OtelcRoot+"/pkg") { + continue + } + seen[modulePath] = true + + replaces = append(replaces, &replaceDirective{ + oldPath: modulePath, + newPath: localModulePath(modulePath), + }) + + goModFile := filepath.Join(localModulePath(modulePath), "go.mod") + if _, err := os.Stat(goModFile); err != nil { + if os.IsNotExist(err) { + continue + } + return nil, ex.Wrapf(err, "checking local module go.mod at %s", goModFile) + } + + modfile, err := parseGoMod(goModFile) + if err != nil { + return nil, err + } + for _, req := range modfile.Require { + if strings.HasPrefix(req.Mod.Path, util.OtelcRoot+"/pkg") { + queue = append(queue, req.Mod.Path) + } + } + } + + return replaces, nil +} + func addReplace(modfile *modfile.File, replace *replaceDirective) (bool, error) { hasReplace := false for _, r := range modfile.Replace { @@ -89,40 +136,27 @@ func (sp *SetupPhase) syncDeps(ctx context.Context, matched []*rule.InstRuleSet, if err != nil { return err } - replaces := make([]*replaceDirective, 0) + modulePaths := make([]string, 0, len(rules)+2) for _, m := range rules { util.Assert(strings.HasPrefix(m.Path, util.OtelcRoot), "sanity check") - oldPath := m.Path - newPath := strings.TrimPrefix(oldPath, util.OtelcRoot) - newPath = filepath.Join(util.GetBuildTempDir(), newPath) - replaces = append(replaces, &replaceDirective{ - oldPath: oldPath, - oldVersion: "", - newPath: newPath, - newVersion: "", - }) + modulePaths = append(modulePaths, m.Path) } // Add replace directive for special pkg module // TODO: Since we haven't published the instrumentation packages yet, // we need to add the replace directive to the local path. // Once the instrumentation packages are published, we can remove this. - replaces = append(replaces, &replaceDirective{ - oldPath: util.OtelcRoot + "/pkg", - oldVersion: "", - newPath: filepath.Join(util.GetBuildTempDir(), unzippedPkgDir), - newVersion: "", - }) + modulePaths = append(modulePaths, util.OtelcRoot+"/pkg") // Add replace directive for special shared module // shared module initializes the OpenTelemetry SDK. It is required by all // hook code to be present. - replaces = append(replaces, &replaceDirective{ - oldPath: util.OtelcRoot + "/pkg/instrumentation/shared", - oldVersion: "", - newPath: filepath.Join(util.GetBuildTempDir(), "pkg/instrumentation/shared"), - newVersion: "", - }) + modulePaths = append(modulePaths, util.OtelcRoot+"/pkg/instrumentation/shared") + + replaces, err := localModuleReplaces(modulePaths...) + if err != nil { + return err + } // Okay, now add all the replace directives to go.mod changed := false diff --git a/tool/internal/setup/sync_test.go b/tool/internal/setup/sync_test.go index 39032f941..c197bb78f 100644 --- a/tool/internal/setup/sync_test.go +++ b/tool/internal/setup/sync_test.go @@ -225,3 +225,65 @@ go 1.21 // At minimum, the pkg replace should be added assert.Contains(t, string(content), "replace") } + +func TestSyncDeps_AddsTransitiveLocalModuleReplaces(t *testing.T) { + tempDir := t.TempDir() + + gomodPath := filepath.Join(tempDir, "go.mod") + err := os.WriteFile(gomodPath, []byte(`module example.com/test + +go 1.25.0 +`), 0o644) + require.NoError(t, err) + + t.Chdir(tempDir) + t.Setenv(util.EnvOtelcWorkDir, tempDir) + + writeModule := func(modulePath, content string) { + t.Helper() + moduleDir := localModulePath(modulePath) + require.NoError(t, os.MkdirAll(moduleDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(moduleDir, "go.mod"), []byte(content), 0o644)) + } + + clientModule := util.OtelcRoot + "/pkg/instrumentation/nethttp/client" + httptraceModule := util.OtelcRoot + "/pkg/instrumentation/nethttp/httptrace" + sharedModule := util.OtelcRoot + "/pkg/instrumentation/shared" + pkgModule := util.OtelcRoot + "/pkg" + + writeModule(clientModule, `module `+clientModule+` + +go 1.25.0 + +require `+httptraceModule+` v0.0.0-00010101000000-000000000000 +`) + writeModule(httptraceModule, `module `+httptraceModule+` + +go 1.25.0 +`) + writeModule(sharedModule, `module `+sharedModule+` + +go 1.25.0 +`) + writeModule(pkgModule, `module `+pkgModule+` + +go 1.25.0 +`) + + sp := &SetupPhase{logger: slog.Default()} + ruleSet := &rule.InstRuleSet{ + FuncRules: map[string][]*rule.InstFuncRule{ + "test.go": {{ + InstBaseRule: rule.InstBaseRule{Name: "http-client"}, + Path: clientModule, + }}, + }, + } + + require.NoError(t, sp.syncDeps(t.Context(), []*rule.InstRuleSet{ruleSet}, tempDir)) + + content, err := os.ReadFile(gomodPath) + require.NoError(t, err) + assert.Contains(t, string(content), "replace "+clientModule+" => "+filepath.Join(util.GetBuildTempDir(), "pkg/instrumentation/nethttp/client")) + assert.Contains(t, string(content), "replace "+httptraceModule+" => "+filepath.Join(util.GetBuildTempDir(), "pkg/instrumentation/nethttp/httptrace")) +}