Merge pull request #215 from grcevski/more_net_debug

Add debugging info for net TLS

.gitignore

@@ -65,4 +65,4 @@ test/integration/components/rubytestserver/testapi/storage/*
 test/integration/components/rubytestserver/testapi/tmp/storage/*
 !test/integration/components/rubytestserver/testapi/tmp/storage/
 !test/integration/components/rubytestserver/testapi/tmp/storage/.keep
-test/integration/components/dotnetserver/obj/Debug
+test/integration/components/dotnetserver/obj/*

bpf/http_sock.c

@@ -90,7 +90,6 @@ int BPF_KPROBE(kprobe_tcp_rcv_established, struct sock *sk, struct sk_buff *skb)
         bpf_map_update_elem(&pid_tid_to_conn, &id, &info, BPF_ANY); // to support SSL on missing handshake
     }
 
-
     return 0;
 }
 
@@ -123,6 +122,8 @@ int BPF_KRETPROBE(kretprobe_sys_accept4, uint fd)
         goto cleanup;
     }
 
+    bpf_dbg_printk("=== accept 4 ret id=%d, sock=%llx, fd=%d ===", id, args->addr, fd);
+
     connection_info_t info = {};
 
     if (parse_accept_socket_info(args, &info)) {
@@ -133,6 +134,7 @@ int BPF_KRETPROBE(kretprobe_sys_accept4, uint fd)
         meta.id = id;
         meta.type = EVENT_HTTP_REQUEST;
         bpf_map_update_elem(&filtered_connections, &info, &meta, BPF_ANY); // On purpose BPF_ANY, we want to overwrite stale
+        bpf_map_update_elem(&pid_tid_to_conn, &id, &info, BPF_ANY); // to support SSL on missing handshake
     }
 
 cleanup:

bpf/http_ssl.h

@@ -151,6 +151,17 @@ static __always_inline void handle_ssl_buf(u64 id, ssl_args_t *args, int bytes_l
 
         bpf_map_delete_elem(&ssl_to_pid_tid, &ssl_ptr);
 
+
+        if (!conn) {
+            connection_info_t c = {};
+            bpf_dbg_printk("setting fake connection info ssl=%llx", ssl);
+            bpf_memcpy(&c.s_addr, &ssl, sizeof(void *));
+            c.d_port = c.s_port = 0;
+
+            bpf_map_update_elem(&ssl_to_conn, &ssl, &c, BPF_ANY);
+            conn = bpf_map_lookup_elem(&ssl_to_conn, &ssl);
+        }
+
         if (conn) {
             void *read_buf = (void *)args->buf;
             char buf[FULL_BUF_SIZE] = {0};

pkg/internal/ebpf/httpfltr/httpfltr.go

@@ -219,9 +219,20 @@ func (p *Tracer) toRequestTrace(record *ringbuf.Record) (any, error) {
 
 	result = HTTPInfo{BPFHTTPInfo: event}
 
-	source, target := event.hostInfo()
-	result.Host = target
-	result.Peer = source
+	// When we can't find the connection info, we signal that through making the
+	// source and destination ports equal to max short. E.g. async SSL
+	if event.ConnInfo.S_port != 0 || event.ConnInfo.D_port != 0 {
+		source, target := event.hostInfo()
+		result.Host = target
+		result.Peer = source
+	} else {
+		host, port := event.hostFromBuf()
+
+		if port >= 0 {
+			result.Host = host
+			result.ConnInfo.D_port = uint16(port)
+		}
+	}
 	result.URL = event.url()
 	result.Method = event.method()
 	if p.Cfg.SystemWide {
@@ -255,6 +266,29 @@ func (event *BPFHTTPInfo) method() string {
 	return buf[:space]
 }
 
+func (event *BPFHTTPInfo) hostFromBuf() (string, int) {
+	buf := string(event.Buf[:])
+	idx := strings.Index(buf, "Host: ")
+
+	if idx < 0 {
+		return "", -1
+	}
+
+	buf = buf[idx+6:]
+
+	rIdx := strings.Index(buf, "\r")
+
+	host, portStr, err := net.SplitHostPort(buf[:rIdx])
+
+	if err != nil {
+		return "", -1
+	}
+
+	port, _ := strconv.Atoi(portStr)
+
+	return host, port
+}
+
 func (event *BPFHTTPInfo) hostInfo() (source, target string) {
 	src := make(net.IP, net.IPv6len)
 	dst := make(net.IP, net.IPv6len)

pkg/internal/ebpf/httpfltr/httpfltr_test.go

@@ -90,6 +90,7 @@ func TestToRequestTrace(t *testing.T) {
 	record.StartMonotimeNs = 123456
 	record.EndMonotimeNs = 789012
 	record.Status = 200
+	record.ConnInfo.D_port = 1
 	record.ConnInfo.S_addr = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 192, 168, 0, 1}
 	record.ConnInfo.D_addr = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 8, 8, 8, 8}
 	copy(record.Buf[:], []byte("GET /hello HTTP/1.1\r\nHost: example.com\r\n\r\n"))
@@ -111,3 +112,33 @@ func TestToRequestTrace(t *testing.T) {
 	}
 	assert.Equal(t, expected, result)
 }
+
+func TestToRequestTraceNoConnection(t *testing.T) {
+	var record BPFHTTPInfo
+	record.Type = 1
+	record.StartMonotimeNs = 123456
+	record.EndMonotimeNs = 789012
+	record.Status = 200
+	record.ConnInfo.S_addr = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 192, 168, 0, 1}
+	record.ConnInfo.D_addr = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 8, 8, 8, 8}
+	copy(record.Buf[:], []byte("GET /hello HTTP/1.1\r\nHost: localhost:7033\r\n\r\nUser-Agent: curl/7.81.0\r\nAccept: */*\r\n"))
+
+	buf := new(bytes.Buffer)
+	err := binary.Write(buf, binary.LittleEndian, &record)
+	assert.NoError(t, err)
+
+	tracer := Tracer{Cfg: &ebpfcommon.TracerConfig{}}
+	result, err := tracer.toRequestTrace(&ringbuf.Record{RawSample: buf.Bytes()})
+	assert.NoError(t, err)
+
+	// change the expected port just before testing
+	record.ConnInfo.D_port = 7033
+	expected := HTTPInfo{
+		BPFHTTPInfo: record,
+		Host:        "localhost",
+		Peer:        "",
+		URL:         "/hello",
+		Method:      "GET",
+	}
+	assert.Equal(t, expected, result)
+}

test/integration/components/beyla/Dockerfile_dbg

@@ -0,0 +1,38 @@
+# Development version of the beyla Dockerfile that compiles for coverage
+# and allows retrieving coverage files.
+# The production-ready minimal image is in the project root's dockerfile.
+FROM golang:1.20 as builder
+
+ARG TARGETARCH
+
+ENV GOARCH=$TARGETARCH
+
+WORKDIR /
+
+COPY test/integration/components/beyla/beyla_wrapper.sh /beyla_wrapper.sh
+
+WORKDIR /src
+
+# Copy the go manifests and source
+COPY .git/ .git/
+COPY bpf/ bpf/
+COPY cmd/ cmd/
+COPY pkg/ pkg/
+COPY vendor/ vendor/
+COPY go.mod go.mod
+COPY go.sum go.sum
+COPY Makefile Makefile
+
+# Build
+RUN make compile-for-coverage
+
+FROM ubuntu:latest
+
+# Copy the native executable into the containers
+COPY --from=builder /src/bin/beyla /beyla
+COPY --from=builder /beyla_wrapper.sh /beyla_wrapper.sh
+
+WORKDIR /
+USER 0:0
+
+CMD [ "/beyla_wrapper.sh" ]

test/integration/components/beyla/beyla_wrapper.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Mount debugfs, we should be running privileged, so should be doable
+mount -t debugfs nodev /sys/kernel/debug
+
+# Start the trace pipe reader
+cat /sys/kernel/debug/tracing/trace_pipe &
+
+# Start the instrumenter
+./beyla "$@" &
+
+# Wait for any process to exit
+wait -n
+
+# Exit with status of process that exited first
+exit $?

test/integration/docker-compose-dotnet.yml

@@ -15,9 +15,9 @@ services:
   autoinstrumenter:
     build:
       context: ../..
-      dockerfile: ./test/integration/components/beyla/Dockerfile
+      dockerfile: ./test/integration/components/beyla/Dockerfile${INSTRUMENT_DOCKERFILE_SUFFIX}
     command:
-      - /beyla
+      - /beyla${INSTRUMENT_COMMAND_SUFFIX}
       - --config=/configs/instrumenter-config-java.yml
     volumes:
       - ./configs/:/configs
@@ -33,8 +33,8 @@ services:
       OPEN_PORT: "${OPEN_PORT}"
       EXECUTABLE_NAME: "${EXECUTABLE_NAME}"
       SERVICE_NAMESPACE: "integration-test"
-      METRICS_INTERVAL: "10ms"
-      BPF_BATCH_TIMEOUT: "10ms"
+      METRICS_INTERVAL: "100ms"
+      BPF_BATCH_TIMEOUT: "100ms"
       LOG_LEVEL: "DEBUG"
       BPF_DEBUG: "TRUE"
       METRICS_REPORT_TARGET: "true"

test/integration/red_test_dotnet.go

@@ -4,6 +4,12 @@ package integration
 
 import (
 	"testing"
+
+	"github.com/mariomac/guara/pkg/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/ebpf-autoinstrument/test/integration/components/prom"
 )
 
 func testREDMetricsDotNetHTTP(t *testing.T) {
@@ -17,16 +23,45 @@ func testREDMetricsDotNetHTTP(t *testing.T) {
 	}
 }
 
-/*
-// Disabled because of: https://github.com/grafana/ebpf-autoinstrument/issues/208
+// Special test without checks for a peer address. With the async nature of SSL on .NET we can't always get
+// this information
+func testREDMetricsForNetHTTPSLibrary(t *testing.T, url string, comm string) {
+	path := "/greeting"
+
+	// Call 3 times the instrumented service, forcing it to:
+	// - take at least 30ms to respond
+	// - returning a 204 code
+	for i := 0; i < 4; i++ {
+		doHTTPGet(t, url+path, 200)
+	}
+
+	// Eventually, Prometheus would make this query visible
+	pq := prom.Client{HostPort: prometheusHostPort}
+	var results []prom.Result
+	test.Eventually(t, testTimeout, func(t require.TestingT) {
+		var err error
+		results, err = pq.Query(`http_server_duration_seconds_count{` +
+			`http_method="GET",` +
+			`http_status_code="200",` +
+			`service_namespace="integration-test",` +
+			`service_name="` + comm + `",` +
+			`http_target="` + path + `"}`)
+		require.NoError(t, err)
+		require.Len(t, results, 1)
+		if len(results) > 0 {
+			res := results[0]
+			require.Len(t, res.Value, 2)
+			assert.LessOrEqual(t, "3", res.Value[1])
+		}
+	})
+}
 func testREDMetricsDotNetHTTPS(t *testing.T) {
 	for _, testCaseURL := range []string{
 		"https://localhost:7034",
 	} {
 		t.Run(testCaseURL, func(t *testing.T) {
 			waitForTestComponents(t, testCaseURL)
-			testREDMetricsForNodeHTTPLibrary(t, testCaseURL, "dotnetserver") // reusing what we do for NodeJS
+			testREDMetricsForNetHTTPSLibrary(t, testCaseURL, "dotnetserver") // reusing what we do for NodeJS
 		})
 	}
 }
-*/

test/integration/suites_test.go

@@ -233,20 +233,19 @@ func TestSuite_DotNet(t *testing.T) {
 	t.Run("BPF pinning folder unmounted", testBPFPinningUnmounted)
 }
 
-/*
 // Disabled for now as we randomly fail to register 3 events, but only get 2
 // Issue: https://github.com/grafana/ebpf-autoinstrument/issues/208
 func TestSuite_DotNetTLS(t *testing.T) {
 	compose, err := docker.ComposeSuite("docker-compose-dotnet.yml", path.Join(pathOutput, "test-suite-dotnet-tls.log"))
 	compose.Env = append(compose.Env, `OPEN_PORT=7033`, `EXECUTABLE_NAME=`, `TEST_SERVICE_PORTS=7034:7033`, `TESTSERVER_DOCKERFILE_SUFFIX=_tls`)
+	// Add these above if you want to get the trace_pipe output in the test logs: `INSTRUMENT_DOCKERFILE_SUFFIX=_dbg`, `INSTRUMENT_COMMAND_SUFFIX=_wrapper.sh`
 	require.NoError(t, err)
 	require.NoError(t, compose.Up())
 	t.Run("DotNet SSL RED metrics", testREDMetricsDotNetHTTPS)
 	t.Run("BPF pinning folder mounted", testBPFPinningMounted)
 	require.NoError(t, compose.Close())
 	t.Run("BPF pinning folder unmounted", testBPFPinningUnmounted)
 }
-*/
 
 func TestSuite_Python(t *testing.T) {
 	compose, err := docker.ComposeSuite("docker-compose-python.yml", path.Join(pathOutput, "test-suite-python.log"))