Merge pull request #52 from hellofresh/feature/check-tracing

EES-4396 Added tracing support

go.mod

@@ -10,5 +10,6 @@ require (
 	github.com/streadway/amqp v1.0.0
 	github.com/stretchr/testify v1.6.1
 	go.mongodb.org/mongo-driver v1.4.1
+	go.opentelemetry.io/otel v0.13.0
 	google.golang.org/grpc v1.31.1
 )

go.sum

@@ -209,6 +209,7 @@ github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=

health.go

@@ -9,6 +9,10 @@ import (
 	"runtime"
 	"sync"
 	"time"
+
+	"go.opentelemetry.io/otel/api/trace"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/label"
 )
 
 // Status type represents health status
@@ -68,6 +72,9 @@ type (
 	Health struct {
 		mu     sync.Mutex
 		checks map[string]Config
+
+		tp                  trace.TracerProvider
+		instrumentationName string
 	}
 
 	checkResponse struct {
@@ -81,6 +88,7 @@ type (
 func New(opts ...Option) (*Health, error) {
 	h := &Health{
 		checks: make(map[string]Config),
+		tp:     trace.NoopTracerProvider(),
 	}
 
 	for _, o := range opts {
@@ -139,15 +147,34 @@ func (h *Health) HandlerFunc(w http.ResponseWriter, r *http.Request) {
 	w.Write(data)
 }
 
+type checkSpan struct {
+	ctx  context.Context
+	span trace.Span
+}
+
+func newCheckSpan(ctx context.Context, tracer trace.Tracer, name string) checkSpan {
+	var cs checkSpan
+	cs.ctx, cs.span = tracer.Start(ctx, name)
+	return cs
+}
+
 // Measure runs all the registered health checks and returns summary status
 func (h *Health) Measure(ctx context.Context) Check {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 
+	tracer := h.tp.Tracer(h.instrumentationName)
+
+	ctx, span := tracer.Start(ctx, "health.Measure")
+	defer span.End()
+
 	status := StatusOK
 	total := len(h.checks)
 	failures := make(map[string]string)
 	resChan := make(chan checkResponse, total)
+	checkSpans := make(map[string]checkSpan)
+
+	span.SetAttributes(label.Int("checks", total))
 
 	var wg sync.WaitGroup
 	wg.Add(total)
@@ -159,6 +186,8 @@ func (h *Health) Measure(ctx context.Context) Check {
 	}()
 
 	for _, c := range h.checks {
+		checkSpans[c.Name] = newCheckSpan(ctx, tracer, c.Name)
+
 		go func(c Config) {
 			defer wg.Done()
 
@@ -174,17 +203,31 @@ func (h *Health) Measure(ctx context.Context) Check {
 			case <-time.After(c.Timeout):
 				failures[c.Name] = string(StatusTimeout)
 				status = getAvailability(status, c.SkipOnErr)
+
+				cs := checkSpans[c.Name]
+				cs.span.SetStatus(codes.Error, string(StatusTimeout))
+				cs.span.End()
+
 				break loop
 			case res := <-resChan:
+				cs := checkSpans[res.name]
+
 				if res.err != nil {
 					failures[res.name] = res.err.Error()
 					status = getAvailability(status, res.skipOnErr)
+
+					cs.span.RecordError(cs.ctx, res.err)
 				}
+
+				cs.span.End()
+
 				break loop
 			}
 		}
 	}
 
+	span.SetAttributes(label.String("status", string(status)))
+
 	return newCheck(status, failures)
 }
 

options.go

@@ -1,6 +1,10 @@
 package health
 
-import "fmt"
+import (
+	"fmt"
+
+	"go.opentelemetry.io/otel/api/trace"
+)
 
 // Option is the health-container options type
 type Option func(*Health) error
@@ -17,3 +21,14 @@ func WithChecks(checks ...Config) Option {
 		return nil
 	}
 }
+
+// WithTracerProvider sets trace provider for the checks and instrumentation name that will be used
+// for tracer from trace provider.
+func WithTracerProvider(tp trace.TracerProvider, instrumentationName string) Option {
+	return func(h *Health) error {
+		h.tp = tp
+		h.instrumentationName = instrumentationName
+
+		return nil
+	}
+}

options_test.go

@@ -1,10 +1,13 @@
 package health
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/api/trace"
 )
 
 func TestWithChecks(t *testing.T) {
@@ -27,3 +30,27 @@ func TestWithChecks(t *testing.T) {
 	}))
 	require.Error(t, err)
 }
+
+type mockTracerProvider struct {
+	mock.Mock
+}
+
+func (m *mockTracerProvider) Tracer(instrumentationName string, opts ...trace.TracerOption) trace.Tracer {
+	args := m.Called(instrumentationName, opts)
+	return args.Get(0).(trace.Tracer)
+}
+
+func TestWithTracerProvider(t *testing.T) {
+	h1, err := New()
+	require.NoError(t, err)
+	assert.Equal(t, "trace.noopTracerProvider", fmt.Sprintf("%T", h1.tp))
+	assert.Equal(t, "", h1.instrumentationName)
+
+	tp := new(mockTracerProvider)
+	instrumentationName := "test.test"
+
+	h2, err := New(WithTracerProvider(tp, instrumentationName))
+	require.NoError(t, err)
+	assert.Same(t, tp, h2.tp)
+	assert.Equal(t, instrumentationName, h2.instrumentationName)
+}