tests: Fix some issues with build output

Before this change `RunTest` may return before all messages are
processed.

This also moves a lot of the logic around to better separate
responsibilities and provide a more flexible interface that is,
hopefully, easier to reason about.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Author: cpuguy83

.github/workflows/ci.yml

@@ -143,6 +143,12 @@ jobs:
         run: docker run --rm --privileged tonistiigi/binfmt:latest --install all
       - name: Setup source policy
         uses: ./.github/actions/setup-source-policy
+      - name: Get docker info
+        run: |
+          docker info
+          docker version
+          docker buildx version
+          containerd --version
       - name: Run integration tests
         run: |
           set -ex

test/gomod_git_auth_test.go

@@ -14,13 +14,13 @@ import (
 	"strconv"
 	"testing"
 
+	"github.com/moby/buildkit/client/llb"
+	gwclient "github.com/moby/buildkit/frontend/gateway/client"
+	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/project-dalec/dalec"
 	"github.com/project-dalec/dalec/test/cmd/git_repo/passwd"
 	gitservices "github.com/project-dalec/dalec/test/git_services"
 	"github.com/project-dalec/dalec/test/testenv"
-	"github.com/moby/buildkit/client/llb"
-	gwclient "github.com/moby/buildkit/frontend/gateway/client"
-	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/agent"
 )
@@ -109,7 +109,8 @@ go {{ .ModFileGoVersion }}
 
 	t.Run("HTTP", func(t *testing.T) {
 		t.Parallel()
-		netHostBuildxEnv.RunTestOptsFirst(ctx, t, []testenv.TestRunnerOpt{
+
+		opts := []testenv.TestRunnerOpt{
 			// This gives buildkit access to a secret with the name
 			// `secretName` and the value `passwd.Password`. On the worker that
 			// fetches the gomod dependencies, a file will be mounted at the
@@ -120,7 +121,9 @@ go {{ .ModFileGoVersion }}
 			// requests. It is necessary but not sufficient for the buildx
 			// instance to have host networking enabled.
 			testenv.WithHostNetworking,
-		}, func(ctx context.Context, client gwclient.Client) {
+		}
+
+		netHostBuildxEnv.RunTest(ctx, t, func(ctx context.Context, client gwclient.Client) {
 			// This MUST be called at the  start of each test. Because we
 			// persist the go mod cache between runs, the git tag of the
 			// private go module needs to be unique. Within a single run of the
@@ -167,7 +170,7 @@ go {{ .ModFileGoVersion }}
 
 			filename := calculateFilename(ctx, t, attr, res)
 			checkFile(ctx, t, filename, res, []byte("bar\n"))
-		})
+		}, opts...)
 	})
 
 	t.Run("SSH", func(t *testing.T) {
@@ -182,12 +185,13 @@ go {{ .ModFileGoVersion }}
 		pubkey, privkey := generateKeyPair(t)
 		agentErrChan := startSSHAgent(t, privkey, sockaddr)
 
-		netHostBuildxEnv.RunTestOptsFirst(ctx, t, []testenv.TestRunnerOpt{
+		opts := []testenv.TestRunnerOpt{
 			// This tells buildkit to forward the SSH Agent socket, giving the
 			// gomod generator worker access to the private key
 			testenv.WithSSHSocket(sshID, sockaddr),
 			testenv.WithHostNetworking,
-		}, func(ctx context.Context, client gwclient.Client) {
+		}
+		netHostBuildxEnv.RunTest(ctx, t, func(ctx context.Context, client gwclient.Client) {
 			// This MUST be called at the  start of each test. Because we
 			// persist the go mod cache between runs, the git tag of the
 			// private go module needs to be unique. Within a single run of the
@@ -245,7 +249,7 @@ go {{ .ModFileGoVersion }}
 
 			filename := calculateFilename(ctx, t, attr, res)
 			checkFile(ctx, t, filename, res, []byte("bar\n"))
-		})
+		}, opts...)
 	})
 }
 

test/linux_target_test.go

@@ -3171,10 +3171,12 @@ func testLinuxPackageTestsFail(ctx context.Context, t *testing.T, cfg testLinuxC
 
 		runTest := func(target string) func(t *testing.T) {
 			return func(t *testing.T) {
+				t.Parallel()
 				ctx := startTestSpan(ctx, t)
 
 				for _, tc := range expectedErrs {
 					t.Run(tc.test.Name, func(t *testing.T) {
+						t.Parallel()
 						ctx := startTestSpan(ctx, t)
 
 						f := newLogFile(t)
@@ -3200,6 +3202,7 @@ func testLinuxPackageTestsFail(ctx context.Context, t *testing.T, cfg testLinuxC
 							sr := newSolveRequest(withSpec(ctx, t, spec), withBuildTarget(target))
 							_, err := client.Solve(ctx, sr)
 							assert.Assert(t, err != nil)
+							t.Logf("Build Error: %v", err)
 							// exit code errors would be here rather than in the buid logs
 							if !tc.isBuildError {
 								return
@@ -3222,9 +3225,7 @@ func testLinuxPackageTestsFail(ctx context.Context, t *testing.T, cfg testLinuxC
 
 						if !bytes.Contains(dt, []byte(tc.err.Error())) {
 							t.Errorf("expected error not found in logs")
-							t.Logf("Expected error:\n\t%v", err)
-							t.Log()
-							t.Log("---- Solve logs ----\n" + string(dt))
+							t.Logf("Expected error:\n\t%v", tc.err)
 						}
 					})
 				}

test/main_test.go

@@ -10,10 +10,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/project-dalec/dalec/test/fixtures"
-	"github.com/project-dalec/dalec/test/testenv"
 	"github.com/moby/buildkit/util/tracing/delegated"
 	"github.com/moby/buildkit/util/tracing/detect"
+	"github.com/project-dalec/dalec/test/fixtures"
+	"github.com/project-dalec/dalec/test/testenv"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/trace"

test/testenv/build.go

@@ -18,7 +18,6 @@ import (
 	gwclient "github.com/moby/buildkit/frontend/gateway/client"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/solver/pb"
-	"github.com/moby/buildkit/util/progress/progressui"
 	"github.com/pkg/errors"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/codes"
@@ -176,6 +175,46 @@ var logBufferPool = &sync.Pool{New: func() any {
 	return bytes.NewBuffer(nil)
 }}
 
+func outputStreamStatusFn(ctx context.Context, t *testing.T) (func(*client.SolveStatus), func()) {
+	var (
+		warnings []*client.VertexWarning
+		errOnce  sync.Once
+
+		done = make(chan struct{})
+		w    = getBuildOutputStream(ctx, t, done)
+	)
+
+	fn := func(msg *client.SolveStatus) {
+		warnings = append(warnings, msg.Warnings...)
+		for _, l := range msg.Logs {
+			if _, err := w.Write(l.Data); err != nil {
+				errOnce.Do(func() {
+					// Don't spam the logs with multiple errors
+					t.Logf("error writing log data: %v", err)
+				})
+			}
+		}
+	}
+
+	return fn, func() {
+		for _, v := range warnings {
+			t.Logf("WARNING: %s", string(v.Short))
+		}
+		close(done)
+	}
+}
+
+func multiStatusFunc(fns ...func(*client.SolveStatus)) func(*client.SolveStatus) {
+	return func(msg *client.SolveStatus) {
+		for _, fn := range fns {
+			if fn == nil {
+				continue
+			}
+			fn(msg)
+		}
+	}
+}
+
 func getBuildOutputStream(ctx context.Context, t *testing.T, done <-chan struct{}) io.Writer {
 	t.Helper()
 
@@ -226,47 +265,42 @@ func getBuildOutputStream(ctx context.Context, t *testing.T, done <-chan struct{
 	return f
 }
 
-func displaySolveStatus(ctx context.Context, t *testing.T) chan *client.SolveStatus {
-	ch := make(chan *client.SolveStatus)
-	done := make(chan struct{})
-
-	output := getBuildOutputStream(ctx, t, done)
-	display, err := progressui.NewDisplay(output, progressui.AutoMode, progressui.WithPhase(t.Name()))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	go func() {
-		defer close(done)
-
-		_, err := display.UpdateFrom(ctx, ch)
-		if err != nil {
-			t.Log(err)
+func fowardToSolveStatusFn(ctx context.Context, ch <-chan *client.SolveStatus, fns ...func(*client.SolveStatus)) {
+	f := multiStatusFunc(fns...)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case msg, ok := <-ch:
+			if msg != nil {
+				f(msg)
+				continue
+			}
+			if !ok {
+				return
+			}
 		}
-	}()
-
-	return ch
+	}
 }
 
 // withProjectRoot adds the current project root as the build context for the solve request.
-func withProjectRoot(t *testing.T, opts *client.SolveOpt) {
-	t.Helper()
-
+func withProjectRoot(opts *client.SolveOpt) error {
 	cwd, err := os.Getwd()
 	if err != nil {
-		t.Fatal(err)
+		return err
 	}
 
 	projectRoot, err := lookupProjectRoot(cwd)
 	if err != nil {
-		t.Fatal(err)
+		return err
 	}
 
 	if opts.LocalDirs == nil {
 		opts.LocalDirs = make(map[string]string)
 	}
 	opts.LocalDirs[dockerui.DefaultLocalNameContext] = projectRoot
 	opts.LocalDirs[dockerui.DefaultLocalNameDockerfile] = projectRoot
+	return nil
 }
 
 // lookupProjectRoot looks up the project root from the current working directory.