diff --git a/.tool-versions b/.tool-versions index 5521e49283..1fd8168ccb 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,3 +1,3 @@ -golang 1.18.1 +golang 1.19.3 shfmt 3.2.0 shellcheck 0.7.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a81623fd5..d05ecc14e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,16 @@ All notable changes to `src-cli` are documented in this file. ### Added +### Changed + +### Fixed + +### Removed + +## 4.2.1 + +### Added + - Batch specs being run locally with `src batch preview` or `src batch apply` can now be run with the `-run-as-root` flag, which will run all step containers as root instead of the default user for the image. This is off by default. [#886](https://github.com/sourcegraph/src-cli/pull/886) ### Changed @@ -21,7 +31,7 @@ All notable changes to `src-cli` are documented in this file. ### Fixed -### Removed +- Batch changes: Git patches are now binary encoded instead of UTF-8 over the wire, fixing support for non-UTF-8 files. [#887](https://github.com/sourcegraph/src-cli/pull/887) ## 4.2.0 diff --git a/cmd/src/batch_apply.go b/cmd/src/batch_apply.go index dcbc061f51..28f715656d 100644 --- a/cmd/src/batch_apply.go +++ b/cmd/src/batch_apply.go @@ -5,10 +5,7 @@ import ( "flag" "fmt" - "github.com/sourcegraph/src-cli/internal/batches/ui" "github.com/sourcegraph/src-cli/internal/cmderrors" - - "github.com/sourcegraph/sourcegraph/lib/output" ) func init() { @@ -47,15 +44,7 @@ Examples: ctx, cancel := contextCancelOnInterrupt(context.Background()) defer cancel() - var execUI ui.ExecUI - if flags.textOnly { - execUI = &ui.JSONLines{} - } else { - out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose}) - execUI = &ui.TUI{Out: out} - } - - if err = executeBatchSpec(ctx, execUI, executeBatchSpecOpts{ + if err = executeBatchSpec(ctx, executeBatchSpecOpts{ flags: flags, client: cfg.apiClient(flags.api, flagSet.Output()), file: file, diff --git a/cmd/src/batch_common.go b/cmd/src/batch_common.go index c387c60055..8a07076421 100644 --- a/cmd/src/batch_common.go +++ b/cmd/src/batch_common.go @@ -18,6 +18,7 @@ import ( "github.com/mattn/go-isatty" "github.com/sourcegraph/sourcegraph/lib/errors" + "github.com/sourcegraph/sourcegraph/lib/output" batcheslib "github.com/sourcegraph/sourcegraph/lib/batches" "github.com/sourcegraph/sourcegraph/lib/batches/template" @@ -253,10 +254,18 @@ type executeBatchSpecOpts struct { // executeBatchSpec performs all the steps required to upload the batch spec to // Sourcegraph, including execution as needed and applying the resulting batch // spec if specified. -func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOpts) (err error) { +func executeBatchSpec(ctx context.Context, opts executeBatchSpecOpts) (err error) { + var execUI ui.ExecUI + if opts.flags.textOnly { + execUI = &ui.JSONLines{} + } else { + out := output.NewOutput(os.Stderr, output.OutputOpts{Verbose: *verbose}) + execUI = &ui.TUI{Out: out} + } + defer func() { if err != nil { - ui.ExecutionError(err) + execUI.ExecutionError(err) } }() @@ -264,9 +273,19 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp Client: opts.client, }) + ffs, err := svc.DetermineFeatureFlags(ctx) + if err != nil { + return err + } + + // Once we know about feature flags, reconfigure the UI if needed. + if opts.flags.textOnly && ffs.BinaryDiffs { + execUI = &ui.JSONLines{BinaryDiffs: true} + } + imageCache := docker.NewImageCache() - if err := validateSourcegraphVersionConstraint(ctx, svc); err != nil { + if err := validateSourcegraphVersionConstraint(ctx, ffs); err != nil { return err } @@ -309,12 +328,12 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp } // Parse flags and build up our service and executor options. - ui.ParsingBatchSpec() + execUI.ParsingBatchSpec() batchSpec, batchSpecDir, rawSpec, err := parseBatchSpec(ctx, opts.file, svc) if err != nil { var multiErr errors.MultiError if errors.As(err, &multiErr) { - ui.ParsingBatchSpecFailure(multiErr) + execUI.ParsingBatchSpecFailure(multiErr) return cmderrors.ExitCode(2, nil) } else { // This shouldn't happen; let's just punt and let the normal @@ -322,32 +341,32 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp return err } } - ui.ParsingBatchSpecSuccess() + execUI.ParsingBatchSpecSuccess() - ui.ResolvingNamespace() + execUI.ResolvingNamespace() namespace, err := svc.ResolveNamespace(ctx, opts.flags.namespace) if err != nil { return err } - ui.ResolvingNamespaceSuccess(namespace.ID) + execUI.ResolvingNamespaceSuccess(namespace.ID) var workspaceCreator workspace.Creator if len(batchSpec.Steps) > 0 { - ui.PreparingContainerImages() + execUI.PreparingContainerImages() images, err := svc.EnsureDockerImages( ctx, imageCache, batchSpec.Steps, parallelism, - ui.PreparingContainerImagesProgress, + execUI.PreparingContainerImagesProgress, ) if err != nil { return err } - ui.PreparingContainerImagesSuccess() + execUI.PreparingContainerImagesSuccess() - ui.DeterminingWorkspaceCreatorType() + execUI.DeterminingWorkspaceCreatorType() var typ workspace.CreatorType workspaceCreator, typ = workspace.NewCreator(ctx, opts.flags.workspace, opts.flags.cacheDir, opts.flags.tempDir, images) if typ == workspace.CreatorTypeVolume { @@ -357,21 +376,21 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp return err } } - ui.DeterminingWorkspaceCreatorTypeSuccess(typ) + execUI.DeterminingWorkspaceCreatorTypeSuccess(typ) } - ui.DeterminingWorkspaces() + execUI.DeterminingWorkspaces() workspaces, repos, err := svc.ResolveWorkspacesForBatchSpec(ctx, batchSpec, opts.flags.allowUnsupported, opts.flags.allowIgnored) if err != nil { if repoSet, ok := err.(batches.UnsupportedRepoSet); ok { - ui.DeterminingWorkspacesSuccess(len(workspaces), len(repos), repoSet, nil) + execUI.DeterminingWorkspacesSuccess(len(workspaces), len(repos), repoSet, nil) } else if repoSet, ok := err.(batches.IgnoredRepoSet); ok { - ui.DeterminingWorkspacesSuccess(len(workspaces), len(repos), nil, repoSet) + execUI.DeterminingWorkspacesSuccess(len(workspaces), len(repos), nil, repoSet) } else { return errors.Wrap(err, "resolving repositories") } } else { - ui.DeterminingWorkspacesSuccess(len(workspaces), len(repos), nil, nil) + execUI.DeterminingWorkspacesSuccess(len(workspaces), len(repos), nil, nil) } archiveRegistry := repozip.NewArchiveRegistry(opts.client, opts.flags.cacheDir, opts.flags.cleanArchives) @@ -389,14 +408,16 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp TempDir: opts.flags.tempDir, GlobalEnv: os.Environ(), ForceRoot: opts.flags.runAsRoot, + BinaryDiffs: ffs.BinaryDiffs, }, - Logger: logManager, - Cache: executor.NewDiskCache(opts.flags.cacheDir), - GlobalEnv: os.Environ(), + Logger: logManager, + Cache: executor.NewDiskCache(opts.flags.cacheDir), + BinaryDiffs: ffs.BinaryDiffs, + GlobalEnv: os.Environ(), }, ) - ui.CheckingCache() + execUI.CheckingCache() tasks := svc.BuildTasks( &template.BatchChangeAttributes{ Name: batchSpec.Name, @@ -421,9 +442,9 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp return err } } - ui.CheckingCacheSuccess(len(specs), len(uncachedTasks)) + execUI.CheckingCacheSuccess(len(specs), len(uncachedTasks)) - taskExecUI := ui.ExecutingTasks(*verbose, parallelism) + taskExecUI := execUI.ExecutingTasks(*verbose, parallelism) freshSpecs, logFiles, execErr := coord.ExecuteAndBuildSpecs(ctx, batchSpec, uncachedTasks, taskExecUI) // Add external changeset specs. importedSpecs, importErr := svc.CreateImportChangesetSpecs(ctx, batchSpec) @@ -440,7 +461,7 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp if err == nil { taskExecUI.Success() } else { - ui.ExecutingTasksSkippingErrors(err) + execUI.ExecutingTasksSkippingErrors(err) } } else { if err != nil { @@ -450,7 +471,7 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp } if len(logFiles) > 0 && opts.flags.keepLogs { - ui.LogFilesKept(logFiles) + execUI.LogFilesKept(logFiles) } specs = append(specs, freshSpecs...) @@ -464,7 +485,7 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp ids := make([]graphql.ChangesetSpecID, len(specs)) if len(specs) > 0 { - ui.UploadingChangesetSpecs(len(specs)) + execUI.UploadingChangesetSpecs(len(specs)) for i, spec := range specs { id, err := svc.CreateChangesetSpec(ctx, spec) @@ -472,21 +493,21 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp return err } ids[i] = id - ui.UploadingChangesetSpecsProgress(i+1, len(specs)) + execUI.UploadingChangesetSpecsProgress(i+1, len(specs)) } - ui.UploadingChangesetSpecsSuccess(ids) + execUI.UploadingChangesetSpecsSuccess(ids) } else if len(repos) == 0 { - ui.NoChangesetSpecs() + execUI.NoChangesetSpecs() } - ui.CreatingBatchSpec() + execUI.CreatingBatchSpec() id, url, err := svc.CreateBatchSpec(ctx, namespace.ID, rawSpec, ids) if err != nil { - return ui.CreatingBatchSpecError(err) + return execUI.CreatingBatchSpecError(err) } previewURL := cfg.Endpoint + url - ui.CreatingBatchSpecSuccess(previewURL) + execUI.CreatingBatchSpecSuccess(previewURL) hasWorkspaceFiles := false for _, step := range batchSpec.Steps { @@ -496,26 +517,26 @@ func executeBatchSpec(ctx context.Context, ui ui.ExecUI, opts executeBatchSpecOp } } if hasWorkspaceFiles { - ui.UploadingWorkspaceFiles() + execUI.UploadingWorkspaceFiles() if err = svc.UploadBatchSpecWorkspaceFiles(ctx, batchSpecDir, string(id), batchSpec.Steps); err != nil { // Since failing to upload workspace files should not stop processing, just warn - ui.UploadingWorkspaceFilesWarning(errors.Wrap(err, "uploading workspace files")) + execUI.UploadingWorkspaceFilesWarning(errors.Wrap(err, "uploading workspace files")) } else { - ui.UploadingWorkspaceFilesSuccess() + execUI.UploadingWorkspaceFilesSuccess() } } if !opts.applyBatchSpec { - ui.PreviewBatchSpec(previewURL) + execUI.PreviewBatchSpec(previewURL) return } - ui.ApplyingBatchSpec() + execUI.ApplyingBatchSpec() batch, err := svc.ApplyBatchChange(ctx, id) if err != nil { return err } - ui.ApplyingBatchSpecSuccess(cfg.Endpoint + batch.URL) + execUI.ApplyingBatchSpecSuccess(cfg.Endpoint + batch.URL) return nil } @@ -617,11 +638,7 @@ func getBatchParallelism(ctx context.Context, flag int) (int, error) { return docker.NCPU(ctx) } -func validateSourcegraphVersionConstraint(ctx context.Context, svc *service.Service) error { - ffs, err := svc.DetermineFeatureFlags(ctx) - if err != nil { - return err - } +func validateSourcegraphVersionConstraint(ctx context.Context, ffs *batches.FeatureFlags) error { if ffs.Sourcegraph40 { return nil } diff --git a/cmd/src/batch_exec.go b/cmd/src/batch_exec.go index d4592ce139..11aadf39e3 100644 --- a/cmd/src/batch_exec.go +++ b/cmd/src/batch_exec.go @@ -37,6 +37,7 @@ type executorModeFlags struct { tempDir string repoDir string workspaceFilesDir string + binaryDiffs bool } func newExecutorModeFlags(flagSet *flag.FlagSet) (f *executorModeFlags) { @@ -47,6 +48,7 @@ func newExecutorModeFlags(flagSet *flag.FlagSet) (f *executorModeFlags) { flagSet.StringVar(&f.tempDir, "tmp", "", "Directory for storing temporary data.") flagSet.StringVar(&f.repoDir, "repo", "", "Path of the checked out repo on disk.") flagSet.StringVar(&f.workspaceFilesDir, "workspaceFiles", "", "Path of workspace files on disk.") + flagSet.BoolVar(&f.binaryDiffs, "binaryDiffs", false, "Whether to encode diffs as base64.") return f } @@ -121,7 +123,7 @@ Examples: } func executeBatchSpecInWorkspaces(ctx context.Context, flags *executorModeFlags) (err error) { - ui := &ui.JSONLines{} + ui := &ui.JSONLines{BinaryDiffs: flags.binaryDiffs} // Ensure the temp dir exists. tempDir := flags.tempDir @@ -211,6 +213,7 @@ func executeBatchSpecInWorkspaces(ctx context.Context, flags *executorModeFlags) RepoArchive: &repozip.NoopArchive{}, UI: taskExecUI.StepsExecutionUI(task), ForceRoot: !flags.runAsImageUser, + BinaryDiffs: flags.binaryDiffs, } results, err := executor.RunSteps(ctx, opts) diff --git a/cmd/src/batch_new.go b/cmd/src/batch_new.go index 577da8bd4b..75fe9a8891 100644 --- a/cmd/src/batch_new.go +++ b/cmd/src/batch_new.go @@ -48,7 +48,12 @@ Examples: Client: cfg.apiClient(apiFlags, flagSet.Output()), }) - if err := validateSourcegraphVersionConstraint(ctx, svc); err != nil { + ffs, err := svc.DetermineFeatureFlags(ctx) + if err != nil { + return err + } + + if err := validateSourcegraphVersionConstraint(ctx, ffs); err != nil { return err } diff --git a/cmd/src/batch_preview.go b/cmd/src/batch_preview.go index 088c58046e..8226bb5459 100644 --- a/cmd/src/batch_preview.go +++ b/cmd/src/batch_preview.go @@ -5,10 +5,7 @@ import ( "flag" "fmt" - "github.com/sourcegraph/src-cli/internal/batches/ui" "github.com/sourcegraph/src-cli/internal/cmderrors" - - "github.com/sourcegraph/sourcegraph/lib/output" ) func init() { @@ -45,15 +42,7 @@ Examples: ctx, cancel := contextCancelOnInterrupt(context.Background()) defer cancel() - var execUI ui.ExecUI - if flags.textOnly { - execUI = &ui.JSONLines{} - } else { - out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose}) - execUI = &ui.TUI{Out: out} - } - - if err = executeBatchSpec(ctx, execUI, executeBatchSpecOpts{ + if err = executeBatchSpec(ctx, executeBatchSpecOpts{ flags: flags, client: cfg.apiClient(flags.api, flagSet.Output()), file: file, diff --git a/cmd/src/batch_remote.go b/cmd/src/batch_remote.go index a5a1a57e1a..dbdcf1c264 100644 --- a/cmd/src/batch_remote.go +++ b/cmd/src/batch_remote.go @@ -52,7 +52,12 @@ Examples: Client: cfg.apiClient(flags.api, flagSet.Output()), }) - if err := validateSourcegraphVersionConstraint(ctx, svc); err != nil { + ffs, err := svc.DetermineFeatureFlags(ctx) + if err != nil { + return err + } + + if err := validateSourcegraphVersionConstraint(ctx, ffs); err != nil { return err } diff --git a/cmd/src/batch_repositories.go b/cmd/src/batch_repositories.go index b3b96d5891..99c64a8688 100644 --- a/cmd/src/batch_repositories.go +++ b/cmd/src/batch_repositories.go @@ -64,7 +64,12 @@ Examples: Client: client, }) - if err := validateSourcegraphVersionConstraint(ctx, svc); err != nil { + ffs, err := svc.DetermineFeatureFlags(ctx) + if err != nil { + return err + } + + if err := validateSourcegraphVersionConstraint(ctx, ffs); err != nil { return err } diff --git a/cmd/src/batch_validate.go b/cmd/src/batch_validate.go index e83ac3f553..b3268ea758 100644 --- a/cmd/src/batch_validate.go +++ b/cmd/src/batch_validate.go @@ -63,7 +63,12 @@ Examples: Client: cfg.apiClient(apiFlags, flagSet.Output()), }) - if err := validateSourcegraphVersionConstraint(ctx, svc); err != nil { + ffs, err := svc.DetermineFeatureFlags(ctx) + if err != nil { + return err + } + + if err := validateSourcegraphVersionConstraint(ctx, ffs); err != nil { ui.ExecutionError(err) return err } diff --git a/go.mod b/go.mod index 360787f63b..ca114bf010 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.18 require ( cloud.google.com/go/storage v1.28.0 - github.com/Masterminds/semver v1.5.0 github.com/creack/goselect v0.1.2 github.com/derision-test/glock v0.0.0-20210316032053-f5b74334bb29 github.com/dineshappavoo/basex v0.0.0-20170425072625-481a6f6dc663 @@ -23,7 +22,7 @@ require ( github.com/sourcegraph/go-diff v0.6.1 github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf github.com/sourcegraph/scip v0.2.3 - github.com/sourcegraph/sourcegraph/lib v0.0.0-20221128221602-4b48d5d69f5d + github.com/sourcegraph/sourcegraph/lib v0.0.0-20221129022201-2804729b069c github.com/stretchr/testify v1.8.0 golang.org/x/net v0.2.0 golang.org/x/sync v0.1.0 @@ -39,11 +38,9 @@ require ( cloud.google.com/go/compute/metadata v0.2.1 // indirect cloud.google.com/go/iam v0.5.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.15.0+incompatible // indirect github.com/alecthomas/chroma v0.10.0 // indirect - github.com/alecthomas/kingpin v2.2.6+incompatible // indirect - github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect - github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/aokoli/goutils v1.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bufbuild/buf v1.4.0 // indirect @@ -52,9 +49,7 @@ require ( github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/dave/jennifer v1.6.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/derision-test/go-mockgen v1.3.7 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect github.com/envoyproxy/protoc-gen-validate v0.3.0-java // indirect github.com/fatih/color v1.13.0 // indirect @@ -85,7 +80,6 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/microcosm-cc/bluemonday v1.0.17 // indirect - github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/go.sum b/go.sum index ca54f1a641..7d93f5e83c 100644 --- a/go.sum +++ b/go.sum @@ -27,12 +27,6 @@ github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqR github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= -github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= -github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -73,15 +67,11 @@ github.com/creack/goselect v0.1.2/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglD github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/dave/jennifer v1.6.0 h1:MQ/6emI2xM7wt0tJzJzyUik2Q3Tcn2eE0vtYgh4GPVI= -github.com/dave/jennifer v1.6.0/go.mod h1:AxTG893FiZKqxy3FP1kL80VMshSMuz2G+EgvszgGRnk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/derision-test/glock v0.0.0-20210316032053-f5b74334bb29 h1:O07j7ewg0eFHRx+7bxCA/7z86HJexl7HLY9kyVMmPDo= github.com/derision-test/glock v0.0.0-20210316032053-f5b74334bb29/go.mod h1:jKtLdBMrF+XQatqvg46wiWdDfDSSDjdhO4dOM2FX9H4= -github.com/derision-test/go-mockgen v1.3.7 h1:b/DXAXL2FkaRPpnbYK3ODdZzklmJAwox0tkc6yyXx74= -github.com/derision-test/go-mockgen v1.3.7/go.mod h1:/TXUePlhtHmDDCaDAi/a4g6xOHqMDz3Wf0r2NPGskB4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -298,8 +288,6 @@ github.com/microcosm-cc/bluemonday v1.0.17 h1:Z1a//hgsQ4yjC+8zEkV8IWySkXnsxmdSY6 github.com/microcosm-cc/bluemonday v1.0.17/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= -github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= @@ -390,10 +378,8 @@ github.com/sourcegraph/log v0.0.0-20220901143117-fc0516a694c9 h1:JjFyvx9hCD5+Jpu github.com/sourcegraph/log v0.0.0-20220901143117-fc0516a694c9/go.mod h1:UxiwB6C3xk3xOySJpW1R0MDUyfGuJRFS5Z8C+SA5p2I= github.com/sourcegraph/scip v0.2.3 h1:6nRsTlNTELn+1V7JxLrDVbBpb9H1gAbUF48N7x5B8Y4= github.com/sourcegraph/scip v0.2.3/go.mod h1:3D/RLCMvrrSam85RtFmC67SE/jhV9++hpCR5MTDzbA0= -github.com/sourcegraph/sourcegraph/lib v0.0.0-20221004162410-237d5855fa13 h1:HEj9QVz35nIU4plPb7iJSFjcTA14zVnfAdfYZF5jsKg= -github.com/sourcegraph/sourcegraph/lib v0.0.0-20221004162410-237d5855fa13/go.mod h1:e3v3msUOgthv/QCWUzhFdfR3+hwUSn37rtuw49IVJg4= -github.com/sourcegraph/sourcegraph/lib v0.0.0-20221128221602-4b48d5d69f5d h1:uIvIqhKNSuTH49fZcOh0fhJoKni/5MO3sQ/0zCfj/MM= -github.com/sourcegraph/sourcegraph/lib v0.0.0-20221128221602-4b48d5d69f5d/go.mod h1:tyHfYAVIPq+U6bCOTbIYra4uQ0gC7PQWA8BqmyF//Q0= +github.com/sourcegraph/sourcegraph/lib v0.0.0-20221129022201-2804729b069c h1:bjTcIdEx6FvUXIbD1Ckjp7SfvV4jTyJ7RozZH/HVj9M= +github.com/sourcegraph/sourcegraph/lib v0.0.0-20221129022201-2804729b069c/go.mod h1:PmWmo2ysXWCHcbtp08cDXKGD+8NQzrkj2UjdRBu9H0k= github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152 h1:z/MpntplPaW6QW95pzcAR/72Z5TWDyDnSo0EOcyij9o= github.com/sourcegraph/yaml v1.0.1-0.20200714132230-56936252f152/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -483,8 +469,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -510,8 +494,6 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= -golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -561,13 +543,9 @@ golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= -golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 h1:EH1Deb8WZJ0xc0WK//leUHXcX9aLE5SymusoTmMZye8= -golang.org/x/term v0.0.0-20220411215600-e5f449aeb171/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -599,8 +577,6 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211102182255-bb4add04ddef/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/api/version_check.go b/internal/api/version_check.go deleted file mode 100644 index dfc8e25f2c..0000000000 --- a/internal/api/version_check.go +++ /dev/null @@ -1,46 +0,0 @@ -package api - -import ( - "github.com/grafana/regexp" - - "github.com/Masterminds/semver" -) - -// NOTE: A version with a prerelease suffix (e.g. the "-rc.3" of "3.35.1-rc.3") is not -// considered by semver to satisfy a constraint without a prerelease suffix, regardless of -// whether or not the major/minor/patch version is greater than or equal to that of the -// constraint. -// -// For example, the version "3.35.1-rc.3" is not considered to satisfy the constraint ">= -// 3.23.0". This is likely not the expected outcome. However, the same version IS -// considered to satisfy the constraint "3.23.0-0". Thus, it is recommended to pass a -// constraint with a minimum prerelease version suffix attached if comparisons to -// prerelease versions are ever expected. See -// https://github.com/Masterminds/semver#working-with-prerelease-versions for more. -func CheckSourcegraphVersion(version, constraint, minDate string) (bool, error) { - if version == "dev" || version == "0.0.0+dev" { - return true, nil - } - - // Since we don't actually care about the abbreviated commit hash at the end of the - // version string, we match on 7 or more characters. Currently, the Sourcegraph version - // is expected to return 12: - // https://sourcegraph.com/github.com/sourcegraph/sourcegraph/-/blob/enterprise/dev/ci/internal/ci/config.go?L96. - buildDate := regexp.MustCompile(`^\d+_(\d{4}-\d{2}-\d{2})_[a-z0-9]{7,}$`) - matches := buildDate.FindStringSubmatch(version) - if len(matches) > 1 { - return matches[1] >= minDate, nil - } - - c, err := semver.NewConstraint(constraint) - if err != nil { - return false, nil - } - - v, err := semver.NewVersion(version) - if err != nil { - return false, err - } - - return c.Check(v), nil -} diff --git a/internal/api/version_check_test.go b/internal/api/version_check_test.go deleted file mode 100644 index fe8e0ee9f7..0000000000 --- a/internal/api/version_check_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package api - -import "testing" - -func TestCheckSourcegraphVersion(t *testing.T) { - for _, tc := range []struct { - currentVersion, constraint, minDate string - expected bool - }{ - { - currentVersion: "3.12.6", - constraint: ">= 3.12.6", - minDate: "2020-01-19", - expected: true, - }, - { - currentVersion: "3.12.6-rc.1", - constraint: ">= 3.12.6-0", - minDate: "2020-01-19", - expected: true, - }, - { - currentVersion: "3.12.6-rc.3", - constraint: ">= 3.10.6-0", - minDate: "2020-01-19", - expected: true, - }, - { - currentVersion: "3.12.6", - constraint: ">= 3.13", - minDate: "2020-01-19", - expected: false, - }, - { - currentVersion: "3.13.0", - constraint: ">= 3.13", - minDate: "2020-01-19", - expected: true, - }, - { - currentVersion: "dev", - constraint: ">= 3.13", - minDate: "2020-01-19", - expected: true, - }, - { - currentVersion: "0.0.0+dev", - constraint: ">= 3.13", - minDate: "2020-01-19", - expected: true, - }, - // 7-character abbreviated hash - { - currentVersion: "54959_2020-01-29_9258595", - minDate: "2020-01-19", - constraint: ">= 999.13", - expected: true, - }, - { - currentVersion: "54959_2020-01-29_9258595", - minDate: "2020-01-30", - constraint: ">= 999.13", - expected: false, - }, - { - currentVersion: "54959_2020-01-29_9258595", - minDate: "2020-01-29", - constraint: ">= 0.0", - expected: true, - }, - // 12-character abbreviated hash - { - currentVersion: "54959_2020-01-29_925859585436", - minDate: "2020-01-19", - constraint: ">= 999.13", - expected: true, - }, - { - currentVersion: "54959_2020-01-29_925859585436", - minDate: "2020-01-30", - constraint: ">= 999.13", - expected: false, - }, - { - currentVersion: "54959_2020-01-29_925859585436", - minDate: "2020-01-29", - constraint: ">= 0.0", - expected: true, - }, - // Full 40-character hash, just for fun - { - currentVersion: "54959_2020-01-29_7db7d396346284fd0f8f79f130f38b16fb1d3d70", - minDate: "2020-01-29", - constraint: ">= 0.0", - expected: true, - }, - } { - actual, err := CheckSourcegraphVersion(tc.currentVersion, tc.constraint, tc.minDate) - if err != nil { - t.Errorf("err: %s", err) - } - - if actual != tc.expected { - t.Errorf("wrong result. want=%t, got=%t (version=%q, constraint=%q)", tc.expected, actual, tc.currentVersion, tc.constraint) - } - } -} diff --git a/internal/batches/executor/coordinator.go b/internal/batches/executor/coordinator.go index d1c16e64b6..921d02ff81 100644 --- a/internal/batches/executor/coordinator.go +++ b/internal/batches/executor/coordinator.go @@ -29,9 +29,10 @@ type Coordinator struct { type NewCoordinatorOpts struct { ExecOpts NewExecutorOpts - Cache cache.Cache - Logger log.LogManager - GlobalEnv []string + Cache cache.Cache + Logger log.LogManager + GlobalEnv []string + BinaryDiffs bool IsRemote bool } @@ -89,7 +90,7 @@ func (c *Coordinator) checkCacheForTask(ctx context.Context, batchSpec *batchesl // add it to the list of specs that are displayed to the user and // send to the server. Instead, we can just report that the task is // complete and move on. - if task.CachedStepResult.Diff == "" { + if len(task.CachedStepResult.Diff) == 0 { return specs, true, nil } @@ -101,6 +102,10 @@ func (c *Coordinator) checkCacheForTask(ctx context.Context, batchSpec *batchesl } func (c *Coordinator) buildChangesetSpecs(task *Task, batchSpec *batcheslib.BatchSpec, result execution.AfterStepResult) ([]*batcheslib.ChangesetSpec, error) { + version := 1 + if c.opts.BinaryDiffs { + version = 2 + } input := &batcheslib.ChangesetSpecInput{ Repository: batcheslib.Repository{ ID: task.Repository.ID, @@ -115,13 +120,14 @@ func (c *Coordinator) buildChangesetSpecs(task *Task, batchSpec *batcheslib.Batc TransformChanges: batchSpec.TransformChanges, Result: execution.AfterStepResult{ + Version: version, Diff: result.Diff, ChangedFiles: result.ChangedFiles, Outputs: result.Outputs, }, } - return batcheslib.BuildChangesetSpecs(input) + return batcheslib.BuildChangesetSpecs(input, c.opts.BinaryDiffs) } func (c *Coordinator) loadCachedStepResults(ctx context.Context, task *Task, globalEnv []string) error { @@ -155,7 +161,7 @@ func (c *Coordinator) buildSpecs(ctx context.Context, batchSpec *batcheslib.Batc // If the steps didn't result in any diff, we don't need to create a // changeset spec that's displayed to the user and send to the server. - if lastStepResult.Diff == "" { + if len(lastStepResult.Diff) == 0 { return nil, nil } diff --git a/internal/batches/executor/coordinator_test.go b/internal/batches/executor/coordinator_test.go index 5ac48da62d..9ce252cda3 100644 --- a/internal/batches/executor/coordinator_test.go +++ b/internal/batches/executor/coordinator_test.go @@ -39,10 +39,11 @@ func TestCoordinator_Execute(t *testing.T) { Body: testChangesetTemplate.Body, Commits: []batcheslib.GitCommitDescription{ { + Version: 2, Message: testChangesetTemplate.Commit.Message, AuthorName: testChangesetTemplate.Commit.Author.Name, AuthorEmail: testChangesetTemplate.Commit.Author.Email, - Diff: `dummydiff1`, + Diff: []byte(`dummydiff1`), }, }, Published: batcheslib.PublishedValue{Val: false}, @@ -78,8 +79,8 @@ func TestCoordinator_Execute(t *testing.T) { executor: &dummyExecutor{ results: []taskResult{ - {task: srcCLITask, stepResults: []execution.AfterStepResult{{Diff: `dummydiff1`}}}, - {task: sourcegraphTask, stepResults: []execution.AfterStepResult{{Diff: `dummydiff2`}}}, + {task: srcCLITask, stepResults: []execution.AfterStepResult{{Version: 2, Diff: []byte(`dummydiff1`)}}}, + {task: sourcegraphTask, stepResults: []execution.AfterStepResult{{Version: 2, Diff: []byte(`dummydiff2`)}}}, }, }, opts: NewCoordinatorOpts{}, @@ -87,10 +88,10 @@ func TestCoordinator_Execute(t *testing.T) { wantCacheEntries: 2, wantSpecs: []*batcheslib.ChangesetSpec{ buildSpecFor(testRepo1, func(spec *batcheslib.ChangesetSpec) { - spec.Commits[0].Diff = `dummydiff1` + spec.Commits[0].Diff = []byte(`dummydiff1`) }), buildSpecFor(testRepo2, func(spec *batcheslib.ChangesetSpec) { - spec.Commits[0].Diff = `dummydiff2` + spec.Commits[0].Diff = []byte(`dummydiff2`) }), }, }, @@ -133,7 +134,8 @@ func TestCoordinator_Execute(t *testing.T) { { task: srcCLITask, stepResults: []execution.AfterStepResult{{ - Diff: `dummydiff1`, + Version: 2, + Diff: []byte(`dummydiff1`), Outputs: map[string]interface{}{ "output1": "myOutputValue1", "output2": map[string]interface{}{ @@ -171,10 +173,11 @@ func TestCoordinator_Execute(t *testing.T) { batch_change_description=the description` spec.Commits = []batcheslib.GitCommitDescription{ { + Version: 2, Message: "output1=myOutputValue1,output2=subFieldValue", AuthorName: "output1=myOutputValue1", AuthorEmail: "output1=myOutputValue1", - Diff: `dummydiff1`, + Diff: []byte(`dummydiff1`), }, } }), @@ -197,8 +200,8 @@ func TestCoordinator_Execute(t *testing.T) { executor: &dummyExecutor{ results: []taskResult{ - {task: srcCLITask, stepResults: []execution.AfterStepResult{{Diff: nestedChangesDiff}}}, - {task: sourcegraphTask, stepResults: []execution.AfterStepResult{{Diff: nestedChangesDiff}}}, + {task: srcCLITask, stepResults: []execution.AfterStepResult{{Version: 2, Diff: nestedChangesDiff}}}, + {task: sourcegraphTask, stepResults: []execution.AfterStepResult{{Version: 2, Diff: nestedChangesDiff}}}, }, }, opts: NewCoordinatorOpts{}, @@ -210,19 +213,19 @@ func TestCoordinator_Execute(t *testing.T) { wantSpecs: []*batcheslib.ChangesetSpec{ buildSpecFor(testRepo1, func(spec *batcheslib.ChangesetSpec) { spec.HeadRef = "refs/heads/" + testChangesetTemplate.Branch - spec.Commits[0].Diff = nestedChangesDiffSubdirA + nestedChangesDiffSubdirB + spec.Commits[0].Diff = []byte(nestedChangesDiffSubdirA + nestedChangesDiffSubdirB) }), buildSpecFor(testRepo2, func(spec *batcheslib.ChangesetSpec) { spec.HeadRef = "refs/heads/in-directory-b" - spec.Commits[0].Diff = nestedChangesDiffSubdirB + nestedChangesDiffSubdirC + spec.Commits[0].Diff = []byte(nestedChangesDiffSubdirB + nestedChangesDiffSubdirC) }), buildSpecFor(testRepo1, func(spec *batcheslib.ChangesetSpec) { spec.HeadRef = "refs/heads/in-directory-c" - spec.Commits[0].Diff = nestedChangesDiffSubdirC + spec.Commits[0].Diff = []byte(nestedChangesDiffSubdirC) }), buildSpecFor(testRepo2, func(spec *batcheslib.ChangesetSpec) { spec.HeadRef = util.EnsureRefPrefix(testChangesetTemplate.Branch) - spec.Commits[0].Diff = nestedChangesDiffSubdirA + spec.Commits[0].Diff = []byte(nestedChangesDiffSubdirA) }), }, }, @@ -245,8 +248,8 @@ func TestCoordinator_Execute(t *testing.T) { executor: &dummyExecutor{ results: []taskResult{ - {task: srcCLITask, stepResults: []execution.AfterStepResult{{Diff: `dummydiff1`, StepIndex: 0}}}, - {task: sourcegraphTask, stepResults: []execution.AfterStepResult{{Diff: `dummydiff2`, StepIndex: 0}}}, + {task: srcCLITask, stepResults: []execution.AfterStepResult{{Version: 2, Diff: []byte(`dummydiff1`), StepIndex: 0}}}, + {task: sourcegraphTask, stepResults: []execution.AfterStepResult{{Version: 2, Diff: []byte(`dummydiff2`), StepIndex: 0}}}, }, }, opts: NewCoordinatorOpts{}, @@ -254,10 +257,10 @@ func TestCoordinator_Execute(t *testing.T) { wantCacheEntries: 2, wantSpecs: []*batcheslib.ChangesetSpec{ buildSpecFor(testRepo1, func(spec *batcheslib.ChangesetSpec) { - spec.Commits[0].Diff = `dummydiff1` + spec.Commits[0].Diff = []byte(`dummydiff1`) }), buildSpecFor(testRepo2, func(spec *batcheslib.ChangesetSpec) { - spec.Commits[0].Diff = `dummydiff2` + spec.Commits[0].Diff = []byte(`dummydiff2`) }), }, }, @@ -280,6 +283,7 @@ func TestCoordinator_Execute(t *testing.T) { tc.opts.Cache = c tc.opts.Logger = logManager + tc.opts.BinaryDiffs = true coord := Coordinator{ exec: tc.executor, @@ -369,9 +373,9 @@ func TestCoordinator_Execute_StepCaching(t *testing.T) { executor.results = []taskResult{{ task: task, stepResults: []execution.AfterStepResult{ - {StepIndex: 0, Diff: `step-0-diff`}, - {StepIndex: 1, Diff: `step-1-diff`}, - {StepIndex: 2, Diff: `step-2-diff`}, + {Version: 2, StepIndex: 0, Diff: []byte(`step-0-diff`)}, + {Version: 2, StepIndex: 1, Diff: []byte(`step-1-diff`)}, + {Version: 2, StepIndex: 2, Diff: []byte(`step-2-diff`)}, }, }} @@ -660,4 +664,4 @@ index 7f96c22..43df362 100644 +var c = 3 ` -const nestedChangesDiff = nestedChangesDiffSubdirA + nestedChangesDiffSubdirB + nestedChangesDiffSubdirC +var nestedChangesDiff = []byte(nestedChangesDiffSubdirA + nestedChangesDiffSubdirB + nestedChangesDiffSubdirC) diff --git a/internal/batches/executor/execution_cache_test.go b/internal/batches/executor/execution_cache_test.go index 0b1ff71877..d5ff754ffc 100644 --- a/internal/batches/executor/execution_cache_test.go +++ b/internal/batches/executor/execution_cache_test.go @@ -29,7 +29,7 @@ var cacheRepo2 = batches.Repository{ FileMatches: []string{"main.go"}, } -const testDiff = `diff --git a/README.md b/README.md +var testDiff = []byte(`diff --git a/README.md b/README.md new file mode 100644 index 0000000..3363c39 --- /dev/null @@ -38,7 +38,7 @@ index 0000000..3363c39 +# README + +This is the readme -` +`) func TestExecutionDiskCache_GetSet(t *testing.T) { ctx := context.Background() @@ -58,7 +58,8 @@ func TestExecutionDiskCache_GetSet(t *testing.T) { } value := execution.AfterStepResult{ - Diff: testDiff, + Version: 2, + Diff: testDiff, ChangedFiles: git.Changes{ Added: []string{"README.md"}, }, diff --git a/internal/batches/executor/executor.go b/internal/batches/executor/executor.go index 657e79e7e5..9ccb94a8fc 100644 --- a/internal/batches/executor/executor.go +++ b/internal/batches/executor/executor.go @@ -69,6 +69,8 @@ type NewExecutorOpts struct { IsRemote bool GlobalEnv []string ForceRoot bool + + BinaryDiffs bool } type executor struct { @@ -180,6 +182,7 @@ func (x *executor) do(ctx context.Context, task *Task, ui TaskExecutionUI) (err RepoArchive: repoArchive, WorkingDirectory: x.opts.WorkingDirectory, ForceRoot: x.opts.ForceRoot, + BinaryDiffs: x.opts.BinaryDiffs, UI: ui.StepsExecutionUI(task), } diff --git a/internal/batches/executor/executor_test.go b/internal/batches/executor/executor_test.go index 2a6040ffdc..c4accc0f87 100644 --- a/internal/batches/executor/executor_test.go +++ b/internal/batches/executor/executor_test.go @@ -455,7 +455,7 @@ func TestExecutor_Integration(t *testing.T) { lastStepResult := taskResult.stepResults[len(taskResult.stepResults)-1] - fileDiffs, err := diff.ParseMultiFileDiff([]byte(lastStepResult.Diff)) + fileDiffs, err := diff.ParseMultiFileDiff(lastStepResult.Diff) if err != nil { t.Fatalf("failed to parse diff: %s", err) } @@ -517,14 +517,14 @@ func TestExecutor_CachedStepResults(t *testing.T) { }, } - cachedDiff := `diff --git README.md README.md + cachedDiff := []byte(`diff --git README.md README.md index 02a19af..c9644dd 100644 --- README.md +++ README.md @@ -1 +1,2 @@ # Welcome to the README +foobar -` +`) task := &Task{ BatchChangeAttributes: &template.BatchChangeAttributes{}, @@ -533,6 +533,7 @@ index 02a19af..c9644dd 100644 }, CachedStepResultFound: true, CachedStepResult: execution.AfterStepResult{ + Version: 2, StepIndex: 0, Diff: cachedDiff, Outputs: map[string]interface{}{}, @@ -579,7 +580,7 @@ This repository is used to test opening and closing pull request with Automation }, } - cachedDiff := `diff --git README.md README.md + cachedDiff := []byte(`diff --git README.md README.md index 1914491..cd2ccbf 100644 --- README.md +++ README.md @@ -598,9 +599,9 @@ index 0000000..888e1ec +++ README.txt @@ -0,0 +1 @@ +this is step 1 -` +`) - wantFinalDiff := `diff --git README.md README.md + wantFinalDiff := []byte(`diff --git README.md README.md index 1914491..d6782d3 100644 --- README.md +++ README.md @@ -628,7 +629,7 @@ index 0000000..257ae8e +++ my-output.txt @@ -0,0 +1 @@ +this is step 5 -` +`) task := &Task{ Repository: testRepo1, @@ -648,6 +649,7 @@ echo "previous_step.modified_files=${{ previous_step.modified_files }}" >> READM }, CachedStepResultFound: true, CachedStepResult: execution.AfterStepResult{ + Version: 2, StepIndex: 2, Diff: cachedDiff, Outputs: map[string]interface{}{ @@ -705,7 +707,7 @@ This repository is used to test opening and closing pull request with Automation }, } - wantFinalDiff := `diff --git README.md README.md + wantFinalDiff := []byte(`diff --git README.md README.md index 3040106..5f2f924 100644 --- README.md +++ README.md @@ -713,7 +715,7 @@ index 3040106..5f2f924 100644 # automation-testing This repository is used to test opening and closing pull request with Automation +hello world -` +`) task := &Task{ Repository: testRepo1, @@ -724,8 +726,9 @@ index 3040106..5f2f924 100644 }, CachedStepResultFound: true, CachedStepResult: execution.AfterStepResult{ + Version: 2, StepIndex: 0, - Diff: "", + Diff: []byte(""), Outputs: map[string]interface{}{}, ChangedFiles: git.Changes{}, Stdout: "hello world", diff --git a/internal/batches/executor/run_steps.go b/internal/batches/executor/run_steps.go index b4911640f9..848362a3b3 100644 --- a/internal/batches/executor/run_steps.go +++ b/internal/batches/executor/run_steps.go @@ -54,6 +54,8 @@ type RunStepsOpts struct { // ForceRoot forces Docker containers to be run as root:root, rather than // whatever the image's default user and group are. ForceRoot bool + + BinaryDiffs bool } func RunSteps(ctx context.Context, opts *RunStepsOpts) (stepResults []execution.AfterStepResult, err error) { @@ -120,8 +122,8 @@ func RunSteps(ctx context.Context, opts *RunStepsOpts) (stepResults []execution. // If the previous steps made any modifications to the workspace yet, // apply them. - if opts.Task.CachedStepResult.Diff != "" { - if err := ws.ApplyDiff(ctx, []byte(opts.Task.CachedStepResult.Diff)); err != nil { + if len(opts.Task.CachedStepResult.Diff) > 0 { + if err := ws.ApplyDiff(ctx, opts.Task.CachedStepResult.Diff); err != nil { return nil, errors.Wrap(err, "applying diff of cache result") } } @@ -195,12 +197,17 @@ func RunSteps(ctx context.Context, opts *RunStepsOpts) (stepResults []execution. return stepResults, errors.Wrap(err, "getting changed files in step") } + version := 1 + if opts.BinaryDiffs { + version = 2 + } stepResult := execution.AfterStepResult{ + Version: version, ChangedFiles: changes, Stdout: stdoutBuffer.String(), Stderr: stderrBuffer.String(), StepIndex: i, - Diff: string(stepDiff), + Diff: stepDiff, // Those will be set below. Outputs: make(map[string]interface{}), } diff --git a/internal/batches/executor/ui.go b/internal/batches/executor/ui.go index 38c24340bb..547a2fb1f1 100644 --- a/internal/batches/executor/ui.go +++ b/internal/batches/executor/ui.go @@ -45,7 +45,7 @@ type StepsExecutionUI interface { StepOutputWriter(context.Context, *Task, int) StepOutputWriter - StepFinished(idx int, diff string, changes git.Changes, outputs map[string]interface{}) + StepFinished(idx int, diff []byte, changes git.Changes, outputs map[string]interface{}) StepFailed(idx int, err error, exitCode int) } @@ -65,7 +65,7 @@ func (noop NoopStepsExecUI) StepStarted(step int, runScript string, env map[stri func (noop NoopStepsExecUI) StepOutputWriter(ctx context.Context, task *Task, step int) StepOutputWriter { return NoopStepOutputWriter{} } -func (noop NoopStepsExecUI) StepFinished(idx int, diff string, changes git.Changes, outputs map[string]interface{}) { +func (noop NoopStepsExecUI) StepFinished(idx int, diff []byte, changes git.Changes, outputs map[string]interface{}) { } func (noop NoopStepsExecUI) StepFailed(idx int, err error, exitCode int) { } diff --git a/internal/batches/features.go b/internal/batches/features.go index 0a020b6bc6..ee319709b7 100644 --- a/internal/batches/features.go +++ b/internal/batches/features.go @@ -1,15 +1,15 @@ package batches import ( + "github.com/sourcegraph/sourcegraph/lib/api" "github.com/sourcegraph/sourcegraph/lib/errors" - - "github.com/sourcegraph/src-cli/internal/api" ) // FeatureFlags represent features that are only available on certain // Sourcegraph versions and we therefore have to detect at runtime. type FeatureFlags struct { Sourcegraph40 bool + BinaryDiffs bool } func (ff *FeatureFlags) SetFromVersion(version string) error { @@ -28,6 +28,7 @@ func (ff *FeatureFlags) SetFromVersion(version string) error { // Example usage: // {&ff.FlagName, ">= 3.23.0-0", "2020-11-24"}, {&ff.Sourcegraph40, ">= 4.0.0-0", "2022-08-24"}, + {&ff.BinaryDiffs, ">= 4.3.0-0", "2022-11-29"}, } { value, err := api.CheckSourcegraphVersion(version, feature.constraint, feature.minDate) if err != nil { diff --git a/internal/batches/ui/json_lines.go b/internal/batches/ui/json_lines.go index f584a25a51..dea8e2d0ee 100644 --- a/internal/batches/ui/json_lines.go +++ b/internal/batches/ui/json_lines.go @@ -23,7 +23,9 @@ import ( var _ ExecUI = &JSONLines{} -type JSONLines struct{} +type JSONLines struct { + BinaryDiffs bool +} func (ui *JSONLines) ParsingBatchSpec() { logOperationStart(batcheslib.LogEventOperationParsingBatchSpec, &batcheslib.ParsingBatchSpecMetadata{}) @@ -89,7 +91,9 @@ func (ui *JSONLines) CheckingCacheSuccess(cachedSpecsFound int, tasksToExecute i } func (ui *JSONLines) ExecutingTasks(_ bool, _ int) executor.TaskExecutionUI { - return &taskExecutionJSONLines{} + return &taskExecutionJSONLines{ + binaryDiffs: ui.BinaryDiffs, + } } func (ui *JSONLines) ExecutingTasksSkippingErrors(err error) { @@ -174,7 +178,8 @@ func (ui *JSONLines) WriteAfterStepResult(key string, value execution.AfterStepR } type taskExecutionJSONLines struct { - linesTasks map[*executor.Task]batcheslib.JSONLinesTask + linesTasks map[*executor.Task]batcheslib.JSONLinesTask + binaryDiffs bool } // seededRand is used in randomID() to generate a "random" number. @@ -256,7 +261,8 @@ func (ui *taskExecutionJSONLines) StepsExecutionUI(task *executor.Task) executor } type stepsExecutionJSONLines struct { - linesTask *batcheslib.JSONLinesTask + linesTask *batcheslib.JSONLinesTask + binaryDiffs bool } const stepFlushDuration = 500 * time.Millisecond @@ -294,7 +300,14 @@ func (ui *stepsExecutionJSONLines) StepPreparingFailed(step int, err error) { } func (ui *stepsExecutionJSONLines) StepStarted(step int, runScript string, env map[string]string) { - logOperationStart(batcheslib.LogEventOperationTaskStep, &batcheslib.TaskStepMetadata{TaskID: ui.linesTask.ID, Step: step, Env: env}) + logOperationStart( + batcheslib.LogEventOperationTaskStep, + &batcheslib.TaskStepMetadata{ + Version: version(ui.binaryDiffs), + TaskID: ui.linesTask.ID, + Step: step, + Env: env, + }) } func (ui *stepsExecutionJSONLines) StepOutputWriter(ctx context.Context, task *executor.Task, step int) executor.StepOutputWriter { @@ -302,19 +315,21 @@ func (ui *stepsExecutionJSONLines) StepOutputWriter(ctx context.Context, task *e logOperationProgress( batcheslib.LogEventOperationTaskStep, &batcheslib.TaskStepMetadata{ - TaskID: ui.linesTask.ID, - Step: step, - Out: data, + Version: version(ui.binaryDiffs), + TaskID: ui.linesTask.ID, + Step: step, + Out: data, }, ) } return NewIntervalProcessWriter(ctx, stepFlushDuration, sink) } -func (ui *stepsExecutionJSONLines) StepFinished(step int, diff string, changes git.Changes, outputs map[string]interface{}) { +func (ui *stepsExecutionJSONLines) StepFinished(step int, diff []byte, changes git.Changes, outputs map[string]interface{}) { logOperationSuccess( batcheslib.LogEventOperationTaskStep, &batcheslib.TaskStepMetadata{ + Version: version(ui.binaryDiffs), TaskID: ui.linesTask.ID, Step: step, Diff: diff, @@ -327,6 +342,7 @@ func (ui *stepsExecutionJSONLines) StepFailed(step int, err error, exitCode int) logOperationFailure( batcheslib.LogEventOperationTaskStep, &batcheslib.TaskStepMetadata{ + Version: version(ui.binaryDiffs), TaskID: ui.linesTask.ID, Step: step, Error: err.Error(), @@ -370,3 +386,10 @@ func logEvent(e batcheslib.LogEvent) { fmt.Fprintln(os.Stderr, err) } } + +func version(binaryDiffs bool) int { + if binaryDiffs { + return 2 + } + return 1 +} diff --git a/internal/batches/ui/task_exec_tui.go b/internal/batches/ui/task_exec_tui.go index 606396ef89..43243324ed 100644 --- a/internal/batches/ui/task_exec_tui.go +++ b/internal/batches/ui/task_exec_tui.go @@ -297,7 +297,7 @@ func (ui *taskExecTUI) TaskChangesetSpecsBuilt(task *executor.Task, specs []*bat var fileDiffs []*diff.FileDiff for _, spec := range specs { - fd, err := diff.ParseMultiFileDiff([]byte(spec.Commits[0].Diff)) + fd, err := diff.ParseMultiFileDiff(spec.Commits[0].Diff) if err != nil { ui.progress.Verbosef("%-*s failed to display status: %s", ui.maxRepoName, ts.displayName, err) return @@ -471,7 +471,7 @@ func (ui stepsExecTUI) StepOutputWriter(ctx context.Context, task *executor.Task return executor.NoopStepOutputWriter{} } -func (ui stepsExecTUI) StepFinished(idx int, diff string, changes git.Changes, outputs map[string]interface{}) { +func (ui stepsExecTUI) StepFinished(idx int, diff []byte, changes git.Changes, outputs map[string]interface{}) { // noop right now } func (ui stepsExecTUI) StepFailed(idx int, err error, exitCode int) { diff --git a/internal/batches/ui/task_exec_tui_test.go b/internal/batches/ui/task_exec_tui_test.go index 572155f11b..e27c671898 100644 --- a/internal/batches/ui/task_exec_tui_test.go +++ b/internal/batches/ui/task_exec_tui_test.go @@ -14,7 +14,7 @@ import ( "github.com/sourcegraph/src-cli/internal/batches/graphql" ) -const progressPrinterDiff = `diff --git README.md README.md +var progressPrinterDiff = []byte(`diff --git README.md README.md new file mode 100644 index 0000000..3363c39 --- /dev/null @@ -37,7 +37,7 @@ index 627c2ae..88f1836 100644 @@ -1 +1 @@ -this is x +this is x (or is it?) -` +`) func TestTaskExecTUI_Integration(t *testing.T) { if runtime.GOOS == "windows" { @@ -153,6 +153,7 @@ func TestTaskExecTUI_Integration(t *testing.T) { Body: "This is my batch change", Commits: []batcheslib.GitCommitDescription{ { + Version: 2, Message: "This is my batch change", Diff: progressPrinterDiff, },