Create initial config file if not on win/mac (managed by Desktop) (#140)

Move configuration options to main provider source file
Add containerized Snyk provider
Run containerized Snyk version if os is linux and no local version of Snyk binary detected/defined
Add existing Snyk config file in the container before doing Auth
Add end to end tests for containerized Snyk provider
Use image sha for containerized Snyk provider
Use Docker CLI client to manage container in containerized Snyk provider
Create initial config file if not on win/mac (managed by Desktop)

Signed-off-by: Guillaume Tardif <guillaume.tardif@gmail.com>
Co-authored-by: Guillaume Lours <guillaume.lours@docker.com>

Author: gtardif

.github/workflows/build-pr.yml

@@ -55,6 +55,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
+      - name: Download binaries
+        run: make -f builder.Makefile download
+
+      - name: Build binary and run tests
+        run: make TAG_NAME=${{ github.event.inputs.tag }} -f builder.Makefile build e2e
+
       - name: Build Cross
         run: make cross
 

Makefile

@@ -19,7 +19,8 @@ BUILD_ARGS := --build-arg GO_VERSION=$(GO_VERSION)\
 	--build-arg ALPINE_VERSION=$(ALPINE_VERSION)\
 	--build-arg GOLANGCI_LINT_VERSION=$(GOLANGCI_LINT_VERSION) \
 	--build-arg TAG_NAME=$(GIT_TAG_NAME) \
-	--build-arg GOTESTSUM_VERSION=$(GOTESTSUM_VERSION)
+	--build-arg GOTESTSUM_VERSION=$(GOTESTSUM_VERSION) \
+	--build-arg SNYK_IMAGE_DIGEST=$(SNYK_IMAGE_DIGEST)
 
 E2E_ENV := --env E2E_TEST_AUTH_TOKEN \
            --env E2E_HUB_URL \

builder.Makefile

@@ -14,7 +14,8 @@ PKG_NAME=github.com/docker/scan-cli-plugin
 STATIC_FLAGS= CGO_ENABLED=0
 LDFLAGS := "-s -w \
   -X $(PKG_NAME)/internal.GitCommit=$(COMMIT) \
-  -X $(PKG_NAME)/internal.Version=$(TAG_NAME)"
+  -X $(PKG_NAME)/internal.Version=$(TAG_NAME) \
+  -X $(PKG_NAME)/internal/provider.ImageDigest=$(SNYK_IMAGE_DIGEST)"
 GO_BUILD = $(STATIC_FLAGS) go build -trimpath -ldflags=$(LDFLAGS)
 
 SNYK_DOWNLOAD_NAME:=snyk-linux

cmd/docker-scan/main.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
+	"runtime"
 	"syscall"
 
 	"github.com/docker/cli/cli-plugins/manager"
@@ -105,13 +106,13 @@ func newScanCmd(ctx context.Context, dockerCli command.Cli) *cobra.Command {
 	return cmd
 }
 
-func configureProvider(ctx context.Context, dockerCli command.Streams, flags options, options ...provider.SnykProviderOps) (provider.Provider, error) {
+func configureProvider(ctx context.Context, dockerCli command.Cli, flags options, options ...provider.Ops) (provider.Provider, error) {
 	conf, err := checkConsent(flags, dockerCli)
 	if err != nil {
 		return nil, err
 	}
 
-	opts := []provider.SnykProviderOps{
+	opts := []provider.Ops{
 		provider.WithContext(ctx),
 		provider.WithPath(conf.Path),
 	}
@@ -141,7 +142,14 @@ func configureProvider(ctx context.Context, dockerCli command.Streams, flags opt
 		}
 		opts = append(opts, provider.WithSeverity(flags.severity))
 	}
-	return provider.NewSnykProvider(opts...)
+	defaultProvider, err := provider.NewProvider(opts...)
+	if err != nil {
+		return nil, err
+	}
+	if runtime.GOOS == "linux" && !provider.UseExternalBinary(defaultProvider) {
+		return provider.NewDockerSnykProvider(dockerCli, defaultProvider)
+	}
+	return provider.NewSnykProvider(defaultProvider)
 }
 
 func checkConsent(flags options, dockerCli command.Streams) (config.Config, error) {
@@ -172,7 +180,7 @@ func checkConsent(flags options, dockerCli command.Streams) (config.Config, erro
 	return conf, nil
 }
 
-func runVersion(ctx context.Context, dockerCli command.Streams, flags options) error {
+func runVersion(ctx context.Context, dockerCli command.Cli, flags options) error {
 	scanProvider, err := configureProvider(ctx, dockerCli, flags)
 	if err != nil {
 		return err
@@ -186,7 +194,7 @@ func runVersion(ctx context.Context, dockerCli command.Streams, flags options) e
 	return nil
 }
 
-func runAuthentication(ctx context.Context, dockerCli command.Streams, flags options, args []string) error {
+func runAuthentication(ctx context.Context, dockerCli command.Cli, flags options, args []string) error {
 	if len(args) != 0 {
 		return fmt.Errorf(`--login flag expects no argument`)
 	}
@@ -211,8 +219,8 @@ func runScan(ctx context.Context, cmd *cobra.Command, dockerCli command.Cli, fla
 		return err
 	}
 	err = scanProvider.Scan(args[0])
-	if exitError, ok := err.(*exec.ExitError); ok {
-		os.Exit(exitError.ExitCode())
+	if _, ok := err.(*exec.ExitError); ok {
+		os.Exit(1)
 	}
 	return err
 }

config/config.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 
 	"github.com/pkg/errors"
 
@@ -38,6 +39,15 @@ type Config struct {
 func ReadConfigFile() (Config, error) {
 	var conf Config
 	path := filepath.Join(cliConfig.Dir(), "scan", "config.json")
+	if runtime.GOOS != "windows" && runtime.GOOS != "darwin" {
+		_, err := os.Stat(path)
+		if err != nil && os.IsNotExist(err) {
+			err := SaveConfigFile(Config{})
+			if err != nil {
+				return conf, errors.Wrapf(err, "failed to create initial scan configuration file %q", path)
+			}
+		}
+	}
 	buf, err := ioutil.ReadFile(path)
 	if err != nil {
 		_ = os.Remove(path)

e2e/auth_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -29,6 +30,9 @@ import (
 )
 
 func TestSnykAuthentication(t *testing.T) {
+	if runtime.GOOS != "darwin" && runtime.GOOS != "windows" {
+		t.Skip("invalid test: only on Docker Desktop")
+	}
 	// Add snyk binary to the path
 	path := os.Getenv("PATH")
 	defer env.Patch(t, "PATH", fmt.Sprintf(pathFormat(), os.Getenv("SNYK_DESKTOP_PATH"), path))()
@@ -50,7 +54,7 @@ func TestSnykAuthentication(t *testing.T) {
 	// snyk config file should be updated
 	buff, err := ioutil.ReadFile(homeDir.Join(".config", "configstore", "snyk.json"))
 	assert.NilError(t, err)
-	assert.Assert(t, strings.Contains(string(buff), token))
+	assert.Assert(t, strings.Contains(string(buff), token), string(buff))
 }
 
 func TestAuthenticationFlagFailsWithImage(t *testing.T) {
@@ -78,3 +82,61 @@ func TestAuthenticationChecksToken(t *testing.T) {
 		Err:      `invalid authentication token "invalid-token"`,
 	})
 }
+
+func TestAuthWithContainerizedSnyk(t *testing.T) {
+	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
+		t.Skip("invalid test on Docker Desktop")
+	}
+	cmd, configDir, cleanup := dockerCli.createTestCommand(false)
+	defer cleanup()
+	createScanConfigFileOptinAndPath(t, configDir, true, "")
+
+	// create Snyk config directories without the config file
+	homeDir, cleanFunction := createSnykConfDirectories(t, false, "")
+	defer cleanFunction()
+
+	token := os.Getenv("E2E_TEST_AUTH_TOKEN")
+	assert.Assert(t, token != "", "E2E_TEST_AUTH_TOKEN needs to be filled")
+
+	cmd.Command = dockerCli.Command("scan", "--accept-license", "--login", "--token", token)
+	cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", homeDir.Path()))
+	icmd.RunCmd(cmd).Assert(t, icmd.Success)
+
+	// snyk config file should be created
+	buff, err := ioutil.ReadFile(homeDir.Join(".config", "configstore", "snyk.json"))
+	assert.NilError(t, err)
+	assert.Assert(t, strings.Contains(string(buff), token))
+}
+
+func TestAuthWithContainerSnykFlagFailsWithImage(t *testing.T) {
+	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
+		t.Skip("invalid test on Docker Desktop")
+	}
+	cmd, configDir, cleanup := dockerCli.createTestCommand(false)
+	defer cleanup()
+	createScanConfigFileOptinAndPath(t, configDir, true, "")
+
+	token := os.Getenv("E2E_TEST_AUTH_TOKEN")
+	assert.Assert(t, token != "", "E2E_TEST_AUTH_TOKEN needs to be filled")
+
+	cmd.Command = dockerCli.Command("scan", "--accept-license", "--login", "--token", token, "example:image")
+	icmd.RunCmd(cmd).Assert(t, icmd.Expected{
+		ExitCode: 1,
+		Err:      "--login flag expects no argument",
+	})
+}
+
+func TestAuthWithContainerSnykChecksToken(t *testing.T) {
+	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
+		t.Skip("invalid test on Docker Desktop")
+	}
+	cmd, configDir, cleanup := dockerCli.createTestCommand(false)
+	defer cleanup()
+	createScanConfigFileOptinAndPath(t, configDir, true, "")
+
+	cmd.Command = dockerCli.Command("scan", "--accept-license", "--login", "--token", "invalid-token")
+	icmd.RunCmd(cmd).Assert(t, icmd.Expected{
+		ExitCode: 1,
+		Err:      `invalid authentication token "invalid-token"`,
+	})
+}

e2e/main_test.go

@@ -38,6 +38,10 @@ type dockerCliCommand struct {
 }
 
 func (d dockerCliCommand) createTestCmd() (icmd.Cmd, string, func()) {
+	return d.createTestCommand(true)
+}
+
+func (d dockerCliCommand) createTestCommand(withSnykBinary bool) (icmd.Cmd, string, func()) {
 	configDir, err := ioutil.TempDir("", "config")
 	if err != nil {
 		panic(err)
@@ -46,7 +50,9 @@ func (d dockerCliCommand) createTestCmd() (icmd.Cmd, string, func()) {
 		panic(err)
 	}
 	sourceDir := os.Getenv("SNYK_DESKTOP_PATH")
-	copyBinary("snyk", sourceDir, filepath.Join(configDir, "scan"))
+	if withSnykBinary {
+		copyBinary("snyk", sourceDir, filepath.Join(configDir, "scan"))
+	}
 
 	configFilePath := filepath.Join(configDir, "config.json")
 	dockerConfig := dockerConfigFile.ConfigFile{

e2e/scan_test.go

@@ -97,12 +97,13 @@ func TestScanSucceedWithDockerHub(t *testing.T) {
 
 	cmd.Command = dockerCli.Command("scan", ImageWithVulnerabilities)
 	result := icmd.RunCmd(cmd)
-	if result.ExitCode == 1 {
-		assert.Assert(t, cmp.Regexp("found .* vulnerabilities", result.Combined()), result.Combined())
-	} else {
+	assert.Assert(t, result.ExitCode == 1)
+	if strings.HasPrefix(result.Combined(), "You") {
 		// We reach the monthly limits of 10 free scans
-		assert.Assert(t, result.ExitCode == 2)
 		assert.Assert(t, strings.Contains(result.Combined(), "You have reached the scan limit of 10 monthly scans without authentication."), result.Combined())
+	} else {
+		assert.Assert(t, cmp.Regexp("found .* vulnerabilities", result.Combined()), result.Combined())
+
 	}
 
 }
@@ -131,12 +132,14 @@ func TestScanWithSnyk(t *testing.T) {
 			exitCode: 0,
 			contains: "no vulnerable paths found",
 		},
-		{
+		// Due to an issue linked to github actions env, we removed the test for the moment
+		// we got the error message that Snyk returns when it can't connect to the engine 'Invalid Docker archive'
+		/*{
 			name:     "invalid-docker-archive",
 			image:    InvalidImage,
-			exitCode: 2,
-			contains: "Invalid Docker archive",
-		},
+			exitCode: 1,
+			contains: "(HTTP code 500) server error - empty export - not implemented",
+		},*/
 		{
 			name:     "image-with-vulnerabilities",
 			image:    ImageWithVulnerabilities,
@@ -146,15 +149,15 @@ func TestScanWithSnyk(t *testing.T) {
 		{
 			name:     "invalid-image-name",
 			image:    "scratch",
-			exitCode: 2,
+			exitCode: 1,
 			contains: "manifest unknown",
 		},
 	}
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
 			cmd.Command = dockerCli.Command("scan", testCase.image)
 			output := icmd.RunCmd(cmd).Assert(t, icmd.Expected{ExitCode: testCase.exitCode}).Combined()
-			assert.Assert(t, strings.Contains(output, testCase.contains))
+			assert.Assert(t, strings.Contains(output, testCase.contains), output)
 		})
 	}
 }
@@ -186,7 +189,7 @@ func TestScanJsonOutput(t *testing.T) {
 		{
 			name:     "invalid-docker-archive",
 			image:    InvalidImage,
-			exitCode: 2,
+			exitCode: 1,
 			isEmpty:  true,
 		},
 		{
@@ -339,12 +342,68 @@ func TestScanWithGroupIssues(t *testing.T) {
 		Err:      "--json flag is mandatory to use --group-issues flag"})
 }
 
-func createSnykConfFile(t *testing.T, token string) (*fs.Dir, func()) {
+func TestScanWithContainerizedSnyk(t *testing.T) {
+	if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
+		t.Skip("Can't run on this ci platform (windows containers or no engine installed)")
+	}
+	homeDir, cleanFunction := createSnykConfFile(t, os.Getenv("E2E_TEST_AUTH_TOKEN"))
+	defer cleanFunction()
+
+	cmd, configDir, cleanup := dockerCli.createTestCmd()
+	defer cleanup()
+	createScanConfigFileOptinAndPath(t, configDir, true, "")
+
+	testCases := []struct {
+		name     string
+		image    string
+		exitCode int
+		contains string
+	}{
+		{
+			name:     "image-without-vulnerabilities",
+			image:    ImageWithoutVulnerabilities,
+			exitCode: 0,
+			contains: "no vulnerable paths found",
+		},
+		{
+			name:     "invalid-docker-archive",
+			image:    InvalidImage,
+			exitCode: 1,
+			contains: "Invalid Docker archive",
+		},
+		{
+			name:     "image-with-vulnerabilities",
+			image:    ImageWithVulnerabilities,
+			exitCode: 1,
+			contains: "vulnerability found",
+		},
+		{
+			name:     "invalid-image-name",
+			image:    "scratch",
+			exitCode: 1,
+			contains: "manifest unknown",
+		},
+	}
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			cmd.Command = dockerCli.Command("scan", testCase.image)
+			cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", homeDir.Path()))
+			output := icmd.RunCmd(cmd).Assert(t, icmd.Expected{ExitCode: testCase.exitCode}).Combined()
+			assert.Assert(t, strings.Contains(output, testCase.contains), output)
+		})
+	}
+}
+
+func createSnykConfDirectories(t *testing.T, withConfFile bool, token string) (*fs.Dir, func()) {
 	content := fmt.Sprintf(`{"api" : "%s"}`, token)
+	var confFiles []fs.PathOp
+	if withConfFile {
+		confFiles = append(confFiles, fs.WithFile("snyk.json", content))
+	}
 	homeDir := fs.NewDir(t, t.Name(),
 		fs.WithDir(".config",
-			fs.WithDir("configstore",
-				fs.WithFile("snyk.json", content))))
+			fs.WithDir("configstore", confFiles...)))
+
 	homeFunc := env.Patch(t, "HOME", homeDir.Path())
 	userProfileFunc := env.Patch(t, "USERPROFILE", homeDir.Path())
 	cleanup := func() {
@@ -356,6 +415,10 @@ func createSnykConfFile(t *testing.T, token string) (*fs.Dir, func()) {
 	return homeDir, cleanup
 }
 
+func createSnykConfFile(t *testing.T, token string) (*fs.Dir, func()) {
+	return createSnykConfDirectories(t, true, token)
+}
+
 func patchConfig(t *testing.T, configDir, url, userName, password string) {
 	buff, err := ioutil.ReadFile(filepath.Join(configDir, "config.json"))
 	assert.NilError(t, err)
@@ -379,8 +442,12 @@ func createScanConfigFile(t *testing.T, configDir string) {
 }
 
 func createScanConfigFileOptin(t *testing.T, configDir string, optin bool) {
+	createScanConfigFileOptinAndPath(t, configDir, optin, filepath.Join(configDir, "scan", "snyk"))
+}
+
+func createScanConfigFileOptinAndPath(t *testing.T, configDir string, optin bool, path string) {
 	conf := config.Config{
-		Path:  filepath.Join(configDir, "scan", "snyk"),
+		Path:  path,
 		Optin: optin,
 	}
 	buf, err := json.MarshalIndent(conf, "", "  ")