Improve docker image and logging

Signed-off-by: Christian Dupuis <cd@atomist.com>

Author: cdupuis

.github/workflows/docker-publish.yaml

@@ -17,7 +17,7 @@ env:
   # Use docker.io for Docker Hub if empty
   REGISTRY: ghcr.io
   # github.repository as <account>/<repo>
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: docker/docker-index
   SHA: ${{ github.event.pull_request.head.sha || github.event.after }}
   DOCKERFILE_PATH: Dockerfile
 

commands/cmd.go

@@ -31,6 +31,7 @@ import (
 	"github.com/docker/index-cli-plugin/query"
 	"github.com/docker/index-cli-plugin/sbom"
 	"github.com/docker/index-cli-plugin/types"
+	"github.com/docker/index-cli-plugin/util"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/moby/term"
 	"github.com/pkg/errors"
@@ -57,10 +58,14 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 	}
 
 	skill.Log.SetOutput(os.Stderr)
-	skill.Log.SetFormatter(&logrus.TextFormatter{
-		DisableTimestamp:       true,
-		DisableLevelTruncation: true,
-	})
+	if dockerCli.Out().IsTerminal() {
+		skill.Log.SetFormatter(&logrus.TextFormatter{
+			DisableTimestamp:       true,
+			DisableLevelTruncation: true,
+		})
+	} else {
+		skill.Log.SetFormatter(&logrus.JSONFormatter{})
+	}
 
 	config := dockerCli.ConfigFile()
 
@@ -112,9 +117,9 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 			var sb *types.Sbom
 
 			if ociDir == "" {
-				sb, _, err = sbom.IndexImage(image, dockerCli.Client())
+				sb, _, err = sbom.IndexImage(image, dockerCli)
 			} else {
-				sb, _, err = sbom.IndexPath(ociDir, image)
+				sb, _, err = sbom.IndexPath(ociDir, image, dockerCli)
 			}
 			if err != nil {
 				return err
@@ -175,9 +180,9 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 			var sb *types.Sbom
 			var img *v1.Image
 			if ociDir == "" {
-				sb, img, err = sbom.IndexImage(image, dockerCli.Client())
+				sb, img, err = sbom.IndexImage(image, dockerCli)
 			} else {
-				sb, img, err = sbom.IndexPath(ociDir, image)
+				sb, img, err = sbom.IndexPath(ociDir, image, dockerCli)
 			}
 			if err != nil {
 				return err
@@ -206,9 +211,9 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 			var img *v1.Image
 
 			if ociDir == "" {
-				sb, img, err = sbom.IndexImage(image, dockerCli.Client())
+				sb, img, err = sbom.IndexImage(image, dockerCli)
 			} else {
-				sb, img, err = sbom.IndexPath(ociDir, image)
+				sb, img, err = sbom.IndexPath(ociDir, image, dockerCli)
 			}
 			if err != nil {
 				return err
@@ -246,7 +251,7 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 					}
 
 					// see if the package comes in via the base image
-					s := StartInfoSpinner("Detecting base image")
+					s := util.StartInfoSpinner("Detecting base image", dockerCli.Out().IsTerminal())
 					defer s.Stop()
 					baseImages, index, _ := query.Detect(img, true, workspace, apiKey)
 					s.Stop()
@@ -261,7 +266,7 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 					}
 
 					if baseImage != nil {
-						s := StartInfoSpinner("Finding alternative base images")
+						s := util.StartInfoSpinner("Finding alternative base images", dockerCli.Out().IsTerminal())
 						defer s.Stop()
 						aBaseImage, _ := query.ForBaseImageWithoutCve(c.SourceId, baseImage.Repository.Name, img, workspace, apiKey)
 						s.Stop()
@@ -294,7 +299,7 @@ func NewRootCmd(name string, isPlugin bool, dockerCli command.Cli) *cobra.Comman
 		Use:   "diff [OPTIONS]",
 		Short: "Diff images",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return sbom.DiffImages(args[0], args[1], dockerCli.Client(), "", "")
+			return sbom.DiffImages(args[0], args[1], dockerCli, "", "")
 		},
 	}
 

commands/spinner.go

@@ -25,9 +25,7 @@
 
                      (or-join [?v ?source-id]
                        [?v :vulnerability/cve-id ?source-id]
-                       (and
-                         [(missing? $ ?v :vulnerability/cve-id)]
-                         [?v :vulnerability/source-id ?source-id]))
+                       [?v :vulnerability/source-id ?source-id])
                      [?v :vulnerability/source ?source]
                      [?v :vulnerability/advisories ?adv]
 

query/package_cve.edn

@@ -81,7 +81,7 @@ func QueryCves(sb *types.Sbom, cve string, workspace string, apiKey string) (*[]
 		return nil, errors.Wrapf(err, "failed to unmarshal response")
 	}
 	if len(result.Query.Data) > 0 {
-		if len(result.Query.Data) == 1 {
+		if len(result.Query.Data[0].Cves) == 1 {
 			skill.Log.Infof("Detected %d vulnerability", len(result.Query.Data[0].Cves))
 		} else {
 			skill.Log.Infof("Detected %d vulnerabilities", len(result.Query.Data[0].Cves))
@@ -99,6 +99,7 @@ func query(query string, name string, workspace string, apiKey string) (*http.Re
 
 	}
 	query = fmt.Sprintf(`{:queries [{:name "query" :query %s}]}`, query)
+	skill.Log.Debugf("Query %s", query)
 	client := &http.Client{}
 	req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(query))
 	if err != nil {
@@ -117,5 +118,6 @@ func query(query string, name string, workspace string, apiKey string) (*http.Re
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to run query")
 	}
+	skill.Log.Debugf("Query response %s", resp.Status)
 	return resp, nil
 }

query/query.go

@@ -18,12 +18,14 @@ package registry
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/atomist-skills/go-skill"
-	"github.com/docker/docker/client"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/index-cli-plugin/util"
 	"github.com/dustin/go-humanize"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
@@ -32,7 +34,6 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 type ImageId struct {
@@ -67,13 +68,13 @@ type ImageCache struct {
 	Ref       *name.Reference
 
 	copy bool
+	cli  command.Cli
 }
 
 func (c *ImageCache) StoreImage() error {
 	if !c.copy {
 		return nil
 	}
-	skill.Log.Infof("Copying image %s", c.Name)
 	skill.Log.Debugf("Copying image to %s", c.ImagePath)
 	u := make(chan v1.Update, 200)
 	errchan := make(chan error)
@@ -87,27 +88,25 @@ func (c *ImageCache) StoreImage() error {
 	var update v1.Update
 	var err error
 	var pp int64
+	spinner := util.StartSpinner("info", "Copying image", c.cli.Out().IsTerminal())
+	defer spinner.Stop()
 	for {
 		select {
 		case update = <-u:
 			p := 100 * update.Complete / update.Total
-			if p%10 == 0 && pp != p {
-				skill.Log.WithFields(logrus.Fields{
-					"event":    "copy",
+			if pp != p {
+				spinner.WithFields(util.Fields{
+					"event":    "progress",
 					"total":    update.Total,
 					"complete": update.Complete,
-				}).Debugf("Copying image %3d%% %s/%s", p, humanize.Bytes(uint64(update.Complete)), humanize.Bytes(uint64(update.Total)))
+				}).Update(fmt.Sprintf("Copying image %d%% %s/%s", p, humanize.Bytes(uint64(update.Complete)), humanize.Bytes(uint64(update.Total))))
 				pp = p
 			}
 		case err = <-errchan:
 			if err != nil {
 				return err
 			} else {
-				skill.Log.WithFields(logrus.Fields{
-					"event":    "copy",
-					"total":    update.Total,
-					"complete": update.Complete,
-				}).Debugf("Copying image completed")
+				spinner.Stop()
 				skill.Log.Infof("Copied image")
 				return nil
 			}
@@ -126,7 +125,7 @@ func (c *ImageCache) Cleanup() {
 }
 
 // SaveImage stores the v1.Image at path returned in OCI format
-func SaveImage(image string, client client.APIClient) (*ImageCache, error) {
+func SaveImage(image string, cli command.Cli) (*ImageCache, error) {
 	ref, err := name.ParseReference(image)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse reference: %s", image)
@@ -155,11 +154,11 @@ func SaveImage(image string, client client.APIClient) (*ImageCache, error) {
 
 	desc, err := remote.Get(ref, withAuth())
 	if err != nil {
-		img, err := daemon.Image(ImageId{name: image}, daemon.WithClient(client))
+		img, err := daemon.Image(ImageId{name: image}, daemon.WithClient(cli.Client()))
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to pull image: %s", image)
 		} else {
-			im, _, err := client.ImageInspectWithRaw(context.Background(), image)
+			im, _, err := cli.Client().ImageInspectWithRaw(context.Background(), image)
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to get local image: %s", image)
 			}
@@ -174,6 +173,7 @@ func SaveImage(image string, client client.APIClient) (*ImageCache, error) {
 				Ref:       &ref,
 				ImagePath: imagePath,
 				copy:      true,
+				cli:       cli,
 			}, nil
 		}
 	} else {
@@ -200,6 +200,7 @@ func SaveImage(image string, client client.APIClient) (*ImageCache, error) {
 			Ref:       &ref,
 			ImagePath: imagePath,
 			copy:      true,
+			cli:       cli,
 		}, nil
 	}
 }

registry/save.go

@@ -22,7 +22,7 @@ import (
 	"sync"
 
 	"github.com/anchore/packageurl-go"
-	"github.com/docker/docker/client"
+	"github.com/docker/cli/cli/command"
 	"github.com/docker/index-cli-plugin/internal"
 	"github.com/docker/index-cli-plugin/types"
 	"github.com/gookit/color"
@@ -58,12 +58,12 @@ func init() {
 	}
 }
 
-func DiffImages(image1 string, image2 string, client client.APIClient, workspace string, apikey string) error {
+func DiffImages(image1 string, image2 string, cli command.Cli, workspace string, apikey string) error {
 	resultChan := make(chan ImageIndexResult, 2)
 	var wg sync.WaitGroup
 	wg.Add(2)
-	go indexImageAsync(&wg, image1, client, resultChan)
-	go indexImageAsync(&wg, image2, client, resultChan)
+	go indexImageAsync(&wg, image1, cli, resultChan)
+	go indexImageAsync(&wg, image2, cli, resultChan)
 	wg.Wait()
 	close(resultChan)
 

sbom/diff.go

@@ -25,11 +25,12 @@ import (
 	"sync"
 
 	"github.com/atomist-skills/go-skill"
-	"github.com/docker/docker/client"
+	"github.com/docker/cli/cli/command"
 	"github.com/docker/index-cli-plugin/internal"
 	"github.com/docker/index-cli-plugin/query"
 	"github.com/docker/index-cli-plugin/registry"
 	"github.com/docker/index-cli-plugin/types"
+	"github.com/docker/index-cli-plugin/util"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/pkg/errors"
@@ -42,9 +43,9 @@ type ImageIndexResult struct {
 	Error error
 }
 
-func indexImageAsync(wg *sync.WaitGroup, image string, client client.APIClient, resultChan chan<- ImageIndexResult) {
+func indexImageAsync(wg *sync.WaitGroup, image string, cli command.Cli, resultChan chan<- ImageIndexResult) {
 	defer wg.Done()
-	sbom, img, err := IndexImage(image, client)
+	sbom, img, err := IndexImage(image, cli)
 	cves, err := query.QueryCves(sbom, "", "", "")
 	if err == nil {
 		sbom.Vulnerabilities = *cves
@@ -57,23 +58,23 @@ func indexImageAsync(wg *sync.WaitGroup, image string, client client.APIClient,
 	}
 }
 
-func IndexPath(path string, name string) (*types.Sbom, *v1.Image, error) {
+func IndexPath(path string, name string, cli command.Cli) (*types.Sbom, *v1.Image, error) {
 	cache, err := registry.ReadImage(name, path)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to read image")
 	}
-	return indexImage(cache)
+	return indexImage(cache, cli)
 }
 
-func IndexImage(image string, client client.APIClient) (*types.Sbom, *v1.Image, error) {
-	cache, err := registry.SaveImage(image, client)
+func IndexImage(image string, cli command.Cli) (*types.Sbom, *v1.Image, error) {
+	cache, err := registry.SaveImage(image, cli)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "failed to copy image")
 	}
-	return indexImage(cache)
+	return indexImage(cache, cli)
 }
 
-func indexImage(cache *registry.ImageCache) (*types.Sbom, *v1.Image, error) {
+func indexImage(cache *registry.ImageCache, cli command.Cli) (*types.Sbom, *v1.Image, error) {
 	// see if we can re-use an existing sbom
 	sbomPath := filepath.Join(cache.Path, "sbom.json")
 	if _, ok := os.LookupEnv("ATOMIST_NO_CACHE"); !ok {
@@ -101,7 +102,8 @@ func indexImage(cache *registry.ImageCache) (*types.Sbom, *v1.Image, error) {
 	lm := createLayerMapping(*cache.Image)
 	skill.Log.Debugf("Created layer mapping")
 
-	skill.Log.Info("Indexing")
+	s := util.StartSpinner("info", "Indexing", cli.Out().IsTerminal())
+	defer s.Stop()
 	trivyResultChan := make(chan types.IndexResult)
 	syftResultChan := make(chan types.IndexResult)
 	go trivySbom(cache.ImagePath, lm, trivyResultChan)
@@ -118,6 +120,7 @@ func indexImage(cache *registry.ImageCache) (*types.Sbom, *v1.Image, error) {
 
 	packages := types.MergePackages(syftResult, trivyResult)
 
+	s.Stop()
 	skill.Log.Infof(`Indexed %d packages`, len(packages))
 
 	manifest, _ := (*cache.Image).RawManifest()