Merge pull request #167 from AkihiroSuda/kube

new driver: kubernetes

Author: tiborvass

README.md

@@ -11,7 +11,7 @@ _buildx is Tech Preview_
 - Multi-node builds for cross-platform images
 - Compose build support
 - WIP: High-level build constructs (`bake`)
-- TODO: In-container driver support
+- In-container driver support (both Docker and Kubernetes)
 
 # Table of Contents
 
@@ -394,12 +394,18 @@ Passes additional driver-specific options. Details for each driver:
 
 - `docker` - No driver options
 - `docker-container`
-  - `image` - Sets the container image to be used for running buildkit.
-  - `network` - Sets the network mode for running the buildkit container.
+  - `image=IMAGE` - Sets the container image to be used for running buildkit.
+  - `network=NETMODE` - Sets the network mode for running the buildkit container.
   - Example:
     ```
     --driver docker-container --driver-opt image=moby/buildkit:master,network=host
     ```
+- `kubernetes`
+  - `image=IMAGE` - Sets the container image to be used for running buildkit.
+  - `namespace=NS` - Sets the Kubernetes namespace. Defaults to the current namespace.
+  - `replicas=N` - Sets the number of `Pod` replicas. Defaults to 1.
+  - `rootless=(true|false)` - Run the container as a non-root user without `securityContext.privileged`. [Using Ubuntu host kernel is recommended](https://github.com/moby/buildkit/blob/master/docs/rootless.md). Defaults to false.
+  - `loadbalance=(sticky|random)` - Load-balancing strategy. If set to "sticky", the pod is chosen using the hash of the context path. Defaults to "sticky"
 
 #### `--leave`
 

cmd/buildx/main.go

@@ -12,8 +12,14 @@ import (
 	cliflags "github.com/docker/cli/cli/flags"
 	"github.com/spf13/cobra"
 
+	// FIXME: "k8s.io/client-go/plugin/pkg/client/auth/azure" is excluded because of compilation error
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
+
 	_ "github.com/docker/buildx/driver/docker"
 	_ "github.com/docker/buildx/driver/docker-container"
+	_ "github.com/docker/buildx/driver/kubernetes"
 )
 
 var experimental string

commands/bake.go

@@ -56,7 +56,9 @@ func runBake(dockerCli command.Cli, targets []string, in bakeOptions) error {
 		return err
 	}
 
-	return buildTargets(ctx, dockerCli, bo, in.progress)
+	contextPathHash, _ := os.Getwd()
+
+	return buildTargets(ctx, dockerCli, bo, in.progress, contextPathHash)
 }
 
 func defaultFiles() ([]string, error) {

commands/build.go

@@ -3,6 +3,7 @@ package commands
 import (
 	"context"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/docker/buildx/build"
@@ -175,11 +176,17 @@ func runBuild(dockerCli command.Cli, in buildOptions) error {
 	}
 	opts.Allow = allow
 
-	return buildTargets(ctx, dockerCli, map[string]build.Options{"default": opts}, in.progress)
+	// key string used for kubernetes "sticky" mode
+	contextPathHash, err := filepath.Abs(in.contextPath)
+	if err != nil {
+		contextPathHash = in.contextPath
+	}
+
+	return buildTargets(ctx, dockerCli, map[string]build.Options{"default": opts}, in.progress, contextPathHash)
 }
 
-func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]build.Options, progressMode string) error {
-	dis, err := getDefaultDrivers(ctx, dockerCli)
+func buildTargets(ctx context.Context, dockerCli command.Cli, opts map[string]build.Options, progressMode, contextPathHash string) error {
+	dis, err := getDefaultDrivers(ctx, dockerCli, contextPathHash)
 	if err != nil {
 		return err
 	}

commands/rm.go

@@ -65,7 +65,7 @@ func rmCmd(dockerCli command.Cli) *cobra.Command {
 }
 
 func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, rm bool) error {
-	dis, err := driversForNodeGroup(ctx, dockerCli, ng)
+	dis, err := driversForNodeGroup(ctx, dockerCli, ng, "")
 	if err != nil {
 		return err
 	}
@@ -88,7 +88,7 @@ func stop(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, rm bo
 }
 
 func stopCurrent(ctx context.Context, dockerCli command.Cli, rm bool) error {
-	dis, err := getDefaultDrivers(ctx, dockerCli)
+	dis, err := getDefaultDrivers(ctx, dockerCli, "")
 	if err != nil {
 		return err
 	}

commands/util.go

@@ -11,9 +11,11 @@ import (
 	"github.com/docker/buildx/util/platformutil"
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/cli/context/docker"
+	"github.com/docker/cli/cli/context/kubernetes"
 	dopts "github.com/docker/cli/opts"
 	dockerclient "github.com/docker/docker/client"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -133,7 +135,7 @@ func getNodeGroup(txn *store.Txn, dockerCli command.Cli, name string) (*store.No
 }
 
 // driversForNodeGroup returns drivers for a nodegroup instance
-func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup) ([]build.DriverInfo, error) {
+func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.NodeGroup, contextPathHash string) ([]build.DriverInfo, error) {
 	eg, _ := errgroup.WithContext(ctx)
 
 	dis := make([]build.DriverInfo, len(ng.Nodes))
@@ -174,7 +176,18 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N
 				// TODO: replace the following line with dockerclient.WithAPIVersionNegotiation option in clientForEndpoint
 				dockerapi.NegotiateAPIVersion(ctx)
 
-				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, n.Flags, n.ConfigFile, n.DriverOpts)
+				contextStore := dockerCli.ContextStore()
+				kcc, err := kubernetes.ConfigFromContext(n.Endpoint, contextStore)
+				if err != nil {
+					// err is returned if n.Endpoint is non-context name like "unix:///var/run/docker.sock".
+					// try again with name="default".
+					// FIXME: n should retain real context name.
+					kcc, err = kubernetes.ConfigFromContext("default", contextStore)
+					if err != nil {
+						logrus.Error(err)
+					}
+				}
+				d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, kcc, n.Flags, n.ConfigFile, n.DriverOpts, contextPathHash)
 				if err != nil {
 					di.Err = err
 					return nil
@@ -235,7 +248,7 @@ func clientForEndpoint(dockerCli command.Cli, name string) (dockerclient.APIClie
 }
 
 // getDefaultDrivers returns drivers based on current cli config
-func getDefaultDrivers(ctx context.Context, dockerCli command.Cli) ([]build.DriverInfo, error) {
+func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, contextPathHash string) ([]build.DriverInfo, error) {
 	txn, release, err := getStore(dockerCli)
 	if err != nil {
 		return nil, err
@@ -248,10 +261,10 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli) ([]build.Driv
 	}
 
 	if ng != nil {
-		return driversForNodeGroup(ctx, dockerCli, ng)
+		return driversForNodeGroup(ctx, dockerCli, ng, contextPathHash)
 	}
 
-	d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), nil, "", nil)
+	d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), nil, nil, "", nil, contextPathHash)
 	if err != nil {
 		return nil, err
 	}
@@ -294,7 +307,7 @@ func loadInfoData(ctx context.Context, d *dinfo) error {
 func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo) error {
 	eg, _ := errgroup.WithContext(ctx)
 
-	dis, err := driversForNodeGroup(ctx, dockerCli, ngi.ng)
+	dis, err := driversForNodeGroup(ctx, dockerCli, ngi.ng, "")
 	if err != nil {
 		return err
 	}
@@ -312,7 +325,30 @@ func loadNodeGroupData(ctx context.Context, dockerCli command.Cli, ngi *nginfo)
 		}(&ngi.drivers[i])
 	}
 
-	return eg.Wait()
+	if eg.Wait(); err != nil {
+		return err
+	}
+	for _, di := range ngi.drivers {
+		// dynamic nodes are used in Kubernetes driver.
+		// Kubernetes pods are dynamically mapped to BuildKit Nodes.
+		if di.info != nil && len(di.info.DynamicNodes) > 0 {
+			var drivers []dinfo
+			for i := 0; i < len(di.info.DynamicNodes); i++ {
+				// all []dinfo share *build.DriverInfo and *driver.Info
+				diClone := di
+				if pl := di.info.DynamicNodes[i].Platforms; len(pl) > 0 {
+					diClone.platforms = pl
+				}
+				drivers = append(drivers, di)
+			}
+			// not append (remove the static nodes in the store)
+			ngi.ng.Nodes = di.info.DynamicNodes
+			ngi.ng.Dynamic = true
+			ngi.drivers = drivers
+			return nil
+		}
+	}
+	return nil
 }
 
 func dockerAPI(dockerCli command.Cli) *api {

driver/bkimage/bkimage.go

@@ -0,0 +1,6 @@
+package bkimage
+
+const (
+	DefaultImage         = "moby/buildkit:buildx-stable-1" // TODO: make this verified
+	DefaultRootlessImage = "moby/buildkit:v0.6.2-rootless"
+)

driver/docker-container/driver.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/docker/buildx/driver"
+	"github.com/docker/buildx/driver/bkimage"
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/docker/api/types"
 	dockertypes "github.com/docker/docker/api/types"
@@ -22,8 +23,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-var defaultBuildkitImage = "moby/buildkit:buildx-stable-1" // TODO: make this verified
-
 type Driver struct {
 	driver.InitConfig
 	factory driver.Factory
@@ -54,7 +53,7 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error {
 }
 
 func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
-	imageName := defaultBuildkitImage
+	imageName := bkimage.DefaultImage
 	if d.image != "" {
 		imageName = d.image
 	}

driver/driver.go

@@ -3,6 +3,7 @@ package driver
 import (
 	"context"
 
+	"github.com/docker/buildx/store"
 	"github.com/docker/buildx/util/progress"
 	"github.com/moby/buildkit/client"
 	"github.com/pkg/errors"
@@ -39,6 +40,8 @@ func (s Status) String() string {
 
 type Info struct {
 	Status Status
+	// DynamicNodes must be empty if the actual nodes are statically listed in the store
+	DynamicNodes []store.Node
 }
 
 type Driver interface {