diff --git a/.gitignore b/.gitignore index 6e2e35e0..88a4ebbb 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ cmd/handbindings/handbindings cmd/handbindings/bindings_test/direct cmd/handbindings/bindings_test/testapp cmd/genbindings/genbindings +cmd/miqt-docker/miqt-docker cmd/miqt-uic/miqt-uic cmd/miqt-rcc/miqt-rcc diff --git a/cmd/miqt-docker/README.md b/cmd/miqt-docker/README.md new file mode 100644 index 00000000..be5b9b02 --- /dev/null +++ b/cmd/miqt-docker/README.md @@ -0,0 +1,33 @@ +# miqt-docker + +This is a helper program to quickly run a dockerized MIQT build environment. + +- Supports all available MIQT docker containers +- Use glob matches to automatically pick the highest version container for target +- Automatically build new docker containers or reuse existing, based on content hash of the Dockerfile +- Automatically bind source code volume from current go.mod / go.work / git repository and preserve relative working directory +- Handles bind-mounting the GOCACHE and GOMODCACHE directories +- Handles using the proper uid+gid on Linux +- Automatically detect sudo requirement on Linux + +## Usage + +Run `miqt-docker` with no arguments to see full usage instructions and all +available embedded dockerfiles: + +```bash +Usage: miqt-docker ENVIRONMENT COMMAND... + +Environment variables: +- DOCKER Override the path to docker + +Available container environments (use * for partial match): +[...] +``` + +Example build commands: + +```bash +miqt-docker macos go build -ldflags '-s -w' +miqt-docker win64*qt6*dynamic go build -ldflags '-s -w -H windowsgui' +``` diff --git a/cmd/miqt-docker/docker.go b/cmd/miqt-docker/docker.go new file mode 100644 index 00000000..77cf728c --- /dev/null +++ b/cmd/miqt-docker/docker.go @@ -0,0 +1,105 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "log" + "os" + "os/exec" +) + +var ( + needsSudo bool +) + +// dockerImage describes an image available in the docker daemon. +type dockerImage struct { + ID string + Repository string + Tag string +} + +// dockerCommand creates an *exec.Cmd for running docker. It respects the global +// `needsSudo` state. +func dockerCommand(args ...string) *exec.Cmd { + docker := os.Getenv("DOCKER") + if docker == "" { + docker = "docker" + } + + if needsSudo { + useArgs := make([]string, 0, len(args)+1) + useArgs = append(useArgs, docker) + useArgs = append(useArgs, args...) + return exec.Command(`sudo`, useArgs...) + } + + return exec.Command(docker, args...) +} + +// dockerListImages lists all the current docker images. +func dockerListImages() ([]dockerImage, error) { + + cmd := dockerCommand(`image`, `ls`, `--format`, `{{json . }}`) + cmd.Stderr = os.Stderr // passthrough + + buff, err := cmd.Output() + if err != nil { + if !needsSudo { + // Retry with sudo + log.Println("Retrying with sudo...") + needsSudo = true + return dockerListImages() + } + + return nil, err + } + + var ret []dockerImage + + dec := json.NewDecoder(bytes.NewReader(buff)) + for { + var entry dockerImage + err = dec.Decode(&entry) + if err != nil { + if errors.Is(err, io.EOF) { + return ret, nil + } + return nil, err // real error + } + + ret = append(ret, entry) + } +} + +// dockerFindImage searches all the current docker images to find one named as +// the supplied `repository:tag`. +func dockerFindImage(repository, tag string) (*dockerImage, error) { + + images, err := dockerListImages() + if err != nil { + return nil, err + } + + for _, im := range images { + if im.Repository == repository && im.Tag == tag { + // found it + return &im, nil + } + } + + // No match + return nil, os.ErrNotExist +} + +// dockerBuild builds the supplied dockerfile and tags it as repository:tag +// as well as repository:latest. +func dockerBuild(dockerfile []byte, repository, tag string) error { + cmd := dockerCommand(`build`, `-t`, repository+`:`+tag, `-t`, repository+`:latest`, `-`) + cmd.Stderr = os.Stderr + cmd.Stdin = bytes.NewReader(dockerfile) + + return cmd.Run() +} diff --git a/cmd/miqt-docker/filepath.go b/cmd/miqt-docker/filepath.go new file mode 100644 index 00000000..5a5661c8 --- /dev/null +++ b/cmd/miqt-docker/filepath.go @@ -0,0 +1,54 @@ +package main + +import ( + "errors" + "path/filepath" + "runtime" + "strings" +) + +// highestCommonParent finds the oldest ancestor of a set of paths. +// If there is no common ancestor, returns / on Linux or an error on Windows. +func highestCommonParent(paths []string) (string, error) { + if len(paths) == 0 { + return "", errors.New("no input") + } + + parts := strings.Split(paths[0], string(filepath.Separator)) + + for _, check := range paths { + checkn := strings.Split(check, string(filepath.Separator)) + + // If this check path is shorter, the common part must also shrink + if len(checkn) < len(parts) { + parts = parts[0:len(checkn)] + } + + for i, checkpart := range checkn[0:len(parts)] { // len(parts) is now <= len(checkn) so this is safe + if parts[i] == checkpart { + continue + } + + // Divergence from i: onwards + parts = parts[0:i] + break + } + + // Early failure case + if len(parts) == 0 { + break + } + + } + + isEmpty := len(parts) == 0 || (len(parts) == 1 && parts[0] == "") + + if isEmpty { + if runtime.GOOS == "windows" { + return "", errors.New("Selected paths have no common ancestor") + } + return `/`, nil + } + + return strings.Join(parts, string(filepath.Separator)), nil +} diff --git a/cmd/miqt-docker/filepath_test.go b/cmd/miqt-docker/filepath_test.go new file mode 100644 index 00000000..2888e768 --- /dev/null +++ b/cmd/miqt-docker/filepath_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "runtime" + "testing" +) + +func TestHighestCommonParent(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("This test uses platform-specific paths") + } + + type testCase struct { + input []string + expect string + } + + cases := []testCase{ + + // Single input + testCase{ + input: []string{`/foo/bar/baz`}, + expect: `/foo/bar/baz`, + }, + + // Duplicated input + testCase{ + input: []string{`/foo/bar/baz`, `/foo/bar/baz`}, + expect: `/foo/bar/baz`, + }, + + // Trailing slashes are preserved if they all included trailing slashes + testCase{ + input: []string{`/foo/bar/baz/`, `/foo/bar/baz/`}, + expect: `/foo/bar/baz/`, + }, + + // Common directory + testCase{ + input: []string{`/foo/bar/baz`, `/foo/quux`}, + expect: `/foo`, + }, + + // Common directory, multiple inputs + testCase{ + input: []string{`/foo/a`, `/foo/b`, `/foo/c`, `/foo/d`}, + expect: `/foo`, + }, + + testCase{ + input: []string{`/foo/bar/baz`, `/unrelated`, `/foo/bar/baz`}, + expect: `/`, + }, + + // No leading forwardslash (single input) + testCase{ + input: []string{`foo/bar/baz`}, + expect: `foo/bar/baz`, + }, + + // No leading forwardslash (empty output assumes /) + testCase{ + input: []string{`foo/bar/baz`, `unrelated`, `foo/bar/baz`}, + expect: `/`, + }, + } + + for idx, tc := range cases { + got, err := highestCommonParent(tc.input) + if err != nil { + t.Errorf("test %d: input(%v) got error=%v", idx, tc.input, err) + continue + } + if got != tc.expect { + t.Errorf("test %d: input(%v) got %q, want %q", idx, tc.input, got, tc.expect) + } + } +} diff --git a/cmd/miqt-docker/main.go b/cmd/miqt-docker/main.go new file mode 100644 index 00000000..6a8cb69f --- /dev/null +++ b/cmd/miqt-docker/main.go @@ -0,0 +1,208 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io/fs" + "log" + "os" + "os/exec" + "os/user" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/mappu/miqt/docker" +) + +// glob2regex converts the glob pattern into a regexp. +// It only supports `*` as a special character. +// The resulting regex is unanchored i.e. can match anywhere within a target string. +func glob2regex(pattern string) *regexp.Regexp { + parts := strings.Split(pattern, `*`) + for i, p := range parts { + parts[i] = regexp.QuoteMeta(p) + } + + return regexp.MustCompile(strings.Join(parts, `.*`)) +} + +// shasum returns the hex sha256 of a byte slice. +func shasum(data []byte) string { + hashdata := sha256.Sum256(data) + return hex.EncodeToString(hashdata[:]) +} + +// usage displays how to use miqt-docker and then exits the process. +func usage(dockerfiles []fs.DirEntry) { + fmt.Fprintf(os.Stderr, "Usage: %s ENVIRONMENT COMMAND...\n", filepath.Base(os.Args[0])) + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, "Environment variables:") + fmt.Fprintln(os.Stderr, "- DOCKER Override the path to docker") + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, "Available container environments (use * for partial match):") + + for _, ff := range dockerfiles { + fmt.Fprintf(os.Stderr, "- %s\n", strings.TrimSuffix(ff.Name(), `.Dockerfile`)) + } + + os.Exit(1) +} + +func main() { + + dockerfiles, err := docker.Dockerfiles.ReadDir(`.`) + if err != nil { + log.Panic(err) + } + + if len(os.Args) < 3 { + usage(dockerfiles) + } + + requestEnvironment := glob2regex(os.Args[1]) + var match string + for _, ff := range dockerfiles { + if !requestEnvironment.MatchString(ff.Name()) { + continue + } + + match = ff.Name() + // continue searching for a later match with higher version number + } + + if match == "" { + log.Fatalf("No available environment matches the request %q\n", os.Args[1]) + } + + if !(match == os.Args[1] || match == os.Args[1]+`.Dockerfile`) { + // An inexact/glob match was involved. Show what it was + log.Printf("Selecting dockerfile: %s", match) + } + + dockerFileContent, err := docker.ReadFile(match) + if err != nil { + log.Panic(err) // shouldn't happen + } + + dockerfileHash := shasum(dockerFileContent)[:8] // First 8 characters of content hash + + // Check to see if this dockerfile has already been turned into an image + + containerName := `miqt-docker/` + strings.TrimSuffix(match, `.Dockerfile`) + + _, err = dockerFindImage(containerName, dockerfileHash) + if err != nil { + if err != os.ErrNotExist { + log.Panic(err) // real error + } + + log.Printf("No matching docker image, creating...") + err = dockerBuild(dockerFileContent, containerName, dockerfileHash) + if err != nil { + log.Panic(err) + } + + // Search again + _, err = dockerFindImage(containerName, dockerfileHash) + if err != nil { + log.Printf("Failed to build container for %s:%s", containerName, dockerfileHash) + log.Panic(err) // Any error now is a real error + } + } + + // Container match found - safe to run our command + + fullCommand := []string{"run"} + + if runtime.GOOS != "windows" { + userinfo, err := user.Current() + if err != nil { + log.Panic(err) + } + + fullCommand = append(fullCommand, `--user`, userinfo.Uid+`:`+userinfo.Gid) + } + + // Find the GOMODCACHE and GOCACHE to populate mapped volumes + gomodcache, err := exec.Command(`go`, `env`, `GOMODCACHE`).Output() + if err != nil { + log.Panic(err) + } + if gomodcache_sz := strings.TrimSpace(string(gomodcache)); len(gomodcache_sz) > 0 { + fullCommand = append(fullCommand, `-v`, gomodcache_sz+`:/go/pkg/mod`, `-e`, `GOMODCACHE=/go/pkg/mod`) + } + + gocache, err := exec.Command(`go`, `env`, `GOCACHE`).Output() + if err != nil { + log.Panic(err) + } + if gocache_sz := strings.TrimSpace(string(gocache)); len(gocache_sz) > 0 { + fullCommand = append(fullCommand, `-v`, gocache_sz+`:/.cache/go-build`, `-e`, `GOCACHE=/.cache/go-build`) + } + + // We need to bind-mount probably not just the current working directory, + // but upwards to the root git repo / go.mod file / go.work file (whichever + // is highest) + + var parentPaths []string + gomod, err := exec.Command(`go`, `env`, `GOMOD`).Output() + if err != nil { + log.Panic(err) + } + if gomod_sz := strings.TrimSpace(string(gomod)); len(gomod_sz) > 0 { + parentPaths = append(parentPaths, gomod_sz) + } + + gowork, err := exec.Command(`go`, `env`, `GOWORK`).Output() + if err != nil { + log.Panic(err) + } + if gowork_sz := strings.TrimSpace(string(gowork)); len(gowork_sz) > 0 { + parentPaths = append(parentPaths, gowork_sz) + } + + gitroot, err := exec.Command(`git`, `rev-parse`, `--show-toplevel`).Output() + if err != nil { + // Maybe this isn't a git repository? Git is optional anyway, there are hg/bzr users + // Don't panic + } else { + if gitroot_sz := strings.TrimSpace(string(gitroot)); len(gitroot_sz) > 0 { + parentPaths = append(parentPaths, gitroot_sz) + } + } + + cwd, err := os.Getwd() + if err != nil { + log.Panic(err) + } + + parentPaths = append(parentPaths, cwd) // It's an option too + + basedir, err := highestCommonParent(parentPaths) + if err != nil { + log.Panic(err) + } + + relCwd, err := filepath.Rel(basedir, cwd) + if err != nil { + log.Panic(err) + } + + fullCommand = append(fullCommand, `-v`, basedir+`:/src`, `-w`, filepath.Join(`/src`, relCwd)) + + // Final standard docker commands + + fullCommand = append(fullCommand, containerName+`:`+dockerfileHash) // , `/bin/bash`, `-c`) + fullCommand = append(fullCommand, os.Args[2:]...) + + cmd := dockerCommand(fullCommand...) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + err = cmd.Run() + if err != nil { + log.Fatal(err) + } +} diff --git a/docker/embed.go b/docker/embed.go new file mode 100644 index 00000000..21c019c4 --- /dev/null +++ b/docker/embed.go @@ -0,0 +1,16 @@ +// This Go file exports all the *.Dockerfile files for miqt-docker to use. +package docker + +import ( + "embed" +) + +//go:embed *.Dockerfile +var Dockerfiles embed.FS + +// ReadFile returns the content of one of the dockerfiles. +// That's because an embed.FS appears out-of-package as a []fs.DirEntry, which +// isn't directly readable. +func ReadFile(name string) ([]byte, error) { + return Dockerfiles.ReadFile(name) +}