genbindings: run multiple clang workers in parallel

cmd/genbindings/clangexec.go

@@ -6,9 +6,16 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"os"
 	"os/exec"
 	"sync"
+	"time"
+)
+
+const (
+	ClangMaxRetries = 5
+	ClangRetryDelay = 3 * time.Second
 )
 
 func clangExec(ctx context.Context, clangBin, inputHeader string, cflags []string) ([]interface{}, error) {
@@ -55,6 +62,26 @@ func clangExec(ctx context.Context, clangBin, inputHeader string, cflags []strin
 	return inner, nil
 }
 
+func mustClangExec(ctx context.Context, clangBin, inputHeader string, cflags []string) []interface{} {
+
+	for i := 0; i < ClangMaxRetries; i++ {
+		astInner, err := clangExec(ctx, clangBin, inputHeader, cflags)
+		if err != nil {
+			// Log and continue with next retry
+			log.Printf("WARNING: Clang execution failed: %v", err)
+			time.Sleep(ClangRetryDelay)
+			log.Printf("Retrying...")
+		}
+
+		// Success
+		return astInner
+	}
+
+	// Failed 5x
+	// Panic
+	panic("Clang failed 5x parsing file " + inputHeader)
+}
+
 // clangStripUpToFile strips all AST nodes from the clang output until we find
 // one that really originated in the source file.
 // This cleans out everything in the translation unit that came from an

cmd/genbindings/main.go

@@ -8,8 +8,13 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
-	"time"
+	"sync"
+)
+
+const (
+	ClangSubprocessCount = 3
 )
 
 func cacheFilePath(inputHeader string) string {
@@ -90,35 +95,30 @@ func main() {
 
 	InsertTypedefs()
 
-	for _, inputHeader := range includeFiles {
+	//
+	// PASS 0 (Fill clang cache)
+	//
 
-		// If we have a cached clang AST, use that instead
+	var clangChan = make(chan string, 0)
-		cacheFile := cacheFilePath(inputHeader)
+	var clangWg sync.WaitGroup
-		astJson, err := ioutil.ReadFile(cacheFile)
-		var astInner []interface{} = nil
-		if err != nil {
 
-			// Nonexistent cache file, regenerate from clang
+	for i := 0; i < ClangSubprocessCount; i++ {
-			log.Printf("No AST cache for file %q, running clang...", filepath.Base(inputHeader))
+		clangWg.Add(1)
+		go func() {
+			defer clangWg.Done()
+			log.Printf("Clang worker: starting")
+
+			for {
+				inputHeader, ok := <-clangChan
+				if !ok {
+					return // Done
+				}
+
+				log.Printf("Clang worker got message for file %q", inputHeader)
 
 				// Parse the file
 				// This seems to intermittently fail, so allow retrying
-		nextRetry:
+				astInner := mustClangExec(ctx, *clang, inputHeader, strings.Fields(*cflags))
-			for retryCt := 0; retryCt < 5; retryCt++ {
-				astInner, err = clangExec(ctx, *clang, inputHeader, strings.Fields(*cflags))
-				if err != nil {
-					// Log and continue with next retry
-					log.Printf("WARNING: Clang execution failed: %v", err)
-					time.Sleep(3 * time.Second)
-					log.Printf("Retrying...")
-
-				} else { // err == nil
-					break nextRetry
-				}
-			}
-			if err != nil {
-				panic("Clang execution failed after 5x retries")
-			}
 
 				// Write to cache
 				jb, err := json.MarshalIndent(astInner, "", "\t")
@@ -126,22 +126,59 @@ func main() {
 					panic(err)
 				}
 
-			err = ioutil.WriteFile(cacheFile, jb, 0644)
+				err = ioutil.WriteFile(cacheFilePath(inputHeader), jb, 0644)
 				if err != nil {
 					panic(err)
 				}
 
-		} else {
+				astInner = nil
-			log.Printf("Reused cache AST for file %q", filepath.Base(inputHeader))
+				jb = nil
+				runtime.GC()
+
+			}
+			log.Printf("Clang worker: exiting")
+		}()
+	}
+
+	for _, inputHeader := range includeFiles {
+
+		// Check if there is a matching cache hit
+		cacheFile := cacheFilePath(inputHeader)
+
+		if _, err := os.Stat(cacheFile); err != nil && os.IsNotExist(err) {
+
+			// Nonexistent cache file, regenerate from clang
+			log.Printf("No AST cache for file %q, running clang...", filepath.Base(inputHeader))
+			clangChan <- inputHeader
+		}
+	}
+
+	// Done with all clang workers
+	close(clangChan)
+	clangWg.Wait()
+
+	// The cache should now be fully populated.
+
+	//
+	// PASS 1 (clang2il)
+	//
+
+	for _, inputHeader := range includeFiles {
+
+		cacheFile := cacheFilePath(inputHeader)
+
+		astJson, err := ioutil.ReadFile(cacheFile)
+		if err != nil {
+			panic("Expected cache to be created for " + inputHeader + ", but got error " + err.Error())
+		}
 
 		// Json decode
+		var astInner []interface{} = nil
 		err = json.Unmarshal(astJson, &astInner)
 		if err != nil {
 			panic(err)
 		}
 
-		}
-
 		// Convert it to our intermediate format
 		parsed, err := parseHeader(astInner, "")
 		if err != nil {