tools/gostatic/tarify/main.go

package main

// tarify implements a minimal, self-contained, hermetic tarball builder.
// It is currently used with gostatic to take a non-hermetic directory and
// turn it into a hermetic tarball via a glob.
//
// For more information about tree artifacts and hermeticity, see:
// https://jmmv.dev/2019/12/bazel-dynamic-execution-tree-artifacts.html

import (
	"archive/tar"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/golang/glog"
)

var (
	flagSite    string
	flagTarball string
)

func init() {
	flag.Set("logtostderr", "true")
}

func main() {
	flag.StringVar(&flagSite, "site", "", "Site sources")
	flag.StringVar(&flagTarball, "tarball", "", "Output tarball")
	flag.Parse()

	if flagSite == "" {
		glog.Exitf("-site must be set")
	}
	if flagTarball == "" {
		glog.Exitf("-tarball must be set")
	}

	f, err := os.Create(flagTarball)
	if err != nil {
		glog.Exitf("Create(%q): %v", flagTarball, err)
	}
	defer f.Close()
	w := tar.NewWriter(f)
	defer w.Close()

	flagSite = strings.TrimSuffix(flagSite, "/")

	// First retrieve all files and sort. This is required for idempotency.
	elems := []struct {
		path string
		info os.FileInfo
	}{}
	err = filepath.Walk(flagSite, func(inPath string, _ os.FileInfo, err error) error {
		// We don't use the given fileinfo, as we want to deref symlinks.
		info, err := os.Stat(inPath)
		if err != nil {
			return fmt.Errorf("Stat: %w", err)
		}
		elems = append(elems, struct {
			path string
			info os.FileInfo
		}{inPath, info})
		return nil
	})
	if err != nil {
		glog.Exitf("Walk(%q, _): %v", flagSite, err)
	}
	sort.Slice(elems, func(i, j int) bool { return elems[i].path < elems[j].path })

	// Now that we have a sorted list, tar 'em up.
	for _, elem := range elems {
		inPath := elem.path
		info := elem.info

		outPath := strings.TrimPrefix(strings.TrimPrefix(inPath, flagSite), "/")
		if outPath == "" {
			continue
		}
		if info.IsDir() {
			glog.Infof("D %s", outPath)
			if err := w.WriteHeader(&tar.Header{
				Typeflag: tar.TypeDir,
				Name:     outPath,
				Mode:     0755,
			}); err != nil {
				glog.Exitf("Writing directory header for %q failed: %v", inPath, err)
			}
		} else {
			glog.Infof("F %s", outPath)
			if err := w.WriteHeader(&tar.Header{
				Typeflag: tar.TypeReg,
				Name:     outPath,
				Mode:     0644,
				// TODO(q3k): this can race (TOCTOU Stat/Open, resulting in "archive/tar: write Too long")
				// No idea, how to handle this better though without reading the entire file into memory,
				// or trying to do filesystem locks? Besides, in practical use with Bazel this will never
				// happen.
				Size: info.Size(),
			}); err != nil {
				glog.Exitf("Writing file header for %q failed: %v", inPath, err)
			}
			r, err := os.Open(inPath)
			if err != nil {
				glog.Exitf("Open(%q): %v", inPath, err)
			}
			defer r.Close()
			if _, err := io.Copy(w, r); err != nil {
				glog.Exitf("Copy(%q): %v", inPath, err)
			}

		}
	}
}