A command to truncate and then concatenate Google Cloud Storage files

concat.go

package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strings"
)

func rm(gsutil, f string) error {
	cmd := exec.Command(gsutil, "rm", f)
	errbuf := bytes.NewBuffer([]byte{})
	cmd.Stderr = errbuf
	err := cmd.Run()
	if err != nil {
		errout := string(errbuf.Bytes())
		if !strings.HasPrefix(errout, "CommandException: No URLs matched") {
			return fmt.Errorf("rm error %s", errout)
		}
	}
	return nil
}

func cp0(gsutil, f string) error {
	cmd := exec.Command(gsutil, "cp", "-", f)
	cmd.Stdin = bytes.NewBuffer([]byte{})
	errbuf := bytes.NewBuffer([]byte{})
	cmd.Stderr = errbuf
	err := cmd.Run()
	if err != nil {
		errout := string(errbuf.Bytes())
		return fmt.Errorf("copy zero bytes error %s", errout)
	}
	return nil
}

func ls(gsutil string, arg ...string) (lines []string, err error) {
	gsarg := []string{"ls"}
	gsarg = append(gsarg, arg...)
	cmd := exec.Command(gsutil, gsarg...)
	errbuf := bytes.NewBuffer([]byte{})
	cmd.Stderr = errbuf
	outbuf := bytes.NewBuffer([]byte{})
	cmd.Stdout = outbuf
	err = cmd.Run()
	if err != nil {
		errout := string(errbuf.Bytes())
		return nil, fmt.Errorf("ls error %s", errout)
	}
	lines = strings.Split(strings.TrimSpace(string(outbuf.Bytes())), "\n")
	return lines, nil
}

const maxComposeFiles = 32

// Append appends all the objects specified in sources to the object dest using the compose command.
func Append(gsutil string, sources []string, dest string, verbose bool) (int, error) {
	cnt := 0
	for _, s := range sources {
		files, err := ls(gsutil, s)
		if err != nil {
			return cnt, err
		}
		batch := []string{}
		for i, f := range files {
			batch = append(batch, f)
			if len(batch) == maxComposeFiles-1 || i == len(files)-1 {
				gscmd := []string{"compose"}
				gscmd = append(gscmd, batch...)
				gscmd = append(gscmd, dest)
				gscmd = append(gscmd, dest)
				cmd := exec.Command(gsutil, gscmd...)
				errbuf := bytes.NewBuffer([]byte{})
				cmd.Stderr = errbuf
				err = cmd.Run()
				if err != nil {
					errout := string(errbuf.Bytes())
					return cnt, fmt.Errorf("compose error %s", errout)
				}

				if verbose {
					for _, b := range batch {
						fmt.Printf("%s\n", b)
					}
				}

				cnt += len(batch)
				batch = []string{}
			}
		}
	}
	return cnt, nil
}

var verbose bool

func init() {
	const (
		defaultVerbose = false
		verboseUsage   = "verbosity of output"
	)
	flag.BoolVar(&verbose, "verbose", defaultVerbose, verboseUsage)
	flag.BoolVar(&verbose, "v", defaultVerbose, verboseUsage+" (shorthand)")
}

func main() {
	flag.Parse()
	if flag.NArg() < 3 {
		fmt.Fprintf(os.Stderr, "Usage: %s [options...] path_to_gsutil sources... destination\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}
	gsutil := flag.Arg(0)
	sources := flag.Args()[1 : flag.NArg()-1]
	dest := flag.Arg(flag.NArg() - 1)

	err := cp0(gsutil, dest)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		os.Exit(1)
	}

	cnt, err := Append(gsutil, sources, dest, verbose)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v", err)
		os.Exit(1)
	}
	fmt.Printf("%d\n", cnt)
}