Initial commit

utils/xopen/fastgzreader.go

@@ -0,0 +1,80 @@
+package xopen
+
+import (
+	"compress/gzip"
+	"io"
+	"os/exec"
+)
+
+// from: https://gist.github.com/rasky/d42a52c16683f1a2f4dccdef80e2712d
+
+// fastGzReader is an API-compatible drop-in replacement
+// for gzip.Reader, that achieves a higher decoding speed
+// by spawning an external gzip instance and pipeing data
+// through it.
+// Go's native gzip implementation is about 2x slower at
+// decompressing data compared to zlib (mostly due to Go compiler
+// inefficiencies). So for tasks where the gzip decoding
+// speed is important, this is a quick workaround that doesn't
+// require cgo.
+// gzip is part of the gzip package and comes preinstalled on
+// most Linux distributions and on OSX.
+type fastGzReader struct {
+	io.ReadCloser
+}
+
+func hasProg(prog ...string) bool {
+	var cmd *exec.Cmd
+	if len(prog) > 1 {
+		cmd = exec.Command(prog[0], prog[1:]...)
+	} else {
+		cmd = exec.Command(prog[0])
+	}
+	err := cmd.Start()
+	has := err == nil
+	cmd.Wait()
+	return has
+}
+
+var hasZlib = hasProg("gzip", "-d")
+var hasPigz = hasProg("pigz", "-d")
+
+func newFastGzReader(r io.Reader) (io.ReadCloser, error) {
+
+	if hasZlib || hasPigz {
+		var gz fastGzReader
+		if err := gz.Reset(r); err != nil {
+			return nil, err
+		}
+		return gz, nil
+	}
+	return gzip.NewReader(r)
+
+}
+
+func (gz *fastGzReader) Reset(r io.Reader) error {
+	if gz.ReadCloser != nil {
+		gz.Close()
+	}
+	var cmd *exec.Cmd
+	if hasPigz {
+		cmd = exec.Command("pigz", "-d")
+	} else {
+		cmd = exec.Command("gzip", "-d")
+	}
+	cmd.Stdin = r
+
+	rpipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		rpipe.Close()
+		return err
+	}
+
+	gz.ReadCloser = rpipe
+	return nil
+}