Initial commit

utils/encrypter.go

@@ -0,0 +1,73 @@
+package utils
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+)
+
+func Encrypt(stringToEncrypt string, keyString string) (encryptedString string) {
+
+	//Since the key is in string, we need to convert decode it to bytes
+	key, _ := hex.DecodeString(keyString)
+	plaintext := []byte(stringToEncrypt)
+
+	//Create a new Cipher Block from the key
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	//Create a new GCM - https://en.wikipedia.org/wiki/Galois/Counter_Mode
+	//https://golang.org/pkg/crypto/cipher/#NewGCM
+	aesGCM, err := cipher.NewGCM(block)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	//Create a nonce. Nonce should be from GCM
+	nonce := make([]byte, aesGCM.NonceSize())
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		panic(err.Error())
+	}
+
+	//Encrypt the data using aesGCM.Seal
+	//Since we don't want to save the nonce somewhere else in this case, we add it as a prefix to the encrypted data. The first nonce argument in Seal is the prefix.
+	ciphertext := aesGCM.Seal(nonce, nonce, plaintext, nil)
+	return fmt.Sprintf("%x", ciphertext)
+}
+
+func Decrypt(encryptedString string, keyString string) (decryptedString string) {
+
+	key, _ := hex.DecodeString(keyString)
+	enc, _ := hex.DecodeString(encryptedString)
+
+	//Create a new Cipher Block from the key
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	//Create a new GCM
+	aesGCM, err := cipher.NewGCM(block)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	//Get the nonce size
+	nonceSize := aesGCM.NonceSize()
+
+	//Extract the nonce from the encrypted data
+	nonce, ciphertext := enc[:nonceSize], enc[nonceSize:]
+
+	//Decrypt the data
+	plaintext, err := aesGCM.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	return fmt.Sprintf("%s", plaintext)
+}

utils/xopen/fastgzreader.go

@@ -0,0 +1,80 @@
+package xopen
+
+import (
+	"compress/gzip"
+	"io"
+	"os/exec"
+)
+
+// from: https://gist.github.com/rasky/d42a52c16683f1a2f4dccdef80e2712d
+
+// fastGzReader is an API-compatible drop-in replacement
+// for gzip.Reader, that achieves a higher decoding speed
+// by spawning an external gzip instance and pipeing data
+// through it.
+// Go's native gzip implementation is about 2x slower at
+// decompressing data compared to zlib (mostly due to Go compiler
+// inefficiencies). So for tasks where the gzip decoding
+// speed is important, this is a quick workaround that doesn't
+// require cgo.
+// gzip is part of the gzip package and comes preinstalled on
+// most Linux distributions and on OSX.
+type fastGzReader struct {
+	io.ReadCloser
+}
+
+func hasProg(prog ...string) bool {
+	var cmd *exec.Cmd
+	if len(prog) > 1 {
+		cmd = exec.Command(prog[0], prog[1:]...)
+	} else {
+		cmd = exec.Command(prog[0])
+	}
+	err := cmd.Start()
+	has := err == nil
+	cmd.Wait()
+	return has
+}
+
+var hasZlib = hasProg("gzip", "-d")
+var hasPigz = hasProg("pigz", "-d")
+
+func newFastGzReader(r io.Reader) (io.ReadCloser, error) {
+
+	if hasZlib || hasPigz {
+		var gz fastGzReader
+		if err := gz.Reset(r); err != nil {
+			return nil, err
+		}
+		return gz, nil
+	}
+	return gzip.NewReader(r)
+
+}
+
+func (gz *fastGzReader) Reset(r io.Reader) error {
+	if gz.ReadCloser != nil {
+		gz.Close()
+	}
+	var cmd *exec.Cmd
+	if hasPigz {
+		cmd = exec.Command("pigz", "-d")
+	} else {
+		cmd = exec.Command("gzip", "-d")
+	}
+	cmd.Stdin = r
+
+	rpipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		rpipe.Close()
+		return err
+	}
+
+	gz.ReadCloser = rpipe
+	return nil
+}

utils/xopen/xopen.go

@@ -0,0 +1,310 @@
+// Package xopen makes it easy to get buffered readers and writers.
+// Ropen opens a (possibly gzipped) file/process/http site for buffered reading.
+// Wopen opens a (possibly gzipped) file for buffered writing.
+// Both will use gzip when appropriate and will user buffered IO.
+package xopen
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
+	"strings"
+
+	gzip "github.com/klauspost/pgzip"
+	//"github.com/klauspost/compress/gzip"
+	// "compress/gzip"
+)
+
+// ErrNoContent means nothing in the stream/file.
+var ErrNoContent = errors.New("xopen: no content")
+
+// ErrDirNotSupported means the path is a directory.
+var ErrDirNotSupported = errors.New("xopen: input is a directory")
+
+// IsGzip returns true buffered Reader has the gzip magic.
+func IsGzip(b *bufio.Reader) (bool, error) {
+	return CheckBytes(b, []byte{0x1f, 0x8b})
+}
+
+// IsStdin checks if we are getting data from stdin.
+func IsStdin() bool {
+	// http://stackoverflow.com/a/26567513
+	stat, err := os.Stdin.Stat()
+	if err != nil {
+		return false
+	}
+	return (stat.Mode() & os.ModeCharDevice) == 0
+}
+
+// ExpandUser expands ~/path and ~otheruser/path appropriately
+func ExpandUser(path string) (string, error) {
+	if path[0] != '~' {
+		return path, nil
+	}
+	var u *user.User
+	var err error
+	if len(path) == 1 || path[1] == '/' {
+		u, err = user.Current()
+	} else {
+		name := strings.Split(path[1:], "/")[0]
+		u, err = user.Lookup(name)
+	}
+	if err != nil {
+		return "", err
+	}
+	home := u.HomeDir
+	path = home + "/" + path[1:]
+	return path, nil
+}
+
+// Exists checks if a local file exits
+func Exists(path string) bool {
+	path, perr := ExpandUser(path)
+	if perr != nil {
+		return false
+	}
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+// CheckBytes peeks at a buffered stream and checks if the first read bytes match.
+func CheckBytes(b *bufio.Reader, buf []byte) (bool, error) {
+
+	m, err := b.Peek(len(buf))
+	if err != nil {
+		return false, ErrNoContent
+	}
+	for i := range buf {
+		if m[i] != buf[i] {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+// Reader is returned by Ropen
+type Reader struct {
+	*bufio.Reader
+	rdr io.Reader
+	gz  io.ReadCloser
+}
+
+// Close the associated files.
+func (r *Reader) Close() error {
+	if r.gz != nil {
+		r.gz.Close()
+	}
+	if c, ok := r.rdr.(io.ReadCloser); ok {
+		c.Close()
+	}
+	return nil
+}
+
+// Writer is returned by Wopen
+type Writer struct {
+	*bufio.Writer
+	wtr *os.File
+	gz  *gzip.Writer
+}
+
+// Close the associated files.
+func (w *Writer) Close() error {
+	w.Flush()
+	if w.gz != nil {
+		w.gz.Close()
+	}
+	w.wtr.Close()
+	return nil
+}
+
+// Flush the writer.
+func (w *Writer) Flush() {
+	w.Writer.Flush()
+	if w.gz != nil {
+		w.gz.Flush()
+	}
+}
+
+var pageSize = os.Getpagesize() * 2
+
+// Buf returns a buffered reader from an io.Reader
+// If f == "-", then it will attempt to read from os.Stdin.
+// If the file is gzipped, it will be read as such.
+func Buf(r io.Reader) (*Reader, error) {
+	b := bufio.NewReaderSize(r, pageSize)
+	var rdr io.ReadCloser
+	if is, err := IsGzip(b); err != nil && err != io.EOF {
+		return nil, err
+	} else if is {
+		// rdr, err = newFastGzReader(b)
+		rdr, err = gzip.NewReader(b)
+		if err != nil {
+			return nil, err
+		}
+		b = bufio.NewReaderSize(rdr, pageSize)
+	}
+
+	// check BOM
+	t, _, err := b.ReadRune()
+	if err != nil {
+		return nil, ErrNoContent
+	}
+	if t != '\uFEFF' {
+		b.UnreadRune()
+	}
+	return &Reader{b, r, rdr}, nil
+}
+
+// XReader returns a reader from a url string or a file.
+func XReader(f string) (io.Reader, error) {
+	if strings.HasPrefix(f, "http://") || strings.HasPrefix(f, "https://") {
+		var rsp *http.Response
+		rsp, err := http.Get(f)
+		if err != nil {
+			return nil, err
+		}
+		if rsp.StatusCode != 200 {
+			return nil, fmt.Errorf("http error downloading %s. status: %s", f, rsp.Status)
+		}
+		rdr := rsp.Body
+		return rdr, nil
+	}
+	f, err := ExpandUser(f)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := os.Stat(f)
+	if err != nil {
+		return nil, err
+	}
+	if fi.IsDir() {
+		return nil, ErrDirNotSupported
+	}
+
+	return os.Open(f)
+}
+
+// Ropen opens a buffered reader.
+func Ropen(f string) (*Reader, error) {
+	var err error
+	var rdr io.Reader
+	if f == "-" {
+		if !IsStdin() {
+			return nil, errors.New("stdin not detected")
+		}
+		b, err := Buf(os.Stdin)
+		return b, err
+	} else if f[0] == '|' {
+		// TODO: use csv to handle quoted file names.
+		cmdStrs := strings.Split(f[1:], " ")
+		var cmd *exec.Cmd
+		if len(cmdStrs) == 2 {
+			cmd = exec.Command(cmdStrs[0], cmdStrs[1:]...)
+		} else {
+			cmd = exec.Command(cmdStrs[0])
+		}
+		rdr, err = cmd.StdoutPipe()
+		if err != nil {
+			return nil, err
+		}
+		err = cmd.Start()
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		rdr, err = XReader(f)
+	}
+	if err != nil {
+		return nil, err
+	}
+	b, err := Buf(rdr)
+	return b, err
+}
+
+// Wopen opens a buffered reader.
+// If f == "-", then stdout will be used.
+// If f endswith ".gz", then the output will be gzipped.
+func Wopen(f string) (*Writer, error) {
+	var wtr *os.File
+	if f == "-" {
+		wtr = os.Stdout
+	} else {
+		dir := filepath.Dir(f)
+		fi, err := os.Stat(dir)
+		if err == nil && !fi.IsDir() {
+			return nil, fmt.Errorf("can not write file into a non-directory path: %s", dir)
+		}
+		if os.IsNotExist(err) {
+			os.MkdirAll(dir, 0755)
+		}
+
+		wtr, err = os.Create(f)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if !strings.HasSuffix(f, ".gz") {
+		return &Writer{bufio.NewWriterSize(wtr, pageSize), wtr, nil}, nil
+	}
+	gz := gzip.NewWriter(wtr)
+	return &Writer{bufio.NewWriterSize(gz, pageSize), wtr, gz}, nil
+}
+
+// WopenGzip opens a buffered gzipped reader.
+// If f == "-", then stdout will be used.
+func WopenGzip(f string) (*Writer, error) {
+	var wtr *os.File
+	if f == "-" {
+		wtr = os.Stdout
+	} else {
+		dir := filepath.Dir(f)
+		fi, err := os.Stat(dir)
+		if err == nil && !fi.IsDir() {
+			return nil, fmt.Errorf("can not write file into a non-directory path: %s", dir)
+		}
+		if os.IsNotExist(err) {
+			os.MkdirAll(dir, 0755)
+		}
+		wtr, err = os.Create(f)
+		if err != nil {
+			return nil, err
+		}
+	}
+	gz := gzip.NewWriter(wtr)
+	return &Writer{bufio.NewWriterSize(gz, pageSize), wtr, gz}, nil
+}
+
+// WopenFile opens a buffered reader.
+// If f == "-", then stdout will be used.
+// If f endswith ".gz", then the output will be gzipped.
+func WopenFile(f string, flag int, perm os.FileMode) (*Writer, error) {
+	var wtr *os.File
+	if f == "-" {
+		wtr = os.Stdout
+	} else {
+		dir := filepath.Dir(f)
+		fi, err := os.Stat(dir)
+		if err == nil && !fi.IsDir() {
+			return nil, fmt.Errorf("can not write file into a non-directory path: %s", dir)
+		}
+		if os.IsNotExist(err) {
+			os.MkdirAll(dir, 0755)
+		}
+		wtr, err = os.OpenFile(f, flag, perm)
+		if err != nil {
+			return nil, err
+		}
+	}
+	if !strings.HasSuffix(f, ".gz") {
+		return &Writer{bufio.NewWriterSize(wtr, pageSize), wtr, nil}, nil
+	}
+	gz := gzip.NewWriter(wtr)
+	return &Writer{bufio.NewWriterSize(gz, pageSize), wtr, gz}, nil
+}

utils/xopen/xopen_test.go

@@ -0,0 +1,154 @@
+package xopen
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type XopenTest struct{}
+
+var _ = Suite(&XopenTest{})
+
+func gzFromString(s string) string {
+	var c bytes.Buffer
+	gz := gzip.NewWriter(&c)
+	gz.Write([]byte(s))
+	return c.String()
+}
+
+var gzTests = []struct {
+	isGz bool
+	data string
+}{
+	{false, "asdf"},
+	{true, gzFromString("asdf")},
+}
+
+func (s *XopenTest) TestIsGzip(c *C) {
+	for _, t := range gzTests {
+		isGz, err := IsGzip(bufio.NewReader(strings.NewReader(t.data)))
+		c.Assert(err, IsNil)
+		c.Assert(t.isGz, Equals, isGz)
+	}
+}
+
+func (s *XopenTest) TestIsStdin(c *C) {
+	r := IsStdin()
+	c.Assert(r, Equals, false)
+}
+
+func (s *XopenTest) TestRopen(c *C) {
+	rdr, err := Ropen("-")
+	c.Assert(err, ErrorMatches, ".* stdin not detected")
+	c.Assert(rdr, IsNil)
+}
+
+func (s *XopenTest) TestWopen(c *C) {
+	for _, f := range []string{"t.gz", "t"} {
+		testString := "ASDF1234"
+		wtr, err := Wopen(f)
+		c.Assert(err, IsNil)
+		_, err = os.Stat(f)
+		c.Assert(err, IsNil)
+		c.Assert(wtr.wtr, NotNil)
+		fmt.Fprint(wtr, testString)
+		wtr.Close()
+
+		rdr, err := Ropen(f)
+		c.Assert(err, IsNil)
+
+		str, err := rdr.ReadString(99)
+		c.Assert(str, Equals, testString)
+		c.Assert(err, Equals, io.EOF)
+		str, err = rdr.ReadString(99)
+		c.Assert(str, Equals, "")
+
+		rdr.Close()
+		os.Remove(f)
+	}
+}
+
+var httpTests = []struct {
+	url         string
+	expectError bool
+}{
+	{"https://raw.githubusercontent.com/brentp/xopen/master/README.md", false},
+	{"http://raw.githubusercontent.com/brentp/xopen/master/README.md", false},
+	{"http://raw.githubusercontent.com/brentp/xopen/master/BAD.md", true},
+}
+
+func (s *XopenTest) TestReadHttp(c *C) {
+	for _, t := range httpTests {
+		rdr, err := Ropen(t.url)
+		if !t.expectError {
+			c.Assert(err, IsNil)
+			v, err := rdr.ReadString(byte('\n'))
+			c.Assert(err, IsNil)
+			c.Assert(len(v), Not(Equals), 0)
+		} else {
+			c.Assert(err, ErrorMatches, ".* 404 Not Found")
+		}
+	}
+}
+
+func (s *XopenTest) TestReadProcess(c *C) {
+	for _, cmd := range []string{"|ls -lh", "|ls", "|ls -lh xopen_test.go"} {
+		rdr, err := Ropen(cmd)
+		c.Assert(err, IsNil)
+		b := make([]byte, 1000)
+		_, err = rdr.Read(b)
+		if err != io.EOF {
+			c.Assert(err, IsNil)
+		}
+		lines := strings.Split(string(b), "\n")
+		has := false
+		for _, line := range lines {
+			if strings.Contains(line, "xopen_test.go") {
+				has = true
+			}
+		}
+		c.Assert(has, Equals, true)
+	}
+}
+
+func (s *XopenTest) TestOpenStdout(c *C) {
+	w, err := Wopen("-")
+	c.Assert(err, IsNil)
+	c.Assert(w.wtr, Equals, os.Stdout)
+}
+
+func (s *XopenTest) TestOpenBadFile(c *C) {
+	r, err := Ropen("XXXXXXXXXXXXXXXXXXXXXXX")
+	c.Assert(r, IsNil)
+	c.Assert(err, ErrorMatches, ".* no such file .*")
+}
+
+func (s *XopenTest) TestWOpenBadFile(c *C) {
+	w, err := Wopen("XX/XXX/XXX/XXX/XXX/XXXXXXXXX")
+	c.Assert(w, IsNil)
+	c.Assert(err, ErrorMatches, ".* no such file .*")
+}
+
+func (s *XopenTest) TestExists(c *C) {
+	c.Assert(Exists("xopen.go"), Equals, true)
+	c.Assert(Exists("____xx"), Equals, false)
+}
+
+func (s *XopenTest) TestUser(c *C) {
+	c.Assert(Exists("~"), Equals, true)
+}
+
+func (s *XopenTest) TestExpand(c *C) {
+	_, err := ExpandUser("~baduser66")
+	c.Assert(err, Not(IsNil))
+}