diff --git a/go.mod b/go.mod index a14bdac0..ccc17a36 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,11 @@ module github.com/schollz/croc/v6 require ( + github.com/cespare/xxhash v1.1.0 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 github.com/denisbrodbeck/machineid v1.0.1 github.com/fatih/color v1.7.0 // indirect + github.com/kalafut/imohash v1.0.0 github.com/mattn/go-colorable v0.1.1 // indirect github.com/mattn/go-isatty v0.0.7 // indirect github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index a14f75bd..9a4b7836 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -7,6 +11,8 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/kalafut/imohash v1.0.0 h1:LgCJ+p/BwM2HKpOxFopkeddpzVCfm15EtXMroXD1SYE= +github.com/kalafut/imohash v1.0.0/go.mod h1:c3RHT80ZAp5C/aYgQI92ZlrOymqkZnRDprU87kg75HI= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -30,6 +36,8 @@ github.com/schollz/progressbar/v2 v2.12.1 h1:0Ce7IBClG+s3lxXN1Noqwh7aToKGL5a3mnM github.com/schollz/progressbar/v2 v2.12.1/go.mod h1:fBI3onORwtNtwCWJHsrXtjE3QnJOtqIZrvr3rDaF7L0= github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9 h1:y08o5oQ/slxXE/F0uh5dd8mdVvb+w4NLcNSDSq4c2F0= github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9/go.mod h1:kCMoQsqzx4MzGJWaALr6tKyCnlrY0kILGLkA1FOiLF4= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -57,3 +65,5 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/tylerb/is.v1 v1.1.2 h1:AB/MANFml2ySf+adwcinvajyHvsYltAOD+rb/8njfSU= +gopkg.in/tylerb/is.v1 v1.1.2/go.mod h1:9yQB2tyIhZ5oph6Kk5Sq7cJMd9c5Jpa1p3hr9kxzPqo= diff --git a/src/utils/utils.go b/src/utils/utils.go index 55e411cc..5debe330 100644 --- a/src/utils/utils.go +++ b/src/utils/utils.go @@ -2,6 +2,7 @@ package utils import ( "bufio" + "bytes" "crypto/md5" "crypto/rand" "crypto/sha256" @@ -9,13 +10,14 @@ import ( "io" "io/ioutil" "log" - "bytes" - "net" "math" + "net" "net/http" "os" "strings" + "github.com/cespare/xxhash" + "github.com/kalafut/imohash" "github.com/schollz/mnemonicode" ) @@ -37,8 +39,12 @@ func GetInput(prompt string) string { return strings.TrimSpace(text) } -// HashFile returns the md5 hash of a file +// HashFile returns the hash of a file func HashFile(fname string) (hash256 []byte, err error) { + return IMOHashFile(fname) +} + +func MD5HashFile(fname string) (hash256 []byte, err error) { f, err := os.Open(fname) if err != nil { return @@ -54,6 +60,30 @@ func HashFile(fname string) (hash256 []byte, err error) { return } +// IMOHashFile returns imohash +func IMOHashFile(fname string) (hash []byte, err error) { + b, err := imohash.SumFile(fname) + hash = b[:] + return +} + +// XXHashFile returns the xxhash of a file +func XXHashFile(fname string) (hash256 []byte, err error) { + f, err := os.Open(fname) + if err != nil { + return + } + defer f.Close() + + h := xxhash.New() + if _, err = io.Copy(h, f); err != nil { + return + } + + hash256 = h.Sum(nil) + return +} + // SHA256 returns sha256 sum func SHA256(s string) string { sha := sha256.New() @@ -112,8 +142,6 @@ func ByteCountDecimal(b int64) string { return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp]) } - - // MissingChunks returns the positions of missing chunks. // If file doesn't exist, it returns an empty chunk list (all chunks). // If the file size is not the same as requested, it returns an empty chunk list (all chunks). diff --git a/src/utils/utils_test.go b/src/utils/utils_test.go index a3e0e1f3..179540c0 100644 --- a/src/utils/utils_test.go +++ b/src/utils/utils_test.go @@ -2,24 +2,60 @@ package utils import ( "fmt" + "io/ioutil" + "math/rand" "testing" "github.com/stretchr/testify/assert" ) -// Exists reports whether the named file or directory exists. +func init() { + rand.Seed(0) + bigBuff := make([]byte, 75000000) + rand.Read(bigBuff) + ioutil.WriteFile("bigfile.test", bigBuff, 0666) +} + +func BenchmarkMD5(b *testing.B) { + for i := 0; i < b.N; i++ { + MD5HashFile("bigfile.test") + } +} + +func BenchmarkXXHash(b *testing.B) { + for i := 0; i < b.N; i++ { + XXHashFile("bigfile.test") + } +} +func BenchmarkImoHash(b *testing.B) { + for i := 0; i < b.N; i++ { + IMOHashFile("bigfile.test") + } +} + func TestExists(t *testing.T) { - assert.True(t, Exists("utils.go")) + assert.True(t, Exists("bigfile.test")) + assert.False(t, Exists("doesnotexist")) } -// HashFile returns the md5 hash of a file -func TestHashFile(t *testing.T) { - b, err := HashFile("utils.go") +func TestMD5HashFile(t *testing.T) { + b, err := MD5HashFile("bigfile.test") assert.Nil(t, err) - assert.Equal(t, "314965d4170cd0c70bd2e4f6c977750a", fmt.Sprintf("%x", b)) + assert.Equal(t, "9fed05acbacbc6a36555c998501c21f6", fmt.Sprintf("%x", b)) +} + +func TestIMOHashFile(t *testing.T) { + b, err := IMOHashFile("bigfile.test") + assert.Nil(t, err) + assert.Equal(t, "c0d1e123a96a598ea801cc503d3db8c0", fmt.Sprintf("%x", b)) +} + +func TestXXHashFile(t *testing.T) { + b, err := XXHashFile("bigfile.test") + assert.Nil(t, err) + assert.Equal(t, "f2da6ee7e75e8324", fmt.Sprintf("%x", b)) } -// SHA256 returns sha256 sum func TestSHA256(t *testing.T) { assert.Equal(t, "09ca7e4eaa6e8ae9c7d261167129184883644d07dfba7cbfbc4c8a2e08360d5b", SHA256("hello, world")) }