mirrors_public/schollz_croc
1
1
Fork 0
mirror of https://github.com/schollz/croc.git synced 2025-10-11 05:11:06 +02:00
Code Activity

Merge pull request #721 from schollz:schollz/issue720

Browse source 
show hash progress on large files
This commit is contained in:
Zack 2024-05-28 16:21:45 -07:00 • committed by GitHub
commit e4c9f2d9fb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 122 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -20,6 +20,8 @@ require (
golang.org/x/time v0.5.0
)
require github.com/minio/highwayhash v1.0.2
require (
github.com/OneOfOne/xxhash v1.2.8 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect

3
go.sum
View file

@ -29,6 +29,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magisterquis/connectproxy v0.0.0-20200725203833-3582e84f0c9b h1:xZ59n7Frzh8CwyfAapUZLSg+gXH5m63YEaFCMpDHhpI=
github.com/magisterquis/connectproxy v0.0.0-20200725203833-3582e84f0c9b/go.mod h1:uDd4sYVYsqcxAB8j+Q7uhL6IJCs/r1kxib1HV4bgOMg=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@ -86,6 +88,7 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

2
src/croc/croc.go
View file

@ -506,7 +506,7 @@ func (c *Client) sendCollectFiles(filesInfo []FileInfo) (err error) {
c.Options.HashAlgorithm = "xxhash"
}
c.FilesToTransfer[i].Hash, err = utils.HashFile(fullPath, c.Options.HashAlgorithm)
c.FilesToTransfer[i].Hash, err = utils.HashFile(fullPath, c.Options.HashAlgorithm, fileInfo.Size > 1e7)
log.Debugf("hashed %s to %x using %s", fullPath, c.FilesToTransfer[i].Hash, c.Options.HashAlgorithm)
totalFilesSize += fileInfo.Size
if err != nil {

101
src/utils/utils.go
View file

@ -25,7 +25,9 @@ import (
"github.com/cespare/xxhash"
"github.com/kalafut/imohash"
"github.com/minio/highwayhash"
"github.com/schollz/mnemonicode"
"github.com/schollz/progressbar/v3"
)
const NbPinNumbers = 4
@ -77,7 +79,11 @@ func GetInput(prompt string) string {
// HashFile returns the hash of a file or, in case of a symlink, the
// SHA256 hash of its target. Takes an argument to specify the algorithm to use.
func HashFile(fname string, algorithm string) (hash256 []byte, err error) {
func HashFile(fname string, algorithm string, showProgress ...bool) (hash256 []byte, err error) {
doShowProgress := false
if len(showProgress) > 0 {
doShowProgress = showProgress[0]
}
var fstats os.FileInfo
fstats, err = os.Lstat(fname)
if err != nil {
@ -95,16 +101,59 @@ func HashFile(fname string, algorithm string) (hash256 []byte, err error) {
case "imohash":
return IMOHashFile(fname)
case "md5":
return MD5HashFile(fname)
return MD5HashFile(fname, doShowProgress)
case "xxhash":
return XXHashFile(fname)
return XXHashFile(fname, doShowProgress)
case "highway":
return HighwayHashFile(fname, doShowProgress)
}
err = fmt.Errorf("unspecified algorithm")
return
}
// HighwayHashFile returns highwayhash of a file
func HighwayHashFile(fname string, doShowProgress bool) (hashHighway []byte, err error) {
f, err := os.Open(fname)
if err != nil {
return
}
defer f.Close()
key, err := hex.DecodeString("1553c5383fb0b86578c3310da665b4f6e0521acf22eb58a99532ffed02a6b115")
if err != nil {
return
}
h, err := highwayhash.New(key)
if err != nil {
err = fmt.Errorf("could not create highwayhash: %s", err.Error())
return
}
if doShowProgress {
stat, _ := f.Stat()
fnameShort := path.Base(fname)
if len(fnameShort) > 20 {
fnameShort = fnameShort[:20] + "..."
}
bar := progressbar.NewOptions64(stat.Size(),
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionShowBytes(true),
progressbar.OptionSetDescription(fmt.Sprintf("Hashing %s", fnameShort)),
progressbar.OptionClearOnFinish(),
)
if _, err = io.Copy(io.MultiWriter(h, bar), f); err != nil {
return
}
} else {
if _, err = io.Copy(h, f); err != nil {
return
}
}
hashHighway = h.Sum(nil)
return
}
// MD5HashFile returns MD5 hash
func MD5HashFile(fname string) (hash256 []byte, err error) {
func MD5HashFile(fname string, doShowProgress bool) (hash256 []byte, err error) {
f, err := os.Open(fname)
if err != nil {
return
@ -112,8 +161,25 @@ func MD5HashFile(fname string) (hash256 []byte, err error) {
defer f.Close()
h := md5.New()
if _, err = io.Copy(h, f); err != nil {
return
if doShowProgress {
stat, _ := f.Stat()
fnameShort := path.Base(fname)
if len(fnameShort) > 20 {
fnameShort = fnameShort[:20] + "..."
}
bar := progressbar.NewOptions64(stat.Size(),
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionShowBytes(true),
progressbar.OptionSetDescription(fmt.Sprintf("Hashing %s", fnameShort)),
progressbar.OptionClearOnFinish(),
)
if _, err = io.Copy(io.MultiWriter(h, bar), f); err != nil {
return
}
} else {
if _, err = io.Copy(h, f); err != nil {
return
}
}
hash256 = h.Sum(nil)
@ -137,7 +203,7 @@ func IMOHashFileFull(fname string) (hash []byte, err error) {
}
// XXHashFile returns the xxhash of a file
func XXHashFile(fname string) (hash256 []byte, err error) {
func XXHashFile(fname string, doShowProgress bool) (hash256 []byte, err error) {
f, err := os.Open(fname)
if err != nil {
return
@ -145,8 +211,25 @@ func XXHashFile(fname string) (hash256 []byte, err error) {
defer f.Close()
h := xxhash.New()
if _, err = io.Copy(h, f); err != nil {
return
if doShowProgress {
stat, _ := f.Stat()
fnameShort := path.Base(fname)
if len(fnameShort) > 20 {
fnameShort = fnameShort[:20] + "..."
}
bar := progressbar.NewOptions64(stat.Size(),
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionShowBytes(true),
progressbar.OptionSetDescription(fmt.Sprintf("Hashing %s", fnameShort)),
progressbar.OptionClearOnFinish(),
)
if _, err = io.Copy(io.MultiWriter(h, bar), f); err != nil {
return
}
} else {
if _, err = io.Copy(h, f); err != nil {
return
}
}
hash256 = h.Sum(nil)

30
src/utils/utils_test.go
View file

@ -24,7 +24,7 @@ func BenchmarkMD5(b *testing.B) {
bigFile()
b.ResetTimer()
for i := 0; i < b.N; i++ {
MD5HashFile("bigfile.test")
MD5HashFile("bigfile.test", false)
}
}
@ -32,7 +32,7 @@ func BenchmarkXXHash(b *testing.B) {
bigFile()
b.ResetTimer()
for i := 0; i < b.N; i++ {
XXHashFile("bigfile.test")
XXHashFile("bigfile.test", false)
}
}
@ -44,6 +44,14 @@ func BenchmarkImoHash(b *testing.B) {
}
}
func BenchmarkHighwayHash(b *testing.B) {
bigFile()
b.ResetTimer()
for i := 0; i < b.N; i++ {
HighwayHashFile("bigfile.test", false)
}
}
func BenchmarkImoHashFull(b *testing.B) {
bigFile()
b.ResetTimer()
@ -78,10 +86,20 @@ func TestExists(t *testing.T) {
func TestMD5HashFile(t *testing.T) {
bigFile()
defer os.Remove("bigfile.test")
b, err := MD5HashFile("bigfile.test")
b, err := MD5HashFile("bigfile.test", false)
assert.Nil(t, err)
assert.Equal(t, "8304ff018e02baad0e3555bade29a405", fmt.Sprintf("%x", b))
_, err = MD5HashFile("bigfile.test.nofile")
_, err = MD5HashFile("bigfile.test.nofile", false)
assert.NotNil(t, err)
}
func TestHighwayHashFile(t *testing.T) {
bigFile()
defer os.Remove("bigfile.test")
b, err := HighwayHashFile("bigfile.test", false)
assert.Nil(t, err)
assert.Equal(t, "3c32999529323ed66a67aeac5720c7bf1301dcc5dca87d8d46595e85ff990329", fmt.Sprintf("%x", b))
_, err = HighwayHashFile("bigfile.test.nofile", false)
assert.NotNil(t, err)
}
@ -96,10 +114,10 @@ func TestIMOHashFile(t *testing.T) {
func TestXXHashFile(t *testing.T) {
bigFile()
defer os.Remove("bigfile.test")
b, err := XXHashFile("bigfile.test")
b, err := XXHashFile("bigfile.test", false)
assert.Nil(t, err)
assert.Equal(t, "4918740eb5ccb6f7", fmt.Sprintf("%x", b))
_, err = XXHashFile("nofile")
_, err = XXHashFile("nofile", false)
assert.NotNil(t, err)
}