diff --git a/README.md b/README.md index 5ecea591..a1a12bd9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ src="https://user-images.githubusercontent.com/6550035/46709024-9b23ad00-cbf6-11e8-9fb2-ca8b20b7dbec.jpg" width="408px" border="0" alt="croc">
-Version +Version +Coverage Build Status @@ -21,7 +22,8 @@ Status"> - enables easy **cross-platform** transfers (Windows, Linux, Mac) - allows **multiple file** transfers - allows **resuming transfers** that are interrupted -- does *not* require a server or port-forwarding +- local server or port-forwarding **not needed** +- **faster** than [wormhole](https://github.com/warner/magic-wormhole), [rsync](https://linux.die.net/man/1/rsync), [scp](https://linux.die.net/man/1/scp) through compression and multiplexing (speedups 1.5x to 4x) For more information about `croc`, see [my blog post](https://schollz.com/software/croc6). diff --git a/go.mod b/go.mod index b3fee5f1..f7f56a33 100644 --- a/go.mod +++ b/go.mod @@ -1,25 +1,28 @@ module github.com/schollz/croc/v6 -go 1.12 +go 1.13 require ( github.com/OneOfOne/xxhash v1.2.5 // indirect github.com/cespare/xxhash v1.1.0 github.com/fatih/color v1.7.0 // indirect github.com/kalafut/imohash v1.0.0 + github.com/kr/pretty v0.1.0 // indirect github.com/mattn/go-colorable v0.1.2 // indirect + github.com/mattn/go-isatty v0.0.9 // indirect github.com/pkg/errors v0.8.1 github.com/schollz/logger v1.0.1 github.com/schollz/mnemonicode v1.0.1 github.com/schollz/pake v1.1.1 - github.com/schollz/peerdiscovery v1.4.0 + github.com/schollz/peerdiscovery v1.4.1 github.com/schollz/progressbar/v2 v2.13.2 github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9 github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/stretchr/testify v1.3.0 - github.com/urfave/cli v1.20.0 - golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 - golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect - golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect + github.com/stretchr/testify v1.4.0 + github.com/urfave/cli v1.21.0 + golang.org/x/crypto v0.0.0-20190907121410-71b5226ff739 + golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect + golang.org/x/sys v0.0.0-20190904154756-749cb33beabd // indirect golang.org/x/text v0.3.2 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 57097ae9..5c33b6c1 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= @@ -11,11 +12,17 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/kalafut/imohash v1.0.0 h1:LgCJ+p/BwM2HKpOxFopkeddpzVCfm15EtXMroXD1SYE= github.com/kalafut/imohash v1.0.0/go.mod h1:c3RHT80ZAp5C/aYgQI92ZlrOymqkZnRDprU87kg75HI= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mitchellh/colorstring v0.0.0-20150917214807-8631ce90f286/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= @@ -28,9 +35,8 @@ github.com/schollz/mnemonicode v1.0.1 h1:LiH5hwADZwjwnfXsaD4xgnMyTAtaKHN+e5AyjRU github.com/schollz/mnemonicode v1.0.1/go.mod h1:cl4UAOhUV0mkdjMj/QYaUZbZZdF8BnOqoz8rHMzwboY= github.com/schollz/pake v1.1.1 h1:QKeojDWzdAdtRC4m89b6HAxw/8gjqrVu7r4SAOxOFg8= github.com/schollz/pake v1.1.1/go.mod h1:aWMxQ1jwqZRwk3StflHcdyzPR+CyW5W7+WIZD6Y3dEY= -github.com/schollz/peerdiscovery v1.4.0 h1:wJWiJUBSMY2io9eIG1+gauXm8WD6sJVN5M+pLd4fYZQ= -github.com/schollz/peerdiscovery v1.4.0/go.mod h1:DXj/7VvxAkUuSZNabx3q8t524uWbrhMPxeX151kvvHs= -github.com/schollz/progressbar/v2 v2.9.1/go.mod h1:l6tn6yU6ZdQoF8lwX/VoAUQ3FjhCbrcZDnl9xeWZzYw= +github.com/schollz/peerdiscovery v1.4.1 h1:xtZ/D8/4eq9O6UEhRupZZiJm4BA8+u1IVUgeHo5VPm4= +github.com/schollz/peerdiscovery v1.4.1/go.mod h1:WDdk0/JVyVHVIA/bmhzTkUg32dhJ20O4tExNqV1u6sk= github.com/schollz/progressbar/v2 v2.13.2 h1:3L9bP5KQOGEnFP8P5V8dz+U0yo5I29iY5Oa9s9EAwn0= github.com/schollz/progressbar/v2 v2.13.2/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8= github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9 h1:y08o5oQ/slxXE/F0uh5dd8mdVvb+w4NLcNSDSq4c2F0= @@ -40,31 +46,43 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tscholl2/siec v0.0.0-20180721101609-21667da05937 h1:lhssCpSe3TjKcbvUoPzFMuv9oUyZDgI3Cmgolfw2C90= github.com/tscholl2/siec v0.0.0-20180721101609-21667da05937/go.mod h1:KL9+ubr1JZdaKjgAaHr+tCytEncXBa1pR6FjbTsOJnw= -github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.21.0 h1:wYSSj06510qPIzGSua9ZqsncMmWE3Zr55KBERygyrxE= +github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/crypto v0.0.0-20190907121410-71b5226ff739 h1:Gc7JIyxvWgD6m+QmVryY0MstDORNYididDGxgZ6Tnpk= +golang.org/x/crypto v0.0.0-20190907121410-71b5226ff739/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/tylerb/is.v1 v1.1.2 h1:AB/MANFml2ySf+adwcinvajyHvsYltAOD+rb/8njfSU= gopkg.in/tylerb/is.v1 v1.1.2/go.mod h1:9yQB2tyIhZ5oph6Kk5Sq7cJMd9c5Jpa1p3hr9kxzPqo= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/src/cli/cli.go b/src/cli/cli.go index fd32351b..f69d4015 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -21,8 +21,10 @@ import ( "github.com/urfave/cli" ) +// Version specifies the version var Version string +// Run will run the command line proram func Run() (err error) { // use all of the processors runtime.GOMAXPROCS(runtime.NumCPU()) @@ -30,7 +32,7 @@ func Run() (err error) { app := cli.NewApp() app.Name = "croc" if Version == "" { - Version = "v6.1.1-82f2c8c" + Version = "v6.1.3-bc6803e" } app.Version = Version app.Compiled = time.Now() @@ -101,7 +103,7 @@ func getConfigDir() (homedir string, err error) { return } homedir = path.Join(homedir, ".config", "croc") - if _, err := os.Stat(homedir); os.IsNotExist(err) { + if _, err = os.Stat(homedir); os.IsNotExist(err) { log.Debugf("creating home directory %s", homedir) err = os.MkdirAll(homedir, 0700) } diff --git a/src/comm/comm_test.go b/src/comm/comm_test.go index e213146d..6f986f4c 100644 --- a/src/comm/comm_test.go +++ b/src/comm/comm_test.go @@ -47,7 +47,7 @@ func TestComm(t *testing.T) { }() time.Sleep(300 * time.Millisecond) - a, err := NewConnection("localhost:" + port) + a, err := NewConnection("localhost:"+port, 10*time.Minute) assert.Nil(t, err) data, err := a.Receive() assert.Equal(t, []byte("hello, world"), data) @@ -56,5 +56,10 @@ func TestComm(t *testing.T) { assert.Nil(t, a.Send([]byte{'\x00'})) assert.Nil(t, a.Send(token)) + _ = a.Connection() + a.Close() + assert.NotNil(t, a.Send(token)) + _, err = a.Write(token) + assert.NotNil(t, err) } diff --git a/src/compress/compress_test.go b/src/compress/compress_test.go index b8682ff6..e48ed92f 100644 --- a/src/compress/compress_test.go +++ b/src/compress/compress_test.go @@ -70,12 +70,18 @@ func TestCompress(t *testing.T) { dataRateSavings := 100 * (1.0 - float64(len(compressedB))/float64(len(fable))) fmt.Printf("Level 9: %2.0f%% percent space savings\n", dataRateSavings) assert.True(t, len(compressedB) < len(fable)) + assert.Equal(t, fable, Decompress(compressedB)) compressedB = CompressWithOption(fable, -2) dataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(fable))) fmt.Printf("Level -2: %2.0f%% percent space savings\n", dataRateSavings) assert.True(t, len(compressedB) < len(fable)) + compressedB = Compress(fable) + dataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(fable))) + fmt.Printf("Level -2: %2.0f%% percent space savings\n", dataRateSavings) + assert.True(t, len(compressedB) < len(fable)) + data := make([]byte, 4096) rand.Read(data) compressedB = CompressWithOption(data, -2) @@ -85,6 +91,7 @@ func TestCompress(t *testing.T) { rand.Read(data) compressedB = CompressWithOption(data, 9) dataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(data))) + fmt.Printf("random, Level 9: %2.0f%% percent space savings\n", dataRateSavings) } diff --git a/src/croc/croc.go b/src/croc/croc.go index 18014cef..ab15e78d 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -36,6 +36,7 @@ func init() { log.SetLevel("debug") } +// Debug toggles debug mode func Debug(debug bool) { if debug { log.SetLevel("debug") @@ -44,6 +45,7 @@ func Debug(debug bool) { } } +// Options specifies user specific options type Options struct { IsSender bool SharedSecret string @@ -55,6 +57,7 @@ type Options struct { DisableLocal bool } +// Client holds the state of the croc transfer type Client struct { Options Options Pake *pake.Pake @@ -72,6 +75,7 @@ type Client struct { // send / receive information of all files FilesToTransfer []FileInfo FilesToTransferCurrentNum int + FilesHasFinished map[int]struct{} // send / receive information of current file CurrentFile *os.File @@ -89,15 +93,20 @@ type Client struct { spinner *spinner.Spinner firstSend bool - mutex *sync.Mutex - quit chan bool + mutex *sync.Mutex + fread *os.File + numfinished int + quit chan bool } +// Chunk contains information about the +// needed bytes type Chunk struct { Bytes []byte `json:"b,omitempty"` Location int64 `json:"l,omitempty"` } +// FileInfo registers the information about the file type FileInfo struct { Name string `json:"n,omitempty"` FolderRemote string `json:"fr,omitempty"` @@ -109,18 +118,21 @@ type FileInfo struct { IsEncrypted bool `json:"e,omitempty"` } +// RemoteFileRequest requests specific bytes type RemoteFileRequest struct { CurrentFileChunkRanges []int64 FilesToTransferCurrentNum int } +// SenderInfo lists the files to be transferred type SenderInfo struct { FilesToTransfer []FileInfo } -// New establishes a new connection for transfering files between two instances. +// New establishes a new connection for transferring files between two instances. func New(ops Options) (c *Client, err error) { c = new(Client) + c.FilesHasFinished = make(map[int]struct{}) // setup basic info c.Options = ops @@ -288,7 +300,7 @@ func (c *Client) Send(options TransferOptions) (err error) { go func() { log.Debugf("establishing connection to %s", c.Options.RelayAddress) var banner string - conn, banner, ipaddr, err := tcp.ConnectToTCPServer(c.Options.RelayAddress, c.Options.SharedSecret) + conn, banner, ipaddr, err := tcp.ConnectToTCPServer(c.Options.RelayAddress, c.Options.SharedSecret, 5*time.Second) log.Debugf("banner: %s", banner) if err != nil { err = errors.Wrap(err, fmt.Sprintf("could not connect to %s", c.Options.RelayAddress)) @@ -449,7 +461,7 @@ func (c *Client) transfer(options TransferOptions) (err error) { break } } - // purge errors that come from succesful transfer + // purge errors that come from successful transfer if c.SuccessfulTransfer { if err != nil { log.Debugf("purging error: %s", err) @@ -667,6 +679,9 @@ func (c *Client) updateState() (err error) { finished := true for i, fileInfo := range c.FilesToTransfer { + if _, ok := c.FilesHasFinished[i]; ok { + continue + } log.Debugf("checking %+v", fileInfo) if i < c.FilesToTransferCurrentNum { continue @@ -733,6 +748,7 @@ func (c *Client) updateState() (err error) { panic(err) } c.SuccessfulTransfer = true + c.FilesHasFinished[c.FilesToTransferCurrentNum] = struct{}{} } // start initiating the process to receive a new file @@ -749,7 +765,7 @@ func (c *Client) updateState() (err error) { c.CurrentFile, errOpen = os.OpenFile( pathToFile, os.O_WRONLY, 0666) - truncate := false + var truncate bool // default false c.CurrentFileChunks = []int64{} c.CurrentFileChunkRanges = []int64{} if errOpen == nil { @@ -810,7 +826,7 @@ func (c *Client) updateState() (err error) { if !c.firstSend { fmt.Fprintf(os.Stderr, "\nSending (->%s)\n", c.ExternalIPConnected) c.firstSend = true - // if there are empty files, show them as already have been transfered now + // if there are empty files, show them as already have been transferred now for i := range c.FilesToTransfer { if c.FilesToTransfer[i].Size == 0 { // setup the progressbar and takedown the progress bar for empty files @@ -837,6 +853,16 @@ func (c *Client) updateState() (err error) { c.setBar() c.TotalSent = 0 log.Debug("beginning sending comms") + pathToFile := path.Join( + c.FilesToTransfer[c.FilesToTransferCurrentNum].FolderSource, + c.FilesToTransfer[c.FilesToTransferCurrentNum].Name, + ) + + c.fread, err = os.Open(pathToFile) + c.numfinished = 0 + if err != nil { + return + } for i := 0; i < len(c.Options.RelayPorts); i++ { log.Debugf("starting sending over comm %d", i) go c.sendData(i) @@ -875,6 +901,7 @@ func (c *Client) setBar() { } func (c *Client) receiveData(i int) { + log.Debugf("%d receiving data", i) for { data, err := c.conn[i+1].Receive() if err != nil { @@ -932,30 +959,23 @@ func (c *Client) receiveData(i int) { func (c *Client) sendData(i int) { defer func() { log.Debugf("finished with %d", i) + c.numfinished++ + if c.numfinished == len(c.Options.RelayPorts) { + log.Debug("closing file") + c.fread.Close() + } }() - pathToFile := path.Join( - c.FilesToTransfer[c.FilesToTransferCurrentNum].FolderSource, - c.FilesToTransfer[c.FilesToTransferCurrentNum].Name, - ) - log.Debugf("opening %s to read", pathToFile) - f, err := os.Open(pathToFile) - if err != nil { - panic(err) - } - defer f.Close() + var readingPos int64 pos := uint64(0) curi := float64(0) for { // Read file data := make([]byte, models.TCP_BUFFER_SIZE/2) - n, err := f.Read(data) - if err != nil { - if err == io.EOF { - break - } - panic(err) - } + // log.Debugf("%d trying to read", i) + n, errRead := c.fread.ReadAt(data, readingPos) + // log.Debugf("%d read %d bytes", i, n) + readingPos += int64(n) if math.Mod(curi, float64(len(c.Options.RelayPorts))) == float64(i) { // check to see if this is a chunk that the recipient wants @@ -997,8 +1017,13 @@ func (c *Client) sendData(i int) { curi++ pos += uint64(n) - } - time.Sleep(10 * time.Second) + if errRead != nil { + if errRead == io.EOF { + break + } + panic(errRead) + } + } return } diff --git a/src/croc/croc_test.go b/src/croc/croc_test.go index 51563a6a..492a05f4 100644 --- a/src/croc/croc_test.go +++ b/src/croc/croc_test.go @@ -1,6 +1,7 @@ package croc import ( + "io/ioutil" "os" "sync" "testing" @@ -8,9 +9,11 @@ import ( "github.com/schollz/croc/v6/src/tcp" log "github.com/schollz/logger" + "github.com/stretchr/testify/assert" ) func TestCroc(t *testing.T) { + log.SetLevel("trace") defer os.Remove("README.md") go tcp.Run("debug", "8081", "8082,8083,8084,8085") go tcp.Run("debug", "8082") @@ -64,3 +67,95 @@ func TestCroc(t *testing.T) { wg.Wait() } + +func TestCrocLocal(t *testing.T) { + log.SetLevel("trace") + defer os.Remove("LICENSE") + defer os.Remove("touched") + time.Sleep(300 * time.Millisecond) + + log.Debug("setting up sender") + sender, err := New(Options{ + IsSender: true, + SharedSecret: "test", + Debug: true, + RelayAddress: "localhost:8181", + RelayPorts: []string{"8181", "8182"}, + Stdout: true, + NoPrompt: true, + DisableLocal: false, + }) + if err != nil { + panic(err) + } + time.Sleep(1 * time.Second) + + log.Debug("setting up receiver") + receiver, err := New(Options{ + IsSender: false, + SharedSecret: "test", + Debug: true, + RelayAddress: "localhost:8181", + Stdout: true, + NoPrompt: true, + DisableLocal: false, + }) + if err != nil { + panic(err) + } + + var wg sync.WaitGroup + os.Create("touched") + wg.Add(2) + go func() { + sender.Send(TransferOptions{ + PathToFiles: []string{"../../LICENSE", "touched"}, + KeepPathInRemote: false, + }) + wg.Done() + }() + time.Sleep(100 * time.Millisecond) + go func() { + receiver.Receive() + wg.Done() + }() + + wg.Wait() +} + +func TestCrocError(t *testing.T) { + content := []byte("temporary file's content") + tmpfile, err := ioutil.TempFile("", "example") + if err != nil { + panic(err) + } + + defer os.Remove(tmpfile.Name()) // clean up + + if _, err := tmpfile.Write(content); err != nil { + panic(err) + } + if err := tmpfile.Close(); err != nil { + panic(err) + } + + Debug(false) + log.SetLevel("warn") + sender, _ := New(Options{ + IsSender: true, + SharedSecret: "test33", + Debug: true, + RelayAddress: "doesntexistok.com:8381", + RelayPorts: []string{"8381", "8382"}, + Stdout: true, + NoPrompt: true, + DisableLocal: true, + }) + err = sender.Send(TransferOptions{ + PathToFiles: []string{tmpfile.Name()}, + KeepPathInRemote: true, + }) + log.Debug(err) + assert.NotNil(t, err) + +} diff --git a/src/crypt/crypt.go b/src/crypt/crypt.go index 743eb910..65ba77c4 100644 --- a/src/crypt/crypt.go +++ b/src/crypt/crypt.go @@ -9,6 +9,8 @@ import ( "golang.org/x/crypto/pbkdf2" ) +// Encryption is the basic type for storing +// the key, passphrase and salt type Encryption struct { key []byte passphrase []byte @@ -36,6 +38,7 @@ func New(passphrase []byte, salt []byte) (e Encryption, err error) { return } +// Salt returns the salt bytes func (e Encryption) Salt() []byte { return e.salt } diff --git a/src/crypt/crypt_test.go b/src/crypt/crypt_test.go index f4fc6b93..14ac6397 100644 --- a/src/crypt/crypt_test.go +++ b/src/crypt/crypt_test.go @@ -45,11 +45,10 @@ func TestEncryption(t *testing.T) { } - func TestNoEncryption(t *testing.T) { bob, err := New(nil, nil) assert.Nil(t, err) - jane, err := New(nil,nil) + jane, err := New(nil, nil) assert.Nil(t, err) enc, err := bob.Encrypt([]byte("hello, world")) assert.Nil(t, err) @@ -57,5 +56,5 @@ func TestNoEncryption(t *testing.T) { assert.Nil(t, err) assert.Equal(t, dec, []byte("hello, world")) assert.Equal(t, enc, []byte("hello, world")) - -} \ No newline at end of file + +} diff --git a/src/message/message.go b/src/message/message.go index 100456d8..2f0abb6c 100644 --- a/src/message/message.go +++ b/src/message/message.go @@ -3,10 +3,10 @@ package message import ( "encoding/json" - log "github.com/schollz/logger" "github.com/schollz/croc/v6/src/comm" "github.com/schollz/croc/v6/src/compress" "github.com/schollz/croc/v6/src/crypt" + log "github.com/schollz/logger" ) // Message is the possible payload for messaging diff --git a/src/message/message_test.go b/src/message/message_test.go index 861500cc..cc4772d6 100644 --- a/src/message/message_test.go +++ b/src/message/message_test.go @@ -1,10 +1,15 @@ package message import ( + "crypto/rand" "fmt" + "net" "testing" + "time" + "github.com/schollz/croc/v6/src/comm" "github.com/schollz/croc/v6/src/crypt" + log "github.com/schollz/logger" "github.com/stretchr/testify/assert" ) @@ -20,4 +25,51 @@ func TestMessage(t *testing.T) { m2, err := Decode(e, b) assert.Nil(t, err) assert.Equal(t, m, m2) + assert.Equal(t, `{"t":"message","m":"hello, world"}`, m.String()) +} + +func TestSend(t *testing.T) { + token := make([]byte, 40000000) + rand.Read(token) + + port := "8801" + go func() { + log.Debugf("starting TCP server on " + port) + server, err := net.Listen("tcp", "0.0.0.0:"+port) + if err != nil { + log.Error(err) + } + defer server.Close() + // spawn a new goroutine whenever a client connects + for { + connection, err := server.Accept() + if err != nil { + log.Error(err) + } + log.Debugf("client %s connected", connection.RemoteAddr().String()) + go func(port string, connection net.Conn) { + c := comm.New(connection) + err = c.Send([]byte("hello, world")) + assert.Nil(t, err) + data, err := c.Receive() + assert.Nil(t, err) + assert.Equal(t, []byte("hello, computer"), data) + data, err = c.Receive() + assert.Nil(t, err) + assert.Equal(t, []byte{'\x00'}, data) + data, err = c.Receive() + assert.Nil(t, err) + assert.Equal(t, token, data) + }(port, connection) + } + }() + + time.Sleep(300 * time.Millisecond) + a, err := comm.NewConnection("localhost:"+port, 10*time.Minute) + assert.Nil(t, err) + m := Message{Type: "message", Message: "hello, world"} + e, err := crypt.New(nil, nil) + assert.Nil(t, err) + + assert.Nil(t, Send(a, e, m)) } diff --git a/src/models/constants.go b/src/models/constants.go index acb20a7b..4c074981 100644 --- a/src/models/constants.go +++ b/src/models/constants.go @@ -1,4 +1,7 @@ package models +// TCP_BUFFER_SIZE is the maximum packet size const TCP_BUFFER_SIZE = 1024 * 64 + +// DEFAULT_RELAY is the default relay used (can be set using --relay) const DEFAULT_RELAY = "142.93.177.120:9009" diff --git a/src/tcp/tcp.go b/src/tcp/tcp.go index 74443a05..473ed261 100644 --- a/src/tcp/tcp.go +++ b/src/tcp/tcp.go @@ -33,6 +33,8 @@ type roomMap struct { sync.Mutex } +var timeToRoomDeletion = 10 * time.Minute + // Run starts a tcp listener, run async func Run(debugLevel, port string, banner ...string) (err error) { s := new(server) @@ -53,7 +55,7 @@ func (s *server) start() (err error) { // delete old rooms go func() { for { - time.Sleep(10 * time.Minute) + time.Sleep(timeToRoomDeletion) roomsToDelete := []string{} s.rooms.Lock() for room := range s.rooms.rooms { @@ -251,6 +253,8 @@ func pipe(conn1 net.Conn, conn2 net.Conn) { } } +// ConnectToTCPServer will initiate a new connection +// to the specified address, room with optional time limit func ConnectToTCPServer(address, room string, timelimit ...time.Duration) (c *comm.Comm, banner string, ipaddr string, err error) { if len(timelimit) > 0 { c, err = comm.NewConnection(address, timelimit[0]) diff --git a/src/tcp/tcp_test.go b/src/tcp/tcp_test.go index 46bcee1f..8c989097 100644 --- a/src/tcp/tcp_test.go +++ b/src/tcp/tcp_test.go @@ -8,14 +8,17 @@ import ( ) func TestTCP(t *testing.T) { - go Run("debug", "8081", "8082") + timeToRoomDeletion = 100 * time.Millisecond + go Run("debug", "8281", "8282") time.Sleep(100 * time.Millisecond) - c1, banner, _, err := ConnectToTCPServer("localhost:8081", "testRoom") - assert.Equal(t, banner, "8082") + c1, banner, _, err := ConnectToTCPServer("localhost:8281", "testRoom", 1*time.Minute) + assert.Equal(t, banner, "8282") assert.Nil(t, err) - c2, _, _, err := ConnectToTCPServer("localhost:8081", "testRoom") + c2, _, _, err := ConnectToTCPServer("localhost:8281", "testRoom") assert.Nil(t, err) - _, _, _, err = ConnectToTCPServer("localhost:8081", "testRoom") + _, _, _, err = ConnectToTCPServer("localhost:8281", "testRoom") + assert.NotNil(t, err) + _, _, _, err = ConnectToTCPServer("localhost:8281", "testRoom", 1*time.Nanosecond) assert.NotNil(t, err) // try sending data diff --git a/src/utils/utils.go b/src/utils/utils.go index adcd017a..c4e288e9 100644 --- a/src/utils/utils.go +++ b/src/utils/utils.go @@ -44,6 +44,7 @@ func HashFile(fname string) (hash256 []byte, err error) { return IMOHashFile(fname) } +// MD5HashFile returns MD5 hash func MD5HashFile(fname string) (hash256 []byte, err error) { f, err := os.Open(fname) if err != nil { @@ -91,6 +92,7 @@ func SHA256(s string) string { return fmt.Sprintf("%x", sha.Sum(nil)) } +// PublicIP returns public ip address func PublicIP() (ip string, err error) { resp, err := http.Get("https://canhazip.com") if err != nil { @@ -108,7 +110,7 @@ func PublicIP() (ip string, err error) { return } -// Get preferred outbound ip of this machine +// LocalIP returns local ip address func LocalIP() string { conn, err := net.Dial("udp", "8.8.8.8:80") if err != nil { @@ -121,6 +123,7 @@ func LocalIP() string { return localAddr.IP.String() } +// GetRandomName returns mnemoicoded random name func GetRandomName() string { result := []string{} bs := make([]byte, 4) @@ -129,6 +132,7 @@ func GetRandomName() string { return strings.Join(result, "-") } +// ByteCountDecimal converts bytes to human readable byte string func ByteCountDecimal(b int64) string { const unit = 1000 if b < unit { @@ -146,17 +150,17 @@ func ByteCountDecimal(b int64) string { // If file doesn't exist, it returns an empty chunk list (all chunks). // If the file size is not the same as requested, it returns an empty chunk list (all chunks). func MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) { - fstat, err := os.Stat(fname) - if fstat.Size() != fsize || err != nil { - return - } - f, err := os.Open(fname) if err != nil { return } defer f.Close() + fstat, err := os.Stat(fname) + if fstat.Size() != fsize || err != nil { + return + } + emptyBuffer := make([]byte, chunkSize) chunkNum := 0 chunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize)))) @@ -196,6 +200,7 @@ func MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int6 return } +// ChunkRangesToChunks converts chunk ranges to list func ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) { if len(chunkRanges) == 0 { return @@ -210,6 +215,7 @@ func ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) { return } +// GetLocalIPs returns all local ips func GetLocalIPs() (ips []string, err error) { addrs, err := net.InterfaceAddrs() if err != nil { diff --git a/src/utils/utils_test.go b/src/utils/utils_test.go index e23a419e..1e4ac707 100644 --- a/src/utils/utils_test.go +++ b/src/utils/utils_test.go @@ -3,8 +3,10 @@ package utils import ( "fmt" "io/ioutil" + "log" "math/rand" "os" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -54,6 +56,8 @@ func TestMD5HashFile(t *testing.T) { b, err := MD5HashFile("bigfile.test") assert.Nil(t, err) assert.Equal(t, "9fed05acbacbc6a36555c998501c21f6", fmt.Sprintf("%x", b)) + _, err = MD5HashFile("bigfile.test.nofile") + assert.NotNil(t, err) } func TestIMOHashFile(t *testing.T) { @@ -70,6 +74,8 @@ func TestXXHashFile(t *testing.T) { b, err := XXHashFile("bigfile.test") assert.Nil(t, err) assert.Equal(t, "f2da6ee7e75e8324", fmt.Sprintf("%x", b)) + _, err = XXHashFile("nofile") + assert.NotNil(t, err) } func TestSHA256(t *testing.T) { @@ -78,6 +84,8 @@ func TestSHA256(t *testing.T) { func TestByteCountDecimal(t *testing.T) { assert.Equal(t, "10.0 kB", ByteCountDecimal(10000)) + assert.Equal(t, "50 B", ByteCountDecimal(50)) + assert.Equal(t, "12.4 MB", ByteCountDecimal(12378517)) } func TestMissingChunks(t *testing.T) { @@ -104,6 +112,29 @@ func TestMissingChunks(t *testing.T) { assert.Equal(t, []int64{0, 40, 50, 70, 80, 90}, chunks) os.Remove("missing.test") + + content := []byte("temporary file's content") + tmpfile, err := ioutil.TempFile("", "example") + if err != nil { + log.Fatal(err) + } + + defer os.Remove(tmpfile.Name()) // clean up + + if _, err := tmpfile.Write(content); err != nil { + panic(err) + } + if err := tmpfile.Close(); err != nil { + panic(err) + } + chunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)), chunkSize) + assert.Empty(t, chunkRanges) + chunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)+10), chunkSize) + assert.Empty(t, chunkRanges) + chunkRanges = MissingChunks(tmpfile.Name()+"ok", int64(len(content)), chunkSize) + assert.Empty(t, chunkRanges) + chunks = ChunkRangesToChunks(chunkRanges) + assert.Empty(t, chunks) } // func Test1(t *testing.T) { @@ -112,3 +143,41 @@ func TestMissingChunks(t *testing.T) { // fmt.Println(ChunkRangesToChunks((chunkRanges))) // assert.Nil(t, nil) // } + +func TestHashFile(t *testing.T) { + content := []byte("temporary file's content") + tmpfile, err := ioutil.TempFile("", "example") + if err != nil { + log.Fatal(err) + } + + defer os.Remove(tmpfile.Name()) // clean up + + if _, err := tmpfile.Write(content); err != nil { + panic(err) + } + if err := tmpfile.Close(); err != nil { + panic(err) + } + hashed, err := HashFile(tmpfile.Name()) + assert.Nil(t, err) + assert.Equal(t, "18c9673a4bb8325d323e7f24fda9ae1e", fmt.Sprintf("%x", hashed)) +} + +func TestPublicIP(t *testing.T) { + ip, err := PublicIP() + fmt.Println(ip) + assert.True(t, strings.Contains(ip, ".")) + assert.Nil(t, err) +} + +func TestLocalIP(t *testing.T) { + ip := LocalIP() + fmt.Println(ip) + assert.True(t, strings.Contains(ip, ".")) +} + +func TestGetRandomName(t *testing.T) { + name := GetRandomName() + assert.NotEmpty(t, name) +}