From 304355af4833cd202124ec7f70b362a7677ddf9f Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:10:36 -0700 Subject: [PATCH 01/32] update pake --- go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 9aa3ab68..ca772692 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/mattn/go-isatty v0.0.4 // indirect github.com/pkg/errors v0.8.0 github.com/schollz/mnemonicode v1.0.1 - github.com/schollz/pake v1.0.2 + github.com/schollz/pake v1.1.0 github.com/schollz/peerdiscovery v1.2.2 github.com/schollz/progressbar/v2 v2.6.0 github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9 @@ -19,6 +19,6 @@ require ( github.com/stretchr/testify v1.2.2 github.com/tscholl2/siec v0.0.0-20180721101609-21667da05937 github.com/urfave/cli v1.20.0 - golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941 + golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e golang.org/x/net v0.0.0-20181005035420-146acd28ed58 // indirect ) From d05e196139ebc2647f5c642e18781df0ccf4aeb2 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:20:23 -0700 Subject: [PATCH 02/32] return errors through websockets --- src/cli/cli.go | 1 + src/croc/croc.go | 2 ++ src/croc/sender.go | 34 +++++++++++++++++++++++++--------- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index dee007fe..9f8a4a97 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -87,6 +87,7 @@ func Run() { } app.Before = func(c *cli.Context) error { cr = croc.Init(c.GlobalBool("debug")) + cr.Version = Version cr.AllowLocalDiscovery = true cr.Address = c.GlobalString("addr") cr.AddressTCPPorts = strings.Split(c.GlobalString("addr-tcp"), ",") diff --git a/src/croc/croc.go b/src/croc/croc.go index 11442c4c..fd11e207 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -17,6 +17,8 @@ func init() { // Croc options type Croc struct { + // Version is the version of croc + Version string // Options for all Debug bool // ShowText will display text on the stderr diff --git a/src/croc/sender.go b/src/croc/sender.go index 7e671115..f4b898c9 100644 --- a/src/croc/sender.go +++ b/src/croc/sender.go @@ -26,7 +26,6 @@ import ( "github.com/schollz/pake" progressbar "github.com/schollz/progressbar/v2" "github.com/schollz/spinner" - "github.com/tscholl2/siec" ) // Send is the async call to send data @@ -37,8 +36,11 @@ func (cr *Croc) startSender(forceSend int, serverAddress string, tcpPorts []stri if err != nil { if !strings.HasPrefix(err.Error(), "websocket: close 100") { fmt.Fprintf(os.Stderr, "\n"+err.Error()) + err = errors.Wrap(err, "error in sender:") + c.WriteMessage(websocket.TextMessage, []byte(err.Error())) + time.Sleep(50 * time.Millisecond) + cr.StateString = err.Error() } - cr.StateString = err.Error() } done <- struct{}{} @@ -91,12 +93,10 @@ func (cr *Croc) send(forceSend int, serverAddress string, tcpPorts []string, isL spin.Writer = os.Stderr defer spin.Stop() - // pick an elliptic curve - curve := siec.SIEC255() // both parties should have a weak key pw := []byte(codephrase) // initialize sender P ("0" indicates sender) - P, err := pake.Init(pw, 0, curve, 1*time.Millisecond) + P, err := pake.InitCurve(pw, 0, cr.CurveType, 1*time.Millisecond) if err != nil { return } @@ -113,6 +113,9 @@ func (cr *Croc) send(forceSend int, serverAddress string, tcpPorts []string, isL if messageType == websocket.TextMessage && bytes.Equal(message, []byte("interrupt")) { return errors.New("\rinterrupted by other party") } + if messageType == websocket.TextMessage && bytes.HasPrefix(message, []byte("err")) { + return errors.New("\r" + string(message)) + } log.Debugf("got %d: %s", messageType, message) switch step { case 0: @@ -123,11 +126,24 @@ func (cr *Croc) send(forceSend int, serverAddress string, tcpPorts []string, isL } else { ip, _ = utils.PublicIP() } - // send my IP address - c.WriteMessage(websocket.BinaryMessage, []byte(ip)) + + initialData := models.Initial{ + CurveType: cr.CurveType, + IPAddress: ip, + VersionString: cr.Version, // version should match + } + bInitialData, _ := json.Marshal(initialData) + // send the initial data + c.WriteMessage(websocket.BinaryMessage, bInitialData) case 1: - // first receive the IP address from the sender - cr.OtherIP = string(message) + // first receive the initial data from the recipient + var initialData models.Initial + err = json.Unmarshal(message, &initialData) + if err != nil { + err = errors.Wrap(err, "incompatible versions of croc") + return + } + cr.OtherIP = initialData.IPAddress log.Debugf("recipient IP: %s", cr.OtherIP) go func() { From 6e10ed25af3fdf17ae3a0ccffcb9a2a81d943040 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:23:48 -0700 Subject: [PATCH 03/32] recipient sends errors through websockets --- src/croc/recipient.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/croc/recipient.go b/src/croc/recipient.go index f05a9f6c..689b2f47 100644 --- a/src/croc/recipient.go +++ b/src/croc/recipient.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "encoding/json" - "errors" "fmt" "io/ioutil" "os" @@ -16,6 +15,7 @@ import ( log "github.com/cihub/seelog" humanize "github.com/dustin/go-humanize" "github.com/gorilla/websocket" + "github.com/pkg/errors" "github.com/schollz/croc/src/comm" "github.com/schollz/croc/src/compress" "github.com/schollz/croc/src/crypt" @@ -26,7 +26,6 @@ import ( "github.com/schollz/pake" "github.com/schollz/progressbar/v2" "github.com/schollz/spinner" - "github.com/tscholl2/siec" ) var DebugLevel string @@ -38,6 +37,10 @@ func (cr *Croc) startRecipient(forceSend int, serverAddress string, tcpPorts []s if err != nil { if !strings.HasPrefix(err.Error(), "websocket: close 100") { fmt.Fprintf(os.Stderr, "\n"+err.Error()) + cr.StateString = err.Error() + err = errors.Wrap(err, "error in recipient:") + c.WriteMessage(websocket.TextMessage, []byte(err.Error())) + time.Sleep(50 * time.Millisecond) } } done <- struct{}{} @@ -74,16 +77,11 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, spin.Start() defer spin.Stop() - // pick an elliptic curve - curve := siec.SIEC255() // both parties should have a weak key pw := []byte(codephrase) // initialize recipient Q ("1" indicates recipient) - Q, err := pake.Init(pw, 1, curve, 1*time.Millisecond) - if err != nil { - return - } + var Q *pake.Pake step := 0 for { @@ -101,10 +99,22 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, log.Debugf("got %d: %s", messageType, message) switch step { case 0: - // sender has initiated, sends their ip address - cr.OtherIP = string(message) + // sender has initiated, sends their initial data + var initialData models.Initial + err = json.Unmarshal(message, &initialData) + if err != nil { + err = errors.Wrap(err, "incompatible versions of croc") + return err + } + cr.OtherIP = initialData.IPAddress log.Debugf("sender IP: %s", cr.OtherIP) + Q, err = pake.InitCurve(pw, 1, initialData.CurveType, 1*time.Millisecond) + if err != nil { + err = errors.Wrap(err, "incompatible curve type") + return err + } + // recipient begins by sending address ip := "" if isLocal { From 9349496111894629eb9fa569863353822c12d09b Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:25:43 -0700 Subject: [PATCH 04/32] recipient sends back initial data --- src/croc/recipient.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/croc/recipient.go b/src/croc/recipient.go index 689b2f47..78ac5a17 100644 --- a/src/croc/recipient.go +++ b/src/croc/recipient.go @@ -53,6 +53,8 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, var progressFile string var resumeFile bool var tcpConnections []comm.Comm + var Q *pake.Pake + dataChan := make(chan []byte, 1024*1024) isConnectedIfUsingTCP := make(chan bool) blocks := []string{} @@ -80,9 +82,6 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, // both parties should have a weak key pw := []byte(codephrase) - // initialize recipient Q ("1" indicates recipient) - var Q *pake.Pake - step := 0 for { messageType, message, err := c.ReadMessage() @@ -109,22 +108,28 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, cr.OtherIP = initialData.IPAddress log.Debugf("sender IP: %s", cr.OtherIP) + // TODO: + // check whether the version strings are compatible + + // initialize the PAKE with the curve sent from the sender Q, err = pake.InitCurve(pw, 1, initialData.CurveType, 1*time.Millisecond) if err != nil { err = errors.Wrap(err, "incompatible curve type") return err } - // recipient begins by sending address + // recipient begins by sending back initial data to sender ip := "" if isLocal { ip = utils.LocalIP() } else { ip, _ = utils.PublicIP() } - c.WriteMessage(websocket.BinaryMessage, []byte(ip)) + initialData.VersionString = cr.Version + initialData.IPAddress = ip + bInitialData, _ := json.Marshal(initialData) + c.WriteMessage(websocket.BinaryMessage, bInitialData) case 1: - // Q receives u log.Debugf("[%d] Q computes k, sends H(k), v back to P", step) if err := Q.Update(message); err != nil { From e591c6afb67467a264818f4f38425a426feb2369 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:31:35 -0700 Subject: [PATCH 05/32] better error message when interrupting --- go.mod | 4 ++-- src/croc/sending.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ca772692..15444550 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/schollz/croc require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 - github.com/dustin/go-humanize v0.0.0-20180713052910-9f541cc9db5d + github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.7.0 // indirect github.com/gorilla/websocket v1.4.0 github.com/mars9/crypt v0.0.0-20150406101210-65899cf653ff // indirect @@ -20,5 +20,5 @@ require ( github.com/tscholl2/siec v0.0.0-20180721101609-21667da05937 github.com/urfave/cli v1.20.0 golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e - golang.org/x/net v0.0.0-20181005035420-146acd28ed58 // indirect + golang.org/x/net v0.0.0-20181017193950-04a2e542c03f // indirect ) diff --git a/src/croc/sending.go b/src/croc/sending.go index 74d972e2..ded75f8e 100644 --- a/src/croc/sending.go +++ b/src/croc/sending.go @@ -178,7 +178,7 @@ func (c *Croc) sendReceive(address, websocketPort string, tcpPorts []string, fna SetDebugLevel("critical") } log.Debug("interrupt") - err = sock.WriteMessage(websocket.TextMessage, []byte("interrupt")) + err = sock.WriteMessage(websocket.TextMessage, []byte("error: interrupted by other party")) if err != nil { return err } From 23d451540c0440e73a34aced00afce006ba7e003 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:33:29 -0700 Subject: [PATCH 06/32] update cli --- src/cli/cli.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index 9f8a4a97..e76c2b88 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -74,7 +74,7 @@ func Run() { cli.BoolFlag{Name: "force-web", Usage: "force websockets"}, cli.StringFlag{Name: "port", Value: "8153", Usage: "port that the websocket listens on"}, cli.StringFlag{Name: "tcp-port", Value: "8154,8155,8156,8157,8158,8159,8160,8161", Usage: "ports that the tcp server listens on"}, - cli.StringFlag{Name: "curve", Value: "siec", Usage: "specify elliptic curve to use (p224, p256, p384, p521, siec)"}, + cli.StringFlag{Name: "curve", Value: "siec", Usage: "specify elliptic curve to use for PAKE (p256, p384, p521, siec)"}, } app.EnableBashCompletion = true app.HideHelp = false From 7371f6f238522cc637fcd8e8134f6fe69e7a73b9 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:38:04 -0700 Subject: [PATCH 07/32] remove utils dependency --- go.mod | 1 - src/croc/sending.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 15444550..7072790d 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/schollz/peerdiscovery v1.2.2 github.com/schollz/progressbar/v2 v2.6.0 github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9 - github.com/schollz/utils v1.0.0 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c github.com/stretchr/testify v1.2.2 github.com/tscholl2/siec v0.0.0-20180721101609-21667da05937 diff --git a/src/croc/sending.go b/src/croc/sending.go index ded75f8e..7ec054bb 100644 --- a/src/croc/sending.go +++ b/src/croc/sending.go @@ -12,8 +12,8 @@ import ( log "github.com/cihub/seelog" "github.com/gorilla/websocket" "github.com/schollz/croc/src/relay" + "github.com/schollz/croc/src/utils" "github.com/schollz/peerdiscovery" - "github.com/schollz/utils" ) // Send the file From ace9b35e3474d2188c80862285bcf50510de4f4c Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:38:11 -0700 Subject: [PATCH 08/32] remove utils dependency --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 7072790d..e34c3396 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.7.0 // indirect github.com/gorilla/websocket v1.4.0 - github.com/mars9/crypt v0.0.0-20150406101210-65899cf653ff // indirect github.com/mattn/go-colorable v0.0.9 // indirect github.com/mattn/go-isatty v0.0.4 // indirect github.com/pkg/errors v0.8.0 From 9e7661c1b098a31e822332175c90b38d80b46d8d Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:43:45 -0700 Subject: [PATCH 09/32] update deps --- go.mod | 4 ++-- src/croc/sending.go | 2 +- src/crypt/crypt.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index e34c3396..50cadb6f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.7.0 // indirect + github.com/golang/crypto v0.0.0-20181015023909-0c41d7ab0a0e github.com/gorilla/websocket v1.4.0 github.com/mattn/go-colorable v0.0.9 // indirect github.com/mattn/go-isatty v0.0.4 // indirect @@ -14,9 +15,8 @@ require ( github.com/schollz/progressbar/v2 v2.6.0 github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c - github.com/stretchr/testify v1.2.2 github.com/tscholl2/siec v0.0.0-20180721101609-21667da05937 github.com/urfave/cli v1.20.0 - golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e + golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e // indirect golang.org/x/net v0.0.0-20181017193950-04a2e542c03f // indirect ) diff --git a/src/croc/sending.go b/src/croc/sending.go index 7ec054bb..86c2e142 100644 --- a/src/croc/sending.go +++ b/src/croc/sending.go @@ -96,7 +96,7 @@ func (c *Croc) Receive(codephrase string) (err error) { log.Debug(errDiscover) } if len(discovered) > 0 { - if discovered[0].Address == utils.GetLocalIP() { + if discovered[0].Address == utils.LocalIP() { discovered[0].Address = "localhost" } log.Debugf("discovered %s:%s", discovered[0].Address, discovered[0].Payload) diff --git a/src/crypt/crypt.go b/src/crypt/crypt.go index 7160ad53..c2512a2a 100644 --- a/src/crypt/crypt.go +++ b/src/crypt/crypt.go @@ -9,7 +9,7 @@ import ( "errors" "strings" - "golang.org/x/crypto/pbkdf2" + "github.com/golang/crypto/pbkdf2" ) // Encryption stores the data From 0d17219c87a916ce7fd369149d75bcf4aa32e63b Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:45:05 -0700 Subject: [PATCH 10/32] add initial struct fixes #98 --- src/models/initial.go | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/models/initial.go diff --git a/src/models/initial.go b/src/models/initial.go new file mode 100644 index 00000000..75b0877b --- /dev/null +++ b/src/models/initial.go @@ -0,0 +1,7 @@ +package models + +type Initial struct { + CurveType string + IPAddress string + VersionString string +} From 1ae117166a24fb7935a7d04aa2f0056de20964c3 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 07:54:59 -0700 Subject: [PATCH 11/32] add config default --- go.mod | 1 + src/croc/config.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 src/croc/config.go diff --git a/go.mod b/go.mod index 50cadb6f..177934f9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/schollz/croc require ( + github.com/BurntSushi/toml v0.3.1 // indirect github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.7.0 // indirect diff --git a/src/croc/config.go b/src/croc/config.go new file mode 100644 index 00000000..0e433427 --- /dev/null +++ b/src/croc/config.go @@ -0,0 +1,59 @@ +package croc + +import ( + "bytes" + "time" + + "github.com/BurntSushi/toml" +) + +type Config struct { + // Relay parameters + RelayWebsocketPort string + RelayTCPPorts []string + + // Sender parameters + CurveType string + + // Options for connecting to server + PublicServerIP string + AddressTCPPorts []string + AddressWebsocketPort string + Timeout time.Duration + LocalOnly bool + NoLocal bool + + // Options for file transfering + UseEncryption bool + UseCompression bool + AllowLocalDiscovery bool + NoRecipientPrompt bool + ForceTCP bool + ForceWebsockets bool + Codephrase string +} + +// DefaultConfig returns the default config +func DefaultConfig() string { + c := Config{} + cr := Init(false) + c.RelayWebsocketPort = cr.RelayWebsocketPort + c.RelayTCPPorts = cr.RelayTCPPorts + c.CurveType = cr.CurveType + c.PublicServerIP = cr.Address + c.AddressTCPPorts = cr.AddressTCPPorts + c.AddressWebsocketPort = cr.AddressWebsocketPort + c.Timeout = cr.Timeout + c.LocalOnly = cr.LocalOnly + c.NoLocal = cr.NoLocal + c.UseEncryption = cr.UseEncryption + c.UseCompression = cr.UseCompression + c.AllowLocalDiscovery = cr.AllowLocalDiscovery + c.NoRecipientPrompt = cr.NoRecipientPrompt + c.ForceTCP = false + c.ForceWebsockets = false + c.Codephrase = "" + buf := new(bytes.Buffer) + toml.NewEncoder(buf).Encode(c) + return buf.String() +} From 81bc06eabb295417186b33770ae6c1f90bf76909 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 08:21:58 -0700 Subject: [PATCH 12/32] add configuration file --- go.mod | 3 +- src/cli/cli.go | 27 +++++++-- src/croc/config.go | 134 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 154 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 177934f9..3f74465a 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/schollz/croc require ( - github.com/BurntSushi/toml v0.3.1 // indirect + github.com/BurntSushi/toml v0.3.1 github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.7.0 // indirect @@ -9,6 +9,7 @@ require ( github.com/gorilla/websocket v1.4.0 github.com/mattn/go-colorable v0.0.9 // indirect github.com/mattn/go-isatty v0.0.4 // indirect + github.com/mitchellh/go-homedir v1.0.0 github.com/pkg/errors v0.8.0 github.com/schollz/mnemonicode v1.0.1 github.com/schollz/pake v1.1.0 diff --git a/src/cli/cli.go b/src/cli/cli.go index e76c2b88..9afe16d4 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -60,6 +60,16 @@ func Run() { return relay(c) }, }, + { + Name: "config", + Usage: "generates a config file", + Description: "the croc config can be used to set static parameters", + Flags: []cli.Flag{}, + HelpName: "croc config", + Action: func(c *cli.Context) error { + return saveDefaultConfig(c) + }, + }, } app.Flags = []cli.Flag{ cli.StringFlag{Name: "addr", Value: "croc4.schollz.com", Usage: "address of the public relay"}, @@ -115,6 +125,10 @@ func Run() { } } +func saveDefaultConfig(c *cli.Context) error { + return croc.SaveDefaultConfig() +} + func send(c *cli.Context) error { stat, _ := os.Stdin.Stat() var fname string @@ -147,11 +161,12 @@ func send(c *cli.Context) error { cr.UseCompression = !c.Bool("no-compress") cr.UseEncryption = !c.Bool("no-encrypt") if c.String("code") != "" { - codePhrase = c.String("code") + cr.Codephrase = c.String("code") } - if len(codePhrase) == 0 { + cr.LoadConfig() + if len(cr.Codephrase) == 0 { // generate code phrase - codePhrase = utils.GetRandomName() + cr.Codephrase = utils.GetRandomName() } // print the text @@ -176,10 +191,10 @@ func send(c *cli.Context) error { humanize.Bytes(uint64(fsize)), fileOrFolder, filename, - codePhrase, - codePhrase, + cr.Codephrase, + cr.Codephrase, ) - return cr.Send(fname, codePhrase) + return cr.Send(fname, cr.Codephrase) } func receive(c *cli.Context) error { diff --git a/src/croc/config.go b/src/croc/config.go index 0e433427..b1750599 100644 --- a/src/croc/config.go +++ b/src/croc/config.go @@ -2,9 +2,16 @@ package croc import ( "bytes" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" "time" "github.com/BurntSushi/toml" + homedir "github.com/mitchellh/go-homedir" + "github.com/schollz/croc/src/utils" ) type Config struct { @@ -33,8 +40,7 @@ type Config struct { Codephrase string } -// DefaultConfig returns the default config -func DefaultConfig() string { +func defaultConfig() Config { c := Config{} cr := Init(false) c.RelayWebsocketPort = cr.RelayWebsocketPort @@ -53,7 +59,129 @@ func DefaultConfig() string { c.ForceTCP = false c.ForceWebsockets = false c.Codephrase = "" + return c +} + +func SaveDefaultConfig() error { + homedir, err := homedir.Dir() + if err != nil { + return err + } + os.MkdirAll(path.Join(homedir, ".config", "croc"), 0644) + c := defaultConfig() buf := new(bytes.Buffer) toml.NewEncoder(buf).Encode(c) - return buf.String() + confTOML := buf.String() + err = ioutil.WriteFile(path.Join(homedir, ".config", "croc", "config.toml"), []byte(confTOML), 0644) + if err == nil { + fmt.Printf("Default config file written at '%s'", filepath.Clean(path.Join(homedir, ".config", "croc", "config.toml"))) + } + return err +} + +// LoadConfig will override parameters +func (cr *Croc) LoadConfig() (err error) { + homedir, err := homedir.Dir() + if err != nil { + return err + } + pathToConfig := path.Join(homedir, ".config", "croc", "config.toml") + if !utils.Exists(pathToConfig) { + // ignore if doesn't exist + return nil + } + + var c Config + _, err = toml.DecodeFile(pathToConfig, &c) + if err != nil { + return + } + + cDefault := defaultConfig() + // only load if things are different than defaults + // just in case the CLI parameters are used + if c.RelayWebsocketPort != cDefault.RelayWebsocketPort && cr.RelayWebsocketPort == cDefault.RelayWebsocketPort { + cr.RelayWebsocketPort = c.RelayWebsocketPort + fmt.Printf("loaded RelayWebsocketPort from config\n") + } + if !slicesEqual(c.RelayTCPPorts, cDefault.RelayTCPPorts) && slicesEqual(cr.RelayTCPPorts, cDefault.RelayTCPPorts) { + cr.RelayTCPPorts = c.RelayTCPPorts + fmt.Printf("loaded RelayTCPPorts from config\n") + } + if c.CurveType != cDefault.CurveType && cr.CurveType == cDefault.CurveType { + cr.CurveType = c.CurveType + fmt.Printf("loaded CurveType from config\n") + } + if c.PublicServerIP != cDefault.PublicServerIP && cr.Address == cDefault.PublicServerIP { + cr.Address = c.PublicServerIP + fmt.Printf("loaded Address from config\n") + } + if !slicesEqual(c.AddressTCPPorts, cDefault.AddressTCPPorts) { + cr.AddressTCPPorts = c.AddressTCPPorts + fmt.Printf("loaded AddressTCPPorts from config\n") + } + if c.AddressWebsocketPort != cDefault.AddressWebsocketPort && cr.AddressWebsocketPort == cDefault.AddressWebsocketPort { + cr.AddressWebsocketPort = c.AddressWebsocketPort + fmt.Printf("loaded AddressWebsocketPort from config\n") + } + if c.Timeout != cDefault.Timeout && cr.Timeout == cDefault.Timeout { + cr.Timeout = c.Timeout + fmt.Printf("loaded Timeout from config\n") + } + if c.LocalOnly != cDefault.LocalOnly && cr.LocalOnly == cDefault.LocalOnly { + cr.LocalOnly = c.LocalOnly + fmt.Printf("loaded LocalOnly from config\n") + } + if c.NoLocal != cDefault.NoLocal && cr.NoLocal == cDefault.NoLocal { + cr.NoLocal = c.NoLocal + fmt.Printf("loaded NoLocal from config\n") + } + if c.UseEncryption != cDefault.UseEncryption && cr.UseEncryption == cDefault.UseEncryption { + cr.UseEncryption = c.UseEncryption + fmt.Printf("loaded UseEncryption from config\n") + } + if c.UseCompression != cDefault.UseCompression && cr.UseCompression == cDefault.UseCompression { + cr.UseCompression = c.UseCompression + fmt.Printf("loaded UseCompression from config\n") + } + if c.AllowLocalDiscovery != cDefault.AllowLocalDiscovery && cr.AllowLocalDiscovery == cDefault.AllowLocalDiscovery { + cr.AllowLocalDiscovery = c.AllowLocalDiscovery + fmt.Printf("loaded AllowLocalDiscovery from config\n") + } + if c.NoRecipientPrompt != cDefault.NoRecipientPrompt && cr.NoRecipientPrompt == cDefault.NoRecipientPrompt { + cr.NoRecipientPrompt = c.NoRecipientPrompt + fmt.Printf("loaded NoRecipientPrompt from config\n") + } + if c.ForceWebsockets { + cr.ForceSend = 1 + } + if c.ForceTCP { + cr.ForceSend = 2 + } + if c.Codephrase != cDefault.Codephrase && cr.Codephrase == cDefault.Codephrase { + cr.Codephrase = c.Codephrase + fmt.Printf("loaded Codephrase from config\n") + } + return +} + +// slicesEqual checcks if two slices are equal +// from https://stackoverflow.com/a/15312097 +func slicesEqual(a, b []string) bool { + // If one is nil, the other must also be nil. + if (a == nil) != (b == nil) { + return false + } + + if len(a) != len(b) { + return false + } + + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true } From dd03cdc6e3ac2c843fd0973e14f9a71f0149e13b Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 08:23:56 -0700 Subject: [PATCH 13/32] update readme about config file --- .github/default.txt | 2 +- README.md | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/default.txt b/.github/default.txt index 9f035513..ec6da57a 100644 --- a/.github/default.txt +++ b/.github/default.txt @@ -33,7 +33,7 @@ install_croc() croc_os="unsupported" croc_arch="unknown" croc_arm="" - croc_version="4.0.8" + croc_version="4.1.0" # Termux on Android has $PREFIX set which already ends with /usr diff --git a/README.md b/README.md index 3a025acd..ce59d2b2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ src="https://user-images.githubusercontent.com/6550035/46709024-9b23ad00-cbf6-11e8-9fb2-ca8b20b7dbec.jpg" width="408px" border="0" alt="croc">
-Version +Version Code coverage Build @@ -122,6 +122,17 @@ You can send files using your relay by entering `-relay` to change the relay tha $ croc -relay Date: Sun, 21 Oct 2018 08:25:53 -0700 Subject: [PATCH 14/32] recipient also uses codephrase --- src/cli/cli.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index 9afe16d4..8e03def9 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -19,7 +19,6 @@ import ( ) var Version string -var codePhrase string var cr *croc.Croc func Run() { @@ -199,20 +198,21 @@ func send(c *cli.Context) error { func receive(c *cli.Context) error { if c.GlobalString("code") != "" { - codePhrase = c.GlobalString("code") + cr.Codephrase = c.GlobalString("code") } if c.Args().First() != "" { - codePhrase = c.Args().First() + cr.Codephrase = c.Args().First() } + cr.LoadConfig() openFolder := false if len(os.Args) == 1 { // open folder since they didn't give any arguments openFolder = true } - if codePhrase == "" { - codePhrase = utils.GetInput("Enter receive code: ") + if cr.Codephrase == "" { + cr.Codephrase = utils.GetInput("Enter receive code: ") } - err := cr.Receive(codePhrase) + err := cr.Receive(cr.Codephrase) if err == nil && openFolder { cwd, _ := os.Getwd() open.Run(cwd) From 45b7eebbb9a7b0c80a8c026557a55464f478a5fa Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 09:05:27 -0700 Subject: [PATCH 15/32] use normal import --- src/crypt/crypt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crypt/crypt.go b/src/crypt/crypt.go index c2512a2a..7160ad53 100644 --- a/src/crypt/crypt.go +++ b/src/crypt/crypt.go @@ -9,7 +9,7 @@ import ( "errors" "strings" - "github.com/golang/crypto/pbkdf2" + "golang.org/x/crypto/pbkdf2" ) // Encryption stores the data From 708e6fab8b1fd23be0ead155649a44ef8c1e2e09 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 10:06:14 -0600 Subject: [PATCH 16/32] fix imports --- go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3f74465a..33ba2563 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ require ( github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.7.0 // indirect - github.com/golang/crypto v0.0.0-20181015023909-0c41d7ab0a0e github.com/gorilla/websocket v1.4.0 github.com/mattn/go-colorable v0.0.9 // indirect github.com/mattn/go-isatty v0.0.4 // indirect @@ -19,6 +18,6 @@ require ( github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c github.com/tscholl2/siec v0.0.0-20180721101609-21667da05937 github.com/urfave/cli v1.20.0 - golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e // indirect + golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e golang.org/x/net v0.0.0-20181017193950-04a2e542c03f // indirect ) From 380ecc7ff917f7712f1bf7a953ba873163441361 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 10:33:15 -0700 Subject: [PATCH 17/32] update gomod --- go.mod | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 33ba2563..457c80e3 100644 --- a/go.mod +++ b/go.mod @@ -16,8 +16,7 @@ require ( github.com/schollz/progressbar/v2 v2.6.0 github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c - github.com/tscholl2/siec v0.0.0-20180721101609-21667da05937 github.com/urfave/cli v1.20.0 - golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e + golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 golang.org/x/net v0.0.0-20181017193950-04a2e542c03f // indirect ) From 3dc6d44c41db09d4d62f89399807efa50e3e0ddc Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 10:46:04 -0700 Subject: [PATCH 18/32] throw error if versions are incompatible --- src/croc/recipient.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/croc/recipient.go b/src/croc/recipient.go index 78ac5a17..d88afef4 100644 --- a/src/croc/recipient.go +++ b/src/croc/recipient.go @@ -108,8 +108,14 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, cr.OtherIP = initialData.IPAddress log.Debugf("sender IP: %s", cr.OtherIP) - // TODO: // check whether the version strings are compatible + versionStringsOther := strings.Split(initialData.VersionString, ".") + versionStringsSelf := strings.Split(cr.Version, ".") + if len(versionStringsOther) == 3 && len(versionStringsSelf) == 3 { + if versionStringsSelf[0] != versionStringsOther[0] || versionStringsSelf[1] != versionStringsOther[1] { + return fmt.Errorf("version sender %s is not compatible with recipient %s", cr.Version, initialData.VersionString) + } + } // initialize the PAKE with the curve sent from the sender Q, err = pake.InitCurve(pw, 1, initialData.CurveType, 1*time.Millisecond) From 47931d6ba200bcfade49a70d08e7af6817a92d01 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Sun, 21 Oct 2018 11:34:31 -0700 Subject: [PATCH 19/32] update readme --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ce59d2b2..89e6e7f2 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,16 @@ Status"> ## Overview -*croc* uses "code phrases" to securely transfer files. A code phrase is a combination of three random words (mnemonicoded 4 bytes) which the sender shares with the recipient. The code phrase is used by the sender and recipient for password authenticated key exchange ([PAKE](https://github.com/schollz/pake)) to validate parties and generate a secure session key for end-to-end encryption. Since a code phrase can only be used once between two parties, an attacker has a chance of less than 1 in *4 billion* to guess the right code phrase to steal the file. Any attacker with the wrong code phrase will fail the PAKE and the sender will be notified. Only two people with the right code phrase will be able to computers transfer encrypted data through a relay. +**Transmit encrypted data with a code phrase** + +*croc* securely transfers data using *code phrases* - a combination of three random words (mnemonicoded 4 bytes). The code phrase is shared between the sender and the recipient for password authenticated key exchange ([PAKE](https://github.com/schollz/pake)), a cryptographic method to use a shared weak key (the "code phrase") to generate a strong key for secure end-to-end encryption. By default, a code phrase can only be used once between two parties so an attacker would have a chance of less than 1 in *4 billion* to guess the code phrase correctly to steal the data. An attacker with the wrong code phrase will fail the PAKE and the sender will be notified without any data transfering. Only two people with the right code phrase will be able to computers transfer encrypted data through a relay. + +**Fast data transfer through TCP** The actual data transfer is accomplished using a relay, either using raw TCP sockets or websockets. If both computers are on the LAN network then *croc* will use a local relay, otherwise a public relay is used. All the data going through the relay is encrypted using the PAKE-generated session key, so the relay can't spy on information passing through it. The data is transferred in blocks, where each block is compressed and encrypted, and the recipient keeps track of blocks received so that it can resume the transfer if interrupted. +**Why another data transfer utility?** + My motivation to write *croc*, as stupid as it sounds, is because I wanted to create a program that made it easy to send a 3GB+ PBS documentary to my friend in a different country. My friend has a Windows computer and is not comfortable using a terminal. So I wanted to write a program that, while secure, is simple to receive a file. *croc* accomplishes this, and now I find myself using it almost everyday at work. To receive a file you can just download the executable and double click on it (sending a file requires opening a terminal still, though). ## Examples From 3064ceef816723b59a0c2d77ace750dcb523d7c2 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 06:36:36 -0700 Subject: [PATCH 20/32] add output folder --- src/cli/cli.go | 4 ++++ src/croc/sender.go | 1 + src/croc/sending.go | 3 +++ 3 files changed, 8 insertions(+) diff --git a/src/cli/cli.go b/src/cli/cli.go index 8e03def9..1059dbfa 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -84,6 +84,7 @@ func Run() { cli.StringFlag{Name: "port", Value: "8153", Usage: "port that the websocket listens on"}, cli.StringFlag{Name: "tcp-port", Value: "8154,8155,8156,8157,8158,8159,8160,8161", Usage: "ports that the tcp server listens on"}, cli.StringFlag{Name: "curve", Value: "siec", Usage: "specify elliptic curve to use for PAKE (p256, p384, p521, siec)"}, + cli.StringFlag{Name: "out", Value: ".", Usage: "specify an output folder to receive the file"}, } app.EnableBashCompletion = true app.HideHelp = false @@ -203,6 +204,9 @@ func receive(c *cli.Context) error { if c.Args().First() != "" { cr.Codephrase = c.Args().First() } + if c.GlobalString("out") != "" { + os.Chdir(c.GlobalString("out")) + } cr.LoadConfig() openFolder := false if len(os.Args) == 1 { diff --git a/src/croc/sender.go b/src/croc/sender.go index f4b898c9..bea4a441 100644 --- a/src/croc/sender.go +++ b/src/croc/sender.go @@ -34,6 +34,7 @@ func (cr *Croc) startSender(forceSend int, serverAddress string, tcpPorts []stri log.Debugf("sending %s", fname) err := cr.send(forceSend, serverAddress, tcpPorts, isLocal, c, fname, codephrase, useCompression, useEncryption) if err != nil { + log.Debug(err) if !strings.HasPrefix(err.Error(), "websocket: close 100") { fmt.Fprintf(os.Stderr, "\n"+err.Error()) err = errors.Wrap(err, "error in sender:") diff --git a/src/croc/sending.go b/src/croc/sending.go index 86c2e142..1e2834fb 100644 --- a/src/croc/sending.go +++ b/src/croc/sending.go @@ -153,6 +153,7 @@ func (c *Croc) sendReceive(address, websocketPort string, tcpPorts []string, fna log.Debugf("connecting to %s", websocketAddress) sock, _, err := websocket.DefaultDialer.Dial(websocketAddress, nil) if err != nil { + log.Error(err) return } defer sock.Close() @@ -160,6 +161,7 @@ func (c *Croc) sendReceive(address, websocketPort string, tcpPorts []string, fna // tell the websockets we are connected err = sock.WriteMessage(websocket.BinaryMessage, []byte("connected")) if err != nil { + log.Error(err) return err } @@ -172,6 +174,7 @@ func (c *Croc) sendReceive(address, websocketPort string, tcpPorts []string, fna for { select { case <-done: + log.Debug("received done signal") return nil case <-interrupt: if !c.Debug { From 3d9bc7193dc59cd5dddbb7c564c87c3a5ff3cae5 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 16:54:57 -0700 Subject: [PATCH 21/32] use latest go --- .travis.yml | 2 +- go.mod | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a294f6c3..987919ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: go go: - - tip + - "1.11" env: - "PATH=/home/travis/gopath/bin:$PATH" diff --git a/go.mod b/go.mod index 457c80e3..96aa8805 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/schollz/progressbar/v2 v2.6.0 github.com/schollz/spinner v0.0.0-20180925172146-6bbc5f7804f9 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c + github.com/stretchr/testify v1.2.2 github.com/urfave/cli v1.20.0 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 golang.org/x/net v0.0.0-20181017193950-04a2e542c03f // indirect From db42e96b7e20313075426ed70f3b105ff0fecf88 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 16:56:19 -0700 Subject: [PATCH 22/32] add go111module on --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 987919ae..70e4a913 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ go: env: - "PATH=/home/travis/gopath/bin:$PATH" + - "GO111MODULE=on" script: - go build -v From ab5df93d10318a502cd5013592b83c5670be16ad Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 19:11:43 -0700 Subject: [PATCH 23/32] move reading to goroutine --- src/croc/croc_test.go | 6 +++--- src/croc/recipient.go | 37 +++++++++++++++++++++++-------------- src/croc/sender.go | 19 ++++++++++++++++++- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/src/croc/croc_test.go b/src/croc/croc_test.go index 65b07c60..ca08c3ca 100644 --- a/src/croc/croc_test.go +++ b/src/croc/croc_test.go @@ -67,9 +67,9 @@ func TestSendReceiveLocalWebsockets(t *testing.T) { sendAndReceive(t, 1, true) } -// func TestSendReceiveLocalTCP(t *testing.T) { -// sendAndReceive(t, 2, true) -// } +func TestSendReceiveLocalTCP(t *testing.T) { + sendAndReceive(t, 2, true) +} func generateRandomFile(megabytes int) (fname string) { // generate a random file diff --git a/src/croc/recipient.go b/src/croc/recipient.go index d88afef4..63bdf683 100644 --- a/src/croc/recipient.go +++ b/src/croc/recipient.go @@ -82,9 +82,26 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, // both parties should have a weak key pw := []byte(codephrase) + // start the reader + websocketMessages := make(chan WebSocketMessage, 1024) + go func() { + defer func() { + if r := recover(); r != nil { + log.Debugf("recovered from %s", r) + } + }() + for { + messageType, message, err := c.ReadMessage() + websocketMessages <- WebSocketMessage{messageType, message, err} + } + }() + step := 0 for { - messageType, message, err := c.ReadMessage() + websocketMessage := <-websocketMessages + messageType := websocketMessage.messageType + message := websocketMessage.message + err := websocketMessage.err if err != nil { return err } @@ -151,6 +168,7 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, // initialize TCP connections if using (possible, but unlikely, race condition) go func() { + log.Debug("initializing TCP connections") if !useWebsockets { log.Debugf("connecting to server") tcpConnections = make([]comm.Comm, len(tcpPorts)) @@ -406,29 +424,20 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, startTime := time.Now() if useWebsockets { for { - var messageType int // read from websockets - messageType, message, err = c.ReadMessage() - if messageType != websocket.BinaryMessage { + websocketMessageData := <-websocketMessages + if websocketMessageData.messageType != websocket.BinaryMessage { continue } if err != nil { log.Error(err) return err } - if bytes.Equal(message, []byte("magic")) { + if bytes.Equal(websocketMessageData.message, []byte("magic")) { log.Debug("got magic") break } - dataChan <- message - // select { - // case dataChan <- message: - // default: - // log.Debug("blocked") - // // no message sent - // // block - // dataChan <- message - // } + dataChan <- websocketMessageData.message } } else { log.Debugf("starting listening with tcp with %d connections", len(tcpConnections)) diff --git a/src/croc/sender.go b/src/croc/sender.go index bea4a441..1ae0bee8 100644 --- a/src/croc/sender.go +++ b/src/croc/sender.go @@ -102,9 +102,26 @@ func (cr *Croc) send(forceSend int, serverAddress string, tcpPorts []string, isL return } + // start the reader + websocketMessages := make(chan WebSocketMessage, 1024) + go func() { + defer func() { + if r := recover(); r != nil { + log.Debugf("recovered from %s", r) + } + }() + for { + messageType, message, err := c.ReadMessage() + websocketMessages <- WebSocketMessage{messageType, message, err} + } + }() + step := 0 for { - messageType, message, errRead := c.ReadMessage() + websocketMessage := <-websocketMessages + messageType := websocketMessage.messageType + message := websocketMessage.message + errRead := websocketMessage.err if errRead != nil { return errRead } From 6a07e1538d014fbf83903af97a6dc53cb7b03137 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 19:39:16 -0700 Subject: [PATCH 24/32] sender listens for recipient to close --- src/croc/sender.go | 51 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/src/croc/sender.go b/src/croc/sender.go index 1ae0bee8..50fb71b3 100644 --- a/src/croc/sender.go +++ b/src/croc/sender.go @@ -429,6 +429,32 @@ func (cr *Croc) send(forceSend int, serverAddress string, tcpPorts []string, isL } else { _ = <-isConnectedIfUsingTCP log.Debug("connected and ready to send on tcp") + + // check to see if any messages are sent + stopMessageSignal := make(chan bool, 1) + errorsDuringTransfer := make(chan error, 24) + go func() { + for { + select { + case sig := <-stopMessageSignal: + errorsDuringTransfer <- nil + log.Debugf("got message signal: %+v", sig) + return + case wsMessage := <-websocketMessages: + log.Debugf("got message: %s", wsMessage.message) + if bytes.HasPrefix(wsMessage.message, []byte("error")) { + log.Debug("stopping transfer") + for i := 0; i < len(tcpConnections)+1; i++ { + errorsDuringTransfer <- fmt.Errorf("%s", wsMessage.message) + } + return + } + default: + continue + } + } + }() + var wg sync.WaitGroup wg.Add(len(tcpConnections)) for i := range tcpConnections { @@ -439,16 +465,23 @@ func (cr *Croc) send(forceSend int, serverAddress string, tcpPorts []string, isL go func(i int, wg *sync.WaitGroup, dataChan <-chan DataChan) { defer wg.Done() for data := range dataChan { + select { + case _ = <-errorsDuringTransfer: + log.Debugf("%d got stop", i) + return + default: + } if data.err != nil { log.Error(data.err) return } cr.Bar.Add(data.bytesRead) // write data to tcp connection - _, err = tcpConnections[i].Write(data.b) - if err != nil { - err = errors.Wrap(err, "problem writing message") - log.Error(err) + _, errTcp := tcpConnections[i].Write(data.b) + if errTcp != nil { + errTcp = errors.Wrap(errTcp, "problem writing message") + log.Debug(errTcp) + errorsDuringTransfer <- errTcp return } if bytes.Equal(data.b, []byte("magic")) { @@ -458,7 +491,17 @@ func (cr *Croc) send(forceSend int, serverAddress string, tcpPorts []string, isL } }(i, &wg, dataChan) } + // block until this is done + log.Debug("waiting for tcp goroutines") wg.Wait() + log.Debug("sending stop message signal") + stopMessageSignal <- true + log.Debug("waiting for error") + errorDuringTransfer := <-errorsDuringTransfer + if errorDuringTransfer != nil { + log.Debugf("got error during transfer: %s", errorDuringTransfer.Error()) + return errorDuringTransfer + } } cr.Bar.Finish() From ea9aa3f8ec95cc0cb3d19f62d3e9568cffeb732d Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 19:41:50 -0700 Subject: [PATCH 25/32] recipient websockets should check for errors --- src/croc/recipient.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/croc/recipient.go b/src/croc/recipient.go index 63bdf683..fe632acc 100644 --- a/src/croc/recipient.go +++ b/src/croc/recipient.go @@ -426,6 +426,9 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, for { // read from websockets websocketMessageData := <-websocketMessages + if bytes.HasPrefix(websocketMessageData.message, []byte("error")) { + return fmt.Errorf("%s", websocketMessageData.message) + } if websocketMessageData.messageType != websocket.BinaryMessage { continue } From e5fcc0d3eeff78cab4fed0ef650d84053220525c Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 19:42:39 -0700 Subject: [PATCH 26/32] add models --- src/croc/models.go | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/croc/models.go diff --git a/src/croc/models.go b/src/croc/models.go new file mode 100644 index 00000000..de2650a3 --- /dev/null +++ b/src/croc/models.go @@ -0,0 +1,7 @@ +package croc + +type WebSocketMessage struct { + messageType int + message []byte + err error +} From c2dd9091ff5432877d874c35ce0b0f34416c55b7 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 19:48:45 -0700 Subject: [PATCH 27/32] recipient listens to sender --- src/croc/croc_test.go | 6 +-- src/croc/recipient.go | 94 ++++++++++++++++++++++++++++++++----------- src/croc/sender.go | 1 + 3 files changed, 75 insertions(+), 26 deletions(-) diff --git a/src/croc/croc_test.go b/src/croc/croc_test.go index ca08c3ca..65b07c60 100644 --- a/src/croc/croc_test.go +++ b/src/croc/croc_test.go @@ -67,9 +67,9 @@ func TestSendReceiveLocalWebsockets(t *testing.T) { sendAndReceive(t, 1, true) } -func TestSendReceiveLocalTCP(t *testing.T) { - sendAndReceive(t, 2, true) -} +// func TestSendReceiveLocalTCP(t *testing.T) { +// sendAndReceive(t, 2, true) +// } func generateRandomFile(megabytes int) (fname string) { // generate a random file diff --git a/src/croc/recipient.go b/src/croc/recipient.go index fe632acc..dd4b2259 100644 --- a/src/croc/recipient.go +++ b/src/croc/recipient.go @@ -444,33 +444,81 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, } } else { log.Debugf("starting listening with tcp with %d connections", len(tcpConnections)) - // using TCP - var wg sync.WaitGroup - wg.Add(len(tcpConnections)) - for i := range tcpConnections { - defer func(i int) { - log.Debugf("closing connection %d", i) - tcpConnections[i].Close() - }(i) - go func(wg *sync.WaitGroup, j int) { - defer wg.Done() - for { - log.Debugf("waiting to read on %d", j) - // read from TCP connection - message, _, _, err := tcpConnections[j].Read() - // log.Debugf("message: %s", message) - if err != nil { - panic(err) - } - if bytes.Equal(message, []byte("magic")) { - log.Debugf("%d got magic, leaving", j) + + // check to see if any messages are sent + stopMessageSignal := make(chan bool, 1) + errorsDuringTransfer := make(chan error, 24) + go func() { + for { + select { + case sig := <-stopMessageSignal: + errorsDuringTransfer <- nil + log.Debugf("got message signal: %+v", sig) + return + case wsMessage := <-websocketMessages: + log.Debugf("got message: %s", wsMessage.message) + if bytes.HasPrefix(wsMessage.message, []byte("error")) { + log.Debug("stopping transfer") + for i := 0; i < len(tcpConnections)+1; i++ { + errorsDuringTransfer <- fmt.Errorf("%s", wsMessage.message) + } return } - dataChan <- message + default: + continue } - }(&wg, i) + } + }() + + // using TCP + go func() { + var wg sync.WaitGroup + wg.Add(len(tcpConnections)) + for i := range tcpConnections { + defer func(i int) { + log.Debugf("closing connection %d", i) + tcpConnections[i].Close() + }(i) + go func(wg *sync.WaitGroup, j int) { + defer wg.Done() + for { + select { + case _ = <-errorsDuringTransfer: + log.Debugf("%d got stop", i) + return + default: + } + + log.Debugf("waiting to read on %d", j) + // read from TCP connection + message, _, _, err := tcpConnections[j].Read() + // log.Debugf("message: %s", message) + if err != nil { + panic(err) + } + if bytes.Equal(message, []byte("magic")) { + log.Debugf("%d got magic, leaving", j) + return + } + dataChan <- message + } + }(&wg, i) + } + log.Debug("waiting for tcp goroutines") + wg.Wait() + errorsDuringTransfer <- nil + }() + + // block until this is done + + log.Debug("waiting for error") + errorDuringTransfer := <-errorsDuringTransfer + log.Debug("sending stop message signal") + stopMessageSignal <- true + if errorDuringTransfer != nil { + log.Debugf("got error during transfer: %s", errorDuringTransfer.Error()) + return errorDuringTransfer } - wg.Wait() } _ = <-finished diff --git a/src/croc/sender.go b/src/croc/sender.go index 50fb71b3..925ba5e9 100644 --- a/src/croc/sender.go +++ b/src/croc/sender.go @@ -491,6 +491,7 @@ func (cr *Croc) send(forceSend int, serverAddress string, tcpPorts []string, isL } }(i, &wg, dataChan) } + // block until this is done log.Debug("waiting for tcp goroutines") wg.Wait() From edc206e73e4b46f747423a3cc4e5827971196962 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 20:01:40 -0700 Subject: [PATCH 28/32] fix travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 70e4a913..987919ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ go: env: - "PATH=/home/travis/gopath/bin:$PATH" - - "GO111MODULE=on" script: - go build -v From 08c84e9f855db7c581e55fd0d096eb82409f8498 Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 20:34:04 -0700 Subject: [PATCH 29/32] connect to tcp in parallel --- src/cli/cli.go | 2 ++ src/croc/recipient.go | 24 +++++++++++++++--------- src/croc/sender.go | 24 +++++++++++++++--------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index 1059dbfa..344357e7 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -8,6 +8,7 @@ import ( "log" "os" "path/filepath" + "runtime" "strings" "time" @@ -22,6 +23,7 @@ var Version string var cr *croc.Croc func Run() { + runtime.GOMAXPROCS(runtime.NumCPU()) app := cli.NewApp() app.Name = "croc" if Version == "" { diff --git a/src/croc/recipient.go b/src/croc/recipient.go index dd4b2259..d41cf758 100644 --- a/src/croc/recipient.go +++ b/src/croc/recipient.go @@ -172,17 +172,23 @@ func (cr *Croc) receive(forceSend int, serverAddress string, tcpPorts []string, if !useWebsockets { log.Debugf("connecting to server") tcpConnections = make([]comm.Comm, len(tcpPorts)) + var wg sync.WaitGroup + wg.Add(len(tcpPorts)) for i, tcpPort := range tcpPorts { - log.Debugf("connecting to %d", i) - var message string - tcpConnections[i], message, err = connectToTCPServer(utils.SHA256(fmt.Sprintf("%d%x", i, sessionKey)), serverAddress+":"+tcpPort) - if err != nil { - log.Error(err) - } - if message != "recipient" { - log.Errorf("got wrong message: %s", message) - } + go func(i int, tcpPort string) { + defer wg.Done() + log.Debugf("connecting to %d", i) + var message string + tcpConnections[i], message, err = connectToTCPServer(utils.SHA256(fmt.Sprintf("%d%x", i, sessionKey)), serverAddress+":"+tcpPort) + if err != nil { + log.Error(err) + } + if message != "recipient" { + log.Errorf("got wrong message: %s", message) + } + }(i, tcpPort) } + wg.Wait() log.Debugf("fully connected") } isConnectedIfUsingTCP <- true diff --git a/src/croc/sender.go b/src/croc/sender.go index 925ba5e9..b6f4248d 100644 --- a/src/croc/sender.go +++ b/src/croc/sender.go @@ -284,17 +284,23 @@ func (cr *Croc) send(forceSend int, serverAddress string, tcpPorts []string, isL go func() { if !useWebsockets { log.Debugf("connecting to server") + var wg sync.WaitGroup + wg.Add(len(tcpPorts)) for i, tcpPort := range tcpPorts { - log.Debugf("connecting to %s on connection %d", tcpPort, i) - var message string - tcpConnections[i], message, err = connectToTCPServer(utils.SHA256(fmt.Sprintf("%d%x", i, sessionKey)), serverAddress+":"+tcpPort) - if err != nil { - log.Error(err) - } - if message != "sender" { - log.Errorf("got wrong message: %s", message) - } + go func(i int, tcpPort string) { + defer wg.Done() + log.Debugf("connecting to %s on connection %d", tcpPort, i) + var message string + tcpConnections[i], message, err = connectToTCPServer(utils.SHA256(fmt.Sprintf("%d%x", i, sessionKey)), serverAddress+":"+tcpPort) + if err != nil { + log.Error(err) + } + if message != "sender" { + log.Errorf("got wrong message: %s", message) + } + }(i, tcpPort) } + wg.Wait() } isConnectedIfUsingTCP <- true }() From 721567fc728469244c2a0b9a4b2cb2c51ae16abc Mon Sep 17 00:00:00 2001 From: Zack Date: Mon, 22 Oct 2018 20:46:20 -0700 Subject: [PATCH 30/32] Create issue_template.md --- .github/issue_template.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/issue_template.md diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 00000000..d2eb4680 --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,17 @@ +## Expected Behavior + + +## Actual Behavior + + +## Steps to Reproduce the Problem + + 1. + 1. + 1. + +## Specifications + + - Version (`croc -version` or let me know latest commit in git log if using dev): + - Platform: + - Subsystem: From 824c49254f7e57fad74bb9bf1b570b39a4c7b83d Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Mon, 22 Oct 2018 20:50:36 -0700 Subject: [PATCH 31/32] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89e6e7f2..c24346f1 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The actual data transfer is accomplished using a relay, either using raw TCP soc **Why another data transfer utility?** -My motivation to write *croc*, as stupid as it sounds, is because I wanted to create a program that made it easy to send a 3GB+ PBS documentary to my friend in a different country. My friend has a Windows computer and is not comfortable using a terminal. So I wanted to write a program that, while secure, is simple to receive a file. *croc* accomplishes this, and now I find myself using it almost everyday at work. To receive a file you can just download the executable and double click on it (sending a file requires opening a terminal still, though). +My motivation to write *croc*, as stupid as it sounds, is because I wanted to create a program that made it easy to send a 3GB+ PBS documentary to my friend in a different country. My friend has a Windows computer and is not comfortable using a terminal. So I wanted to write a program that, while secure, is simple to receive a file. *croc* accomplishes this, and now I find myself using it almost everyday at work. To receive a file you can just download the executable and double click on it (sending a file requires opening a terminal still, though). The name is inspired by the [fable of the frog and the crocodile](https://web.archive.org/web/20180926035731/http://allaboutfrogs.org/stories/crocodile.html). ## Examples From 61d14be70d007759cc0ce789c4882eeca50fa64c Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Tue, 23 Oct 2018 05:28:30 -0700 Subject: [PATCH 32/32] add compress test --- src/compress/compress.go | 9 +++- src/compress/compress_test.go | 78 +++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 src/compress/compress_test.go diff --git a/src/compress/compress.go b/src/compress/compress.go index bd1a1b77..e23dd3e2 100644 --- a/src/compress/compress.go +++ b/src/compress/compress.go @@ -6,10 +6,17 @@ import ( "io" ) +// CompressWithOption returns compressed data using the specified level +func CompressWithOption(src []byte, level int) []byte { + compressedData := new(bytes.Buffer) + compress(src, compressedData, level) + return compressedData.Bytes() +} + // Compress returns a compressed byte slice. func Compress(src []byte) []byte { compressedData := new(bytes.Buffer) - compress(src, compressedData, 9) + compress(src, compressedData, -2) return compressedData.Bytes() } diff --git a/src/compress/compress_test.go b/src/compress/compress_test.go new file mode 100644 index 00000000..563883df --- /dev/null +++ b/src/compress/compress_test.go @@ -0,0 +1,78 @@ +package compress + +import ( + "crypto/rand" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +var fable = []byte(`The Frog and the Crocodile +Once, there was a frog who lived in the middle of a swamp. His entire family had lived in that swamp for generations, but this particular frog decided that he had had quite enough wetness to last him a lifetime. He decided that he was going to find a dry place to live instead. + +The only thing that separated him from dry land was a swampy, muddy, swiftly flowing river. But the river was home to all sorts of slippery, slittering snakes that loved nothing better than a good, plump frog for dinner, so Frog didn't dare try to swim across. + +So for many days, the frog stayed put, hopping along the bank, trying to think of a way to get across. + +The snakes hissed and jeered at him, daring him to come closer, but he refused. Occasionally they would slither closer, jaws open to attack, but the frog always leaped out of the way. But no matter how far upstream he searched or how far downstream, the frog wasn't able to find a way across the water. + +He had felt certain that there would be a bridge, or a place where the banks came together, yet all he found was more reeds and water. After a while, even the snakes stopped teasing him and went off in search of easier prey. + +The frog sighed in frustration and sat to sulk in the rushes. Suddenly, he spotted two big eyes staring at him from the water. The giant log-shaped animal opened its mouth and asked him, "What are you doing, Frog? Surely there are enough flies right there for a meal." + +The frog croaked in surprise and leaped away from the crocodile. That creature could swallow him whole in a moment without thinking about it! Once he was a satisfied that he was a safe distance away, he answered. "I'm tired of living in swampy waters, and I want to travel to the other side of the river. But if I swim across, the snakes will eat me." + +The crocodile harrumphed in agreement and sat, thinking, for a while. "Well, if you're afraid of the snakes, I could give you a ride across," he suggested. + +"Oh no, I don't think so," Frog answered quickly. "You'd eat me on the way over, or go underwater so the snakes could get me!" + +"Now why would I let the snakes get you? I think they're a terrible nuisance with all their hissing and slithering! The river would be much better off without them altogether! Anyway, if you're so worried that I might eat you, you can ride on my tail." + +The frog considered his offer. He did want to get to dry ground very badly, and there didn't seem to be any other way across the river. He looked at the crocodile from his short, squat buggy eyes and wondered about the crocodile's motives. But if he rode on the tail, the croc couldn't eat him anyway. And he was right about the snakes--no self-respecting crocodile would give a meal to the snakes. + +"Okay, it sounds like a good plan to me. Turn around so I can hop on your tail." + +The crocodile flopped his tail into the marshy mud and let the frog climb on, then he waddled out to the river. But he couldn't stick his tail into the water as a rudder because the frog was on it -- and if he put his tail in the water, the snakes would eat the frog. They clumsily floated downstream for a ways, until the crocodile said, "Hop onto my back so I can steer straight with my tail." The frog moved, and the journey smoothed out. + +From where he was sitting, the frog couldn't see much except the back of Crocodile's head. "Why don't you hop up on my head so you can see everything around us?" Crocodile invited. `) + +func BenchmarkCompressLevelMinusTwo(b *testing.B) { + for i := 0; i < b.N; i++ { + CompressWithOption(fable, -2) + } +} + +func BenchmarkCompressLevelNine(b *testing.B) { + for i := 0; i < b.N; i++ { + CompressWithOption(fable, 9) + } +} + +func BenchmarkCompressLevelMinusTwoBinary(b *testing.B) { + data := make([]byte, 1000000) + rand.Read(data) + for i := 0; i < b.N; i++ { + CompressWithOption(data, -2) + } +} + +func BenchmarkCompressLevelNineBinary(b *testing.B) { + data := make([]byte, 1000000) + rand.Read(data) + for i := 0; i < b.N; i++ { + CompressWithOption(data, 9) + } +} + +func TestCompress(t *testing.T) { + compressedB := CompressWithOption(fable, 9) + dataRateSavings := 100 * (1.0 - float64(len(compressedB))/float64(len(fable))) + fmt.Printf("Level 9: %2.0f%% percent space savings\n", dataRateSavings) + assert.True(t, len(compressedB) < len(fable)) + + compressedB = CompressWithOption(fable, -2) + dataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(fable))) + fmt.Printf("Level -2: %2.0f%% percent space savings\n", dataRateSavings) + assert.True(t, len(compressedB) < len(fable)) +}