0
0
Fork 0
mirror of https://github.com/schollz/croc.git synced 2025-10-11 13:21:00 +02:00

Merge branch 'master' of https://github.com/schollz/croc into install_script_rewrite

This commit is contained in:
Micheal Quinn 2019-09-10 07:29:24 -05:00
commit c0f7c3c250
No known key found for this signature in database
GPG key ID: 0E7217F3C30BA059
17 changed files with 363 additions and 67 deletions

View file

@ -21,8 +21,10 @@ import (
"github.com/urfave/cli"
)
// Version specifies the version
var Version string
// Run will run the command line proram
func Run() (err error) {
// use all of the processors
runtime.GOMAXPROCS(runtime.NumCPU())
@ -30,7 +32,7 @@ func Run() (err error) {
app := cli.NewApp()
app.Name = "croc"
if Version == "" {
Version = "v6.1.1-82f2c8c"
Version = "v6.1.3-bc6803e"
}
app.Version = Version
app.Compiled = time.Now()
@ -101,7 +103,7 @@ func getConfigDir() (homedir string, err error) {
return
}
homedir = path.Join(homedir, ".config", "croc")
if _, err := os.Stat(homedir); os.IsNotExist(err) {
if _, err = os.Stat(homedir); os.IsNotExist(err) {
log.Debugf("creating home directory %s", homedir)
err = os.MkdirAll(homedir, 0700)
}

View file

@ -47,7 +47,7 @@ func TestComm(t *testing.T) {
}()
time.Sleep(300 * time.Millisecond)
a, err := NewConnection("localhost:" + port)
a, err := NewConnection("localhost:"+port, 10*time.Minute)
assert.Nil(t, err)
data, err := a.Receive()
assert.Equal(t, []byte("hello, world"), data)
@ -56,5 +56,10 @@ func TestComm(t *testing.T) {
assert.Nil(t, a.Send([]byte{'\x00'}))
assert.Nil(t, a.Send(token))
_ = a.Connection()
a.Close()
assert.NotNil(t, a.Send(token))
_, err = a.Write(token)
assert.NotNil(t, err)
}

View file

@ -70,12 +70,18 @@ func TestCompress(t *testing.T) {
dataRateSavings := 100 * (1.0 - float64(len(compressedB))/float64(len(fable)))
fmt.Printf("Level 9: %2.0f%% percent space savings\n", dataRateSavings)
assert.True(t, len(compressedB) < len(fable))
assert.Equal(t, fable, Decompress(compressedB))
compressedB = CompressWithOption(fable, -2)
dataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(fable)))
fmt.Printf("Level -2: %2.0f%% percent space savings\n", dataRateSavings)
assert.True(t, len(compressedB) < len(fable))
compressedB = Compress(fable)
dataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(fable)))
fmt.Printf("Level -2: %2.0f%% percent space savings\n", dataRateSavings)
assert.True(t, len(compressedB) < len(fable))
data := make([]byte, 4096)
rand.Read(data)
compressedB = CompressWithOption(data, -2)
@ -85,6 +91,7 @@ func TestCompress(t *testing.T) {
rand.Read(data)
compressedB = CompressWithOption(data, 9)
dataRateSavings = 100 * (1.0 - float64(len(compressedB))/float64(len(data)))
fmt.Printf("random, Level 9: %2.0f%% percent space savings\n", dataRateSavings)
}

View file

@ -36,6 +36,7 @@ func init() {
log.SetLevel("debug")
}
// Debug toggles debug mode
func Debug(debug bool) {
if debug {
log.SetLevel("debug")
@ -44,6 +45,7 @@ func Debug(debug bool) {
}
}
// Options specifies user specific options
type Options struct {
IsSender bool
SharedSecret string
@ -55,6 +57,7 @@ type Options struct {
DisableLocal bool
}
// Client holds the state of the croc transfer
type Client struct {
Options Options
Pake *pake.Pake
@ -72,6 +75,7 @@ type Client struct {
// send / receive information of all files
FilesToTransfer []FileInfo
FilesToTransferCurrentNum int
FilesHasFinished map[int]struct{}
// send / receive information of current file
CurrentFile *os.File
@ -89,15 +93,20 @@ type Client struct {
spinner *spinner.Spinner
firstSend bool
mutex *sync.Mutex
quit chan bool
mutex *sync.Mutex
fread *os.File
numfinished int
quit chan bool
}
// Chunk contains information about the
// needed bytes
type Chunk struct {
Bytes []byte `json:"b,omitempty"`
Location int64 `json:"l,omitempty"`
}
// FileInfo registers the information about the file
type FileInfo struct {
Name string `json:"n,omitempty"`
FolderRemote string `json:"fr,omitempty"`
@ -109,18 +118,21 @@ type FileInfo struct {
IsEncrypted bool `json:"e,omitempty"`
}
// RemoteFileRequest requests specific bytes
type RemoteFileRequest struct {
CurrentFileChunkRanges []int64
FilesToTransferCurrentNum int
}
// SenderInfo lists the files to be transferred
type SenderInfo struct {
FilesToTransfer []FileInfo
}
// New establishes a new connection for transfering files between two instances.
// New establishes a new connection for transferring files between two instances.
func New(ops Options) (c *Client, err error) {
c = new(Client)
c.FilesHasFinished = make(map[int]struct{})
// setup basic info
c.Options = ops
@ -288,7 +300,7 @@ func (c *Client) Send(options TransferOptions) (err error) {
go func() {
log.Debugf("establishing connection to %s", c.Options.RelayAddress)
var banner string
conn, banner, ipaddr, err := tcp.ConnectToTCPServer(c.Options.RelayAddress, c.Options.SharedSecret)
conn, banner, ipaddr, err := tcp.ConnectToTCPServer(c.Options.RelayAddress, c.Options.SharedSecret, 5*time.Second)
log.Debugf("banner: %s", banner)
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("could not connect to %s", c.Options.RelayAddress))
@ -449,7 +461,7 @@ func (c *Client) transfer(options TransferOptions) (err error) {
break
}
}
// purge errors that come from succesful transfer
// purge errors that come from successful transfer
if c.SuccessfulTransfer {
if err != nil {
log.Debugf("purging error: %s", err)
@ -667,6 +679,9 @@ func (c *Client) updateState() (err error) {
finished := true
for i, fileInfo := range c.FilesToTransfer {
if _, ok := c.FilesHasFinished[i]; ok {
continue
}
log.Debugf("checking %+v", fileInfo)
if i < c.FilesToTransferCurrentNum {
continue
@ -733,6 +748,7 @@ func (c *Client) updateState() (err error) {
panic(err)
}
c.SuccessfulTransfer = true
c.FilesHasFinished[c.FilesToTransferCurrentNum] = struct{}{}
}
// start initiating the process to receive a new file
@ -749,7 +765,7 @@ func (c *Client) updateState() (err error) {
c.CurrentFile, errOpen = os.OpenFile(
pathToFile,
os.O_WRONLY, 0666)
truncate := false
var truncate bool // default false
c.CurrentFileChunks = []int64{}
c.CurrentFileChunkRanges = []int64{}
if errOpen == nil {
@ -810,7 +826,7 @@ func (c *Client) updateState() (err error) {
if !c.firstSend {
fmt.Fprintf(os.Stderr, "\nSending (->%s)\n", c.ExternalIPConnected)
c.firstSend = true
// if there are empty files, show them as already have been transfered now
// if there are empty files, show them as already have been transferred now
for i := range c.FilesToTransfer {
if c.FilesToTransfer[i].Size == 0 {
// setup the progressbar and takedown the progress bar for empty files
@ -837,6 +853,16 @@ func (c *Client) updateState() (err error) {
c.setBar()
c.TotalSent = 0
log.Debug("beginning sending comms")
pathToFile := path.Join(
c.FilesToTransfer[c.FilesToTransferCurrentNum].FolderSource,
c.FilesToTransfer[c.FilesToTransferCurrentNum].Name,
)
c.fread, err = os.Open(pathToFile)
c.numfinished = 0
if err != nil {
return
}
for i := 0; i < len(c.Options.RelayPorts); i++ {
log.Debugf("starting sending over comm %d", i)
go c.sendData(i)
@ -875,6 +901,7 @@ func (c *Client) setBar() {
}
func (c *Client) receiveData(i int) {
log.Debugf("%d receiving data", i)
for {
data, err := c.conn[i+1].Receive()
if err != nil {
@ -932,30 +959,23 @@ func (c *Client) receiveData(i int) {
func (c *Client) sendData(i int) {
defer func() {
log.Debugf("finished with %d", i)
c.numfinished++
if c.numfinished == len(c.Options.RelayPorts) {
log.Debug("closing file")
c.fread.Close()
}
}()
pathToFile := path.Join(
c.FilesToTransfer[c.FilesToTransferCurrentNum].FolderSource,
c.FilesToTransfer[c.FilesToTransferCurrentNum].Name,
)
log.Debugf("opening %s to read", pathToFile)
f, err := os.Open(pathToFile)
if err != nil {
panic(err)
}
defer f.Close()
var readingPos int64
pos := uint64(0)
curi := float64(0)
for {
// Read file
data := make([]byte, models.TCP_BUFFER_SIZE/2)
n, err := f.Read(data)
if err != nil {
if err == io.EOF {
break
}
panic(err)
}
// log.Debugf("%d trying to read", i)
n, errRead := c.fread.ReadAt(data, readingPos)
// log.Debugf("%d read %d bytes", i, n)
readingPos += int64(n)
if math.Mod(curi, float64(len(c.Options.RelayPorts))) == float64(i) {
// check to see if this is a chunk that the recipient wants
@ -997,8 +1017,13 @@ func (c *Client) sendData(i int) {
curi++
pos += uint64(n)
}
time.Sleep(10 * time.Second)
if errRead != nil {
if errRead == io.EOF {
break
}
panic(errRead)
}
}
return
}

View file

@ -1,6 +1,7 @@
package croc
import (
"io/ioutil"
"os"
"sync"
"testing"
@ -8,9 +9,11 @@ import (
"github.com/schollz/croc/v6/src/tcp"
log "github.com/schollz/logger"
"github.com/stretchr/testify/assert"
)
func TestCroc(t *testing.T) {
log.SetLevel("trace")
defer os.Remove("README.md")
go tcp.Run("debug", "8081", "8082,8083,8084,8085")
go tcp.Run("debug", "8082")
@ -64,3 +67,95 @@ func TestCroc(t *testing.T) {
wg.Wait()
}
func TestCrocLocal(t *testing.T) {
log.SetLevel("trace")
defer os.Remove("LICENSE")
defer os.Remove("touched")
time.Sleep(300 * time.Millisecond)
log.Debug("setting up sender")
sender, err := New(Options{
IsSender: true,
SharedSecret: "test",
Debug: true,
RelayAddress: "localhost:8181",
RelayPorts: []string{"8181", "8182"},
Stdout: true,
NoPrompt: true,
DisableLocal: false,
})
if err != nil {
panic(err)
}
time.Sleep(1 * time.Second)
log.Debug("setting up receiver")
receiver, err := New(Options{
IsSender: false,
SharedSecret: "test",
Debug: true,
RelayAddress: "localhost:8181",
Stdout: true,
NoPrompt: true,
DisableLocal: false,
})
if err != nil {
panic(err)
}
var wg sync.WaitGroup
os.Create("touched")
wg.Add(2)
go func() {
sender.Send(TransferOptions{
PathToFiles: []string{"../../LICENSE", "touched"},
KeepPathInRemote: false,
})
wg.Done()
}()
time.Sleep(100 * time.Millisecond)
go func() {
receiver.Receive()
wg.Done()
}()
wg.Wait()
}
func TestCrocError(t *testing.T) {
content := []byte("temporary file's content")
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
panic(err)
}
defer os.Remove(tmpfile.Name()) // clean up
if _, err := tmpfile.Write(content); err != nil {
panic(err)
}
if err := tmpfile.Close(); err != nil {
panic(err)
}
Debug(false)
log.SetLevel("warn")
sender, _ := New(Options{
IsSender: true,
SharedSecret: "test33",
Debug: true,
RelayAddress: "doesntexistok.com:8381",
RelayPorts: []string{"8381", "8382"},
Stdout: true,
NoPrompt: true,
DisableLocal: true,
})
err = sender.Send(TransferOptions{
PathToFiles: []string{tmpfile.Name()},
KeepPathInRemote: true,
})
log.Debug(err)
assert.NotNil(t, err)
}

View file

@ -9,6 +9,8 @@ import (
"golang.org/x/crypto/pbkdf2"
)
// Encryption is the basic type for storing
// the key, passphrase and salt
type Encryption struct {
key []byte
passphrase []byte
@ -36,6 +38,7 @@ func New(passphrase []byte, salt []byte) (e Encryption, err error) {
return
}
// Salt returns the salt bytes
func (e Encryption) Salt() []byte {
return e.salt
}

View file

@ -45,11 +45,10 @@ func TestEncryption(t *testing.T) {
}
func TestNoEncryption(t *testing.T) {
bob, err := New(nil, nil)
assert.Nil(t, err)
jane, err := New(nil,nil)
jane, err := New(nil, nil)
assert.Nil(t, err)
enc, err := bob.Encrypt([]byte("hello, world"))
assert.Nil(t, err)
@ -57,5 +56,5 @@ func TestNoEncryption(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, dec, []byte("hello, world"))
assert.Equal(t, enc, []byte("hello, world"))
}
}

View file

@ -3,10 +3,10 @@ package message
import (
"encoding/json"
log "github.com/schollz/logger"
"github.com/schollz/croc/v6/src/comm"
"github.com/schollz/croc/v6/src/compress"
"github.com/schollz/croc/v6/src/crypt"
log "github.com/schollz/logger"
)
// Message is the possible payload for messaging

View file

@ -1,10 +1,15 @@
package message
import (
"crypto/rand"
"fmt"
"net"
"testing"
"time"
"github.com/schollz/croc/v6/src/comm"
"github.com/schollz/croc/v6/src/crypt"
log "github.com/schollz/logger"
"github.com/stretchr/testify/assert"
)
@ -20,4 +25,51 @@ func TestMessage(t *testing.T) {
m2, err := Decode(e, b)
assert.Nil(t, err)
assert.Equal(t, m, m2)
assert.Equal(t, `{"t":"message","m":"hello, world"}`, m.String())
}
func TestSend(t *testing.T) {
token := make([]byte, 40000000)
rand.Read(token)
port := "8801"
go func() {
log.Debugf("starting TCP server on " + port)
server, err := net.Listen("tcp", "0.0.0.0:"+port)
if err != nil {
log.Error(err)
}
defer server.Close()
// spawn a new goroutine whenever a client connects
for {
connection, err := server.Accept()
if err != nil {
log.Error(err)
}
log.Debugf("client %s connected", connection.RemoteAddr().String())
go func(port string, connection net.Conn) {
c := comm.New(connection)
err = c.Send([]byte("hello, world"))
assert.Nil(t, err)
data, err := c.Receive()
assert.Nil(t, err)
assert.Equal(t, []byte("hello, computer"), data)
data, err = c.Receive()
assert.Nil(t, err)
assert.Equal(t, []byte{'\x00'}, data)
data, err = c.Receive()
assert.Nil(t, err)
assert.Equal(t, token, data)
}(port, connection)
}
}()
time.Sleep(300 * time.Millisecond)
a, err := comm.NewConnection("localhost:"+port, 10*time.Minute)
assert.Nil(t, err)
m := Message{Type: "message", Message: "hello, world"}
e, err := crypt.New(nil, nil)
assert.Nil(t, err)
assert.Nil(t, Send(a, e, m))
}

View file

@ -1,4 +1,7 @@
package models
// TCP_BUFFER_SIZE is the maximum packet size
const TCP_BUFFER_SIZE = 1024 * 64
// DEFAULT_RELAY is the default relay used (can be set using --relay)
const DEFAULT_RELAY = "142.93.177.120:9009"

View file

@ -33,6 +33,8 @@ type roomMap struct {
sync.Mutex
}
var timeToRoomDeletion = 10 * time.Minute
// Run starts a tcp listener, run async
func Run(debugLevel, port string, banner ...string) (err error) {
s := new(server)
@ -53,7 +55,7 @@ func (s *server) start() (err error) {
// delete old rooms
go func() {
for {
time.Sleep(10 * time.Minute)
time.Sleep(timeToRoomDeletion)
roomsToDelete := []string{}
s.rooms.Lock()
for room := range s.rooms.rooms {
@ -251,6 +253,8 @@ func pipe(conn1 net.Conn, conn2 net.Conn) {
}
}
// ConnectToTCPServer will initiate a new connection
// to the specified address, room with optional time limit
func ConnectToTCPServer(address, room string, timelimit ...time.Duration) (c *comm.Comm, banner string, ipaddr string, err error) {
if len(timelimit) > 0 {
c, err = comm.NewConnection(address, timelimit[0])

View file

@ -8,14 +8,17 @@ import (
)
func TestTCP(t *testing.T) {
go Run("debug", "8081", "8082")
timeToRoomDeletion = 100 * time.Millisecond
go Run("debug", "8281", "8282")
time.Sleep(100 * time.Millisecond)
c1, banner, _, err := ConnectToTCPServer("localhost:8081", "testRoom")
assert.Equal(t, banner, "8082")
c1, banner, _, err := ConnectToTCPServer("localhost:8281", "testRoom", 1*time.Minute)
assert.Equal(t, banner, "8282")
assert.Nil(t, err)
c2, _, _, err := ConnectToTCPServer("localhost:8081", "testRoom")
c2, _, _, err := ConnectToTCPServer("localhost:8281", "testRoom")
assert.Nil(t, err)
_, _, _, err = ConnectToTCPServer("localhost:8081", "testRoom")
_, _, _, err = ConnectToTCPServer("localhost:8281", "testRoom")
assert.NotNil(t, err)
_, _, _, err = ConnectToTCPServer("localhost:8281", "testRoom", 1*time.Nanosecond)
assert.NotNil(t, err)
// try sending data

View file

@ -44,6 +44,7 @@ func HashFile(fname string) (hash256 []byte, err error) {
return IMOHashFile(fname)
}
// MD5HashFile returns MD5 hash
func MD5HashFile(fname string) (hash256 []byte, err error) {
f, err := os.Open(fname)
if err != nil {
@ -91,6 +92,7 @@ func SHA256(s string) string {
return fmt.Sprintf("%x", sha.Sum(nil))
}
// PublicIP returns public ip address
func PublicIP() (ip string, err error) {
resp, err := http.Get("https://canhazip.com")
if err != nil {
@ -108,7 +110,7 @@ func PublicIP() (ip string, err error) {
return
}
// Get preferred outbound ip of this machine
// LocalIP returns local ip address
func LocalIP() string {
conn, err := net.Dial("udp", "8.8.8.8:80")
if err != nil {
@ -121,6 +123,7 @@ func LocalIP() string {
return localAddr.IP.String()
}
// GetRandomName returns mnemoicoded random name
func GetRandomName() string {
result := []string{}
bs := make([]byte, 4)
@ -129,6 +132,7 @@ func GetRandomName() string {
return strings.Join(result, "-")
}
// ByteCountDecimal converts bytes to human readable byte string
func ByteCountDecimal(b int64) string {
const unit = 1000
if b < unit {
@ -146,17 +150,17 @@ func ByteCountDecimal(b int64) string {
// If file doesn't exist, it returns an empty chunk list (all chunks).
// If the file size is not the same as requested, it returns an empty chunk list (all chunks).
func MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) {
fstat, err := os.Stat(fname)
if fstat.Size() != fsize || err != nil {
return
}
f, err := os.Open(fname)
if err != nil {
return
}
defer f.Close()
fstat, err := os.Stat(fname)
if fstat.Size() != fsize || err != nil {
return
}
emptyBuffer := make([]byte, chunkSize)
chunkNum := 0
chunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))
@ -196,6 +200,7 @@ func MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int6
return
}
// ChunkRangesToChunks converts chunk ranges to list
func ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) {
if len(chunkRanges) == 0 {
return
@ -210,6 +215,7 @@ func ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) {
return
}
// GetLocalIPs returns all local ips
func GetLocalIPs() (ips []string, err error) {
addrs, err := net.InterfaceAddrs()
if err != nil {

View file

@ -3,8 +3,10 @@ package utils
import (
"fmt"
"io/ioutil"
"log"
"math/rand"
"os"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@ -54,6 +56,8 @@ func TestMD5HashFile(t *testing.T) {
b, err := MD5HashFile("bigfile.test")
assert.Nil(t, err)
assert.Equal(t, "9fed05acbacbc6a36555c998501c21f6", fmt.Sprintf("%x", b))
_, err = MD5HashFile("bigfile.test.nofile")
assert.NotNil(t, err)
}
func TestIMOHashFile(t *testing.T) {
@ -70,6 +74,8 @@ func TestXXHashFile(t *testing.T) {
b, err := XXHashFile("bigfile.test")
assert.Nil(t, err)
assert.Equal(t, "f2da6ee7e75e8324", fmt.Sprintf("%x", b))
_, err = XXHashFile("nofile")
assert.NotNil(t, err)
}
func TestSHA256(t *testing.T) {
@ -78,6 +84,8 @@ func TestSHA256(t *testing.T) {
func TestByteCountDecimal(t *testing.T) {
assert.Equal(t, "10.0 kB", ByteCountDecimal(10000))
assert.Equal(t, "50 B", ByteCountDecimal(50))
assert.Equal(t, "12.4 MB", ByteCountDecimal(12378517))
}
func TestMissingChunks(t *testing.T) {
@ -104,6 +112,29 @@ func TestMissingChunks(t *testing.T) {
assert.Equal(t, []int64{0, 40, 50, 70, 80, 90}, chunks)
os.Remove("missing.test")
content := []byte("temporary file's content")
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
log.Fatal(err)
}
defer os.Remove(tmpfile.Name()) // clean up
if _, err := tmpfile.Write(content); err != nil {
panic(err)
}
if err := tmpfile.Close(); err != nil {
panic(err)
}
chunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)), chunkSize)
assert.Empty(t, chunkRanges)
chunkRanges = MissingChunks(tmpfile.Name(), int64(len(content)+10), chunkSize)
assert.Empty(t, chunkRanges)
chunkRanges = MissingChunks(tmpfile.Name()+"ok", int64(len(content)), chunkSize)
assert.Empty(t, chunkRanges)
chunks = ChunkRangesToChunks(chunkRanges)
assert.Empty(t, chunks)
}
// func Test1(t *testing.T) {
@ -112,3 +143,41 @@ func TestMissingChunks(t *testing.T) {
// fmt.Println(ChunkRangesToChunks((chunkRanges)))
// assert.Nil(t, nil)
// }
func TestHashFile(t *testing.T) {
content := []byte("temporary file's content")
tmpfile, err := ioutil.TempFile("", "example")
if err != nil {
log.Fatal(err)
}
defer os.Remove(tmpfile.Name()) // clean up
if _, err := tmpfile.Write(content); err != nil {
panic(err)
}
if err := tmpfile.Close(); err != nil {
panic(err)
}
hashed, err := HashFile(tmpfile.Name())
assert.Nil(t, err)
assert.Equal(t, "18c9673a4bb8325d323e7f24fda9ae1e", fmt.Sprintf("%x", hashed))
}
func TestPublicIP(t *testing.T) {
ip, err := PublicIP()
fmt.Println(ip)
assert.True(t, strings.Contains(ip, "."))
assert.Nil(t, err)
}
func TestLocalIP(t *testing.T) {
ip := LocalIP()
fmt.Println(ip)
assert.True(t, strings.Contains(ip, "."))
}
func TestGetRandomName(t *testing.T) {
name := GetRandomName()
assert.NotEmpty(t, name)
}