From ce8fb796b3691a2c9faa03d3dcde1e5a3c98454a Mon Sep 17 00:00:00 2001 From: RCL98 <48773399+RCL98@users.noreply.github.com> Date: Mon, 21 Feb 2022 20:16:27 +0200 Subject: [PATCH 1/3] #431 added file informations retrival functions to replace getPaths --- src/croc/croc.go | 124 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 100 insertions(+), 24 deletions(-) diff --git a/src/croc/croc.go b/src/croc/croc.go index 5dc565b0..1afa5c7b 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -16,6 +16,7 @@ import ( "strings" "sync" "time" + "golang.org/x/time/rate" "github.com/denisbrodbeck/machineid" @@ -85,7 +86,7 @@ type Client struct { // steps involved in forming relationship Step1ChannelSecured bool - Step2FileInfoTransferred bool + Step2FileInfoTransferred bool Step3RecipientRequestFile bool Step4FileTransfer bool Step5CloseChannels bool @@ -103,10 +104,10 @@ type Client struct { CurrentFileIsClosed bool LastFolder string - TotalSent int64 + TotalSent int64 TotalChunksTransferred int - chunkMap map[uint64]struct{} - limiter *rate.Limiter + chunkMap map[uint64]struct{} + limiter *rate.Limiter // tcp connections conn []*comm.Comm @@ -115,11 +116,11 @@ type Client struct { longestFilename int firstSend bool - mutex *sync.Mutex - fread *os.File - numfinished int - quit chan bool - finishedNum int + mutex *sync.Mutex + fread *os.File + numfinished int + quit chan bool + finishedNum int numberOfTransferredFiles int } @@ -132,15 +133,16 @@ type Chunk struct { // FileInfo registers the information about the file type FileInfo struct { - Name string `json:"n,omitempty"` - FolderRemote string `json:"fr,omitempty"` - FolderSource string `json:"fs,omitempty"` - Hash []byte `json:"h,omitempty"` - Size int64 `json:"s,omitempty"` - ModTime time.Time `json:"m,omitempty"` - IsCompressed bool `json:"c,omitempty"` - IsEncrypted bool `json:"e,omitempty"` - Symlink string `json:"sy,omitempty"` + Name string `json:"n,omitempty"` + FolderRemote string `json:"fr,omitempty"` + FolderSource string `json:"fs,omitempty"` + Hash []byte `json:"h,omitempty"` + Size int64 `json:"s,omitempty"` + ModTime time.Time `json:"m,omitempty"` + IsCompressed bool `json:"c,omitempty"` + IsEncrypted bool `json:"e,omitempty"` + Symlink string `json:"sy,omitempty"` + Mode os.FileMode `json:"md,omitempty"` } // RemoteFileRequest requests specific bytes @@ -188,11 +190,11 @@ func New(ops Options) (c *Client, err error) { var rt rate.Limit switch unit := string(c.Options.ThrottleUpload[len(c.Options.ThrottleUpload)-1:]); unit { case "g", "G": - uploadLimit = uploadLimit*1024*1024*1024 + uploadLimit = uploadLimit * 1024 * 1024 * 1024 case "m", "M": - uploadLimit = uploadLimit*1024*1024 + uploadLimit = uploadLimit * 1024 * 1024 case "k", "K": - uploadLimit = uploadLimit*1024 + uploadLimit = uploadLimit * 1024 default: uploadLimit, err = strconv.ParseInt(c.Options.ThrottleUpload, 10, 64) if err != nil { @@ -200,8 +202,8 @@ func New(ops Options) (c *Client, err error) { } } // Somehow 4* is neccessary - rt = rate.Every(time.Second / (4*time.Duration(uploadLimit))) - if (int(uploadLimit) > minBurstSize) { + rt = rate.Every(time.Second / (4 * time.Duration(uploadLimit))) + if int(uploadLimit) > minBurstSize { minBurstSize = int(uploadLimit) } c.limiter = rate.NewLimiter(rt, minBurstSize) @@ -226,6 +228,80 @@ type TransferOptions struct { KeepPathInRemote bool } +// This function retrives the important file informations +// for every file that will be transfered +func GetFilesInfo(fnames []string) (filesInfo []FileInfo, err error) { + // fnames: the relativ/absolute paths of files/folders that will be transfered + + var paths []string + for _, fname := range fnames { + // Support wildcard + if strings.Contains(fname, "*") { + matches, errGlob := filepath.Glob(fname) + if errGlob != nil { + err = errGlob + return + } + paths = append(paths, matches...) + continue + } else { + paths = append(paths, fname) + } + } + + for _, path := range paths { + stat, errStat := os.Lstat(path) + + if errStat != nil { + err = errStat + return + } + + absPath, errAbs := filepath.Abs(path) + absPath = filepath.ToSlash(absPath) + + if errAbs != nil { + err = errAbs + return + } + + if stat.IsDir() { + err = filepath.Walk(absPath, + func(pathName string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + filesInfo = append(filesInfo, FileInfo{ + Name: info.Name(), + FolderRemote: strings.TrimPrefix(filepath.Dir(filepath.ToSlash(pathName)), + filepath.Dir(absPath)+"/"), + FolderSource: filepath.Dir(pathName), + Size: info.Size(), + ModTime: info.ModTime(), + Mode: info.Mode(), + }) + } + return nil + }) + if err != nil { + return + } + } else { + filesInfo = append(filesInfo, FileInfo{ + Name: stat.Name(), + FolderRemote: "./", + FolderSource: filepath.Dir(absPath), + Size: stat.Size(), + ModTime: stat.ModTime(), + Mode: stat.Mode(), + }) + } + + } + return +} + func (c *Client) sendCollectFiles(options TransferOptions) (err error) { c.FilesToTransfer = make([]FileInfo, len(options.PathToFiles)) totalFilesSize := int64(0) @@ -1559,7 +1635,7 @@ func (c *Client) sendData(i int) { n, errRead := c.fread.ReadAt(data, readingPos) // log.Debugf("%d read %d bytes", i, n) readingPos += int64(n) - if (c.limiter != nil) { + if c.limiter != nil { r := c.limiter.ReserveN(time.Now(), n) log.Debugf("Limiting Upload for %d", r.Delay()) time.Sleep(r.Delay()) From 32a188fa85f2fe49bad391a39476c4300e67c443 Mon Sep 17 00:00:00 2001 From: iulius98 Date: Mon, 21 Feb 2022 20:41:19 +0200 Subject: [PATCH 2/3] #431 Integrate GetFilesInfo function with the code --- src/cli/cli.go | 48 ++----------------------- src/croc/croc.go | 84 +++++++++++++------------------------------ src/croc/croc_test.go | 26 ++++++++------ 3 files changed, 42 insertions(+), 116 deletions(-) diff --git a/src/cli/cli.go b/src/cli/cli.go index 1923b0ff..65fef581 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -267,7 +267,7 @@ func send(c *cli.Context) (err error) { crocOptions.SharedSecret = utils.GetRandomName() } - paths, haveFolder, err := getPaths(fnames) + minimalFileInfos, err := croc.GetFilesInfo(fnames) if err != nil { return } @@ -280,10 +280,7 @@ func send(c *cli.Context) (err error) { // save the config saveConfig(c, crocOptions) - err = cr.Send(croc.TransferOptions{ - PathToFiles: paths, - KeepPathInRemote: haveFolder, - }) + err = cr.Send(minimalFileInfos) return } @@ -324,47 +321,6 @@ func makeTempFileWithString(s string) (fnames []string, err error) { return } -func getPaths(fnames []string) (paths []string, haveFolder bool, err error) { - haveFolder = false - paths = []string{} - for _, fname := range fnames { - // Support wildcard - if strings.Contains(fname, "*") { - matches, errGlob := filepath.Glob(fname) - if errGlob != nil { - err = errGlob - return - } - paths = append(paths, matches...) - continue - } - - stat, errStat := os.Lstat(fname) - if errStat != nil { - err = errStat - return - } - if stat.IsDir() { - haveFolder = true - err = filepath.Walk(fname, - func(pathName string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - paths = append(paths, filepath.ToSlash(pathName)) - } - return nil - }) - if err != nil { - return - } - } else { - paths = append(paths, filepath.ToSlash(fname)) - } - } - return -} func saveConfig(c *cli.Context, crocOptions croc.Options) { if c.Bool("remember") { diff --git a/src/croc/croc.go b/src/croc/croc.go index 1afa5c7b..2cdedb5e 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -302,72 +302,38 @@ func GetFilesInfo(fnames []string) (filesInfo []FileInfo, err error) { return } -func (c *Client) sendCollectFiles(options TransferOptions) (err error) { - c.FilesToTransfer = make([]FileInfo, len(options.PathToFiles)) +func (c *Client) sendCollectFiles(filesInfo []FileInfo) (err error) { + c.FilesToTransfer = filesInfo totalFilesSize := int64(0) - for i, pathToFile := range options.PathToFiles { - var fstats os.FileInfo - var fullPath string - fullPath, err = filepath.Abs(pathToFile) - if err != nil { - return - } - fullPath = filepath.Clean(fullPath) - var folderName string - folderName, _ = filepath.Split(fullPath) - fstats, err = os.Lstat(fullPath) - if err != nil { - return + for i, fileInfo := range c.FilesToTransfer { + var fullPath string + fullPath = fileInfo.FolderSource + string(os.PathSeparator) + fileInfo.Name + fullPath = filepath.Clean(fullPath) + + if len(fileInfo.Name) > c.longestFilename { + c.longestFilename = len(fileInfo.Name) } - if len(fstats.Name()) > c.longestFilename { - c.longestFilename = len(fstats.Name()) - } - c.FilesToTransfer[i] = FileInfo{ - Name: fstats.Name(), - FolderRemote: ".", - FolderSource: folderName, - Size: fstats.Size(), - ModTime: fstats.ModTime(), - } - if fstats.Mode()&os.ModeSymlink != 0 { - log.Debugf("%s is symlink", fstats.Name()) - c.FilesToTransfer[i].Symlink, err = os.Readlink(pathToFile) + + if fileInfo.Mode&os.ModeSymlink != 0 { + log.Debugf("%s is symlink", fileInfo.Name) + c.FilesToTransfer[i].Symlink, err = os.Readlink(fullPath) if err != nil { log.Debugf("error getting symlink: %s", err.Error()) } log.Debugf("%+v", c.FilesToTransfer[i]) } + if c.Options.HashAlgorithm == "" { c.Options.HashAlgorithm = "xxhash" } + c.FilesToTransfer[i].Hash, err = utils.HashFile(fullPath, c.Options.HashAlgorithm) log.Debugf("hashed %s to %x using %s", fullPath, c.FilesToTransfer[i].Hash, c.Options.HashAlgorithm) - totalFilesSize += fstats.Size() + totalFilesSize += fileInfo.Size if err != nil { return } - if options.KeepPathInRemote { - var curFolder string - curFolder, err = os.Getwd() - if err != nil { - return - } - curFolder, err = filepath.Abs(curFolder) - if err != nil { - return - } - if !strings.HasPrefix(folderName, curFolder) { - err = fmt.Errorf("remote directory must be relative to current") - return - } - c.FilesToTransfer[i].FolderRemote = strings.TrimPrefix(folderName, curFolder) - c.FilesToTransfer[i].FolderRemote = filepath.ToSlash(c.FilesToTransfer[i].FolderRemote) - c.FilesToTransfer[i].FolderRemote = strings.TrimPrefix(c.FilesToTransfer[i].FolderRemote, "/") - if c.FilesToTransfer[i].FolderRemote == "" { - c.FilesToTransfer[i].FolderRemote = "." - } - } log.Debugf("file %d info: %+v", i, c.FilesToTransfer[i]) fmt.Fprintf(os.Stderr, "\r ") fmt.Fprintf(os.Stderr, "\rSending %d files (%s)", i, utils.ByteCountDecimal(totalFilesSize)) @@ -440,7 +406,7 @@ func (c *Client) broadcastOnLocalNetwork(useipv6 bool) { } } -func (c *Client) transferOverLocalRelay(options TransferOptions, errchan chan<- error) { +func (c *Client) transferOverLocalRelay(errchan chan<- error) { time.Sleep(500 * time.Millisecond) log.Debug("establishing connection") var banner string @@ -472,12 +438,12 @@ func (c *Client) transferOverLocalRelay(options TransferOptions, errchan chan<- c.Options.RelayPorts = []string{c.Options.RelayPorts[0]} } c.ExternalIP = ipaddr - errchan <- c.transfer(options) + errchan <- c.transfer() } // Send will send the specified file -func (c *Client) Send(options TransferOptions) (err error) { - err = c.sendCollectFiles(options) +func (c *Client) Send(filesInfo []FileInfo) (err error) { + err = c.sendCollectFiles(filesInfo) if err != nil { return } @@ -507,7 +473,7 @@ func (c *Client) Send(options TransferOptions) (err error) { go c.broadcastOnLocalNetwork(false) // broadcast on ipv6 go c.broadcastOnLocalNetwork(true) - go c.transferOverLocalRelay(options, errchan) + go c.transferOverLocalRelay(errchan) } if !c.Options.OnlyLocal { @@ -591,7 +557,7 @@ func (c *Client) Send(options TransferOptions) (err error) { } c.ExternalIP = ipaddr log.Debug("exchanged header message") - errchan <- c.transfer(options) + errchan <- c.transfer() }() } @@ -809,7 +775,7 @@ func (c *Client) Receive() (err error) { } log.Debug("exchanged header message") fmt.Fprintf(os.Stderr, "\rsecuring channel...") - err = c.transfer(TransferOptions{}) + err = c.transfer() if err == nil { if c.numberOfTransferredFiles == 0 { fmt.Fprintf(os.Stderr, "\rNo files transferred.") @@ -818,7 +784,7 @@ func (c *Client) Receive() (err error) { return } -func (c *Client) transfer(options TransferOptions) (err error) { +func (c *Client) transfer() (err error) { // connect to the server // quit with c.quit <- true @@ -1635,7 +1601,7 @@ func (c *Client) sendData(i int) { n, errRead := c.fread.ReadAt(data, readingPos) // log.Debugf("%d read %d bytes", i, n) readingPos += int64(n) - if c.limiter != nil { + if (c.limiter != nil) { r := c.limiter.ReserveN(time.Now(), n) log.Debugf("Limiting Upload for %d", r.Delay()) time.Sleep(r.Delay()) diff --git a/src/croc/croc_test.go b/src/croc/croc_test.go index 140ab63a..5c61ef2b 100644 --- a/src/croc/croc_test.go +++ b/src/croc/croc_test.go @@ -63,9 +63,11 @@ func TestCrocReadme(t *testing.T) { var wg sync.WaitGroup wg.Add(2) go func() { - err := sender.Send(TransferOptions{ - PathToFiles: []string{"../../README.md"}, - }) + filesInfo, errGet := GetFilesInfo([]string{"../../README.md"}) + if errGet != nil { + t.Errorf("failed to get minimal info: %v", errGet) + } + err := sender.Send(filesInfo) if err != nil { t.Errorf("send failed: %v", err) } @@ -129,10 +131,11 @@ func TestCrocLocal(t *testing.T) { os.Create("touched") wg.Add(2) go func() { - err := sender.Send(TransferOptions{ - PathToFiles: []string{"../../LICENSE", "touched"}, - KeepPathInRemote: false, - }) + filesInfo, errGet := GetFilesInfo([]string{"../../LICENSE", "touched"}) + if errGet != nil { + t.Errorf("failed to get minimal info: %v", errGet) + } + err := sender.Send(filesInfo) if err != nil { t.Errorf("send failed: %v", err) } @@ -181,10 +184,11 @@ func TestCrocError(t *testing.T) { Curve: "siec", Overwrite: true, }) - err = sender.Send(TransferOptions{ - PathToFiles: []string{tmpfile.Name()}, - KeepPathInRemote: true, - }) + filesInfo, errGet := GetFilesInfo([]string{tmpfile.Name()}) + if errGet != nil { + t.Errorf("failed to get minimal info: %v", errGet) + } + err = sender.Send(filesInfo) log.Debug(err) assert.NotNil(t, err) From 7c1a59c1022e3819b838512cb1a1b9e0e1e9a540 Mon Sep 17 00:00:00 2001 From: RCL98 <48773399+RCL98@users.noreply.github.com> Date: Mon, 21 Feb 2022 21:50:24 +0200 Subject: [PATCH 3/3] #431 fixed remote folder name for Windows --- src/croc/croc.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/croc/croc.go b/src/croc/croc.go index 2cdedb5e..6b4532f7 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -258,7 +258,6 @@ func GetFilesInfo(fnames []string) (filesInfo []FileInfo, err error) { } absPath, errAbs := filepath.Abs(path) - absPath = filepath.ToSlash(absPath) if errAbs != nil { err = errAbs @@ -272,10 +271,12 @@ func GetFilesInfo(fnames []string) (filesInfo []FileInfo, err error) { return err } if !info.IsDir() { + + remoteFolder := strings.TrimPrefix(filepath.Dir(pathName), + filepath.Dir(absPath)+string(os.PathSeparator)) filesInfo = append(filesInfo, FileInfo{ - Name: info.Name(), - FolderRemote: strings.TrimPrefix(filepath.Dir(filepath.ToSlash(pathName)), - filepath.Dir(absPath)+"/"), + Name: info.Name(), + FolderRemote: strings.Replace(remoteFolder, string(os.PathSeparator), "/", -1) + "/", FolderSource: filepath.Dir(pathName), Size: info.Size(), ModTime: info.ModTime(), @@ -1601,7 +1602,7 @@ func (c *Client) sendData(i int) { n, errRead := c.fread.ReadAt(data, readingPos) // log.Debugf("%d read %d bytes", i, n) readingPos += int64(n) - if (c.limiter != nil) { + if c.limiter != nil { r := c.limiter.ReserveN(time.Now(), n) log.Debugf("Limiting Upload for %d", r.Delay()) time.Sleep(r.Delay())