diff --git a/src/cli/cli.go b/src/cli/cli.go index 3c7d0e4b..c553877e 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -267,7 +267,7 @@ func send(c *cli.Context) (err error) { crocOptions.SharedSecret = utils.GetRandomName() } - minimalFileInfos, err := croc.GetFilesInfo(fnames) + minimalFileInfos, emptyFoldersToTransfer, totalNumberFolders, err := croc.GetFilesInfo(fnames) if err != nil { return } @@ -280,7 +280,7 @@ func send(c *cli.Context) (err error) { // save the config saveConfig(c, crocOptions) - err = cr.Send(minimalFileInfos) + err = cr.Send(minimalFileInfos, emptyFoldersToTransfer, totalNumberFolders) return } diff --git a/src/croc/croc.go b/src/croc/croc.go index 5dcf4629..ab6bfa76 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -88,12 +88,15 @@ type Client struct { Step1ChannelSecured bool Step2FileInfoTransferred bool Step3RecipientRequestFile bool - Step4FileTransfer bool + Step4FileTransferred bool Step5CloseChannels bool SuccessfulTransfer bool // send / receive information of all files FilesToTransfer []FileInfo + EmptyFoldersToTransfer []FileInfo + TotalNumberOfContents int + TotalNumberFolders int FilesToTransferCurrentNum int FilesHasFinished map[int]struct{} @@ -154,12 +157,14 @@ type RemoteFileRequest struct { // SenderInfo lists the files to be transferred type SenderInfo struct { - FilesToTransfer []FileInfo - MachineID string - Ask bool - SendingText bool - NoCompress bool - HashAlgorithm string + FilesToTransfer []FileInfo + EmptyFoldersToTransfer []FileInfo + TotalNumberFolders int + MachineID string + Ask bool + SendingText bool + NoCompress bool + HashAlgorithm string } // New establishes a new connection for transferring files between two instances. @@ -228,11 +233,25 @@ type TransferOptions struct { KeepPathInRemote bool } +func isEmptyFolder(folderPath string) (bool, error) { + f, err := os.Open(folderPath) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + return false, nil +} + // This function retrives the important file informations // for every file that will be transfered -func GetFilesInfo(fnames []string) (filesInfo []FileInfo, err error) { +func GetFilesInfo(fnames []string) (filesInfo []FileInfo, emptyFolders []FileInfo, totalNumberFolders int, err error) { // fnames: the relativ/absolute paths of files/folders that will be transfered - + totalNumberFolders = 0 var paths []string for _, fname := range fnames { // Support wildcard @@ -270,10 +289,9 @@ func GetFilesInfo(fnames []string) (filesInfo []FileInfo, err error) { if err != nil { return err } + remoteFolder := strings.TrimPrefix(filepath.Dir(pathName), + filepath.Dir(absPath)+string(os.PathSeparator)) if !info.IsDir() { - - remoteFolder := strings.TrimPrefix(filepath.Dir(pathName), - filepath.Dir(absPath)+string(os.PathSeparator)) filesInfo = append(filesInfo, FileInfo{ Name: info.Name(), FolderRemote: strings.Replace(remoteFolder, string(os.PathSeparator), "/", -1) + "/", @@ -282,6 +300,16 @@ func GetFilesInfo(fnames []string) (filesInfo []FileInfo, err error) { ModTime: info.ModTime(), Mode: info.Mode(), }) + } else { + totalNumberFolders++ + isEmptyFolder, _ := isEmptyFolder(pathName) + if isEmptyFolder { + emptyFolders = append(emptyFolders, FileInfo{ + // Name: info.Name(), + FolderRemote: strings.Replace(strings.TrimPrefix(pathName, + filepath.Dir(absPath)+string(os.PathSeparator)), string(os.PathSeparator), "/", -1) + "/", + }) + } } return nil }) @@ -341,6 +369,7 @@ func (c *Client) sendCollectFiles(filesInfo []FileInfo) (err error) { } log.Debugf("longestFilename: %+v", c.longestFilename) fname := fmt.Sprintf("%d files", len(c.FilesToTransfer)) + folderName := fmt.Sprintf("%d folders", c.TotalNumberFolders) if len(c.FilesToTransfer) == 1 { fname = fmt.Sprintf("'%s'", c.FilesToTransfer[0].Name) } @@ -352,7 +381,7 @@ func (c *Client) sendCollectFiles(filesInfo []FileInfo) (err error) { } fmt.Fprintf(os.Stderr, "\r ") - fmt.Fprintf(os.Stderr, "\rSending %s (%s)\n", fname, utils.ByteCountDecimal(totalFilesSize)) + fmt.Fprintf(os.Stderr, "\rSending %s and %s (%s)\n", fname, folderName, utils.ByteCountDecimal(totalFilesSize)) return } @@ -443,8 +472,12 @@ func (c *Client) transferOverLocalRelay(errchan chan<- error) { } // Send will send the specified file -func (c *Client) Send(filesInfo []FileInfo) (err error) { +func (c *Client) Send(filesInfo []FileInfo, emptyFoldersToTransfer []FileInfo, totalNumberFolders int) (err error) { + c.EmptyFoldersToTransfer = emptyFoldersToTransfer + c.TotalNumberFolders = totalNumberFolders + c.TotalNumberOfContents = len(filesInfo) err = c.sendCollectFiles(filesInfo) + if err != nil { return } @@ -778,7 +811,7 @@ func (c *Client) Receive() (err error) { fmt.Fprintf(os.Stderr, "\rsecuring channel...") err = c.transfer() if err == nil { - if c.numberOfTransferredFiles == 0 { + if c.numberOfTransferredFiles+len(c.EmptyFoldersToTransfer) == 0 { fmt.Fprintf(os.Stderr, "\rNo files transferred.") } } @@ -860,6 +893,28 @@ func (c *Client) transfer() (err error) { return } +func (c *Client) createEmptyFolder(i int) (err error) { + err = os.MkdirAll(c.EmptyFoldersToTransfer[i].FolderRemote, os.ModePerm) + if err != nil { + return + } + fmt.Fprintf(os.Stderr, "%s\n", c.EmptyFoldersToTransfer[i].FolderRemote) + c.bar = progressbar.NewOptions64(1, + progressbar.OptionOnCompletion(func() { + c.fmtPrintUpdate() + }), + progressbar.OptionSetWidth(20), + progressbar.OptionSetDescription(" "), + progressbar.OptionSetRenderBlankState(true), + progressbar.OptionShowBytes(true), + progressbar.OptionShowCount(), + progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetVisibility(!c.Options.SendingText), + ) + c.bar.Finish() + return +} + func (c *Client) processMessageFileInfo(m message.Message) (done bool, err error) { var senderInfo SenderInfo err = json.Unmarshal(m.Bytes, &senderInfo) @@ -870,6 +925,17 @@ func (c *Client) processMessageFileInfo(m message.Message) (done bool, err error c.Options.SendingText = senderInfo.SendingText c.Options.NoCompress = senderInfo.NoCompress c.Options.HashAlgorithm = senderInfo.HashAlgorithm + c.EmptyFoldersToTransfer = senderInfo.EmptyFoldersToTransfer + c.TotalNumberFolders = senderInfo.TotalNumberFolders + c.FilesToTransfer = senderInfo.FilesToTransfer + c.TotalNumberOfContents = 0 + if c.FilesToTransfer != nil { + c.TotalNumberOfContents += len(c.FilesToTransfer) + } + if c.EmptyFoldersToTransfer != nil { + c.TotalNumberOfContents += len(c.EmptyFoldersToTransfer) + } + if c.Options.HashAlgorithm == "" { c.Options.HashAlgorithm = "xxhash" } @@ -880,8 +946,9 @@ func (c *Client) processMessageFileInfo(m message.Message) (done bool, err error if c.Options.SendingText { c.Options.Stdout = true } - c.FilesToTransfer = senderInfo.FilesToTransfer + fname := fmt.Sprintf("%d files", len(c.FilesToTransfer)) + folderName := fmt.Sprintf("%d folders", c.TotalNumberFolders) if len(c.FilesToTransfer) == 1 { fname = fmt.Sprintf("'%s'", c.FilesToTransfer[0].Name) } @@ -909,7 +976,7 @@ func (c *Client) processMessageFileInfo(m message.Message) (done bool, err error machID, _ := machineid.ID() fmt.Fprintf(os.Stderr, "\rYour machine id is '%s'.\n%s %s (%s) from '%s'? (Y/n) ", machID, action, fname, utils.ByteCountDecimal(totalSize), senderInfo.MachineID) } else { - fmt.Fprintf(os.Stderr, "\r%s %s (%s)? (Y/n) ", action, fname, utils.ByteCountDecimal(totalSize)) + fmt.Fprintf(os.Stderr, "\r%s %s and %s (%s)? (Y/n) ", action, fname, folderName, utils.ByteCountDecimal(totalSize)) } choice := strings.ToLower(utils.GetInput("")) if choice != "" && choice != "y" && choice != "yes" { @@ -927,6 +994,42 @@ func (c *Client) processMessageFileInfo(m message.Message) (done bool, err error } fmt.Fprintf(os.Stderr, "\nReceiving (<-%s)\n", c.ExternalIPConnected) + for i := 0; i < len(c.EmptyFoldersToTransfer); i += 1 { + _, errExists := os.Stat(c.EmptyFoldersToTransfer[i].FolderRemote) + if os.IsNotExist(errExists) { + err = c.createEmptyFolder(i) + if err != nil { + return + } + } else { + isEmpty, _ := isEmptyFolder(c.EmptyFoldersToTransfer[i].FolderRemote) + if !isEmpty { + log.Debug("asking to overwrite") + prompt := fmt.Sprintf("\n%s already has some content in it. \nDo you want"+ + " to overwrite it with an empty folder? (y/N) ", c.EmptyFoldersToTransfer[i].FolderRemote) + choice := strings.ToLower(utils.GetInput(prompt)) + if choice == "y" || choice == "yes" { + err = c.createEmptyFolder(i) + if err != nil { + return + } + } + } + } + } + + // if no files are to be transfered, then we can end the file transfer process + if c.FilesToTransfer == nil { + c.SuccessfulTransfer = true + c.Step3RecipientRequestFile = true + c.Step4FileTransferred = true + errStopTransfer := message.Send(c.conn[0], c.Key, message.Message{ + Type: message.TypeFinished, + }) + if errStopTransfer != nil { + err = errStopTransfer + } + } log.Debug(c.FilesToTransfer) c.Step2FileInfoTransferred = true return @@ -1121,14 +1224,14 @@ func (c *Client) processMessage(payload []byte) (done bool, err error) { case message.TypeCloseSender: c.bar.Finish() log.Debug("close-sender received...") - c.Step4FileTransfer = false + c.Step4FileTransferred = false c.Step3RecipientRequestFile = false log.Debug("sending close-recipient") err = message.Send(c.conn[0], c.Key, message.Message{ Type: message.TypeCloseRecipient, }) case message.TypeCloseRecipient: - c.Step4FileTransfer = false + c.Step4FileTransferred = false c.Step3RecipientRequestFile = false } if err != nil { @@ -1148,12 +1251,14 @@ func (c *Client) updateIfSenderChannelSecured() (err error) { var b []byte machID, _ := machineid.ID() b, err = json.Marshal(SenderInfo{ - FilesToTransfer: c.FilesToTransfer, - MachineID: machID, - Ask: c.Options.Ask, - SendingText: c.Options.SendingText, - NoCompress: c.Options.NoCompress, - HashAlgorithm: c.Options.HashAlgorithm, + FilesToTransfer: c.FilesToTransfer, + EmptyFoldersToTransfer: c.EmptyFoldersToTransfer, + MachineID: machID, + Ask: c.Options.Ask, + TotalNumberFolders: c.TotalNumberFolders, + SendingText: c.Options.SendingText, + NoCompress: c.Options.NoCompress, + HashAlgorithm: c.Options.HashAlgorithm, }) if err != nil { log.Error(err) @@ -1198,7 +1303,7 @@ func (c *Client) recipientInitializeFile() (err error) { if errOpen == nil { stat, _ := c.CurrentFile.Stat() truncate = stat.Size() != c.FilesToTransfer[c.FilesToTransferCurrentNum].Size - if truncate == false { + if !truncate { // recipient requests the file and chunks (if empty, then should receive all chunks) // TODO: determine the missing chunks c.CurrentFileChunkRanges = utils.MissingChunks( @@ -1306,8 +1411,8 @@ func (c *Client) createEmptyFileAndFinish(fileInfo FileInfo, i int) (err error) // setup the progressbar description := fmt.Sprintf("%-*s", c.longestFilename, c.FilesToTransfer[i].Name) if len(c.FilesToTransfer) == 1 { - // description = c.FilesToTransfer[i].Name - description = "" + description = c.FilesToTransfer[i].Name + // description = "" } else { description = " " + description } @@ -1334,7 +1439,6 @@ func (c *Client) updateIfRecipientHasFileInfo() (err error) { // find the next file to transfer and send that number // if the files are the same size, then look for missing chunks finished := true - for i, fileInfo := range c.FilesToTransfer { if _, ok := c.FilesHasFinished[i]; ok { continue @@ -1402,14 +1506,14 @@ func (c *Client) updateIfRecipientHasFileInfo() (err error) { break } } - err = c.recipientGetFileReady(finished) + c.recipientGetFileReady(finished) return } func (c *Client) fmtPrintUpdate() { c.finishedNum++ - if len(c.FilesToTransfer) > 1 { - fmt.Fprintf(os.Stderr, " %d/%d\n", c.finishedNum, len(c.FilesToTransfer)) + if c.TotalNumberOfContents > 1 { + fmt.Fprintf(os.Stderr, " %d/%d\n", c.finishedNum, c.TotalNumberOfContents) } else { fmt.Fprintf(os.Stderr, "\n") } @@ -1426,7 +1530,7 @@ func (c *Client) updateState() (err error) { return } - if c.Options.IsSender && c.Step3RecipientRequestFile && !c.Step4FileTransfer { + if c.Options.IsSender && c.Step3RecipientRequestFile && !c.Step4FileTransferred { log.Debug("start sending data!") if !c.firstSend { @@ -1438,8 +1542,8 @@ func (c *Client) updateState() (err error) { // setup the progressbar and takedown the progress bar for empty files description := fmt.Sprintf("%-*s", c.longestFilename, c.FilesToTransfer[i].Name) if len(c.FilesToTransfer) == 1 { - // description = c.FilesToTransfer[i].Name - description = "" + description = c.FilesToTransfer[i].Name + // description = "" } c.bar = progressbar.NewOptions64(1, progressbar.OptionOnCompletion(func() { @@ -1457,7 +1561,7 @@ func (c *Client) updateState() (err error) { } } } - c.Step4FileTransfer = true + c.Step4FileTransferred = true // setup the progressbar c.setBar() c.TotalSent = 0 @@ -1483,9 +1587,9 @@ func (c *Client) updateState() (err error) { func (c *Client) setBar() { description := fmt.Sprintf("%-*s", c.longestFilename, c.FilesToTransfer[c.FilesToTransferCurrentNum].Name) - if len(c.FilesToTransfer) == 1 { - // description = c.FilesToTransfer[c.FilesToTransferCurrentNum].Name - description = "" + folder, _ := filepath.Split(c.FilesToTransfer[c.FilesToTransferCurrentNum].FolderRemote) + if folder == "./" { + description = c.FilesToTransfer[c.FilesToTransferCurrentNum].Name } else if !c.Options.IsSender { description = " " + description } @@ -1559,6 +1663,8 @@ func (c *Client) receiveData(i int) { log.Debug("finished receiving!") if err := c.CurrentFile.Close(); err != nil { log.Debugf("error closing %s: %v", c.CurrentFile.Name(), err) + } else { + log.Debugf("Successful closing %s", c.CurrentFile.Name()) } if c.Options.Stdout || c.Options.SendingText { pathToFile := path.Join( diff --git a/src/croc/croc_test.go b/src/croc/croc_test.go index b55249e9..c825b9d8 100644 --- a/src/croc/croc_test.go +++ b/src/croc/croc_test.go @@ -4,6 +4,7 @@ import ( "os" "path" "path/filepath" + "runtime" "sync" "testing" "time" @@ -65,11 +66,77 @@ func TestCrocReadme(t *testing.T) { var wg sync.WaitGroup wg.Add(2) go func() { - filesInfo, errGet := GetFilesInfo([]string{"../../README.md"}) + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../README.md"}) if errGet != nil { t.Errorf("failed to get minimal info: %v", errGet) } - err := sender.Send(filesInfo) + err := sender.Send(filesInfo, emptyFolders, totalNumberFolders) + if err != nil { + t.Errorf("send failed: %v", err) + } + wg.Done() + }() + time.Sleep(100 * time.Millisecond) + go func() { + err := receiver.Receive() + if err != nil { + t.Errorf("receive failed: %v", err) + } + wg.Done() + }() + + wg.Wait() +} + +func TestCrocEmptyFolder(t *testing.T) { + pathName := "../../testEmpty" + defer os.RemoveAll(pathName) + defer os.RemoveAll("./testEmpty") + os.MkdirAll(pathName, 0755) + + log.Debug("setting up sender") + sender, err := New(Options{ + IsSender: true, + SharedSecret: "8123-testingthecroc", + Debug: true, + RelayAddress: "localhost:8281", + RelayPorts: []string{"8281"}, + RelayPassword: "pass123", + Stdout: false, + NoPrompt: true, + DisableLocal: true, + Curve: "siec", + Overwrite: true, + }) + if err != nil { + panic(err) + } + + log.Debug("setting up receiver") + receiver, err := New(Options{ + IsSender: false, + SharedSecret: "8123-testingthecroc", + Debug: true, + RelayAddress: "localhost:8281", + RelayPassword: "pass123", + Stdout: false, + NoPrompt: true, + DisableLocal: true, + Curve: "siec", + Overwrite: true, + }) + if err != nil { + panic(err) + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}) + if errGet != nil { + t.Errorf("failed to get minimal info: %v", errGet) + } + err := sender.Send(filesInfo, emptyFolders, totalNumberFolders) if err != nil { t.Errorf("send failed: %v", err) } @@ -90,6 +157,7 @@ func TestCrocReadme(t *testing.T) { func TestCrocSymlink(t *testing.T) { pathName := "../link-in-folder" defer os.RemoveAll(pathName) + defer os.RemoveAll("./link-in-folder") os.MkdirAll(pathName, 0755) os.Symlink("../../README.md", filepath.Join(pathName, "README.link")) @@ -131,11 +199,11 @@ func TestCrocSymlink(t *testing.T) { var wg sync.WaitGroup wg.Add(2) go func() { - filesInfo, errGet := GetFilesInfo([]string{pathName}) + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}) if errGet != nil { t.Errorf("failed to get minimal info: %v", errGet) } - err := sender.Send(filesInfo) + err := sender.Send(filesInfo, emptyFolders, totalNumberFolders) if err != nil { t.Errorf("send failed: %v", err) } @@ -153,7 +221,8 @@ func TestCrocSymlink(t *testing.T) { wg.Wait() s, err := filepath.EvalSymlinks(path.Join(pathName, "README.link")) - if s != "../../README.md" { + if s != "../../README.md" && s != "..\\..\\README.md" { + log.Debug(s) t.Errorf("symlink failed to transfer in folder") } if err != nil { @@ -207,11 +276,11 @@ func TestCrocLocal(t *testing.T) { os.Create("touched") wg.Add(2) go func() { - filesInfo, errGet := GetFilesInfo([]string{"../../LICENSE", "touched"}) + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../LICENSE", "touched"}) if errGet != nil { t.Errorf("failed to get minimal info: %v", errGet) } - err := sender.Send(filesInfo) + err := sender.Send(filesInfo, emptyFolders, totalNumberFolders) if err != nil { t.Errorf("send failed: %v", err) } @@ -260,12 +329,42 @@ func TestCrocError(t *testing.T) { Curve: "siec", Overwrite: true, }) - filesInfo, errGet := GetFilesInfo([]string{tmpfile.Name()}) + filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}) if errGet != nil { t.Errorf("failed to get minimal info: %v", errGet) } - err = sender.Send(filesInfo) + err = sender.Send(filesInfo, emptyFolders, totalNumberFolders) log.Debug(err) assert.NotNil(t, err) - +} + +func TestCleanUp(t *testing.T) { + // windows allows files to be deleted only if they + // are not open by another program so the remove actions + // from the above tests will not always do a good clean up + // This "test" will make sure + operatingSystem := runtime.GOOS + log.Debugf("The operating system is %s", operatingSystem) + if operatingSystem == "windows" { + time.Sleep(1 * time.Second) + log.Debug("Full cleanup") + var err error + + for _, file := range []string{"README.md", "./README.md"} { + err = os.Remove(file) + if err == nil { + log.Debugf("Successfuly purged %s", file) + } else { + log.Debugf("%s was already purged.", file) + } + } + for _, folder := range []string{"./testEmpty", "./link-in-folder"} { + err = os.RemoveAll(folder) + if err == nil { + log.Debugf("Successfuly purged %s", folder) + } else { + log.Debugf("%s was already purged.", folder) + } + } + } }