From 9f9b93cf47ccee5fd50dc61d19511db3f968592b Mon Sep 17 00:00:00 2001 From: Zack Scholl Date: Fri, 3 May 2019 07:30:35 -0700 Subject: [PATCH] share chunk ranges instead of chunks Fixes #125 --- src/croc/croc.go | 21 +++++++++++++++------ src/message/message.go | 4 +++- src/utils/utils.go | 37 +++++++++++++++++++++++++++++++++---- src/utils/utils_test.go | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 11 deletions(-) diff --git a/src/croc/croc.go b/src/croc/croc.go index 875af348..a78a63cf 100644 --- a/src/croc/croc.go +++ b/src/croc/croc.go @@ -77,7 +77,9 @@ type Client struct { // send / receive information of current file CurrentFile *os.File + CurrentFileChunkRanges []int64 CurrentFileChunks []int64 + TotalSent int64 TotalChunksTransfered int chunkMap map[uint64]struct{} @@ -111,7 +113,7 @@ type FileInfo struct { } type RemoteFileRequest struct { - CurrentFileChunks []int64 + CurrentFileChunkRanges []int64 FilesToTransferCurrentNum int } @@ -548,7 +550,8 @@ func (c *Client) processMessage(payload []byte) (done bool, err error) { return } c.FilesToTransferCurrentNum = remoteFile.FilesToTransferCurrentNum - c.CurrentFileChunks = remoteFile.CurrentFileChunks + c.CurrentFileChunkRanges = remoteFile.CurrentFileChunkRanges + c.CurrentFileChunks = utils.ChunkRangesToChunks(c.CurrentFileChunkRanges) log.Debugf("current file chunks: %+v", c.CurrentFileChunks) c.chunkMap = make(map[uint64]struct{}) for _, chunk := range c.CurrentFileChunks { @@ -611,6 +614,7 @@ func (c *Client) updateState() (err error) { continue } fileHash, errHash := utils.HashFile(path.Join(fileInfo.FolderRemote, fileInfo.Name)) + log.Debugf("%s %+x %+x %+v",fileInfo.Name,fileHash,fileInfo.Hash,errHash) if errHash != nil || !bytes.Equal(fileHash, fileInfo.Hash) { if errHash != nil { // probably can't find, its okay @@ -655,13 +659,14 @@ func (c *Client) updateState() (err error) { os.O_WRONLY, 0666) truncate := false c.CurrentFileChunks = []int64{} + c.CurrentFileChunkRanges = []int64{} if errOpen == nil { stat, _ := c.CurrentFile.Stat() truncate = stat.Size() != c.FilesToTransfer[c.FilesToTransferCurrentNum].Size if truncate == false { // recipient requests the file and chunks (if empty, then should receive all chunks) // TODO: determine the missing chunks - c.CurrentFileChunks = utils.MissingChunks( + c.CurrentFileChunkRanges = utils.MissingChunks( pathToFile, c.FilesToTransfer[c.FilesToTransferCurrentNum].Size, models.TCP_BUFFER_SIZE/2, @@ -685,13 +690,17 @@ func (c *Client) updateState() (err error) { } } - // setup the progressbar - c.setBar() c.TotalSent = 0 bRequest, _ := json.Marshal(RemoteFileRequest{ - CurrentFileChunks: c.CurrentFileChunks, + CurrentFileChunkRanges: c.CurrentFileChunkRanges, FilesToTransferCurrentNum: c.FilesToTransferCurrentNum, }) + log.Debug("converting to chunk range") + c.CurrentFileChunks = utils.ChunkRangesToChunks(c.CurrentFileChunkRanges) + + // setup the progressbar + c.setBar() + log.Debugf("sending recipient ready with %d chunks",len(c.CurrentFileChunks)) err = message.Send(c.conn[0], c.Key, message.Message{ Type: "recipientready", diff --git a/src/message/message.go b/src/message/message.go index 2c43c8c9..356e81c5 100644 --- a/src/message/message.go +++ b/src/message/message.go @@ -2,7 +2,8 @@ package message import ( "encoding/json" - + + log "github.com/cihub/seelog" "github.com/schollz/croc/v6/src/comm" "github.com/schollz/croc/v6/src/compress" "github.com/schollz/croc/v6/src/crypt" @@ -27,6 +28,7 @@ func Send(c *comm.Comm, key crypt.Encryption, m Message) (err error) { if err != nil { return } + log.Debugf("writing %s message (%d bytes)",m.Type,len(mSend)) _, err = c.Write(mSend) return } diff --git a/src/utils/utils.go b/src/utils/utils.go index 5debe330..78a869f7 100644 --- a/src/utils/utils.go +++ b/src/utils/utils.go @@ -41,7 +41,7 @@ func GetInput(prompt string) string { // HashFile returns the hash of a file func HashFile(fname string) (hash256 []byte, err error) { - return IMOHashFile(fname) + return XXHashFile(fname) } func MD5HashFile(fname string) (hash256 []byte, err error) { @@ -145,7 +145,7 @@ func ByteCountDecimal(b int64) string { // MissingChunks returns the positions of missing chunks. // If file doesn't exist, it returns an empty chunk list (all chunks). // If the file size is not the same as requested, it returns an empty chunk list (all chunks). -func MissingChunks(fname string, fsize int64, chunkSize int) (chunks []int64) { +func MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) { fstat, err := os.Stat(fname) if fstat.Size() != fsize || err != nil { return @@ -159,7 +159,7 @@ func MissingChunks(fname string, fsize int64, chunkSize int) (chunks []int64) { emptyBuffer := make([]byte, chunkSize) chunkNum := 0 - chunks = make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize)))) + chunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize)))) var currentLocation int64 for { buffer := make([]byte, chunkSize) @@ -174,9 +174,38 @@ func MissingChunks(fname string, fsize int64, chunkSize int) (chunks []int64) { currentLocation += int64(bytesread) } if chunkNum == 0 { - chunks = []int64{} + chunkRanges = []int64{} } else { chunks = chunks[:chunkNum] + chunkRanges = []int64{int64(chunkSize), chunks[0]} + curCount := 0 + for i, chunk := range chunks { + if i == 0 { + continue + } + curCount++ + if chunk-chunks[i-1] > int64(chunkSize) { + chunkRanges = append(chunkRanges, int64(curCount)) + chunkRanges = append(chunkRanges, chunk) + curCount = 0 + } + } + chunkRanges = append(chunkRanges, int64(curCount+1)) + chunks = chunkRanges + } + return +} + +func ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) { + if len(chunkRanges) == 0 { + return + } + chunkSize := chunkRanges[0] + chunks = []int64{} + for i := 1; i < len(chunkRanges); i += 2 { + for j := int64(0); j < (chunkRanges[i+1]); j++ { + chunks = append(chunks, chunkRanges[i]+j*chunkSize) + } } return } diff --git a/src/utils/utils_test.go b/src/utils/utils_test.go index 179540c0..520dacb3 100644 --- a/src/utils/utils_test.go +++ b/src/utils/utils_test.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "math/rand" + "os" "testing" "github.com/stretchr/testify/assert" @@ -63,3 +64,36 @@ func TestSHA256(t *testing.T) { func TestByteCountDecimal(t *testing.T) { assert.Equal(t, "10.0 kB", ByteCountDecimal(10000)) } + +func TestMissingChunks(t *testing.T) { + fileSize := 100 + chunkSize := 10 + rand.Seed(1) + bigBuff := make([]byte, fileSize) + rand.Read(bigBuff) + ioutil.WriteFile("missing.test", bigBuff, 0644) + empty := make([]byte, chunkSize) + f, err := os.OpenFile("missing.test", os.O_RDWR, 0644) + assert.Nil(t, err) + for block := 0; block < fileSize/chunkSize; block++ { + if block == 0 || block == 4 || block == 5 || block >= 7 { + f.WriteAt(empty, int64(block*chunkSize)) + } + } + f.Close() + + chunkRanges := MissingChunks("missing.test", int64(fileSize), chunkSize) + assert.Equal(t, []int64{10, 0, 1, 40, 2, 70, 3}, chunkRanges) + + chunks := ChunkRangesToChunks(chunkRanges) + assert.Equal(t, []int64{0, 40, 50, 70, 80, 90}, chunks) + + os.Remove("missing.test") +} + +// func Test1(t *testing.T) { +// chunkRanges := MissingChunks("../../m/bigfile.test", int64(75000000), 1024*64/2) +// fmt.Println(chunkRanges) +// fmt.Println(ChunkRangesToChunks((chunkRanges))) +// assert.Nil(t, nil) +// }