share chunk ranges instead of chunks Fixes #125

src/croc/croc.go

@@ -77,7 +77,9 @@ type Client struct {
 
 	// send / receive information of current file
 	CurrentFile           *os.File
+	CurrentFileChunkRanges     []int64
 	CurrentFileChunks     []int64
+	
 	TotalSent             int64
 	TotalChunksTransfered int
 	chunkMap              map[uint64]struct{}
@@ -111,7 +113,7 @@ type FileInfo struct {
 }
 
 type RemoteFileRequest struct {
-	CurrentFileChunks         []int64
+	CurrentFileChunkRanges        []int64
 	FilesToTransferCurrentNum int
 }
 
@@ -548,7 +550,8 @@ func (c *Client) processMessage(payload []byte) (done bool, err error) {
 			return
 		}
 		c.FilesToTransferCurrentNum = remoteFile.FilesToTransferCurrentNum
-		c.CurrentFileChunks = remoteFile.CurrentFileChunks
+		c.CurrentFileChunkRanges = remoteFile.CurrentFileChunkRanges
+		c.CurrentFileChunks = utils.ChunkRangesToChunks(c.CurrentFileChunkRanges)
 		log.Debugf("current file chunks: %+v", c.CurrentFileChunks)
 		c.chunkMap = make(map[uint64]struct{})
 		for _, chunk := range c.CurrentFileChunks {
@@ -611,6 +614,7 @@ func (c *Client) updateState() (err error) {
 				continue
 			}
 			fileHash, errHash := utils.HashFile(path.Join(fileInfo.FolderRemote, fileInfo.Name))
+			log.Debugf("%s %+x %+x %+v",fileInfo.Name,fileHash,fileInfo.Hash,errHash)
 			if errHash != nil || !bytes.Equal(fileHash, fileInfo.Hash) {
 				if errHash != nil {
 					// probably can't find, its okay
@@ -655,13 +659,14 @@ func (c *Client) updateState() (err error) {
 			os.O_WRONLY, 0666)
 		truncate := false
 		c.CurrentFileChunks = []int64{}
+		c.CurrentFileChunkRanges = []int64{}
 		if errOpen == nil {
 			stat, _ := c.CurrentFile.Stat()
 			truncate = stat.Size() != c.FilesToTransfer[c.FilesToTransferCurrentNum].Size
 			if truncate == false {
 				// recipient requests the file and chunks (if empty, then should receive all chunks)
 				// TODO: determine the missing chunks
-				c.CurrentFileChunks = utils.MissingChunks(
+				c.CurrentFileChunkRanges = utils.MissingChunks(
 					pathToFile,
 					c.FilesToTransfer[c.FilesToTransferCurrentNum].Size,
 					models.TCP_BUFFER_SIZE/2,
@@ -685,13 +690,17 @@ func (c *Client) updateState() (err error) {
 			}
 		}
 
-		// setup the progressbar
-		c.setBar()
 		c.TotalSent = 0
 		bRequest, _ := json.Marshal(RemoteFileRequest{
-			CurrentFileChunks:         c.CurrentFileChunks,
+			CurrentFileChunkRanges:         c.CurrentFileChunkRanges,
 			FilesToTransferCurrentNum: c.FilesToTransferCurrentNum,
 		})
+		log.Debug("converting to chunk range")
+		c.CurrentFileChunks = utils.ChunkRangesToChunks(c.CurrentFileChunkRanges)
+
+		// setup the progressbar
+		c.setBar()
+
 		log.Debugf("sending recipient ready with %d chunks",len(c.CurrentFileChunks))
 		err = message.Send(c.conn[0], c.Key, message.Message{
 			Type:  "recipientready",

src/message/message.go

@@ -2,7 +2,8 @@ package message
 
 import (
 	"encoding/json"
-
+	
+	log "github.com/cihub/seelog"
 	"github.com/schollz/croc/v6/src/comm"
 	"github.com/schollz/croc/v6/src/compress"
 	"github.com/schollz/croc/v6/src/crypt"
@@ -27,6 +28,7 @@ func Send(c *comm.Comm, key crypt.Encryption, m Message) (err error) {
 	if err != nil {
 		return
 	}
+	log.Debugf("writing %s message (%d bytes)",m.Type,len(mSend))
 	_, err = c.Write(mSend)
 	return
 }

src/utils/utils.go

@@ -41,7 +41,7 @@ func GetInput(prompt string) string {
 
 // HashFile returns the hash of a file
 func HashFile(fname string) (hash256 []byte, err error) {
-	return IMOHashFile(fname)
+	return XXHashFile(fname)
 }
 
 func MD5HashFile(fname string) (hash256 []byte, err error) {
@@ -145,7 +145,7 @@ func ByteCountDecimal(b int64) string {
 // MissingChunks returns the positions of missing chunks.
 // If file doesn't exist, it returns an empty chunk list (all chunks).
 // If the file size is not the same as requested, it returns an empty chunk list (all chunks).
-func MissingChunks(fname string, fsize int64, chunkSize int) (chunks []int64) {
+func MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) {
 	fstat, err := os.Stat(fname)
 	if fstat.Size() != fsize || err != nil {
 		return
@@ -159,7 +159,7 @@ func MissingChunks(fname string, fsize int64, chunkSize int) (chunks []int64) {
 
 	emptyBuffer := make([]byte, chunkSize)
 	chunkNum := 0
-	chunks = make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))
+	chunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))
 	var currentLocation int64
 	for {
 		buffer := make([]byte, chunkSize)
@@ -174,9 +174,38 @@ func MissingChunks(fname string, fsize int64, chunkSize int) (chunks []int64) {
 		currentLocation += int64(bytesread)
 	}
 	if chunkNum == 0 {
-		chunks = []int64{}
+		chunkRanges = []int64{}
 	} else {
 		chunks = chunks[:chunkNum]
+		chunkRanges = []int64{int64(chunkSize), chunks[0]}
+		curCount := 0
+		for i, chunk := range chunks {
+			if i == 0 {
+				continue
+			}
+			curCount++
+			if chunk-chunks[i-1] > int64(chunkSize) {
+				chunkRanges = append(chunkRanges, int64(curCount))
+				chunkRanges = append(chunkRanges, chunk)
+				curCount = 0
+			}
+		}
+		chunkRanges = append(chunkRanges, int64(curCount+1))
+		chunks = chunkRanges
+	}
+	return
+}
+
+func ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) {
+	if len(chunkRanges) == 0 {
+		return
+	}
+	chunkSize := chunkRanges[0]
+	chunks = []int64{}
+	for i := 1; i < len(chunkRanges); i += 2 {
+		for j := int64(0); j < (chunkRanges[i+1]); j++ {
+			chunks = append(chunks, chunkRanges[i]+j*chunkSize)
+		}
 	}
 	return
 }

src/utils/utils_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -63,3 +64,36 @@ func TestSHA256(t *testing.T) {
 func TestByteCountDecimal(t *testing.T) {
 	assert.Equal(t, "10.0 kB", ByteCountDecimal(10000))
 }
+
+func TestMissingChunks(t *testing.T) {
+	fileSize := 100
+	chunkSize := 10
+	rand.Seed(1)
+	bigBuff := make([]byte, fileSize)
+	rand.Read(bigBuff)
+	ioutil.WriteFile("missing.test", bigBuff, 0644)
+	empty := make([]byte, chunkSize)
+	f, err := os.OpenFile("missing.test", os.O_RDWR, 0644)
+	assert.Nil(t, err)
+	for block := 0; block < fileSize/chunkSize; block++ {
+		if block == 0 || block == 4 || block == 5 || block >= 7 {
+			f.WriteAt(empty, int64(block*chunkSize))
+		}
+	}
+	f.Close()
+
+	chunkRanges := MissingChunks("missing.test", int64(fileSize), chunkSize)
+	assert.Equal(t, []int64{10, 0, 1, 40, 2, 70, 3}, chunkRanges)
+
+	chunks := ChunkRangesToChunks(chunkRanges)
+	assert.Equal(t, []int64{0, 40, 50, 70, 80, 90}, chunks)
+
+	os.Remove("missing.test")
+}
+
+// func Test1(t *testing.T) {
+// 	chunkRanges := MissingChunks("../../m/bigfile.test", int64(75000000), 1024*64/2)
+// 	fmt.Println(chunkRanges)
+// 	fmt.Println(ChunkRangesToChunks((chunkRanges)))
+// 	assert.Nil(t, nil)
+// }