mirror of
https://github.com/schollz/croc.git
synced 2025-10-11 13:21:00 +02:00
share chunk ranges instead of chunks Fixes #125
This commit is contained in:
parent
7d91f8200c
commit
9f9b93cf47
4 changed files with 85 additions and 11 deletions
|
@ -77,7 +77,9 @@ type Client struct {
|
||||||
|
|
||||||
// send / receive information of current file
|
// send / receive information of current file
|
||||||
CurrentFile *os.File
|
CurrentFile *os.File
|
||||||
|
CurrentFileChunkRanges []int64
|
||||||
CurrentFileChunks []int64
|
CurrentFileChunks []int64
|
||||||
|
|
||||||
TotalSent int64
|
TotalSent int64
|
||||||
TotalChunksTransfered int
|
TotalChunksTransfered int
|
||||||
chunkMap map[uint64]struct{}
|
chunkMap map[uint64]struct{}
|
||||||
|
@ -111,7 +113,7 @@ type FileInfo struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type RemoteFileRequest struct {
|
type RemoteFileRequest struct {
|
||||||
CurrentFileChunks []int64
|
CurrentFileChunkRanges []int64
|
||||||
FilesToTransferCurrentNum int
|
FilesToTransferCurrentNum int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -548,7 +550,8 @@ func (c *Client) processMessage(payload []byte) (done bool, err error) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c.FilesToTransferCurrentNum = remoteFile.FilesToTransferCurrentNum
|
c.FilesToTransferCurrentNum = remoteFile.FilesToTransferCurrentNum
|
||||||
c.CurrentFileChunks = remoteFile.CurrentFileChunks
|
c.CurrentFileChunkRanges = remoteFile.CurrentFileChunkRanges
|
||||||
|
c.CurrentFileChunks = utils.ChunkRangesToChunks(c.CurrentFileChunkRanges)
|
||||||
log.Debugf("current file chunks: %+v", c.CurrentFileChunks)
|
log.Debugf("current file chunks: %+v", c.CurrentFileChunks)
|
||||||
c.chunkMap = make(map[uint64]struct{})
|
c.chunkMap = make(map[uint64]struct{})
|
||||||
for _, chunk := range c.CurrentFileChunks {
|
for _, chunk := range c.CurrentFileChunks {
|
||||||
|
@ -611,6 +614,7 @@ func (c *Client) updateState() (err error) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
fileHash, errHash := utils.HashFile(path.Join(fileInfo.FolderRemote, fileInfo.Name))
|
fileHash, errHash := utils.HashFile(path.Join(fileInfo.FolderRemote, fileInfo.Name))
|
||||||
|
log.Debugf("%s %+x %+x %+v",fileInfo.Name,fileHash,fileInfo.Hash,errHash)
|
||||||
if errHash != nil || !bytes.Equal(fileHash, fileInfo.Hash) {
|
if errHash != nil || !bytes.Equal(fileHash, fileInfo.Hash) {
|
||||||
if errHash != nil {
|
if errHash != nil {
|
||||||
// probably can't find, its okay
|
// probably can't find, its okay
|
||||||
|
@ -655,13 +659,14 @@ func (c *Client) updateState() (err error) {
|
||||||
os.O_WRONLY, 0666)
|
os.O_WRONLY, 0666)
|
||||||
truncate := false
|
truncate := false
|
||||||
c.CurrentFileChunks = []int64{}
|
c.CurrentFileChunks = []int64{}
|
||||||
|
c.CurrentFileChunkRanges = []int64{}
|
||||||
if errOpen == nil {
|
if errOpen == nil {
|
||||||
stat, _ := c.CurrentFile.Stat()
|
stat, _ := c.CurrentFile.Stat()
|
||||||
truncate = stat.Size() != c.FilesToTransfer[c.FilesToTransferCurrentNum].Size
|
truncate = stat.Size() != c.FilesToTransfer[c.FilesToTransferCurrentNum].Size
|
||||||
if truncate == false {
|
if truncate == false {
|
||||||
// recipient requests the file and chunks (if empty, then should receive all chunks)
|
// recipient requests the file and chunks (if empty, then should receive all chunks)
|
||||||
// TODO: determine the missing chunks
|
// TODO: determine the missing chunks
|
||||||
c.CurrentFileChunks = utils.MissingChunks(
|
c.CurrentFileChunkRanges = utils.MissingChunks(
|
||||||
pathToFile,
|
pathToFile,
|
||||||
c.FilesToTransfer[c.FilesToTransferCurrentNum].Size,
|
c.FilesToTransfer[c.FilesToTransferCurrentNum].Size,
|
||||||
models.TCP_BUFFER_SIZE/2,
|
models.TCP_BUFFER_SIZE/2,
|
||||||
|
@ -685,13 +690,17 @@ func (c *Client) updateState() (err error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// setup the progressbar
|
|
||||||
c.setBar()
|
|
||||||
c.TotalSent = 0
|
c.TotalSent = 0
|
||||||
bRequest, _ := json.Marshal(RemoteFileRequest{
|
bRequest, _ := json.Marshal(RemoteFileRequest{
|
||||||
CurrentFileChunks: c.CurrentFileChunks,
|
CurrentFileChunkRanges: c.CurrentFileChunkRanges,
|
||||||
FilesToTransferCurrentNum: c.FilesToTransferCurrentNum,
|
FilesToTransferCurrentNum: c.FilesToTransferCurrentNum,
|
||||||
})
|
})
|
||||||
|
log.Debug("converting to chunk range")
|
||||||
|
c.CurrentFileChunks = utils.ChunkRangesToChunks(c.CurrentFileChunkRanges)
|
||||||
|
|
||||||
|
// setup the progressbar
|
||||||
|
c.setBar()
|
||||||
|
|
||||||
log.Debugf("sending recipient ready with %d chunks",len(c.CurrentFileChunks))
|
log.Debugf("sending recipient ready with %d chunks",len(c.CurrentFileChunks))
|
||||||
err = message.Send(c.conn[0], c.Key, message.Message{
|
err = message.Send(c.conn[0], c.Key, message.Message{
|
||||||
Type: "recipientready",
|
Type: "recipientready",
|
||||||
|
|
|
@ -3,6 +3,7 @@ package message
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
|
||||||
|
log "github.com/cihub/seelog"
|
||||||
"github.com/schollz/croc/v6/src/comm"
|
"github.com/schollz/croc/v6/src/comm"
|
||||||
"github.com/schollz/croc/v6/src/compress"
|
"github.com/schollz/croc/v6/src/compress"
|
||||||
"github.com/schollz/croc/v6/src/crypt"
|
"github.com/schollz/croc/v6/src/crypt"
|
||||||
|
@ -27,6 +28,7 @@ func Send(c *comm.Comm, key crypt.Encryption, m Message) (err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
log.Debugf("writing %s message (%d bytes)",m.Type,len(mSend))
|
||||||
_, err = c.Write(mSend)
|
_, err = c.Write(mSend)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -41,7 +41,7 @@ func GetInput(prompt string) string {
|
||||||
|
|
||||||
// HashFile returns the hash of a file
|
// HashFile returns the hash of a file
|
||||||
func HashFile(fname string) (hash256 []byte, err error) {
|
func HashFile(fname string) (hash256 []byte, err error) {
|
||||||
return IMOHashFile(fname)
|
return XXHashFile(fname)
|
||||||
}
|
}
|
||||||
|
|
||||||
func MD5HashFile(fname string) (hash256 []byte, err error) {
|
func MD5HashFile(fname string) (hash256 []byte, err error) {
|
||||||
|
@ -145,7 +145,7 @@ func ByteCountDecimal(b int64) string {
|
||||||
// MissingChunks returns the positions of missing chunks.
|
// MissingChunks returns the positions of missing chunks.
|
||||||
// If file doesn't exist, it returns an empty chunk list (all chunks).
|
// If file doesn't exist, it returns an empty chunk list (all chunks).
|
||||||
// If the file size is not the same as requested, it returns an empty chunk list (all chunks).
|
// If the file size is not the same as requested, it returns an empty chunk list (all chunks).
|
||||||
func MissingChunks(fname string, fsize int64, chunkSize int) (chunks []int64) {
|
func MissingChunks(fname string, fsize int64, chunkSize int) (chunkRanges []int64) {
|
||||||
fstat, err := os.Stat(fname)
|
fstat, err := os.Stat(fname)
|
||||||
if fstat.Size() != fsize || err != nil {
|
if fstat.Size() != fsize || err != nil {
|
||||||
return
|
return
|
||||||
|
@ -159,7 +159,7 @@ func MissingChunks(fname string, fsize int64, chunkSize int) (chunks []int64) {
|
||||||
|
|
||||||
emptyBuffer := make([]byte, chunkSize)
|
emptyBuffer := make([]byte, chunkSize)
|
||||||
chunkNum := 0
|
chunkNum := 0
|
||||||
chunks = make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))
|
chunks := make([]int64, int64(math.Ceil(float64(fsize)/float64(chunkSize))))
|
||||||
var currentLocation int64
|
var currentLocation int64
|
||||||
for {
|
for {
|
||||||
buffer := make([]byte, chunkSize)
|
buffer := make([]byte, chunkSize)
|
||||||
|
@ -174,9 +174,38 @@ func MissingChunks(fname string, fsize int64, chunkSize int) (chunks []int64) {
|
||||||
currentLocation += int64(bytesread)
|
currentLocation += int64(bytesread)
|
||||||
}
|
}
|
||||||
if chunkNum == 0 {
|
if chunkNum == 0 {
|
||||||
chunks = []int64{}
|
chunkRanges = []int64{}
|
||||||
} else {
|
} else {
|
||||||
chunks = chunks[:chunkNum]
|
chunks = chunks[:chunkNum]
|
||||||
|
chunkRanges = []int64{int64(chunkSize), chunks[0]}
|
||||||
|
curCount := 0
|
||||||
|
for i, chunk := range chunks {
|
||||||
|
if i == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
curCount++
|
||||||
|
if chunk-chunks[i-1] > int64(chunkSize) {
|
||||||
|
chunkRanges = append(chunkRanges, int64(curCount))
|
||||||
|
chunkRanges = append(chunkRanges, chunk)
|
||||||
|
curCount = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chunkRanges = append(chunkRanges, int64(curCount+1))
|
||||||
|
chunks = chunkRanges
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func ChunkRangesToChunks(chunkRanges []int64) (chunks []int64) {
|
||||||
|
if len(chunkRanges) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
chunkSize := chunkRanges[0]
|
||||||
|
chunks = []int64{}
|
||||||
|
for i := 1; i < len(chunkRanges); i += 2 {
|
||||||
|
for j := int64(0); j < (chunkRanges[i+1]); j++ {
|
||||||
|
chunks = append(chunks, chunkRanges[i]+j*chunkSize)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
@ -63,3 +64,36 @@ func TestSHA256(t *testing.T) {
|
||||||
func TestByteCountDecimal(t *testing.T) {
|
func TestByteCountDecimal(t *testing.T) {
|
||||||
assert.Equal(t, "10.0 kB", ByteCountDecimal(10000))
|
assert.Equal(t, "10.0 kB", ByteCountDecimal(10000))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMissingChunks(t *testing.T) {
|
||||||
|
fileSize := 100
|
||||||
|
chunkSize := 10
|
||||||
|
rand.Seed(1)
|
||||||
|
bigBuff := make([]byte, fileSize)
|
||||||
|
rand.Read(bigBuff)
|
||||||
|
ioutil.WriteFile("missing.test", bigBuff, 0644)
|
||||||
|
empty := make([]byte, chunkSize)
|
||||||
|
f, err := os.OpenFile("missing.test", os.O_RDWR, 0644)
|
||||||
|
assert.Nil(t, err)
|
||||||
|
for block := 0; block < fileSize/chunkSize; block++ {
|
||||||
|
if block == 0 || block == 4 || block == 5 || block >= 7 {
|
||||||
|
f.WriteAt(empty, int64(block*chunkSize))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.Close()
|
||||||
|
|
||||||
|
chunkRanges := MissingChunks("missing.test", int64(fileSize), chunkSize)
|
||||||
|
assert.Equal(t, []int64{10, 0, 1, 40, 2, 70, 3}, chunkRanges)
|
||||||
|
|
||||||
|
chunks := ChunkRangesToChunks(chunkRanges)
|
||||||
|
assert.Equal(t, []int64{0, 40, 50, 70, 80, 90}, chunks)
|
||||||
|
|
||||||
|
os.Remove("missing.test")
|
||||||
|
}
|
||||||
|
|
||||||
|
// func Test1(t *testing.T) {
|
||||||
|
// chunkRanges := MissingChunks("../../m/bigfile.test", int64(75000000), 1024*64/2)
|
||||||
|
// fmt.Println(chunkRanges)
|
||||||
|
// fmt.Println(ChunkRangesToChunks((chunkRanges)))
|
||||||
|
// assert.Nil(t, nil)
|
||||||
|
// }
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue