mirror of
https://github.com/schollz/croc.git
synced 2025-10-11 13:21:00 +02:00
feat: add support to respect .gitignore files
This commit is contained in:
PThorpe92
parent
f91c7a9948
commit
0af35d7149
5 changed files with 158 additions and 18 deletions
125
src/croc/croc.go
125
src/croc/croc.go
|
|
@ -20,6 +20,7 @@ import (
|
|||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/denisbrodbeck/machineid"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
log "github.com/schollz/logger"
|
||||
"github.com/schollz/pake/v3"
|
||||
"github.com/schollz/peerdiscovery"
|
||||
|
|
@ -77,6 +78,7 @@ type Options struct {
|
|||
ThrottleUpload string
|
||||
ZipFolder bool
|
||||
TestFlag bool
|
||||
GitIgnore bool
|
||||
}
|
||||
|
||||
// Client holds the state of the croc transfer
|
||||
|
|
@ -101,6 +103,7 @@ type Client struct {
|
|||
TotalNumberFolders int
|
||||
FilesToTransferCurrentNum int
|
||||
FilesHasFinished map[int]struct{}
|
||||
TotalFilesIgnored int
|
||||
|
||||
// send / receive information of current file
|
||||
CurrentFile *os.File
|
||||
|
|
@ -149,6 +152,7 @@ type FileInfo struct {
|
|||
Symlink string `json:"sy,omitempty"`
|
||||
Mode os.FileMode `json:"md,omitempty"`
|
||||
TempFile bool `json:"tf,omitempty"`
|
||||
IsIgnored bool `json:"ig,omitempty"`
|
||||
}
|
||||
|
||||
// RemoteFileRequest requests specific bytes
|
||||
|
|
@ -251,9 +255,51 @@ func isEmptyFolder(folderPath string) (bool, error) {
|
|||
return false, nil
|
||||
}
|
||||
|
||||
// helper function to walk each subfolder and parses against an ignore file.
|
||||
// returns a hashmap Key: Absolute filepath, Value: boolean (true=ignore)
|
||||
func gitWalk(dir string, gitObj *ignore.GitIgnore, files map[string]bool) {
|
||||
var ignoredDir bool
|
||||
var current string
|
||||
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isChild(current, path) && ignoredDir {
|
||||
files[path] = true
|
||||
return nil
|
||||
}
|
||||
if info.IsDir() && filepath.Base(path) == filepath.Base(dir) {
|
||||
ignoredDir = false // Skip applying ignore rules for root directory
|
||||
return nil
|
||||
}
|
||||
if gitObj.MatchesPath(info.Name()) {
|
||||
files[path] = true
|
||||
ignoredDir = true
|
||||
current = path
|
||||
return nil
|
||||
} else {
|
||||
files[path] = false
|
||||
ignoredDir = false
|
||||
return nil
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("filepath error")
|
||||
}
|
||||
}
|
||||
|
||||
func isChild(parentPath, childPath string) bool {
|
||||
relPath, err := filepath.Rel(parentPath, childPath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return !strings.HasPrefix(relPath, "..")
|
||||
|
||||
}
|
||||
|
||||
// This function retrieves the important file information
|
||||
// for every file that will be transferred
|
||||
func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyFolders []FileInfo, totalNumberFolders int, err error) {
|
||||
func GetFilesInfo(fnames []string, zipfolder bool, ignoreGit bool) (filesInfo []FileInfo, emptyFolders []FileInfo, totalNumberFolders int, err error) {
|
||||
// fnames: the relative/absolute paths of files/folders that will be transferred
|
||||
totalNumberFolders = 0
|
||||
var paths []string
|
||||
|
|
@ -271,7 +317,44 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
|
|||
paths = append(paths, fname)
|
||||
}
|
||||
}
|
||||
|
||||
var ignoredPaths = make(map[string]bool)
|
||||
if ignoreGit {
|
||||
wd, wdErr := os.Stat(".gitignore")
|
||||
if wdErr == nil {
|
||||
gitIgnore, gitErr := ignore.CompileIgnoreFile(wd.Name())
|
||||
if gitErr == nil {
|
||||
for _, path := range paths {
|
||||
abs, absErr := filepath.Abs(path)
|
||||
if absErr != nil {
|
||||
err = absErr
|
||||
return
|
||||
}
|
||||
if gitIgnore.MatchesPath(path) {
|
||||
ignoredPaths[abs] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, path := range paths {
|
||||
abs, absErr := filepath.Abs(path)
|
||||
if absErr != nil {
|
||||
err = absErr
|
||||
return
|
||||
}
|
||||
file, fileErr := os.Stat(path)
|
||||
if fileErr == nil && file.IsDir() {
|
||||
_, subErr := os.Stat(filepath.Join(path, ".gitignore"))
|
||||
if subErr == nil {
|
||||
gitObj, gitObjErr := ignore.CompileIgnoreFile(filepath.Join(path, ".gitignore"))
|
||||
if gitObjErr != nil {
|
||||
err = gitObjErr
|
||||
return
|
||||
}
|
||||
gitWalk(abs, gitObj, ignoredPaths)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, fpath := range paths {
|
||||
stat, errStat := os.Lstat(fpath)
|
||||
|
||||
|
|
@ -286,7 +369,6 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
|
|||
err = errAbs
|
||||
return
|
||||
}
|
||||
|
||||
if stat.IsDir() && zipfolder {
|
||||
if fpath[len(fpath)-1:] != "/" {
|
||||
fpath += "/"
|
||||
|
|
@ -304,7 +386,8 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
|
|||
err = errAbs
|
||||
return
|
||||
}
|
||||
filesInfo = append(filesInfo, FileInfo{
|
||||
|
||||
fInfo := FileInfo{
|
||||
Name: stat.Name(),
|
||||
FolderRemote: "./",
|
||||
FolderSource: filepath.Dir(absPath),
|
||||
|
|
@ -312,7 +395,12 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
|
|||
ModTime: stat.ModTime(),
|
||||
Mode: stat.Mode(),
|
||||
TempFile: true,
|
||||
})
|
||||
IsIgnored: ignoredPaths[absPath],
|
||||
}
|
||||
if fInfo.IsIgnored {
|
||||
continue
|
||||
}
|
||||
filesInfo = append(filesInfo, fInfo)
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -325,7 +413,7 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
|
|||
remoteFolder := strings.TrimPrefix(filepath.Dir(pathName),
|
||||
filepath.Dir(absPath)+string(os.PathSeparator))
|
||||
if !info.IsDir() {
|
||||
filesInfo = append(filesInfo, FileInfo{
|
||||
fInfo := FileInfo{
|
||||
Name: info.Name(),
|
||||
FolderRemote: strings.ReplaceAll(remoteFolder, string(os.PathSeparator), "/") + "/",
|
||||
FolderSource: filepath.Dir(pathName),
|
||||
|
|
@ -333,10 +421,19 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
|
|||
ModTime: info.ModTime(),
|
||||
Mode: info.Mode(),
|
||||
TempFile: false,
|
||||
})
|
||||
IsIgnored: ignoredPaths[pathName],
|
||||
}
|
||||
if fInfo.IsIgnored && ignoreGit {
|
||||
return nil
|
||||
} else {
|
||||
filesInfo = append(filesInfo, fInfo)
|
||||
}
|
||||
} else {
|
||||
totalNumberFolders++
|
||||
if ignoredPaths[pathName] {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
isEmptyFolder, _ := isEmptyFolder(pathName)
|
||||
totalNumberFolders++
|
||||
if isEmptyFolder {
|
||||
emptyFolders = append(emptyFolders, FileInfo{
|
||||
// Name: info.Name(),
|
||||
|
|
@ -352,7 +449,7 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
|
|||
}
|
||||
|
||||
} else {
|
||||
filesInfo = append(filesInfo, FileInfo{
|
||||
fInfo := FileInfo{
|
||||
Name: stat.Name(),
|
||||
FolderRemote: "./",
|
||||
FolderSource: filepath.Dir(absPath),
|
||||
|
|
@ -360,9 +457,14 @@ func GetFilesInfo(fnames []string, zipfolder bool) (filesInfo []FileInfo, emptyF
|
|||
ModTime: stat.ModTime(),
|
||||
Mode: stat.Mode(),
|
||||
TempFile: false,
|
||||
})
|
||||
IsIgnored: ignoredPaths[absPath],
|
||||
}
|
||||
if fInfo.IsIgnored && ignoreGit {
|
||||
continue
|
||||
} else {
|
||||
filesInfo = append(filesInfo, fInfo)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -517,7 +619,6 @@ func (c *Client) Send(filesInfo []FileInfo, emptyFoldersToTransfer []FileInfo, t
|
|||
c.TotalNumberFolders = totalNumberFolders
|
||||
c.TotalNumberOfContents = len(filesInfo)
|
||||
err = c.sendCollectFiles(filesInfo)
|
||||
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
|
@ -41,6 +42,7 @@ func TestCrocReadme(t *testing.T) {
|
|||
DisableLocal: true,
|
||||
Curve: "siec",
|
||||
Overwrite: true,
|
||||
GitIgnore: false,
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -66,7 +68,7 @@ func TestCrocReadme(t *testing.T) {
|
|||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../README.md"}, false)
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../README.md"}, false, false)
|
||||
if errGet != nil {
|
||||
t.Errorf("failed to get minimal info: %v", errGet)
|
||||
}
|
||||
|
|
@ -132,7 +134,7 @@ func TestCrocEmptyFolder(t *testing.T) {
|
|||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false)
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)
|
||||
if errGet != nil {
|
||||
t.Errorf("failed to get minimal info: %v", errGet)
|
||||
}
|
||||
|
|
@ -174,6 +176,7 @@ func TestCrocSymlink(t *testing.T) {
|
|||
DisableLocal: true,
|
||||
Curve: "siec",
|
||||
Overwrite: true,
|
||||
GitIgnore: false,
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -199,7 +202,7 @@ func TestCrocSymlink(t *testing.T) {
|
|||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false)
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{pathName}, false, false)
|
||||
if errGet != nil {
|
||||
t.Errorf("failed to get minimal info: %v", errGet)
|
||||
}
|
||||
|
|
@ -229,6 +232,32 @@ func TestCrocSymlink(t *testing.T) {
|
|||
t.Errorf("symlink transfer failed: %s", err.Error())
|
||||
}
|
||||
}
|
||||
func testCrocIgnoreGit(t *testing.T) {
|
||||
log.SetLevel("trace")
|
||||
defer os.Remove(".gitignore")
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
file, err := os.Create(".gitignore")
|
||||
if err != nil {
|
||||
log.Errorf("error creating file")
|
||||
}
|
||||
_, err = file.WriteString("LICENSE")
|
||||
if err != nil {
|
||||
log.Errorf("error writing to file")
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
// due to how files are ignored in this function, all we have to do to test is make sure LICENSE doesn't get included in FilesInfo.
|
||||
filesInfo, _, _, errGet := GetFilesInfo([]string{"../../LICENSE", ".gitignore", "croc.go"}, false, true)
|
||||
if errGet != nil {
|
||||
t.Errorf("failed to get minimal info: %v", errGet)
|
||||
}
|
||||
for _, file := range filesInfo {
|
||||
if strings.Contains(file.Name, "LICENSE") {
|
||||
t.Errorf("test failed, should ignore LICENSE")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrocLocal(t *testing.T) {
|
||||
log.SetLevel("trace")
|
||||
|
|
@ -249,6 +278,7 @@ func TestCrocLocal(t *testing.T) {
|
|||
DisableLocal: false,
|
||||
Curve: "siec",
|
||||
Overwrite: true,
|
||||
GitIgnore: false,
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -276,7 +306,7 @@ func TestCrocLocal(t *testing.T) {
|
|||
os.Create("touched")
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../LICENSE", "touched"}, false)
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{"../../LICENSE", "touched"}, false, false)
|
||||
if errGet != nil {
|
||||
t.Errorf("failed to get minimal info: %v", errGet)
|
||||
}
|
||||
|
|
@ -329,7 +359,7 @@ func TestCrocError(t *testing.T) {
|
|||
Curve: "siec",
|
||||
Overwrite: true,
|
||||
})
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false)
|
||||
filesInfo, emptyFolders, totalNumberFolders, errGet := GetFilesInfo([]string{tmpfile.Name()}, false, false)
|
||||
if errGet != nil {
|
||||
t.Errorf("failed to get minimal info: %v", errGet)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue