diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..173454b --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# Created by .ignore support plugin (hsz.mobi) +### Example user template template +### Example user template + +# IntelliJ project files +.idea +*.iml +out +gen diff --git a/matcher.go b/matcher.go new file mode 100644 index 0000000..eb05c28 --- /dev/null +++ b/matcher.go @@ -0,0 +1,33 @@ +package grsync + +import ( + "regexp" +) + +type matcher struct { + regExp *regexp.Regexp +} + +func (m matcher) Match(data string) bool { + return m.regExp.Match([]byte(data)) +} + +func (m matcher) Extract(data string) string { + const submatchCount = 1 + matches := m.regExp.FindAllStringSubmatch(data, submatchCount) + if len(matches) == 0 || len(matches[0]) < 2 { + return "" + } + + return matches[0][1] +} + +func (m matcher) ExtractAllStringSubmatch(data string, submatchCount int) [][]string { + return m.regExp.FindAllStringSubmatch(data, submatchCount) +} + +func newMatcher(regExpString string) *matcher { + return &matcher{ + regExp: regexp.MustCompile(regExpString), + } +} diff --git a/rsync.go b/rsync.go new file mode 100644 index 0000000..825f00c --- /dev/null +++ b/rsync.go @@ -0,0 +1,554 @@ +package grsync + +import ( + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" +) + +// Rsync is wrapper under rsync +type Rsync struct { + Source string + Destination string + + cmd *exec.Cmd +} + +// RsyncOptions for rsync +type RsyncOptions struct { + // Verbose increase verbosity + Verbose bool + // Quet suppress non-error messages + Quiet bool + // Checksum skip based on checksum, not mod-time & size + Checksum bool + // Archve is archive mode; equals -rlptgoD (no -H,-A,-X) + Archive bool + // Recurse into directories + Recursive bool + // Relative option to use relative path names + Relative bool + // NoImliedDirs don't send implied dirs with --relative + NoImpliedDirs bool + // Update skip files that are newer on the receiver + Update bool + // Inplace update destination files in-place + Inplace bool + // Append data onto shorter files + Append bool + // AppendVerify --append w/old data in file checksum + AppendVerify bool + // Dirs transfer directories without recursing + Dirs bool + // Links copy symlinks as symlinks + Links bool + // CopyLinks transform symlink into referent file/dir + CopyLinks bool + // CopyUnsafeLinks only "unsafe" symlinks are transformed + CopyUnsafeLinks bool + // SafeLinks ignore symlinks that point outside the tree + SafeLinks bool + // CopyDirLinks transform symlink to dir into referent dir + CopyDirLinks bool + // KeepDirLinks treat symlinked dir on receiver as dir + KeepDirLinks bool + // HardLinks preserve hard links + HardLinks bool + // Perms preserve permissions + Perms bool + // Executability preserve executability + Executability bool + // CHMOD affect file and/or directory permissions + CHMOD os.FileMode + // Acls preserve ACLs (implies -p) + ACLs bool + // XAttrs preserve extended attributes + XAttrs bool + // Owner preserve owner (super-user only) + Owner bool + // Group preserve group + Group bool + // Devices preserve device files (super-user only) + Devices bool + // Specials preserve special files + Specials bool + // Times preserve modification times + Times bool + // omit directories from --times + OmitDirTimes bool + // Super receiver attempts super-user activities + Super bool + // FakeSuper store/recover privileged attrs using xattrs + FakeSuper bool + // Sparce handle sparse files efficiently + Sparse bool + // DryRun perform a trial run with no changes made + DryRun bool + // WholeFile copy files whole (w/o delta-xfer algorithm) + WholeFile bool + // OneFileSystem don't cross filesystem boundaries + OneFileSystem bool + // BlockSize block-size=SIZE force a fixed checksum block-size + BlockSize int + // Rsh -rsh=COMMAND specify the remote shell to use + Rsh string + // RsyncProgramm rsync-path=PROGRAM specify the rsync to run on remote machine + RsyncProgramm string + // Existing skip creating new files on receiver + Existing bool + // IgnoreExisting skip updating files that exist on receiver + IgnoreExisting bool + // RemoveSourceFiles sender removes synchronized files (non-dir) + RemoveSourceFiles bool + // Delete delete extraneous files from dest dirs + Delete bool + // DeleteBefore receiver deletes before transfer, not during + DeleteBefore bool + // DeleteDuring receiver deletes during the transfer + DeleteDuring bool + // DeleteDelay find deletions during, delete after + DeleteDelay bool + // DeleteAfter receiver deletes after transfer, not during + DeleteAfter bool + // DeleteExcluded also delete excluded files from dest dirs + DeleteExcluded bool + // IgnoreErrors delete even if there are I/O errors + IgnoreErrors bool + // Force deletion of dirs even if not empty + Force bool + // MaxDelete max-delete=NUM don't delete more than NUM files + MaxDelete int + // MaxSize max-size=SIZE don't transfer any file larger than SIZE + MaxSize int + // MinSize don't transfer any file smaller than SIZE + MinSize int + // Partial keep partially transferred files + Partial bool + // PartialDir partial-dir=DIR + PartialDir string + // DelayUpdates put all updated files into place at end + DelayUpdates bool + // PruneEmptyDirs prune empty directory chains from file-list + PruneEmptyDirs bool + // NumericIDs don't map uid/gid values by user/group name + NumericIDs bool + // Timeout timeout=SECONDS set I/O timeout in seconds + Timeout int + // Contimeout contimeout=SECONDS set daemon connection timeout in seconds + Contimeout int + // IgnoreTimes don't skip files that match size and time + IgnoreTimes bool + // SizeOnly skip files that match in size + SizeOnly bool + // ModifyWindow modify-window=NUM compare mod-times with reduced accuracy + ModifyWindow bool + // TempDir temp-dir=DIR create temporary files in directory DIR + TempDir string + // Fuzzy find similar file for basis if no dest file + Fuzzy bool + // CompareDest compare-dest=DIR also compare received files relative to DIR + CompareDest string + // CopyDest copy-dest=DIR ... and include copies of unchanged files + CopyDest string + // LinkDest link-dest=DIR hardlink to files in DIR when unchanged + LinkDest string + // Compress file data during the transfer + Compress bool + // CompressLevel explicitly set compression level + CompressLevel int + // SkipCompress skip-compress=LIST skip compressing files with suffix in LIST + SkipCompress []string + // CVSExclude auto-ignore files in the same way CVS does + CVSExclude bool + // Stats give some file-transfer stats + Stats bool + // HumanReadable output numbers in a human-readable format + HumanReadable bool + // Progress show progress during transfer + Progress bool + // 端口 + Port int + // 密钥文件 + PasswordFile string + // Info + Info string + + // ipv4 + IPv4 bool + // ipv6 + IPv6 bool +} + +// StdoutPipe returns a pipe that will be connected to the command's +// standard output when the command starts. +func (r Rsync) StdoutPipe() (io.ReadCloser, error) { + return r.cmd.StdoutPipe() +} + +// StderrPipe returns a pipe that will be connected to the command's +// standard error when the command starts. +func (r Rsync) StderrPipe() (io.ReadCloser, error) { + return r.cmd.StderrPipe() +} + +// Run start rsync task +func (r Rsync) Run() error { + if !isExist(r.Destination) { + if err := createDir(r.Destination); err != nil { + return err + } + } + + if err := r.cmd.Start(); err != nil { + return err + } + + return r.cmd.Wait() +} + +// NewRsync returns task with described options +func NewRsync(source, destination string, options RsyncOptions) *Rsync { + arguments := append(getArguments(options), source, destination) + return &Rsync{ + Source: source, + Destination: destination, + cmd: exec.Command("rsync", arguments...), + } +} + +func getArguments(options RsyncOptions) []string { + arguments := []string{} + if options.Verbose { + arguments = append(arguments, "--verbose") + } + + if options.Checksum { + arguments = append(arguments, "--checksum") + } + + if options.Quiet { + arguments = append(arguments, "--quiet") + } + + if options.Archive { + arguments = append(arguments, "--archive") + } + + if options.Recursive { + arguments = append(arguments, "--recursive") + } + + if options.Relative { + arguments = append(arguments, "--relative") + } + + if options.NoImpliedDirs { + arguments = append(arguments, "--no-implied-dirs") + } + + if options.Update { + arguments = append(arguments, "--update") + } + + if options.Inplace { + arguments = append(arguments, "--inplace") + } + + if options.Append { + arguments = append(arguments, "--append") + } + + if options.AppendVerify { + arguments = append(arguments, "--append-verify") + } + + if options.Dirs { + arguments = append(arguments, "--dirs") + } + + if options.Links { + arguments = append(arguments, "--links") + } + + if options.CopyLinks { + arguments = append(arguments, "--copy-links") + } + + if options.CopyUnsafeLinks { + arguments = append(arguments, "--copy-unsafe-links") + } + + if options.SafeLinks { + arguments = append(arguments, "--safe-links") + } + + if options.CopyDirLinks { + arguments = append(arguments, "--copy-dir-links") + } + + if options.KeepDirLinks { + arguments = append(arguments, "--keep-dir-links") + } + + if options.HardLinks { + arguments = append(arguments, "--hard-links") + } + + if options.Perms { + arguments = append(arguments, "--perms") + } + + if options.Executability { + arguments = append(arguments, "--executability") + } + + if options.ACLs { + arguments = append(arguments, "--acls") + } + + if options.XAttrs { + arguments = append(arguments, "--xattrs") + } + + if options.Owner { + arguments = append(arguments, "--owner") + } + + if options.Group { + arguments = append(arguments, "--group") + } + + if options.Devices { + arguments = append(arguments, "--devices") + } + + if options.Port > 0 { + arguments = append(arguments, fmt.Sprintf("--port=%d", options.Port)) + } + + if options.Specials { + arguments = append(arguments, "--specials") + } + + if options.Times { + arguments = append(arguments, "--times") + } + + if options.OmitDirTimes { + arguments = append(arguments, "--omit-dir-times") + } + + if options.Super { + arguments = append(arguments, "--super") + } + + if options.FakeSuper { + arguments = append(arguments, "--fake-super") + } + + if options.Sparse { + arguments = append(arguments, "--sparse") + } + + if options.DryRun { + arguments = append(arguments, "--dry-run") + } + + if options.WholeFile { + arguments = append(arguments, "--whole-file") + } + + if options.OneFileSystem { + arguments = append(arguments, "--one-file-system") + } + + if options.BlockSize > 0 { + arguments = append(arguments, "--block-size", strconv.Itoa(options.BlockSize)) + } + + if options.Rsh != "" { + arguments = append(arguments, "--rsh", options.Rsh) + } + + if options.RsyncProgramm != "" { + arguments = append(arguments, "--rsync-programm", options.RsyncProgramm) + } + + if options.Existing { + arguments = append(arguments, "--existing") + } + + if options.IgnoreExisting { + arguments = append(arguments, "--ignore-existing") + } + + if options.RemoveSourceFiles { + arguments = append(arguments, "--remove-source-files") + } + + if options.Delete { + arguments = append(arguments, "--delete") + } + + if options.DeleteBefore { + arguments = append(arguments, "--delete-before") + } + + if options.DeleteDuring { + arguments = append(arguments, "--delete-during") + } + + if options.DeleteDelay { + arguments = append(arguments, "--delete-delay") + } + + if options.DeleteAfter { + arguments = append(arguments, "--delete-after") + } + + if options.DeleteExcluded { + arguments = append(arguments, "--delete-excluded") + } + + if options.IgnoreErrors { + arguments = append(arguments, "--ignore-errors") + } + + if options.Force { + arguments = append(arguments, "--force") + } + + if options.MaxDelete > 0 { + arguments = append(arguments, "--max-delete", strconv.Itoa(options.MaxDelete)) + } + + if options.MaxSize > 0 { + arguments = append(arguments, "--max-size", strconv.Itoa(options.MaxSize)) + } + + if options.MinSize > 0 { + arguments = append(arguments, "--min-size", strconv.Itoa(options.MinSize)) + } + + if options.Partial { + arguments = append(arguments, "--partial") + } + + if options.PartialDir != "" { + arguments = append(arguments, "--partial-dir", options.PartialDir) + } + + if options.DelayUpdates { + arguments = append(arguments, "--delay-updates") + } + + if options.PruneEmptyDirs { + arguments = append(arguments, "--prune-empty-dirs") + } + + if options.NumericIDs { + arguments = append(arguments, "--numeric-ids") + } + + if options.Timeout > 0 { + arguments = append(arguments, "--timeout", strconv.Itoa(options.Timeout)) + } + + if options.Contimeout > 0 { + arguments = append(arguments, "--contimeout", strconv.Itoa(options.Contimeout)) + } + + if options.IgnoreTimes { + arguments = append(arguments, "--ignore-times") + } + + if options.SizeOnly { + arguments = append(arguments, "--size-only") + } + + if options.ModifyWindow { + arguments = append(arguments, "--modify-window") + } + + if options.TempDir != "" { + arguments = append(arguments, "--temp-dir", options.TempDir) + } + + if options.Fuzzy { + arguments = append(arguments, "--fuzzy") + } + + if options.CompareDest != "" { + arguments = append(arguments, "--compare-dest", options.CompareDest) + } + + if options.CopyDest != "" { + arguments = append(arguments, "--copy-dest", options.CopyDest) + } + + if options.LinkDest != "" { + arguments = append(arguments, "--link-dest", options.LinkDest) + } + + if options.Compress { + arguments = append(arguments, "--compress") + } + + if options.CompressLevel > 0 { + arguments = append(arguments, "--compress-level", strconv.Itoa(options.CompressLevel)) + } + + if len(options.SkipCompress) > 0 { + arguments = append(arguments, "--skip-compress", strings.Join(options.SkipCompress, ",")) + } + + if options.CVSExclude { + arguments = append(arguments, "--cvs-exclude") + } + + if options.Stats { + arguments = append(arguments, "--stats") + } + + if options.HumanReadable { + arguments = append(arguments, "--human-readable") + } + + if options.Progress { + arguments = append(arguments, "--progress") + } + + if options.IPv4 { + arguments = append(arguments, "--ipv4") + } + + if options.IPv6 { + arguments = append(arguments, "--ipv6") + } + + if options.Info != "" { + arguments = append(arguments, "--info", options.Info) + } + + if options.PasswordFile != "" { + arguments = append(arguments, "--password-file", options.PasswordFile) + } + + return arguments +} + +func createDir(dir string) error { + cmd := exec.Command("mkdir", "-p", dir) + if err := cmd.Start(); err != nil { + return err + } + return cmd.Wait() +} + +func isExist(p string) bool { + stat, err := os.Stat(p) + return os.IsExist(err) && stat.IsDir() +} diff --git a/task.go b/task.go new file mode 100644 index 0000000..3fdb191 --- /dev/null +++ b/task.go @@ -0,0 +1,140 @@ +package grsync + +import ( + "bufio" + "io" + "math" + "strconv" + "strings" +) + +// Task is high-level API under rsync +type Task struct { + rsync *Rsync + + state *State + log *Log +} + +// State contains information about rsync process +type State struct { + Remain int `json:"remain"` + Total int `json:"total"` + Speed string `json:"speed"` + Progress float64 `json:"progress"` +} + +// Log contains raw stderr and stdout outputs +type Log struct { + Stderr string `json:"stderr"` + Stdout string `json:"stdout"` +} + +// State returns inforation about rsync processing task +func (t Task) State() State { + return *t.state +} + +// Log return structure which contains raw stderr and stdout outputs +func (t Task) Log() Log { + return Log{ + Stderr: t.log.Stderr, + Stdout: t.log.Stdout, + } +} + +// Run starts rsync process with options +func (t *Task) Run() error { + stderr, err := t.rsync.StderrPipe() + if err != nil { + return err + } + defer stderr.Close() + + stdout, err := t.rsync.StdoutPipe() + if err != nil { + return err + } + defer stdout.Close() + + go processStdout(t, stdout) + go processStderr(t, stderr) + + return t.rsync.Run() +} + +// NewTask returns new rsync task +func NewTask(source, destination string, rsyncOptions RsyncOptions) *Task { + // Force set required options + rsyncOptions.HumanReadable = true + rsyncOptions.Partial = true + rsyncOptions.Progress = true + rsyncOptions.Archive = true + + return &Task{ + rsync: NewRsync(source, destination, rsyncOptions), + state: &State{}, + log: &Log{}, + } +} + +func processStdout(task *Task, stdout io.Reader) { + const maxPercents = float64(100) + const minDivider = 1 + + progressMatcher := newMatcher(`\(.+-chk=(\d+.\d+)`) + speedMatcher := newMatcher(`(\d+\.\d+.{2}\/s)`) + + // Extract data from strings: + // 999,999 99% 999.99kB/s 0:00:59 (xfr#9, to-chk=999/9999) + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + logStr := scanner.Text() + if progressMatcher.Match(logStr) { + task.state.Remain, task.state.Total = getTaskProgress(progressMatcher.Extract(logStr)) + + copiedCount := float64(task.state.Total - task.state.Remain) + task.state.Progress = copiedCount / math.Max(float64(task.state.Total), float64(minDivider)) * maxPercents + } + + if speedMatcher.Match(logStr) { + task.state.Speed = getTaskSpeed(speedMatcher.ExtractAllStringSubmatch(logStr, 2)) + } + + task.log.Stdout += logStr + "\n" + } +} + +func processStderr(task *Task, stderr io.Reader) { + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + task.log.Stderr += scanner.Text() + "\n" + } +} + +func getTaskProgress(remTotalString string) (int, int) { + const remTotalSeparator = "/" + const numbersCount = 2 + const ( + indexRem = iota + indexTotal + ) + + info := strings.Split(remTotalString, remTotalSeparator) + if len(info) < numbersCount { + return 0, 0 + } + + remain, _ := strconv.Atoi(info[indexRem]) + total, _ := strconv.Atoi(info[indexTotal]) + + return remain, total +} + +func getTaskSpeed(data [][]string) string { + if len(data) < 2 || len(data[1]) < 2 { + return "" + } + + return data[1][1] +}