diff --git a/README.md b/README.md index 25352a1..3c94d5e 100644 --- a/README.md +++ b/README.md @@ -1345,15 +1345,23 @@ sh.mkdirp('/test/mkdirp', function(err) { Rsync copies files locally on the current host (currently no remote functionality). The srcPath can be either a file or a directory, with the destPath being the destination directory. If the destination directory does not exist, it will be created. -Options object can currently have the following attributes: +Options object can currently have the following attributes (and their default values): recursive: true //default 'false' size: 5 //default 750. File chunk size in Kb. +checksum: false //default 'false'. False will skip files if their size AND modified times are the same (regardless of content difference) Example: ```javascript -sh.rsync('/test', '/test2', { recursive: true }, function(err) { +fs.writeFile('/1.txt','This is my file.', 'utf8', function(err) { if(err) throw err; + shell.rsync('/1.txt', '/test', { size: 5 }, function(err) { + if(err) throw err; + fs.readFile('/test/1.txt', 'utf8', function(err, data){ + if(err) throw err; + //data will equal 'This is my file.' + }); + }); }); ``` \ No newline at end of file diff --git a/src/rsync.js b/src/rsync.js index f9554ae..9bf7141 100644 --- a/src/rsync.js +++ b/src/rsync.js @@ -95,10 +95,14 @@ define(function(require) { callback = opts; options = {}; options.size = 750; + options.checksum = false; + } + else { + options = opts || {}; + options.size = options.size || 750; + options.checksum = options.checksum || false; + callback = callback || function() {}; } - options = opts || {}; - options.size = options.size || 750; - callback = callback || function() {}; if(srcPath === null || srcPath === '/' || srcPath === '') { callback (new Errors.EINVAL('invalid source path')); return; @@ -111,7 +115,7 @@ define(function(require) { callback(err); return; } - if(stats.type === 'DIRECTORY') { + if(stats.isDirectory()) { self.fs.readdir(path, function(err, entries) { if(err) { callback(err); @@ -128,9 +132,10 @@ define(function(require) { var entry = { path: Path.basename(name), modified: stats.mtime, + size: stats.size, type: stats.type }; - if(options.recursive && stats.type === 'DIRECTORY') { + if(options.recursive && stats.isDirectory()) { getSrcList(Path.join(srcPath, entry.path), function(error, items) { if(error) { callback(error); @@ -140,7 +145,7 @@ define(function(require) { result.push(entry); callback(); }); - } else if(stats.type === 'FILE') { + } else if(stats.isFile()) { result.push(entry); callback(); } else { @@ -158,6 +163,7 @@ define(function(require) { var entry = { path: Path.basename(path), modified: stats.mtime, + size: stats.size, type: stats.type }; result.push(entry); @@ -166,7 +172,7 @@ define(function(require) { }); } - function getChecksums(destPath, srcList, callback){ + function getChecksums(destPath, srcList, callback) { var result = []; function getDirChecksums(entry, callback) { var item = { path: entry.path }; @@ -180,18 +186,38 @@ define(function(require) { result.push(item); callback(); }); - } else if(entry.type === 'FILE'){ - checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) { - if(err) { - callback(err); - return; - } - item.checksum = checksums; - result.push(item); - callback(); - }); + } else if(entry.type === 'FILE') { + if(options.checksum === false) { + self.fs.stat(Path.join(destPath, entry.path), function(err, stat) { + if(!err && stat.mtime === entry.modified && stat.size === entry.size) { + callback(); + } + else { + checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) { + if(err) { + callback(err); + return; + } + item.checksum = checksums; + result.push(item); + callback(); + }); + } + }); + } + else { + checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) { + if(err) { + callback(err); + return; + } + item.checksum = checksums; + result.push(item); + callback(); + }); + } } - else{ + else { callback(); } } @@ -200,23 +226,27 @@ define(function(require) { }); } - getSrcList(srcPath, function(err, result){ + getSrcList(srcPath, function(err, result) { if(err) { callback(err); return; } - self.mkdirp(destPath, function(err){ - getChecksums(destPath, result, function(err, result){ - if(err){ + self.mkdirp(destPath, function(err) { + getChecksums(destPath, result, function(err, result) { + if(err) { callback(err); return; } + else if (result.length === 0) { + callback(); + return; + } diff.call(self, srcPath, result, function(err, diffs) { - if(err){ + if(err) { callback(err); return; } - sync.call(self, destPath, diffs, function(err){ + sync.call(self, destPath, diffs, function(err) { callback(err); }); }); @@ -228,11 +258,11 @@ define(function(require) { function checksum (path, callback) { var self = this; self.fs.readFile(path, function (err, data) { - if (!err){ + if (!err) { // cache file cache[path] = data; } - else if (err && err.code === 'ENOENT'){ + else if (err && err.code === 'ENOENT') { cache[path] = []; } else { @@ -265,12 +295,10 @@ define(function(require) { function diff(path, checksums, callback) { var self = this; - if (!checksums.length) - return callback(new Error('attribute does not exist'), null); // roll through the file var diffs = []; - self.fs.stat(path, function(err, stat){ - if(stat.type ==='DIRECTORY') { + self.fs.stat(path, function(err, stat) { + if(stat.isDirectory()) { async.each(checksums, getDiff, function(err) { callback(err, diffs); }); @@ -316,7 +344,7 @@ define(function(require) { function sync(path, diff, callback) { var self = this; - function syncEach(entry, callback){ + function syncEach(entry, callback) { //get slice of raw file from block's index function rawslice(index) { @@ -326,7 +354,7 @@ define(function(require) { } if(entry.hasOwnProperty('contents')) { - sync.call(self, Path.join(path, entry.path), entry.contents, function(err){ + sync.call(self, Path.join(path, entry.path), entry.contents, function(err) { if(err) { callback(err); return; @@ -338,7 +366,7 @@ define(function(require) { var raw = cache[Path.join(path,entry.path)]; var i = 0; var len = entry.diff.length; - if(typeof raw === 'undefined'){ + if(typeof raw === 'undefined') { return callback(new Error('must do checksum() first'), null); } @@ -355,8 +383,8 @@ define(function(require) { } } delete cache[Path.join(path,entry.path)]; - self.fs.writeFile(Path.join(path,entry.path), buf, function(err){ - if(err){ + self.fs.writeFile(Path.join(path,entry.path), buf, function(err) { + if(err) { callback(err); return; } diff --git a/tests/spec/shell/rsync.spec.js b/tests/spec/shell/rsync.spec.js index 8bb0bf8..09e75ed 100644 --- a/tests/spec/shell/rsync.spec.js +++ b/tests/spec/shell/rsync.spec.js @@ -52,7 +52,7 @@ define(["Filer", "util"], function(Filer, util) { }); }); - it('should succeed if the source file is altered in content but not length from the destination file. (Destination edited)', function(done) { + it('should succeed if the source file is different in content but not length from the destination file. (Destination edited)', function(done) { var fs = util.fs(); var shell = fs.Shell(); @@ -60,11 +60,11 @@ define(["Filer", "util"], function(Filer, util) { expect(err).to.not.exist; fs.writeFile('/1.txt','This is my file. It does not have any typos.','utf8',function(err) { expect(err).to.not.exist; - fs.writeFile('/test/1.txt','This is my fivth file. It doez not have any topos,', 'utf8', function(err) { + fs.writeFile('/test/1.txt','This iz mi fiel. It doez not have any topos,', 'utf8', function(err) { expect(err).to.not.exist; shell.rsync('/1.txt', '/test', { size: 5 }, function(err) { expect(err).to.not.exist; - fs.readFile('/test/1.txt', 'utf8', function(err, data){ + fs.readFile('/test/1.txt', 'utf8', function(err, data) { expect(err).to.not.exist; expect(data).to.exist; expect(data).to.equal('This is my file. It does not have any typos.'); @@ -145,6 +145,83 @@ define(["Filer", "util"], function(Filer, util) { }); }); + it('should succeed if no options are provided', function(done) { + var fs = util.fs(); + var shell = fs.Shell(); + + fs.mkdir('/test', function(err) { + expect(err).to.not.exist; + fs.writeFile('/1.txt','This is my file. It does not exist in the destination folder.', 'utf8', function(err) { + expect(err).to.not.exist; + shell.rsync('/1.txt', '/test', function(err) { + expect(err).to.not.exist; + fs.readFile('/test/1.txt', 'utf8', function(err, data){ + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('This is my file. It does not exist in the destination folder.'); + done(); + }); + }); + }); + }); + }); + + it('should do nothing if the source file and destination file have the same mtime and size with \'checksum = false\' flag (Default)', function(done){ + var fs = util.fs(); + var shell = fs.Shell(); + var date = Date.parse('1 Oct 2000 15:33:22'); + fs.mkdir('/test', function(err) { + expect(err).to.not.exist; + fs.writeFile('/1.txt', 'This is a file.', 'utf8', function(err) { + expect(err).to.not.exist; + fs.writeFile('/test/1.txt', 'Different file.', 'utf8', function(err) { + expect(err).to.not.exist; + fs.utimes('/1.txt', date, date, function(err) { + expect(err).to.not.exist; + fs.utimes('/test/1.txt', date, date, function(err) { + expect(err).to.not.exist; + shell.ls('/', {recursive: true}, function(err, stuff){ + shell.rsync('/1.txt', '/test', {size: 5, checksum: false }, function(err) { + expect(err).to.not.exist; + fs.readFile('/test/1.txt', 'utf8', function(err, data) { + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('Different file.'); + done(); + }); + }); + }); + }); + }); + }); + }); + }); + }); + + it('should suceed if the source file and destination file have the same mtime and size with \'checksum = true\' flag', function(done){ + var fs = util.fs(); + var shell = fs.Shell(); + + fs.mkdir('/test', function(err) { + expect(err).to.not.exist; + fs.writeFile('/1.txt', 'This is a file.', 'utf8', function(err) { + expect(err).to.not.exist; + fs.writeFile('/test/1.txt', 'Different file.', 'utf8', function(err) { + expect(err).to.not.exist; + shell.rsync('/1.txt', '/test', {size: 5, checksum: true }, function(err) { + expect(err).to.not.exist; + fs.readFile('/test/1.txt', 'utf8', function(err, data) { + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('This is a file.'); + done(); + }); + }); + }); + }); + }); + }); + it('should succeed if the destination folder does not exist (Destination file created)', function(done) { var fs = util.fs(); var shell = fs.Shell(); @@ -224,79 +301,59 @@ define(["Filer", "util"], function(Filer, util) { }); }); - it('should succeed syncing a directory recursively (recursive: true)', function(done) { + it('should succeed syncing a directory recursively, skipping same-size and time files (recursive: true)', function(done) { var fs = util.fs(); var shell = fs.Shell(); - fs.mkdir('/test', function(err) { + var date = Date.parse('1 Oct 2000 15:33:22'); + + shell.mkdirp('/test/sync', function(err){ expect(err).to.not.exist; - fs.mkdir('/test2', function(err) { + shell.mkdirp('/test2/sync', function(err){ expect(err).to.not.exist; - fs.mkdir('/test/sync', function(err) { + fs.writeFile('/test/1.txt','This is my 1st file.', 'utf8', function(err) { expect(err).to.not.exist; - fs.mkdir('/test2/sync', function(err) { + fs.writeFile('/test/sync/2.txt','This is my 2nd file.', 'utf8', function(err) { expect(err).to.not.exist; - - fs.writeFile('/test/1.txt','This is my 1st file. It does not have any typos.', 'utf8', function(err) { + fs.writeFile('/test/sync/3.txt','This is my 3rd file.', 'utf8', function(err) { expect(err).to.not.exist; - fs.writeFile('/test/2.txt','This is my 2nd file. It is longer than the destination file.', 'utf8', function(err) { + fs.writeFile('/test2/sync/3.txt','This shouldn\'t sync.', 'utf8', function(err) { expect(err).to.not.exist; - fs.writeFile('/test/sync/3.txt','This is my 3rd file.', 'utf8', function(err) { - expect(err).to.not.exist; - fs.writeFile('/test/sync/5.txt','This is my 5th file. It does not exist in the destination folder.', 'utf8', function(err) { - expect(err).to.not.exist; - fs.writeFile('/test2/1.txt','This is my 1st file. It doez not have any topos,', 'utf8', function(err) { - expect(err).to.not.exist; - fs.writeFile('/test2/2.txt','This is my 2nd file.', 'utf8', function(err) { - expect(err).to.not.exist; - fs.writeFile('/test2/sync/3.txt','This is my 3rd file. It is longer than the source version.', 'utf8', function(err) { - expect(err).to.not.exist; - fs.writeFile('/test2/sync/4.txt','This is my 4th file. It does not exist in the source folder.', 'utf8', function(err) { - expect(err).to.not.exist; - - shell.rsync('/test', '/test2', { recursive: true, size: 5 }, function(err) { - expect(err).to.not.exist; - fs.readFile('/test2/1.txt', 'utf8', function(err, data){ - expect(err).to.not.exist; - expect(data).to.exist; - expect(data).to.equal('This is my 1st file. It does not have any typos.'); - fs.readFile('/test2/2.txt', 'utf8', function(err, data){ - expect(err).to.not.exist; - expect(data).to.exist; - expect(data).to.equal('This is my 2nd file. It is longer than the destination file.'); - fs.readFile('/test2/sync/3.txt', 'utf8', function(err, data){ - expect(err).to.not.exist; - expect(data).to.exist; - expect(data).to.equal('This is my 3rd file.') - fs.readFile('/test2/sync/4.txt', 'utf8', function(err, data){ - expect(err).to.not.exist; - expect(data).to.exist; - expect(data).to.equal('This is my 4th file. It does not exist in the source folder.') - fs.readFile('/test2/sync/5.txt', 'utf8', function(err, data){ - expect(err).to.not.exist; - expect(data).to.exist; - expect(data).to.equal('This is my 5th file. It does not exist in the destination folder.') - done(); - }); - }); - }); - }); - }); - }); + fs.utimes('/test/sync/3.txt', date, date, function(err) { + expect(err).to.not.exist; + fs.utimes('/test2/sync/3.txt', date, date, function(err) { + expect(err).to.not.exist; + + shell.rsync('/test', '/test2', { recursive: true, size: 5 }, function(err) { + expect(err).to.not.exist; + fs.readFile('/test2/1.txt', 'utf8', function(err, data){ + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('This is my 1st file.'); + fs.readFile('/test2/sync/2.txt', 'utf8', function(err, data){ + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('This is my 2nd file.'); + fs.readFile('/test2/sync/3.txt', 'utf8', function(err, data){ + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('This shouldn\'t sync.') + + done(); }); }); - }); + }); }); - }); - }); + + }); + }); + }); }); - }); }); }); }); - }); });