LumeWeb
/
filer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

RSync functional. Tests done 

Functional flags:
size
recursive
checksum
This commit is contained in:
Petr Bouianov 2014-03-31 00:20:19 -04:00
parent 8d92d421e9
commit 4572612319
3 changed files with 188 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
README.md
View File

@ -1345,15 +1345,23 @@ sh.mkdirp('/test/mkdirp', function(err) {
Rsync copies files locally on the current host (currently no remote functionality). Rsync copies files locally on the current host (currently no remote functionality).
The srcPath can be either a file or a directory, with the destPath being the The srcPath can be either a file or a directory, with the destPath being the
destination directory. If the destination directory does not exist, it will be created. destination directory. If the destination directory does not exist, it will be created.
Options object can currently have the following attributes: Options object can currently have the following attributes (and their default values):
recursive: true //default 'false' recursive: true //default 'false'
size: 5 //default 750. File chunk size in Kb. size: 5 //default 750. File chunk size in Kb.
checksum: false //default 'false'. False will skip files if their size AND modified times are the same (regardless of content difference)
Example: Example:
```javascript ```javascript
sh.rsync('/test', '/test2', { recursive: true }, function(err) { fs.writeFile('/1.txt','This is my file.', 'utf8', function(err) {
if(err) throw err; if(err) throw err;
shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
if(err) throw err;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
if(err) throw err;
//data will equal 'This is my file.'
});
});
}); });
``` ```

98
src/rsync.js
View File

@ -95,10 +95,14 @@ define(function(require) {
callback = opts; callback = opts;
options = {}; options = {};
options.size = 750; options.size = 750;
options.checksum = false;
}
else {
options = opts || {};
options.size = options.size || 750;
options.checksum = options.checksum || false;
callback = callback || function() {};
} }
options = opts || {};
options.size = options.size || 750;
callback = callback || function() {};
if(srcPath === null || srcPath === '/' || srcPath === '') { if(srcPath === null || srcPath === '/' || srcPath === '') {
callback (new Errors.EINVAL('invalid source path')); callback (new Errors.EINVAL('invalid source path'));
return; return;
@ -111,7 +115,7 @@ define(function(require) {
callback(err); callback(err);
return; return;
} }
if(stats.type === 'DIRECTORY') { if(stats.isDirectory()) {
self.fs.readdir(path, function(err, entries) { self.fs.readdir(path, function(err, entries) {
if(err) { if(err) {
callback(err); callback(err);
@ -128,9 +132,10 @@ define(function(require) {
var entry = { var entry = {
path: Path.basename(name), path: Path.basename(name),
modified: stats.mtime, modified: stats.mtime,
size: stats.size,
type: stats.type type: stats.type
}; };
if(options.recursive && stats.type === 'DIRECTORY') { if(options.recursive && stats.isDirectory()) {
getSrcList(Path.join(srcPath, entry.path), function(error, items) { getSrcList(Path.join(srcPath, entry.path), function(error, items) {
if(error) { if(error) {
callback(error); callback(error);
@ -140,7 +145,7 @@ define(function(require) {
result.push(entry); result.push(entry);
callback(); callback();
}); });
} else if(stats.type === 'FILE') { } else if(stats.isFile()) {
result.push(entry); result.push(entry);
callback(); callback();
} else { } else {
@ -158,6 +163,7 @@ define(function(require) {
var entry = { var entry = {
path: Path.basename(path), path: Path.basename(path),
modified: stats.mtime, modified: stats.mtime,
size: stats.size,
type: stats.type type: stats.type
}; };
result.push(entry); result.push(entry);
@ -166,7 +172,7 @@ define(function(require) {
}); });
} }
function getChecksums(destPath, srcList, callback){ function getChecksums(destPath, srcList, callback) {
var result = []; var result = [];
function getDirChecksums(entry, callback) { function getDirChecksums(entry, callback) {
var item = { path: entry.path }; var item = { path: entry.path };
@ -180,18 +186,38 @@ define(function(require) {
result.push(item); result.push(item);
callback(); callback();
}); });
} else if(entry.type === 'FILE'){ } else if(entry.type === 'FILE') {
checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) { if(options.checksum === false) {
if(err) { self.fs.stat(Path.join(destPath, entry.path), function(err, stat) {
callback(err); if(!err && stat.mtime === entry.modified && stat.size === entry.size) {
return; callback();
} }
item.checksum = checksums; else {
result.push(item); checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) {
callback(); if(err) {
}); callback(err);
return;
}
item.checksum = checksums;
result.push(item);
callback();
});
}
});
}
else {
checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) {
if(err) {
callback(err);
return;
}
item.checksum = checksums;
result.push(item);
callback();
});
}
} }
else{ else {
callback(); callback();
} }
} }
@ -200,23 +226,27 @@ define(function(require) {
}); });
} }
getSrcList(srcPath, function(err, result){ getSrcList(srcPath, function(err, result) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
} }
self.mkdirp(destPath, function(err){ self.mkdirp(destPath, function(err) {
getChecksums(destPath, result, function(err, result){ getChecksums(destPath, result, function(err, result) {
if(err){ if(err) {
callback(err); callback(err);
return; return;
} }
else if (result.length === 0) {
callback();
return;
}
diff.call(self, srcPath, result, function(err, diffs) { diff.call(self, srcPath, result, function(err, diffs) {
if(err){ if(err) {
callback(err); callback(err);
return; return;
} }
sync.call(self, destPath, diffs, function(err){ sync.call(self, destPath, diffs, function(err) {
callback(err); callback(err);
}); });
}); });
@ -228,11 +258,11 @@ define(function(require) {
function checksum (path, callback) { function checksum (path, callback) {
var self = this; var self = this;
self.fs.readFile(path, function (err, data) { self.fs.readFile(path, function (err, data) {
if (!err){ if (!err) {
// cache file // cache file
cache[path] = data; cache[path] = data;
} }
else if (err && err.code === 'ENOENT'){ else if (err && err.code === 'ENOENT') {
cache[path] = []; cache[path] = [];
} }
else { else {
@ -265,12 +295,10 @@ define(function(require) {
function diff(path, checksums, callback) { function diff(path, checksums, callback) {
var self = this; var self = this;
if (!checksums.length)
return callback(new Error('attribute does not exist'), null);
// roll through the file // roll through the file
var diffs = []; var diffs = [];
self.fs.stat(path, function(err, stat){ self.fs.stat(path, function(err, stat) {
if(stat.type ==='DIRECTORY') { if(stat.isDirectory()) {
async.each(checksums, getDiff, function(err) { async.each(checksums, getDiff, function(err) {
callback(err, diffs); callback(err, diffs);
}); });
@ -316,7 +344,7 @@ define(function(require) {
function sync(path, diff, callback) { function sync(path, diff, callback) {
var self = this; var self = this;
function syncEach(entry, callback){ function syncEach(entry, callback) {
//get slice of raw file from block's index //get slice of raw file from block's index
function rawslice(index) { function rawslice(index) {
@ -326,7 +354,7 @@ define(function(require) {
} }
if(entry.hasOwnProperty('contents')) { if(entry.hasOwnProperty('contents')) {
sync.call(self, Path.join(path, entry.path), entry.contents, function(err){ sync.call(self, Path.join(path, entry.path), entry.contents, function(err) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
@ -338,7 +366,7 @@ define(function(require) {
var raw = cache[Path.join(path,entry.path)]; var raw = cache[Path.join(path,entry.path)];
var i = 0; var i = 0;
var len = entry.diff.length; var len = entry.diff.length;
if(typeof raw === 'undefined'){ if(typeof raw === 'undefined') {
return callback(new Error('must do checksum() first'), null); return callback(new Error('must do checksum() first'), null);
} }
@ -355,8 +383,8 @@ define(function(require) {
} }
} }
delete cache[Path.join(path,entry.path)]; delete cache[Path.join(path,entry.path)];
self.fs.writeFile(Path.join(path,entry.path), buf, function(err){ self.fs.writeFile(Path.join(path,entry.path), buf, function(err) {
if(err){ if(err) {
callback(err); callback(err);
return; return;
} }

155
tests/spec/shell/rsync.spec.js
View File

@ -52,7 +52,7 @@ define(["Filer", "util"], function(Filer, util) {
}); });
}); });
it('should succeed if the source file is altered in content but not length from the destination file. (Destination edited)', function(done) { it('should succeed if the source file is different in content but not length from the destination file. (Destination edited)', function(done) {
var fs = util.fs(); var fs = util.fs();
var shell = fs.Shell(); var shell = fs.Shell();
@ -60,11 +60,11 @@ define(["Filer", "util"], function(Filer, util) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/1.txt','This is my file. It does not have any typos.','utf8',function(err) { fs.writeFile('/1.txt','This is my file. It does not have any typos.','utf8',function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/test/1.txt','This is my fivth file. It doez not have any topos,', 'utf8', function(err) { fs.writeFile('/test/1.txt','This iz mi fiel. It doez not have any topos,', 'utf8', function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', { size: 5 }, function(err) { shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){ fs.readFile('/test/1.txt', 'utf8', function(err, data) {
expect(err).to.not.exist; expect(err).to.not.exist;
expect(data).to.exist; expect(data).to.exist;
expect(data).to.equal('This is my file. It does not have any typos.'); expect(data).to.equal('This is my file. It does not have any typos.');
@ -145,6 +145,83 @@ define(["Filer", "util"], function(Filer, util) {
}); });
}); });
it('should succeed if no options are provided', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt','This is my file. It does not exist in the destination folder.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my file. It does not exist in the destination folder.');
done();
});
});
});
});
});
it('should do nothing if the source file and destination file have the same mtime and size with \'checksum = false\' flag (Default)', function(done){
var fs = util.fs();
var shell = fs.Shell();
var date = Date.parse('1 Oct 2000 15:33:22');
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt', 'This is a file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt', 'Different file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.utimes('/1.txt', date, date, function(err) {
expect(err).to.not.exist;
fs.utimes('/test/1.txt', date, date, function(err) {
expect(err).to.not.exist;
shell.ls('/', {recursive: true}, function(err, stuff){
shell.rsync('/1.txt', '/test', {size: 5, checksum: false }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data) {
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('Different file.');
done();
});
});
});
});
});
});
});
});
});
it('should suceed if the source file and destination file have the same mtime and size with \'checksum = true\' flag', function(done){
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt', 'This is a file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt', 'Different file.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', {size: 5, checksum: true }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data) {
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is a file.');
done();
});
});
});
});
});
});
it('should succeed if the destination folder does not exist (Destination file created)', function(done) { it('should succeed if the destination folder does not exist (Destination file created)', function(done) {
var fs = util.fs(); var fs = util.fs();
var shell = fs.Shell(); var shell = fs.Shell();
@ -224,79 +301,59 @@ define(["Filer", "util"], function(Filer, util) {
}); });
}); });
it('should succeed syncing a directory recursively (recursive: true)', function(done) { it('should succeed syncing a directory recursively, skipping same-size and time files (recursive: true)', function(done) {
var fs = util.fs(); var fs = util.fs();
var shell = fs.Shell(); var shell = fs.Shell();
fs.mkdir('/test', function(err) { var date = Date.parse('1 Oct 2000 15:33:22');
shell.mkdirp('/test/sync', function(err){
expect(err).to.not.exist; expect(err).to.not.exist;
fs.mkdir('/test2', function(err) { shell.mkdirp('/test2/sync', function(err){
expect(err).to.not.exist; expect(err).to.not.exist;
fs.mkdir('/test/sync', function(err) { fs.writeFile('/test/1.txt','This is my 1st file.', 'utf8', function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.mkdir('/test2/sync', function(err) { fs.writeFile('/test/sync/2.txt','This is my 2nd file.', 'utf8', function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/test/sync/3.txt','This is my 3rd file.', 'utf8', function(err) {
fs.writeFile('/test/1.txt','This is my 1st file. It does not have any typos.', 'utf8', function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/test/2.txt','This is my 2nd file. It is longer than the destination file.', 'utf8', function(err) { fs.writeFile('/test2/sync/3.txt','This shouldn\'t sync.', 'utf8', function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/test/sync/3.txt','This is my 3rd file.', 'utf8', function(err) {
fs.utimes('/test/sync/3.txt', date, date, function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/test/sync/5.txt','This is my 5th file. It does not exist in the destination folder.', 'utf8', function(err) { fs.utimes('/test2/sync/3.txt', date, date, function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/test2/1.txt','This is my 1st file. It doez not have any topos,', 'utf8', function(err) {
shell.rsync('/test', '/test2', { recursive: true, size: 5 }, function(err) {
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/test2/2.txt','This is my 2nd file.', 'utf8', function(err) { fs.readFile('/test2/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/test2/sync/3.txt','This is my 3rd file. It is longer than the source version.', 'utf8', function(err) { expect(data).to.exist;
expect(data).to.equal('This is my 1st file.');
fs.readFile('/test2/sync/2.txt', 'utf8', function(err, data){
expect(err).to.not.exist; expect(err).to.not.exist;
fs.writeFile('/test2/sync/4.txt','This is my 4th file. It does not exist in the source folder.', 'utf8', function(err) { expect(data).to.exist;
expect(data).to.equal('This is my 2nd file.');
fs.readFile('/test2/sync/3.txt', 'utf8', function(err, data){
expect(err).to.not.exist; expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This shouldn\'t sync.')
shell.rsync('/test', '/test2', { recursive: true, size: 5 }, function(err) { done();
expect(err).to.not.exist;
fs.readFile('/test2/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 1st file. It does not have any typos.');
fs.readFile('/test2/2.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 2nd file. It is longer than the destination file.');
fs.readFile('/test2/sync/3.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 3rd file.')
fs.readFile('/test2/sync/4.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 4th file. It does not exist in the source folder.')
fs.readFile('/test2/sync/5.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 5th file. It does not exist in the destination folder.')
done();
});
});
});
});
});
});
}); });
}); });
}); });
}); });
}); });
}); });
}); });
}); });
}); });
}); });
}); });
}); });
}); });
}); });