LumeWeb
/
filer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

RSync functional. Tests done 

Functional flags:
size
recursive
checksum
This commit is contained in:
Petr Bouianov 2014-03-31 00:20:19 -04:00
parent 8d92d421e9
commit 4572612319
3 changed files with 188 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
README.md
View File

@ -1345,15 +1345,23 @@ sh.mkdirp('/test/mkdirp', function(err) {
Rsync copies files locally on the current host (currently no remote functionality).
The srcPath can be either a file or a directory, with the destPath being the
destination directory. If the destination directory does not exist, it will be created.
Options object can currently have the following attributes:
Options object can currently have the following attributes (and their default values):
recursive: true //default 'false'
size: 5 //default 750. File chunk size in Kb.
checksum: false //default 'false'. False will skip files if their size AND modified times are the same (regardless of content difference)
Example:
```javascript
sh.rsync('/test', '/test2', { recursive: true }, function(err) {
fs.writeFile('/1.txt','This is my file.', 'utf8', function(err) {
if(err) throw err;
shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
if(err) throw err;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
if(err) throw err;
//data will equal 'This is my file.'
});
});
});
```

98
src/rsync.js
View File

@ -95,10 +95,14 @@ define(function(require) {
callback = opts;
options = {};
options.size = 750;
options.checksum = false;
}
else {
options = opts || {};
options.size = options.size || 750;
options.checksum = options.checksum || false;
callback = callback || function() {};
}
options = opts || {};
options.size = options.size || 750;
callback = callback || function() {};
if(srcPath === null || srcPath === '/' || srcPath === '') {
callback (new Errors.EINVAL('invalid source path'));
return;
@ -111,7 +115,7 @@ define(function(require) {
callback(err);
return;
}
if(stats.type === 'DIRECTORY') {
if(stats.isDirectory()) {
self.fs.readdir(path, function(err, entries) {
if(err) {
callback(err);
@ -128,9 +132,10 @@ define(function(require) {
var entry = {
path: Path.basename(name),
modified: stats.mtime,
size: stats.size,
type: stats.type
};
if(options.recursive && stats.type === 'DIRECTORY') {
if(options.recursive && stats.isDirectory()) {
getSrcList(Path.join(srcPath, entry.path), function(error, items) {
if(error) {
callback(error);
@ -140,7 +145,7 @@ define(function(require) {
result.push(entry);
callback();
});
} else if(stats.type === 'FILE') {
} else if(stats.isFile()) {
result.push(entry);
callback();
} else {
@ -158,6 +163,7 @@ define(function(require) {
var entry = {
path: Path.basename(path),
modified: stats.mtime,
size: stats.size,
type: stats.type
};
result.push(entry);
@ -166,7 +172,7 @@ define(function(require) {
});
}
function getChecksums(destPath, srcList, callback){
function getChecksums(destPath, srcList, callback) {
var result = [];
function getDirChecksums(entry, callback) {
var item = { path: entry.path };
@ -180,18 +186,38 @@ define(function(require) {
result.push(item);
callback();
});
} else if(entry.type === 'FILE'){
checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) {
if(err) {
callback(err);
return;
}
item.checksum = checksums;
result.push(item);
callback();
});
} else if(entry.type === 'FILE') {
if(options.checksum === false) {
self.fs.stat(Path.join(destPath, entry.path), function(err, stat) {
if(!err && stat.mtime === entry.modified && stat.size === entry.size) {
callback();
}
else {
checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) {
if(err) {
callback(err);
return;
}
item.checksum = checksums;
result.push(item);
callback();
});
}
});
}
else {
checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) {
if(err) {
callback(err);
return;
}
item.checksum = checksums;
result.push(item);
callback();
});
}
}
else{
else {
callback();
}
}
@ -200,23 +226,27 @@ define(function(require) {
});
}
getSrcList(srcPath, function(err, result){
getSrcList(srcPath, function(err, result) {
if(err) {
callback(err);
return;
}
self.mkdirp(destPath, function(err){
getChecksums(destPath, result, function(err, result){
if(err){
self.mkdirp(destPath, function(err) {
getChecksums(destPath, result, function(err, result) {
if(err) {
callback(err);
return;
}
else if (result.length === 0) {
callback();
return;
}
diff.call(self, srcPath, result, function(err, diffs) {
if(err){
if(err) {
callback(err);
return;
}
sync.call(self, destPath, diffs, function(err){
sync.call(self, destPath, diffs, function(err) {
callback(err);
});
});
@ -228,11 +258,11 @@ define(function(require) {
function checksum (path, callback) {
var self = this;
self.fs.readFile(path, function (err, data) {
if (!err){
if (!err) {
// cache file
cache[path] = data;
}
else if (err && err.code === 'ENOENT'){
else if (err && err.code === 'ENOENT') {
cache[path] = [];
}
else {
@ -265,12 +295,10 @@ define(function(require) {
function diff(path, checksums, callback) {
var self = this;
if (!checksums.length)
return callback(new Error('attribute does not exist'), null);
// roll through the file
var diffs = [];
self.fs.stat(path, function(err, stat){
if(stat.type ==='DIRECTORY') {
self.fs.stat(path, function(err, stat) {
if(stat.isDirectory()) {
async.each(checksums, getDiff, function(err) {
callback(err, diffs);
});
@ -316,7 +344,7 @@ define(function(require) {
function sync(path, diff, callback) {
var self = this;
function syncEach(entry, callback){
function syncEach(entry, callback) {
//get slice of raw file from block's index
function rawslice(index) {
@ -326,7 +354,7 @@ define(function(require) {
}
if(entry.hasOwnProperty('contents')) {
sync.call(self, Path.join(path, entry.path), entry.contents, function(err){
sync.call(self, Path.join(path, entry.path), entry.contents, function(err) {
if(err) {
callback(err);
return;
@ -338,7 +366,7 @@ define(function(require) {
var raw = cache[Path.join(path,entry.path)];
var i = 0;
var len = entry.diff.length;
if(typeof raw === 'undefined'){
if(typeof raw === 'undefined') {
return callback(new Error('must do checksum() first'), null);
}
@ -355,8 +383,8 @@ define(function(require) {
}
}
delete cache[Path.join(path,entry.path)];
self.fs.writeFile(Path.join(path,entry.path), buf, function(err){
if(err){
self.fs.writeFile(Path.join(path,entry.path), buf, function(err) {
if(err) {
callback(err);
return;
}

173
tests/spec/shell/rsync.spec.js
View File

@ -52,7 +52,7 @@ define(["Filer", "util"], function(Filer, util) {
});
});
it('should succeed if the source file is altered in content but not length from the destination file. (Destination edited)', function(done) {
it('should succeed if the source file is different in content but not length from the destination file. (Destination edited)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
@ -60,11 +60,11 @@ define(["Filer", "util"], function(Filer, util) {
expect(err).to.not.exist;
fs.writeFile('/1.txt','This is my file. It does not have any typos.','utf8',function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt','This is my fivth file. It doez not have any topos,', 'utf8', function(err) {
fs.writeFile('/test/1.txt','This iz mi fiel. It doez not have any topos,', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
fs.readFile('/test/1.txt', 'utf8', function(err, data) {
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my file. It does not have any typos.');
@ -145,6 +145,83 @@ define(["Filer", "util"], function(Filer, util) {
});
});
it('should succeed if no options are provided', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt','This is my file. It does not exist in the destination folder.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my file. It does not exist in the destination folder.');
done();
});
});
});
});
});
it('should do nothing if the source file and destination file have the same mtime and size with \'checksum = false\' flag (Default)', function(done){
var fs = util.fs();
var shell = fs.Shell();
var date = Date.parse('1 Oct 2000 15:33:22');
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt', 'This is a file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt', 'Different file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.utimes('/1.txt', date, date, function(err) {
expect(err).to.not.exist;
fs.utimes('/test/1.txt', date, date, function(err) {
expect(err).to.not.exist;
shell.ls('/', {recursive: true}, function(err, stuff){
shell.rsync('/1.txt', '/test', {size: 5, checksum: false }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data) {
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('Different file.');
done();
});
});
});
});
});
});
});
});
});
it('should suceed if the source file and destination file have the same mtime and size with \'checksum = true\' flag', function(done){
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt', 'This is a file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt', 'Different file.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', {size: 5, checksum: true }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data) {
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is a file.');
done();
});
});
});
});
});
});
it('should succeed if the destination folder does not exist (Destination file created)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
@ -224,79 +301,59 @@ define(["Filer", "util"], function(Filer, util) {
});
});
it('should succeed syncing a directory recursively (recursive: true)', function(done) {
it('should succeed syncing a directory recursively, skipping same-size and time files (recursive: true)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
var date = Date.parse('1 Oct 2000 15:33:22');
shell.mkdirp('/test/sync', function(err){
expect(err).to.not.exist;
fs.mkdir('/test2', function(err) {
shell.mkdirp('/test2/sync', function(err){
expect(err).to.not.exist;
fs.mkdir('/test/sync', function(err) {
fs.writeFile('/test/1.txt','This is my 1st file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.mkdir('/test2/sync', function(err) {
fs.writeFile('/test/sync/2.txt','This is my 2nd file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt','This is my 1st file. It does not have any typos.', 'utf8', function(err) {
fs.writeFile('/test/sync/3.txt','This is my 3rd file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/2.txt','This is my 2nd file. It is longer than the destination file.', 'utf8', function(err) {
fs.writeFile('/test2/sync/3.txt','This shouldn\'t sync.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/sync/3.txt','This is my 3rd file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/sync/5.txt','This is my 5th file. It does not exist in the destination folder.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test2/1.txt','This is my 1st file. It doez not have any topos,', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test2/2.txt','This is my 2nd file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test2/sync/3.txt','This is my 3rd file. It is longer than the source version.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test2/sync/4.txt','This is my 4th file. It does not exist in the source folder.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/test', '/test2', { recursive: true, size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test2/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 1st file. It does not have any typos.');
fs.readFile('/test2/2.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 2nd file. It is longer than the destination file.');
fs.readFile('/test2/sync/3.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 3rd file.')
fs.readFile('/test2/sync/4.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 4th file. It does not exist in the source folder.')
fs.readFile('/test2/sync/5.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 5th file. It does not exist in the destination folder.')
done();
});
});
});
});
});
});
fs.utimes('/test/sync/3.txt', date, date, function(err) {
expect(err).to.not.exist;
fs.utimes('/test2/sync/3.txt', date, date, function(err) {
expect(err).to.not.exist;
shell.rsync('/test', '/test2', { recursive: true, size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test2/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 1st file.');
fs.readFile('/test2/sync/2.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 2nd file.');
fs.readFile('/test2/sync/3.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This shouldn\'t sync.')
done();
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});