LumeWeb
/
filer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

README started. Tests added

This commit is contained in:
Petr Bouianov 2014-03-25 14:41:25 -04:00
parent 8b2f2914de
commit 8d92d421e9
5 changed files with 382 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@ -1133,6 +1133,7 @@ var sh = fs.Shell();
* [sh.rm(path, [options], callback)](#rm) * [sh.rm(path, [options], callback)](#rm)
* [sh.tempDir(callback)](#tempDir) * [sh.tempDir(callback)](#tempDir)
* [sh.mkdirp(path, callback)](#mkdirp) * [sh.mkdirp(path, callback)](#mkdirp)
* [sh.rsync(srcPath, destPath, [options], callback)](#rsync)
#### sh.cd(path, callback)<a name="cd"></a> #### sh.cd(path, callback)<a name="cd"></a>
@ -1338,3 +1339,21 @@ sh.mkdirp('/test/mkdirp', function(err) {
// the root '/' now contains a directory 'test' containing the directory 'mkdirp' // the root '/' now contains a directory 'test' containing the directory 'mkdirp'
}); });
``` ```
#### sh.rsync(srcPath, destPath, [options], callback)<a name="rsync"></a>
Rsync copies files locally on the current host (currently no remote functionality).
The srcPath can be either a file or a directory, with the destPath being the
destination directory. If the destination directory does not exist, it will be created.
Options object can currently have the following attributes:
recursive: true //default 'false'
size: 5 //default 750. File chunk size in Kb.
Example:
```javascript
sh.rsync('/test', '/test2', { recursive: true }, function(err) {
if(err) throw err;
});
```

2
src/hash.js
View File

@ -7,7 +7,7 @@ define(function(require) {
require("crypto-js/rollups/md5"); require("crypto-js/rollups/md5");
function md5(data) { function md5(data) {
return CryptoJS.MD5(String.fromCharCode.apply(null, data)).toString(); return CryptoJS.MD5(String.fromCharCode(data)).toString();
} }
function weak32(data, prev, start, end) { function weak32(data, prev, start, end) {
var a = 0, var a = 0,

104
src/rsync.js
View File

@ -4,6 +4,7 @@
define(function(require) { define(function(require) {
var Path = require('src/path'); var Path = require('src/path');
var Errors = require('src/errors');
var async = require('async'); var async = require('async');
var _md5 = require('./hash').md5; var _md5 = require('./hash').md5;
var _weak16 = require('./hash').weak16; var _weak16 = require('./hash').weak16;
@ -41,6 +42,7 @@ define(function(require) {
var weak = _weak32(data, prevRollingWeak, start, end); var weak = _weak32(data, prevRollingWeak, start, end);
var weak16 = _weak16(weak.sum); var weak16 = _weak16(weak.sum);
var match = false; var match = false;
var d;
prevRollingWeak = weak; prevRollingWeak = weak;
if (hashtable[weak16]) { if (hashtable[weak16]) {
var len = hashtable[weak16].length; var len = hashtable[weak16].length;
@ -59,13 +61,13 @@ define(function(require) {
} }
if (match) { if (match) {
if(start < lastMatchedEnd) { if(start < lastMatchedEnd) {
var d = data.subarray(lastMatchedEnd - 1, end); d = data.subarray(lastMatchedEnd - 1, end);
results.push({ results.push({
data: d, data: d,
index: match.index index: match.index
}); });
} else if (start - lastMatchedEnd > 0) { } else if (start - lastMatchedEnd > 0) {
var d = data.subarray(lastMatchedEnd, start); d = data.subarray(lastMatchedEnd, start);
results.push({ results.push({
data: d, data: d,
index: match.index index: match.index
@ -78,7 +80,7 @@ define(function(require) {
lastMatchedEnd = end; lastMatchedEnd = end;
} else if (end === length) { } else if (end === length) {
// No match and last block // No match and last block
var d = data.subarray(lastMatchedEnd); d = data.subarray(lastMatchedEnd);
results.push({ results.push({
data: d data: d
}); });
@ -89,29 +91,36 @@ define(function(require) {
function rsync (srcPath, destPath, opts, callback) { function rsync (srcPath, destPath, opts, callback) {
var self = this; var self = this;
if(typeof options === 'function') { if(typeof opts === 'function') {
callback = options; callback = opts;
options = {}; options = {};
options.size = 750;
} }
opts = opts || {}; options = opts || {};
opts.size = opts.size || 750; options.size = options.size || 750;
options = opts;
callback = callback || function() {}; callback = callback || function() {};
if(srcPath === null || srcPath === '/' || srcPath === '') {
callback (new Errors.EINVAL('invalid source path'));
return;
}
function getSrcList(path, callback) { function getSrcList(path, callback) {
var result = []; var result = [];
fs.stat(path, function(err, stats) { self.fs.stat(path, function(err, stats) {
if(err) callback(err); if(err) {
callback(err);
return;
}
if(stats.type === 'DIRECTORY') { if(stats.type === 'DIRECTORY') {
fs.readdir(path, function(err, entries) { self.fs.readdir(path, function(err, entries) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
}; }
function getSrcContents(name, callback) { function getSrcContents(_name, callback) {
var name = Path.join(path, name); var name = Path.join(path, _name);
fs.stat(name, function(error, stats) { self.fs.stat(name, function(error, stats) {
if(error) { if(error) {
callback(error); callback(error);
return; return;
@ -192,26 +201,33 @@ define(function(require) {
} }
getSrcList(srcPath, function(err, result){ getSrcList(srcPath, function(err, result){
console.log(result); if(err) {
fs.exists(destPath, function(res){ callback(err);
self.mkdirp(destPath, function(err){ return;
getChecksums(destPath, result, function(err, result){ }
console.log(result); self.mkdirp(destPath, function(err){
diff.call(self, srcPath, result, function(err, diffs) { getChecksums(destPath, result, function(err, result){
console.log(diffs); if(err){
sync.call(self, destPath, diffs, function(err){ callback(err);
callback(err); return;
}) }
diff.call(self, srcPath, result, function(err, diffs) {
if(err){
callback(err);
return;
}
sync.call(self, destPath, diffs, function(err){
callback(err);
}); });
}); });
}); });
}); });
}); });
}; }
function checksum (path, callback) { function checksum (path, callback) {
var self = this; var self = this;
fs.readFile(path, function (err, data) { self.fs.readFile(path, function (err, data) {
if (!err){ if (!err){
// cache file // cache file
cache[path] = data; cache[path] = data;
@ -253,14 +269,14 @@ define(function(require) {
return callback(new Error('attribute does not exist'), null); return callback(new Error('attribute does not exist'), null);
// roll through the file // roll through the file
var diffs = []; var diffs = [];
fs.stat(path, function(err, stat){ self.fs.stat(path, function(err, stat){
if(stat.type ==='DIRECTORY') { if(stat.type ==='DIRECTORY') {
async.each(checksums, getDiff, function(err) { async.each(checksums, getDiff, function(err) {
callback(err, diffs); callback(err, diffs);
}); });
} }
else { else {
fs.readFile(path, function (err, data) { self.fs.readFile(path, function (err, data) {
if (err) { return callback(err); } if (err) { return callback(err); }
diffs.push({ diffs.push({
diff: roll(data, checksums[0].checksum, options.size), diff: roll(data, checksums[0].checksum, options.size),
@ -285,7 +301,7 @@ define(function(require) {
callback(); callback();
}); });
} else { } else {
fs.readFile(Path.join(path,entry.path), function (err, data) { self.fs.readFile(Path.join(path,entry.path), function (err, data) {
if (err) { return callback(err); } if (err) { return callback(err); }
diffs.push({ diffs.push({
diff: roll(data, entry.checksum, options.size), diff: roll(data, entry.checksum, options.size),
@ -301,6 +317,14 @@ define(function(require) {
var self = this; var self = this;
function syncEach(entry, callback){ function syncEach(entry, callback){
//get slice of raw file from block's index
function rawslice(index) {
var start = index*options.size;
var end = start + options.size > raw.length ? raw.length : start + options.size;
return raw.subarray(start, end);
}
if(entry.hasOwnProperty('contents')) { if(entry.hasOwnProperty('contents')) {
sync.call(self, Path.join(path, entry.path), entry.contents, function(err){ sync.call(self, Path.join(path, entry.path), entry.contents, function(err){
if(err) { if(err) {
@ -314,42 +338,30 @@ define(function(require) {
var raw = cache[Path.join(path,entry.path)]; var raw = cache[Path.join(path,entry.path)];
var i = 0; var i = 0;
var len = entry.diff.length; var len = entry.diff.length;
if(typeof raw === 'undefined') if(typeof raw === 'undefined'){
return callback(new Error('must do checksum() first'), null); return callback(new Error('must do checksum() first'), null);
//get slice of raw file from block's index
function rawslice(index) {
var start = index*options.size;
var end = start + options.size > raw.length
? raw.length
: start + options.size;
return raw.subarray(start, end);
} }
var synced = '';
var buf = new Uint8Array(); var buf = new Uint8Array();
for(; i < len; i++) { for(; i < len; i++) {
var chunk = entry.diff[i]; var chunk = entry.diff[i];
if(typeof chunk.data === 'undefined') { //use slice of original file if(typeof chunk.data === 'undefined') { //use slice of original file
buf = appendBuffer(buf, rawslice(chunk.index)); buf = appendBuffer(buf, rawslice(chunk.index));
synced += String.fromCharCode.apply(null, rawslice(chunk.index));
} else { } else {
buf = appendBuffer(buf, chunk.data); buf = appendBuffer(buf, chunk.data);
synced += String.fromCharCode.apply(null, chunk.data);
if(typeof chunk.index !== 'undefined') { if(typeof chunk.index !== 'undefined') {
buf = appendBuffer(buf, rawslice(chunk.index)); buf = appendBuffer(buf, rawslice(chunk.index));
synced += String.fromCharCode.apply(null, rawslice(chunk.index));
} }
} }
} }
delete cache[Path.join(path,entry.path)]; delete cache[Path.join(path,entry.path)];
fs.writeFile(Path.join(path,entry.path), buf, function(err){ self.fs.writeFile(Path.join(path,entry.path), buf, function(err){
if(err){ if(err){
callback(err); callback(err);
return; return;
} }
return callback(null); return callback(null);
}) });
} }
} }

303
tests/spec/shell/rsync.spec.js Normal file
View File

@ -0,0 +1,303 @@
define(["Filer", "util"], function(Filer, util) {
describe('FileSystemShell.rsync', function() {
beforeEach(util.setup);
afterEach(util.cleanup);
it('should be a function', function() {
var shell = util.shell();
expect(shell.rsync).to.be.a('function');
});
it('should fail without a source path provided (null)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
shell.rsync(null, '/', function(err) {
expect(err).to.exist;
expect(err.code).to.equal('EINVAL');
done();
});
});
it('should fail without a source path provided (\'\')', function(done) {
var fs = util.fs();
var shell = fs.Shell();
shell.rsync('', '/', function(err) {
expect(err).to.exist;
expect(err.code).to.equal('EINVAL');
done();
});
});
it('should fail with root provided (\'/\')', function(done) {
var fs = util.fs();
var shell = fs.Shell();
shell.rsync('/', '/', function(err) {
expect(err).to.exist;
expect(err.code).to.equal('EINVAL');
done();
});
});
it('should fail with a non-existant path', function(done) {
var fs = util.fs();
var shell = fs.Shell();
shell.rsync('/1.txt', '/', function(err) {
expect(err).to.exist;
expect(err.code).to.equal('ENOENT');
done();
});
});
it('should succeed if the source file is altered in content but not length from the destination file. (Destination edited)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt','This is my file. It does not have any typos.','utf8',function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt','This is my fivth file. It doez not have any topos,', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my file. It does not have any typos.');
done();
});
});
});
});
});
});
it('should succeed if the source file is longer than the destination file. (Destination appended)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt','This is my file. It is longer than the destination file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt','This is my file.','utf8',function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my file. It is longer than the destination file.');
done();
});
});
});
});
});
});
it('should succeed if the source file shorter than the destination file. (Destination truncated)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt','This is my file.','utf8',function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt','This is my file. It is longer than the source version.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my file.');
done();
});
});
});
});
});
});
it('should succeed if the source file does not exist in the destination folder (Destination file created)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt','This is my file. It does not exist in the destination folder.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my file. It does not exist in the destination folder.');
done();
});
});
});
});
});
it('should succeed if the destination folder does not exist (Destination file created)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.writeFile('/1.txt','This is my file. It does not exist in the destination folder.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my file. It does not exist in the destination folder.');
done();
});
});
});
});
it('should succeed syncing a directory if the destination directory is empty', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.mkdir('/test2', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt','This is my 1st file. It does not have any typos.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/2.txt','This is my 2nd file. It is longer than the destination file.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/test', '/test2', { size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test2/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 1st file. It does not have any typos.');
fs.readFile('/test2/2.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 2nd file. It is longer than the destination file.');
done();
});
});
});
});
});
});
});
});
it('should succeed syncing a directory if the destination directory doesn\'t exist', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt','This is my 1st file. It does not have any typos.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/2.txt','This is my 2nd file. It is longer than the destination file.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/test', '/test2', { size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test2/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 1st file. It does not have any typos.');
fs.readFile('/test2/2.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 2nd file. It is longer than the destination file.');
done();
});
});
});
});
});
});
});
it('should succeed syncing a directory recursively (recursive: true)', function(done) {
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.mkdir('/test2', function(err) {
expect(err).to.not.exist;
fs.mkdir('/test/sync', function(err) {
expect(err).to.not.exist;
fs.mkdir('/test2/sync', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/1.txt','This is my 1st file. It does not have any typos.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/2.txt','This is my 2nd file. It is longer than the destination file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/sync/3.txt','This is my 3rd file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test/sync/5.txt','This is my 5th file. It does not exist in the destination folder.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test2/1.txt','This is my 1st file. It doez not have any topos,', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test2/2.txt','This is my 2nd file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test2/sync/3.txt','This is my 3rd file. It is longer than the source version.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.writeFile('/test2/sync/4.txt','This is my 4th file. It does not exist in the source folder.', 'utf8', function(err) {
expect(err).to.not.exist;
shell.rsync('/test', '/test2', { recursive: true, size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test2/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 1st file. It does not have any typos.');
fs.readFile('/test2/2.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 2nd file. It is longer than the destination file.');
fs.readFile('/test2/sync/3.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 3rd file.')
fs.readFile('/test2/sync/4.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 4th file. It does not exist in the source folder.')
fs.readFile('/test2/sync/5.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my 5th file. It does not exist in the destination folder.')
done();
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});
});

1
tests/test-manifest.js
View File

@ -57,6 +57,7 @@ define([
"spec/shell/rm.spec", "spec/shell/rm.spec",
"spec/shell/env.spec", "spec/shell/env.spec",
"spec/shell/mkdirp.spec", "spec/shell/mkdirp.spec",
"spec/shell/rsync.spec",
// Ported node.js tests (filenames match names in https://github.com/joyent/node/tree/master/test) // Ported node.js tests (filenames match names in https://github.com/joyent/node/tree/master/test)
"spec/node-js/simple/test-fs-mkdir", "spec/node-js/simple/test-fs-mkdir",