diff --git a/README.md b/README.md
index fec7d2d..25352a1 100644
--- a/README.md
+++ b/README.md
@@ -1133,6 +1133,7 @@ var sh = fs.Shell();
* [sh.rm(path, [options], callback)](#rm)
* [sh.tempDir(callback)](#tempDir)
* [sh.mkdirp(path, callback)](#mkdirp)
+* [sh.rsync(srcPath, destPath, [options], callback)](#rsync)
#### sh.cd(path, callback)
@@ -1338,3 +1339,21 @@ sh.mkdirp('/test/mkdirp', function(err) {
// the root '/' now contains a directory 'test' containing the directory 'mkdirp'
});
```
+
+#### sh.rsync(srcPath, destPath, [options], callback)
+
+Rsync copies files locally on the current host (currently no remote functionality).
+The srcPath can be either a file or a directory, with the destPath being the
+destination directory. If the destination directory does not exist, it will be created.
+Options object can currently have the following attributes:
+
+recursive: true //default 'false'
+size: 5 //default 750. File chunk size in Kb.
+
+Example:
+
+```javascript
+sh.rsync('/test', '/test2', { recursive: true }, function(err) {
+ if(err) throw err;
+});
+```
\ No newline at end of file
diff --git a/src/hash.js b/src/hash.js
index 74c146c..c4006a3 100644
--- a/src/hash.js
+++ b/src/hash.js
@@ -7,7 +7,7 @@ define(function(require) {
require("crypto-js/rollups/md5");
function md5(data) {
- return CryptoJS.MD5(String.fromCharCode.apply(null, data)).toString();
+ return CryptoJS.MD5(String.fromCharCode(data)).toString();
}
function weak32(data, prev, start, end) {
var a = 0,
diff --git a/src/rsync.js b/src/rsync.js
index 4c5d1e5..f9554ae 100644
--- a/src/rsync.js
+++ b/src/rsync.js
@@ -4,6 +4,7 @@
define(function(require) {
var Path = require('src/path');
+ var Errors = require('src/errors');
var async = require('async');
var _md5 = require('./hash').md5;
var _weak16 = require('./hash').weak16;
@@ -41,6 +42,7 @@ define(function(require) {
var weak = _weak32(data, prevRollingWeak, start, end);
var weak16 = _weak16(weak.sum);
var match = false;
+ var d;
prevRollingWeak = weak;
if (hashtable[weak16]) {
var len = hashtable[weak16].length;
@@ -59,13 +61,13 @@ define(function(require) {
}
if (match) {
if(start < lastMatchedEnd) {
- var d = data.subarray(lastMatchedEnd - 1, end);
+ d = data.subarray(lastMatchedEnd - 1, end);
results.push({
data: d,
index: match.index
});
} else if (start - lastMatchedEnd > 0) {
- var d = data.subarray(lastMatchedEnd, start);
+ d = data.subarray(lastMatchedEnd, start);
results.push({
data: d,
index: match.index
@@ -78,7 +80,7 @@ define(function(require) {
lastMatchedEnd = end;
} else if (end === length) {
// No match and last block
- var d = data.subarray(lastMatchedEnd);
+ d = data.subarray(lastMatchedEnd);
results.push({
data: d
});
@@ -89,29 +91,36 @@ define(function(require) {
function rsync (srcPath, destPath, opts, callback) {
var self = this;
- if(typeof options === 'function') {
- callback = options;
+ if(typeof opts === 'function') {
+ callback = opts;
options = {};
+ options.size = 750;
}
- opts = opts || {};
- opts.size = opts.size || 750;
- options = opts;
+ options = opts || {};
+ options.size = options.size || 750;
callback = callback || function() {};
+ if(srcPath === null || srcPath === '/' || srcPath === '') {
+ callback (new Errors.EINVAL('invalid source path'));
+ return;
+ }
function getSrcList(path, callback) {
var result = [];
- fs.stat(path, function(err, stats) {
- if(err) callback(err);
+ self.fs.stat(path, function(err, stats) {
+ if(err) {
+ callback(err);
+ return;
+ }
if(stats.type === 'DIRECTORY') {
- fs.readdir(path, function(err, entries) {
+ self.fs.readdir(path, function(err, entries) {
if(err) {
callback(err);
return;
- };
+ }
- function getSrcContents(name, callback) {
- var name = Path.join(path, name);
- fs.stat(name, function(error, stats) {
+ function getSrcContents(_name, callback) {
+ var name = Path.join(path, _name);
+ self.fs.stat(name, function(error, stats) {
if(error) {
callback(error);
return;
@@ -192,26 +201,33 @@ define(function(require) {
}
getSrcList(srcPath, function(err, result){
- console.log(result);
- fs.exists(destPath, function(res){
- self.mkdirp(destPath, function(err){
- getChecksums(destPath, result, function(err, result){
- console.log(result);
- diff.call(self, srcPath, result, function(err, diffs) {
- console.log(diffs);
- sync.call(self, destPath, diffs, function(err){
- callback(err);
- })
+ if(err) {
+ callback(err);
+ return;
+ }
+ self.mkdirp(destPath, function(err){
+ getChecksums(destPath, result, function(err, result){
+ if(err){
+ callback(err);
+ return;
+ }
+ diff.call(self, srcPath, result, function(err, diffs) {
+ if(err){
+ callback(err);
+ return;
+ }
+ sync.call(self, destPath, diffs, function(err){
+ callback(err);
});
});
});
});
});
- };
+ }
function checksum (path, callback) {
var self = this;
- fs.readFile(path, function (err, data) {
+ self.fs.readFile(path, function (err, data) {
if (!err){
// cache file
cache[path] = data;
@@ -253,14 +269,14 @@ define(function(require) {
return callback(new Error('attribute does not exist'), null);
// roll through the file
var diffs = [];
- fs.stat(path, function(err, stat){
+ self.fs.stat(path, function(err, stat){
if(stat.type ==='DIRECTORY') {
async.each(checksums, getDiff, function(err) {
callback(err, diffs);
});
}
else {
- fs.readFile(path, function (err, data) {
+ self.fs.readFile(path, function (err, data) {
if (err) { return callback(err); }
diffs.push({
diff: roll(data, checksums[0].checksum, options.size),
@@ -285,7 +301,7 @@ define(function(require) {
callback();
});
} else {
- fs.readFile(Path.join(path,entry.path), function (err, data) {
+ self.fs.readFile(Path.join(path,entry.path), function (err, data) {
if (err) { return callback(err); }
diffs.push({
diff: roll(data, entry.checksum, options.size),
@@ -301,6 +317,14 @@ define(function(require) {
var self = this;
function syncEach(entry, callback){
+
+ //get slice of raw file from block's index
+ function rawslice(index) {
+ var start = index*options.size;
+ var end = start + options.size > raw.length ? raw.length : start + options.size;
+ return raw.subarray(start, end);
+ }
+
if(entry.hasOwnProperty('contents')) {
sync.call(self, Path.join(path, entry.path), entry.contents, function(err){
if(err) {
@@ -314,42 +338,30 @@ define(function(require) {
var raw = cache[Path.join(path,entry.path)];
var i = 0;
var len = entry.diff.length;
- if(typeof raw === 'undefined')
+ if(typeof raw === 'undefined'){
return callback(new Error('must do checksum() first'), null);
-
- //get slice of raw file from block's index
- function rawslice(index) {
- var start = index*options.size;
- var end = start + options.size > raw.length
- ? raw.length
- : start + options.size;
- return raw.subarray(start, end);
}
- var synced = '';
var buf = new Uint8Array();
for(; i < len; i++) {
var chunk = entry.diff[i];
if(typeof chunk.data === 'undefined') { //use slice of original file
buf = appendBuffer(buf, rawslice(chunk.index));
- synced += String.fromCharCode.apply(null, rawslice(chunk.index));
} else {
buf = appendBuffer(buf, chunk.data);
- synced += String.fromCharCode.apply(null, chunk.data);
if(typeof chunk.index !== 'undefined') {
buf = appendBuffer(buf, rawslice(chunk.index));
- synced += String.fromCharCode.apply(null, rawslice(chunk.index));
}
}
}
delete cache[Path.join(path,entry.path)];
- fs.writeFile(Path.join(path,entry.path), buf, function(err){
+ self.fs.writeFile(Path.join(path,entry.path), buf, function(err){
if(err){
callback(err);
return;
}
- return callback(null);
- })
+ return callback(null);
+ });
}
}
diff --git a/tests/spec/shell/rsync.spec.js b/tests/spec/shell/rsync.spec.js
new file mode 100644
index 0000000..8bb0bf8
--- /dev/null
+++ b/tests/spec/shell/rsync.spec.js
@@ -0,0 +1,303 @@
+define(["Filer", "util"], function(Filer, util) {
+
+ describe('FileSystemShell.rsync', function() {
+ beforeEach(util.setup);
+ afterEach(util.cleanup);
+
+ it('should be a function', function() {
+ var shell = util.shell();
+ expect(shell.rsync).to.be.a('function');
+ });
+
+ it('should fail without a source path provided (null)', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ shell.rsync(null, '/', function(err) {
+ expect(err).to.exist;
+ expect(err.code).to.equal('EINVAL');
+ done();
+ });
+ });
+
+ it('should fail without a source path provided (\'\')', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ shell.rsync('', '/', function(err) {
+ expect(err).to.exist;
+ expect(err.code).to.equal('EINVAL');
+ done();
+ });
+ });
+
+ it('should fail with root provided (\'/\')', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ shell.rsync('/', '/', function(err) {
+ expect(err).to.exist;
+ expect(err.code).to.equal('EINVAL');
+ done();
+ });
+ });
+
+ it('should fail with a non-existant path', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+ shell.rsync('/1.txt', '/', function(err) {
+ expect(err).to.exist;
+ expect(err.code).to.equal('ENOENT');
+ done();
+ });
+ });
+
+ it('should succeed if the source file is altered in content but not length from the destination file. (Destination edited)', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ fs.mkdir('/test', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/1.txt','This is my file. It does not have any typos.','utf8',function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/1.txt','This is my fivth file. It doez not have any topos,', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
+ expect(err).to.not.exist;
+ fs.readFile('/test/1.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my file. It does not have any typos.');
+ done();
+ });
+ });
+ });
+ });
+ });
+ });
+
+ it('should succeed if the source file is longer than the destination file. (Destination appended)', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ fs.mkdir('/test', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/1.txt','This is my file. It is longer than the destination file.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/1.txt','This is my file.','utf8',function(err) {
+ expect(err).to.not.exist;
+ shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
+ expect(err).to.not.exist;
+ fs.readFile('/test/1.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my file. It is longer than the destination file.');
+ done();
+ });
+ });
+ });
+ });
+ });
+ });
+
+ it('should succeed if the source file shorter than the destination file. (Destination truncated)', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ fs.mkdir('/test', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/1.txt','This is my file.','utf8',function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/1.txt','This is my file. It is longer than the source version.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
+ expect(err).to.not.exist;
+ fs.readFile('/test/1.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my file.');
+ done();
+ });
+ });
+ });
+ });
+ });
+ });
+
+ it('should succeed if the source file does not exist in the destination folder (Destination file created)', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ fs.mkdir('/test', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/1.txt','This is my file. It does not exist in the destination folder.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
+ expect(err).to.not.exist;
+ fs.readFile('/test/1.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my file. It does not exist in the destination folder.');
+ done();
+ });
+ });
+ });
+ });
+ });
+
+ it('should succeed if the destination folder does not exist (Destination file created)', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ fs.writeFile('/1.txt','This is my file. It does not exist in the destination folder.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ shell.rsync('/1.txt', '/test', { size: 5 }, function(err) {
+ expect(err).to.not.exist;
+ fs.readFile('/test/1.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my file. It does not exist in the destination folder.');
+ done();
+ });
+ });
+ });
+ });
+
+ it('should succeed syncing a directory if the destination directory is empty', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ fs.mkdir('/test', function(err) {
+ expect(err).to.not.exist;
+ fs.mkdir('/test2', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/1.txt','This is my 1st file. It does not have any typos.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/2.txt','This is my 2nd file. It is longer than the destination file.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ shell.rsync('/test', '/test2', { size: 5 }, function(err) {
+ expect(err).to.not.exist;
+ fs.readFile('/test2/1.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my 1st file. It does not have any typos.');
+ fs.readFile('/test2/2.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my 2nd file. It is longer than the destination file.');
+ done();
+ });
+ });
+ });
+ });
+ });
+ });
+ });
+ });
+
+ it('should succeed syncing a directory if the destination directory doesn\'t exist', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+
+ fs.mkdir('/test', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/1.txt','This is my 1st file. It does not have any typos.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/2.txt','This is my 2nd file. It is longer than the destination file.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ shell.rsync('/test', '/test2', { size: 5 }, function(err) {
+ expect(err).to.not.exist;
+ fs.readFile('/test2/1.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my 1st file. It does not have any typos.');
+ fs.readFile('/test2/2.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my 2nd file. It is longer than the destination file.');
+ done();
+ });
+ });
+ });
+ });
+ });
+ });
+ });
+
+ it('should succeed syncing a directory recursively (recursive: true)', function(done) {
+ var fs = util.fs();
+ var shell = fs.Shell();
+ fs.mkdir('/test', function(err) {
+ expect(err).to.not.exist;
+ fs.mkdir('/test2', function(err) {
+ expect(err).to.not.exist;
+ fs.mkdir('/test/sync', function(err) {
+ expect(err).to.not.exist;
+ fs.mkdir('/test2/sync', function(err) {
+ expect(err).to.not.exist;
+
+ fs.writeFile('/test/1.txt','This is my 1st file. It does not have any typos.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/2.txt','This is my 2nd file. It is longer than the destination file.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/sync/3.txt','This is my 3rd file.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test/sync/5.txt','This is my 5th file. It does not exist in the destination folder.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test2/1.txt','This is my 1st file. It doez not have any topos,', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test2/2.txt','This is my 2nd file.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test2/sync/3.txt','This is my 3rd file. It is longer than the source version.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+ fs.writeFile('/test2/sync/4.txt','This is my 4th file. It does not exist in the source folder.', 'utf8', function(err) {
+ expect(err).to.not.exist;
+
+ shell.rsync('/test', '/test2', { recursive: true, size: 5 }, function(err) {
+ expect(err).to.not.exist;
+ fs.readFile('/test2/1.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my 1st file. It does not have any typos.');
+ fs.readFile('/test2/2.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my 2nd file. It is longer than the destination file.');
+ fs.readFile('/test2/sync/3.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my 3rd file.')
+ fs.readFile('/test2/sync/4.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my 4th file. It does not exist in the source folder.')
+ fs.readFile('/test2/sync/5.txt', 'utf8', function(err, data){
+ expect(err).to.not.exist;
+ expect(data).to.exist;
+ expect(data).to.equal('This is my 5th file. It does not exist in the destination folder.')
+ done();
+ });
+ });
+ });
+ });
+ });
+ });
+
+ });
+ });
+ });
+ });
+ });
+ });
+ });
+ });
+
+ });
+ });
+ });
+ });
+
+ });
+
+ });
+});
diff --git a/tests/test-manifest.js b/tests/test-manifest.js
index 1e5a908..368ab94 100644
--- a/tests/test-manifest.js
+++ b/tests/test-manifest.js
@@ -57,6 +57,7 @@ define([
"spec/shell/rm.spec",
"spec/shell/env.spec",
"spec/shell/mkdirp.spec",
+ "spec/shell/rsync.spec",
// Ported node.js tests (filenames match names in https://github.com/joyent/node/tree/master/test)
"spec/node-js/simple/test-fs-mkdir",