LumeWeb
/
filer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changes as per PR. Flags added 

Made changes as per PR notes
time flag added (preserves mtime)
links flag added (copies symlinks instead of resolving)
This commit is contained in:
Petr Bouianov 2014-04-15 20:05:09 -04:00
parent 4572612319
commit 2bf6162c00
4 changed files with 334 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -1324,7 +1324,7 @@ sh.tempDir(function(err, tmp) {
});
```
#### sh.mkdirp(callback)<a name="mkdirp"></a>
#### sh.mkdirp(path, callback)<a name="mkdirp"></a>
Recursively creates the directory at the provided path. If the
directory already exists, no error is returned. All parents must
@ -1349,7 +1349,9 @@ Options object can currently have the following attributes (and their default va
recursive: true //default 'false'
size: 5 //default 750. File chunk size in Kb.
checksum: false //default 'false'. False will skip files if their size AND modified times are the same (regardless of content difference)
checksum: false //default 'false'. False will skip files if their size AND modified times are the same (regardless of content difference).
time: true //default 'false'. Preserves file modified time when syncing.
links: true //default 'false'. Copies symlinks as links instead of resolving.
Example:

53
src/hash.js
View File

@ -1,53 +0,0 @@
// RSync hashing algorithms
// Based on node.js Anchor's hash.js
// Used under MIT License
// https://github.com/ttezel/anchor
define(function(require) {
require("crypto-js/rollups/md5");
function md5(data) {
return CryptoJS.MD5(String.fromCharCode(data)).toString();
}
function weak32(data, prev, start, end) {
var a = 0,
b = 0,
sum = 0,
M = 1 << 16;
if (!prev) {
var len = start >= 0 && end >= 0 ? end - start : data.length,
i = 0;
for (; i < len; i++) {
a += data[i];
b += a;
}
a %= M;
b %= M;
} else {
var k = start,
l = end - 1,
prev_k = k - 1,
prev_l = l - 1,
prev_first = data[prev_k],
prev_last = data[prev_l],
curr_first = data[k],
curr_last = data[l];
a = (prev.a - prev_first + curr_last) % M
b = (prev.b - (prev_l - prev_k + 1) * prev_first + a) % M
}
return { a: a, b: b, sum: a + b * M };
}
function weak16(data) {
return 0xffff & (data >> 16 ^ data*1009);
}
return {
md5: md5,
weak16: weak16,
weak32: weak32
};
});

205
src/rsync.js
View File

@ -1,17 +1,71 @@
// RSync Module for Filer
// Based on the Anchor module for node.js (https://github.com/ttezel/anchor)
// Used under MIT
define(function(require) {
var Path = require('src/path');
var Errors = require('src/errors');
var async = require('async');
var _md5 = require('./hash').md5;
var _weak16 = require('./hash').weak16;
var _weak32 = require('./hash').weak32;
var cache = {};
var options;
//MD5 hashing for RSync
//Used from Node.js Anchor module
//MIT Licensed
//https://github.com/ttezel/anchor
function _md5(data) {
return CryptoJS.MD5(String.fromCharCode(data)).toString();
}
//Weak32 hashing for RSync
//Used from Node.js Anchor module
//MIT Licensed
//https://github.com/ttezel/anchor
function _weak32(data, prev, start, end) {
var a = 0;
var b = 0;
var sum = 0;
var M = 1 << 16;
if (!prev) {
var len = start >= 0 && end >= 0 ? end - start : data.length;
var i = 0;
for (; i < len; i++) {
a += data[i];
b += a;
}
a %= M;
b %= M;
} else {
var k = start;
var l = end - 1;
var prev_k = k - 1;
var prev_l = l - 1;
var prev_first = data[prev_k];
var prev_last = data[prev_l];
var curr_first = data[k];
var curr_last = data[l];
a = (prev.a - prev_first + curr_last) % M;
b = (prev.b - (prev_l - prev_k + 1) * prev_first + a) % M;
}
return { a: a, b: b, sum: a + b * M };
}
//Weak16 hashing for RSync
//Used from Node.js Anchor module
//MIT Licensed
//https://github.com/ttezel/anchor
function _weak16(data) {
return 0xffff & (data >> 16 ^ data*1009);
}
/* RSync Algorithm function
* Copyright(c) 2011 Mihai Tomescu <matomesc@gmail.com>
* Copyright(c) 2011 Tolga Tezel <tolgatezel11@gmail.com>
* https://github.com/ttezel/anchor
* MIT Licensed
*/
function createHashtable(checksums) {
var hashtable = {};
var len = checksums.length;
@ -28,6 +82,12 @@ define(function(require) {
return hashtable;
}
/* RSync Algorithm function
* Copyright(c) 2011 Mihai Tomescu <matomesc@gmail.com>
* Copyright(c) 2011 Tolga Tezel <tolgatezel11@gmail.com>
* https://github.com/ttezel/anchor
* MIT Licensed
*/
function roll(data, checksums, chunkSize) {
var results = [];
var hashtable = createHashtable(checksums);
@ -96,11 +156,17 @@ define(function(require) {
options = {};
options.size = 750;
options.checksum = false;
options.recursive = false;
options.time = false;
options.links = false;
}
else {
options = opts || {};
options.size = options.size || 750;
options.checksum = options.checksum || false;
options.recursive = options.recursive || false;
options.time = options.time || false;
options.links = options.links || false;
callback = callback || function() {};
}
if(srcPath === null || srcPath === '/' || srcPath === '') {
@ -110,7 +176,7 @@ define(function(require) {
function getSrcList(path, callback) {
var result = [];
self.fs.stat(path, function(err, stats) {
self.fs.lstat(path, function(err, stats) {
if(err) {
callback(err);
return;
@ -124,11 +190,13 @@ define(function(require) {
function getSrcContents(_name, callback) {
var name = Path.join(path, _name);
self.fs.stat(name, function(error, stats) {
self.fs.lstat(name, function(error, stats) {
if(error) {
callback(error);
return;
}
var entry = {
path: Path.basename(name),
modified: stats.mtime,
@ -145,10 +213,11 @@ define(function(require) {
result.push(entry);
callback();
});
} else if(stats.isFile()) {
} else if(stats.isFile() || !options.links) {
result.push(entry);
callback();
} else {
} else if (entry.type === 'SYMLINK'){
result.push(entry);
callback();
}
});
@ -162,9 +231,9 @@ define(function(require) {
else {
var entry = {
path: Path.basename(path),
modified: stats.mtime,
size: stats.size,
type: stats.type
type: stats.type,
modified: stats.mtime
};
result.push(entry);
callback(err, result);
@ -186,8 +255,8 @@ define(function(require) {
result.push(item);
callback();
});
} else if(entry.type === 'FILE') {
if(options.checksum === false) {
} else if(entry.type === 'FILE' || !options.links) {
if(!options.checksum) {
self.fs.stat(Path.join(destPath, entry.path), function(err, stat) {
if(!err && stat.mtime === entry.modified && stat.size === entry.size) {
callback();
@ -199,6 +268,7 @@ define(function(require) {
return;
}
item.checksum = checksums;
item.modified = entry.modified;
result.push(item);
callback();
});
@ -212,14 +282,30 @@ define(function(require) {
return;
}
item.checksum = checksums;
item.modified = entry.modified;
result.push(item);
callback();
});
}
}
else {
else if(entry.type === 'SYMLINK'){
if(!options.checksum) {
self.fs.stat(Path.join(destPath, entry.path), function(err, stat){
if(!err && stat.mtime === entry.modified && stat.size === entry.size) {
callback();
}
else {
item.link = true;
result.push(item);
callback();
}
});
} else {
item.link = true;
result.push(item);
callback();
}
}
}
async.each(srcList, getDirChecksums, function(error) {
callback(error, result);
@ -255,6 +341,13 @@ define(function(require) {
});
}
/* RSync Checksum Function
* Based on Node.js Anchor module checksum function
* Copyright(c) 2011 Mihai Tomescu <matomesc@gmail.com>
* Copyright(c) 2011 Tolga Tezel <tolgatezel11@gmail.com>
* https://github.com/ttezel/anchor
* MIT Licensed
*/
function checksum (path, callback) {
var self = this;
self.fs.readFile(path, function (err, data) {
@ -293,26 +386,54 @@ define(function(require) {
});
}
/* RSync Checksum Function
* Based on Node.js Anchor module diff function
* Copyright(c) 2011 Mihai Tomescu <matomesc@gmail.com>
* Copyright(c) 2011 Tolga Tezel <tolgatezel11@gmail.com>
* https://github.com/ttezel/anchor
* MIT Licensed
*/
function diff(path, checksums, callback) {
var self = this;
// roll through the file
var diffs = [];
self.fs.stat(path, function(err, stat) {
self.fs.lstat(path, function(err, stat) {
if(stat.isDirectory()) {
async.each(checksums, getDiff, function(err) {
callback(err, diffs);
});
}
else {
else if (stat.isFile() || !options.links) {
self.fs.readFile(path, function (err, data) {
if (err) { return callback(err); }
diffs.push({
diff: roll(data, checksums[0].checksum, options.size),
modified: checksums[0].modified,
path: checksums[0].path
});
callback(err, diffs);
});
}
else if (stat.isSymbolicLink()) {
self.fs.readlink(path, function(err, linkContents) {
if(err) {
callback(err);
return;
}
self.fs.lstat(path, function(err, stats){
if(err) {
callback(err);
return;
}
diffs.push({
link: linkContents,
modified: stats.mtime,
path: path
});
callback(err, diffs);
});
});
}
});
function getDiff(entry, callback) {
@ -328,11 +449,31 @@ define(function(require) {
});
callback();
});
} else if (entry.hasOwnProperty('link')) {
fs.readlink(Path.join(path, entry.path), function(err, linkContents) {
if(err) {
callback(err);
return;
}
fs.lstat(Path.join(path, entry.path), function(err, stats){
if(err) {
callback(err);
return;
}
diffs.push({
link: linkContents,
modified: stats.mtime,
path: entry.path
});
callback(err, diffs);
});
});
} else {
self.fs.readFile(Path.join(path,entry.path), function (err, data) {
if (err) { return callback(err); }
diffs.push({
diff: roll(data, entry.checksum, options.size),
modified: entry.modified,
path: entry.path
});
callback(err, diffs);
@ -341,6 +482,13 @@ define(function(require) {
}
}
/* RSync Checksum Function
* Based on Node.js Anchor module sync function
* Copyright(c) 2011 Mihai Tomescu <matomesc@gmail.com>
* Copyright(c) 2011 Tolga Tezel <tolgatezel11@gmail.com>
* https://github.com/ttezel/anchor
* MIT Licensed
*/
function sync(path, diff, callback) {
var self = this;
@ -362,7 +510,17 @@ define(function(require) {
callback();
});
}
else {
else if (entry.hasOwnProperty('link')) {
var syncPath = Path.join(path,entry.path);
self.fs.symlink(entry.link, syncPath, function(err){
if(err) {
callback(err);
return;
}
return callback();
});
} else {
var raw = cache[Path.join(path,entry.path)];
var i = 0;
var len = entry.diff.length;
@ -388,7 +546,18 @@ define(function(require) {
callback(err);
return;
}
return callback(null);
if(options.time) {
self.fs.utimes(Path.join(path,entry.path), entry.modified, entry.modified, function(err) {
if(err) {
callback(err);
return;
}
return callback();
});
}
else {
return callback();
}
});
}

142
tests/spec/shell/rsync.spec.js
View File

@ -9,7 +9,7 @@ define(["Filer", "util"], function(Filer, util) {
expect(shell.rsync).to.be.a('function');
});
it('should fail without a source path provided (null)', function(done) {
it('should fail if source path is null', function(done) {
var fs = util.fs();
var shell = fs.Shell();
@ -20,7 +20,7 @@ define(["Filer", "util"], function(Filer, util) {
});
});
it('should fail without a source path provided (\'\')', function(done) {
it('should fail if source path is empty string', function(done) {
var fs = util.fs();
var shell = fs.Shell();
@ -42,7 +42,7 @@ define(["Filer", "util"], function(Filer, util) {
});
});
it('should fail with a non-existant path', function(done) {
it('should fail if source path doesn\'t exist', function(done) {
var fs = util.fs();
var shell = fs.Shell();
shell.rsync('/1.txt', '/', function(err) {
@ -198,7 +198,7 @@ define(["Filer", "util"], function(Filer, util) {
});
});
it('should suceed if the source file and destination file have the same mtime and size with \'checksum = true\' flag', function(done){
it('should succeed if the source file and destination file have the same mtime and size with \'checksum = true\' flag', function(done){
var fs = util.fs();
var shell = fs.Shell();
@ -222,7 +222,139 @@ define(["Filer", "util"], function(Filer, util) {
});
});
it('should succeed if the destination folder does not exist (Destination file created)', function(done) {
it('should succeed and update mtime with \'time = true\' flag', function(done) {
var fs = util.fs();
var shell = fs.Shell();
var mtime;
fs.mkdir('/test', function(err) {
expect(err).to.not.exist;
fs.writeFile('/1.txt','This is my file.', 'utf8', function(err) {
expect(err).to.not.exist;
fs.stat('/1.txt', function(err, stats){
expect(err).to.not.exist;
expect(stats).to.exist;
mtime = stats.mtime;
shell.rsync('/1.txt', '/test', { time: true, size: 5 }, function(err) {
expect(err).to.not.exist;
fs.readFile('/test/1.txt', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is my file.');
fs.stat('/test/1.txt', function(err, stats){
expect(err).to.not.exist;
expect(stats).to.exist;
expect(stats.mtime).to.equal(mtime);
done();
});
});
});
})
});
});
});
it('should copy a symlink as a file with \'links = false\' flag (Default)', function(done){
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err){
expect(err).to.not.exist;
fs.writeFile('/1.txt', 'This is a file', function(err){
expect(err).to.not.exist;
fs.symlink('/1.txt', '/2', function(err){
expect(err).to.not.exist;
shell.rsync('/2', '/test', function(err){
expect(err).to.not.exist;
fs.unlink('/1.txt', function(err){
expect(err).to.not.exist;
fs.lstat('/test/2', function(err, stats){
expect(err).to.not.exist;
expect(stats).to.exist;
expect(stats.type).to.equal('FILE');
fs.readFile('/test/2', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is a file');
done();
});
});
});
});
});
});
});
});
it('should copy a symlink as a file with \'links = true\' flag', function(done){
var fs = util.fs();
var shell = fs.Shell();
fs.mkdir('/test', function(err){
expect(err).to.not.exist;
fs.writeFile('/apple.txt', 'This is a file', function(err){
expect(err).to.not.exist;
fs.symlink('/apple.txt', '/apple', function(err){
expect(err).to.not.exist;
shell.rsync('/apple', '/test', { links:true }, function(err){
expect(err).to.not.exist;
fs.lstat('/test/apple', function(err, stats){
expect(err).to.not.exist;
expect(stats).to.exist;
expect(stats.type).to.equal('SYMLINK');
fs.readFile('/test/apple', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is a file');
done();
});
});
});
});
});
});
});
it('should copy a symlink as a file with \'links = false\' flag and update time with \'time: true\' flag', function(done){
var fs = util.fs();
var shell = fs.Shell();
var mtime;
fs.mkdir('/test', function(err){
expect(err).to.not.exist;
fs.writeFile('/1.txt', 'This is a file', function(err){
expect(err).to.not.exist;
fs.symlink('/1.txt', '/2', function(err){
expect(err).to.not.exist;
fs.lstat('/2', function(err, stats){
expect(err).to.not.exist;
expect(stats).to.exist;
mtime = stats.mtime;
shell.rsync('/2', '/test', { time: true }, function(err){
expect(err).to.not.exist;
fs.unlink('/1.txt', function(err){
expect(err).to.not.exist;
fs.lstat('/test/2', function(err, stats){
expect(err).to.not.exist;
expect(stats).to.exist;
expect(stats.mtime).to.equal(mtime);
expect(stats.type).to.equal('FILE');
fs.readFile('/test/2', 'utf8', function(err, data){
expect(err).to.not.exist;
expect(data).to.exist;
expect(data).to.equal('This is a file');
done();
});
});
});
});
})
});
});
});
});
it('should succeed if the destination folder does not exist (Destination directory created)', function(done) {
var fs = util.fs();
var shell = fs.Shell();