From 2bf6162c00b273cc23cb09a3448904640b0c050f Mon Sep 17 00:00:00 2001 From: Petr Bouianov Date: Tue, 15 Apr 2014 20:05:09 -0400 Subject: [PATCH] Changes as per PR. Flags added Made changes as per PR notes time flag added (preserves mtime) links flag added (copies symlinks instead of resolving) --- README.md | 10 +- src/hash.js | 53 -------- src/rsync.js | 213 +++++++++++++++++++++++++++++---- tests/spec/shell/rsync.spec.js | 142 +++++++++++++++++++++- 4 files changed, 334 insertions(+), 84 deletions(-) delete mode 100644 src/hash.js diff --git a/README.md b/README.md index 3c94d5e..15eb4ab 100644 --- a/README.md +++ b/README.md @@ -850,7 +850,7 @@ Examples: ```javascript // Append UTF8 text file fs.writeFile('/myfile.txt', "More...", function (err) { - if (err) throw err; + if (err) throw err; }); fs.appendFile('/myfile.txt', "Data...", function (err) { if (err) throw err; @@ -861,7 +861,7 @@ fs.appendFile('/myfile.txt', "Data...", function (err) { var more = new Uint8Array([1, 2, 3, 4]); var data = new Uint8Array([5, 6, 7, 8]); fs.writeFile('/myfile', more, function (err) { - if (err) throw err; + if (err) throw err; }); fs.appendFile('/myfile', buffer, function (err) { if (err) throw err; @@ -1324,7 +1324,7 @@ sh.tempDir(function(err, tmp) { }); ``` -#### sh.mkdirp(callback) +#### sh.mkdirp(path, callback) Recursively creates the directory at the provided path. If the directory already exists, no error is returned. All parents must @@ -1349,7 +1349,9 @@ Options object can currently have the following attributes (and their default va recursive: true //default 'false' size: 5 //default 750. File chunk size in Kb. -checksum: false //default 'false'. False will skip files if their size AND modified times are the same (regardless of content difference) +checksum: false //default 'false'. False will skip files if their size AND modified times are the same (regardless of content difference). +time: true //default 'false'. Preserves file modified time when syncing. +links: true //default 'false'. Copies symlinks as links instead of resolving. Example: diff --git a/src/hash.js b/src/hash.js deleted file mode 100644 index c4006a3..0000000 --- a/src/hash.js +++ /dev/null @@ -1,53 +0,0 @@ -// RSync hashing algorithms -// Based on node.js Anchor's hash.js -// Used under MIT License -// https://github.com/ttezel/anchor - -define(function(require) { - require("crypto-js/rollups/md5"); - - function md5(data) { - return CryptoJS.MD5(String.fromCharCode(data)).toString(); - } - function weak32(data, prev, start, end) { - var a = 0, - b = 0, - sum = 0, - M = 1 << 16; - - if (!prev) { - var len = start >= 0 && end >= 0 ? end - start : data.length, - i = 0; - - for (; i < len; i++) { - a += data[i]; - b += a; - } - - a %= M; - b %= M; - } else { - var k = start, - l = end - 1, - prev_k = k - 1, - prev_l = l - 1, - prev_first = data[prev_k], - prev_last = data[prev_l], - curr_first = data[k], - curr_last = data[l]; - - a = (prev.a - prev_first + curr_last) % M - b = (prev.b - (prev_l - prev_k + 1) * prev_first + a) % M - } - return { a: a, b: b, sum: a + b * M }; - } - function weak16(data) { - return 0xffff & (data >> 16 ^ data*1009); - } - - return { - md5: md5, - weak16: weak16, - weak32: weak32 - }; -}); diff --git a/src/rsync.js b/src/rsync.js index 9bf7141..ff04455 100644 --- a/src/rsync.js +++ b/src/rsync.js @@ -1,17 +1,71 @@ // RSync Module for Filer -// Based on the Anchor module for node.js (https://github.com/ttezel/anchor) -// Used under MIT define(function(require) { var Path = require('src/path'); var Errors = require('src/errors'); var async = require('async'); - var _md5 = require('./hash').md5; - var _weak16 = require('./hash').weak16; - var _weak32 = require('./hash').weak32; var cache = {}; var options; + //MD5 hashing for RSync + //Used from Node.js Anchor module + //MIT Licensed + //https://github.com/ttezel/anchor + function _md5(data) { + return CryptoJS.MD5(String.fromCharCode(data)).toString(); + } + + //Weak32 hashing for RSync + //Used from Node.js Anchor module + //MIT Licensed + //https://github.com/ttezel/anchor + function _weak32(data, prev, start, end) { + var a = 0; + var b = 0; + var sum = 0; + var M = 1 << 16; + + if (!prev) { + var len = start >= 0 && end >= 0 ? end - start : data.length; + var i = 0; + + for (; i < len; i++) { + a += data[i]; + b += a; + } + + a %= M; + b %= M; + } else { + var k = start; + var l = end - 1; + var prev_k = k - 1; + var prev_l = l - 1; + var prev_first = data[prev_k]; + var prev_last = data[prev_l]; + var curr_first = data[k]; + var curr_last = data[l]; + + a = (prev.a - prev_first + curr_last) % M; + b = (prev.b - (prev_l - prev_k + 1) * prev_first + a) % M; + } + return { a: a, b: b, sum: a + b * M }; + } + + //Weak16 hashing for RSync + //Used from Node.js Anchor module + //MIT Licensed + //https://github.com/ttezel/anchor + function _weak16(data) { + return 0xffff & (data >> 16 ^ data*1009); + } + + /* RSync Algorithm function + * Copyright(c) 2011 Mihai Tomescu + * Copyright(c) 2011 Tolga Tezel + * https://github.com/ttezel/anchor + * MIT Licensed + */ function createHashtable(checksums) { var hashtable = {}; var len = checksums.length; @@ -28,6 +82,12 @@ define(function(require) { return hashtable; } + /* RSync Algorithm function + * Copyright(c) 2011 Mihai Tomescu + * Copyright(c) 2011 Tolga Tezel + * https://github.com/ttezel/anchor + * MIT Licensed + */ function roll(data, checksums, chunkSize) { var results = []; var hashtable = createHashtable(checksums); @@ -96,11 +156,17 @@ define(function(require) { options = {}; options.size = 750; options.checksum = false; + options.recursive = false; + options.time = false; + options.links = false; } else { options = opts || {}; options.size = options.size || 750; options.checksum = options.checksum || false; + options.recursive = options.recursive || false; + options.time = options.time || false; + options.links = options.links || false; callback = callback || function() {}; } if(srcPath === null || srcPath === '/' || srcPath === '') { @@ -110,7 +176,7 @@ define(function(require) { function getSrcList(path, callback) { var result = []; - self.fs.stat(path, function(err, stats) { + self.fs.lstat(path, function(err, stats) { if(err) { callback(err); return; @@ -124,11 +190,13 @@ define(function(require) { function getSrcContents(_name, callback) { var name = Path.join(path, _name); - self.fs.stat(name, function(error, stats) { + self.fs.lstat(name, function(error, stats) { + if(error) { callback(error); return; } + var entry = { path: Path.basename(name), modified: stats.mtime, @@ -145,10 +213,11 @@ define(function(require) { result.push(entry); callback(); }); - } else if(stats.isFile()) { + } else if(stats.isFile() || !options.links) { result.push(entry); callback(); - } else { + } else if (entry.type === 'SYMLINK'){ + result.push(entry); callback(); } }); @@ -159,12 +228,12 @@ define(function(require) { }); }); } - else{ + else { var entry = { path: Path.basename(path), - modified: stats.mtime, size: stats.size, - type: stats.type + type: stats.type, + modified: stats.mtime }; result.push(entry); callback(err, result); @@ -186,12 +255,12 @@ define(function(require) { result.push(item); callback(); }); - } else if(entry.type === 'FILE') { - if(options.checksum === false) { + } else if(entry.type === 'FILE' || !options.links) { + if(!options.checksum) { self.fs.stat(Path.join(destPath, entry.path), function(err, stat) { if(!err && stat.mtime === entry.modified && stat.size === entry.size) { - callback(); - } + callback(); + } else { checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) { if(err) { @@ -199,6 +268,7 @@ define(function(require) { return; } item.checksum = checksums; + item.modified = entry.modified; result.push(item); callback(); }); @@ -212,13 +282,29 @@ define(function(require) { return; } item.checksum = checksums; + item.modified = entry.modified; result.push(item); callback(); }); } } - else { - callback(); + else if(entry.type === 'SYMLINK'){ + if(!options.checksum) { + self.fs.stat(Path.join(destPath, entry.path), function(err, stat){ + if(!err && stat.mtime === entry.modified && stat.size === entry.size) { + callback(); + } + else { + item.link = true; + result.push(item); + callback(); + } + }); + } else { + item.link = true; + result.push(item); + callback(); + } } } async.each(srcList, getDirChecksums, function(error) { @@ -255,6 +341,13 @@ define(function(require) { }); } + /* RSync Checksum Function + * Based on Node.js Anchor module checksum function + * Copyright(c) 2011 Mihai Tomescu + * Copyright(c) 2011 Tolga Tezel + * https://github.com/ttezel/anchor + * MIT Licensed + */ function checksum (path, callback) { var self = this; self.fs.readFile(path, function (err, data) { @@ -293,26 +386,54 @@ define(function(require) { }); } + /* RSync Checksum Function + * Based on Node.js Anchor module diff function + * Copyright(c) 2011 Mihai Tomescu + * Copyright(c) 2011 Tolga Tezel + * https://github.com/ttezel/anchor + * MIT Licensed + */ function diff(path, checksums, callback) { var self = this; // roll through the file var diffs = []; - self.fs.stat(path, function(err, stat) { + self.fs.lstat(path, function(err, stat) { if(stat.isDirectory()) { async.each(checksums, getDiff, function(err) { callback(err, diffs); }); } - else { + else if (stat.isFile() || !options.links) { self.fs.readFile(path, function (err, data) { if (err) { return callback(err); } diffs.push({ diff: roll(data, checksums[0].checksum, options.size), + modified: checksums[0].modified, path: checksums[0].path }); callback(err, diffs); }); } + else if (stat.isSymbolicLink()) { + self.fs.readlink(path, function(err, linkContents) { + if(err) { + callback(err); + return; + } + self.fs.lstat(path, function(err, stats){ + if(err) { + callback(err); + return; + } + diffs.push({ + link: linkContents, + modified: stats.mtime, + path: path + }); + callback(err, diffs); + }); + }); + } }); function getDiff(entry, callback) { @@ -328,11 +449,31 @@ define(function(require) { }); callback(); }); + } else if (entry.hasOwnProperty('link')) { + fs.readlink(Path.join(path, entry.path), function(err, linkContents) { + if(err) { + callback(err); + return; + } + fs.lstat(Path.join(path, entry.path), function(err, stats){ + if(err) { + callback(err); + return; + } + diffs.push({ + link: linkContents, + modified: stats.mtime, + path: entry.path + }); + callback(err, diffs); + }); + }); } else { self.fs.readFile(Path.join(path,entry.path), function (err, data) { if (err) { return callback(err); } diffs.push({ diff: roll(data, entry.checksum, options.size), + modified: entry.modified, path: entry.path }); callback(err, diffs); @@ -341,6 +482,13 @@ define(function(require) { } } + /* RSync Checksum Function + * Based on Node.js Anchor module sync function + * Copyright(c) 2011 Mihai Tomescu + * Copyright(c) 2011 Tolga Tezel + * https://github.com/ttezel/anchor + * MIT Licensed + */ function sync(path, diff, callback) { var self = this; @@ -362,7 +510,17 @@ define(function(require) { callback(); }); } - else { + else if (entry.hasOwnProperty('link')) { + + var syncPath = Path.join(path,entry.path); + self.fs.symlink(entry.link, syncPath, function(err){ + if(err) { + callback(err); + return; + } + return callback(); + }); + } else { var raw = cache[Path.join(path,entry.path)]; var i = 0; var len = entry.diff.length; @@ -388,7 +546,18 @@ define(function(require) { callback(err); return; } - return callback(null); + if(options.time) { + self.fs.utimes(Path.join(path,entry.path), entry.modified, entry.modified, function(err) { + if(err) { + callback(err); + return; + } + return callback(); + }); + } + else { + return callback(); + } }); } diff --git a/tests/spec/shell/rsync.spec.js b/tests/spec/shell/rsync.spec.js index 09e75ed..f771416 100644 --- a/tests/spec/shell/rsync.spec.js +++ b/tests/spec/shell/rsync.spec.js @@ -9,7 +9,7 @@ define(["Filer", "util"], function(Filer, util) { expect(shell.rsync).to.be.a('function'); }); - it('should fail without a source path provided (null)', function(done) { + it('should fail if source path is null', function(done) { var fs = util.fs(); var shell = fs.Shell(); @@ -20,7 +20,7 @@ define(["Filer", "util"], function(Filer, util) { }); }); - it('should fail without a source path provided (\'\')', function(done) { + it('should fail if source path is empty string', function(done) { var fs = util.fs(); var shell = fs.Shell(); @@ -42,7 +42,7 @@ define(["Filer", "util"], function(Filer, util) { }); }); - it('should fail with a non-existant path', function(done) { + it('should fail if source path doesn\'t exist', function(done) { var fs = util.fs(); var shell = fs.Shell(); shell.rsync('/1.txt', '/', function(err) { @@ -198,7 +198,7 @@ define(["Filer", "util"], function(Filer, util) { }); }); - it('should suceed if the source file and destination file have the same mtime and size with \'checksum = true\' flag', function(done){ + it('should succeed if the source file and destination file have the same mtime and size with \'checksum = true\' flag', function(done){ var fs = util.fs(); var shell = fs.Shell(); @@ -222,7 +222,139 @@ define(["Filer", "util"], function(Filer, util) { }); }); - it('should succeed if the destination folder does not exist (Destination file created)', function(done) { + it('should succeed and update mtime with \'time = true\' flag', function(done) { + var fs = util.fs(); + var shell = fs.Shell(); + var mtime; + + fs.mkdir('/test', function(err) { + expect(err).to.not.exist; + fs.writeFile('/1.txt','This is my file.', 'utf8', function(err) { + expect(err).to.not.exist; + fs.stat('/1.txt', function(err, stats){ + expect(err).to.not.exist; + expect(stats).to.exist; + mtime = stats.mtime; + shell.rsync('/1.txt', '/test', { time: true, size: 5 }, function(err) { + expect(err).to.not.exist; + fs.readFile('/test/1.txt', 'utf8', function(err, data){ + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('This is my file.'); + fs.stat('/test/1.txt', function(err, stats){ + expect(err).to.not.exist; + expect(stats).to.exist; + expect(stats.mtime).to.equal(mtime); + done(); + }); + }); + }); + }) + }); + }); + }); + + it('should copy a symlink as a file with \'links = false\' flag (Default)', function(done){ + var fs = util.fs(); + var shell = fs.Shell(); + + fs.mkdir('/test', function(err){ + expect(err).to.not.exist; + fs.writeFile('/1.txt', 'This is a file', function(err){ + expect(err).to.not.exist; + fs.symlink('/1.txt', '/2', function(err){ + expect(err).to.not.exist; + shell.rsync('/2', '/test', function(err){ + expect(err).to.not.exist; + fs.unlink('/1.txt', function(err){ + expect(err).to.not.exist; + fs.lstat('/test/2', function(err, stats){ + expect(err).to.not.exist; + expect(stats).to.exist; + expect(stats.type).to.equal('FILE'); + fs.readFile('/test/2', 'utf8', function(err, data){ + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('This is a file'); + done(); + }); + }); + }); + }); + }); + }); + }); + }); + + it('should copy a symlink as a file with \'links = true\' flag', function(done){ + var fs = util.fs(); + var shell = fs.Shell(); + + fs.mkdir('/test', function(err){ + expect(err).to.not.exist; + fs.writeFile('/apple.txt', 'This is a file', function(err){ + expect(err).to.not.exist; + fs.symlink('/apple.txt', '/apple', function(err){ + expect(err).to.not.exist; + shell.rsync('/apple', '/test', { links:true }, function(err){ + expect(err).to.not.exist; + fs.lstat('/test/apple', function(err, stats){ + expect(err).to.not.exist; + expect(stats).to.exist; + expect(stats.type).to.equal('SYMLINK'); + fs.readFile('/test/apple', 'utf8', function(err, data){ + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('This is a file'); + done(); + }); + }); + }); + }); + }); + }); + }); + + it('should copy a symlink as a file with \'links = false\' flag and update time with \'time: true\' flag', function(done){ + var fs = util.fs(); + var shell = fs.Shell(); + var mtime; + + fs.mkdir('/test', function(err){ + expect(err).to.not.exist; + fs.writeFile('/1.txt', 'This is a file', function(err){ + expect(err).to.not.exist; + fs.symlink('/1.txt', '/2', function(err){ + expect(err).to.not.exist; + fs.lstat('/2', function(err, stats){ + expect(err).to.not.exist; + expect(stats).to.exist; + mtime = stats.mtime; + shell.rsync('/2', '/test', { time: true }, function(err){ + expect(err).to.not.exist; + fs.unlink('/1.txt', function(err){ + expect(err).to.not.exist; + fs.lstat('/test/2', function(err, stats){ + expect(err).to.not.exist; + expect(stats).to.exist; + expect(stats.mtime).to.equal(mtime); + expect(stats.type).to.equal('FILE'); + fs.readFile('/test/2', 'utf8', function(err, data){ + expect(err).to.not.exist; + expect(data).to.exist; + expect(data).to.equal('This is a file'); + done(); + }); + }); + }); + }); + }) + }); + }); + }); + }); + + it('should succeed if the destination folder does not exist (Destination directory created)', function(done) { var fs = util.fs(); var shell = fs.Shell();