LumeWeb
/
filer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added fs flag to sync to a 2nd local filesystem

This commit is contained in:
Petr Bouianov 2014-04-28 20:24:00 -04:00
parent 2bf6162c00
commit 4e86f469e8
2 changed files with 40 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -1352,6 +1352,7 @@ size: 5 //default 750. File chunk size in Kb.
checksum: false //default 'false'. False will skip files if their size AND modified times are the same (regardless of content difference). checksum: false //default 'false'. False will skip files if their size AND modified times are the same (regardless of content difference).
time: true //default 'false'. Preserves file modified time when syncing. time: true //default 'false'. Preserves file modified time when syncing.
links: true //default 'false'. Copies symlinks as links instead of resolving. links: true //default 'false'. Copies symlinks as links instead of resolving.
fs: new Filer.FileSystem() //takes an optional FS reference to sync the changes to. If absent syncs to/from the same FS.
Example: Example:

64
src/rsync.js
View File

@ -6,6 +6,7 @@ define(function(require) {
var async = require('async'); var async = require('async');
var cache = {}; var cache = {};
var options; var options;
var destFS;
//MD5 hashing for RSync //MD5 hashing for RSync
//Used from Node.js Anchor module //Used from Node.js Anchor module
@ -150,7 +151,7 @@ define(function(require) {
} }
function rsync (srcPath, destPath, opts, callback) { function rsync (srcPath, destPath, opts, callback) {
var self = this; var srcFS = this.fs;
if(typeof opts === 'function') { if(typeof opts === 'function') {
callback = opts; callback = opts;
options = {}; options = {};
@ -159,6 +160,7 @@ define(function(require) {
options.recursive = false; options.recursive = false;
options.time = false; options.time = false;
options.links = false; options.links = false;
destFS = srcFS;
} }
else { else {
options = opts || {}; options = opts || {};
@ -167,6 +169,7 @@ define(function(require) {
options.recursive = options.recursive || false; options.recursive = options.recursive || false;
options.time = options.time || false; options.time = options.time || false;
options.links = options.links || false; options.links = options.links || false;
destFS = options.fs || srcFS;
callback = callback || function() {}; callback = callback || function() {};
} }
if(srcPath === null || srcPath === '/' || srcPath === '') { if(srcPath === null || srcPath === '/' || srcPath === '') {
@ -176,13 +179,13 @@ define(function(require) {
function getSrcList(path, callback) { function getSrcList(path, callback) {
var result = []; var result = [];
self.fs.lstat(path, function(err, stats) { srcFS.lstat(path, function(err, stats) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
} }
if(stats.isDirectory()) { if(stats.isDirectory()) {
self.fs.readdir(path, function(err, entries) { srcFS.readdir(path, function(err, entries) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
@ -190,7 +193,7 @@ define(function(require) {
function getSrcContents(_name, callback) { function getSrcContents(_name, callback) {
var name = Path.join(path, _name); var name = Path.join(path, _name);
self.fs.lstat(name, function(error, stats) { srcFS.lstat(name, function(error, stats) {
if(error) { if(error) {
callback(error); callback(error);
@ -257,12 +260,12 @@ define(function(require) {
}); });
} else if(entry.type === 'FILE' || !options.links) { } else if(entry.type === 'FILE' || !options.links) {
if(!options.checksum) { if(!options.checksum) {
self.fs.stat(Path.join(destPath, entry.path), function(err, stat) { destFS.stat(Path.join(destPath, entry.path), function(err, stat) {
if(!err && stat.mtime === entry.modified && stat.size === entry.size) { if(!err && stat.mtime === entry.modified && stat.size === entry.size) {
callback(); callback();
} }
else { else {
checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) { checksum.call(destFS, Path.join(destPath, entry.path), function(err, checksums) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
@ -276,7 +279,7 @@ define(function(require) {
}); });
} }
else { else {
checksum.call(self, Path.join(destPath, entry.path), function(err, checksums) { checksum.call(destFS, Path.join(destPath, entry.path), function(err, checksums) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
@ -290,7 +293,7 @@ define(function(require) {
} }
else if(entry.type === 'SYMLINK'){ else if(entry.type === 'SYMLINK'){
if(!options.checksum) { if(!options.checksum) {
self.fs.stat(Path.join(destPath, entry.path), function(err, stat){ destFS.stat(Path.join(destPath, entry.path), function(err, stat){
if(!err && stat.mtime === entry.modified && stat.size === entry.size) { if(!err && stat.mtime === entry.modified && stat.size === entry.size) {
callback(); callback();
} }
@ -317,7 +320,7 @@ define(function(require) {
callback(err); callback(err);
return; return;
} }
self.mkdirp(destPath, function(err) { destFS.Shell().mkdirp(destPath, function(err) {
getChecksums(destPath, result, function(err, result) { getChecksums(destPath, result, function(err, result) {
if(err) { if(err) {
callback(err); callback(err);
@ -327,18 +330,20 @@ define(function(require) {
callback(); callback();
return; return;
} }
diff.call(self, srcPath, result, function(err, diffs) { diff.call(srcFS, srcPath, result, function(err, diffs) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
} }
sync.call(self, destPath, diffs, function(err) { destFS.readdir('/', function(err, stuff){
sync(destPath, diffs, function(err) {
callback(err); callback(err);
}); });
}); });
}); });
}); });
}); });
});
} }
/* RSync Checksum Function /* RSync Checksum Function
@ -349,8 +354,8 @@ define(function(require) {
* MIT Licensed * MIT Licensed
*/ */
function checksum (path, callback) { function checksum (path, callback) {
var self = this; //var destFS = this;
self.fs.readFile(path, function (err, data) { destFS.readFile(path, function (err, data) {
if (!err) { if (!err) {
// cache file // cache file
cache[path] = data; cache[path] = data;
@ -394,17 +399,17 @@ define(function(require) {
* MIT Licensed * MIT Licensed
*/ */
function diff(path, checksums, callback) { function diff(path, checksums, callback) {
var self = this; var srcFS = this;
// roll through the file // roll through the file
var diffs = []; var diffs = [];
self.fs.lstat(path, function(err, stat) { srcFS.lstat(path, function(err, stat) {
if(stat.isDirectory()) { if(stat.isDirectory()) {
async.each(checksums, getDiff, function(err) { async.each(checksums, getDiff, function(err) {
callback(err, diffs); callback(err, diffs);
}); });
} }
else if (stat.isFile() || !options.links) { else if (stat.isFile() || !options.links) {
self.fs.readFile(path, function (err, data) { srcFS.readFile(path, function (err, data) {
if (err) { return callback(err); } if (err) { return callback(err); }
diffs.push({ diffs.push({
diff: roll(data, checksums[0].checksum, options.size), diff: roll(data, checksums[0].checksum, options.size),
@ -415,12 +420,12 @@ define(function(require) {
}); });
} }
else if (stat.isSymbolicLink()) { else if (stat.isSymbolicLink()) {
self.fs.readlink(path, function(err, linkContents) { srcFS.readlink(path, function(err, linkContents) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
} }
self.fs.lstat(path, function(err, stats){ srcFS.lstat(path, function(err, stats){
if(err) { if(err) {
callback(err); callback(err);
return; return;
@ -438,7 +443,7 @@ define(function(require) {
function getDiff(entry, callback) { function getDiff(entry, callback) {
if(entry.hasOwnProperty('contents')) { if(entry.hasOwnProperty('contents')) {
diff.call(self, Path.join(path, entry.path), entry.contents, function(err, stuff) { diff.call(srcFS, Path.join(path, entry.path), entry.contents, function(err, stuff) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
@ -469,7 +474,7 @@ define(function(require) {
}); });
}); });
} else { } else {
self.fs.readFile(Path.join(path,entry.path), function (err, data) { srcFS.readFile(Path.join(path,entry.path), function (err, data) {
if (err) { return callback(err); } if (err) { return callback(err); }
diffs.push({ diffs.push({
diff: roll(data, entry.checksum, options.size), diff: roll(data, entry.checksum, options.size),
@ -490,8 +495,6 @@ define(function(require) {
* MIT Licensed * MIT Licensed
*/ */
function sync(path, diff, callback) { function sync(path, diff, callback) {
var self = this;
function syncEach(entry, callback) { function syncEach(entry, callback) {
//get slice of raw file from block's index //get slice of raw file from block's index
@ -500,9 +503,8 @@ define(function(require) {
var end = start + options.size > raw.length ? raw.length : start + options.size; var end = start + options.size > raw.length ? raw.length : start + options.size;
return raw.subarray(start, end); return raw.subarray(start, end);
} }
if(entry.hasOwnProperty('contents')) { if(entry.hasOwnProperty('contents')) {
sync.call(self, Path.join(path, entry.path), entry.contents, function(err) { sync(Path.join(path, entry.path), entry.contents, function(err) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
@ -511,9 +513,8 @@ define(function(require) {
}); });
} }
else if (entry.hasOwnProperty('link')) { else if (entry.hasOwnProperty('link')) {
var syncPath = Path.join(path,entry.path); var syncPath = Path.join(path,entry.path);
self.fs.symlink(entry.link, syncPath, function(err){ destFS.symlink(entry.link, syncPath, function(err){
if(err) { if(err) {
callback(err); callback(err);
return; return;
@ -541,13 +542,13 @@ define(function(require) {
} }
} }
delete cache[Path.join(path,entry.path)]; delete cache[Path.join(path,entry.path)];
self.fs.writeFile(Path.join(path,entry.path), buf, function(err) { destFS.writeFile(Path.join(path,entry.path), buf, function(err) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
} }
if(options.time) { if(options.time) {
self.fs.utimes(Path.join(path,entry.path), entry.modified, entry.modified, function(err) { destFS.utimes(Path.join(path,entry.path), entry.modified, entry.modified, function(err) {
if(err) { if(err) {
callback(err); callback(err);
return; return;
@ -562,10 +563,15 @@ define(function(require) {
} }
} }
destFS.mkdir(path, function(err){
if(err && err.code != "EEXIST"){
callback(err);
return;
}
async.each(diff, syncEach, function(err) { async.each(diff, syncEach, function(err) {
callback(err); callback(err);
}); });
});
} }
function appendBuffer( buffer1, buffer2 ) { function appendBuffer( buffer1, buffer2 ) {