Merge pull request #331 from humphd/issue329

Fix #329 - Implement Shell.find()
This commit is contained in:
Alan K 2014-12-02 11:23:43 -05:00
commit d6c77abfa0
7 changed files with 424 additions and 5 deletions

View File

@ -1171,6 +1171,7 @@ var sh = new fs.Shell();
* [sh.cd(path, callback)](#cd)
* [sh.pwd()](#pwd)
* [sh.find(dir, [options], callback)](#find)
* [sh.ls(dir, [options], callback)](#ls)
* [sh.exec(path, [args], callback)](#exec)
* [sh.touch(path, [options], callback)](#touch)
@ -1199,6 +1200,66 @@ sh.cd('/dir1', function(err) {
Returns the shell's current working directory. See [sh.cd()](#cd).
#### sh.find(dir, [options], callback)<a name="find"></a>
Recursively walk a directory tree, reporting back all paths that were
found along the way. Asynchronous [find(1)](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/find.html
)
If given no options, `find` walks the given dir path
and the callback gives `function(err, found)`, where `found` is an array of
all paths discovered during a depth-first walk.
Valid options include a `regex` for pattern matching paths, allowing paths
to be ignored (e.g., ```regex: /\.bak$/``` to find all `.bak` files). You can
also use `name` and `path` to provide a [match pattern](https://github.com/isaacs/minimatch) for the basename and
dirname respectively (e.g., `{name: '*.js'}` to find all JavaScript files or
`{path: '*-modules'}` to only look in folders named `base-modules`, `foo-modules`, etc.).
Finally, you can also provide an `exec` function of the form `function(path, next)` where
`path` is the current path that was found and matches any provided `regex`
(NOTE: dir paths have an '/' appended), and `next` is a callback to call
when you are done processing the path.
Example:
```javascript
function processPath(path, next) {
// Process the path somehow, in this case we print it.
// Dir paths end with /
if(path.endsWith('/')) {
console.log('Found dir: ' + path);
} else {
console.log('Found file: ' + path);
}
// All done, let the process continue by invoking second arg:
next();
}
// Get every path (NOTE: no name or regex provided) below the root, depth first
sh.find('/', {exec: processPath}, function(err, found) {
/* find command is finished, `found` contains the flattened list as an Array */
});
// Find all files that look like map201.jpg, map202.jpg in the /data dir
sh.find('/data', {regex: /map20\d\.jpg$/, exec: processPath}, function(err) {
/* find command is finished */
});
// Find and delete all *.bak files under /app/user
sh.find('/app/user', {
name: '*.bak',
exec: function(path, next) {
sh.rm(path, next);
}
}, function callback(err, found) {
if(err) throw err;
if(found.length) {
console.log('Deleted the following ' + found.length + ' files: ', found);
}
});
```
#### sh.ls(dir, [options], callback)<a name="ls"></a>
Get the listing of a directory, returning an array of directory entries

View File

@ -25,7 +25,8 @@
},
"dependencies": {
"bower": "~1.3.8",
"base64-arraybuffer": "^0.1.2"
"base64-arraybuffer": "^0.1.2",
"minimatch": "^1.0.0"
},
"devDependencies": {
"chai": "~1.9.1",

View File

@ -66,7 +66,7 @@ function resolve() {
resolvedAbsolute = false;
for (var i = arguments.length - 1; i >= -1 && !resolvedAbsolute; i--) {
// XXXidbfs: we don't have process.cwd() so we use '/' as a fallback
// XXXfiler: we don't have process.cwd() so we use '/' as a fallback
var path = (i >= 0) ? arguments[i] : '/';
// Skip empty and invalid entries
@ -184,7 +184,7 @@ function basename(path, ext) {
if (ext && f.substr(-1 * ext.length) === ext) {
f = f.substr(0, f.length - ext.length);
}
// XXXidbfs: node.js just does `return f`
// XXXfiler: node.js just does `return f`
return f === "" ? "/" : f;
}
@ -206,7 +206,19 @@ function isNull(path) {
return false;
}
// XXXidbfs: we don't support path.exists() or path.existsSync(), which
// Make sure we don't double-add a trailing slash (e.g., '/' -> '//')
function addTrailing(path) {
return path.replace(/\/*$/, '/');
}
// Deal with multiple slashes at the end, one, or none
// and make sure we don't return the empty string.
function removeTrailing(path) {
path = path.replace(/\/*$/, '');
return path === '' ? '/' : path;
}
// XXXfiler: we don't support path.exists() or path.existsSync(), which
// are deprecated, and need a FileSystem instance to work. Use fs.stat().
module.exports = {
@ -220,5 +232,8 @@ module.exports = {
basename: basename,
extname: extname,
isAbsolute: isAbsolute,
isNull: isNull
isNull: isNull,
// Non-node but useful...
addTrailing: addTrailing,
removeTrailing: removeTrailing
};

View File

@ -3,6 +3,7 @@ var Errors = require('../errors.js');
var Environment = require('./environment.js');
var async = require('../../lib/async.js');
var Encoding = require('../encoding.js');
var minimatch = require('minimatch');
function Shell(fs, options) {
options = options || {};
@ -426,4 +427,120 @@ Shell.prototype.mkdirp = function(path, callback) {
_mkdirp(path, callback);
};
/**
* Recursively walk a directory tree, reporting back all paths
* that were found along the way. The `path` must be a dir.
* Valid options include a `regex` for pattern matching paths
* and an `exec` function of the form `function(path, next)` where
* `path` is the current path that was found (dir paths have an '/'
* appended) and `next` is a callback to call when done processing
* the current path, passing any error object back as the first argument.
* `find` returns a flat array of absolute paths for all matching/found
* paths as the final argument to the callback.
*/
Shell.prototype.find = function(path, options, callback) {
var sh = this;
var fs = sh.fs;
if(typeof options === 'function') {
callback = options;
options = {};
}
options = options || {};
callback = callback || function(){};
var exec = options.exec || function(path, next) { next(); };
var found = [];
if(!path) {
callback(new Errors.EINVAL('Missing path argument'));
return;
}
function processPath(path, callback) {
exec(path, function(err) {
if(err) {
callback(err);
return;
}
found.push(path);
callback();
});
}
function maybeProcessPath(path, callback) {
// Test the path against the user's regex, name, path primaries (if any)
// and remove any trailing slashes added previously.
var rawPath = Path.removeTrailing(path);
// Check entire path against provided regex, if any
if(options.regex && !options.regex.test(rawPath)) {
callback();
return;
}
// Check basename for matches against name primary, if any
if(options.name && !minimatch(Path.basename(rawPath), options.name)) {
callback();
return;
}
// Check dirname for matches against path primary, if any
if(options.path && !minimatch(Path.dirname(rawPath), options.path)) {
callback();
return;
}
processPath(path, callback);
}
function walk(path, callback) {
path = Path.resolve(sh.pwd(), path);
// The path is either a file or dir, and instead of doing
// a stat() to determine it first, we just try to readdir()
// and it will either work or not, and we handle the non-dir error.
fs.readdir(path, function(err, entries) {
if(err) {
if(err.code === 'ENOTDIR' /* file case, ignore error */) {
maybeProcessPath(path, callback);
} else {
callback(err);
}
return;
}
// Path is really a dir, add a trailing / and report it found
maybeProcessPath(Path.addTrailing(path), function(err) {
if(err) {
callback(err);
return;
}
entries = entries.map(function(entry) {
return Path.join(path, entry);
});
async.eachSeries(entries, walk, function(err) {
callback(err, found);
});
});
});
}
// Make sure we are starting with a dir path
fs.stat(path, function(err, stats) {
if(err) {
callback(err);
return;
}
if(!stats.isDirectory()) {
callback(new Errors.ENOTDIR(null, path));
return;
}
walk(path, callback);
});
};
module.exports = Shell;

View File

@ -55,6 +55,7 @@ require("./spec/shell/ls.spec");
require("./spec/shell/rm.spec");
require("./spec/shell/env.spec");
require("./spec/shell/mkdirp.spec");
require("./spec/shell/find.spec");
// Ported node.js tests (filenames match names in https://github.com/joyent/node/tree/master/test)
require("./spec/node-js/simple/test-fs-mkdir");

View File

@ -223,4 +223,22 @@ describe('path resolution', function() {
});
});
});
it('should properly add trailing slashes with Path.addTrailing()', function() {
var Path = Filer.Path;
expect(Path.addTrailing('/')).to.equal('/');
expect(Path.addTrailing('/////')).to.equal('/');
expect(Path.addTrailing('.')).to.equal('./');
expect(Path.addTrailing('/dir')).to.equal('/dir/');
expect(Path.addTrailing('/dir/')).to.equal('/dir/');
});
it('should properly remove trailing slashes with Path.removeTrailing()', function() {
var Path = Filer.Path;
expect(Path.removeTrailing('/')).to.equal('/');
expect(Path.removeTrailing('/////')).to.equal('/');
expect(Path.removeTrailing('./')).to.equal('.');
expect(Path.removeTrailing('/dir/')).to.equal('/dir');
expect(Path.removeTrailing('/dir//')).to.equal('/dir');
});
});

View File

@ -0,0 +1,206 @@
var Filer = require('../../..');
var util = require('../../lib/test-utils.js');
var expect = require('chai').expect;
describe('FileSystemShell.find', function() {
beforeEach(function(done) {
util.setup(function() {
var fs = util.fs();
/**
* Create a basic fs layout for each test:
*
* /
* --file1
* --file2
* --dir1/
* --file3
* --subdir1/
* --dir2/
* --file4
*/
fs.writeFile('/file1', 'file1', function(err) {
if(err) throw err;
fs.writeFile('/file2', 'file2', function(err) {
if(err) throw err;
fs.mkdir('/dir1', function(err) {
if(err) throw err;
fs.writeFile('/dir1/file3', 'file3', function(err) {
if(err) throw err;
fs.mkdir('/dir1/subdir1', function(err) {
if(err) throw err;
fs.mkdir('/dir2', function(err) {
if(err) throw err;
fs.writeFile('/dir2/file4', 'file4', function(err) {
if(err) throw err;
done();
});
});
});
});
});
});
});
});
});
afterEach(util.cleanup);
it('should be a function', function() {
var shell = util.shell();
expect(shell.find).to.be.a('function');
});
it('should fail when path does not exist', function(done) {
var shell = util.shell();
shell.find('/no-such-folder', function(err, found) {
expect(err).to.exist;
expect(err.code).to.equal('ENOENT');
expect(found).not.to.exist;
done();
});
});
it('should fail when path exists but is non-dir', function(done) {
var shell = util.shell();
shell.find('/file1', function(err, found) {
expect(err).to.exist;
expect(err.code).to.equal('ENOTDIR');
expect(found).not.to.exist;
done();
});
});
it('should find all paths in the filesystem with no options', function(done) {
var shell = util.shell();
shell.find('/', function(err, found) {
expect(err).not.to.exist;
var expected = [
'/',
'/file1',
'/file2',
'/dir1/',
'/dir1/file3',
'/dir1/subdir1/',
'/dir2/',
'/dir2/file4'
];
expect(found).to.deep.equal(expected);
done();
});
});
it('should get same paths in exec as are found when complete', function(done) {
var shell = util.shell();
var pathsSeen = [];
function processPath(path, next) {
pathsSeen.push(path);
next();
}
shell.find('/', {exec: processPath}, function(err, found) {
expect(err).not.to.exist;
expect(found).to.deep.equal(pathsSeen);
done();
});
});
it('should return only paths that match a regex pattern', function(done) {
var shell = util.shell();
shell.find('/', {regex: /file\d$/}, function(err, found) {
expect(err).not.to.exist;
var expected = [
'/file1',
'/file2',
'/dir1/file3',
'/dir2/file4'
];
expect(found).to.deep.equal(expected);
done();
});
});
it('should append a / to the end of a dir path', function(done) {
var shell = util.shell();
var dirsSeen = 0;
function endsWith(str, suffix) {
var lastIndex = str.lastIndexOf(suffix);
return (lastIndex !== -1) && (lastIndex + suffix.length === str.length);
}
function processPath(path, next) {
expect(endsWith(path, '/')).to.be.true;
dirsSeen++;
next();
}
shell.find('/', {regex: /dir\d$/, exec: processPath}, function(err) {
expect(err).not.to.exist;
expect(dirsSeen).to.equal(3);
done();
});
});
it('should only look below the specified dir path', function(done) {
var shell = util.shell();
shell.find('/dir1', function(err, found) {
expect(err).not.to.exist;
var expected = [
'/dir1/',
'/dir1/file3',
'/dir1/subdir1/'
];
expect(found).to.deep.equal(expected);
done();
});
});
it('should allow using options.name to match basename with a pattern', function(done) {
var shell = util.shell();
shell.find('/', {name: 'file*'}, function(err, found) {
expect(err).not.to.exist;
var expected = [
'/file1',
'/file2',
'/dir1/file3',
'/dir2/file4'
];
expect(found).to.deep.equal(expected);
done();
});
});
it('should allow using options.path to match dirname with a pattern', function(done) {
var shell = util.shell();
shell.find('/', {name: '*ir1*'}, function(err, found) {
expect(err).not.to.exist;
var expected = [
'/dir1/',
'/dir1/subdir1/'
];
expect(found).to.deep.equal(expected);
done();
});
});
});