Merge pull request #331 from humphd/issue329

Fix #329 - Implement Shell.find()

README.md

@@ -1171,6 +1171,7 @@ var sh = new fs.Shell();
 
 * [sh.cd(path, callback)](#cd)
 * [sh.pwd()](#pwd)
+* [sh.find(dir, [options], callback)](#find)
 * [sh.ls(dir, [options], callback)](#ls)
 * [sh.exec(path, [args], callback)](#exec)
 * [sh.touch(path, [options], callback)](#touch)
@@ -1199,6 +1200,66 @@ sh.cd('/dir1', function(err) {
 
 Returns the shell's current working directory. See [sh.cd()](#cd).
 
+#### sh.find(dir, [options], callback)<a name="find"></a>
+
+Recursively walk a directory tree, reporting back all paths that were
+found along the way. Asynchronous [find(1)](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/find.html
+)
+If given no options, `find` walks the given dir path
+and the callback gives `function(err, found)`, where `found` is an array of
+all paths discovered during a depth-first walk.
+
+Valid options include a `regex` for pattern matching paths, allowing paths
+to be ignored (e.g., ```regex: /\.bak$/``` to find all `.bak` files). You can
+also use `name` and `path` to provide a [match pattern](https://github.com/isaacs/minimatch) for the basename and
+dirname respectively (e.g., `{name: '*.js'}` to find all JavaScript files or
+`{path: '*-modules'}` to only look in folders named `base-modules`, `foo-modules`, etc.).
+Finally, you can also provide an `exec` function of the form `function(path, next)` where
+`path` is the current path that was found and matches any provided `regex`
+(NOTE: dir paths have an '/' appended), and `next` is a callback to call
+when you are done processing the path.
+
+Example:
+
+```javascript
+function processPath(path, next) {
+  // Process the path somehow, in this case we print it.
+  // Dir paths end with /
+  if(path.endsWith('/')) {
+    console.log('Found dir: ' + path);
+  } else {
+    console.log('Found file: ' + path);
+  }
+
+  // All done, let the process continue by invoking second arg:
+  next();
+}
+
+// Get every path (NOTE: no name or regex provided) below the root, depth first
+sh.find('/', {exec: processPath}, function(err, found) {
+  /* find command is finished, `found` contains the flattened list as an Array */
+});
+
+// Find all files that look like map201.jpg, map202.jpg in the /data dir
+sh.find('/data', {regex: /map20\d\.jpg$/, exec: processPath}, function(err) {
+  /* find command is finished */
+});
+
+// Find and delete all *.bak files under /app/user
+sh.find('/app/user', {
+  name: '*.bak',
+  exec: function(path, next) {
+    sh.rm(path, next);
+  }
+}, function callback(err, found) {
+  if(err) throw err;
+
+  if(found.length) {
+    console.log('Deleted the following ' + found.length + ' files: ', found);
+  }
+});
+```
+
 #### sh.ls(dir, [options], callback)<a name="ls"></a>
 
 Get the listing of a directory, returning an array of directory entries

package.json

@@ -25,7 +25,8 @@
   },
   "dependencies": {
     "bower": "~1.3.8",
-    "base64-arraybuffer": "^0.1.2"
+    "base64-arraybuffer": "^0.1.2",
+    "minimatch": "^1.0.0"
   },
   "devDependencies": {
     "chai": "~1.9.1",

src/path.js

@@ -66,7 +66,7 @@ function resolve() {
       resolvedAbsolute = false;
 
   for (var i = arguments.length - 1; i >= -1 && !resolvedAbsolute; i--) {
-    // XXXidbfs: we don't have process.cwd() so we use '/' as a fallback
+    // XXXfiler: we don't have process.cwd() so we use '/' as a fallback
     var path = (i >= 0) ? arguments[i] : '/';
 
     // Skip empty and invalid entries
@@ -184,7 +184,7 @@ function basename(path, ext) {
   if (ext && f.substr(-1 * ext.length) === ext) {
     f = f.substr(0, f.length - ext.length);
   }
-  // XXXidbfs: node.js just does `return f`
+  // XXXfiler: node.js just does `return f`
   return f === "" ? "/" : f;
 }
 
@@ -206,7 +206,19 @@ function isNull(path) {
   return false;
 }
 
-// XXXidbfs: we don't support path.exists() or path.existsSync(), which
+// Make sure we don't double-add a trailing slash (e.g., '/' -> '//')
+function addTrailing(path) {
+  return path.replace(/\/*$/, '/');
+}
+
+// Deal with multiple slashes at the end, one, or none
+// and make sure we don't return the empty string.
+function removeTrailing(path) {
+  path = path.replace(/\/*$/, '');
+  return path === '' ? '/' : path;
+}
+
+// XXXfiler: we don't support path.exists() or path.existsSync(), which
 // are deprecated, and need a FileSystem instance to work. Use fs.stat().
 
 module.exports = {
@@ -220,5 +232,8 @@ module.exports = {
   basename: basename,
   extname: extname,
   isAbsolute: isAbsolute,
-  isNull: isNull
+  isNull: isNull,
+  // Non-node but useful...
+  addTrailing: addTrailing,
+  removeTrailing: removeTrailing
 };

src/shell/shell.js

@@ -3,6 +3,7 @@ var Errors = require('../errors.js');
 var Environment = require('./environment.js');
 var async = require('../../lib/async.js');
 var Encoding = require('../encoding.js');
+var minimatch = require('minimatch');
 
 function Shell(fs, options) {
   options = options || {};
@@ -426,4 +427,120 @@ Shell.prototype.mkdirp = function(path, callback) {
   _mkdirp(path, callback);
 };
 
+/**
+ * Recursively walk a directory tree, reporting back all paths
+ * that were found along the way. The `path` must be a dir.
+ * Valid options include a `regex` for pattern matching paths
+ * and an `exec` function of the form `function(path, next)` where
+ * `path` is the current path that was found (dir paths have an '/'
+ * appended) and `next` is a callback to call when done processing
+ * the current path, passing any error object back as the first argument.
+ * `find` returns a flat array of absolute paths for all matching/found
+ * paths as the final argument to the callback.
+ */
+ Shell.prototype.find = function(path, options, callback) {
+  var sh = this;
+  var fs = sh.fs;
+  if(typeof options === 'function') {
+    callback = options;
+    options = {};
+  }
+  options = options || {};
+  callback = callback || function(){};
+
+  var exec = options.exec || function(path, next) { next(); };
+  var found = [];
+
+  if(!path) {
+    callback(new Errors.EINVAL('Missing path argument'));
+    return;
+  }
+
+  function processPath(path, callback) {
+    exec(path, function(err) {
+      if(err) {
+        callback(err);
+        return;
+      }
+
+      found.push(path);
+      callback();
+    });
+  }
+
+  function maybeProcessPath(path, callback) {
+    // Test the path against the user's regex, name, path primaries (if any)
+    // and remove any trailing slashes added previously.
+    var rawPath = Path.removeTrailing(path);
+
+    // Check entire path against provided regex, if any
+    if(options.regex && !options.regex.test(rawPath)) {
+      callback();
+      return;
+    }
+
+    // Check basename for matches against name primary, if any
+    if(options.name && !minimatch(Path.basename(rawPath), options.name)) {
+      callback();
+      return;
+    }
+
+    // Check dirname for matches against path primary, if any
+    if(options.path && !minimatch(Path.dirname(rawPath), options.path)) {
+      callback();
+      return;
+    }
+
+    processPath(path, callback);
+  }
+
+  function walk(path, callback) {
+    path = Path.resolve(sh.pwd(), path);
+
+    // The path is either a file or dir, and instead of doing
+    // a stat() to determine it first, we just try to readdir()
+    // and it will either work or not, and we handle the non-dir error.
+    fs.readdir(path, function(err, entries) {
+      if(err) {
+        if(err.code === 'ENOTDIR' /* file case, ignore error */) {
+          maybeProcessPath(path, callback);
+        } else {
+          callback(err);
+        }
+        return;
+      }
+
+      // Path is really a dir, add a trailing / and report it found
+      maybeProcessPath(Path.addTrailing(path), function(err) {
+        if(err) {
+          callback(err);
+          return;
+        }
+
+        entries = entries.map(function(entry) {
+          return Path.join(path, entry);
+        });
+
+        async.eachSeries(entries, walk, function(err) {
+          callback(err, found);
+        });
+      });
+    });
+  }
+
+  // Make sure we are starting with a dir path
+  fs.stat(path, function(err, stats) {
+    if(err) {
+      callback(err);
+      return;
+    }
+    if(!stats.isDirectory()) {
+      callback(new Errors.ENOTDIR(null, path));
+      return;
+    }
+
+    walk(path, callback);
+  });
+};
+
 module.exports = Shell;

tests/index.js

@@ -55,6 +55,7 @@ require("./spec/shell/ls.spec");
 require("./spec/shell/rm.spec");
 require("./spec/shell/env.spec");
 require("./spec/shell/mkdirp.spec");
+require("./spec/shell/find.spec");
 
 // Ported node.js tests (filenames match names in https://github.com/joyent/node/tree/master/test)
 require("./spec/node-js/simple/test-fs-mkdir");

tests/spec/path-resolution.spec.js

@@ -223,4 +223,22 @@ describe('path resolution', function() {
       });
     });
   });
+
+  it('should properly add trailing slashes with Path.addTrailing()', function() {
+    var Path = Filer.Path;
+    expect(Path.addTrailing('/')).to.equal('/');
+    expect(Path.addTrailing('/////')).to.equal('/');
+    expect(Path.addTrailing('.')).to.equal('./');
+    expect(Path.addTrailing('/dir')).to.equal('/dir/');
+    expect(Path.addTrailing('/dir/')).to.equal('/dir/');
+  });
+
+  it('should properly remove trailing slashes with Path.removeTrailing()', function() {
+    var Path = Filer.Path;
+    expect(Path.removeTrailing('/')).to.equal('/');
+    expect(Path.removeTrailing('/////')).to.equal('/');
+    expect(Path.removeTrailing('./')).to.equal('.');
+    expect(Path.removeTrailing('/dir/')).to.equal('/dir');
+    expect(Path.removeTrailing('/dir//')).to.equal('/dir');
+  });
 });

tests/spec/shell/find.spec.js

@@ -0,0 +1,206 @@
+var Filer = require('../../..');
+var util = require('../../lib/test-utils.js');
+var expect = require('chai').expect;
+
+describe('FileSystemShell.find', function() {
+  beforeEach(function(done) {
+    util.setup(function() {
+      var fs = util.fs();
+      /**
+       * Create a basic fs layout for each test:
+       *
+       * /
+       * --file1
+       * --file2
+       * --dir1/
+       *   --file3
+       *   --subdir1/
+       * --dir2/
+       *   --file4
+       */
+      fs.writeFile('/file1', 'file1', function(err) {
+        if(err) throw err;
+
+        fs.writeFile('/file2', 'file2', function(err) {
+          if(err) throw err;
+
+          fs.mkdir('/dir1', function(err) {
+            if(err) throw err;
+
+            fs.writeFile('/dir1/file3', 'file3', function(err) {
+              if(err) throw err;
+
+              fs.mkdir('/dir1/subdir1', function(err) {
+                if(err) throw err;
+
+                fs.mkdir('/dir2', function(err) {
+                  if(err) throw err;
+
+                  fs.writeFile('/dir2/file4', 'file4', function(err) {
+                    if(err) throw err;
+
+                    done();
+                  });
+                });
+              });
+            });
+          });
+        });
+      });
+    });
+  });
+  afterEach(util.cleanup);
+
+  it('should be a function', function() {
+    var shell = util.shell();
+    expect(shell.find).to.be.a('function');
+  });
+
+  it('should fail when path does not exist', function(done) {
+    var shell = util.shell();
+
+    shell.find('/no-such-folder', function(err, found) {
+      expect(err).to.exist;
+      expect(err.code).to.equal('ENOENT');
+      expect(found).not.to.exist;
+      done();
+    });
+  });
+
+  it('should fail when path exists but is non-dir', function(done) {
+    var shell = util.shell();
+
+    shell.find('/file1', function(err, found) {
+      expect(err).to.exist;
+      expect(err.code).to.equal('ENOTDIR');
+      expect(found).not.to.exist;
+      done();
+    });
+  });
+
+  it('should find all paths in the filesystem with no options', function(done) {
+    var shell = util.shell();
+
+    shell.find('/', function(err, found) {
+      expect(err).not.to.exist;
+
+      var expected = [
+        '/',
+        '/file1',
+        '/file2',
+        '/dir1/',
+        '/dir1/file3',
+        '/dir1/subdir1/',
+        '/dir2/',
+        '/dir2/file4'
+      ];
+      expect(found).to.deep.equal(expected);
+      done();
+    });
+  });
+
+  it('should get same paths in exec as are found when complete', function(done) {
+    var shell = util.shell();
+    var pathsSeen = [];
+
+    function processPath(path, next) {
+      pathsSeen.push(path);
+      next();
+    }
+
+    shell.find('/', {exec: processPath}, function(err, found) {
+      expect(err).not.to.exist;
+
+      expect(found).to.deep.equal(pathsSeen);
+      done();
+    });
+  });
+
+  it('should return only paths that match a regex pattern', function(done) {
+    var shell = util.shell();
+
+    shell.find('/', {regex: /file\d$/}, function(err, found) {
+      expect(err).not.to.exist;
+
+      var expected = [
+        '/file1',
+        '/file2',
+        '/dir1/file3',
+        '/dir2/file4'
+      ];
+      expect(found).to.deep.equal(expected);
+      done();
+    });    
+  });
+
+  it('should append a / to the end of a dir path', function(done) {
+    var shell = util.shell();
+    var dirsSeen = 0;
+
+    function endsWith(str, suffix) {
+      var lastIndex = str.lastIndexOf(suffix);
+      return (lastIndex !== -1) && (lastIndex + suffix.length === str.length);
+    }
+
+    function processPath(path, next) {
+      expect(endsWith(path, '/')).to.be.true;
+      dirsSeen++;
+      next();
+    }
+
+    shell.find('/', {regex: /dir\d$/, exec: processPath}, function(err) {
+      expect(err).not.to.exist;
+      expect(dirsSeen).to.equal(3);
+      done();
+    });    
+  });
+
+  it('should only look below the specified dir path', function(done) {
+    var shell = util.shell();
+
+    shell.find('/dir1', function(err, found) {
+      expect(err).not.to.exist;
+
+      var expected = [
+        '/dir1/',
+        '/dir1/file3',
+        '/dir1/subdir1/'
+      ];
+      expect(found).to.deep.equal(expected);
+      done();
+    });    
+  });
+
+  it('should allow using options.name to match basename with a pattern', function(done) {
+    var shell = util.shell();
+
+    shell.find('/', {name: 'file*'}, function(err, found) {
+      expect(err).not.to.exist;
+
+      var expected = [
+        '/file1',
+        '/file2',
+        '/dir1/file3',
+        '/dir2/file4'
+      ];
+      expect(found).to.deep.equal(expected);
+      done();
+    });
+  });
+
+  it('should allow using options.path to match dirname with a pattern', function(done) {
+    var shell = util.shell();
+
+    shell.find('/', {name: '*ir1*'}, function(err, found) {
+      expect(err).not.to.exist;
+
+      var expected = [
+        '/dir1/',
+        '/dir1/subdir1/'
+      ];
+      expect(found).to.deep.equal(expected);
+      done();
+    });
+  });
+
+});