Add parallel recursive directory iterator.

This adds a new walk type in the `ignore` crate, `WalkParallel`, which
provides a way for recursively iterating over a set of paths in parallel
while respecting various ignore rules.

The API is a bit strange, as a closure producing a closure isn't
something one often sees, but it does seem to work well.

This also allowed us to simplify much of the worker logic in ripgrep
proper, where MultiWorker is now gone.

ignore/examples/walk.rs

@@ -1,28 +1,92 @@
-/*
+#![allow(dead_code, unused_imports, unused_mut, unused_variables)]
+
+extern crate crossbeam;
 extern crate ignore;
 extern crate walkdir;
 
 use std::env;
 use std::io::{self, Write};
-use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::thread;
 
-use ignore::ignore::IgnoreBuilder;
+use crossbeam::sync::MsQueue;
+use ignore::WalkBuilder;
 use walkdir::WalkDir;
 
 fn main() {
-    let path = env::args().nth(1).unwrap();
-    let ig = IgnoreBuilder::new().build();
-    let wd = WalkDir::new(path);
-    let walker = ignore::walk::Iter::new(ig, wd);
-
-    let mut stdout = io::BufWriter::new(io::stdout());
-    // let mut count = 0;
-    for dirent in walker {
-        // count += 1;
-        stdout.write(dirent.path().as_os_str().as_bytes()).unwrap();
-        stdout.write(b"\n").unwrap();
+    let mut path = env::args().nth(1).unwrap();
+    let mut parallel = false;
+    let mut simple = false;
+    let queue: Arc<MsQueue<Option<DirEntry>>> = Arc::new(MsQueue::new());
+    if path == "parallel" {
+        path = env::args().nth(2).unwrap();
+        parallel = true;
+    } else if path == "walkdir" {
+        path = env::args().nth(2).unwrap();
+        simple = true;
     }
-    // println!("{}", count);
+
+    let stdout_queue = queue.clone();
+    let stdout_thread = thread::spawn(move || {
+        let mut stdout = io::BufWriter::new(io::stdout());
+        while let Some(dent) = stdout_queue.pop() {
+            write_path(&mut stdout, dent.path());
+        }
+    });
+
+    if parallel {
+        let walker = WalkBuilder::new(path).threads(6).build_parallel();
+        walker.run(|| {
+            let queue = queue.clone();
+            Box::new(move |result| {
+                use ignore::WalkState::*;
+
+                queue.push(Some(DirEntry::Y(result.unwrap())));
+                Continue
+            })
+        });
+    } else if simple {
+        let mut stdout = io::BufWriter::new(io::stdout());
+        let walker = WalkDir::new(path);
+        for result in walker {
+            queue.push(Some(DirEntry::X(result.unwrap())));
+        }
+    } else {
+        let mut stdout = io::BufWriter::new(io::stdout());
+        let walker = WalkBuilder::new(path).build();
+        for result in walker {
+            queue.push(Some(DirEntry::Y(result.unwrap())));
+        }
+    }
+    queue.push(None);
+    stdout_thread.join().unwrap();
+}
+
+enum DirEntry {
+    X(walkdir::DirEntry),
+    Y(ignore::DirEntry),
+}
+
+impl DirEntry {
+    fn path(&self) -> &Path {
+        match *self {
+            DirEntry::X(ref x) => x.path(),
+            DirEntry::Y(ref y) => y.path(),
+        }
+    }
+}
+
+#[cfg(unix)]
+fn write_path<W: Write>(mut wtr: W, path: &Path) {
+    use std::os::unix::ffi::OsStrExt;
+    wtr.write(path.as_os_str().as_bytes()).unwrap();
+    wtr.write(b"\n").unwrap();
+}
+
+#[cfg(not(unix))]
+fn write_path<W: Write>(mut wtr: W, path: &Path) {
+    wtr.write(path.to_string_lossy().as_bytes()).unwrap();
+    wtr.write(b"\n").unwrap();
 }
-*/
-fn main() {}