lostecho/ripgrep
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

libripgrep: initial commit introducing libripgrep

Browse Source 
libripgrep is not any one library, but rather, a collection of libraries
that roughly separate the following key distinct phases in a grep
implementation:

  1. Pattern matching (e.g., by a regex engine).
  2. Searching a file using a pattern matcher.
  3. Printing results.

Ultimately, both (1) and (3) are defined by de-coupled interfaces, of
which there may be multiple implementations. Namely, (1) is satisfied by
the `Matcher` trait in the `grep-matcher` crate and (3) is satisfied by
the `Sink` trait in the `grep2` crate. The searcher (2) ties everything
together and finds results using a matcher and reports those results
using a `Sink` implementation.

Closes #162
This commit is contained in:
Andrew Gallant
2018-04-29 09:29:52 -04:00
parent 0958837ee1
commit d9ca529356
68 changed files with 18010 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

592
grep-searcher/src/searcher/core.rs Normal file
View File

@@ -0,0 +1,592 @@
use std::cmp;
use memchr::memchr;
use grep_matcher::{LineMatchKind, Matcher};
use lines::{self, LineStep};
use line_buffer::BinaryDetection;
use searcher::{Config, Range, Searcher};
use sink::{
Sink, SinkError,
SinkFinish, SinkContext, SinkContextKind, SinkMatch,
};
#[derive(Debug)]
pub struct Core<'s, M: 's, S> {
config: &'s Config,
matcher: M,
searcher: &'s Searcher,
sink: S,
binary: bool,
pos: usize,
absolute_byte_offset: u64,
binary_byte_offset: Option<usize>,
line_number: Option<u64>,
last_line_counted: usize,
last_line_visited: usize,
after_context_left: usize,
has_sunk: bool,
}
impl<'s, M: Matcher, S: Sink> Core<'s, M, S> {
pub fn new(
searcher: &'s Searcher,
matcher: M,
sink: S,
binary: bool,
) -> Core<'s, M, S> {
let line_number =
if searcher.config.line_number {
Some(1)
} else {
None
};
let core = Core {
config: &searcher.config,
matcher: matcher,
searcher: searcher,
sink: sink,
binary: binary,
pos: 0,
absolute_byte_offset: 0,
binary_byte_offset: None,
line_number: line_number,
last_line_counted: 0,
last_line_visited: 0,
after_context_left: 0,
has_sunk: false,
};
if !core.searcher.multi_line_with_matcher(&core.matcher) {
if core.is_line_by_line_fast() {
trace!("searcher core: will use fast line searcher");
} else {
trace!("searcher core: will use slow line searcher");
}
}
core
}
pub fn pos(&self) -> usize {
self.pos
}
pub fn set_pos(&mut self, pos: usize) {
self.pos = pos;
}
pub fn binary_byte_offset(&self) -> Option<u64> {
self.binary_byte_offset.map(|offset| offset as u64)
}
pub fn matcher(&self) -> &M {
&self.matcher
}
pub fn matched(
&mut self,
buf: &[u8],
range: &Range,
) -> Result<bool, S::Error> {
self.sink_matched(buf, range)
}
pub fn begin(&mut self) -> Result<bool, S::Error> {
self.sink.begin(&self.searcher)
}
pub fn finish(
&mut self,
byte_count: u64,
binary_byte_offset: Option<u64>,
) -> Result<(), S::Error> {
self.sink.finish(
&self.searcher,
&SinkFinish {
byte_count,
binary_byte_offset,
})
}
pub fn match_by_line(&mut self, buf: &[u8]) -> Result<bool, S::Error> {
if self.is_line_by_line_fast() {
self.match_by_line_fast(buf)
} else {
self.match_by_line_slow(buf)
}
}
pub fn roll(&mut self, buf: &[u8]) -> usize {
let consumed =
if self.config.max_context() == 0 {
buf.len()
} else {
// It might seem like all we need to care about here is just
// the "before context," but in order to sink the context
// separator (when before_context==0 and after_context>0), we
// need to know something about the position of the previous
// line visited, even if we're at the beginning of the buffer.
let context_start = lines::preceding(
buf,
self.config.line_term.as_byte(),
self.config.max_context(),
);
let consumed = cmp::max(context_start, self.last_line_visited);
consumed
};
self.count_lines(buf, consumed);
self.absolute_byte_offset += consumed as u64;
self.last_line_counted = 0;
self.last_line_visited = 0;
self.set_pos(buf.len() - consumed);
consumed
}
pub fn detect_binary(&mut self, buf: &[u8], range: &Range) -> bool {
if self.binary_byte_offset.is_some() {
return true;
}
let binary_byte = match self.config.binary.0 {
BinaryDetection::Quit(b) => b,
_ => return false,
};
if let Some(i) = memchr(binary_byte, &buf[*range]) {
self.binary_byte_offset = Some(range.start() + i);
true
} else {
false
}
}
pub fn before_context_by_line(
&mut self,
buf: &[u8],
upto: usize,
) -> Result<bool, S::Error> {
if self.config.before_context == 0 {
return Ok(true);
}
let range = Range::new(self.last_line_visited, upto);
if range.is_empty() {
return Ok(true);
}
let before_context_start = range.start() + lines::preceding(
&buf[range],
self.config.line_term.as_byte(),
self.config.before_context - 1,
);
let range = Range::new(before_context_start, range.end());
let mut stepper = LineStep::new(
self.config.line_term.as_byte(),
range.start(),
range.end(),
);
while let Some(line) = stepper.next_match(buf) {
if !self.sink_break_context(line.start())? {
return Ok(false);
}
if !self.sink_before_context(buf, &line)? {
return Ok(false);
}
}
Ok(true)
}
pub fn after_context_by_line(
&mut self,
buf: &[u8],
upto: usize,
) -> Result<bool, S::Error> {
if self.after_context_left == 0 {
return Ok(true);
}
let range = Range::new(self.last_line_visited, upto);
let mut stepper = LineStep::new(
self.config.line_term.as_byte(),
range.start(),
range.end(),
);
while let Some(line) = stepper.next_match(buf) {
if !self.sink_after_context(buf, &line)? {
return Ok(false);
}
if self.after_context_left == 0 {
break;
}
}
Ok(true)
}
pub fn other_context_by_line(
&mut self,
buf: &[u8],
upto: usize,
) -> Result<bool, S::Error> {
let range = Range::new(self.last_line_visited, upto);
let mut stepper = LineStep::new(
self.config.line_term.as_byte(),
range.start(),
range.end(),
);
while let Some(line) = stepper.next_match(buf) {
if !self.sink_other_context(buf, &line)? {
return Ok(false);
}
}
Ok(true)
}
fn match_by_line_slow(&mut self, buf: &[u8]) -> Result<bool, S::Error> {
debug_assert!(!self.searcher.multi_line_with_matcher(&self.matcher));
let range = Range::new(self.pos(), buf.len());
let mut stepper = LineStep::new(
self.config.line_term.as_byte(),
range.start(),
range.end(),
);
while let Some(line) = stepper.next_match(buf) {
let matched = {
// Stripping the line terminator is necessary to prevent some
// classes of regexes from matching the empty position *after*
// the end of the line. For example, `(?m)^$` will match at
// position (2, 2) in the string `a\n`.
let slice = lines::without_terminator(
&buf[line],
self.config.line_term,
);
match self.matcher.shortest_match(slice) {
Err(err) => return Err(S::Error::error_message(err)),
Ok(result) => result.is_some(),
}
};
self.set_pos(line.end());
if matched != self.config.invert_match {
if !self.before_context_by_line(buf, line.start())? {
return Ok(false);
}
if !self.sink_matched(buf, &line)? {
return Ok(false);
}
} else if self.after_context_left >= 1 {
if !self.sink_after_context(buf, &line)? {
return Ok(false);
}
} else if self.config.passthru {
if !self.sink_other_context(buf, &line)? {
return Ok(false);
}
}
}
Ok(true)
}
fn match_by_line_fast(&mut self, buf: &[u8]) -> Result<bool, S::Error> {
debug_assert!(!self.config.passthru);
while !buf[self.pos()..].is_empty() {
if self.config.invert_match {
if !self.match_by_line_fast_invert(buf)? {
return Ok(false);
}
} else if let Some(line) = self.find_by_line_fast(buf)? {
if self.config.max_context() > 0 {
if !self.after_context_by_line(buf, line.start())? {
return Ok(false);
}
if !self.before_context_by_line(buf, line.start())? {
return Ok(false);
}
}
self.set_pos(line.end());
if !self.sink_matched(buf, &line)? {
return Ok(false);
}
} else {
break;
}
}
if !self.after_context_by_line(buf, buf.len())? {
return Ok(false);
}
self.set_pos(buf.len());
Ok(true)
}
#[inline(always)]
fn match_by_line_fast_invert(
&mut self,
buf: &[u8],
) -> Result<bool, S::Error> {
assert!(self.config.invert_match);
let invert_match = match self.find_by_line_fast(buf)? {
None => {
let range = Range::new(self.pos(), buf.len());
self.set_pos(range.end());
range
}
Some(line) => {
let range = Range::new(self.pos(), line.start());
self.set_pos(line.end());
range
}
};
if invert_match.is_empty() {
return Ok(true);
}
if !self.after_context_by_line(buf, invert_match.start())? {
return Ok(false);
}
if !self.before_context_by_line(buf, invert_match.start())? {
return Ok(false);
}
let mut stepper = LineStep::new(
self.config.line_term.as_byte(),
invert_match.start(),
invert_match.end(),
);
while let Some(line) = stepper.next_match(buf) {
if !self.sink_matched(buf, &line)? {
return Ok(false);
}
}
Ok(true)
}
#[inline(always)]
fn find_by_line_fast(
&self,
buf: &[u8],
) -> Result<Option<Range>, S::Error> {
debug_assert!(!self.searcher.multi_line_with_matcher(&self.matcher));
debug_assert!(self.is_line_by_line_fast());
let mut pos = self.pos();
while !buf[pos..].is_empty() {
match self.matcher.find_candidate_line(&buf[pos..]) {
Err(err) => return Err(S::Error::error_message(err)),
Ok(None) => return Ok(None),
Ok(Some(LineMatchKind::Confirmed(i))) => {
let line = lines::locate(
buf,
self.config.line_term.as_byte(),
Range::zero(i).offset(pos),
);
// If we matched beyond the end of the buffer, then we
// don't report this as a match.
if line.start() == buf.len() {
pos = buf.len();
continue;
}
return Ok(Some(line));
}
Ok(Some(LineMatchKind::Candidate(i))) => {
let line = lines::locate(
buf,
self.config.line_term.as_byte(),
Range::zero(i).offset(pos),
);
// We need to strip the line terminator here to match the
// semantics of line-by-line searching. Namely, regexes
// like `(?m)^$` can match at the final position beyond a
// line terminator, which is non-sensical in line oriented
// matching.
let slice = lines::without_terminator(
&buf[line],
self.config.line_term,
);
match self.matcher.is_match(slice) {
Err(err) => return Err(S::Error::error_message(err)),
Ok(true) => return Ok(Some(line)),
Ok(false) => {
pos = line.end();
continue;
}
}
}
}
}
Ok(None)
}
#[inline(always)]
fn sink_matched(
&mut self,
buf: &[u8],
range: &Range,
) -> Result<bool, S::Error> {
if self.binary && self.detect_binary(buf, range) {
return Ok(false);
}
if !self.sink_break_context(range.start())? {
return Ok(false);
}
self.count_lines(buf, range.start());
let offset = self.absolute_byte_offset + range.start() as u64;
let linebuf =
if self.config.line_term.is_crlf() {
// Normally, a line terminator is never part of a match, but
// if the line terminator is CRLF, then it's possible for `\r`
// to end up in the match, which we generally don't want. So
// we strip it here.
lines::without_terminator(&buf[*range], self.config.line_term)
} else {
&buf[*range]
};
let keepgoing = self.sink.matched(
&self.searcher,
&SinkMatch {
line_term: self.config.line_term,
bytes: linebuf,
absolute_byte_offset: offset,
line_number: self.line_number,
},
)?;
if !keepgoing {
return Ok(false);
}
self.last_line_visited = range.end();
self.after_context_left = self.config.after_context;
self.has_sunk = true;
Ok(true)
}
fn sink_before_context(
&mut self,
buf: &[u8],
range: &Range,
) -> Result<bool, S::Error> {
if self.binary && self.detect_binary(buf, range) {
return Ok(false);
}
self.count_lines(buf, range.start());
let offset = self.absolute_byte_offset + range.start() as u64;
let keepgoing = self.sink.context(
&self.searcher,
&SinkContext {
line_term: self.config.line_term,
bytes: &buf[*range],
kind: SinkContextKind::Before,
absolute_byte_offset: offset,
line_number: self.line_number,
},
)?;
if !keepgoing {
return Ok(false);
}
self.last_line_visited = range.end();
self.has_sunk = true;
Ok(true)
}
fn sink_after_context(
&mut self,
buf: &[u8],
range: &Range,
) -> Result<bool, S::Error> {
assert!(self.after_context_left >= 1);
if self.binary && self.detect_binary(buf, range) {
return Ok(false);
}
self.count_lines(buf, range.start());
let offset = self.absolute_byte_offset + range.start() as u64;
let keepgoing = self.sink.context(
&self.searcher,
&SinkContext {
line_term: self.config.line_term,
bytes: &buf[*range],
kind: SinkContextKind::After,
absolute_byte_offset: offset,
line_number: self.line_number,
},
)?;
if !keepgoing {
return Ok(false);
}
self.last_line_visited = range.end();
self.after_context_left -= 1;
self.has_sunk = true;
Ok(true)
}
fn sink_other_context(
&mut self,
buf: &[u8],
range: &Range,
) -> Result<bool, S::Error> {
if self.binary && self.detect_binary(buf, range) {
return Ok(false);
}
self.count_lines(buf, range.start());
let offset = self.absolute_byte_offset + range.start() as u64;
let keepgoing = self.sink.context(
&self.searcher,
&SinkContext {
line_term: self.config.line_term,
bytes: &buf[*range],
kind: SinkContextKind::Other,
absolute_byte_offset: offset,
line_number: self.line_number,
},
)?;
if !keepgoing {
return Ok(false);
}
self.last_line_visited = range.end();
self.has_sunk = true;
Ok(true)
}
fn sink_break_context(
&mut self,
start_of_line: usize,
) -> Result<bool, S::Error> {
let is_gap = self.last_line_visited < start_of_line;
let any_context =
self.config.before_context > 0
|| self.config.after_context > 0;
if !any_context || !self.has_sunk || !is_gap {
Ok(true)
} else {
self.sink.context_break(&self.searcher)
}
}
fn count_lines(&mut self, buf: &[u8], upto: usize) {
if let Some(ref mut line_number) = self.line_number {
if self.last_line_counted >= upto {
return;
}
let slice = &buf[self.last_line_counted..upto];
let count = lines::count(slice, self.config.line_term.as_byte());
*line_number += count;
self.last_line_counted = upto;
}
}
fn is_line_by_line_fast(&self) -> bool {
debug_assert!(!self.searcher.multi_line_with_matcher(&self.matcher));
if self.config.passthru {
return false;
}
if let Some(line_term) = self.matcher.line_terminator() {
if line_term == self.config.line_term {
return true;
}
}
if let Some(non_matching) = self.matcher.non_matching_bytes() {
// If the line terminator is CRLF, we don't actually need to care
// whether the regex can match `\r` or not. Namely, a `\r` is
// neither necessary nor sufficient to terminate a line. A `\n` is
// always required.
if non_matching.contains(self.config.line_term.as_byte()) {
return true;
}
}
false
}
}

1506
grep-searcher/src/searcher/glue.rs Normal file
View File

File diff suppressed because it is too large Load Diff

106
grep-searcher/src/searcher/mmap.rs Normal file
View File

@@ -0,0 +1,106 @@
use std::fs::File;
use std::path::Path;
use memmap::Mmap;
/// Controls the strategy used for determining when to use memory maps.
///
/// If a searcher is called in circumstances where it is possible to use memory
/// maps, and memory maps are enabled, then it will attempt to do so if it
/// believes it will make the search faster.
///
/// By default, memory maps are disabled.
#[derive(Clone, Debug)]
pub struct MmapChoice(MmapChoiceImpl);
#[derive(Clone, Debug)]
enum MmapChoiceImpl {
Auto,
Never,
}
impl Default for MmapChoice {
fn default() -> MmapChoice {
MmapChoice(MmapChoiceImpl::Never)
}
}
impl MmapChoice {
/// Use memory maps when they are believed to be advantageous.
///
/// The heuristics used to determine whether to use a memory map or not
/// may depend on many things, including but not limited to, file size
/// and platform.
///
/// If memory maps are unavailable or cannot be used for a specific input,
/// then normal OS read calls are used instead.
///
/// # Safety
///
/// This constructor is not safe because there is no obvious way to
/// encapsulate the safety of file backed memory maps on all platforms
/// without simultaneously negating some or all of their benefits.
///
/// The specific contract the caller is required to uphold isn't precise,
/// but it basically amounts to something like, "the caller guarantees that
/// the underlying file won't be mutated." This, of course, isn't feasible
/// in many environments. However, command line tools may still decide to
/// take the risk of, say, a `SIGBUS` occurring while attempting to read a
/// memory map.
pub unsafe fn auto() -> MmapChoice {
MmapChoice(MmapChoiceImpl::Auto)
}
/// Never use memory maps, no matter what. This is the default.
pub fn never() -> MmapChoice {
MmapChoice(MmapChoiceImpl::Never)
}
/// Return a memory map if memory maps are enabled and if creating a
/// memory from the given file succeeded and if memory maps are believed
/// to be advantageous for performance.
///
/// If this does attempt to open a memory map and it fails, then `None`
/// is returned and the corresponding error (along with the file path, if
/// present) is logged at the debug level.
pub(crate) fn open(
&self,
file: &File,
path: Option<&Path>,
) -> Option<Mmap> {
if !self.is_enabled() {
return None;
}
if cfg!(target_os = "macos") {
// I guess memory maps on macOS aren't great. Should re-evaluate.
return None;
}
// SAFETY: This is acceptable because the only way `MmapChoiceImpl` can
// be `Auto` is if the caller invoked the `auto` constructor, which
// is itself not safe. Thus, this is a propagation of the caller's
// assertion that using memory maps is safe.
match unsafe { Mmap::map(file) } {
Ok(mmap) => Some(mmap),
Err(err) => {
if let Some(path) = path {
debug!(
"{}: failed to open memory map: {}",
path.display(),
err
);
} else {
debug!("failed to open memory map: {}", err);
}
None
}
}
}
/// Whether this strategy may employ memory maps or not.
pub(crate) fn is_enabled(&self) -> bool {
match self.0 {
MmapChoiceImpl::Auto => true,
MmapChoiceImpl::Never => false,
}
}
}

956
grep-searcher/src/searcher/mod.rs Normal file
View File

@@ -0,0 +1,956 @@
use std::cell::RefCell;
use std::cmp;
use std::fmt;
use std::fs::File;
use std::io::{self, Read};
use std::path::Path;
use encoding_rs;
use encoding_rs_io::DecodeReaderBytesBuilder;
use grep_matcher::{LineTerminator, Match, Matcher};
use line_buffer::{
self, BufferAllocation, LineBuffer, LineBufferBuilder, LineBufferReader,
DEFAULT_BUFFER_CAPACITY, alloc_error,
};
use searcher::glue::{ReadByLine, SliceByLine, MultiLine};
use sink::{Sink, SinkError};
pub use self::mmap::MmapChoice;
mod core;
mod glue;
mod mmap;
/// We use this type alias since we want the ergonomics of a matcher's `Match`
/// type, but in practice, we use it for arbitrary ranges, so give it a more
/// accurate name. This is only used in the searcher's internals.
type Range = Match;
/// The behavior of binary detection while searching.
///
/// Binary detection is the process of _heuristically_ identifying whether a
/// given chunk of data is binary or not, and then taking an action based on
/// the result of that heuristic. The motivation behind detecting binary data
/// is that binary data often indicates data that is undesirable to search
/// using textual patterns. Of course, there are many cases in which this isn't
/// true, which is why binary detection is disabled by default.
///
/// Unfortunately, binary detection works differently depending on the type of
/// search being executed:
///
/// 1. When performing a search using a fixed size buffer, binary detection is
/// applied to the buffer's contents as it is filled. Binary detection must
/// be applied to the buffer directly because binary files may not contain
/// line terminators, which could result in exorbitant memory usage.
/// 2. When performing a search using memory maps or by reading data off the
/// heap, then binary detection is only guaranteed to be applied to the
/// parts corresponding to a match. When `Quit` is enabled, then the first
/// few KB of the data are searched for binary data.
#[derive(Clone, Debug, Default)]
pub struct BinaryDetection(line_buffer::BinaryDetection);
impl BinaryDetection {
/// No binary detection is performed. Data reported by the searcher may
/// contain arbitrary bytes.
///
/// This is the default.
pub fn none() -> BinaryDetection {
BinaryDetection(line_buffer::BinaryDetection::None)
}
/// Binary detection is performed by looking for the given byte.
///
/// When searching is performed using a fixed size buffer, then the
/// contents of that buffer are always searched for the presence of this
/// byte. If it is found, then the underlying data is considered binary
/// and the search stops as if it reached EOF.
///
/// When searching is performed with the entire contents mapped into
/// memory, then binary detection is more conservative. Namely, only a
/// fixed sized region at the beginning of the contents are detected for
/// binary data. As a compromise, any subsequent matching (or context)
/// lines are also searched for binary data. If binary data is detected at
/// any point, then the search stops as if it reached EOF.
pub fn quit(binary_byte: u8) -> BinaryDetection {
BinaryDetection(line_buffer::BinaryDetection::Quit(binary_byte))
}
// TODO(burntsushi): Figure out how to make binary conversion work. This
// permits implementing GNU grep's default behavior, which is to zap NUL
// bytes but still execute a search (if a match is detected, then GNU grep
// stops and reports that a match was found but doesn't print the matching
// line itself).
//
// This behavior is pretty simple to implement using the line buffer (and
// in fact, it is already implemented and tested), since there's a fixed
// size buffer that we can easily write to. The issue arises when searching
// a `&[u8]` (whether on the heap or via a memory map), since this isn't
// something we can easily write to.
/// The given byte is searched in all contents read by the line buffer. If
/// it occurs, then it is replaced by the line terminator. The line buffer
/// guarantees that this byte will never be observable by callers.
#[allow(dead_code)]
fn convert(binary_byte: u8) -> BinaryDetection {
BinaryDetection(line_buffer::BinaryDetection::Convert(binary_byte))
}
}
/// An encoding to use when searching.
///
/// An encoding can be used to configure a
/// [`SearcherBuilder`](struct.SearchBuilder.html)
/// to transcode source data from an encoding to UTF-8 before searching.
///
/// An `Encoding` will always be cheap to clone.
#[derive(Clone, Debug)]
pub struct Encoding(&'static encoding_rs::Encoding);
impl Encoding {
/// Create a new encoding for the specified label.
///
/// The encoding label provided is mapped to an encoding via the set of
/// available choices specified in the
/// [Encoding Standard](https://encoding.spec.whatwg.org/#concept-encoding-get).
/// If the given label does not correspond to a valid encoding, then this
/// returns an error.
pub fn new(label: &str) -> Result<Encoding, ConfigError> {
let label = label.as_bytes();
match encoding_rs::Encoding::for_label_no_replacement(label) {
Some(encoding) => Ok(Encoding(encoding)),
None => {
Err(ConfigError::UnknownEncoding { label: label.to_vec() })
}
}
}
}
/// The internal configuration of a searcher. This is shared among several
/// search related types, but is only ever written to by the SearcherBuilder.
#[derive(Clone, Debug)]
pub struct Config {
/// The line terminator to use.
line_term: LineTerminator,
/// Whether to invert matching.
invert_match: bool,
/// The number of lines after a match to include.
after_context: usize,
/// The number of lines before a match to include.
before_context: usize,
/// Whether to enable unbounded context or not.
passthru: bool,
/// Whether to count line numbers.
line_number: bool,
/// The maximum amount of heap memory to use.
///
/// When not given, no explicit limit is enforced. When set to `0`, then
/// only the memory map search strategy is available.
heap_limit: Option<usize>,
/// The memory map strategy.
mmap: MmapChoice,
/// The binary data detection strategy.
binary: BinaryDetection,
/// Whether to enable matching across multiple lines.
multi_line: bool,
/// An encoding that, when present, causes the searcher to transcode all
/// input from the encoding to UTF-8.
encoding: Option<Encoding>,
}
impl Default for Config {
fn default() -> Config {
Config {
line_term: LineTerminator::default(),
invert_match: false,
after_context: 0,
before_context: 0,
passthru: false,
line_number: true,
heap_limit: None,
mmap: MmapChoice::default(),
binary: BinaryDetection::default(),
multi_line: false,
encoding: None,
}
}
}
impl Config {
/// Return the maximal amount of lines needed to fulfill this
/// configuration's context.
///
/// If this returns `0`, then no context is ever needed.
fn max_context(&self) -> usize {
cmp::max(self.before_context, self.after_context)
}
/// Build a line buffer from this configuration.
fn line_buffer(&self) -> LineBuffer {
let mut builder = LineBufferBuilder::new();
builder
.line_terminator(self.line_term.as_byte())
.binary_detection(self.binary.0);
if let Some(limit) = self.heap_limit {
let (capacity, additional) =
if limit <= DEFAULT_BUFFER_CAPACITY {
(limit, 0)
} else {
(DEFAULT_BUFFER_CAPACITY, limit - DEFAULT_BUFFER_CAPACITY)
};
builder
.capacity(capacity)
.buffer_alloc(BufferAllocation::Error(additional));
}
builder.build()
}
}
/// An error that can occur when building a searcher.
///
/// This error occurs when a non-sensical configuration is present when trying
/// to construct a `Searcher` from a `SearcherBuilder`.
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum ConfigError {
/// Indicates that the heap limit configuration prevents all possible
/// search strategies from being used. For example, if the heap limit is
/// set to 0 and memory map searching is disabled or unavailable.
SearchUnavailable,
/// Occurs when a matcher reports a line terminator that is different than
/// the one configured in the searcher.
MismatchedLineTerminators {
/// The matcher's line terminator.
matcher: LineTerminator,
/// The searcher's line terminator.
searcher: LineTerminator,
},
/// Occurs when no encoding could be found for a particular label.
UnknownEncoding {
/// The provided encoding label that could not be found.
label: Vec<u8>,
},
/// Hints that destructuring should not be exhaustive.
///
/// This enum may grow additional variants, so this makes sure clients
/// don't count on exhaustive matching. (Otherwise, adding a new variant
/// could break existing code.)
#[doc(hidden)]
__Nonexhaustive,
}
impl ::std::error::Error for ConfigError {
fn description(&self) -> &str { "grep-searcher configuration error" }
}
impl fmt::Display for ConfigError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
ConfigError::SearchUnavailable => {
write!(f, "grep config error: no available searchers")
}
ConfigError::MismatchedLineTerminators { matcher, searcher } => {
write!(
f,
"grep config error: mismatched line terminators, \
matcher has {:?} but searcher has {:?}",
matcher,
searcher
)
}
ConfigError::UnknownEncoding { ref label } => {
write!(
f,
"grep config error: unknown encoding: {}",
String::from_utf8_lossy(label),
)
}
_ => panic!("BUG: unexpected variant found"),
}
}
}
/// A builder for configuring a searcher.
///
/// A search builder permits specifying the configuration of a searcher,
/// including options like whether to invert the search or to enable multi
/// line search.
///
/// Once a searcher has been built, it is beneficial to reuse that searcher
/// for multiple searches, if possible.
#[derive(Clone, Debug)]
pub struct SearcherBuilder {
config: Config,
}
impl Default for SearcherBuilder {
fn default() -> SearcherBuilder {
SearcherBuilder::new()
}
}
impl SearcherBuilder {
/// Create a new searcher builder with a default configuration.
pub fn new() -> SearcherBuilder {
SearcherBuilder {
config: Config::default(),
}
}
/// Build a searcher with the given matcher.
pub fn build(&self) -> Searcher {
let mut config = self.config.clone();
if config.passthru {
config.before_context = 0;
config.after_context = 0;
}
let mut decode_builder = DecodeReaderBytesBuilder::new();
decode_builder
.encoding(self.config.encoding.as_ref().map(|e| e.0))
.utf8_passthru(true)
.bom_override(true);
Searcher {
config: config,
decode_builder: decode_builder,
decode_buffer: RefCell::new(vec![0; 8 * (1<<10)]),
line_buffer: RefCell::new(self.config.line_buffer()),
multi_line_buffer: RefCell::new(vec![]),
}
}
/// Set the line terminator that is used by the searcher.
///
/// When using a searcher, if the matcher provided has a line terminator
/// set, then it must be the same as this one. If they aren't, building
/// a searcher will return an error.
///
/// By default, this is set to `b'\n'`.
pub fn line_terminator(
&mut self,
line_term: LineTerminator,
) -> &mut SearcherBuilder {
self.config.line_term = line_term;
self
}
/// Whether to invert matching, whereby lines that don't match are reported
/// instead of reporting lines that do match.
///
/// By default, this is disabled.
pub fn invert_match(&mut self, yes: bool) -> &mut SearcherBuilder {
self.config.invert_match = yes;
self
}
/// Whether to count and include line numbers with matching lines.
///
/// This is enabled by default. There is a small performance penalty
/// associated with computing line numbers, so this can be disabled when
/// this isn't desirable.
pub fn line_number(&mut self, yes: bool) -> &mut SearcherBuilder {
self.config.line_number = yes;
self
}
/// Whether to enable multi line search or not.
///
/// When multi line search is enabled, matches *may* match across multiple
/// lines. Conversely, when multi line search is disabled, it is impossible
/// for any match to span more than one line.
///
/// **Warning:** multi line search requires having the entire contents to
/// search mapped in memory at once. When searching files, memory maps
/// will be used if possible and if they are enabled, which avoids using
/// your program's heap. However, if memory maps cannot be used (e.g.,
/// for searching streams like `stdin` or if transcoding is necessary),
/// then the entire contents of the stream are read on to the heap before
/// starting the search.
///
/// This is disabled by default.
pub fn multi_line(&mut self, yes: bool) -> &mut SearcherBuilder {
self.config.multi_line = yes;
self
}
/// Whether to include a fixed number of lines after every match.
///
/// When this is set to a non-zero number, then the searcher will report
/// `line_count` contextual lines after every match.
///
/// This is set to `0` by default.
pub fn after_context(
&mut self,
line_count: usize,
) -> &mut SearcherBuilder {
self.config.after_context = line_count;
self
}
/// Whether to include a fixed number of lines before every match.
///
/// When this is set to a non-zero number, then the searcher will report
/// `line_count` contextual lines before every match.
///
/// This is set to `0` by default.
pub fn before_context(
&mut self,
line_count: usize,
) -> &mut SearcherBuilder {
self.config.before_context = line_count;
self
}
/// Whether to enable the "passthru" feature or not.
///
/// When passthru is enabled, it effectively treats all non-matching lines
/// as contextual lines. In other words, enabling this is akin to
/// requesting an unbounded number of before and after contextual lines.
///
/// When passthru mode is enabled, any `before_context` or `after_context`
/// settings are ignored by setting them to `0`.
///
/// This is disabled by default.
pub fn passthru(&mut self, yes: bool) -> &mut SearcherBuilder {
self.config.passthru = yes;
self
}
/// Set an approximate limit on the amount of heap space used by a
/// searcher.
///
/// The heap limit is enforced in two scenarios:
///
/// * When searching using a fixed size buffer, the heap limit controls
/// how big this buffer is allowed to be. Assuming contexts are disabled,
/// the minimum size of this buffer is the length (in bytes) of the
/// largest single line in the contents being searched. If any line
/// exceeds the heap limit, then an error will be returned.
/// * When performing a multi line search, a fixed size buffer cannot be
/// used. Thus, the only choices are to read the entire contents on to
/// the heap, or use memory maps. In the former case, the heap limit set
/// here is enforced.
///
/// If a heap limit is set to `0`, then no heap space is used. If there are
/// no alternative strategies available for searching without heap space
/// (e.g., memory maps are disabled), then the searcher wil return an error
/// immediately.
///
/// By default, no limit is set.
pub fn heap_limit(
&mut self,
bytes: Option<usize>,
) -> &mut SearcherBuilder {
self.config.heap_limit = bytes;
self
}
/// Set the strategy to employ use of memory maps.
///
/// Currently, there are only two strategies that can be employed:
///
/// * **Automatic** - A searcher will use heuristics, including but not
/// limited to file size and platform, to determine whether to use memory
/// maps or not.
/// * **Never** - Memory maps will never be used. If multi line search is
/// enabled, then the entire contents will be read on to the heap before
/// searching begins.
///
/// The default behavior is **never**. Generally speaking, and perhaps
/// against conventional wisdom, memory maps don't necessarily enable
/// faster searching. For example, depending on the platform, using memory
/// maps while searching a large directory can actually be quite a bit
/// slower than using normal read calls because of the overhead of managing
/// the memory maps.
///
/// Memory maps can be faster in some cases however. On some platforms,
/// when searching a very large file that *is already in memory*, it can
/// be slightly faster to search it as a memory map instead of using
/// normal read calls.
///
/// Finally, memory maps have a somewhat complicated safety story in Rust.
/// If you aren't sure whether enabling memory maps is worth it, then just
/// don't bother with it.
///
/// **WARNING**: If your process is searching a file backed memory map
/// at the same time that file is truncated, then it's possible for the
/// process to terminate with a bus error.
pub fn memory_map(
&mut self,
strategy: MmapChoice,
) -> &mut SearcherBuilder {
self.config.mmap = strategy;
self
}
/// Set the binary detection strategy.
///
/// The binary detection strategy determines not only how the searcher
/// detects binary data, but how it responds to the presence of binary
/// data. See the [`BinaryDetection`](struct.BinaryDetection.html) type
/// for more information.
///
/// By default, binary detection is disabled.
pub fn binary_detection(
&mut self,
detection: BinaryDetection,
) -> &mut SearcherBuilder {
self.config.binary = detection;
self
}
/// Set the encoding used to read the source data before searching.
///
/// When an encoding is provided, then the source data is _unconditionally_
/// transcoded using the encoding, unless a BOM is present. If a BOM is
/// present, then the encoding indicated by the BOM is used instead. If the
/// transcoding process encounters an error, then bytes are replaced with
/// the Unicode replacement codepoint.
///
/// When no encoding is specified (the default), then BOM sniffing is used
/// to determine whether the source data is UTF-8 or UTF-16, and
/// transcoding will be performed automatically. If no BOM could be found,
/// then the source data is searched _as if_ it were UTF-8. However, so
/// long as the source data is at least ASCII compatible, then it is
/// possible for a search to produce useful results.
pub fn encoding(
&mut self,
encoding: Option<Encoding>,
) -> &mut SearcherBuilder {
self.config.encoding = encoding;
self
}
}
/// A searcher executes searches over a haystack and writes results to a caller
/// provided sink.
///
/// Matches are detected via implementations of the `Matcher` trait, which must
/// be provided by the caller when executing a search.
///
/// When possible, a searcher should be reused.
#[derive(Clone, Debug)]
pub struct Searcher {
/// The configuration for this searcher.
///
/// We make most of these settings available to users of `Searcher` via
/// public API methods, which can be queried in implementations of `Sink`
/// if necessary.
config: Config,
/// A builder for constructing a streaming reader that transcodes source
/// data according to either an explicitly specified encoding or via an
/// automatically detected encoding via BOM sniffing.
///
/// When no transcoding is needed, then the transcoder built will pass
/// through the underlying bytes with no additional overhead.
decode_builder: DecodeReaderBytesBuilder,
/// A buffer that is used for transcoding scratch space.
decode_buffer: RefCell<Vec<u8>>,
/// A line buffer for use in line oriented searching.
///
/// We wrap it in a RefCell to permit lending out borrows of `Searcher`
/// to sinks. We still require a mutable borrow to execute a search, so
/// we statically prevent callers from causing RefCell to panic at runtime
/// due to a borrowing violation.
line_buffer: RefCell<LineBuffer>,
/// A buffer in which to store the contents of a reader when performing a
/// multi line search. In particular, multi line searches cannot be
/// performed incrementally, and need the entire haystack in memory at
/// once.
multi_line_buffer: RefCell<Vec<u8>>,
}
impl Searcher {
/// Create a new searcher with a default configuration.
///
/// To configure the searcher (e.g., invert matching, enable memory maps,
/// enable contexts, etc.), use the
/// [`SearcherBuilder`](struct.SearcherBuilder.html).
pub fn new() -> Searcher {
SearcherBuilder::new().build()
}
/// Execute a search over the file with the given path and write the
/// results to the given sink.
///
/// If memory maps are enabled and the searcher heuristically believes
/// memory maps will help the search run faster, then this will use
/// memory maps. For this reason, callers should prefer using this method
/// or `search_file` over the more generic `search_reader` when possible.
pub fn search_path<P, M, S>(
&mut self,
matcher: M,
path: P,
write_to: S,
) -> Result<(), S::Error>
where P: AsRef<Path>,
M: Matcher,
S: Sink,
{
let path = path.as_ref();
let file = File::open(path).map_err(S::Error::error_io)?;
self.search_file_maybe_path(matcher, Some(path), &file, write_to)
}
/// Execute a search over a file and write the results to the given sink.
///
/// If memory maps are enabled and the searcher heuristically believes
/// memory maps will help the search run faster, then this will use
/// memory maps. For this reason, callers should prefer using this method
/// or `search_path` over the more generic `search_reader` when possible.
pub fn search_file<M, S>(
&mut self,
matcher: M,
file: &File,
write_to: S,
) -> Result<(), S::Error>
where M: Matcher,
S: Sink,
{
self.search_file_maybe_path(matcher, None, file, write_to)
}
fn search_file_maybe_path<M, S>(
&mut self,
matcher: M,
path: Option<&Path>,
file: &File,
write_to: S,
) -> Result<(), S::Error>
where M: Matcher,
S: Sink,
{
if let Some(mmap) = self.config.mmap.open(file, path) {
trace!("{:?}: searching via memory map", path);
return self.search_slice(matcher, &mmap, write_to);
}
// Fast path for multi-line searches of files when memory maps are
// not enabled. This pre-allocates a buffer roughly the size of the
// file, which isn't possible when searching an arbitrary io::Read.
if self.multi_line_with_matcher(&matcher) {
trace!("{:?}: reading entire file on to heap for mulitline", path);
self.fill_multi_line_buffer_from_file::<S>(file)?;
trace!("{:?}: searching via multiline strategy", path);
MultiLine::new(
self,
matcher,
&*self.multi_line_buffer.borrow(),
write_to,
).run()
} else {
trace!("{:?}: searching using generic reader", path);
self.search_reader(matcher, file, write_to)
}
}
/// Execute a search over any implementation of `io::Read` and write the
/// results to the given sink.
///
/// When possible, this implementation will search the reader incrementally
/// without reading it into memory. In some cases---for example, if multi
/// line search is enabled---an incremental search isn't possible and the
/// given reader is consumed completely and placed on the heap before
/// searching begins. For this reason, when multi line search is enabled,
/// one should try to use higher level APIs (e.g., searching by file or
/// file path) so that memory maps can be used if they are available and
/// enabled.
pub fn search_reader<M, R, S>(
&mut self,
matcher: M,
read_from: R,
write_to: S,
) -> Result<(), S::Error>
where M: Matcher,
R: io::Read,
S: Sink,
{
self.check_config(&matcher).map_err(S::Error::error_config)?;
let mut decode_buffer = self.decode_buffer.borrow_mut();
let read_from = self.decode_builder
.build_with_buffer(read_from, &mut *decode_buffer)
.map_err(S::Error::error_io)?;
if self.multi_line_with_matcher(&matcher) {
trace!("generic reader: reading everything to heap for multiline");
self.fill_multi_line_buffer_from_reader::<_, S>(read_from)?;
trace!("generic reader: searching via multiline strategy");
MultiLine::new(
self,
matcher,
&*self.multi_line_buffer.borrow(),
write_to,
).run()
} else {
let mut line_buffer = self.line_buffer.borrow_mut();
let rdr = LineBufferReader::new(read_from, &mut *line_buffer);
trace!("generic reader: searching via roll buffer strategy");
ReadByLine::new(self, matcher, rdr, write_to).run()
}
}
/// Execute a search over the given slice and write the results to the
/// given sink.
pub fn search_slice<M, S>(
&mut self,
matcher: M,
slice: &[u8],
write_to: S,
) -> Result<(), S::Error>
where M: Matcher,
S: Sink,
{
self.check_config(&matcher).map_err(S::Error::error_config)?;
// We can search the slice directly, unless we need to do transcoding.
if self.slice_needs_transcoding(slice) {
trace!("slice reader: needs transcoding, using generic reader");
return self.search_reader(matcher, slice, write_to);
}
if self.multi_line_with_matcher(&matcher) {
trace!("slice reader: searching via multiline strategy");
MultiLine::new(self, matcher, slice, write_to).run()
} else {
trace!("slice reader: searching via slice-by-line strategy");
SliceByLine::new(self, matcher, slice, write_to).run()
}
}
/// Check that the searcher's configuration and the matcher are consistent
/// with each other.
fn check_config<M: Matcher>(&self, matcher: M) -> Result<(), ConfigError> {
if self.config.heap_limit == Some(0)
&& !self.config.mmap.is_enabled()
{
return Err(ConfigError::SearchUnavailable);
}
let matcher_line_term = match matcher.line_terminator() {
None => return Ok(()),
Some(line_term) => line_term,
};
if matcher_line_term != self.config.line_term {
return Err(ConfigError::MismatchedLineTerminators {
matcher: matcher_line_term,
searcher: self.config.line_term,
});
}
Ok(())
}
/// Returns true if and only if the given slice needs to be transcoded.
fn slice_needs_transcoding(&self, slice: &[u8]) -> bool {
self.config.encoding.is_some() || slice_has_utf16_bom(slice)
}
}
/// The following methods permit querying the configuration of a searcher.
/// These can be useful in generic implementations of
/// [`Sink`](trait.Sink.html),
/// where the output may be tailored based on how the searcher is configured.
impl Searcher {
/// Returns the line terminator used by this searcher.
#[inline]
pub fn line_terminator(&self) -> LineTerminator {
self.config.line_term
}
/// Returns true if and only if this searcher is configured to invert its
/// search results. That is, matching lines are lines that do **not** match
/// the searcher's matcher.
#[inline]
pub fn invert_match(&self) -> bool {
self.config.invert_match
}
/// Returns true if and only if this searcher is configured to count line
/// numbers.
#[inline]
pub fn line_number(&self) -> bool {
self.config.line_number
}
/// Returns true if and only if this searcher is configured to perform
/// multi line search.
#[inline]
pub fn multi_line(&self) -> bool {
self.config.multi_line
}
/// Returns true if and only if this searcher will choose a multi-line
/// strategy given the provided matcher.
///
/// This may diverge from the result of `multi_line` in cases where the
/// searcher has been configured to execute a search that can report
/// matches over multiple lines, but where the matcher guarantees that it
/// will never produce a match over multiple lines.
pub fn multi_line_with_matcher<M: Matcher>(&self, matcher: M) -> bool {
if !self.multi_line() {
return false;
}
if let Some(line_term) = matcher.line_terminator() {
if line_term == self.line_terminator() {
return false;
}
}
if let Some(non_matching) = matcher.non_matching_bytes() {
// If the line terminator is CRLF, we don't actually need to care
// whether the regex can match `\r` or not. Namely, a `\r` is
// neither necessary nor sufficient to terminate a line. A `\n` is
// always required.
if non_matching.contains(self.line_terminator().as_byte()) {
return false;
}
}
true
}
/// Returns the number of "after" context lines to report. When context
/// reporting is not enabled, this returns `0`.
#[inline]
pub fn after_context(&self) -> usize {
self.config.after_context
}
/// Returns the number of "before" context lines to report. When context
/// reporting is not enabled, this returns `0`.
#[inline]
pub fn before_context(&self) -> usize {
self.config.before_context
}
/// Returns true if and only if the searcher has "passthru" mode enabled.
#[inline]
pub fn passthru(&self) -> bool {
self.config.passthru
}
/// Fill the buffer for use with multi-line searching from the given file.
/// This reads from the file until EOF or until an error occurs. If the
/// contents exceed the configured heap limit, then an error is returned.
fn fill_multi_line_buffer_from_file<S: Sink>(
&self,
file: &File,
) -> Result<(), S::Error> {
assert!(self.config.multi_line);
let mut decode_buffer = self.decode_buffer.borrow_mut();
let mut read_from = self.decode_builder
.build_with_buffer(file, &mut *decode_buffer)
.map_err(S::Error::error_io)?;
// If we don't have a heap limit, then we can defer to std's
// read_to_end implementation. fill_multi_line_buffer_from_reader will
// do this too, but since we have a File, we can be a bit smarter about
// pre-allocating here.
//
// If we're transcoding, then our pre-allocation might not be exact,
// but is probably still better than nothing.
if self.config.heap_limit.is_none() {
let mut buf = self.multi_line_buffer.borrow_mut();
buf.clear();
let cap = file
.metadata()
.map(|m| m.len() as usize + 1)
.unwrap_or(0);
buf.reserve(cap);
read_from.read_to_end(&mut *buf).map_err(S::Error::error_io)?;
return Ok(());
}
self.fill_multi_line_buffer_from_reader::<_, S>(read_from)
}
/// Fill the buffer for use with multi-line searching from the given
/// reader. This reads from the reader until EOF or until an error occurs.
/// If the contents exceed the configured heap limit, then an error is
/// returned.
fn fill_multi_line_buffer_from_reader<R: io::Read, S: Sink>(
&self,
mut read_from: R,
) -> Result<(), S::Error> {
assert!(self.config.multi_line);
let mut buf = self.multi_line_buffer.borrow_mut();
buf.clear();
// If we don't have a heap limit, then we can defer to std's
// read_to_end implementation...
let heap_limit = match self.config.heap_limit {
Some(heap_limit) => heap_limit,
None => {
read_from.read_to_end(&mut *buf).map_err(S::Error::error_io)?;
return Ok(());
}
};
if heap_limit == 0 {
return Err(S::Error::error_io(alloc_error(heap_limit)));
}
// ... otherwise we need to roll our own. This is likely quite a bit
// slower than what is optimal, but we avoid worry about memory safety
// until there's a compelling reason to speed this up.
buf.resize(cmp::min(DEFAULT_BUFFER_CAPACITY, heap_limit), 0);
let mut pos = 0;
loop {
let nread = match read_from.read(&mut buf[pos..]) {
Ok(nread) => nread,
Err(ref err) if err.kind() == io::ErrorKind::Interrupted => {
continue;
}
Err(err) => return Err(S::Error::error_io(err)),
};
if nread == 0 {
buf.resize(pos, 0);
return Ok(());
}
pos += nread;
if buf[pos..].is_empty() {
let additional = heap_limit - buf.len();
if additional == 0 {
return Err(S::Error::error_io(alloc_error(heap_limit)));
}
let limit = buf.len() + additional;
let doubled = 2 * buf.len();
buf.resize(cmp::min(doubled, limit), 0);
}
}
}
}
/// Returns true if and only if the given slice begins with a UTF-16 BOM.
///
/// This is used by the searcher to determine if a transcoder is necessary.
/// Otherwise, it is advantageous to search the slice directly.
fn slice_has_utf16_bom(slice: &[u8]) -> bool {
let enc = match encoding_rs::Encoding::for_bom(slice) {
None => return false,
Some((enc, _)) => enc,
};
[encoding_rs::UTF_16LE, encoding_rs::UTF_16BE].contains(&enc)
}
#[cfg(test)]
mod tests {
use testutil::{KitchenSink, RegexMatcher};
use super::*;
#[test]
fn config_error_heap_limit() {
let matcher = RegexMatcher::new("");
let sink = KitchenSink::new();
let mut searcher = SearcherBuilder::new()
.heap_limit(Some(0))
.build();
let res = searcher.search_slice(matcher, &[], sink);
assert!(res.is_err());
}
#[test]
fn config_error_line_terminator() {
let mut matcher = RegexMatcher::new("");
matcher.set_line_term(Some(LineTerminator::byte(b'z')));
let sink = KitchenSink::new();
let mut searcher = Searcher::new();
let res = searcher.search_slice(matcher, &[], sink);
assert!(res.is_err());
}
}