1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
// Copyright (c) Peter Sanders. All rights reserved. // Date: 2018-02-04 extern crate buf_redux; extern crate num; extern crate regex; use std::io::Read; use std::io::BufRead; use std::marker::Sized; use std::str; use buf_redux::BufReader; use regex::Regex; // For regex "delim" use num::Integer; use num::Float; #[cfg(test)] mod tests; const DEFAULT_BUF_SIZE: usize = 1024 * 8; /// Rust implementation of java.util.Scanner pub struct Scanner<R: Read + Sized> { stream: BufReader<R>, // Underlying stream object we are handling. delim: Regex, // Delimiter used to specify word boundaries. radix: u32, // Base in which we parse numeric types. // See `impl BufRead for Scanner` block for details. // TODO(hxtk): Implement BufRead. Pending Issue #5. } /// Implements the meta-methods of Scanner that affect how the data stream /// is processed, e.g., delimiter, parsing radix, etc. impl<R: Read + Sized> Scanner<R> { /// Sets the delimiter to be some pre-compiled regex and return it /// for behavioral consistency. pub fn set_delim(&mut self, delim: Regex) -> &Regex { self.delim = delim; &self.delim } /// Sets the delimiter to be a string literal. The resulting delimiting /// expression is guaranteed to only interpret the literal passed in, /// i.e., this method **cannot** be used to simultaneously compile and set /// an arbitrary regular expression. /// /// We return the compiled delimiting expression. pub fn set_delim_str(&mut self, delim: &str) -> &Regex { // We escape any regex metacharacters, so the result is a // string literal that is guaranteed to be a safe regex. self.delim = Regex::new(regex::escape(delim).as_str()).unwrap(); &self.delim } /// Return the delimiter for `Scanner.next()` /// and methods that depend on it. pub fn get_delim(&self) -> &Regex { &self.delim } /// Sets the radix in which numbers are parsed. This value must be on /// the closed range [2, 36], such that alphabet characters represent /// values greater than 9 in bases exceeding 10. /// /// We return the postcondition value of the radix, which is the input /// if the input is within the valid range or the precondition value /// otherwise. pub fn set_radix(&mut self, radix: u32) -> u32 { if 1 < radix && radix <= 36 { self.radix = radix; } self.radix } /// Retrieve the radix on which we perform numeric parsing. pub fn get_radix(&self) -> u32 { self.radix } } /// Implements the methods of Scanner that affect the underlying data stream impl<R: Read + Sized> Scanner<R> { /// Creates a new instance of Scanner on some object implementing `Read` pub fn new(stream: R) -> Scanner<R> { Scanner { stream: BufReader::new(stream), // We can safely unwrap this regex because it is hard-coded. delim: Regex::new(r"\s+").unwrap(), radix: 10, } } /// Creates a new instance of Scanner using a BufReader with a specified /// buffer size. /// /// This instantiator allows the user to specify the capacity of the buffer. /// Its primary use-case is unit testing this module, i.e., it would be /// cumbersome to write 64KB test strings so one might specify a /// capacity of only a few bytes in order to test what happens at the pub fn with_capacity(size: usize, stream: R) -> Scanner<R> { Scanner { stream: BufReader::with_capacity(size, stream), // We can safely unwrap this regex because it is hard-coded. delim: Regex::new(r"\s+").unwrap(), radix: 10, } } /// Returns `Some(String)` containing the next string if there is one. /// Otherwise returns `None`. /// /// We first consume all leading `delim`s that fit within the buffer of the /// underlying `BufRead`, then attempt to read everything until /// (but excluding) the next `delim` which is entirely contained within a /// single buffer. We guarantee this will behave as expected if the longest /// single precendent delimiter is no larger than the size of the buffer. /// /// Otherwise it will fail. pub fn next(&mut self) -> Option<String> { self.consume_leading_delims(); let delim_idx; let mut res = String::new(); let mut last_length = 0; loop { println!("result string: \"{}\"", res.as_str()); let delta = { if let Ok(_size) = self.stream.read_into_buf() { let buf = self.stream.get_buf(); // If the buffer is not a valid utf-8 string, we exit the // method with `None` result. if str::from_utf8(buf).is_err() { return None; } // The check above guarantees `unwrap` will succeed. res = String::from(str::from_utf8(buf).unwrap()); let old_len = last_length; last_length = buf.len(); buf.len() - old_len } else { 0 } }; if delta == 0 { delim_idx = res.len(); break; } // If a delimiter is found within the result string, we stop reading // and mark the location. Everything up to here should be consumed. if let Some(found) = self.delim.find(res.as_str()) { delim_idx = found.start(); break; } else { self.stream.grow(DEFAULT_BUF_SIZE); } } self.stream.consume(delim_idx); res.truncate(delim_idx); res.shrink_to_fit(); Some(res) } /// Read up to (but excluding) the next `\n` character. /// If there are any leading `delim`s, they will be included in the /// returned string. /// /// NOTE: unlike `next()` we do consume the trailing `\n`, if it exists. pub fn next_line(&mut self) -> Option<String> { let mut res = String::new(); if let Ok(_size) = self.stream.read_line(&mut res) { if let Some(end) = res.pop() { if end == '\n' { Some(res) } else { res.push(end); Some(res) } } else { None } } else { None } } /// Attempts to retrieve the next integer of the specified (or inferred) /// type. Even if this fails, we still consume `next`. /// /// The default radix for this parsing is 10, but users may specify a /// one-time arbitrary radix using `Scanner.next_int_radix(u32)` /// or persistently using `Scanner.set_radix(u32)`. pub fn next_int<T: Integer>(&mut self) -> Option<T> { if let Some(mut input) = self.next() { // Strip commas. Numbers with commas are considered valid // but Rust does not recognize them in its default behavior. while let Some(comma_idx) = input.rfind(',') { input.remove(comma_idx); } match <T>::from_str_radix(input.as_str(), self.radix) { Ok(res) => Some(res), Err(_e) => None, } } else { None } } /// Returns the next integer in some arbitrary base on [2, 36]. /// /// If the radix provided is outside of this range, we do nothing. /// Otherwise, we will consume `next()` even if it is not a valid integer. /// /// NOTE: If one means to repeatedly parse in a fixed, arbitrary base, /// it is more efficient to use `Scanner.set_radix(u32)` followed by /// `Scanner.next_int` with no radix argument. pub fn next_int_radix<T: Integer>(&mut self, radix: u32) -> Option<T> { if radix < 2 || radix > 36 { None } else { let old_radix = self.radix; self.set_radix(radix); let res = self.next_int::<T>(); self.set_radix(old_radix); res } } /// Attempts to retrieve the next floating-point number of the specified /// (or inferred) type. Even if this fails, we still consume `next`. /// /// Note that this method is based on `Scanner.next()`, so the delimiter /// is still the same. pub fn next_float<T: Float>(&mut self) -> Option<T> { if let Some(mut input) = self.next() { // Strip commas. Numbers with commas are considered valid // but Rust does not recognize them in its default behavior. while let Some(comma_idx) = input.rfind(',') { input.remove(comma_idx); } match <T>::from_str_radix(input.as_str(), self.radix) { Ok(res) => Some(res), Err(_e) => None, } } else { None } } /// Returns the next float in some arbitrary base on [2, 36]. /// /// If the radix provided is outside of this range, we do nothing. /// Otherwise, we will consume `next()` even if it is not a valid integer. /// /// NOTE: If one means to repeatedly parse in a fixed, arbitrary base, /// it is more efficient to use `Scanner.set_radix(u32)` followed by /// `Scanner.next_float` with no radix argument. pub fn next_float_radix<T: Float>(&mut self, radix: u32) -> Option<T> { if radix < 2 || radix > 36 { None } else { let old_radix = self.radix; self.set_radix(radix); let res = self.next_float::<T>(); self.set_radix(old_radix); res } } } /// Private helper functions for Scanner impl<R: Read + Sized> Scanner<R> { /// When we read `Scanner.next()`, we must first skip over any strings /// in the delimiting language before we begin reading the target text. fn consume_leading_delims(&mut self) { loop { let length = { if let Ok(buf) = self.stream.fill_buf() { if let Ok(text) = str::from_utf8(buf) { if let Some(found) = self.delim.find(text) { if found.start() > 0 { return; } found.end() } else { 0 } } else { 0 } } else { 0 } }; if length == 0 { return; } else { self.stream.consume(length); self.stream.make_room(); } } } }