alpha
/
crafting-interpreters
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

iterator all the things

Browse Source
main
Alpha Chen 2 years ago
parent 4525c22af9
commit 9b69d3db20
Signed by: alpha
SSH Key Fingerprint: SHA256:3fOT8fiYQG/aK9ntivV3Bqtg8AYQ7q4nV6ZgihOA20g
2 changed files with 91 additions and 93 deletions
Show Stats Download Patch File Download Diff File

9
rust/src/compiler.rs
Unescape View File

@ -5,8 +5,7 @@ pub fn compile(source: &str) -> Result<()> {
let mut scanner = Scanner::new(source); let mut scanner = Scanner::new(source);
let mut line = None; let mut line = None;
loop { for token in &mut scanner {
let token = scanner.scan();
if Some(token.line) != line { if Some(token.line) != line {
print!("{:4} ", token.line); print!("{:4} ", token.line);
line = Some(token.line); line = Some(token.line);
@ -14,9 +13,7 @@ pub fn compile(source: &str) -> Result<()> {
print!(" | "); print!(" | ");
} }
println!("{:2?} '{}'", token.kind, token.value); println!("{:2?} '{}'", token.kind, token.value);
if token.kind == TokenKind::Eof {
return Ok(());
}
} }
Ok(())
} }

175
rust/src/scanner.rs
Unescape View File

@ -15,61 +15,7 @@ impl<'a> Scanner<'a> {
} }
} }
pub fn scan(&mut self) -> Token { fn make_token(&self, kind: TokenKind) -> Token<'a> {
self.skip_whitespace();
if self.peek().is_none() {
return self.make_token(TokenKind::Eof);
}
match self.advance() {
c if is_alpha(c) => self.identifier(),
c if c.is_ascii_digit() => self.number(),
'(' => self.make_token(TokenKind::LeftParen),
')' => self.make_token(TokenKind::RightParen),
'{' => self.make_token(TokenKind::LeftBrace),
'}' => self.make_token(TokenKind::RightBrace),
';' => self.make_token(TokenKind::Semicolon),
',' => self.make_token(TokenKind::Comma),
'.' => self.make_token(TokenKind::Dot),
'-' => self.make_token(TokenKind::Minus),
'+' => self.make_token(TokenKind::Plus),
'/' => self.make_token(TokenKind::Slash),
'*' => self.make_token(TokenKind::Star),
'!' => {
if self.is_match('=') {
self.make_token(TokenKind::BangEqual)
} else {
self.make_token(TokenKind::Bang)
}
}
'=' => {
if self.is_match('=') {
self.make_token(TokenKind::EqualEqual)
} else {
self.make_token(TokenKind::Equal)
}
}
'<' => {
if self.is_match('=') {
self.make_token(TokenKind::LessEqual)
} else {
self.make_token(TokenKind::Less)
}
}
'>' => {
if self.is_match('=') {
self.make_token(TokenKind::GreaterEqual)
} else {
self.make_token(TokenKind::Greater)
}
}
'"' => self.string(),
_ => self.error_token("Unexpected character."),
}
}
fn make_token(&self, kind: TokenKind) -> Token {
Token { Token {
kind, kind,
value: std::str::from_utf8(&self.source[..self.current]).unwrap(), value: std::str::from_utf8(&self.source[..self.current]).unwrap(),
@ -77,7 +23,7 @@ impl<'a> Scanner<'a> {
} }
} }
fn error_token(&self, message: &'static str) -> Token { fn error_token(&self, message: &'static str) -> Token<'a> {
Token { Token {
kind: TokenKind::Error, kind: TokenKind::Error,
value: message, value: message,
@ -135,7 +81,7 @@ impl<'a> Scanner<'a> {
self.source.get(self.current + 1).map(|&x| x as char) self.source.get(self.current + 1).map(|&x| x as char)
} }
fn string(&mut self) -> Token { fn string(&mut self) -> Token<'a> {
while matches!(self.peek(), Some(c) if c != '"') { while matches!(self.peek(), Some(c) if c != '"') {
if self.peek() == Some('\n') { if self.peek() == Some('\n') {
self.line += 1; self.line += 1;
@ -152,7 +98,7 @@ impl<'a> Scanner<'a> {
self.make_token(TokenKind::String) self.make_token(TokenKind::String)
} }
fn number(&mut self) -> Token { fn number(&mut self) -> Token<'a> {
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) { while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
self.advance(); self.advance();
} }
@ -168,7 +114,7 @@ impl<'a> Scanner<'a> {
self.make_token(TokenKind::Number) self.make_token(TokenKind::Number)
} }
fn identifier(&mut self) -> Token { fn identifier(&mut self) -> Token<'a> {
while matches!(self.peek(), Some(c) if is_alpha(c) || c.is_ascii_digit()) { while matches!(self.peek(), Some(c) if is_alpha(c) || c.is_ascii_digit()) {
self.advance(); self.advance();
} }
@ -205,6 +151,61 @@ impl<'a> Scanner<'a> {
} }
} }
impl<'a> Iterator for Scanner<'a> {
type Item = Token<'a>;
fn next(&mut self) -> Option<Self::Item> {
self.skip_whitespace();
self.peek()?;
Some(match self.advance() {
c if is_alpha(c) => self.identifier(),
c if c.is_ascii_digit() => self.number(),
'(' => self.make_token(TokenKind::LeftParen),
')' => self.make_token(TokenKind::RightParen),
'{' => self.make_token(TokenKind::LeftBrace),
'}' => self.make_token(TokenKind::RightBrace),
';' => self.make_token(TokenKind::Semicolon),
',' => self.make_token(TokenKind::Comma),
'.' => self.make_token(TokenKind::Dot),
'-' => self.make_token(TokenKind::Minus),
'+' => self.make_token(TokenKind::Plus),
'/' => self.make_token(TokenKind::Slash),
'*' => self.make_token(TokenKind::Star),
'!' => {
if self.is_match('=') {
self.make_token(TokenKind::BangEqual)
} else {
self.make_token(TokenKind::Bang)
}
}
'=' => {
if self.is_match('=') {
self.make_token(TokenKind::EqualEqual)
} else {
self.make_token(TokenKind::Equal)
}
}
'<' => {
if self.is_match('=') {
self.make_token(TokenKind::LessEqual)
} else {
self.make_token(TokenKind::Less)
}
}
'>' => {
if self.is_match('=') {
self.make_token(TokenKind::GreaterEqual)
} else {
self.make_token(TokenKind::Greater)
}
}
'"' => self.string(),
_ => self.error_token("Unexpected character."),
})
}
}
fn is_alpha(c: char) -> bool { fn is_alpha(c: char) -> bool {
c.is_ascii_alphabetic() || c == '_' c.is_ascii_alphabetic() || c == '_'
} }
@ -271,54 +272,54 @@ mod tests {
#[test] #[test]
fn test_scan() { fn test_scan() {
assert_source_scan("(", TokenKind::LeftParen); assert_source_scan("(", Some(TokenKind::LeftParen));
assert_source_scan("!", TokenKind::Bang); assert_source_scan("!", Some(TokenKind::Bang));
assert_source_scan("!=", TokenKind::BangEqual); assert_source_scan("!=", Some(TokenKind::BangEqual));
assert_source_scan("!a", TokenKind::Bang); assert_source_scan("!a", Some(TokenKind::Bang));
assert_source_scan("a", TokenKind::Identifier); assert_source_scan("a", Some(TokenKind::Identifier));
assert_source_scan("_a", TokenKind::Identifier); assert_source_scan("_a", Some(TokenKind::Identifier));
assert_source_scan("a1", TokenKind::Identifier); assert_source_scan("a1", Some(TokenKind::Identifier));
assert_source_scan("\"foo\"", TokenKind::String); assert_source_scan("\"foo\"", Some(TokenKind::String));
assert_source_scan("1a", TokenKind::Number); assert_source_scan("1a", Some(TokenKind::Number));
assert_source_scan("1.1", TokenKind::Number); assert_source_scan("1.1", Some(TokenKind::Number));
assert_source_scan("a", TokenKind::Identifier); assert_source_scan("a", Some(TokenKind::Identifier));
assert_source_scan("an", TokenKind::Identifier); assert_source_scan("an", Some(TokenKind::Identifier));
assert_source_scan("and", TokenKind::And); assert_source_scan("and", Some(TokenKind::And));
assert_source_scan("andy", TokenKind::Identifier); assert_source_scan("andy", Some(TokenKind::Identifier));
assert_source_scan("false", TokenKind::False); assert_source_scan("false", Some(TokenKind::False));
assert_source_scan("fa", TokenKind::Identifier); assert_source_scan("fa", Some(TokenKind::Identifier));
assert_source_scan("@", TokenKind::Error); assert_source_scan("@", Some(TokenKind::Error));
assert_source_scan("", TokenKind::Eof); assert_source_scan("", None);
} }
#[test] #[test]
fn test_multi_scan() { fn test_multi_scan() {
let mut scanner = Scanner::new("()"); let mut scanner = Scanner::new("()");
assert_scan(&mut scanner, TokenKind::LeftParen); assert_scan(&mut scanner, Some(TokenKind::LeftParen));
assert_scan(&mut scanner, TokenKind::RightParen); assert_scan(&mut scanner, Some(TokenKind::RightParen));
assert_scan(&mut scanner, TokenKind::Eof); assert_scan(&mut scanner, None);
} }
#[test] #[test]
fn test_whitespace() { fn test_whitespace() {
assert_source_scan(" foo", TokenKind::Identifier); assert_source_scan(" foo", Some(TokenKind::Identifier));
assert_source_scan("\tfoo", TokenKind::Identifier); assert_source_scan("\tfoo", Some(TokenKind::Identifier));
assert_source_scan("// \n", TokenKind::Eof); assert_source_scan("// \n", None);
} }
fn assert_source_scan(source: &str, kind: TokenKind) { fn assert_source_scan(source: &str, kind: Option<TokenKind>) {
let mut scanner = Scanner::new(source); let mut scanner = Scanner::new(source);
assert_scan(&mut scanner, kind); assert_scan(&mut scanner, kind);
} }
fn assert_scan(scanner: &mut Scanner, kind: TokenKind) { fn assert_scan(scanner: &mut Scanner, kind: Option<TokenKind>) {
let token = scanner.scan(); let token = scanner.next();
assert_eq!(token.kind, kind); assert_eq!(token.map(|x| x.kind), kind);
} }
} }

Write Preview
Loading…
Cancel
Save