|
|
@ -15,61 +15,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub fn scan(&mut self) -> Token {
|
|
|
|
fn make_token(&self, kind: TokenKind) -> Token<'a> {
|
|
|
|
self.skip_whitespace();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if self.peek().is_none() {
|
|
|
|
|
|
|
|
return self.make_token(TokenKind::Eof);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
match self.advance() {
|
|
|
|
|
|
|
|
c if is_alpha(c) => self.identifier(),
|
|
|
|
|
|
|
|
c if c.is_ascii_digit() => self.number(),
|
|
|
|
|
|
|
|
'(' => self.make_token(TokenKind::LeftParen),
|
|
|
|
|
|
|
|
')' => self.make_token(TokenKind::RightParen),
|
|
|
|
|
|
|
|
'{' => self.make_token(TokenKind::LeftBrace),
|
|
|
|
|
|
|
|
'}' => self.make_token(TokenKind::RightBrace),
|
|
|
|
|
|
|
|
';' => self.make_token(TokenKind::Semicolon),
|
|
|
|
|
|
|
|
',' => self.make_token(TokenKind::Comma),
|
|
|
|
|
|
|
|
'.' => self.make_token(TokenKind::Dot),
|
|
|
|
|
|
|
|
'-' => self.make_token(TokenKind::Minus),
|
|
|
|
|
|
|
|
'+' => self.make_token(TokenKind::Plus),
|
|
|
|
|
|
|
|
'/' => self.make_token(TokenKind::Slash),
|
|
|
|
|
|
|
|
'*' => self.make_token(TokenKind::Star),
|
|
|
|
|
|
|
|
'!' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::BangEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Bang)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'=' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::EqualEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Equal)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'<' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::LessEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Less)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'>' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::GreaterEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Greater)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'"' => self.string(),
|
|
|
|
|
|
|
|
_ => self.error_token("Unexpected character."),
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn make_token(&self, kind: TokenKind) -> Token {
|
|
|
|
|
|
|
|
Token {
|
|
|
|
Token {
|
|
|
|
kind,
|
|
|
|
kind,
|
|
|
|
value: std::str::from_utf8(&self.source[..self.current]).unwrap(),
|
|
|
|
value: std::str::from_utf8(&self.source[..self.current]).unwrap(),
|
|
|
@ -77,7 +23,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn error_token(&self, message: &'static str) -> Token {
|
|
|
|
fn error_token(&self, message: &'static str) -> Token<'a> {
|
|
|
|
Token {
|
|
|
|
Token {
|
|
|
|
kind: TokenKind::Error,
|
|
|
|
kind: TokenKind::Error,
|
|
|
|
value: message,
|
|
|
|
value: message,
|
|
|
@ -135,7 +81,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
self.source.get(self.current + 1).map(|&x| x as char)
|
|
|
|
self.source.get(self.current + 1).map(|&x| x as char)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn string(&mut self) -> Token {
|
|
|
|
fn string(&mut self) -> Token<'a> {
|
|
|
|
while matches!(self.peek(), Some(c) if c != '"') {
|
|
|
|
while matches!(self.peek(), Some(c) if c != '"') {
|
|
|
|
if self.peek() == Some('\n') {
|
|
|
|
if self.peek() == Some('\n') {
|
|
|
|
self.line += 1;
|
|
|
|
self.line += 1;
|
|
|
@ -152,7 +98,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
self.make_token(TokenKind::String)
|
|
|
|
self.make_token(TokenKind::String)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn number(&mut self) -> Token {
|
|
|
|
fn number(&mut self) -> Token<'a> {
|
|
|
|
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
|
|
|
|
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
|
|
|
|
self.advance();
|
|
|
|
self.advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -168,7 +114,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
self.make_token(TokenKind::Number)
|
|
|
|
self.make_token(TokenKind::Number)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn identifier(&mut self) -> Token {
|
|
|
|
fn identifier(&mut self) -> Token<'a> {
|
|
|
|
while matches!(self.peek(), Some(c) if is_alpha(c) || c.is_ascii_digit()) {
|
|
|
|
while matches!(self.peek(), Some(c) if is_alpha(c) || c.is_ascii_digit()) {
|
|
|
|
self.advance();
|
|
|
|
self.advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -205,6 +151,61 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
impl<'a> Iterator for Scanner<'a> {
|
|
|
|
|
|
|
|
type Item = Token<'a>;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn next(&mut self) -> Option<Self::Item> {
|
|
|
|
|
|
|
|
self.skip_whitespace();
|
|
|
|
|
|
|
|
self.peek()?;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Some(match self.advance() {
|
|
|
|
|
|
|
|
c if is_alpha(c) => self.identifier(),
|
|
|
|
|
|
|
|
c if c.is_ascii_digit() => self.number(),
|
|
|
|
|
|
|
|
'(' => self.make_token(TokenKind::LeftParen),
|
|
|
|
|
|
|
|
')' => self.make_token(TokenKind::RightParen),
|
|
|
|
|
|
|
|
'{' => self.make_token(TokenKind::LeftBrace),
|
|
|
|
|
|
|
|
'}' => self.make_token(TokenKind::RightBrace),
|
|
|
|
|
|
|
|
';' => self.make_token(TokenKind::Semicolon),
|
|
|
|
|
|
|
|
',' => self.make_token(TokenKind::Comma),
|
|
|
|
|
|
|
|
'.' => self.make_token(TokenKind::Dot),
|
|
|
|
|
|
|
|
'-' => self.make_token(TokenKind::Minus),
|
|
|
|
|
|
|
|
'+' => self.make_token(TokenKind::Plus),
|
|
|
|
|
|
|
|
'/' => self.make_token(TokenKind::Slash),
|
|
|
|
|
|
|
|
'*' => self.make_token(TokenKind::Star),
|
|
|
|
|
|
|
|
'!' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::BangEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Bang)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'=' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::EqualEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Equal)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'<' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::LessEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Less)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'>' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::GreaterEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Greater)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'"' => self.string(),
|
|
|
|
|
|
|
|
_ => self.error_token("Unexpected character."),
|
|
|
|
|
|
|
|
})
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn is_alpha(c: char) -> bool {
|
|
|
|
fn is_alpha(c: char) -> bool {
|
|
|
|
c.is_ascii_alphabetic() || c == '_'
|
|
|
|
c.is_ascii_alphabetic() || c == '_'
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -271,54 +272,54 @@ mod tests {
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[test]
|
|
|
|
fn test_scan() {
|
|
|
|
fn test_scan() {
|
|
|
|
assert_source_scan("(", TokenKind::LeftParen);
|
|
|
|
assert_source_scan("(", Some(TokenKind::LeftParen));
|
|
|
|
|
|
|
|
|
|
|
|
assert_source_scan("!", TokenKind::Bang);
|
|
|
|
assert_source_scan("!", Some(TokenKind::Bang));
|
|
|
|
assert_source_scan("!=", TokenKind::BangEqual);
|
|
|
|
assert_source_scan("!=", Some(TokenKind::BangEqual));
|
|
|
|
assert_source_scan("!a", TokenKind::Bang);
|
|
|
|
assert_source_scan("!a", Some(TokenKind::Bang));
|
|
|
|
|
|
|
|
|
|
|
|
assert_source_scan("a", TokenKind::Identifier);
|
|
|
|
assert_source_scan("a", Some(TokenKind::Identifier));
|
|
|
|
assert_source_scan("_a", TokenKind::Identifier);
|
|
|
|
assert_source_scan("_a", Some(TokenKind::Identifier));
|
|
|
|
assert_source_scan("a1", TokenKind::Identifier);
|
|
|
|
assert_source_scan("a1", Some(TokenKind::Identifier));
|
|
|
|
|
|
|
|
|
|
|
|
assert_source_scan("\"foo\"", TokenKind::String);
|
|
|
|
assert_source_scan("\"foo\"", Some(TokenKind::String));
|
|
|
|
|
|
|
|
|
|
|
|
assert_source_scan("1a", TokenKind::Number);
|
|
|
|
assert_source_scan("1a", Some(TokenKind::Number));
|
|
|
|
assert_source_scan("1.1", TokenKind::Number);
|
|
|
|
assert_source_scan("1.1", Some(TokenKind::Number));
|
|
|
|
|
|
|
|
|
|
|
|
assert_source_scan("a", TokenKind::Identifier);
|
|
|
|
assert_source_scan("a", Some(TokenKind::Identifier));
|
|
|
|
assert_source_scan("an", TokenKind::Identifier);
|
|
|
|
assert_source_scan("an", Some(TokenKind::Identifier));
|
|
|
|
assert_source_scan("and", TokenKind::And);
|
|
|
|
assert_source_scan("and", Some(TokenKind::And));
|
|
|
|
assert_source_scan("andy", TokenKind::Identifier);
|
|
|
|
assert_source_scan("andy", Some(TokenKind::Identifier));
|
|
|
|
assert_source_scan("false", TokenKind::False);
|
|
|
|
assert_source_scan("false", Some(TokenKind::False));
|
|
|
|
assert_source_scan("fa", TokenKind::Identifier);
|
|
|
|
assert_source_scan("fa", Some(TokenKind::Identifier));
|
|
|
|
|
|
|
|
|
|
|
|
assert_source_scan("@", TokenKind::Error);
|
|
|
|
assert_source_scan("@", Some(TokenKind::Error));
|
|
|
|
assert_source_scan("", TokenKind::Eof);
|
|
|
|
assert_source_scan("", None);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[test]
|
|
|
|
fn test_multi_scan() {
|
|
|
|
fn test_multi_scan() {
|
|
|
|
let mut scanner = Scanner::new("()");
|
|
|
|
let mut scanner = Scanner::new("()");
|
|
|
|
assert_scan(&mut scanner, TokenKind::LeftParen);
|
|
|
|
assert_scan(&mut scanner, Some(TokenKind::LeftParen));
|
|
|
|
assert_scan(&mut scanner, TokenKind::RightParen);
|
|
|
|
assert_scan(&mut scanner, Some(TokenKind::RightParen));
|
|
|
|
assert_scan(&mut scanner, TokenKind::Eof);
|
|
|
|
assert_scan(&mut scanner, None);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
#[test]
|
|
|
|
fn test_whitespace() {
|
|
|
|
fn test_whitespace() {
|
|
|
|
assert_source_scan(" foo", TokenKind::Identifier);
|
|
|
|
assert_source_scan(" foo", Some(TokenKind::Identifier));
|
|
|
|
assert_source_scan("\tfoo", TokenKind::Identifier);
|
|
|
|
assert_source_scan("\tfoo", Some(TokenKind::Identifier));
|
|
|
|
assert_source_scan("// \n", TokenKind::Eof);
|
|
|
|
assert_source_scan("// \n", None);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn assert_source_scan(source: &str, kind: TokenKind) {
|
|
|
|
fn assert_source_scan(source: &str, kind: Option<TokenKind>) {
|
|
|
|
let mut scanner = Scanner::new(source);
|
|
|
|
let mut scanner = Scanner::new(source);
|
|
|
|
assert_scan(&mut scanner, kind);
|
|
|
|
assert_scan(&mut scanner, kind);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn assert_scan(scanner: &mut Scanner, kind: TokenKind) {
|
|
|
|
fn assert_scan(scanner: &mut Scanner, kind: Option<TokenKind>) {
|
|
|
|
let token = scanner.scan();
|
|
|
|
let token = scanner.next();
|
|
|
|
assert_eq!(token.kind, kind);
|
|
|
|
assert_eq!(token.map(|x| x.kind), kind);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|