|
|
@ -4,6 +4,10 @@ pub struct Scanner<'a> {
|
|
|
|
line: usize,
|
|
|
|
line: usize,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn is_alpha(c: char) -> bool {
|
|
|
|
|
|
|
|
c.is_ascii_alphabetic() || c == '_'
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl<'a> Scanner<'a> {
|
|
|
|
impl<'a> Scanner<'a> {
|
|
|
|
pub fn new(source: &'a str) -> Self {
|
|
|
|
pub fn new(source: &'a str) -> Self {
|
|
|
|
Self {
|
|
|
|
Self {
|
|
|
@ -13,22 +17,6 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn make_token(&self, kind: TokenKind) -> Token<'a> {
|
|
|
|
|
|
|
|
Token {
|
|
|
|
|
|
|
|
kind,
|
|
|
|
|
|
|
|
value: std::str::from_utf8(&self.source[..self.current]).unwrap(),
|
|
|
|
|
|
|
|
line: self.line,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn error_token(&self, message: &'static str) -> Token<'a> {
|
|
|
|
|
|
|
|
Token {
|
|
|
|
|
|
|
|
kind: TokenKind::Error,
|
|
|
|
|
|
|
|
value: message,
|
|
|
|
|
|
|
|
line: self.line,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn advance(&mut self) -> char {
|
|
|
|
fn advance(&mut self) -> char {
|
|
|
|
self.current += 1;
|
|
|
|
self.current += 1;
|
|
|
|
self.source[self.current - 1] as char
|
|
|
|
self.source[self.current - 1] as char
|
|
|
@ -75,11 +63,11 @@ impl<'a> Scanner<'a> {
|
|
|
|
self.source.get(self.current).map(|&x| x as char)
|
|
|
|
self.source.get(self.current).map(|&x| x as char)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn peek_next(&mut self) -> Option<char> {
|
|
|
|
fn peek_next(&self) -> Option<char> {
|
|
|
|
self.source.get(self.current + 1).map(|&x| x as char)
|
|
|
|
self.source.get(self.current + 1).map(|&x| x as char)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn string(&mut self) -> Token<'a> {
|
|
|
|
fn string(&'a mut self) -> Token<'a> {
|
|
|
|
while matches!(self.peek(), Some(c) if c != '"') {
|
|
|
|
while matches!(self.peek(), Some(c) if c != '"') {
|
|
|
|
if self.peek() == Some('\n') {
|
|
|
|
if self.peek() == Some('\n') {
|
|
|
|
self.line += 1;
|
|
|
|
self.line += 1;
|
|
|
@ -89,14 +77,15 @@ impl<'a> Scanner<'a> {
|
|
|
|
|
|
|
|
|
|
|
|
if self.peek().is_none() {
|
|
|
|
if self.peek().is_none() {
|
|
|
|
// Maybe this should return an Err?
|
|
|
|
// Maybe this should return an Err?
|
|
|
|
return self.error_token("Unterminated string.");
|
|
|
|
return Token::error(&self, "Unterminated string.");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
self.advance();
|
|
|
|
self.advance();
|
|
|
|
|
|
|
|
// Token::new(&self, TokenKind::String)
|
|
|
|
self.make_token(TokenKind::String)
|
|
|
|
self.make_token(TokenKind::String)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn number(&mut self) -> Token<'a> {
|
|
|
|
fn number(&'a mut self) -> Token<'a> {
|
|
|
|
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
|
|
|
|
while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
|
|
|
|
self.advance();
|
|
|
|
self.advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -109,14 +98,16 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Token::new(&self, TokenKind::Number)
|
|
|
|
self.make_token(TokenKind::Number)
|
|
|
|
self.make_token(TokenKind::Number)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn identifier(&mut self) -> Token<'a> {
|
|
|
|
fn identifier(&mut self) -> Token {
|
|
|
|
while matches!(self.peek(), Some(c) if is_alpha(c) || c.is_ascii_digit()) {
|
|
|
|
while matches!(self.peek(), Some(c) if is_alpha(c) || c.is_ascii_digit()) {
|
|
|
|
self.advance();
|
|
|
|
self.advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Token::new(&self, self.identifier_type())
|
|
|
|
self.make_token(self.identifier_type())
|
|
|
|
self.make_token(self.identifier_type())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -147,6 +138,14 @@ impl<'a> Scanner<'a> {
|
|
|
|
_ => TokenKind::Identifier,
|
|
|
|
_ => TokenKind::Identifier,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn make_token(&'a self, kind: TokenKind) -> Token<'a> {
|
|
|
|
|
|
|
|
Token {
|
|
|
|
|
|
|
|
kind: TokenKind::And,
|
|
|
|
|
|
|
|
value: std::str::from_utf8(&self.source[..self.current]).unwrap(),
|
|
|
|
|
|
|
|
line: self.line,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl<'a> Iterator for Scanner<'a> {
|
|
|
|
impl<'a> Iterator for Scanner<'a> {
|
|
|
@ -157,55 +156,60 @@ impl<'a> Iterator for Scanner<'a> {
|
|
|
|
self.peek()?;
|
|
|
|
self.peek()?;
|
|
|
|
|
|
|
|
|
|
|
|
Some(match self.advance() {
|
|
|
|
Some(match self.advance() {
|
|
|
|
c if is_alpha(c) => self.identifier(),
|
|
|
|
// _ => self.make_token(TokenKind::And),
|
|
|
|
c if c.is_ascii_digit() => self.number(),
|
|
|
|
_ => Token {
|
|
|
|
'(' => self.make_token(TokenKind::LeftParen),
|
|
|
|
kind: TokenKind::And,
|
|
|
|
')' => self.make_token(TokenKind::RightParen),
|
|
|
|
value: std::str::from_utf8(&self.source[..self.current]).unwrap(),
|
|
|
|
'{' => self.make_token(TokenKind::LeftBrace),
|
|
|
|
line: self.line,
|
|
|
|
'}' => self.make_token(TokenKind::RightBrace),
|
|
|
|
},
|
|
|
|
';' => self.make_token(TokenKind::Semicolon),
|
|
|
|
|
|
|
|
',' => self.make_token(TokenKind::Comma),
|
|
|
|
|
|
|
|
'.' => self.make_token(TokenKind::Dot),
|
|
|
|
|
|
|
|
'-' => self.make_token(TokenKind::Minus),
|
|
|
|
|
|
|
|
'+' => self.make_token(TokenKind::Plus),
|
|
|
|
|
|
|
|
'/' => self.make_token(TokenKind::Slash),
|
|
|
|
|
|
|
|
'*' => self.make_token(TokenKind::Star),
|
|
|
|
|
|
|
|
'!' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::BangEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Bang)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'=' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::EqualEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Equal)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'<' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::LessEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Less)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'>' => {
|
|
|
|
|
|
|
|
if self.is_match('=') {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::GreaterEqual)
|
|
|
|
|
|
|
|
} else {
|
|
|
|
|
|
|
|
self.make_token(TokenKind::Greater)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
'"' => self.string(),
|
|
|
|
|
|
|
|
_ => self.error_token("Unexpected character."),
|
|
|
|
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn is_alpha(c: char) -> bool {
|
|
|
|
// Some(match self.advance() {
|
|
|
|
c.is_ascii_alphabetic() || c == '_'
|
|
|
|
// c if is_alpha(c) => self.identifier(),
|
|
|
|
|
|
|
|
// c if c.is_ascii_digit() => self.number(),
|
|
|
|
|
|
|
|
// '(' => self.make_token(TokenKind::LeftParen),
|
|
|
|
|
|
|
|
// ')' => self.make_token(TokenKind::RightParen),
|
|
|
|
|
|
|
|
// '{' => self.make_token(TokenKind::LeftBrace),
|
|
|
|
|
|
|
|
// '}' => self.make_token(TokenKind::RightBrace),
|
|
|
|
|
|
|
|
// ';' => self.make_token(TokenKind::Semicolon),
|
|
|
|
|
|
|
|
// ',' => self.make_token(TokenKind::Comma),
|
|
|
|
|
|
|
|
// '.' => self.make_token(TokenKind::Dot),
|
|
|
|
|
|
|
|
// '-' => self.make_token(TokenKind::Minus),
|
|
|
|
|
|
|
|
// '+' => self.make_token(TokenKind::Plus),
|
|
|
|
|
|
|
|
// '/' => self.make_token(TokenKind::Slash),
|
|
|
|
|
|
|
|
// '*' => self.make_token(TokenKind::Star),
|
|
|
|
|
|
|
|
// '!' => {
|
|
|
|
|
|
|
|
// if self.is_match('=') {
|
|
|
|
|
|
|
|
// self.make_token(TokenKind::BangEqual)
|
|
|
|
|
|
|
|
// } else {
|
|
|
|
|
|
|
|
// self.make_token(TokenKind::Bang)
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// '=' => {
|
|
|
|
|
|
|
|
// if self.is_match('=') {
|
|
|
|
|
|
|
|
// self.make_token(TokenKind::EqualEqual)
|
|
|
|
|
|
|
|
// } else {
|
|
|
|
|
|
|
|
// self.make_token(TokenKind::Equal)
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// '<' => {
|
|
|
|
|
|
|
|
// if self.is_match('=') {
|
|
|
|
|
|
|
|
// self.make_token(TokenKind::LessEqual)
|
|
|
|
|
|
|
|
// } else {
|
|
|
|
|
|
|
|
// self.make_token(TokenKind::Less)
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// '>' => {
|
|
|
|
|
|
|
|
// if self.is_match('=') {
|
|
|
|
|
|
|
|
// self.make_token(TokenKind::GreaterEqual)
|
|
|
|
|
|
|
|
// } else {
|
|
|
|
|
|
|
|
// self.make_token(TokenKind::Greater)
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// }
|
|
|
|
|
|
|
|
// '"' => self.string(),
|
|
|
|
|
|
|
|
// _ => Token::error(&self, "Unexpected character."),
|
|
|
|
|
|
|
|
// })
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
pub struct Token<'a> {
|
|
|
|
pub struct Token<'a> {
|
|
|
@ -214,6 +218,24 @@ pub struct Token<'a> {
|
|
|
|
pub line: usize,
|
|
|
|
pub line: usize,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
impl<'a> Token<'a> {
|
|
|
|
|
|
|
|
fn new(scanner: &'a Scanner, kind: TokenKind) -> Self {
|
|
|
|
|
|
|
|
Token {
|
|
|
|
|
|
|
|
kind,
|
|
|
|
|
|
|
|
value: std::str::from_utf8(&scanner.source[..scanner.current]).unwrap(),
|
|
|
|
|
|
|
|
line: scanner.line,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fn error(scanner: &Scanner, message: &'static str) -> Self {
|
|
|
|
|
|
|
|
Token {
|
|
|
|
|
|
|
|
kind: TokenKind::Error,
|
|
|
|
|
|
|
|
value: message,
|
|
|
|
|
|
|
|
line: scanner.line,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
#[derive(Debug, PartialEq)]
|
|
|
|
#[derive(Debug, PartialEq)]
|
|
|
|
pub enum TokenKind {
|
|
|
|
pub enum TokenKind {
|
|
|
|
// Single-character tokens.
|
|
|
|
// Single-character tokens.
|
|
|
|