|
|
@ -2,7 +2,6 @@ use color_eyre::eyre::Result;
|
|
|
|
|
|
|
|
|
|
|
|
pub struct Scanner<'a> {
|
|
|
|
pub struct Scanner<'a> {
|
|
|
|
source: &'a [u8],
|
|
|
|
source: &'a [u8],
|
|
|
|
start: usize,
|
|
|
|
|
|
|
|
current: usize,
|
|
|
|
current: usize,
|
|
|
|
line: usize,
|
|
|
|
line: usize,
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -11,7 +10,6 @@ impl<'a> Scanner<'a> {
|
|
|
|
pub fn new(source: &'a str) -> Self {
|
|
|
|
pub fn new(source: &'a str) -> Self {
|
|
|
|
Self {
|
|
|
|
Self {
|
|
|
|
source: source.as_bytes(),
|
|
|
|
source: source.as_bytes(),
|
|
|
|
start: 0,
|
|
|
|
|
|
|
|
current: 0,
|
|
|
|
current: 0,
|
|
|
|
line: 1,
|
|
|
|
line: 1,
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -20,8 +18,6 @@ impl<'a> Scanner<'a> {
|
|
|
|
pub fn scan(&mut self) -> Result<Token> {
|
|
|
|
pub fn scan(&mut self) -> Result<Token> {
|
|
|
|
self.skip_whitespace();
|
|
|
|
self.skip_whitespace();
|
|
|
|
|
|
|
|
|
|
|
|
self.start = self.current;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if self.is_at_end() {
|
|
|
|
if self.is_at_end() {
|
|
|
|
return self.make_token(TokenKind::Eof);
|
|
|
|
return self.make_token(TokenKind::Eof);
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -30,7 +26,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
if is_alpha(c) {
|
|
|
|
if is_alpha(c) {
|
|
|
|
return self.identifier();
|
|
|
|
return self.identifier();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if c.is_digit(10) {
|
|
|
|
if c.is_ascii_digit() {
|
|
|
|
return self.number();
|
|
|
|
return self.number();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -86,7 +82,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
fn make_token(&self, kind: TokenKind) -> Result<Token> {
|
|
|
|
fn make_token(&self, kind: TokenKind) -> Result<Token> {
|
|
|
|
Ok(Token {
|
|
|
|
Ok(Token {
|
|
|
|
kind,
|
|
|
|
kind,
|
|
|
|
value: std::str::from_utf8(&self.source[self.start..self.current])?,
|
|
|
|
value: std::str::from_utf8(&self.source[..self.current])?,
|
|
|
|
line: self.line,
|
|
|
|
line: self.line,
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -175,14 +171,14 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn number(&mut self) -> Result<Token> {
|
|
|
|
fn number(&mut self) -> Result<Token> {
|
|
|
|
while self.peek().is_digit(10) {
|
|
|
|
while self.peek().is_ascii_digit() {
|
|
|
|
self.advance();
|
|
|
|
self.advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if self.peek() == '.' && self.peek_next().is_digit(10) {
|
|
|
|
if self.peek() == '.' && self.peek_next().is_ascii_digit() {
|
|
|
|
self.advance();
|
|
|
|
self.advance();
|
|
|
|
|
|
|
|
|
|
|
|
while self.peek().is_digit(10) {
|
|
|
|
while self.peek().is_ascii_digit() {
|
|
|
|
self.advance();
|
|
|
|
self.advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
@ -191,7 +187,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn identifier(&mut self) -> Result<Token> {
|
|
|
|
fn identifier(&mut self) -> Result<Token> {
|
|
|
|
while is_alpha(self.peek()) || self.peek().is_digit(10) {
|
|
|
|
while is_alpha(self.peek()) || self.peek().is_ascii_digit() {
|
|
|
|
self.advance();
|
|
|
|
self.advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -199,7 +195,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn identifier_type(&self) -> TokenKind {
|
|
|
|
fn identifier_type(&self) -> TokenKind {
|
|
|
|
match self.source[self.start] as char {
|
|
|
|
match self.source[0] as char {
|
|
|
|
'a' => self.check_keyword(1, "nd", TokenKind::And),
|
|
|
|
'a' => self.check_keyword(1, "nd", TokenKind::And),
|
|
|
|
'c' => self.check_keyword(1, "lass", TokenKind::Class),
|
|
|
|
'c' => self.check_keyword(1, "lass", TokenKind::Class),
|
|
|
|
'e' => self.check_keyword(1, "lse", TokenKind::Else),
|
|
|
|
'e' => self.check_keyword(1, "lse", TokenKind::Else),
|
|
|
@ -212,8 +208,8 @@ impl<'a> Scanner<'a> {
|
|
|
|
'v' => self.check_keyword(1, "ar", TokenKind::Var),
|
|
|
|
'v' => self.check_keyword(1, "ar", TokenKind::Var),
|
|
|
|
'w' => self.check_keyword(1, "hile", TokenKind::While),
|
|
|
|
'w' => self.check_keyword(1, "hile", TokenKind::While),
|
|
|
|
'f' => {
|
|
|
|
'f' => {
|
|
|
|
if self.current - self.start > 1 {
|
|
|
|
if self.current > 1 {
|
|
|
|
match self.source[self.start + 1] as char {
|
|
|
|
match self.source[1] as char {
|
|
|
|
'a' => self.check_keyword(2, "lse", TokenKind::False),
|
|
|
|
'a' => self.check_keyword(2, "lse", TokenKind::False),
|
|
|
|
'o' => self.check_keyword(2, "r", TokenKind::For),
|
|
|
|
'o' => self.check_keyword(2, "r", TokenKind::For),
|
|
|
|
'u' => self.check_keyword(2, "n", TokenKind::Fun),
|
|
|
|
'u' => self.check_keyword(2, "n", TokenKind::Fun),
|
|
|
@ -224,8 +220,8 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
't' => {
|
|
|
|
't' => {
|
|
|
|
if self.current - self.start > 1 {
|
|
|
|
if self.current > 1 {
|
|
|
|
match self.source[self.start + 1] as char {
|
|
|
|
match self.source[1] as char {
|
|
|
|
'h' => self.check_keyword(2, "is", TokenKind::This),
|
|
|
|
'h' => self.check_keyword(2, "is", TokenKind::This),
|
|
|
|
'r' => self.check_keyword(2, "ue", TokenKind::True),
|
|
|
|
'r' => self.check_keyword(2, "ue", TokenKind::True),
|
|
|
|
_ => TokenKind::Identifier
|
|
|
|
_ => TokenKind::Identifier
|
|
|
@ -239,8 +235,7 @@ impl<'a> Scanner<'a> {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
fn check_keyword(&self, start: usize, rest: &str, kind: TokenKind) -> TokenKind {
|
|
|
|
fn check_keyword(&self, start: usize, rest: &str, kind: TokenKind) -> TokenKind {
|
|
|
|
// Do I have an off-by-one error here?
|
|
|
|
if &self.source[start..self.current] == rest.as_bytes() {
|
|
|
|
if &self.source[self.start + start..self.current] == rest.as_bytes() {
|
|
|
|
|
|
|
|
kind
|
|
|
|
kind
|
|
|
|
} else {
|
|
|
|
} else {
|
|
|
|
TokenKind::Identifier
|
|
|
|
TokenKind::Identifier
|
|
|
|