fail tokenizing if there are errors

FossilOrigin-Name: 070ae9c41a27c56f13509c7a7ceffb7e0b16fe61b21248b8271c363e3c5b9abe
private
alpha 2 years ago
parent ccf9e0e19c
commit 3330fe5d62

@ -96,11 +96,15 @@ module Lox
while WHILE
].each_slice(2).to_h.transform_values(&:to_sym)
State = Struct.new(:ss, :tokens, :errors, :line) do
class State < Struct.new(:ss, :tokens, :errors, :line)
def eos? = ss.eos?
def scan(re) = ss.scan(re)
def pos = ss.pos
def initialize(src)
super(StringScanner.new(src), [], [], 1)
end
def add_token(type, text: nil, literal: nil)
text ||= ss.matched
self.tokens << Token.new(type, text, literal, line)
@ -108,7 +112,7 @@ module Lox
end
def scan(src)
state = State.new(StringScanner.new(src), [], [], 1)
state = State.new(src)
until state.eos?
case
@ -135,6 +139,8 @@ module Lox
end
end
fail unless state.errors.empty?
state.add_token(:EOF, text: "")
state.tokens
end

@ -86,7 +86,9 @@ class TestScanner < Minitest::Test
Token.new(:EOF, "", nil, 1),
], @scanner.scan('""')
assert_equal [Token.new(:EOF, "", nil, 1)], @scanner.scan('"') # TODO test the error once it's exposed
assert_raises do
@scanner.scan('"')
end
assert_equal [
Token.new(:STRING, '"foo"', "foo", 1),
@ -130,6 +132,8 @@ class TestScanner < Minitest::Test
Token.new(:EOF, "", nil, 5),
], tokens
assert_equal [Token.new(:EOF, "", nil, 1)], @scanner.scan("/*")
assert_raises do
@scanner.scan("/*")
end
end
end

Loading…
Cancel
Save