From 3330fe5d62cff14f18c4888e5e0b084ede92c10b Mon Sep 17 00:00:00 2001
From: alpha <alpha@kejadlen.dev>
Date: Tue, 19 Jul 2022 22:40:41 +0000
Subject: [PATCH] fail tokenizing if there are errors

FossilOrigin-Name: 070ae9c41a27c56f13509c7a7ceffb7e0b16fe61b21248b8271c363e3c5b9abe
---
 ruby/lox.rb      | 10 ++++++++--
 ruby/test_lox.rb |  8 ++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/ruby/lox.rb b/ruby/lox.rb
index 5a33abe..ebce4ab 100755
--- a/ruby/lox.rb
+++ b/ruby/lox.rb
@@ -96,11 +96,15 @@ module Lox
       while  WHILE
     ].each_slice(2).to_h.transform_values(&:to_sym)
 
-    State = Struct.new(:ss, :tokens, :errors, :line) do
+    class State < Struct.new(:ss, :tokens, :errors, :line)
       def eos? = ss.eos?
       def scan(re) = ss.scan(re)
       def pos = ss.pos
 
+      def initialize(src)
+        super(StringScanner.new(src), [], [], 1)
+      end
+
       def add_token(type, text: nil, literal: nil)
         text ||= ss.matched
         self.tokens << Token.new(type, text, literal, line)
@@ -108,7 +112,7 @@ module Lox
     end
 
     def scan(src)
-      state = State.new(StringScanner.new(src), [], [], 1)
+      state = State.new(src)
 
       until state.eos?
         case
@@ -135,6 +139,8 @@ module Lox
         end
       end
 
+      fail unless state.errors.empty?
+
       state.add_token(:EOF, text: "")
       state.tokens
     end
diff --git a/ruby/test_lox.rb b/ruby/test_lox.rb
index 464361e..27f4ff3 100644
--- a/ruby/test_lox.rb
+++ b/ruby/test_lox.rb
@@ -86,7 +86,9 @@ class TestScanner < Minitest::Test
       Token.new(:EOF, "", nil, 1),
     ], @scanner.scan('""')
 
-    assert_equal [Token.new(:EOF, "", nil, 1)], @scanner.scan('"') # TODO test the error once it's exposed
+    assert_raises do
+      @scanner.scan('"')
+    end
 
     assert_equal [
       Token.new(:STRING, '"foo"', "foo", 1),
@@ -130,6 +132,8 @@ class TestScanner < Minitest::Test
       Token.new(:EOF, "", nil, 5),
     ], tokens
 
-    assert_equal [Token.new(:EOF, "", nil, 1)], @scanner.scan("/*")
+    assert_raises do
+      @scanner.scan("/*")
+    end
   end
 end