From a2a685bc471b3cebc2734e0b29972af44336a4bc Mon Sep 17 00:00:00 2001 From: alpha Date: Wed, 20 Jul 2022 04:41:02 +0000 Subject: [PATCH] split out lox.rb into multiple files FossilOrigin-Name: 9f8294ceb7aa3c4ad7ea96bafc90afceb8211dc8fabae66a4ace9560aa99eba4 --- ruby/lib/lox.rb | 195 +----------------------------- ruby/lib/lox/ast_printer.rb | 16 +++ ruby/lib/lox/error.rb | 11 ++ ruby/lib/lox/expr.rb | 19 +++ ruby/lib/lox/scanner.rb | 142 ++++++++++++++++++++++ ruby/lib/lox/token.rb | 7 ++ ruby/test/lox/test_ast_printer.rb | 22 ++++ ruby/test/lox/test_scanner.rb | 108 +++++++++++++++++ ruby/test/test_helper.rb | 6 + ruby/test/test_lox.rb | 134 +------------------- 10 files changed, 341 insertions(+), 319 deletions(-) create mode 100644 ruby/lib/lox/ast_printer.rb create mode 100644 ruby/lib/lox/error.rb create mode 100644 ruby/lib/lox/expr.rb create mode 100644 ruby/lib/lox/scanner.rb create mode 100644 ruby/lib/lox/token.rb create mode 100644 ruby/test/lox/test_ast_printer.rb create mode 100644 ruby/test/lox/test_scanner.rb create mode 100644 ruby/test/test_helper.rb diff --git a/ruby/lib/lox.rb b/ruby/lib/lox.rb index 416c12b..45c5ad7 100644 --- a/ruby/lib/lox.rb +++ b/ruby/lib/lox.rb @@ -1,22 +1,11 @@ #!/usr/bin/env ruby -w -require "strscan" +require_relative "lox/error" +require_relative "lox/expr" +require_relative "lox/scanner" +require_relative "lox/token" module Lox - class Error < StandardError - def initialize(line:, where: "", message:) - @line, @where, @message = line, where, message - end - - def to_s - "[line #@line] Error#@where: #@message" - end - end - - def self.error(line, msg) - raise Error(line:, message:) - end - class Runner def initialize(scanner: Scanner.new) @scanner = scanner @@ -28,180 +17,4 @@ module Lox end end end - - class Scanner - TOKENS = %w[ - ( LEFT_PAREN - ) RIGHT_PAREN - { LEFT_BRACE - } RIGHT_BRACE - , COMMA - . DOT - - MINUS - + PLUS - ; SEMICOLON - * STAR - != BANG_EQUAL - ! BANG - == EQUAL_EQUAL - = EQUAL - <= LESS_EQUAL - < LESS - >= GREATER_EQUAL - > GREATER - / SLASH - ].each_slice(2).to_h.transform_values(&:to_sym) - TOKENS_RE = Regexp.union(TOKENS.keys) - - KEYWORDS = %w[ - and AND - class CLASS - else ELSE - false FALSE - for FOR - fun FUN - if IF - nil NIL - or OR - print PRINT - return RETURN - super SUPER - this THIS - true TRUE - var VAR - while WHILE - ].each_slice(2).to_h.transform_values(&:to_sym) - - class State < Struct.new(:ss, :tokens, :errors, :line) - def eos? = ss.eos? - def scan(re) = ss.scan(re) - def pos = ss.pos - - def initialize(src) - super(StringScanner.new(src), [], [], 1) - end - - def add_token(type, text: nil, literal: nil) - text ||= ss.matched - self.tokens << Token.new(type, text, literal, line) - end - end - - def scan(src) - state = State.new(src) - - until state.eos? - case - when state.scan(/\/\/(?~\n)/) - # ignore line comment - when state.scan(/\/\*/) - scan_block_comment(state) - when matched = state.scan(TOKENS_RE) - state.add_token(TOKENS.fetch(matched)) - when state.scan(/[ \r\t]/) - # ignore whitespace - when state.scan(/\n/) - state.line += 1 - when state.scan(/"/) - scan_str(state) - when number = state.scan(/\d+(\.\d+)?/) - state.add_token(:NUMBER, literal: number.to_f) - when identifier = state.scan(/[a-zA-Z_]\w*/) - type = KEYWORDS.fetch(identifier, :IDENTIFIER) - state.add_token(type) - else - state.errors << Error.new(line: state.line, message: "Unexpected character.") - state.scan(/./) # keep scanning - end - end - - fail unless state.errors.empty? - - state.add_token(:EOF, text: "") - state.tokens - end - - private - - def scan_str(state) - text = ?" - loop do - case - when state.scan(/"/) - text << ?" - state.add_token(:STRING, text:, literal: text[1..-2]) - return - when state.scan(/\n/) - text << ?\n - state.line += 1 - when state.eos? - state.errors << Error.new(line: state.line, message: "Unterminated string.") - return - when c = state.scan(/(?~"|\n)/) - text << c - else - fail "unreachable!" - end - end - end - - def scan_block_comment(state) - loop do - case - when state.scan(/\/\*/) - scan_block_comment(state) - when state.scan(/\*\//) - return - when state.scan(/\n/) - state.line += 1 - when state.eos? - state.errors << Error.new(line: state.line, message: "Unterminated block comment.") - return - when state.scan(/./) - # no-op - else - fail "unreachable!" - end - end - end - end - - Token = Struct.new(:type, :lexeme, :literal, :line) do - def to_s - "#{type} #{lexeme} #{literal}" - end - end - - module Expr - Binary = Struct.new(:left, :op, :right) do - def accept(visitor) = visitor.visit_binary(self) - end - - Grouping = Struct.new(:expr) do - def accept(visitor) = visitor.visit_grouping(self) - end - - Literal = Struct.new(:value) do - def accept(visitor) = visitor.visit_literal(self) - end - - Unary = Struct.new(:op, :right) do - def accept(visitor) = visitor.visit_unary(self) - end - end - - class AstPrinter - def print(expr) = expr.accept(self) - - def visit_binary(expr) = parenthesize(expr.op.lexeme, expr.left, expr.right) - def visit_grouping(expr) = parenthesize("group", expr.expr) - def visit_literal(expr) = expr.value&.to_s || "nil" - def visit_unary(expr) = parenthesize(expr.op.lexeme, expr.right) - - private - - def parenthesize(name, *exprs) - "(#{name} #{exprs.map {|expr| expr.accept(self) }.join(" ")})" - end - end end diff --git a/ruby/lib/lox/ast_printer.rb b/ruby/lib/lox/ast_printer.rb new file mode 100644 index 0000000..3a23375 --- /dev/null +++ b/ruby/lib/lox/ast_printer.rb @@ -0,0 +1,16 @@ +module Lox + class AstPrinter + def print(expr) = expr.accept(self) + + def visit_binary(expr) = parenthesize(expr.op.lexeme, expr.left, expr.right) + def visit_grouping(expr) = parenthesize("group", expr.expr) + def visit_literal(expr) = expr.value&.to_s || "nil" + def visit_unary(expr) = parenthesize(expr.op.lexeme, expr.right) + + private + + def parenthesize(name, *exprs) + "(#{name} #{exprs.map {|expr| expr.accept(self) }.join(" ")})" + end + end +end diff --git a/ruby/lib/lox/error.rb b/ruby/lib/lox/error.rb new file mode 100644 index 0000000..4120812 --- /dev/null +++ b/ruby/lib/lox/error.rb @@ -0,0 +1,11 @@ +module Lox + class Error < StandardError + def initialize(line:, where: "", message:) + @line, @where, @message = line, where, message + end + + def to_s + "[line #@line] Error#@where: #@message" + end + end +end diff --git a/ruby/lib/lox/expr.rb b/ruby/lib/lox/expr.rb new file mode 100644 index 0000000..38860a2 --- /dev/null +++ b/ruby/lib/lox/expr.rb @@ -0,0 +1,19 @@ +module Lox + module Expr + Binary = Struct.new(:left, :op, :right) do + def accept(visitor) = visitor.visit_binary(self) + end + + Grouping = Struct.new(:expr) do + def accept(visitor) = visitor.visit_grouping(self) + end + + Literal = Struct.new(:value) do + def accept(visitor) = visitor.visit_literal(self) + end + + Unary = Struct.new(:op, :right) do + def accept(visitor) = visitor.visit_unary(self) + end + end +end diff --git a/ruby/lib/lox/scanner.rb b/ruby/lib/lox/scanner.rb new file mode 100644 index 0000000..ca3d477 --- /dev/null +++ b/ruby/lib/lox/scanner.rb @@ -0,0 +1,142 @@ +require "strscan" + +require_relative "token" + +module Lox + class Scanner + TOKENS = %w[ + ( LEFT_PAREN + ) RIGHT_PAREN + { LEFT_BRACE + } RIGHT_BRACE + , COMMA + . DOT + - MINUS + + PLUS + ; SEMICOLON + * STAR + != BANG_EQUAL + ! BANG + == EQUAL_EQUAL + = EQUAL + <= LESS_EQUAL + < LESS + >= GREATER_EQUAL + > GREATER + / SLASH + ].each_slice(2).to_h.transform_values(&:to_sym) + TOKENS_RE = Regexp.union(TOKENS.keys) + + KEYWORDS = %w[ + and AND + class CLASS + else ELSE + false FALSE + for FOR + fun FUN + if IF + nil NIL + or OR + print PRINT + return RETURN + super SUPER + this THIS + true TRUE + var VAR + while WHILE + ].each_slice(2).to_h.transform_values(&:to_sym) + + class State < Struct.new(:ss, :tokens, :errors, :line) + def eos? = ss.eos? + def scan(re) = ss.scan(re) + def pos = ss.pos + + def initialize(src) + super(StringScanner.new(src), [], [], 1) + end + + def add_token(type, text: nil, literal: nil) + text ||= ss.matched + self.tokens << Lox::Token.new(type, text, literal, line) + end + end + + def scan(src) + state = State.new(src) + + until state.eos? + case + when state.scan(/\/\/(?~\n)/) + # ignore line comment + when state.scan(/\/\*/) + scan_block_comment(state) + when matched = state.scan(TOKENS_RE) + state.add_token(TOKENS.fetch(matched)) + when state.scan(/[ \r\t]/) + # ignore whitespace + when state.scan(/\n/) + state.line += 1 + when state.scan(/"/) + scan_str(state) + when number = state.scan(/\d+(\.\d+)?/) + state.add_token(:NUMBER, literal: number.to_f) + when identifier = state.scan(/[a-zA-Z_]\w*/) + type = KEYWORDS.fetch(identifier, :IDENTIFIER) + state.add_token(type) + else + state.errors << Error.new(line: state.line, message: "Unexpected character.") + state.scan(/./) # keep scanning + end + end + + fail unless state.errors.empty? + + state.add_token(:EOF, text: "") + state.tokens + end + + private + + def scan_str(state) + text = ?" + loop do + case + when state.scan(/"/) + text << ?" + state.add_token(:STRING, text:, literal: text[1..-2]) + return + when state.scan(/\n/) + text << ?\n + state.line += 1 + when state.eos? + state.errors << Error.new(line: state.line, message: "Unterminated string.") + return + when c = state.scan(/(?~"|\n)/) + text << c + else + fail "unreachable!" + end + end + end + + def scan_block_comment(state) + loop do + case + when state.scan(/\/\*/) + scan_block_comment(state) + when state.scan(/\*\//) + return + when state.scan(/\n/) + state.line += 1 + when state.eos? + state.errors << Error.new(line: state.line, message: "Unterminated block comment.") + return + when state.scan(/./) + # no-op + else + fail "unreachable!" + end + end + end + end +end diff --git a/ruby/lib/lox/token.rb b/ruby/lib/lox/token.rb new file mode 100644 index 0000000..f1166ad --- /dev/null +++ b/ruby/lib/lox/token.rb @@ -0,0 +1,7 @@ +module Lox + Token = Struct.new(:type, :lexeme, :literal, :line) do + def to_s + "#{type} #{lexeme} #{literal}" + end + end +end diff --git a/ruby/test/lox/test_ast_printer.rb b/ruby/test/lox/test_ast_printer.rb new file mode 100644 index 0000000..2a0319a --- /dev/null +++ b/ruby/test/lox/test_ast_printer.rb @@ -0,0 +1,22 @@ +require_relative "../test_helper" + +require "lox/ast_printer" +require "lox/expr" +require "lox/token" + +class TestAstPrinter < Lox::Test + def test_ast_printer + expr = Lox::Expr::Binary.new( + Lox::Expr::Unary.new( + Lox::Token.new(:MINUS, ?-, nil, 1), + Lox::Expr::Literal.new(123), + ), + Lox::Token.new(:STAR, ?*, nil, 1), + Lox::Expr::Grouping.new( + Lox::Expr::Literal.new(45.67), + ), + ) + + assert_equal "(* (- 123) (group 45.67))", Lox::AstPrinter.new.print(expr) + end +end diff --git a/ruby/test/lox/test_scanner.rb b/ruby/test/lox/test_scanner.rb new file mode 100644 index 0000000..1523a2e --- /dev/null +++ b/ruby/test/lox/test_scanner.rb @@ -0,0 +1,108 @@ +require_relative "../test_helper" + +require "lox/scanner" + +class TestScanner < Lox::Test + def setup + @scanner = Lox::Scanner.new + end + + def test_basic_tokens + %w[( LEFT_PAREN + ) RIGHT_PAREN + { LEFT_BRACE + } RIGHT_BRACE + , COMMA + . DOT + - MINUS + + PLUS + ; SEMICOLON + * STAR + != BANG_EQUAL + ! BANG + == EQUAL_EQUAL + = EQUAL + <= LESS_EQUAL + < LESS + >= GREATER_EQUAL + > GREATER + / SLASH].each_slice(2).to_h.transform_values(&:to_sym).each do |str, token_type| + assert_equal token_type.to_sym, @scanner.scan(str)[0].type + end + end + + def test_comments_and_whitespace + tokens = @scanner.scan(<<~SRC) + (\t) // here lies a comment + . // + SRC + + assert_equal %i[LEFT_PAREN RIGHT_PAREN DOT EOF], tokens.map(&:type) + end + + def test_line_numbers + tokens = @scanner.scan(<<~SRC) + ( + ) + SRC + + assert_equal [1, 2, 3], tokens.map(&:line) + end + + def test_strings + assert_equal [ + Lox::Token.new(:STRING, '""', "", 1), + Lox::Token.new(:EOF, "", nil, 1), + ], @scanner.scan('""') + + assert_raises do + @scanner.scan('"') + end + + assert_equal [ + Lox::Token.new(:STRING, '"foo"', "foo", 1), + Lox::Token.new(:EOF, "", nil, 1), + ], @scanner.scan('"foo"') + + assert_equal [ + Lox::Token.new(:STRING, "\"foo\nbar\"", "foo\nbar", 2), + Lox::Token.new(:EOF, "", nil, 2), + ], @scanner.scan("\"foo\nbar\"") + end + + def test_numbers + assert_equal [ + Lox::Token.new(:NUMBER, "123", 123.0, 1), + Lox::Token.new(:NUMBER, "123.4", 123.4, 1), + Lox::Token.new(:EOF, "", nil, 1), + ], @scanner.scan("123 123.4") + end + + def test_identifiers + assert_equal [ + Lox::Token.new(:OR, "or", nil, 1), + Lox::Token.new(:IDENTIFIER, "orchid", nil, 1), + Lox::Token.new(:IDENTIFIER, "o", nil, 1), + Lox::Token.new(:EOF, "", nil, 1), + ], @scanner.scan("or orchid o") + end + + def test_block_comments + tokens = @scanner.scan(<<~SRC) + foo + /* here lies a /* nested */ block comment + with newlines */ + bar + SRC + + assert_equal [ + Lox::Token.new(:IDENTIFIER, "foo", nil, 1), + Lox::Token.new(:IDENTIFIER, "bar", nil, 4), + Lox::Token.new(:EOF, "", nil, 5), + ], tokens + + assert_raises do + @scanner.scan("/*") + end + end +end diff --git a/ruby/test/test_helper.rb b/ruby/test/test_helper.rb new file mode 100644 index 0000000..2b59721 --- /dev/null +++ b/ruby/test/test_helper.rb @@ -0,0 +1,6 @@ +require "minitest" + +module Lox + class Test < Minitest::Test + end +end diff --git a/ruby/test/test_lox.rb b/ruby/test/test_lox.rb index 1aeff9e..eca0bb0 100644 --- a/ruby/test/test_lox.rb +++ b/ruby/test/test_lox.rb @@ -1,13 +1,12 @@ +require_relative "test_helper" + require "lox" -include Lox require "open3" -require "minitest/autorun" require "mocktail" -require "pry" -class TestLox < Minitest::Test +class TestLox < Lox::Test def test_error_on_more_than_one_arg lox_path = File.expand_path("../bin/lox", __dir__) o, s = Open3.capture2(lox_path, "foo", "bar") @@ -16,7 +15,7 @@ class TestLox < Minitest::Test end end -class TestRunner < Minitest::Test +class TestRunner < Lox::Test include Mocktail::DSL def teardown @@ -24,8 +23,8 @@ class TestRunner < Minitest::Test end def test_returns_tokens - scanner = Mocktail.of(Scanner) - runner = Runner.new(scanner:) + scanner = Mocktail.of(Lox::Scanner) + runner = Lox::Runner.new(scanner:) stubs { scanner.scan("src") }.with { %w[ some tokens ] } tokens = runner.run("src") @@ -34,124 +33,3 @@ class TestRunner < Minitest::Test end end -class TestScanner < Minitest::Test - def setup - @scanner = Scanner.new - end - - def test_basic_tokens - %w[( LEFT_PAREN - ) RIGHT_PAREN - { LEFT_BRACE - } RIGHT_BRACE - , COMMA - . DOT - - MINUS - + PLUS - ; SEMICOLON - * STAR - != BANG_EQUAL - ! BANG - == EQUAL_EQUAL - = EQUAL - <= LESS_EQUAL - < LESS - >= GREATER_EQUAL - > GREATER - / SLASH].each_slice(2).to_h.transform_values(&:to_sym).each do |str, token_type| - assert_equal token_type.to_sym, @scanner.scan(str)[0].type - end - end - - def test_comments_and_whitespace - tokens = @scanner.scan(<<~SRC) - (\t) // here lies a comment - . // - SRC - - assert_equal %i[LEFT_PAREN RIGHT_PAREN DOT EOF], tokens.map(&:type) - end - - def test_line_numbers - tokens = @scanner.scan(<<~SRC) - ( - ) - SRC - - assert_equal [1, 2, 3], tokens.map(&:line) - end - - def test_strings - assert_equal [ - Token.new(:STRING, '""', "", 1), - Token.new(:EOF, "", nil, 1), - ], @scanner.scan('""') - - assert_raises do - @scanner.scan('"') - end - - assert_equal [ - Token.new(:STRING, '"foo"', "foo", 1), - Token.new(:EOF, "", nil, 1), - ], @scanner.scan('"foo"') - - assert_equal [ - Token.new(:STRING, "\"foo\nbar\"", "foo\nbar", 2), - Token.new(:EOF, "", nil, 2), - ], @scanner.scan("\"foo\nbar\"") - end - - def test_numbers - assert_equal [ - Token.new(:NUMBER, "123", 123.0, 1), - Token.new(:NUMBER, "123.4", 123.4, 1), - Token.new(:EOF, "", nil, 1), - ], @scanner.scan("123 123.4") - end - - def test_identifiers - assert_equal [ - Token.new(:OR, "or", nil, 1), - Token.new(:IDENTIFIER, "orchid", nil, 1), - Token.new(:IDENTIFIER, "o", nil, 1), - Token.new(:EOF, "", nil, 1), - ], @scanner.scan("or orchid o") - end - - def test_block_comments - tokens = @scanner.scan(<<~SRC) - foo - /* here lies a /* nested */ block comment - with newlines */ - bar - SRC - - assert_equal [ - Token.new(:IDENTIFIER, "foo", nil, 1), - Token.new(:IDENTIFIER, "bar", nil, 4), - Token.new(:EOF, "", nil, 5), - ], tokens - - assert_raises do - @scanner.scan("/*") - end - end -end - -class TestAstPrinter < Minitest::Test - def test_ast_printer - expr = Expr::Binary.new( - Expr::Unary.new( - Token.new(:MINUS, ?-, nil, 1), - Expr::Literal.new(123), - ), - Token.new(:STAR, ?*, nil, 1), - Expr::Grouping.new( - Expr::Literal.new(45.67), - ), - ) - - assert_equal "(* (- 123) (group 45.67))", AstPrinter.new.print(expr) - end -end