alpha
/
crafting-interpreters
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

split out lox.rb into multiple files

Browse Source 
FossilOrigin-Name: 9f8294ceb7aa3c4ad7ea96bafc90afceb8211dc8fabae66a4ace9560aa99eba4
private
alpha 2 years ago
parent 3cd5ba6587
commit a2a685bc47
10 changed files with 341 additions and 319 deletions
Show Stats Download Patch File Download Diff File

195
ruby/lib/lox.rb
Unescape View File

@ -1,22 +1,11 @@
#!/usr/bin/env ruby -w #!/usr/bin/env ruby -w
require "strscan" require_relative "lox/error"
require_relative "lox/expr"
require_relative "lox/scanner"
require_relative "lox/token"
module Lox module Lox
class Error < StandardError
def initialize(line:, where: "", message:)
@line, @where, @message = line, where, message
end
def to_s
"[line #@line] Error#@where: #@message"
end
end
def self.error(line, msg)
raise Error(line:, message:)
end
class Runner class Runner
def initialize(scanner: Scanner.new) def initialize(scanner: Scanner.new)
@scanner = scanner @scanner = scanner
@ -28,180 +17,4 @@ module Lox
end end
end end
end end
class Scanner
TOKENS = %w[
( LEFT_PAREN
) RIGHT_PAREN
{ LEFT_BRACE
} RIGHT_BRACE
, COMMA
. DOT
- MINUS
+ PLUS
; SEMICOLON
* STAR
!= BANG_EQUAL
! BANG
== EQUAL_EQUAL
= EQUAL
<= LESS_EQUAL
< LESS
>= GREATER_EQUAL
> GREATER
/ SLASH
].each_slice(2).to_h.transform_values(&:to_sym)
TOKENS_RE = Regexp.union(TOKENS.keys)
KEYWORDS = %w[
and AND
class CLASS
else ELSE
false FALSE
for FOR
fun FUN
if IF
nil NIL
or OR
print PRINT
return RETURN
super SUPER
this THIS
true TRUE
var VAR
while WHILE
].each_slice(2).to_h.transform_values(&:to_sym)
class State < Struct.new(:ss, :tokens, :errors, :line)
def eos? = ss.eos?
def scan(re) = ss.scan(re)
def pos = ss.pos
def initialize(src)
super(StringScanner.new(src), [], [], 1)
end
def add_token(type, text: nil, literal: nil)
text ||= ss.matched
self.tokens << Token.new(type, text, literal, line)
end
end
def scan(src)
state = State.new(src)
until state.eos?
case
when state.scan(/\/\/(?~\n)/)
# ignore line comment
when state.scan(/\/\*/)
scan_block_comment(state)
when matched = state.scan(TOKENS_RE)
state.add_token(TOKENS.fetch(matched))
when state.scan(/[ \r\t]/)
# ignore whitespace
when state.scan(/\n/)
state.line += 1
when state.scan(/"/)
scan_str(state)
when number = state.scan(/\d+(\.\d+)?/)
state.add_token(:NUMBER, literal: number.to_f)
when identifier = state.scan(/[a-zA-Z_]\w*/)
type = KEYWORDS.fetch(identifier, :IDENTIFIER)
state.add_token(type)
else
state.errors << Error.new(line: state.line, message: "Unexpected character.")
state.scan(/./) # keep scanning
end
end
fail unless state.errors.empty?
state.add_token(:EOF, text: "")
state.tokens
end
private
def scan_str(state)
text = ?"
loop do
case
when state.scan(/"/)
text << ?"
state.add_token(:STRING, text:, literal: text[1..-2])
return
when state.scan(/\n/)
text << ?\n
state.line += 1
when state.eos?
state.errors << Error.new(line: state.line, message: "Unterminated string.")
return
when c = state.scan(/(?~"|\n)/)
text << c
else
fail "unreachable!"
end
end
end
def scan_block_comment(state)
loop do
case
when state.scan(/\/\*/)
scan_block_comment(state)
when state.scan(/\*\//)
return
when state.scan(/\n/)
state.line += 1
when state.eos?
state.errors << Error.new(line: state.line, message: "Unterminated block comment.")
return
when state.scan(/./)
# no-op
else
fail "unreachable!"
end
end
end
end
Token = Struct.new(:type, :lexeme, :literal, :line) do
def to_s
"#{type} #{lexeme} #{literal}"
end
end
module Expr
Binary = Struct.new(:left, :op, :right) do
def accept(visitor) = visitor.visit_binary(self)
end
Grouping = Struct.new(:expr) do
def accept(visitor) = visitor.visit_grouping(self)
end
Literal = Struct.new(:value) do
def accept(visitor) = visitor.visit_literal(self)
end
Unary = Struct.new(:op, :right) do
def accept(visitor) = visitor.visit_unary(self)
end
end
class AstPrinter
def print(expr) = expr.accept(self)
def visit_binary(expr) = parenthesize(expr.op.lexeme, expr.left, expr.right)
def visit_grouping(expr) = parenthesize("group", expr.expr)
def visit_literal(expr) = expr.value&.to_s || "nil"
def visit_unary(expr) = parenthesize(expr.op.lexeme, expr.right)
private
def parenthesize(name, *exprs)
"(#{name} #{exprs.map {|expr| expr.accept(self) }.join(" ")})"
end
end
end end

16
ruby/lib/lox/ast_printer.rb
Unescape View File

@ -0,0 +1,16 @@
module Lox
class AstPrinter
def print(expr) = expr.accept(self)
def visit_binary(expr) = parenthesize(expr.op.lexeme, expr.left, expr.right)
def visit_grouping(expr) = parenthesize("group", expr.expr)
def visit_literal(expr) = expr.value&.to_s || "nil"
def visit_unary(expr) = parenthesize(expr.op.lexeme, expr.right)
private
def parenthesize(name, *exprs)
"(#{name} #{exprs.map {|expr| expr.accept(self) }.join(" ")})"
end
end
end

11
ruby/lib/lox/error.rb
Unescape View File

@ -0,0 +1,11 @@
module Lox
class Error < StandardError
def initialize(line:, where: "", message:)
@line, @where, @message = line, where, message
end
def to_s
"[line #@line] Error#@where: #@message"
end
end
end

19
ruby/lib/lox/expr.rb
Unescape View File

@ -0,0 +1,19 @@
module Lox
module Expr
Binary = Struct.new(:left, :op, :right) do
def accept(visitor) = visitor.visit_binary(self)
end
Grouping = Struct.new(:expr) do
def accept(visitor) = visitor.visit_grouping(self)
end
Literal = Struct.new(:value) do
def accept(visitor) = visitor.visit_literal(self)
end
Unary = Struct.new(:op, :right) do
def accept(visitor) = visitor.visit_unary(self)
end
end
end

142
ruby/lib/lox/scanner.rb
Unescape View File

@ -0,0 +1,142 @@
require "strscan"
require_relative "token"
module Lox
class Scanner
TOKENS = %w[
( LEFT_PAREN
) RIGHT_PAREN
{ LEFT_BRACE
} RIGHT_BRACE
, COMMA
. DOT
- MINUS
+ PLUS
; SEMICOLON
* STAR
!= BANG_EQUAL
! BANG
== EQUAL_EQUAL
= EQUAL
<= LESS_EQUAL
< LESS
>= GREATER_EQUAL
> GREATER
/ SLASH
].each_slice(2).to_h.transform_values(&:to_sym)
TOKENS_RE = Regexp.union(TOKENS.keys)
KEYWORDS = %w[
and AND
class CLASS
else ELSE
false FALSE
for FOR
fun FUN
if IF
nil NIL
or OR
print PRINT
return RETURN
super SUPER
this THIS
true TRUE
var VAR
while WHILE
].each_slice(2).to_h.transform_values(&:to_sym)
class State < Struct.new(:ss, :tokens, :errors, :line)
def eos? = ss.eos?
def scan(re) = ss.scan(re)
def pos = ss.pos
def initialize(src)
super(StringScanner.new(src), [], [], 1)
end
def add_token(type, text: nil, literal: nil)
text ||= ss.matched
self.tokens << Lox::Token.new(type, text, literal, line)
end
end
def scan(src)
state = State.new(src)
until state.eos?
case
when state.scan(/\/\/(?~\n)/)
# ignore line comment
when state.scan(/\/\*/)
scan_block_comment(state)
when matched = state.scan(TOKENS_RE)
state.add_token(TOKENS.fetch(matched))
when state.scan(/[ \r\t]/)
# ignore whitespace
when state.scan(/\n/)
state.line += 1
when state.scan(/"/)
scan_str(state)
when number = state.scan(/\d+(\.\d+)?/)
state.add_token(:NUMBER, literal: number.to_f)
when identifier = state.scan(/[a-zA-Z_]\w*/)
type = KEYWORDS.fetch(identifier, :IDENTIFIER)
state.add_token(type)
else
state.errors << Error.new(line: state.line, message: "Unexpected character.")
state.scan(/./) # keep scanning
end
end
fail unless state.errors.empty?
state.add_token(:EOF, text: "")
state.tokens
end
private
def scan_str(state)
text = ?"
loop do
case
when state.scan(/"/)
text << ?"
state.add_token(:STRING, text:, literal: text[1..-2])
return
when state.scan(/\n/)
text << ?\n
state.line += 1
when state.eos?
state.errors << Error.new(line: state.line, message: "Unterminated string.")
return
when c = state.scan(/(?~"|\n)/)
text << c
else
fail "unreachable!"
end
end
end
def scan_block_comment(state)
loop do
case
when state.scan(/\/\*/)
scan_block_comment(state)
when state.scan(/\*\//)
return
when state.scan(/\n/)
state.line += 1
when state.eos?
state.errors << Error.new(line: state.line, message: "Unterminated block comment.")
return
when state.scan(/./)
# no-op
else
fail "unreachable!"
end
end
end
end
end

7
ruby/lib/lox/token.rb
Unescape View File

@ -0,0 +1,7 @@
module Lox
Token = Struct.new(:type, :lexeme, :literal, :line) do
def to_s
"#{type} #{lexeme} #{literal}"
end
end
end

22
ruby/test/lox/test_ast_printer.rb
Unescape View File

@ -0,0 +1,22 @@
require_relative "../test_helper"
require "lox/ast_printer"
require "lox/expr"
require "lox/token"
class TestAstPrinter < Lox::Test
def test_ast_printer
expr = Lox::Expr::Binary.new(
Lox::Expr::Unary.new(
Lox::Token.new(:MINUS, ?-, nil, 1),
Lox::Expr::Literal.new(123),
),
Lox::Token.new(:STAR, ?*, nil, 1),
Lox::Expr::Grouping.new(
Lox::Expr::Literal.new(45.67),
),
)
assert_equal "(* (- 123) (group 45.67))", Lox::AstPrinter.new.print(expr)
end
end

108
ruby/test/lox/test_scanner.rb
Unescape View File

@ -0,0 +1,108 @@
require_relative "../test_helper"
require "lox/scanner"
class TestScanner < Lox::Test
def setup
@scanner = Lox::Scanner.new
end
def test_basic_tokens
%w[( LEFT_PAREN
) RIGHT_PAREN
{ LEFT_BRACE
} RIGHT_BRACE
, COMMA
. DOT
- MINUS
+ PLUS
; SEMICOLON
* STAR
!= BANG_EQUAL
! BANG
== EQUAL_EQUAL
= EQUAL
<= LESS_EQUAL
< LESS
>= GREATER_EQUAL
> GREATER
/ SLASH].each_slice(2).to_h.transform_values(&:to_sym).each do |str, token_type|
assert_equal token_type.to_sym, @scanner.scan(str)[0].type
end
end
def test_comments_and_whitespace
tokens = @scanner.scan(<<~SRC)
(\t) // here lies a comment
. //
SRC
assert_equal %i[LEFT_PAREN RIGHT_PAREN DOT EOF], tokens.map(&:type)
end
def test_line_numbers
tokens = @scanner.scan(<<~SRC)
(
)
SRC
assert_equal [1, 2, 3], tokens.map(&:line)
end
def test_strings
assert_equal [
Lox::Token.new(:STRING, '""', "", 1),
Lox::Token.new(:EOF, "", nil, 1),
], @scanner.scan('""')
assert_raises do
@scanner.scan('"')
end
assert_equal [
Lox::Token.new(:STRING, '"foo"', "foo", 1),
Lox::Token.new(:EOF, "", nil, 1),
], @scanner.scan('"foo"')
assert_equal [
Lox::Token.new(:STRING, "\"foo\nbar\"", "foo\nbar", 2),
Lox::Token.new(:EOF, "", nil, 2),
], @scanner.scan("\"foo\nbar\"")
end
def test_numbers
assert_equal [
Lox::Token.new(:NUMBER, "123", 123.0, 1),
Lox::Token.new(:NUMBER, "123.4", 123.4, 1),
Lox::Token.new(:EOF, "", nil, 1),
], @scanner.scan("123 123.4")
end
def test_identifiers
assert_equal [
Lox::Token.new(:OR, "or", nil, 1),
Lox::Token.new(:IDENTIFIER, "orchid", nil, 1),
Lox::Token.new(:IDENTIFIER, "o", nil, 1),
Lox::Token.new(:EOF, "", nil, 1),
], @scanner.scan("or orchid o")
end
def test_block_comments
tokens = @scanner.scan(<<~SRC)
foo
/* here lies a /* nested */ block comment
with newlines */
bar
SRC
assert_equal [
Lox::Token.new(:IDENTIFIER, "foo", nil, 1),
Lox::Token.new(:IDENTIFIER, "bar", nil, 4),
Lox::Token.new(:EOF, "", nil, 5),
], tokens
assert_raises do
@scanner.scan("/*")
end
end
end

6
ruby/test/test_helper.rb
Unescape View File

@ -0,0 +1,6 @@
require "minitest"
module Lox
class Test < Minitest::Test
end
end

134
ruby/test/test_lox.rb
Unescape View File

@ -1,13 +1,12 @@
require_relative "test_helper"
require "lox" require "lox"
include Lox
require "open3" require "open3"
require "minitest/autorun"
require "mocktail" require "mocktail"
require "pry"
class TestLox < Minitest::Test class TestLox < Lox::Test
def test_error_on_more_than_one_arg def test_error_on_more_than_one_arg
lox_path = File.expand_path("../bin/lox", __dir__) lox_path = File.expand_path("../bin/lox", __dir__)
o, s = Open3.capture2(lox_path, "foo", "bar") o, s = Open3.capture2(lox_path, "foo", "bar")
@ -16,7 +15,7 @@ class TestLox < Minitest::Test
end end
end end
class TestRunner < Minitest::Test class TestRunner < Lox::Test
include Mocktail::DSL include Mocktail::DSL
def teardown def teardown
@ -24,8 +23,8 @@ class TestRunner < Minitest::Test
end end
def test_returns_tokens def test_returns_tokens
scanner = Mocktail.of(Scanner) scanner = Mocktail.of(Lox::Scanner)
runner = Runner.new(scanner:) runner = Lox::Runner.new(scanner:)
stubs { scanner.scan("src") }.with { %w[ some tokens ] } stubs { scanner.scan("src") }.with { %w[ some tokens ] }
tokens = runner.run("src") tokens = runner.run("src")
@ -34,124 +33,3 @@ class TestRunner < Minitest::Test
end end
end end
class TestScanner < Minitest::Test
def setup
@scanner = Scanner.new
end
def test_basic_tokens
%w[( LEFT_PAREN
) RIGHT_PAREN
{ LEFT_BRACE
} RIGHT_BRACE
, COMMA
. DOT
- MINUS
+ PLUS
; SEMICOLON
* STAR
!= BANG_EQUAL
! BANG
== EQUAL_EQUAL
= EQUAL
<= LESS_EQUAL
< LESS
>= GREATER_EQUAL
> GREATER
/ SLASH].each_slice(2).to_h.transform_values(&:to_sym).each do |str, token_type|
assert_equal token_type.to_sym, @scanner.scan(str)[0].type
end
end
def test_comments_and_whitespace
tokens = @scanner.scan(<<~SRC)
(\t) // here lies a comment
. //
SRC
assert_equal %i[LEFT_PAREN RIGHT_PAREN DOT EOF], tokens.map(&:type)
end
def test_line_numbers
tokens = @scanner.scan(<<~SRC)
(
)
SRC
assert_equal [1, 2, 3], tokens.map(&:line)
end
def test_strings
assert_equal [
Token.new(:STRING, '""', "", 1),
Token.new(:EOF, "", nil, 1),
], @scanner.scan('""')
assert_raises do
@scanner.scan('"')
end
assert_equal [
Token.new(:STRING, '"foo"', "foo", 1),
Token.new(:EOF, "", nil, 1),
], @scanner.scan('"foo"')
assert_equal [
Token.new(:STRING, "\"foo\nbar\"", "foo\nbar", 2),
Token.new(:EOF, "", nil, 2),
], @scanner.scan("\"foo\nbar\"")
end
def test_numbers
assert_equal [
Token.new(:NUMBER, "123", 123.0, 1),
Token.new(:NUMBER, "123.4", 123.4, 1),
Token.new(:EOF, "", nil, 1),
], @scanner.scan("123 123.4")
end
def test_identifiers
assert_equal [
Token.new(:OR, "or", nil, 1),
Token.new(:IDENTIFIER, "orchid", nil, 1),
Token.new(:IDENTIFIER, "o", nil, 1),
Token.new(:EOF, "", nil, 1),
], @scanner.scan("or orchid o")
end
def test_block_comments
tokens = @scanner.scan(<<~SRC)
foo
/* here lies a /* nested */ block comment
with newlines */
bar
SRC
assert_equal [
Token.new(:IDENTIFIER, "foo", nil, 1),
Token.new(:IDENTIFIER, "bar", nil, 4),
Token.new(:EOF, "", nil, 5),
], tokens
assert_raises do
@scanner.scan("/*")
end
end
end
class TestAstPrinter < Minitest::Test
def test_ast_printer
expr = Expr::Binary.new(
Expr::Unary.new(
Token.new(:MINUS, ?-, nil, 1),
Expr::Literal.new(123),
),
Token.new(:STAR, ?*, nil, 1),
Expr::Grouping.new(
Expr::Literal.new(45.67),
),
)
assert_equal "(* (- 123) (group 45.67))", AstPrinter.new.print(expr)
end
end

Write Preview
Loading…
Cancel
Save