You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

480 lines
12 KiB

require "test_helper"
class Minitest::ThesisTest < Minitest::Thesis
class Failure < StandardError; end
def test_finds_small_list
(0...10).each do |seed|
out, _ = capture_io do
assert_raises(Minitest::Assertion) do
run_test("finds_small_list", database: {}, random: Random.new(seed)) do |test_case|
ls = test_case.any(lists(integers(0, 10_000)))
assert ls.sum <= 1_000
end
end
end
assert_equal <<~OUT, out
any(lists(integers(0, 10000))): [1001]
OUT
end
end
# Minithesis can't really handle shrinking arbitrary monadic bind, but length
# parameters are a common case of monadic bind that it has a little bit of
# special casing for. This test ensures that that special casing works.
#
# The problem is that if you generate a list by drawing a length and then
# drawing that many elements, you can end up with something like ``[1001, 0,
# 0]`` then deleting those zeroes in the middle is a pain. minithesis will
# solve this by first sorting those elements, so that we have ``[0, 0,
# 1001]``, and then lowering the length by two, turning it into ``[1001]`` as
# desired.
def test_finds_small_list_even_with_bad_lists
bad_list = Possibility.new(
->(tc) { n = tc.choice(10); Array.new(n) { tc.choice(10_000) }},
name: "bad_list",
)
(0...10).each do |seed|
out, _ = capture_io do
assert_raises(Minitest::Assertion) do
run_test("finds_small_list_even_with_bad_lists", database: {}, random: Random.new(seed)) do |test_case|
ls = test_case.any(bad_list)
assert ls.sum <= 1_000
end
end
end
assert_equal <<~OUT, out
any(bad_list): [1001]
OUT
end
end
def test_reduces_additive_pairs
out, _ = capture_io do
assert_raises(Minitest::Assertion) do
run_test("reduces_additive_pairs", database: {}, max_examples: 10_000) do |test_case|
m = test_case.choice(1000)
n = test_case.choice(1000)
assert m + n <= 1000
end
end
end
assert_equal <<~OUT, out
choice(1000): 1
choice(1000): 1000
OUT
end
def test_reuses_results_from_the_database
Dir.mktmpdir do |tmpdir|
db = DirectoryDb.new(tmpdir)
count = 0
run = -> {
assert_raises(Minitest::Assertion) do
run_test("reuses_results_from_the_database", database: db, quiet: true) do |test_case|
count += 1
assert test_case.choice(10_000) < 10
end
end
}
run.()
assert_equal 1, Dir.children(tmpdir).length
prev_count = count
run.()
assert_equal 1, Dir.children(tmpdir).length
assert_equal prev_count + 2, count
end
end
def test_test_cases_satisfy_preconditions
run_test("test_cases_satisfy_preconditions", database: {}) do |test_case|
n = test_case.choice(10)
test_case.assume(n != 0)
refute_equal 0, n
end
end
def test_error_on_too_strict_precondition
assert_raises(Unsatisfiable) do
run_test("error_on_too_strict_precondition", database: {}) do |test_case|
n = test_case.choice(10)
test_case.reject
end
end
end
def test_error_on_unbounded_test_function
orig_buffer_size = BUFFER_SIZE
suppress_warnings do
Minitest::Thesis.const_set(:BUFFER_SIZE, 10)
end
assert_raises(Unsatisfiable) do
run_test("error_on_unbounded_test_function", database: {}, max_examples: 5) do |test_case|
loop do
test_case.choice(10)
end
end
end
ensure
suppress_warnings do
Minitest::Thesis.const_set(:BUFFER_SIZE, orig_buffer_size)
end
end
def test_function_cache
tf = ->(tc) do
tc.mark_status(Status::INTERESTING) if tc.choice(1_000) >= 200
tc.reject if tc.choice(1).zero?
end
state = TestingState.new(random: Random.new(0), test_function: tf, max_examples: 100)
cache = CachedTestFunction.new {|tc| state.test_function(tc) }
assert_equal Status::VALID, cache.([1, 1])
assert_equal Status::OVERRUN, cache.([1])
assert_equal Status::INTERESTING, cache.([1_000])
assert_equal Status::INTERESTING, cache.([1_000])
assert_equal Status::INTERESTING, cache.([1_000, 1])
assert_equal 2, state.calls
end
# Targeting has a number of places it checks for whether we've exceeded the
# generation limits. This makes sure we've checked them all.
def test_max_examples_is_not_exceeded
(1...100).each do |max_examples|
calls = 0
run_test(
"max_examples_is_not_exceeded",
database: {},
random: Random.new(0),
max_examples:,
) do |tc|
m = 10000
n = tc.choice(m)
calls += 1
tc.target(n * (m - n))
end
assert_equal max_examples, calls
end
end
# Targeting has a number of places it checks for whether we've exceeded the
# generation limits. This makes sure we've checked them all.
def test_finds_a_local_maximum
(0...100).each do |seed|
assert_raises(Minitest::Assertion) do
run_test(
"finds_a_local_maximum",
database: {},
random: Random.new(seed),
max_examples: 200,
quiet: true
) do |tc|
m = tc.choice(1000)
n = tc.choice(1000)
score = -((m - 500) ** 2 + (n - 500) ** 2)
tc.target(score)
assert m != 500 || n != 500
end
end
end
end
def test_can_target_a_score_upwards_to_interesting
out, _ = capture_io do
assert_raises(Minitest::Assertion) do
run_test("can_target_a_score_upwards_to_interesting", database: {}, max_examples: 1000) do |test_case|
n = test_case.choice(1000)
m = test_case.choice(1000)
score = n + m
test_case.target(score)
assert score < 2000
end
end
end
assert_equal <<~OUT, out
choice(1000): 1000
choice(1000): 1000
OUT
end
def test_can_target_a_score_upwards_without_failing
max_score = 0
run_test("can_target_a_score_upwards_without_failing", database: {}, max_examples: 1000) do |test_case|
n = test_case.choice(1000)
m = test_case.choice(1000)
score = n + m
test_case.target(score)
max_score = [score, max_score].max
end
assert_equal 2000, max_score
end
def test_targeting_when_most_do_not_benefit
big = 10_000
out, _ = capture_io do
assert_raises(Minitest::Assertion) do
run_test("targeting_when_most_do_not_benefit", database: {}, max_examples: 1000) do |test_case|
test_case.choice(1000)
test_case.choice(1000)
score = test_case.choice(big)
test_case.target(score)
assert score < big
end
end
end
assert_equal <<~OUT, out
choice(1000): 0
choice(1000): 0
choice(#{big}): #{big}
OUT
end
def test_can_target_a_score_downwards
out, _ = capture_io do
assert_raises(Minitest::Assertion) do
run_test("can_target_a_score_downwards", database: {}, max_examples: 1000) do |test_case|
n = test_case.choice(1000)
m = test_case.choice(1000)
score = n + m
test_case.target(-score)
assert score.positive?
end
end
end
assert_equal <<~OUT, out
choice(1000): 0
choice(1000): 0
OUT
end
def test_prints_a_top_level_weighted
out, _ = capture_io do
assert_raises(Minitest::Assertion) do
run_test("prints_a_top_level_weighted", database: {}, max_examples: 1000) do |test_case|
assert test_case.weighted(0.5).nonzero?
end
end
end
assert_equal <<~OUT, out
weighted(0.5): 0
OUT
end
def test_errors_when_using_frozen
tc = TestCase.for_choices([0])
tc.status = Status::VALID
assert_raises(Frozen) do
tc.mark_status(Status::INTERESTING)
end
assert_raises(Frozen) do
tc.choice(10)
end
assert_raises(Frozen) do
tc.forced_choice(10)
end
end
def test_errors_on_too_large_choice
tc = TestCase.for_choices([0])
assert_raises(RangeError) do
tc.choice(2 ** 64)
end
end
def test_can_choose_full_64_bits
run_test("can_choose_full_64_bits", database: {}) do |tc|
tc.choice(2 ** 64 - 1)
end
end
def test_mapped_possibility
run_test("mapped_possibility", database: {}) do |tc|
n = tc.any(integers(0, 5).map {|n| n * 2 })
assert n.even?
end
end
def test_selected_possibility
run_test("selected_possibility", database: {}) do |tc|
n = tc.any(integers(0, 5).satisfying(&:even?))
assert n.even?
end
end
def test_bound_possibility
run_test("bound_possibility", database: {}) do |tc|
m, n = tc.any(
integers(0, 5).bind {|m| tuples(just(m), integers(m, m + 10)) }
)
assert (m..m+10).cover?(n)
end
end
def test_cannot_witness_nothing
assert_raises(Unsatisfiable) do
run_test("cannot_witness_nothing", database: {}) do |tc|
tc.any(nothing)
end
end
end
def test_cannot_witness_empty_mix_of
assert_raises(Unsatisfiable) do
run_test("cannot_witness_empty_mix_of", database: {}) do |tc|
tc.any(mix_of)
end
end
end
def test_can_draw_mixture
run_test("can_draw_mixture", database: {}) do |tc|
m = tc.any(mix_of(integers(-5, 0), integers(2, 5)))
assert (-5..5).cover?(m)
refute_equal 1, m
end
end
# This test is very hard to trigger without targeting, and targeting will
# tend to overshoot the score, so we will see multiple interesting test cases
# before shrinking.
def test_target_and_reduce
out, _ = capture_io do
assert_raises(Minitest::Assertion) do
run_test("target_and_reduce", database: {}) do |tc|
m = tc.choice(100_000)
tc.target(m)
assert m <= 99_900
end
end
end
assert_equal <<~OUT, out
choice(100000): 99901
OUT
end
def test_impossible_weighted
assert_raises(Failure) do
run_test("impossible_weighted", database: {}, quiet: true) do |tc|
tc.choice(1)
10.times do
assert false unless tc.weighted(0.0).zero?
end
raise Failure if tc.choice(1).zero?
end
end
end
def test_guaranteed_weighted
assert_raises(Failure) do
run_test("guaranteed_weighted", database: {}, quiet: true) do |tc|
if tc.weighted(1.0).nonzero?
tc.choice(1)
raise Failure
else
assert false
end
end
end
end
def test_size_bounds_on_list
run_test("size_bounds_on_list", database: {}) do |tc|
ls = tc.any(lists(integers(0, 10), min_size: 1, max_size: 3))
assert (1..3).cover?(ls.length)
end
end
def test_forced_choice_bounds
assert_raises(RangeError) do
run_test("forced_choice_bounds", database: {}) do |tc|
tc.forced_choice(2 ** 64)
end
end
end
def test_failure_from_hypothesis_1
assert_raises(Failure) do
run_test("failure_from_hypothesis_1", database: {}, random: Random.new(100), max_examples: 1000, quiet: true) do |tc|
n1 = tc.weighted(0.0)
if n1.zero?
n2 = tc.choice(511)
if n2 == 112
n3 = tc.choice(511)
if n3 == 124
raise Failure
elsif n3 == 93
raise Failure
else
tc.mark_status(Status::INVALID)
end
elsif n2 == 93
raise Failure
else
tc.mark_status(Status::INVALID)
end
end
end
end
end
def test_failure_from_hypothesis_2
assert_raises(Failure) do
run_test("failure_from_hypothesis_2", database: {}, random: Random.new(0), max_examples: 1000, quiet: true) do |tc|
n1 = tc.choice(6)
if n1 == 6
n2 = tc.weighted(0.0)
if n2.zero?
raise Failure
end
elsif n1 == 4
n3 = tc.choice(0)
if n3 == 0
raise Failure
else
tc.mark_status(Status::INVALID)
end
elsif n1 == 2
raise Failure
else
tc.mark_status(Status::INVALID)
end
end
end
end
private
def suppress_warnings
original_verbosity = $VERBOSE
$VERBOSE = nil
yield
$VERBOSE = original_verbosity
end
end