minitest-thesis/test/minitest/thesis_test.rb

require "test_helper"

class Minitest::ThesisTest < Minitest::Thesis::Test
  class Failure < StandardError; end

  def test_finds_small_list
    (0...10).each do |seed|
      out, _ = capture_io do
        assert_raises(Minitest::Assertion) do
          run_test("finds_small_list", database: {}, random: Random.new(seed)) do |test_case|
            ls = test_case.any(lists(integers(0, 10_000)))
            assert ls.sum <= 1_000
          end
        end
      end

      assert_equal <<~OUT, out
        any(lists(integers(0, 10000))): [1001]
      OUT
    end
  end

  # Minithesis can't really handle shrinking arbitrary monadic bind, but length
  # parameters are a common case of monadic bind that it has a little bit of
  # special casing for. This test ensures that that special casing works.
  #
  # The problem is that if you generate a list by drawing a length and then
  # drawing that many elements, you can end up with something like ``[1001, 0,
  # 0]`` then deleting those zeroes in the middle is a pain. minithesis will
  # solve this by first sorting those elements, so that we have ``[0, 0,
  # 1001]``, and then lowering the length by two, turning it into ``[1001]`` as
  # desired.
  def test_finds_small_list_even_with_bad_lists
    bad_list = Possibility.new("bad_list") {|tc|
      n = tc.choice(10)
      Array.new(n) { tc.choice(10_000) }
    }

    (0...10).each do |seed|
      out, _ = capture_io do
        assert_raises(Minitest::Assertion) do
          run_test("finds_small_list_even_with_bad_lists", database: {}, random: Random.new(seed)) do |test_case|
            ls = test_case.any(bad_list)
            assert ls.sum <= 1_000
          end
        end
      end

      assert_equal <<~OUT, out
        any(bad_list): [1001]
      OUT
    end
  end

  def test_reduces_additive_pairs
    out, _ = capture_io do
      assert_raises(Minitest::Assertion) do
        run_test("reduces_additive_pairs", database: {}, max_examples: 10_000) do |test_case|
          m = test_case.choice(1000)
          n = test_case.choice(1000)
          assert m + n <= 1000
        end
      end
    end

    assert_equal <<~OUT, out
      choice(1000): 1
      choice(1000): 1000
    OUT
  end

  def test_reuses_results_from_the_database
    Dir.mktmpdir do |tmpdir|
      db = DirectoryDb.new(tmpdir)
      count = 0

      run = -> {
        assert_raises(Minitest::Assertion) do
          run_test("reuses_results_from_the_database", database: db, quiet: true) do |test_case|
            count += 1
            assert test_case.choice(10_000) < 10
          end
        end
      }

      run.()

      assert_equal 1, Dir.children(tmpdir).length
      prev_count = count

      run.()

      assert_equal 1, Dir.children(tmpdir).length
      assert_equal prev_count + 2, count
    end
  end

  def test_test_cases_satisfy_preconditions
    run_test("test_cases_satisfy_preconditions", database: {}) do |test_case|
      n = test_case.choice(10)
      test_case.assume(n != 0)
      refute_equal 0, n
    end
  end

  def test_error_on_too_strict_precondition
    assert_raises(Unsatisfiable) do
      run_test("error_on_too_strict_precondition", database: {}) do |test_case|
        n = test_case.choice(10)
        test_case.reject
      end
    end
  end

  def test_error_on_unbounded_test_function
    orig_buffer_size = BUFFER_SIZE
    suppress_warnings do
      Minitest::Thesis.const_set(:BUFFER_SIZE, 10)
    end

    assert_raises(Unsatisfiable) do
      run_test("error_on_unbounded_test_function", database: {}, max_examples: 5) do |test_case|
        loop do
          test_case.choice(10)
        end
      end
    end
  ensure
    suppress_warnings do
      Minitest::Thesis.const_set(:BUFFER_SIZE, orig_buffer_size)
    end
  end

  def test_function_cache
    tf = ->(tc) do
      tc.mark_status(Status::INTERESTING) if tc.choice(1_000) >= 200
      tc.reject if tc.choice(1).zero?
    end

    state = TestingState.new(random: Random.new(0), test_function: tf, max_examples: 100)
    cache = CachedTestFunction.new {|tc| state.test_function(tc) }

    assert_equal Status::VALID, cache.([1, 1])
    assert_equal Status::OVERRUN, cache.([1])
    assert_equal Status::INTERESTING, cache.([1_000])
    assert_equal Status::INTERESTING, cache.([1_000])
    assert_equal Status::INTERESTING, cache.([1_000, 1])

    assert_equal 2, state.calls
  end

  # Targeting has a number of places it checks for whether we've exceeded the
  # generation limits. This makes sure we've checked them all.
  def test_max_examples_is_not_exceeded
    (1...100).each do |max_examples|
      calls = 0

      run_test(
        "max_examples_is_not_exceeded",
        database: {},
        random: Random.new(0),
        max_examples:,
      ) do |tc|
        m = 10000
        n = tc.choice(m)
        calls += 1
        tc.target(n * (m - n))
      end

      assert_equal max_examples, calls
    end
  end


  # Targeting has a number of places it checks for whether we've exceeded the
  # generation limits. This makes sure we've checked them all.
  def test_finds_a_local_maximum
    (0...100).each do |seed|
      assert_raises(Minitest::Assertion) do
        run_test(
          "finds_a_local_maximum",
          database: {},
          random: Random.new(seed),
          max_examples: 200,
          quiet: true
        ) do |tc|
          m = tc.choice(1000)
          n = tc.choice(1000)
          score = -((m - 500) ** 2 + (n - 500) ** 2)
          tc.target(score)
          assert m != 500 || n != 500
        end
      end
    end
  end

  def test_can_target_a_score_upwards_to_interesting
    out, _ = capture_io do
      assert_raises(Minitest::Assertion) do
        run_test("can_target_a_score_upwards_to_interesting", database: {}, max_examples: 1000) do |test_case|
          n = test_case.choice(1000)
          m = test_case.choice(1000)
          score = n + m
          test_case.target(score)
          assert score < 2000
        end
      end
    end

    assert_equal <<~OUT, out
      choice(1000): 1000
      choice(1000): 1000
    OUT
  end

  def test_can_target_a_score_upwards_without_failing
    max_score = 0

    run_test("can_target_a_score_upwards_without_failing", database: {}, max_examples: 1000) do |test_case|
      n = test_case.choice(1000)
      m = test_case.choice(1000)
      score = n + m
      test_case.target(score)
      max_score = [score, max_score].max
    end

    assert_equal 2000, max_score
  end

  def test_targeting_when_most_do_not_benefit
    big = 10_000

    out, _ = capture_io do
      assert_raises(Minitest::Assertion) do
        run_test("targeting_when_most_do_not_benefit", database: {}, max_examples: 1000) do |test_case|
          test_case.choice(1000)
          test_case.choice(1000)
          score = test_case.choice(big)
          test_case.target(score)
          assert score < big
        end
      end
    end

    assert_equal <<~OUT, out
      choice(1000): 0
      choice(1000): 0
      choice(#{big}): #{big}
    OUT
  end

  def test_can_target_a_score_downwards
    out, _ = capture_io do
      assert_raises(Minitest::Assertion) do
        run_test("can_target_a_score_downwards", database: {}, max_examples: 1000) do |test_case|
          n = test_case.choice(1000)
          m = test_case.choice(1000)
          score = n + m
          test_case.target(-score)
          assert score.positive?
        end
      end
    end

    assert_equal <<~OUT, out
      choice(1000): 0
      choice(1000): 0
    OUT
  end

  def test_prints_a_top_level_weighted
    out, _ = capture_io do
      assert_raises(Minitest::Assertion) do
        run_test("prints_a_top_level_weighted", database: {}, max_examples: 1000) do |test_case|
          assert test_case.weighted(0.5).nonzero?
        end
      end
    end

    assert_equal <<~OUT, out
      weighted(0.5): 0
    OUT
  end

  def test_errors_when_using_frozen
    tc = TestCase.for_choices([0])
    tc.status = Status::VALID

    assert_raises(Frozen) do
      tc.mark_status(Status::INTERESTING)
    end

    assert_raises(Frozen) do
      tc.choice(10)
    end

    assert_raises(Frozen) do
      tc.forced_choice(10)
    end
  end

  def test_errors_on_too_large_choice
    tc = TestCase.for_choices([0])
    assert_raises(RangeError) do
      tc.choice(2 ** 64)
    end
  end

  def test_can_choose_full_64_bits
    run_test("can_choose_full_64_bits", database: {}) do |tc|
      tc.choice(2 ** 64 - 1)
    end
  end

  def test_mapped_possibility
    run_test("mapped_possibility", database: {}) do |tc|
      n = tc.any(integers(0, 5).map {|n| n * 2 })
      assert n.even?
    end
  end

  def test_selected_possibility
    run_test("selected_possibility", database: {}) do |tc|
      n = tc.any(
        integers(0, 5)
          .satisfying(&:even?)
      )
      assert n.even?
    end
  end

  def test_bound_possibility
    run_test("bound_possibility", database: {}) do |tc|
      m, n = tc.any(
        integers(0, 5).bind {|m| tuples(just(m), integers(m, m + 10)) }
      )
      assert (m..m+10).cover?(n)
    end
  end

  def test_cannot_witness_nothing
    assert_raises(Unsatisfiable) do
      run_test("cannot_witness_nothing", database: {}) do |tc|
        tc.any(nothing)
      end
    end
  end

  def test_cannot_witness_empty_mix_of
    assert_raises(Unsatisfiable) do
      run_test("cannot_witness_empty_mix_of", database: {}) do |tc|
        tc.any(mix_of)
      end
    end
  end

  def test_can_draw_mixture
    run_test("can_draw_mixture", database: {}) do |tc|
      m = tc.any(mix_of(integers(-5, 0), integers(2, 5)))
      assert (-5..5).cover?(m)
      refute_equal 1, m
    end
  end

  # This test is very hard to trigger without targeting, and targeting will
  # tend to overshoot the score, so we will see multiple interesting test cases
  # before shrinking.
  def test_target_and_reduce
    out, _ = capture_io do
      assert_raises(Minitest::Assertion) do
        run_test("target_and_reduce", database: {}) do |tc|
          m = tc.choice(100_000)
          tc.target(m)
          assert m <= 99_900
        end
      end
    end

    assert_equal <<~OUT, out
      choice(100000): 99901
    OUT
  end

  def test_impossible_weighted
    assert_raises(Failure) do
      run_test("impossible_weighted", database: {}, quiet: true) do |tc|
        tc.choice(1)
        10.times do
          assert false unless tc.weighted(0.0).zero?
        end
        raise Failure if tc.choice(1).zero?
      end
    end
  end

  def test_guaranteed_weighted
    assert_raises(Failure) do
      run_test("guaranteed_weighted", database: {}, quiet: true) do |tc|
        if tc.weighted(1.0).nonzero?
          tc.choice(1)
          raise Failure
        else
          assert false
        end
      end
    end
  end

  def test_size_bounds_on_list
    run_test("size_bounds_on_list", database: {}) do |tc|
      ls = tc.any(lists(integers(0, 10), min_size: 1, max_size: 3))
      assert (1..3).cover?(ls.length)
    end
  end

  def test_forced_choice_bounds
    assert_raises(RangeError) do
      run_test("forced_choice_bounds", database: {}) do |tc|
        tc.forced_choice(2 ** 64)
      end
    end
  end

  def test_failure_from_hypothesis_1
    assert_raises(Failure) do
      run_test("failure_from_hypothesis_1", database: {}, random: Random.new(100), max_examples: 1000, quiet: true) do |tc|
        n1 = tc.weighted(0.0)
        if n1.zero?
          n2 = tc.choice(511)
          if n2 == 112
            n3 = tc.choice(511)
            if n3 == 124
              raise Failure
            elsif n3 == 93
              raise Failure
            else
              tc.mark_status(Status::INVALID)
            end
          elsif n2 == 93
            raise Failure
          else
            tc.mark_status(Status::INVALID)
          end
        end
      end
    end
  end

  def test_failure_from_hypothesis_2
    assert_raises(Failure) do
      run_test("failure_from_hypothesis_2", database: {}, random: Random.new(0), max_examples: 1000, quiet: true) do |tc|
        n1 = tc.choice(6)
        if n1 == 6
          n2 = tc.weighted(0.0)
          if n2.zero?
            raise Failure
          end
        elsif n1 == 4
          n3 = tc.choice(0)
          if n3 == 0
            raise Failure
          else
            tc.mark_status(Status::INVALID)
          end
        elsif n1 == 2
          raise Failure
        else
          tc.mark_status(Status::INVALID)
        end
      end
    end
  end

  def test_refactoring_cache
    old = ->(node, choices, status) {
      choices.each.with_index do |c, i|
        if i + 1 < choices.length || status == Status::OVERRUN
          node = if node.has_key?(c)
                   node[c]
                 else
                   node[c] = {}
                 end
        else
          node[c] = status
        end
      end
    }

    new = ->(node, choices, status) {
      *rest, last = choices
      rest.each do |c|
        node = if node.has_key?(c)
                 node[c]
               else
                 node[c] = {}
               end
      end
      unless last.nil?
        node[last] = status == Status::OVERRUN ? {} : status
      end
    }

    run_test("refactoring_cache", database: {}) do |tc|
      old_tree = {}
      new_tree = {}

      choices = tc.any(lists(integers(0, 10)))
      status = Status.new(tc.choice(4))

      old.(old_tree, choices, status)
      new.(new_tree, choices, status)

      assert_equal old_tree, new_tree
    end
  end

  private

  def suppress_warnings
    original_verbosity = $VERBOSE
    $VERBOSE = nil
    yield
    $VERBOSE = original_verbosity
  end
end