From 22ef61b805b393dbfdf18b5e0b52dd6157c2e28e Mon Sep 17 00:00:00 2001 From: Alpha Chen Date: Wed, 16 Nov 2022 13:14:53 -0800 Subject: [PATCH] mu --- .gitignore | 8 + .ruby-version | 1 + CODE_OF_CONDUCT.md | 74 ++++ Gemfile | 6 + Gemfile.lock | 22 + LICENSE.txt | 21 + README.md | 43 ++ Rakefile | 6 + bin/console | 14 + bin/setup | 8 + lib/minitest/thesis.rb | 762 +++++++++++++++++++++++++++++++++++ minitest-thesis.gemspec | 41 ++ test/minitest/thesis_test.rb | 479 ++++++++++++++++++++++ test/test_helper.rb | 4 + 14 files changed, 1489 insertions(+) create mode 100644 .gitignore create mode 100644 .ruby-version create mode 100644 CODE_OF_CONDUCT.md create mode 100644 Gemfile create mode 100644 Gemfile.lock create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 Rakefile create mode 100755 bin/console create mode 100755 bin/setup create mode 100644 lib/minitest/thesis.rb create mode 100644 minitest-thesis.gemspec create mode 100644 test/minitest/thesis_test.rb create mode 100644 test/test_helper.rb diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9106b2a --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..8c50098 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.1 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f07ce6a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at alpha@kejadlen.dev. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..ba3e064 --- /dev/null +++ b/Gemfile @@ -0,0 +1,6 @@ +source "https://rubygems.org" + +git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } + +# Specify your gem's dependencies in minitest-thesis.gemspec +gemspec diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..3509d0a --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,22 @@ +PATH + remote: . + specs: + minitest-thesis (0.1.0) + +GEM + remote: https://rubygems.org/ + specs: + minitest (5.16.3) + rake (13.0.6) + +PLATFORMS + arm64-darwin-21 + +DEPENDENCIES + bundler (~> 2.3) + minitest (~> 5.16) + minitest-thesis! + rake (~> 13.0) + +BUNDLED WITH + 2.3.25 diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..f62e7d0 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2022 Alpha Chen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6009d01 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# Minitest::Thesis + +Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/minitest/thesis`. To experiment with that code, run `bin/console` for an interactive prompt. + +TODO: Delete this and the text above, and describe your gem + +## Installation + +Add this line to your application's Gemfile: + +```ruby +gem 'minitest-thesis' +``` + +And then execute: + + $ bundle + +Or install it yourself as: + + $ gem install minitest-thesis + +## Usage + +TODO: Write usage instructions here + +## Development + +After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. + +To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/minitest-thesis. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). + +## Code of Conduct + +Everyone interacting in the Minitest::Thesis project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/minitest-thesis/blob/master/CODE_OF_CONDUCT.md). diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..7fb47ba --- /dev/null +++ b/Rakefile @@ -0,0 +1,6 @@ +require "bundler/gem_tasks" +require "minitest/test_task" + +Minitest::TestTask.create(:test) + +task :default => :test diff --git a/bin/console b/bin/console new file mode 100755 index 0000000..37150c1 --- /dev/null +++ b/bin/console @@ -0,0 +1,14 @@ +#!/usr/bin/env ruby + +require "bundler/setup" +require "minitest/thesis" + +# You can add fixtures and/or initialization code here to make experimenting +# with your gem easier. You can also use a different console, if you like. + +# (If you use this, don't forget to add pry to your Gemfile!) +# require "pry" +# Pry.start + +require "irb" +IRB.start(__FILE__) diff --git a/bin/setup b/bin/setup new file mode 100755 index 0000000..dce67d8 --- /dev/null +++ b/bin/setup @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +set -vx + +bundle install + +# Do any other automated setup that you need to do here diff --git a/lib/minitest/thesis.rb b/lib/minitest/thesis.rb new file mode 100644 index 0000000..7560336 --- /dev/null +++ b/lib/minitest/thesis.rb @@ -0,0 +1,762 @@ +require "digest" + +require "minitest" + +module Minitest + class Thesis < Test + VERSION = "0.1.0" + + # Runs a test. Usage is: + # + # run_test do |test_case| + # n = test_case.choice(1000) + # end + # + # The block takes a `TestCase` argument, and should raise an exception to + # indicate a test failure. It will either run silently or print drawn + # values and then fail with an exception if minithesis finds some test case + # that fails. + # + # Arguments: + # * max_examples: the maximum number of valid test cases to run for. + # Note that under some circumstances the test may run fewer test + # cases than this. + # * random: An instance of random.Random that will be used for all + # nondeterministic choices. + # * database: A Hash-like object in which results will be cached and resumed + # from, ensuring that if a test is run twice it fails in the same way. + # * quiet: Will not print anything on failure if True. + def run_test( + name, + max_examples: 100, + random: Random.new, + database: DirectoryDb.new(".minitest-thesis-cache"), + quiet: false, + &test + ) + mark_failures_interesting = ->(test_case) do + test.(test_case) + rescue Exception + raise unless test_case.status.nil? + + test_case.mark_status(Status::INTERESTING) + end + + state = TestingState.new(random:, test_function: mark_failures_interesting, max_examples:) + + prev_failure = database[name] + + unless prev_failure.nil? + choices = prev_failure.unpack("Q>*") + state.test_function(TestCase.for_choices(choices)) + end + + if state.result.nil? + state.run + end + + if state.valid_test_cases.zero? + raise Unsatisfiable + end + + if state.result.nil? + database.delete(name) + else + database[name] = state.result.pack("Q>*") + end + + unless state.result.nil? + test.(TestCase.for_choices(state.result, print_results: !quiet)) + end + end + + # Represents a single generated test case, which consists of an underlying + # set of choices that produce possibilities. + class TestCase + + # Returns a test case that makes this series of choices. + def self.for_choices(choices, print_results: false) + self.new(prefix: choices, random: nil, max_size: choices.length, print_results:) + end + + attr_accessor :status + attr_reader :choices, :targeting_score + + def initialize(prefix:, random:, max_size: Float::INFINITY, print_results: false) + @prefix, @random, @max_size, @print_results = prefix, random, max_size, print_results + @choices = [] + @status = nil + @depth = 0 + @targeting_score = nil + end + + # Returns a number in the range [0, n] + def choice(n) + result = make_choice(n) { @random.rand(n) } + + puts "choice(#{n}): #{result}" if should_print? + + result + end + + # Return True with probability `p`. + def weighted(p) + if p.zero? || p.negative? + result = forced_choice(0) + elsif p >= 1 + result = forced_choice(1) + else + result = make_choice(1) { (@random.rand <= p) ? 1 : 0 } + end + + puts "weighted(#{p}): #{result}" if should_print? + + result + end + + # Inserts a fake choice into the choice sequence, as if some call to + # choice() had returned `n`. You almost never need this, but sometimes it + # can be a useful hint to the shrinker. + def forced_choice(n) + + raise RangeError.new("Invalid choice #{n}") if n.bit_length > 64 || n.negative? + raise Frozen unless @status.nil? + + mark_status(Status::OVERRUN) if @choices.length >= @max_size + + choices << n + n + end + + # Mark this test case as invalid. + def reject = mark_status(Status::INVALID) + + # If this precondition is not met, abort the test and mark this test case as invalid. + def assume(precondition) + return if precondition + reject + end + + # Set a score to maximize. Multiple calls to this function will override previous ones. + # + # The name and idea come from Löscher, Andreas, and Konstantinos Sagonas. + # "Targeted property-based testing." ISSTA. 2017, but the implementation + # is based on that found in Hypothesis, which is not that similar to + # anything described in the paper. + def target(score) = @targeting_score = score + + # Return a possible value from `possibility`. + def any(possibility) + begin + @depth += 1 + result = possibility.produce.(self) + ensure + @depth -= 1 + end + + puts "any(#{possibility}): #{result}" if should_print? + + result + end + + # Set the status and raise StopTest. + def mark_status(status) + raise Frozen unless self.status.nil? + + @status = status + raise StopTest + end + + private + + def should_print? = @print_results && @depth.zero? + + # Make a choice in [0, n], by calling rnd_method if randomness is needed. + def make_choice(n, &rnd_method) + raise RangeError.new("Invalid choice #{n}") if n.bit_length > 64 || n.negative? + raise Frozen unless @status.nil? + + mark_status(Status::OVERRUN) if @choices.length >= @max_size + + result = if @choices.length < @prefix.length + @prefix[@choices.length] + else + rnd_method.() + end + @choices << result + + mark_status(Status::INVALID) if result > n + + result + end + end + + # Represents some range of values that might be used in a test, that can be + # requested from a `TestCase`. Pass one of these to TestCase.any to get a + # concrete value. + class Possibility + attr_reader :produce, :name + + def initialize(produce, name: "TODO") + @produce = produce + @name = name + end + + def inspect = name + def to_s = name + + # "Returns a `Possibility` where values come from applying `f` to some possible value for `self`." + def map(&f) + self.class.new( + ->(test_case) { f.(test_case.any(self)) }, + name: "#{name}.map(TODO)", + ) + end + + # Returns a `Possibility` where values come from applying `f` (which + # should return a new `Possibility` to some possible value for `self` + # then returning a possible value from that. + def bind(&f) + produce = ->(test_case) { test_case.any(f.(test_case.any(self))) } + self.class.new(produce, name: "#{name}.bind(TODO)") + end + + # Returns a `Possibility` whose values are any possible value of `self` + # for which `f` returns True. + def satisfying(&f) + produce = ->(test_case) { + 3.times do + candidate = test_case.any(self) + return candidate if f.(candidate) + end + test_case.reject + } + + self.class.new(produce, name: "#{name}.select(TODO)") + end + end + + # Any integer in the range [m, n] is possible + def integers(m, n) = Possibility.new(->(tc) { m + tc.choice(n - m) }, name: "integers(#{m}, #{n})") + + # Any lists whose elements are possible values from `elements` are possible. + def lists(elements, min_size: 0, max_size: Float::INFINITY) + produce = ->(test_case) { + result = [] + loop do + if result.length < min_size + test_case.forced_choice(1) + elsif result.length + 1 >= max_size + test_case.forced_choice(0) + break + elsif test_case.weighted(0.9).zero? + break + end + result << test_case.any(elements) + end + result + } + + Possibility.new(produce, name: "lists(#{elements.name})") + end + + # Only `value` is possible. + def just(value) = Possibility.new(->(_) { value }, name: "just(#{value})") + + # No possible values. i.e. Any call to `any` will reject the test case. + def nothing = Possibility.new(->(tc) { tc.reject }) + + # Possible values can be any value possible for one of `possibilities`. + def mix_of(*possibilities) + return nothing if possibilities.empty? + + Possibility.new( + ->(tc) { tc.any(possibilities[tc.choice(possibilities.length - 1)]) }, + name: "mix_of(#{possibilities.map(&:name).join(", ")})", + ) + end + + # Any tuple t of of length len(possibilities) such that t[i] is possible + # for possibilities[i] is possible. + def tuples(*possibilities) + Possibility.new( + ->(tc) { possibilities.map {|p| tc.any(p) } }, + name: "tuples(#{possibilities.map(&:name).join(", ")})", + ) + end + + # We cap the maximum amount of entropy a test case can use. + # This prevents cases where the generated test case size explodes + # by effectively rejection + BUFFER_SIZE = 8 * 1024 + + # Returns a cached version of a function that maps a choice sequence to the + # status of calling a test function on a test case populated with it. Is + # able to take advantage of the structure of the test function to predict + # the result even if exact sequence of choices has not been seen + # previously. + # + # You can safely omit implementing this at the cost of somewhat increased + # shrinking time. + class CachedTestFunction + def initialize(&test_function) + @test_function = test_function + + # Tree nodes are either a point at which a choice occurs + # in which case they map the result of the choice to the + # tree node we are in after, or a Status object indicating + # mark_status was called at this point and all future + # choices are irrelevant. + # + # Note that a better implementation of this would use + # a Patricia trie, which implements long non-branching + # paths as an array inline. For simplicity we don't + # do that here. + @tree = {} + end + + def call(choices) + node = @tree + begin + choices.each do |c| + node = node.fetch(c) + # mark_status was called, thus future choices + # will be ignored. + if node.is_a?(Status) + fail if node == Status::OVERRUN + return node + end + end + # If we never entered an unknown region of the tree + # or hit a Status value, then we know that another + # choice will be made next and the result will overrun. + return Status::OVERRUN + rescue KeyError + end + + # We now have to actually call the test function to find out what + # happens. + test_case = TestCase.for_choices(choices) + @test_function.(test_case) + fail if test_case.status.nil? + + # We enter the choices made in a tree. + node = @tree + test_case.choices.each.with_index do |c, i| + if i + 1 < test_case.choices.length || test_case.status == Status::OVERRUN + if node.has_key?(c) + node = node[c] + else + node = node[c] = {} + end + else + node[c] = test_case.status + end + end + + test_case.status + end + end + + class TestingState + attr_reader :result, :valid_test_cases, :calls + + def initialize(random:, test_function:, max_examples:) + @random, @_test_function, @max_examples = random, test_function, max_examples + @valid_test_cases = 0 + @calls = 0 + @test_is_trivial = false + end + + def test_function(test_case) + begin + @_test_function.(test_case) + rescue StopTest + end + + if test_case.status.nil? + test_case.status = Status::VALID + end + + @calls += 1 + + if test_case.status >= Status::INVALID && test_case.choices.length.zero? + @test_is_trivial = true + end + + if test_case.status >= Status::VALID + @valid_test_cases += 1 + + unless test_case.targeting_score.nil? + relevant_info = [test_case.targeting_score, test_case.choices] + if @best_scoring.nil? + @best_scoring = relevant_info + else + best, _ = @best_scoring + if test_case.targeting_score > best + @best_scoring = relevant_info + end + end + end + end + + if test_case.status == Status::INTERESTING && ( + @result.nil? || ((sort_key(test_case.choices) <=> sort_key(@result)) == -1) + ) + @result = test_case.choices + end + end + + # If any test cases have had `target()` called on them, do a simple + # hill climbing algorithm to attempt to optimise that target score. + def target + return if !@result.nil? || @best_scoring.nil? + + # Can we improve the score by changing choices[i] by `step`? + adjust = ->(i, step) do + fail if @best_scoring.nil? + + score, choices = @best_scoring + return false if choices[i] + step < 0 || choices[i].bit_length >= 64 + + attempt = choices.dup + attempt[i] += step + test_case = TestCase.new( + prefix: attempt, random: @random, max_size: BUFFER_SIZE + ) + test_function(test_case) + + fail if test_case.status.nil? + + test_case.status >= Status::VALID && + !test_case.targeting_score.nil? && + test_case.targeting_score > score + end + + while keep_generating? + i = @random.rand(@best_scoring[1].length) + sign = 0 + [1, -1].each do |k| + return unless keep_generating? + + if adjust.(i, k) + sign = k + break + end + end + + next if sign.zero? + + k = 1 + k *= 2 while keep_generating? && adjust.(i, sign * k) + + while k.positive? + while keep_generating? && adjust.(i, sign * k) + end + k /= 2 + end + end + end + + def run + generate + target + shrink + end + + def keep_generating? + !@test_is_trivial && + result.nil? && + @valid_test_cases < @max_examples && + # We impose a limit on the maximum number of calls as + # well as the maximum number of valid examples. This is + # to avoid taking a prohibitively long time on tests which + # have hard or impossible to satisfy preconditions. + @calls < @max_examples * 10 + end + + # Run random generation until either we have found an interesting test + # case or hit the limit of how many test cases we should evaluate. + def generate + while keep_generating? && (@best_scoring.nil? || @valid_test_cases < @max_examples / 2) + test_function(TestCase.new(prefix: [], random: @random, max_size: BUFFER_SIZE)) + end + end + + # If we have found an interesting example, try shrinking it so that the + # choice sequence leading to our best example is shortlex smaller than + # the one we originally found. This improves the quality of the generated + # test case, as per our paper. + # + # https://drmaciver.github.io/papers/reduction-via-generation-preview.pdf + def shrink + # if not self.result: + # return + return if @result.nil? || @result.empty? + + # Shrinking will typically try the same choice sequences over and over + # again, so we cache the test function in order to not end up + # reevaluating it in those cases. This also allows us to catch cases + # where we try something that is e.g. a prefix of something we've + # previously tried, which is guaranteed not to work. + cached = CachedTestFunction.new {|tc| test_function(tc) } + + consider = ->(choices) do + return true if choices == @result + + cached.(choices) == Status::INTERESTING + end + + fail unless consider.(@result) + + # We are going to perform a number of transformations to the current + # result, iterating until none of them make any progress - i.e. until + # we make it through an entire iteration of the loop without changing + # the result. + prev = nil + while prev != @result + prev = @result + + # A note on weird loop order: We iterate backwards through the choice + # sequence rather than forwards, because later bits tend to depend on + # earlier bits so it's easier to make changes near the end and + # deleting bits at the end may allow us to make changes earlier on + # that we we'd have missed. + # + # Note that we do not restart the loop at the end when we find a + # successful shrink. This is because things we've already tried are + # less likely to work. + # + # If this guess is wrong, that's OK, this isn't a correctness + # problem, because if we made a successful reduction then we are not + # at a fixed point and will restart the loop at the end the next time + # round. In some cases this can result in performance issues, but the + # end result should still be fine. + + # First try deleting each choice we made in chunks. We try longer + # chunks because this allows us to delete whole composite elements: + # e.g. deleting an element from a generated list requires us to + # delete both the choice of whether to include it and also the + # element itself, which may involve more than one choice. Some things + # will take more than 8 choices in the sequence. That's too bad, we + # may not be able to delete those. In Hypothesis proper we record the + # boundaries corresponding to `any` calls so that we can try deleting + # those, but that's pretty high overhead and also a bunch of slightly + # annoying code that it's not worth porting. + # + # We could instead do a quadratic amount of work to try all + # boundaries, but in general we don't want to do that because even a + # shrunk test case can involve a relatively large number of choices. + k = 8 + while k.positive? + i = @result.length - k - 1 + until i.negative? + if i >= @result.length + # Can happen if we successfully lowered the value at i - 1 + i -= 1 + next + end + attempt = @result[0...i] + (@result[i + k..] || []) + + fail unless attempt.length < @result.length + + unless consider.(attempt) + # This fixes a common problem that occurs + # when you have dependencies on some + # length parameter. e.g. draw a number + # between 0 and 10 and then draw that + # many elements. This can't delete + # everything that occurs that way, but + # it can delete some things and often + # will get us unstuck when nothing else + # does. + if i.positive? && attempt[i - 1].positive? + attempt[i - 1] -= 1 + i += 1 if consider.(attempt) + end + + i -= 1 + end + end + + k /= 2 + end + + # Attempts to replace some indices in the current result with new + # values. Useful for some purely lexicographic reductions that we are + # about to perform. + replace = ->(values) do + fail if @result.nil? + attempt = @result.dup + values.each do |i, v| + # The size of self.result can change during shrinking. If that + # happens, stop attempting to make use of these replacements + # because some other shrink pass is better to run now. + return false if i >= attempt.length + attempt[i] = v + end + consider.(attempt) + end + + # Now we try replacing blocks of choices with zeroes. Note that + # unlike the above we skip k = 1 because we handle that in the next + # step. Often (but not always) a block of all zeroes is the shortlex + # smallest value that a region can be. + k = 8 + + while k > 1 + i = @result.length - k + until i.negative? + if replace.((i...i+k).to_h {|i| [i, 0]}) + # If we've succeeded then all of [i, i + k] is zero so we + # adjust i so that the next region does not overlap with this + # at all. + i -= k + else + # Otherwise we might still be able to zero some of these values + # but not the last one, so we just go back one. + i -= 1 + end + end + k /= 2 + end + + # Now try replacing each choice with a smaller value by doing a + # binary search. This will replace n with 0 or n - 1 if possible, but + # will also more efficiently replace it with a smaller number than + # doing multiple subtractions would. + i = @result.length - 1 + until i.negative? + # Attempt to replace + bin_search_down(0, @result[i]) {|v| replace.({i => v}) } + i -= 1 + end + + # NB from here on this is just showing off cool shrinker tricks and + # you probably don't need to worry about it and can skip these bits + # unless they're easy and you want bragging rights for how much + # better you are at shrinking than the local QuickCheck equivalent. + + # Try sorting out of order ranges of choices, as `sort(x) <= x`, so + # this is always a lexicographic reduction. + k = 8 + # while k > 1: + while k > 1 + (@result.length - k - 1).downto(0).each do |i| + consider.(@result[0...i] + @result[i...i+k].sort + @result[i+k..]) + end + k /= 2 + end + + # Try adjusting nearby pairs of integers by redistributing value + # between them. This is useful for tests that depend on the sum of + # some generated values. + [2, 1].each do |k| + (@result.length - k - 1).downto(0).each do |i| + j = i + k + # This check is necessary because the previous changes might have + # shrunk the size of result, but also it's tedious to write tests + # for this so I didn't. + if j < @result.length + # Try swapping out of order pairs + if @result[i] > @result[j] + replace.({j => @result[i], i => @result[j]}) + end + # j could be out of range if the previous swap succeeded. + if j < @result.length && @result[i].positive? + prev_i = @result[i] + prev_j = @result[j] + bin_search_down(0, prev_i) {|v| + replace.({i => v, j => prev_j + (prev_i - v)}) + } + end + end + end + end + end + end + + private + + # Returns a key that can be used for the shrinking order of test cases. + def sort_key(choices) = [choices.length, choices] + + # Returns n in [lo, hi] such that f(n) is True, where it is assumed and + # will not be checked that f(hi) is True. + # + # Will return `lo` if `f(lo)` is True, otherwise the only guarantee that is + # made is that `f(n - 1)` is False and `f(n)` is True. In particular this + # does *not* guarantee to find the smallest value, only a locally minimal + # one. + def bin_search_down(low, high, &f) + return low if f.(low) + while low + 1 < high + mid = low + (high - low) / 2 + if f.(mid) + high = mid + else + low = mid + end + end + high + end + end + + class DirectoryDb + def initialize(dir) + @dir = dir + Dir.mkdir(@dir) + rescue SystemCallError => e + raise unless e.errno == Errno::EEXIST::Errno + end + + def [](key) + f = file(key) + return nil unless File.exist?(f) + + File.read(f) + end + + def []=(key, value) + File.write(file(key), value) + end + + private + + def file(key) + File.join(@dir, Digest::SHA1.hexdigest(key)[0...10]) + end + end + + class Error< StandardError; end + + # Attempted to make choices on a test case that has been completed. + class Frozen < Error; end + + # Raised when a test should stop executing early. + class StopTest < Error; end + + # Raised when a test has no valid examples. + class Unsatisfiable < Error; end + + class Status < Struct.new(:value) + # Test case didn't have enough data to complete + OVERRUN = self.new(0) + + # Test case contained something that prevented completion + INVALID = self.new(1) + + # Test case completed just fine but was boring + VALID = self.new(2) + + # Test case completed and was interesting + INTERESTING = self.new(3) + + include Comparable + + def <=>(other) + value <=> other.value + end + end + end +end diff --git a/minitest-thesis.gemspec b/minitest-thesis.gemspec new file mode 100644 index 0000000..da221cc --- /dev/null +++ b/minitest-thesis.gemspec @@ -0,0 +1,41 @@ +lib = File.expand_path("../lib", __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require "minitest/thesis" + +Gem::Specification.new do |spec| + spec.name = "minitest-thesis" + spec.version = Minitest::Thesis::VERSION + spec.authors = ["Alpha Chen"] + spec.email = ["alpha@kejadlen.dev"] + + spec.summary = "" + # spec.description = %q{TODO: Write a longer description or delete this line.} + spec.homepage = "https://git.kejadlen.dev/alpha/minitest-thesis" + spec.license = "MIT" + + # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host' + # to allow pushing to a single host or delete this section to allow pushing to any host. + if spec.respond_to?(:metadata) + spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'" + + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = "https://git.kejadlen.dev/alpha/minitest-thesis" + # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here." + else + raise "RubyGems 2.0 or newer is required to protect against " \ + "public gem pushes." + end + + # Specify which files should be added to the gem when it is released. + # The `git ls-files -z` loads the files in the RubyGem that have been added into git. + spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do + `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + end + spec.bindir = "exe" + spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } + spec.require_paths = ["lib"] + + spec.add_development_dependency "bundler", "~> 2.3" + spec.add_development_dependency "rake", "~> 13.0" + spec.add_development_dependency "minitest", "~> 5.16" +end diff --git a/test/minitest/thesis_test.rb b/test/minitest/thesis_test.rb new file mode 100644 index 0000000..49a56d6 --- /dev/null +++ b/test/minitest/thesis_test.rb @@ -0,0 +1,479 @@ +require "test_helper" + +class Minitest::ThesisTest < Minitest::Thesis + class Failure < StandardError; end + + def test_finds_small_list + (0...10).each do |seed| + out, _ = capture_io do + assert_raises(Minitest::Assertion) do + run_test("finds_small_list", database: {}, random: Random.new(seed)) do |test_case| + ls = test_case.any(lists(integers(0, 10_000))) + assert ls.sum <= 1_000 + end + end + end + + assert_equal <<~OUT, out + any(lists(integers(0, 10000))): [1001] + OUT + end + end + + # Minithesis can't really handle shrinking arbitrary monadic bind, but length + # parameters are a common case of monadic bind that it has a little bit of + # special casing for. This test ensures that that special casing works. + # + # The problem is that if you generate a list by drawing a length and then + # drawing that many elements, you can end up with something like ``[1001, 0, + # 0]`` then deleting those zeroes in the middle is a pain. minithesis will + # solve this by first sorting those elements, so that we have ``[0, 0, + # 1001]``, and then lowering the length by two, turning it into ``[1001]`` as + # desired. + def test_finds_small_list_even_with_bad_lists + bad_list = Possibility.new( + ->(tc) { n = tc.choice(10); Array.new(n) { tc.choice(10_000) }}, + name: "bad_list", + ) + + (0...10).each do |seed| + out, _ = capture_io do + assert_raises(Minitest::Assertion) do + run_test("finds_small_list_even_with_bad_lists", database: {}, random: Random.new(seed)) do |test_case| + ls = test_case.any(bad_list) + assert ls.sum <= 1_000 + end + end + end + + assert_equal <<~OUT, out + any(bad_list): [1001] + OUT + end + end + + def test_reduces_additive_pairs + out, _ = capture_io do + assert_raises(Minitest::Assertion) do + run_test("reduces_additive_pairs", database: {}, max_examples: 10_000) do |test_case| + m = test_case.choice(1000) + n = test_case.choice(1000) + assert m + n <= 1000 + end + end + end + + assert_equal <<~OUT, out + choice(1000): 1 + choice(1000): 1000 + OUT + end + + def test_reuses_results_from_the_database + Dir.mktmpdir do |tmpdir| + db = DirectoryDb.new(tmpdir) + count = 0 + + run = -> { + assert_raises(Minitest::Assertion) do + run_test("reuses_results_from_the_database", database: db, quiet: true) do |test_case| + count += 1 + assert test_case.choice(10_000) < 10 + end + end + } + + run.() + + assert_equal 1, Dir.children(tmpdir).length + prev_count = count + + run.() + + assert_equal 1, Dir.children(tmpdir).length + assert_equal prev_count + 2, count + end + end + + def test_test_cases_satisfy_preconditions + run_test("test_cases_satisfy_preconditions", database: {}) do |test_case| + n = test_case.choice(10) + test_case.assume(n != 0) + refute_equal 0, n + end + end + + def test_error_on_too_strict_precondition + assert_raises(Unsatisfiable) do + run_test("error_on_too_strict_precondition", database: {}) do |test_case| + n = test_case.choice(10) + test_case.reject + end + end + end + + def test_error_on_unbounded_test_function + orig_buffer_size = BUFFER_SIZE + suppress_warnings do + Minitest::Thesis.const_set(:BUFFER_SIZE, 10) + end + + assert_raises(Unsatisfiable) do + run_test("error_on_unbounded_test_function", database: {}, max_examples: 5) do |test_case| + loop do + test_case.choice(10) + end + end + end + ensure + suppress_warnings do + Minitest::Thesis.const_set(:BUFFER_SIZE, orig_buffer_size) + end + end + + def test_function_cache + tf = ->(tc) do + tc.mark_status(Status::INTERESTING) if tc.choice(1_000) >= 200 + tc.reject if tc.choice(1).zero? + end + + state = TestingState.new(random: Random.new(0), test_function: tf, max_examples: 100) + cache = CachedTestFunction.new {|tc| state.test_function(tc) } + + assert_equal Status::VALID, cache.([1, 1]) + assert_equal Status::OVERRUN, cache.([1]) + assert_equal Status::INTERESTING, cache.([1_000]) + assert_equal Status::INTERESTING, cache.([1_000]) + assert_equal Status::INTERESTING, cache.([1_000, 1]) + + assert_equal 2, state.calls + end + + # Targeting has a number of places it checks for whether we've exceeded the + # generation limits. This makes sure we've checked them all. + def test_max_examples_is_not_exceeded + (1...100).each do |max_examples| + calls = 0 + + run_test( + "max_examples_is_not_exceeded", + database: {}, + random: Random.new(0), + max_examples:, + ) do |tc| + m = 10000 + n = tc.choice(m) + calls += 1 + tc.target(n * (m - n)) + end + + assert_equal max_examples, calls + end + end + + + # Targeting has a number of places it checks for whether we've exceeded the + # generation limits. This makes sure we've checked them all. + def test_finds_a_local_maximum + (0...100).each do |seed| + assert_raises(Minitest::Assertion) do + run_test( + "finds_a_local_maximum", + database: {}, + random: Random.new(seed), + max_examples: 200, + quiet: true + ) do |tc| + m = tc.choice(1000) + n = tc.choice(1000) + score = -((m - 500) ** 2 + (n - 500) ** 2) + tc.target(score) + assert m != 500 || n != 500 + end + end + end + end + + def test_can_target_a_score_upwards_to_interesting + out, _ = capture_io do + assert_raises(Minitest::Assertion) do + run_test("can_target_a_score_upwards_to_interesting", database: {}, max_examples: 1000) do |test_case| + n = test_case.choice(1000) + m = test_case.choice(1000) + score = n + m + test_case.target(score) + assert score < 2000 + end + end + end + + assert_equal <<~OUT, out + choice(1000): 1000 + choice(1000): 1000 + OUT + end + + def test_can_target_a_score_upwards_without_failing + max_score = 0 + + run_test("can_target_a_score_upwards_without_failing", database: {}, max_examples: 1000) do |test_case| + n = test_case.choice(1000) + m = test_case.choice(1000) + score = n + m + test_case.target(score) + max_score = [score, max_score].max + end + + assert_equal 2000, max_score + end + + def test_targeting_when_most_do_not_benefit + big = 10_000 + + out, _ = capture_io do + assert_raises(Minitest::Assertion) do + run_test("targeting_when_most_do_not_benefit", database: {}, max_examples: 1000) do |test_case| + test_case.choice(1000) + test_case.choice(1000) + score = test_case.choice(big) + test_case.target(score) + assert score < big + end + end + end + + assert_equal <<~OUT, out + choice(1000): 0 + choice(1000): 0 + choice(#{big}): #{big} + OUT + end + + def test_can_target_a_score_downwards + out, _ = capture_io do + assert_raises(Minitest::Assertion) do + run_test("can_target_a_score_downwards", database: {}, max_examples: 1000) do |test_case| + n = test_case.choice(1000) + m = test_case.choice(1000) + score = n + m + test_case.target(-score) + assert score.positive? + end + end + end + + assert_equal <<~OUT, out + choice(1000): 0 + choice(1000): 0 + OUT + end + + def test_prints_a_top_level_weighted + out, _ = capture_io do + assert_raises(Minitest::Assertion) do + run_test("prints_a_top_level_weighted", database: {}, max_examples: 1000) do |test_case| + assert test_case.weighted(0.5).nonzero? + end + end + end + + assert_equal <<~OUT, out + weighted(0.5): 0 + OUT + end + + def test_errors_when_using_frozen + tc = TestCase.for_choices([0]) + tc.status = Status::VALID + + assert_raises(Frozen) do + tc.mark_status(Status::INTERESTING) + end + + assert_raises(Frozen) do + tc.choice(10) + end + + assert_raises(Frozen) do + tc.forced_choice(10) + end + end + + def test_errors_on_too_large_choice + tc = TestCase.for_choices([0]) + assert_raises(RangeError) do + tc.choice(2 ** 64) + end + end + + def test_can_choose_full_64_bits + run_test("can_choose_full_64_bits", database: {}) do |tc| + tc.choice(2 ** 64 - 1) + end + end + + def test_mapped_possibility + run_test("mapped_possibility", database: {}) do |tc| + n = tc.any(integers(0, 5).map {|n| n * 2 }) + assert n.even? + end + end + + def test_selected_possibility + run_test("selected_possibility", database: {}) do |tc| + n = tc.any(integers(0, 5).satisfying(&:even?)) + assert n.even? + end + end + + def test_bound_possibility + run_test("bound_possibility", database: {}) do |tc| + m, n = tc.any( + integers(0, 5).bind {|m| tuples(just(m), integers(m, m + 10)) } + ) + assert (m..m+10).cover?(n) + end + end + + def test_cannot_witness_nothing + assert_raises(Unsatisfiable) do + run_test("cannot_witness_nothing", database: {}) do |tc| + tc.any(nothing) + end + end + end + + def test_cannot_witness_empty_mix_of + assert_raises(Unsatisfiable) do + run_test("cannot_witness_empty_mix_of", database: {}) do |tc| + tc.any(mix_of) + end + end + end + + def test_can_draw_mixture + run_test("can_draw_mixture", database: {}) do |tc| + m = tc.any(mix_of(integers(-5, 0), integers(2, 5))) + assert (-5..5).cover?(m) + refute_equal 1, m + end + end + + # This test is very hard to trigger without targeting, and targeting will + # tend to overshoot the score, so we will see multiple interesting test cases + # before shrinking. + def test_target_and_reduce + out, _ = capture_io do + assert_raises(Minitest::Assertion) do + run_test("target_and_reduce", database: {}) do |tc| + m = tc.choice(100_000) + tc.target(m) + assert m <= 99_900 + end + end + end + + assert_equal <<~OUT, out + choice(100000): 99901 + OUT + end + + def test_impossible_weighted + assert_raises(Failure) do + run_test("impossible_weighted", database: {}, quiet: true) do |tc| + tc.choice(1) + 10.times do + assert false unless tc.weighted(0.0).zero? + end + raise Failure if tc.choice(1).zero? + end + end + end + + def test_guaranteed_weighted + assert_raises(Failure) do + run_test("guaranteed_weighted", database: {}, quiet: true) do |tc| + if tc.weighted(1.0).nonzero? + tc.choice(1) + raise Failure + else + assert false + end + end + end + end + + def test_size_bounds_on_list + run_test("size_bounds_on_list", database: {}) do |tc| + ls = tc.any(lists(integers(0, 10), min_size: 1, max_size: 3)) + assert (1..3).cover?(ls.length) + end + end + + def test_forced_choice_bounds + assert_raises(RangeError) do + run_test("forced_choice_bounds", database: {}) do |tc| + tc.forced_choice(2 ** 64) + end + end + end + + def test_failure_from_hypothesis_1 + assert_raises(Failure) do + run_test("failure_from_hypothesis_1", database: {}, random: Random.new(100), max_examples: 1000, quiet: true) do |tc| + n1 = tc.weighted(0.0) + if n1.zero? + n2 = tc.choice(511) + if n2 == 112 + n3 = tc.choice(511) + if n3 == 124 + raise Failure + elsif n3 == 93 + raise Failure + else + tc.mark_status(Status::INVALID) + end + elsif n2 == 93 + raise Failure + else + tc.mark_status(Status::INVALID) + end + end + end + end + end + + def test_failure_from_hypothesis_2 + assert_raises(Failure) do + run_test("failure_from_hypothesis_2", database: {}, random: Random.new(0), max_examples: 1000, quiet: true) do |tc| + n1 = tc.choice(6) + if n1 == 6 + n2 = tc.weighted(0.0) + if n2.zero? + raise Failure + end + elsif n1 == 4 + n3 = tc.choice(0) + if n3 == 0 + raise Failure + else + tc.mark_status(Status::INVALID) + end + elsif n1 == 2 + raise Failure + else + tc.mark_status(Status::INVALID) + end + end + end + end + + private + + def suppress_warnings + original_verbosity = $VERBOSE + $VERBOSE = nil + yield + $VERBOSE = original_verbosity + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb new file mode 100644 index 0000000..e5cfcda --- /dev/null +++ b/test/test_helper.rb @@ -0,0 +1,4 @@ +$LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) +require "minitest/thesis" + +require "minitest/autorun"