of816/platforms/GoSXB/run-tests.rb

#!/usr/bin/ruby

require 'open3'
require 'yaml'
require 'stringio'
require 'timeout'
require 'optparse'

@opts = {
    test_dir: '../../test',
    suites: '*',
    output_file: nil,
    cov_file: nil,
    verbose: $DEBUG,
    list_only: false
}

@coverage = Hash.new(0)

OptionParser.new do |o|
    o.banner = "Usage: #{$0} [options]"

    o.on('-d', '--dir TEST_DIR', 'Specify test directory.') do |d|
        @opts[:test_dir] = d
    end

    o.on('-s', '--suites PATTERN', 'Specify suites as shell glob.') do |p|
        @opts[:suites] = p
    end

    o.on('-o', '--outfile FILENAME', 'Specify test output file (default none).') do |f|
        @opts[:output_file] = f
    end

    o.on('-c', '--coverage FILENAME', 'Specify test coverage file (default none).') do |f|
        @opts[:cov_file] = f
    end

    o.on('-l', '--list-suites', 'List available test suites.') do |d|
        @opts[:list_only] = true
    end

    o.on("-v", "--[no-]verbose", "Run verbosely") do |v|
        @opts[:verbose] = v
    end

    o.on_tail("-h", "--help", "Show this message") do
        puts o
        exit
    end
end.parse!

@total_errors = 0

def verbose
    @opts[:verbose]
end

def run_suite(suite, outfile = nil)
    errors = 0
    suite_text = []
    outbuf = StringIO.new
    errbuf = StringIO.new
    suite['load'].each do |file|
        suite_text += File.readlines("#{@opts[:test_dir]}/#{file}")
    end
    colons = {}
    suite_text << "\nbye\n"
    Open3.popen3('./gosxb-of816.sh') do |stdin, stdout, stderr, wait_thr|
        until [stdout, stderr].find {|f| !f.eof}.nil?
            readable, writable, errored = IO.select([stdout,stderr],[stdin],[],0)
            if writable.include?(stdin)
                if line = suite_text.shift
                    puts ">> #{line}" if verbose
                    stdin.write(line)
                else
                    puts "Lines complete."
                    stdin.flush
                    begin
                        # Give test 30 seconds to complete
                        Timeout.timeout(30) do
                            outbuf.write(stdout.read)
                        end
                    rescue Timeout::Error
                        begin
                            outbuf.write(stdout.read_nonblock(1024))
                        rescue IO::EAGAINWaitReadable
                            # nothing
                        end
                        puts outbuf.string unless verbose
                        STDERR.puts "Emulator did not exit on its own."
                        errors += 1
                    end
                    stdin.close
                    puts outbuf.string if verbose
                    break
                end
            end
            if readable.include?(stdout)
                text = stdout.readline
                puts "<< #{text}" if verbose
                outbuf.write(text)
            end
            if readable.include?(stderr)
                stdin.flush
                errbuf.write(stderr.read)
                puts "Unexpected output on stderr:"
                puts outbuf.string
                puts errbuf.string
                stdout.close
                stderr.close
                exit 2
            end
            break unless wait_thr.alive?
        end
        outfile.write(outbuf.string) if outfile
        prevline = ""
        outbuf.string.lines.each do |line|
            cs_line = line.gsub(/\\.+$/,'')
            case line
            when /Exception/, /Def not found/, /Stack u/
                STDERR.puts prevline, line
                errors += 1
            when /WRONG/, /INCORRECT/
                unless prevline =~ /\[OK\]/
                    STDERR.puts line
                    errors += 1
                end
            when /TESTING/i
                puts line unless line.start_with?(':') || line.start_with?('\\')
            end
            if cs_line =~ /\s*:\s+(\S+)/
                colons[$1.downcase] = true
            end
            if cs_line =~ /\s*[tT]\{\s+:\s+(\S+)/
                colons[$1.downcase] = true
            end
            if cs_line =~ /^\s*T\{\s+(.+)\s+\-\>\s+/i
                words = $1.split(/\s+/)
                words.each do |word|
                    next if word == '->'
                    @coverage[word.downcase] += 1 unless colons[word.downcase]
                end
            end
            if line =~ /\\.*\s+covers:\s*(\S+)\s*$/
              @coverage[$1.downcase] += 1
            end
            if prevline =~ /expect:\s*\"(.+)\"\s*$/
                unless line.chomp == $1
                    STDERR.puts prevline
                    STDERR.puts "Unexpected: #{line.chomp.inspect}"
                    errors += 1
                end
            elsif prevline =~ /expect:\s*\/(.+)\/\s*$/
                rexp = Regexp.new($1)
                unless line.chomp =~ rexp
                    STDERR.puts prevline
                    STDERR.puts "Unexpected: #{line.chomp.inspect}"
                    errors += 1
                end
            end

            prevline = line
        end
        puts "Errors = #{errors}"
    end
    puts "Suite complete."
    return errors
end

outfile = nil
begin
    manifest = YAML.load(File.read("#{@opts[:test_dir]}/test-manifest.yaml"))
    #puts manifest.inspect

    unless @opts[:verbose]
        outfile = File.open(@opts[:output_file], 'w') if @opts[:output_file]
    end

    manifest.each do |suite|
        if File.fnmatch(@opts[:suites], suite['name'])
            print "Executing suite: " unless @opts[:list_only]
            puts suite['name']
            @total_errors += run_suite(suite, outfile) unless @opts[:list_only]
        end
    end
rescue RuntimeError => e
    STDERR.puts e.to_s
ensure
    outfile.close if outfile
end

exit 0 if @opts[:list_only]

File.write(@opts[:cov_file], @coverage.to_yaml) if @opts[:cov_file]

if @total_errors > 0
    STDERR.puts "Tests complete, total errors: #{@total_errors}"
    exit 1
end

STDOUT.puts "Tests complete, no errors."
exit 0