From c57c8f87b4baba6d6d52d4d4806d2926c95b81ad Mon Sep 17 00:00:00 2001 From: transistor Date: Fri, 9 Sep 2022 23:17:33 -0700 Subject: [PATCH] Added test running for Tom Harte's ProcessorTests test suite --- Cargo.toml | 3 +- src/error.rs | 9 + tests/ProcessorTests | 1 + tests/harte_tests/Cargo.toml | 12 ++ tests/harte_tests/README.md | 18 ++ tests/harte_tests/previous.txt | 130 ++++++++++++++ tests/harte_tests/src/main.rs | 314 +++++++++++++++++++++++++++++++++ 7 files changed, 485 insertions(+), 2 deletions(-) create mode 160000 tests/ProcessorTests create mode 100644 tests/harte_tests/Cargo.toml create mode 100644 tests/harte_tests/README.md create mode 100644 tests/harte_tests/previous.txt create mode 100644 tests/harte_tests/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 2e69cdf..1e3a8b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,8 +4,7 @@ version = "0.1.0" edition = "2018" [workspace] -members = [".", "frontends/moa-common", "frontends/moa-console", "frontends/moa-minifb"] +members = [".", "frontends/moa-common", "frontends/moa-console", "frontends/moa-minifb", "tests/harte_tests"] default-members = ["frontends/moa-console"] [dependencies] - diff --git a/src/error.rs b/src/error.rs index f702694..453e9f0 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,7 @@ #[derive(Copy, Clone, Debug, PartialEq)] pub enum ErrorType { + Assertion, Emulator, Processor, Breakpoint, @@ -37,6 +38,14 @@ impl Error { msg: msg.to_string(), } } + + pub fn assertion(msg: &str) -> Error { + Error { + err: ErrorType::Assertion, + native: 0, + msg: msg.to_string(), + } + } } diff --git a/tests/ProcessorTests b/tests/ProcessorTests new file mode 160000 index 0000000..2048468 --- /dev/null +++ b/tests/ProcessorTests @@ -0,0 +1 @@ +Subproject commit 204846854d522fea3de0fdd260808d18bb180b05 diff --git a/tests/harte_tests/Cargo.toml b/tests/harte_tests/Cargo.toml new file mode 100644 index 0000000..4163bcc --- /dev/null +++ b/tests/harte_tests/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "harte_tests" +version = "0.1.0" +edition = "2018" + +[dependencies] +moa = { path = "../../" } +serde = "1.0" +serde_json = "1.0" +serde_derive = "1.0" +flate2 = "1.0" +clap = { version = "3.2.20", features = ["derive"] } diff --git a/tests/harte_tests/README.md b/tests/harte_tests/README.md new file mode 100644 index 0000000..8da029b --- /dev/null +++ b/tests/harte_tests/README.md @@ -0,0 +1,18 @@ + +Tom Harte Test Suite +==================== + +This is a test running for moa that uses the [Tom Harte Test Suite](https://github.com/TomHarte/ProcessorTests). + +To run, the ProcessorTests repository must be cloned into tests/ and then from the moa project root: +```shell +cargo run -p harte_tests -- [FILTER] +``` + +An optional filter can be specified, which will only run test files who's file name starts with the +filter text. Timing tests are not done by default, but can be run with `-t` or `--timing`. The output +can be increased or decreased with the `--debug` or `--quiet` flags, respectively. + +Special thanks to [Tom](https://github.com/TomHarte) for painstakingly constructing this test suite. +Emulators everywhere will be better for your efforts! + diff --git a/tests/harte_tests/previous.txt b/tests/harte_tests/previous.txt new file mode 100644 index 0000000..92ab107 --- /dev/null +++ b/tests/harte_tests/previous.txt @@ -0,0 +1,130 @@ +Last run on 2022-09-09 + +ABCD.json.gz completed: 150 passed, 7915 FAILED +ADD.b.json.gz completed: 4924 passed, 3141 FAILED +ADD.l.json.gz completed: 3636 passed, 4429 FAILED +ADD.w.json.gz completed: 3669 passed, 4396 FAILED +ADDA.l.json.gz completed: 4843 passed, 3222 FAILED +ADDA.w.json.gz completed: 4916 passed, 3149 FAILED +ADDX.b.json.gz completed: 4051 passed, 4014 FAILED +ADDX.l.json.gz completed: 4065 passed, 4000 FAILED +ADDX.w.json.gz completed: 4012 passed, 4053 FAILED +AND.b.json.gz completed: 5274 passed, 2791 FAILED +AND.l.json.gz completed: 3263 passed, 4802 FAILED +AND.w.json.gz completed: 3293 passed, 4772 FAILED +ANDItoCCR.json.gz completed, all passed! +ANDItoSR.json.gz completed, all passed! +ASL.b.json.gz completed: 3941 passed, 4124 FAILED +ASL.l.json.gz completed: 3578 passed, 4487 FAILED +ASL.w.json.gz completed: 3246 passed, 4819 FAILED +ASR.b.json.gz completed: 6316 passed, 1749 FAILED +ASR.l.json.gz completed: 7007 passed, 1058 FAILED +ASR.w.json.gz completed: 5414 passed, 2651 FAILED +BCHG.json.gz completed: 4316 passed, 3749 FAILED +BCLR.json.gz completed: 4701 passed, 3364 FAILED +BSET.json.gz completed: 4475 passed, 3590 FAILED +BSR.json.gz completed: 4078 passed, 3987 FAILED +BTST.json.gz completed: 6851 passed, 1214 FAILED +Bcc.json.gz completed: 5861 passed, 2204 FAILED +CHK.json.gz completed: 0 passed, 8065 FAILED +CLR.b.json.gz completed: 6594 passed, 1471 FAILED +CLR.l.json.gz completed: 4301 passed, 3764 FAILED +CLR.w.json.gz completed: 4327 passed, 3738 FAILED +CMP.b.json.gz completed: 6627 passed, 1438 FAILED +CMP.l.json.gz completed: 4634 passed, 3431 FAILED +CMP.w.json.gz completed: 4597 passed, 3468 FAILED +CMPA.l.json.gz completed: 4998 passed, 3067 FAILED +CMPA.w.json.gz completed: 3895 passed, 4170 FAILED +DBcc.json.gz completed: 5194 passed, 2871 FAILED +DIVS.json.gz completed: 539 passed, 7526 FAILED +DIVU.json.gz completed: 1093 passed, 6972 FAILED +EOR.b.json.gz completed: 4359 passed, 3706 FAILED +EOR.l.json.gz completed: 2898 passed, 5167 FAILED +EOR.w.json.gz completed: 2936 passed, 5129 FAILED +EORItoCCR.json.gz completed: 1067 passed, 6998 FAILED +EORItoSR.json.gz completed: 141 passed, 7924 FAILED +EXG.json.gz completed, all passed! +EXT.l.json.gz completed, all passed! +EXT.w.json.gz completed, all passed! +JMP.json.gz completed: 533 passed, 7532 FAILED +JSR.json.gz completed: 152 passed, 7913 FAILED +LEA.json.gz completed: 5726 passed, 2339 FAILED +LINK.json.gz completed: 7060 passed, 1005 FAILED +LSL.b.json.gz completed: 7774 passed, 291 FAILED +LSL.l.json.gz completed: 7017 passed, 1048 FAILED +LSL.w.json.gz completed: 6151 passed, 1914 FAILED +LSR.b.json.gz completed: 7797 passed, 268 FAILED +LSR.l.json.gz completed: 7044 passed, 1021 FAILED +LSR.w.json.gz completed: 6157 passed, 1908 FAILED +MOVE.b.json.gz completed: 5383 passed, 2682 FAILED +MOVE.l.json.gz completed: 2626 passed, 5439 FAILED +MOVE.q.json.gz completed, all passed! +MOVE.w.json.gz completed: 2709 passed, 5356 FAILED +MOVEA.l.json.gz completed: 4827 passed, 3238 FAILED +MOVEA.w.json.gz completed: 4813 passed, 3252 FAILED +MOVEM.l.json.gz completed: 3286 passed, 4779 FAILED +MOVEM.w.json.gz completed: 3324 passed, 4741 FAILED +MOVEP.l.json.gz completed: 4036 passed, 4029 FAILED +MOVEP.w.json.gz completed: 4046 passed, 4019 FAILED +MOVEfromSR.json.gz completed: 4456 passed, 3609 FAILED +MOVEfromUSP.json.gz completed, all passed! +MOVEtoCCR.json.gz completed: 541 passed, 7524 FAILED +MOVEtoSR.json.gz completed: 90 passed, 7975 FAILED +MOVEtoUSP.json.gz completed, all passed! +MULS.json.gz completed: 2241 passed, 5824 FAILED +MULU.json.gz completed: 4388 passed, 3677 FAILED +NBCD.json.gz completed: 0 passed, 8065 FAILED +NEG.b.json.gz completed: 4372 passed, 3693 FAILED +NEG.l.json.gz completed: 2991 passed, 5074 FAILED +NEG.w.json.gz completed: 2870 passed, 5195 FAILED +NEGX.b.json.gz completed: 0 passed, 8065 FAILED +NEGX.l.json.gz completed: 0 passed, 8065 FAILED +NEGX.w.json.gz completed: 0 passed, 8065 FAILED +NOP.json.gz completed, all passed! +NOT.b.json.gz completed: 4424 passed, 3641 FAILED +NOT.l.json.gz completed: 2915 passed, 5150 FAILED +NOT.w.json.gz completed: 2944 passed, 5121 FAILED +OR.b.json.gz completed: 5220 passed, 2845 FAILED +OR.l.json.gz completed: 3294 passed, 4771 FAILED +OR.w.json.gz completed: 3204 passed, 4861 FAILED +ORItoCCR.json.gz completed: 987 passed, 7078 FAILED +ORItoSR.json.gz completed: 118 passed, 7947 FAILED +PEA.json.gz completed: 5798 passed, 2267 FAILED +RESET.json.gz completed: 0 passed, 8065 FAILED +ROL.b.json.gz completed, all passed! +ROL.l.json.gz completed, all passed! +ROL.w.json.gz completed: 6560 passed, 1505 FAILED +ROR.b.json.gz completed, all passed! +ROR.l.json.gz completed, all passed! +ROR.w.json.gz completed: 6511 passed, 1554 FAILED +ROXL.b.json.gz completed: 8039 passed, 26 FAILED +ROXL.l.json.gz completed: 8029 passed, 36 FAILED +ROXL.w.json.gz completed: 6534 passed, 1531 FAILED +ROXR.b.json.gz completed: 8037 passed, 28 FAILED +ROXR.l.json.gz completed: 8022 passed, 43 FAILED +ROXR.w.json.gz completed: 6531 passed, 1534 FAILED +RTE.json.gz completed: 0 passed, 8065 FAILED +RTR.json.gz completed: 0 passed, 8065 FAILED +RTS.json.gz completed: 11 passed, 8054 FAILED +SBCD.json.gz completed: 439 passed, 7626 FAILED +SUB.b.json.gz completed: 5037 passed, 3028 FAILED +SUB.l.json.gz completed: 3635 passed, 4430 FAILED +SUB.w.json.gz completed: 3603 passed, 4462 FAILED +SUBA.l.json.gz completed: 4787 passed, 3278 FAILED +SUBA.w.json.gz completed: 4842 passed, 3223 FAILED +SUBX.b.json.gz completed: 3861 passed, 4204 FAILED +SUBX.l.json.gz completed: 3955 passed, 4110 FAILED +SUBX.w.json.gz completed: 3876 passed, 4189 FAILED +SWAP.json.gz completed: 509 passed, 7556 FAILED +Scc.json.gz completed: 6637 passed, 1428 FAILED +TAS.json.gz completed: 4409 passed, 3656 FAILED +TRAP.json.gz completed: 0 passed, 8065 FAILED +TRAPV.json.gz completed: 3970 passed, 4095 FAILED +TST.b.json.gz completed: 6566 passed, 1499 FAILED +TST.l.json.gz completed: 4381 passed, 3684 FAILED +TST.w.json.gz completed: 4362 passed, 3703 FAILED +UNLINK.json.gz completed, all passed! + +passed: 541447, failed: 458613, total: 54% +completed in 17m 28s + diff --git a/tests/harte_tests/src/main.rs b/tests/harte_tests/src/main.rs new file mode 100644 index 0000000..c63e340 --- /dev/null +++ b/tests/harte_tests/src/main.rs @@ -0,0 +1,314 @@ + +const HART_TESTS: &str = "tests/ProcessorTests/680x0/68000/v1/"; + +use std::io::prelude::*; +use std::fmt::Debug; +use std::path::PathBuf; +use std::time::SystemTime; +use std::fs::{self, File}; + +use clap::Parser; +use flate2::read::GzDecoder; +use serde_derive::Deserialize; + +use moa::error::Error; +use moa::system::System; +use moa::memory::{MemoryBlock, BusPort}; +use moa::devices::{Addressable, Steppable, wrap_transmutable}; + +use moa::cpus::m68k::{M68k, M68kType}; +use moa::cpus::m68k::state::Status; + +#[derive(Parser)] +struct Args { + /// Filter the tests by gzip file name + filter: Option, + /// Dump the CPU state when a test fails + #[clap(short, long)] + debug: bool, + /// Only print a summary for each test file + #[clap(short, long)] + quiet: bool, + /// Also test instruction timing + #[clap(short, long)] + timing: bool, +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +enum InfoLevel { + Quiet, + Normal, + Debug, +} + +fn main() { + let args = Args::parse(); + + let level = if args.debug { + InfoLevel::Debug + } else if args.quiet { + InfoLevel::Quiet + } else { + InfoLevel::Normal + }; + + run_all_tests(args, level); +} + + +#[derive(Debug, Deserialize)] +struct TestState { + d0: u32, + d1: u32, + d2: u32, + d3: u32, + d4: u32, + d5: u32, + d6: u32, + d7: u32, + a0: u32, + a1: u32, + a2: u32, + a3: u32, + a4: u32, + a5: u32, + a6: u32, + usp: u32, + ssp: u32, + sr: u16, + pc: u32, + prefetch: Vec, + ram: Vec<(u32, u8)>, +} + +#[derive(Debug, Deserialize)] +struct TestCase { + name: String, + #[serde(rename(deserialize = "initial"))] + initial_state: TestState, + #[serde(rename(deserialize = "final"))] + final_state: TestState, + length: usize +} + +fn init_execute_test(cputype: M68kType, state: &TestState) -> Result<(M68k, System), Error> { + let mut system = System::new(); + + // Insert basic initialization + let data = vec![0; 0x01000000]; + let mem = MemoryBlock::new(data); + system.add_addressable_device(0x00000000, wrap_transmutable(mem)).unwrap(); + + let port = if cputype <= M68kType::MC68010 { + BusPort::new(0, 24, 16, system.bus.clone()) + } else { + BusPort::new(0, 32, 32, system.bus.clone()) + }; + let mut cpu = M68k::new(cputype, 10_000_000, port); + cpu.state.status = Status::Running; + + load_state(&mut cpu, &mut system, state)?; + + Ok((cpu, system)) +} + +fn assert_value(actual: T, expected: T, message: &str) -> Result<(), Error> { + if actual == expected { + Ok(()) + } else { + Err(Error::assertion(&format!("{:?} != {:?}, {}", actual, expected, message))) + } +} + +fn load_state(cpu: &mut M68k, system: &mut System, initial: &TestState) -> Result<(), Error> { + cpu.state.d_reg[0] = initial.d0; + cpu.state.d_reg[1] = initial.d1; + cpu.state.d_reg[2] = initial.d2; + cpu.state.d_reg[3] = initial.d3; + cpu.state.d_reg[4] = initial.d4; + cpu.state.d_reg[5] = initial.d5; + cpu.state.d_reg[6] = initial.d6; + cpu.state.d_reg[7] = initial.d7; + cpu.state.a_reg[0] = initial.a0; + cpu.state.a_reg[1] = initial.a1; + cpu.state.a_reg[2] = initial.a2; + cpu.state.a_reg[3] = initial.a3; + cpu.state.a_reg[4] = initial.a4; + cpu.state.a_reg[5] = initial.a5; + cpu.state.a_reg[6] = initial.a6; + + cpu.state.usp = initial.usp; + cpu.state.ssp = initial.ssp; + cpu.state.sr = initial.sr; + cpu.state.pc = initial.pc; + + // Load instructions into memory + for (i, ins) in initial.prefetch.iter().enumerate() { + system.get_bus().write_beu16((initial.pc + (i as u32 * 2)) as u64, *ins)?; + } + + // Load data bytes into memory + for (addr, byte) in initial.ram.iter() { + system.get_bus().write_u8(*addr as u64, *byte)?; + } + + Ok(()) +} + +fn assert_state(cpu: &M68k, system: &System, expected: &TestState) -> Result<(), Error> { + assert_value(cpu.state.d_reg[0], expected.d0, "d0")?; + assert_value(cpu.state.d_reg[1], expected.d1, "d1")?; + assert_value(cpu.state.d_reg[2], expected.d2, "d2")?; + assert_value(cpu.state.d_reg[3], expected.d3, "d3")?; + assert_value(cpu.state.d_reg[4], expected.d4, "d4")?; + assert_value(cpu.state.d_reg[5], expected.d5, "d5")?; + assert_value(cpu.state.d_reg[6], expected.d6, "d6")?; + assert_value(cpu.state.d_reg[7], expected.d7, "d7")?; + assert_value(cpu.state.a_reg[0], expected.a0, "a0")?; + assert_value(cpu.state.a_reg[1], expected.a1, "a1")?; + assert_value(cpu.state.a_reg[2], expected.a2, "a2")?; + assert_value(cpu.state.a_reg[3], expected.a3, "a3")?; + assert_value(cpu.state.a_reg[4], expected.a4, "a4")?; + assert_value(cpu.state.a_reg[5], expected.a5, "a5")?; + assert_value(cpu.state.a_reg[6], expected.a6, "a6")?; + + assert_value(cpu.state.usp, expected.usp, "usp")?; + assert_value(cpu.state.ssp, expected.ssp, "ssp")?; + assert_value(cpu.state.sr, expected.sr, "sr")?; + assert_value(cpu.state.pc, expected.pc, "pc")?; + + // Load instructions into memory + for (i, ins) in expected.prefetch.iter().enumerate() { + let addr = expected.pc + (i as u32 * 2); + let actual = system.get_bus().read_beu16(addr as u64)?; + assert_value(actual, *ins, &format!("prefetch at {}", addr))?; + } + + // Load data bytes into memory + for (addr, byte) in expected.ram.iter() { + let actual = system.get_bus().read_u8(*addr as u64)?; + assert_value(actual, *byte, &format!("ram at {}", addr))?; + } + + Ok(()) +} + +fn step_cpu_and_assert(cpu: &mut M68k, system: &System, case: &TestCase, test_timing: bool) -> Result<(), Error> { + let clock_elapsed = cpu.step(&system)?; + let cycles = clock_elapsed / (1_000_000_000 / cpu.frequency as u64); + + assert_state(&cpu, &system, &case.final_state)?; + + if test_timing { + assert_value(cycles, case.length as u64, "clock cycles")?; + } + Ok(()) +} + +fn run_test(case: &TestCase, level: InfoLevel, test_timing: bool) -> Result<(), Error> { + let (mut cpu, system) = init_execute_test(M68kType::MC68010, &case.initial_state).unwrap(); + + let result = step_cpu_and_assert(&mut cpu, &system, case, test_timing); + + match result { + Ok(()) => Ok(()), + Err(err) => { + if level > InfoLevel::Quiet { + if level == InfoLevel::Debug { + cpu.dump_state(&system); + } + println!("FAILED: {}", err.msg); + } + Err(err) + }, + } +} + +fn test_json_file(path: PathBuf, level: InfoLevel, test_timing: bool) -> (usize, usize, String) { + let file = File::open(&path).unwrap(); + let mut decoder = GzDecoder::new(file); + let mut data = String::new(); + decoder.read_to_string(&mut data).unwrap(); + let cases: Vec = serde_json::from_str(&data).unwrap(); + + let mut passed = 0; + let mut failed = 0; + for case in cases { + if level > InfoLevel::Quiet { + println!("Running test {}", case.name); + } + let result = run_test(&case, level, test_timing); + + if let Err(err) = result { + failed += 1; + if level > InfoLevel::Quiet { + println!("FAILED: {:?}", err); + } + } else { + passed += 1 + } + } + + let name = path.file_name().unwrap().to_str().unwrap(); + let message = if failed == 0 { + format!("{} completed, all passed!", name) + } else { + format!("{} completed: {} passed, {} FAILED", name, passed, failed) + }; + + (passed, failed, message) +} + + +fn run_all_tests(args: Args, level: InfoLevel) { + let mut passed = 0; + let mut failed = 0; + let mut messages = vec![]; + + + let mut tests: Vec = fs::read_dir(HART_TESTS) + .unwrap() + .map(|dirent| dirent.unwrap().path()) + .collect(); + tests.sort(); + + let start = SystemTime::now(); + for path in tests { + // Only test gzip files (the repo has .md files as well) + if path.extension().unwrap() != "gz" { + continue; + } + + // If specified, only test files that start with a given string + if let Some(filter) = &args.filter { + if !path.file_name().unwrap().to_str().unwrap().starts_with(filter) { + continue; + } + } + + // Run every test in the file + let (test_passed, test_failed, message) = test_json_file(path, level, args.timing); + + // In quiet mode, print each summary as it's received to give a progress update + if level == InfoLevel::Quiet { + println!("{}", message); + } + + passed += test_passed; + failed += test_failed; + messages.push(message); + } + let elapsed_secs = start.elapsed().unwrap().as_secs(); + + // Print the stored summary if not in quite mode + if level > InfoLevel::Quiet { + for message in messages { + println!("{}", message); + } + } + + println!(""); + println!("passed: {}, failed: {}, total {}%", passed, failed, (passed / (passed + failed)) * 100); + println!("completed in {}m {}s", elapsed_secs / 60, elapsed_secs % 60); +} +