Make C64/C16 tests completely automated.

6502/C64/Makefile

@@ -14,24 +14,43 @@ test_files_petscii = $(patsubst tests/%, cbmfiles/%, $(test_files))
 vf_blk_fth: $(vf_blk_fth_files)
 
 
+clean:
+	rm -f cbmfiles/*.fr cbmfiles/%.fth *.log *.result
+
+
 # Convenience targets
 
+test: test-c64.result test-c16.result
+
+test64: test-c64.result
+
 run-devenv: emulator/devenv.T64
 	emulator/run-in-vice.sh devenv
 
 run-testbase: emulator/testbase.T64
 	emulator/run-in-vice.sh testbase
 
-test: emulator/testbase.T64 $(test_files_petscii)
-	emulator/run-in-vice.sh testbase \
-	  "include run-vf-tests.fth\n"
-
 run-testbase16: emulator/testbase16.T64
 	VICE=xplus4 emulator/run-in-vice.sh testbase16
 
-test16: emulator/testbase.T64 $(test_files_petscii)
+
+# Core test targets
+
+test-c64.result: emulator/testbase.T64 $(test_files_petscii) \
+    emulator/run-in-vice.sh tests/evaluate-test.sh tests/test-c64.golden
+	rm -f test-c64.log test-c64.result
+	emulator/run-in-vice.sh testbase \
+	  "include run-vf-tests.fth\n1234567890\ndos s0:notdone\n"
+	petscii2ascii cbmfiles/test.log test-c64.log
+	tests/evaluate-test.sh test-c64
+
+test-c16.result: emulator/testbase16.T64 $(test_files_petscii) \
+    emulator/run-in-vice.sh tests/evaluate-test.sh tests/test-c16.golden
+	rm -f test-c16.log test-c16.result
 	VICE=xplus4 emulator/run-in-vice.sh testbase16 \
-	  "include run-vf-tests.fth\n"
+	  "include run-vf-tests.fth\n1234567890\ndos s0:notdone\n"
+	petscii2ascii cbmfiles/test.log test-c16.log
+	tests/evaluate-test.sh test-c16
 
 
 # Rules for building Forth binaries on top of the plain vanilla

6502/C64/tests/evaluate-test.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+testsdir="$(realpath --relative-to="$PWD" "$(dirname "${BASH_SOURCE[0]}")")"
+basedir="$(realpath --relative-to="$PWD" "${testsdir}/..")"
+
+testname="$1"
+
+diff "${testsdir}/${testname}.golden" "${basedir}/${testname}.log" > tmp.result
+exitcode=$?
+test $exitcode -eq 0 \
+  && echo "PASS: ${testname}" >> tmp.result \
+  || echo "FAIL: ${testname}" >> tmp.result
+cat tmp.result
+mv tmp.result "${basedir}/${testname}.result"
+exit $exitcode

6502/C64/tests/run-vf-tests.fth

@@ -17,13 +17,16 @@ include core.fr
 
 include coreplustest.fth
 
-include utilities.fth
+\ The C16 VolksForth has LIMIT at $8000.
-include errorreport.fth
+\ More tests than up to here fill the dictionary.
 
-include coreexttest.fth
+(64 include utilities.fth C)
+(64 include errorreport.fth C)
 
-include doubletest.fth
+(64 include coreexttest.fth C)
 
-REPORT-ERRORS
+(64 include doubletest.fth C)
+
+(64 REPORT-ERRORS C)
 
 logclose

6502/C64/tests/test-c16.golden

@@ -0,0 +1,74 @@
+
+ans-shim.fth CHAR exists  0> exists 
+
+prelimtest.fth
+
+CR CR SOURCE TYPE ( Preliminary test ) CR
+SOURCE ( These lines test SOURCE, TYPE, CR and parenthetic comments ) TYPE CR
+( The next line of output should be blank to test CR ) SOURCE TYPE CR CR
+
+( Pass #1: testing 0 >IN +! ) 0 >IN +! SOURCE TYPE CR
+( Pass #2: testing 1 >IN +! ) 1 >IN +! xSOURCE TYPE CR
+( Pass #3: testing 1+ ) 1 1+ >IN +! xxSOURCE TYPE CR
+( Pass #4: testing @ ! BASE ) 0 1+ 1+ BASE ! BASE @ >IN +! xxSOURCE TYPE CR
+( Pass #5: testing decimal BASE ) BASE @ >IN +! xxxxxxxxxxSOURCE TYPE CR
+( Pass #6: testing : ; ) : .SRC SOURCE TYPE CR ; 6 >IN +! xxxxxx.SRC
+( Pass #7: testing number input ) 19 >IN +! xxxxxxxxxxxxxxxxxxx.SRC
+( Pass #8: testing VARIABLE ) VARIABLE Y 2 Y ! Y @ >IN +! xx.SRC
+( Pass #9: testing WORD COUNT ) 5 MSG abcdef) Y ! Y ! >IN +! xxxxx.SRC
+( Pass #10: testing WORD COUNT ) MSG ab) >IN +! xxY ! .SRC
+Pass #11: testing WORD COUNT .MSG
+Pass #12: testing = returns all 1's for true
+Pass #13: testing = returns 0 for false
+Pass #14: testing -1 interpreted correctly
+Pass #15: testing 2*
+Pass #16: testing 2*
+Pass #17: testing AND
+Pass #18: testing AND
+Pass #19: testing AND
+Pass #20: testing ?F~ ?~~ Pass Error
+Pass #21: testing ?~
+Pass #22: testing EMIT
+Pass #23: testing S"
+
+Results: 
+
+Pass messages #1 to #23 should be displayed above
+and no error messages
+
+0 tests failed out of 57 additional tests
+
+
+--- End of Preliminary Tests --- 
+
+tester.fth ERROR exists 
+core.fr
+*********************YOU SHOULD SEE THE STANDARD GRAPHIC CHARACTERS:
+ !"#$%&'()*+,-./0123456789:;<=>?@
+abcdefghijklmnopqrstuvwxyz[\]^_`
+ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~
+YOU SHOULD SEE 0-9 SEPARATED BY A SPACE:
+0 1 2 3 4 5 6 7 8 9 
+YOU SHOULD SEE 0-9 (WITH NO SPACES):
+0123456789
+YOU SHOULD SEE A-G SEPARATED BY A SPACE:
+A B C D E F G 
+YOU SHOULD SEE 0-5 SEPARATED BY TWO SPACES:
+0  1  2  3  4  5  
+YOU SHOULD SEE TWO SEPARATE LINES:
+LINE 1
+LINE 2
+YOU SHOULD SEE THE NUMBER RANGES OF SIGNED AND UNSIGNED NUMBERS:
+  SIGNED: -8000 7FFF 
+UNSIGNED: 0 FFFF 
+*
+PLEASE TYPE UP TO 80 CHARACTERS:
+1234567890 
+RECEIVED: "1234567890"
+* GDX exists 
+End of Core word set tests
+
+coreplustest.fth********
+You should see 2345: 2345
+*****
+End of additional Core tests

6502/C64/tests/test-c64.golden

@@ -0,0 +1,150 @@
+
+ans-shim.fth CHAR exists  0> exists 
+
+prelimtest.fth
+
+CR CR SOURCE TYPE ( Preliminary test ) CR
+SOURCE ( These lines test SOURCE, TYPE, CR and parenthetic comments ) TYPE CR
+( The next line of output should be blank to test CR ) SOURCE TYPE CR CR
+
+( Pass #1: testing 0 >IN +! ) 0 >IN +! SOURCE TYPE CR
+( Pass #2: testing 1 >IN +! ) 1 >IN +! xSOURCE TYPE CR
+( Pass #3: testing 1+ ) 1 1+ >IN +! xxSOURCE TYPE CR
+( Pass #4: testing @ ! BASE ) 0 1+ 1+ BASE ! BASE @ >IN +! xxSOURCE TYPE CR
+( Pass #5: testing decimal BASE ) BASE @ >IN +! xxxxxxxxxxSOURCE TYPE CR
+( Pass #6: testing : ; ) : .SRC SOURCE TYPE CR ; 6 >IN +! xxxxxx.SRC
+( Pass #7: testing number input ) 19 >IN +! xxxxxxxxxxxxxxxxxxx.SRC
+( Pass #8: testing VARIABLE ) VARIABLE Y 2 Y ! Y @ >IN +! xx.SRC
+( Pass #9: testing WORD COUNT ) 5 MSG abcdef) Y ! Y ! >IN +! xxxxx.SRC
+( Pass #10: testing WORD COUNT ) MSG ab) >IN +! xxY ! .SRC
+Pass #11: testing WORD COUNT .MSG
+Pass #12: testing = returns all 1's for true
+Pass #13: testing = returns 0 for false
+Pass #14: testing -1 interpreted correctly
+Pass #15: testing 2*
+Pass #16: testing 2*
+Pass #17: testing AND
+Pass #18: testing AND
+Pass #19: testing AND
+Pass #20: testing ?F~ ?~~ Pass Error
+Pass #21: testing ?~
+Pass #22: testing EMIT
+Pass #23: testing S"
+
+Results: 
+
+Pass messages #1 to #23 should be displayed above
+and no error messages
+
+0 tests failed out of 57 additional tests
+
+
+--- End of Preliminary Tests --- 
+
+tester.fth ERROR exists 
+core.fr
+*********************YOU SHOULD SEE THE STANDARD GRAPHIC CHARACTERS:
+ !"#$%&'()*+,-./0123456789:;<=>?@
+abcdefghijklmnopqrstuvwxyz[\]^_`
+ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~
+YOU SHOULD SEE 0-9 SEPARATED BY A SPACE:
+0 1 2 3 4 5 6 7 8 9 
+YOU SHOULD SEE 0-9 (WITH NO SPACES):
+0123456789
+YOU SHOULD SEE A-G SEPARATED BY A SPACE:
+A B C D E F G 
+YOU SHOULD SEE 0-5 SEPARATED BY TWO SPACES:
+0  1  2  3  4  5  
+YOU SHOULD SEE TWO SEPARATE LINES:
+LINE 1
+LINE 2
+YOU SHOULD SEE THE NUMBER RANGES OF SIGNED AND UNSIGNED NUMBERS:
+  SIGNED: -8000 7FFF 
+UNSIGNED: 0 FFFF 
+*
+PLEASE TYPE UP TO 80 CHARACTERS:
+1234567890 
+RECEIVED: "1234567890"
+* GDX exists 
+End of Core word set tests
+
+coreplustest.fth********
+You should see 2345: 2345
+*****
+End of additional Core tests
+
+utilities.fth ?DEFTEST1 exists 
+Test utilities loaded
+
+errorreport.fth
+coreexttest.fth**************
+
+Output from .(
+You should see -9876: -9876 
+and again: -9876
+
+
+On the next 2 lines you should see First then Second messages:
+First message via .( 
+Second message via ."
+
+*
+
+Output from .R and U.R
+You should see lines duplicated:
+indented by 0 spaces
+30278 
+30278
+-31871 
+-31871
+30278 
+30278
+33665 
+33665
+
+indented by 0 spaces
+30278 
+30278
+-31871 
+-31871
+30278 
+30278
+33665 
+33665
+
+indented by 5 spaces
+     30278 
+     30278
+     -31871 
+     -31871
+     30278 
+     30278
+     33665 
+     33665
+
+***
+End of Core Extension word tests
+
+doubletest.fth*****************
+End of Double-Number word tests
+
+---------------------------
+        Error Report
+Word Set             Errors
+---------------------------
+Core                    0
+Core extension          0
+Block                   -
+Double number           0
+Exception               -
+Facility                -
+File-access             -
+Locals                  -
+Memory-allocation       -
+Programming-tools       -
+Search-order            -
+String                  -
+---------------------------
+Total                   0
+---------------------------
+