From d7a804d120812401b389163996af28551cc2013b Mon Sep 17 00:00:00 2001 From: Spiro Trikaliotis Date: Sat, 12 Nov 2022 23:12:25 +0100 Subject: [PATCH] Split stdin and stdout in test/asm/ Some tests pipe the stdout and stderr of the compiler or assembler into a file and compare the result with a reference. This has the subtle problem that both stdout and stderr and buffered i/o streams that may or may not work the same on different OSs or even shells. That means the order of the lines in the resulting file is not guaranteed. In practise it is "mostly" not an issue, but "we" still stumbled about one test where different behaviour can be triggered depending on running it in cmd.exe or bash (msys): test/asm/listing output of 010-paramcount.bin differs from the reference output when running the test from cmd.exe. The solution is most likely to have two reference files and not redirect into one file. This patch fixes the issue (cf. #1912) for the directory test/asm/. --- test/asm/listing/Makefile | 54 +++++++++++++++---- test/asm/listing/ref/010-paramcount.err-ref | 9 ---- test/asm/listing/ref/010-paramcount.err2-ref | 9 ++++ ...ss.err-ref => 030-assert-success.err2-ref} | 0 ...rr-ref => 030-assert-success.ld65err2-ref} | 0 ...rror.err-ref => 031-assert-error.err2-ref} | 0 ...err-ref => 032-assert-error2.ld65err2-ref} | 0 ...err-ref => 032-assert-error3.ld65err2-ref} | 0 ...err-ref => 032-assert-error4.ld65err2-ref} | 0 ...err-ref => 032-assert-error5.ld65err2-ref} | 0 ...err-ref => 032-assert-error6.ld65err2-ref} | 0 ...err-ref => 032-assert-error7.ld65err2-ref} | 0 ...err-ref => 032-assert-error8.ld65err2-ref} | 0 ...033-assert-ldwarning-success.ld65err2-ref} | 0 ...1.err-ref => 034-assert-lderror1.err2-ref} | 0 ...r-ref => 034-assert-lderror2.ld65err2-ref} | 0 ...r-ref => 034-assert-lderror3.ld65err2-ref} | 0 ...r-ref => 034-assert-lderror4.ld65err2-ref} | 0 ...r-ref => 034-assert-lderror6.ld65err2-ref} | 0 ...r-ref => 034-assert-lderror7.ld65err2-ref} | 0 ...r-ref => 034-assert-lderror8.ld65err2-ref} | 0 ...ign.ld65err-ref => 040-align.ld65err2-ref} | 0 ...se-on-1.err-ref => 050-case-on-1.err2-ref} | 0 ...se-on-2.err-ref => 050-case-on-2.err2-ref} | 0 ...se-on-3.err-ref => 050-case-on-3.err2-ref} | 0 ...se-on-4.err-ref => 050-case-on-4.err2-ref} | 0 ...se-on-5.err-ref => 050-case-on-5.err2-ref} | 0 27 files changed, 53 insertions(+), 19 deletions(-) create mode 100644 test/asm/listing/ref/010-paramcount.err2-ref rename test/asm/listing/ref/{030-assert-success.err-ref => 030-assert-success.err2-ref} (100%) rename test/asm/listing/ref/{030-assert-success.ld65err-ref => 030-assert-success.ld65err2-ref} (100%) rename test/asm/listing/ref/{031-assert-error.err-ref => 031-assert-error.err2-ref} (100%) rename test/asm/listing/ref/{032-assert-error2.ld65err-ref => 032-assert-error2.ld65err2-ref} (100%) rename test/asm/listing/ref/{032-assert-error3.ld65err-ref => 032-assert-error3.ld65err2-ref} (100%) rename test/asm/listing/ref/{032-assert-error4.ld65err-ref => 032-assert-error4.ld65err2-ref} (100%) rename test/asm/listing/ref/{032-assert-error5.ld65err-ref => 032-assert-error5.ld65err2-ref} (100%) rename test/asm/listing/ref/{032-assert-error6.ld65err-ref => 032-assert-error6.ld65err2-ref} (100%) rename test/asm/listing/ref/{032-assert-error7.ld65err-ref => 032-assert-error7.ld65err2-ref} (100%) rename test/asm/listing/ref/{032-assert-error8.ld65err-ref => 032-assert-error8.ld65err2-ref} (100%) rename test/asm/listing/ref/{033-assert-ldwarning-success.ld65err-ref => 033-assert-ldwarning-success.ld65err2-ref} (100%) rename test/asm/listing/ref/{034-assert-lderror1.err-ref => 034-assert-lderror1.err2-ref} (100%) rename test/asm/listing/ref/{034-assert-lderror2.ld65err-ref => 034-assert-lderror2.ld65err2-ref} (100%) rename test/asm/listing/ref/{034-assert-lderror3.ld65err-ref => 034-assert-lderror3.ld65err2-ref} (100%) rename test/asm/listing/ref/{034-assert-lderror4.ld65err-ref => 034-assert-lderror4.ld65err2-ref} (100%) rename test/asm/listing/ref/{034-assert-lderror6.ld65err-ref => 034-assert-lderror6.ld65err2-ref} (100%) rename test/asm/listing/ref/{034-assert-lderror7.ld65err-ref => 034-assert-lderror7.ld65err2-ref} (100%) rename test/asm/listing/ref/{034-assert-lderror8.ld65err-ref => 034-assert-lderror8.ld65err2-ref} (100%) rename test/asm/listing/ref/{040-align.ld65err-ref => 040-align.ld65err2-ref} (100%) rename test/asm/listing/ref/{050-case-on-1.err-ref => 050-case-on-1.err2-ref} (100%) rename test/asm/listing/ref/{050-case-on-2.err-ref => 050-case-on-2.err2-ref} (100%) rename test/asm/listing/ref/{050-case-on-3.err-ref => 050-case-on-3.err2-ref} (100%) rename test/asm/listing/ref/{050-case-on-4.err-ref => 050-case-on-4.err2-ref} (100%) rename test/asm/listing/ref/{050-case-on-5.err-ref => 050-case-on-5.err2-ref} (100%) diff --git a/test/asm/listing/Makefile b/test/asm/listing/Makefile index d0d045a02..3c4c404af 100644 --- a/test/asm/listing/Makefile +++ b/test/asm/listing/Makefile @@ -9,7 +9,6 @@ ifdef CMD_EXE EXE = .exe MKDIR = mkdir $(subst /,\,$1) RMDIR = -rmdir /q /s $(subst /,\,$1) - ERRDIR = 2>&1 TRUE = exit 0 CAT = type $(subst /,\,$1) else @@ -17,7 +16,6 @@ else EXE = MKDIR = mkdir -p $1 RMDIR = $(RM) -r $1 - ERRDIR = 2>&1 TRUE = true CAT = cat $1 endif @@ -58,14 +56,14 @@ $(WORKDIR)/$1.bin: $1.s $(ISEQUAL) # compile without generating listing ifeq ($(wildcard control/$1.err),) - $(CA65) -t none -o $$(@:.bin=.o) $$< > $$(@:.bin=.err) $(ERRDIR) + $(CA65) -t none -o $$(@:.bin=.o) $$< > $$(@:.bin=.err) 2> $$(@:.bin=.err2) ifeq ($(wildcard control/$1.no-ld65),) - $(LD65) -t none -o $$@ $$(@:.bin=.o) none.lib > $$(@:.bin=.ld65-err) $(ERRDIR) + $(LD65) -t none -o $$@ $$(@:.bin=.o) none.lib > $$(@:.bin=.ld65-err) 2> $$(@:.bin=.ld65-err2) endif else - $(CA65) -t none -o $$(@:.bin=.o) $$< > $$(@:.bin=.err) $(ERRDIR) || $(TRUE) + $(CA65) -t none -o $$(@:.bin=.o) $$< > $$(@:.bin=.err) 2> $$(@:.bin=.err2) || $(TRUE) ifeq ($(wildcard control/$1.no-ld65),) - $(LD65) -t none -o $$@ $$(@:.bin=.o) none.lib > $$(@:.bin=.ld65-err) $(ERRDIR) || $(TRUE) + $(LD65) -t none -o $$@ $$(@:.bin=.o) none.lib > $$(@:.bin=.ld65-err) 2> $$(@:.bin=.ld65-err2) || $(TRUE) endif endif @@ -75,6 +73,12 @@ else $(ISEQUAL) --empty $$(@:.bin=.err) endif +ifneq ($(wildcard ref/$1.err2-ref),) + $(ISEQUAL) ref/$1.err2-ref $$(@:.bin=.err2) +else + $(ISEQUAL) --empty $$(@:.bin=.err2) +endif + ifneq ($(wildcard ref/$1.bin-ref),) $(ISEQUAL) --binary ref/$1.bin-ref $$@ endif @@ -95,16 +99,30 @@ ifneq ($(wildcard $(WORKDIR)/$1.ld65-err),) endif endif +ifneq ($(wildcard ref/$1.ld65err2-ref),) + @echo $(CAT) $$(@:.bin=.ld65-err2) +# FIXME: somehow this refuses to work in cmd.exe +ifndef CMD_EXE + $(call CAT,$$(@:.bin=.ld65-err2)) + -diff -u ref/$1.ld65err2-ref $$(@:.bin=.ld65-err2) +endif + $(ISEQUAL) --wildcards ref/$1.ld65err2-ref $$(@:.bin=.ld65-err2) +else +ifneq ($(wildcard $(WORKDIR)/$1.ld65-err2),) + $(ISEQUAL) --empty $$(@:.bin=.ld65-err2) +endif +endif + # compile with listing file ifeq ($(wildcard control/$1.err),) - $(CA65) -t none -l $$(@:.bin=.list-lst) -o $$(@:.bin=.list-o) $$< > $$(@:.bin=.list-err) $(ERRDIR) + $(CA65) -t none -l $$(@:.bin=.list-lst) -o $$(@:.bin=.list-o) $$< > $$(@:.bin=.list-err) 2> $$(@:.bin=.list-err2) ifeq ($(wildcard control/$1.no-ld65),) - $(LD65) -t none -o $$(@:.bin=.list-bin) $$(@:.bin=.list-o) none.lib > $$(@:.bin=.list-ld65-err) $(ERRDIR) + $(LD65) -t none -o $$(@:.bin=.list-bin) $$(@:.bin=.list-o) none.lib > $$(@:.bin=.list-ld65-err) 2> $$(@:.bin=.list-ld65-err2) endif else - $(CA65) -t none -l $$(@:.bin=.list-lst) -o $$(@:.bin=.list-o) $$< > $$(@:.bin=.list-err) $(ERRDIR) || $(TRUE) + $(CA65) -t none -l $$(@:.bin=.list-lst) -o $$(@:.bin=.list-o) $$< > $$(@:.bin=.list-err) 2> $$(@:.bin=.list-err2) || $(TRUE) ifeq ($(wildcard control/$1.no-ld65),) - $(LD65) -t none -o $$(@:.bin=.list-bin) $$(@:.bin=.list-o) none.lib > $$(@:.bin=.list-ld65-err) $(ERRDIR) || $(TRUE) + $(LD65) -t none -o $$(@:.bin=.list-bin) $$(@:.bin=.list-o) none.lib > $$(@:.bin=.list-ld65-err) 2> $$(@:.bin=.list-ld65-err2) || $(TRUE) endif endif @@ -122,10 +140,26 @@ ifneq ($(wildcard $(WORKDIR)/$1.list-ld65-err),) endif endif +ifneq ($(wildcard ref/$1.err2-ref),) + $(ISEQUAL) ref/$1.err2-ref $$(@:.bin=.list-err2) +else + $(ISEQUAL) --empty $$(@:.bin=.list-err2) +endif + +ifneq ($(wildcard ref/$1.ld65err2-ref),) + $(ISEQUAL) --wildcards ref/$1.ld65err2-ref $$(@:.bin=.list-ld65-err2) +else +ifneq ($(wildcard $(WORKDIR)/$1.list-ld65-err2),) + $(ISEQUAL) --empty $$(@:.bin=.list-ld65-err2) +endif +endif + # check if the result bin is the same as without listing file ifeq ($(wildcard control/$1.err),) +ifeq ($(wildcard control/$1.err2),) $(ISEQUAL) $$@ $$(@:.bin=.list-bin) endif +endif ifneq ($(wildcard ref/$1.list-ref),) # we have a reference file, compare that, too diff --git a/test/asm/listing/ref/010-paramcount.err-ref b/test/asm/listing/ref/010-paramcount.err-ref index a66162eb5..baf73f50f 100644 --- a/test/asm/listing/ref/010-paramcount.err-ref +++ b/test/asm/listing/ref/010-paramcount.err-ref @@ -1,15 +1,6 @@ .paramcount = 3 .paramcount = 5 -010-paramcount.s:18: Warning: User warning: r1 is blank! -010-paramcount.s:14: Note: Macro was defined here -010-paramcount.s:8: Note: Macro was defined here .paramcount = 3 .paramcount = 5 -010-paramcount.s:19: Warning: User warning: r1 is blank! -010-paramcount.s:14: Note: Macro was defined here -010-paramcount.s:8: Note: Macro was defined here .paramcount = 1 .paramcount = 5 -010-paramcount.s:20: Warning: User warning: r1 is blank! -010-paramcount.s:14: Note: Macro was defined here -010-paramcount.s:8: Note: Macro was defined here diff --git a/test/asm/listing/ref/010-paramcount.err2-ref b/test/asm/listing/ref/010-paramcount.err2-ref new file mode 100644 index 000000000..304c9de61 --- /dev/null +++ b/test/asm/listing/ref/010-paramcount.err2-ref @@ -0,0 +1,9 @@ +010-paramcount.s:18: Warning: User warning: r1 is blank! +010-paramcount.s:14: Note: Macro was defined here +010-paramcount.s:8: Note: Macro was defined here +010-paramcount.s:19: Warning: User warning: r1 is blank! +010-paramcount.s:14: Note: Macro was defined here +010-paramcount.s:8: Note: Macro was defined here +010-paramcount.s:20: Warning: User warning: r1 is blank! +010-paramcount.s:14: Note: Macro was defined here +010-paramcount.s:8: Note: Macro was defined here diff --git a/test/asm/listing/ref/030-assert-success.err-ref b/test/asm/listing/ref/030-assert-success.err2-ref similarity index 100% rename from test/asm/listing/ref/030-assert-success.err-ref rename to test/asm/listing/ref/030-assert-success.err2-ref diff --git a/test/asm/listing/ref/030-assert-success.ld65err-ref b/test/asm/listing/ref/030-assert-success.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/030-assert-success.ld65err-ref rename to test/asm/listing/ref/030-assert-success.ld65err2-ref diff --git a/test/asm/listing/ref/031-assert-error.err-ref b/test/asm/listing/ref/031-assert-error.err2-ref similarity index 100% rename from test/asm/listing/ref/031-assert-error.err-ref rename to test/asm/listing/ref/031-assert-error.err2-ref diff --git a/test/asm/listing/ref/032-assert-error2.ld65err-ref b/test/asm/listing/ref/032-assert-error2.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/032-assert-error2.ld65err-ref rename to test/asm/listing/ref/032-assert-error2.ld65err2-ref diff --git a/test/asm/listing/ref/032-assert-error3.ld65err-ref b/test/asm/listing/ref/032-assert-error3.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/032-assert-error3.ld65err-ref rename to test/asm/listing/ref/032-assert-error3.ld65err2-ref diff --git a/test/asm/listing/ref/032-assert-error4.ld65err-ref b/test/asm/listing/ref/032-assert-error4.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/032-assert-error4.ld65err-ref rename to test/asm/listing/ref/032-assert-error4.ld65err2-ref diff --git a/test/asm/listing/ref/032-assert-error5.ld65err-ref b/test/asm/listing/ref/032-assert-error5.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/032-assert-error5.ld65err-ref rename to test/asm/listing/ref/032-assert-error5.ld65err2-ref diff --git a/test/asm/listing/ref/032-assert-error6.ld65err-ref b/test/asm/listing/ref/032-assert-error6.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/032-assert-error6.ld65err-ref rename to test/asm/listing/ref/032-assert-error6.ld65err2-ref diff --git a/test/asm/listing/ref/032-assert-error7.ld65err-ref b/test/asm/listing/ref/032-assert-error7.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/032-assert-error7.ld65err-ref rename to test/asm/listing/ref/032-assert-error7.ld65err2-ref diff --git a/test/asm/listing/ref/032-assert-error8.ld65err-ref b/test/asm/listing/ref/032-assert-error8.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/032-assert-error8.ld65err-ref rename to test/asm/listing/ref/032-assert-error8.ld65err2-ref diff --git a/test/asm/listing/ref/033-assert-ldwarning-success.ld65err-ref b/test/asm/listing/ref/033-assert-ldwarning-success.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/033-assert-ldwarning-success.ld65err-ref rename to test/asm/listing/ref/033-assert-ldwarning-success.ld65err2-ref diff --git a/test/asm/listing/ref/034-assert-lderror1.err-ref b/test/asm/listing/ref/034-assert-lderror1.err2-ref similarity index 100% rename from test/asm/listing/ref/034-assert-lderror1.err-ref rename to test/asm/listing/ref/034-assert-lderror1.err2-ref diff --git a/test/asm/listing/ref/034-assert-lderror2.ld65err-ref b/test/asm/listing/ref/034-assert-lderror2.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/034-assert-lderror2.ld65err-ref rename to test/asm/listing/ref/034-assert-lderror2.ld65err2-ref diff --git a/test/asm/listing/ref/034-assert-lderror3.ld65err-ref b/test/asm/listing/ref/034-assert-lderror3.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/034-assert-lderror3.ld65err-ref rename to test/asm/listing/ref/034-assert-lderror3.ld65err2-ref diff --git a/test/asm/listing/ref/034-assert-lderror4.ld65err-ref b/test/asm/listing/ref/034-assert-lderror4.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/034-assert-lderror4.ld65err-ref rename to test/asm/listing/ref/034-assert-lderror4.ld65err2-ref diff --git a/test/asm/listing/ref/034-assert-lderror6.ld65err-ref b/test/asm/listing/ref/034-assert-lderror6.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/034-assert-lderror6.ld65err-ref rename to test/asm/listing/ref/034-assert-lderror6.ld65err2-ref diff --git a/test/asm/listing/ref/034-assert-lderror7.ld65err-ref b/test/asm/listing/ref/034-assert-lderror7.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/034-assert-lderror7.ld65err-ref rename to test/asm/listing/ref/034-assert-lderror7.ld65err2-ref diff --git a/test/asm/listing/ref/034-assert-lderror8.ld65err-ref b/test/asm/listing/ref/034-assert-lderror8.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/034-assert-lderror8.ld65err-ref rename to test/asm/listing/ref/034-assert-lderror8.ld65err2-ref diff --git a/test/asm/listing/ref/040-align.ld65err-ref b/test/asm/listing/ref/040-align.ld65err2-ref similarity index 100% rename from test/asm/listing/ref/040-align.ld65err-ref rename to test/asm/listing/ref/040-align.ld65err2-ref diff --git a/test/asm/listing/ref/050-case-on-1.err-ref b/test/asm/listing/ref/050-case-on-1.err2-ref similarity index 100% rename from test/asm/listing/ref/050-case-on-1.err-ref rename to test/asm/listing/ref/050-case-on-1.err2-ref diff --git a/test/asm/listing/ref/050-case-on-2.err-ref b/test/asm/listing/ref/050-case-on-2.err2-ref similarity index 100% rename from test/asm/listing/ref/050-case-on-2.err-ref rename to test/asm/listing/ref/050-case-on-2.err2-ref diff --git a/test/asm/listing/ref/050-case-on-3.err-ref b/test/asm/listing/ref/050-case-on-3.err2-ref similarity index 100% rename from test/asm/listing/ref/050-case-on-3.err-ref rename to test/asm/listing/ref/050-case-on-3.err2-ref diff --git a/test/asm/listing/ref/050-case-on-4.err-ref b/test/asm/listing/ref/050-case-on-4.err2-ref similarity index 100% rename from test/asm/listing/ref/050-case-on-4.err-ref rename to test/asm/listing/ref/050-case-on-4.err2-ref diff --git a/test/asm/listing/ref/050-case-on-5.err-ref b/test/asm/listing/ref/050-case-on-5.err2-ref similarity index 100% rename from test/asm/listing/ref/050-case-on-5.err-ref rename to test/asm/listing/ref/050-case-on-5.err2-ref