Port source generation test harness

I haven't figured out how to make the AppendText(text, color) extension
work right with the WPF version of RichTextBox, but I was able to work
around the issue, and it's not really worth sinking time into.

CommonWPF/WPFExtensions.cs

@@ -18,6 +18,7 @@ using System.Diagnostics;
 using System.Windows;
 using System.Windows.Controls;
 using System.Windows.Controls.Primitives;
+using System.Windows.Documents;
 using System.Windows.Input;
 using System.Windows.Media;
 
@@ -217,4 +218,31 @@ namespace CommonWPF {
         }
 #endif
     }
+
+    /// <summary>
+    /// RichTextBox extensions.
+    /// </summary>
+    public static class RichTextBoxExtensions {
+        /// <summary>
+        /// Overloads RichTextBox.AppendText() with a version that takes a color as an argument.
+        /// NOTE: color is "sticky", and will affect the next call to the built-in AppendText()
+        /// method.
+        /// </summary>
+        /// <remarks>
+        /// Adapted from https://stackoverflow.com/a/23402165/294248
+        ///
+        /// TODO(someday): figure out how to reset the color for future calls.
+        /// </remarks>
+        public static void AppendText(this RichTextBox box, string text, Color color) {
+
+            TextRange tr = new TextRange(box.Document.ContentEnd, box.Document.ContentEnd);
+            tr.Text = text;
+            try {
+                tr.ApplyPropertyValue(TextElement.ForegroundProperty,
+                    new SolidColorBrush(color));
+            } catch (FormatException ex) {
+                Debug.WriteLine("RTB AppendText extension failed: " + ex);
+            }
+        }
+    }
 }

SourceGenWPF/MainController.cs

@@ -2249,5 +2249,14 @@ namespace SourceGenWPF {
         }
 
         #endregion Info panel
+
+        #region Debug features
+
+        public void RunSourceGenerationTests() {
+            Tests.WpfGui.GenTestRunner dlg = new Tests.WpfGui.GenTestRunner(mMainWin);
+            dlg.ShowDialog();
+        }
+
+        #endregion
     }
 }

SourceGenWPF/SourceGenWPF.csproj

@@ -73,6 +73,11 @@
     <Compile Include="AsmGen\IGenerator.cs" />
     <Compile Include="AsmGen\LabelLocalizer.cs" />
     <Compile Include="AsmGen\StringGather.cs" />
+    <Compile Include="Tests\GenTest.cs" />
+    <Compile Include="Tests\ProgressMessage.cs" />
+    <Compile Include="Tests\WpfGui\GenTestRunner.xaml.cs">
+      <DependentUpon>GenTestRunner.xaml</DependentUpon>
+    </Compile>
     <Compile Include="WpfGui\EditAppSettings.xaml.cs">
       <DependentUpon>EditAppSettings.xaml</DependentUpon>
     </Compile>
@@ -176,6 +181,10 @@
     <Resource Include="Res\Logo.png" />
   </ItemGroup>
   <ItemGroup>
+    <Page Include="Tests\WpfGui\GenTestRunner.xaml">
+      <SubType>Designer</SubType>
+      <Generator>MSBuild:Compile</Generator>
+    </Page>
     <Page Include="WpfGui\EditAppSettings.xaml">
       <SubType>Designer</SubType>
       <Generator>MSBuild:Compile</Generator>

SourceGenWPF/Tests/GenTest.cs

@@ -0,0 +1,620 @@
+/*
+ * Copyright 2019 faddenSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.IO;
+using System.Text.RegularExpressions;
+using System.Windows.Media;
+
+using CommonUtil;
+using SourceGenWPF.AsmGen;
+
+namespace SourceGenWPF.Tests {
+    /// <summary>
+    /// Source code generation regression test.
+    ///
+    /// The generator is tested in two ways: (1) by comparing the output to known-good
+    /// sources, and (2) by running it through the assembler.  Assembling the sources is
+    /// important to ensure that we don't get bad sources in the "known-good" set.
+    ///
+    /// This does not take assembler version into account, so it will not be helpful for
+    /// monitoring compatibility with old versions of assemblers.
+    /// </summary>
+    public class GenTest {
+        private const string TEST_DIR_NAME = "SGTestData";
+        private const string EXPECTED_DIR_NAME = "Expected";
+
+        //private static char[] sInvalidChars = new char[] { '.', '_' };
+        private const string TestCasePattern = @"^\d\d\d\d-[A-Za-z0-9-]+$";
+        private static Regex sTestCaseRegex = new Regex(TestCasePattern);
+
+        /// <summary>
+        /// Whitelist of removable names for ScrubWorkDirectory().
+        /// </summary>
+        private static string[] sScrubList = new string[] {
+            "_FileInformation.txt",             // created by Merlin 32
+            "error_output.txt",                 // created by Merlin 32 (only when errors found)
+        };
+
+        /// <summary>
+        /// Test result.  One of these will be created for every {test-case, assembler} pair.
+        /// </summary>
+        public class GenTestResults {
+            public string PathName { get; private set; }
+            public string FileName { get; private set; }
+            public AssemblerInfo.Id AsmId { get; private set; }
+
+            public FileLoadReport ProjectLoadReport { get; set; }
+
+            public bool GenerateOkay { get; set; }
+            public bool AssembleOkay { get; set; }
+            public AssemblerResults AsmResults { get; set; }    // may be null
+
+            public TaskTimer Timer { get; set; }                // may be null
+
+            public GenTestResults(string pathName, AssemblerInfo.Id asmId) {
+                PathName = pathName;
+                AsmId = asmId;
+
+                FileName = Path.GetFileName(pathName);
+            }
+
+            // Return a string for use in the UI combo box.
+            public override string ToString() {
+                return (GenerateOkay && AssembleOkay ? "OK" : "FAIL") + " - " +
+                    FileName + " - " + AsmId.ToString();
+            }
+        }
+
+        /// <summary>
+        /// If true, don't scrub directories.
+        /// </summary>
+        public bool RetainOutput { get; set; }
+
+        /// <summary>
+        /// Directory with test cases.
+        /// </summary>
+        private string mTestDir;
+
+        private BackgroundWorker mWorker;
+
+        private List<GenTestResults> mResults = new List<GenTestResults>();
+
+
+        /// <summary>
+        /// Runs generate/assemble test cases.  Main entry point.
+        /// </summary>
+        /// <param name="worker">Background worker object from dialog box.</param>
+        public List<GenTestResults> Run(BackgroundWorker worker) {
+            Debug.Assert(mWorker == null);  // don't re-use object
+
+            mWorker = worker;
+            string runtimeDir = RuntimeDataAccess.GetDirectory();
+            mTestDir = Path.Combine(Path.GetDirectoryName(runtimeDir), TEST_DIR_NAME);
+
+            if (!Directory.Exists(mTestDir)) {
+                ReportErrMsg("Regression test directory not found: " + mTestDir);
+                ReportFailure();
+                return null;
+            }
+
+            List<string> testCases = new List<string>();
+            foreach (string pathName in Directory.EnumerateFiles(mTestDir)) {
+                // Filter out everything that doesn't look like "1000-nifty-test".  We
+                // want to ignore .dis65 files and assembler output (which has the name
+                // of the assembler following an underscore).
+                string fileName = Path.GetFileName(pathName);
+                MatchCollection matches = sTestCaseRegex.Matches(fileName);
+                if (matches.Count == 0) {
+                    //ReportProgress("Ignoring " + fileName + "\r\n", Color.Gray);
+                    continue;
+                }
+
+                ReportProgress("Found " + fileName + "\r\n");
+                testCases.Add(pathName);
+            }
+
+            ReportProgress("Processing " + testCases.Count + " test cases...\r\n");
+            DateTime startWhen = DateTime.Now;
+
+            int successCount = 0;
+            foreach (string pathName in testCases) {
+                if (GenerateAndAssemble(pathName)) {
+                    successCount++;
+                }
+
+                if (worker.CancellationPending) {
+                    ReportProgress("\r\nCancelled.\r\n", Colors.Red);
+                    return mResults;
+                }
+            }
+
+            DateTime endWhen = DateTime.Now;
+
+            if (successCount == testCases.Count) {
+                ReportProgress(string.Format("All " + testCases.Count +
+                    " tests passed in {0:N3} sec\r\n",
+                    (endWhen - startWhen).TotalSeconds), Colors.Green);
+            } else {
+                ReportProgress(successCount + " of " + testCases.Count + " tests passed\r\n");
+            }
+
+            return mResults;
+        }
+
+        private void ReportProgress(string msg) {
+            mWorker.ReportProgress(0, new ProgressMessage(msg));
+        }
+
+        private void ReportProgress(string msg, Color color) {
+            mWorker.ReportProgress(0, new ProgressMessage(msg, color));
+        }
+
+        private void ReportErrMsg(string msg) {
+            ReportProgress(" [" + msg + "] ", Colors.Blue);
+        }
+
+        private void ReportSuccess() {
+            ReportProgress(" success\r\n", Colors.Green);
+        }
+
+        private void ReportFailure() {
+            ReportProgress(" failed\r\n", Colors.Red);
+        }
+
+        /// <summary>
+        /// Extracts the test's number from the pathname.
+        /// </summary>
+        /// <param name="pathName">Full or partial path to test file.</param>
+        /// <returns>Test number.</returns>
+        private int GetTestNum(string pathName) {
+            // Should always succeed if pathName matched on our regex.
+            string fileName = Path.GetFileName(pathName);
+            return int.Parse(fileName.Substring(0, 4));
+        }
+
+        /// <summary>
+        /// Generates source code for the specified test case, assembles it, and compares
+        /// the output of both steps to expected values.  The process is repeated for every
+        /// known assembler.
+        ///
+        /// If an assembler is known but not configured, the assembly step is skipped, and
+        /// does not count as a failure.
+        /// </summary>
+        /// <param name="pathName">Full path to test case.</param>
+        /// <returns>True if all assemblers worked as expected.</returns>
+        private bool GenerateAndAssemble(string pathName) {
+            ReportProgress(Path.GetFileName(pathName) + "...\r\n");
+
+            // Create DisasmProject object, either as a new project for a plain data file,
+            // or from a project file.
+            DisasmProject project = InstantiateProject(pathName,
+                out FileLoadReport projectLoadReport);
+            if (project == null) {
+                ReportFailure();
+                return false;
+            }
+
+            int testNum = GetTestNum(pathName);
+
+            // Create a temporary directory to work in.
+            string workDir = CreateWorkDirectory(pathName);
+            if (string.IsNullOrEmpty(workDir)) {
+                ReportFailure();
+                project.Cleanup();
+                return false;
+            }
+
+            AppSettings settings = CreateNormalizedSettings();
+            ApplyProjectSettings(settings, project);
+
+            // Iterate through all known assemblers.
+            bool didFail = false;
+            foreach (AssemblerInfo.Id asmId in
+                    (AssemblerInfo.Id[])Enum.GetValues(typeof(AssemblerInfo.Id))) {
+                if (asmId == AssemblerInfo.Id.Unknown) {
+                    continue;
+                }
+
+                string fileName = Path.GetFileName(pathName);
+                TaskTimer timer = new TaskTimer();
+                timer.StartTask("Full Test Duration");
+
+                // Create results object and add it to the list.  We'll add stuff to it for
+                // as far as we get.
+                GenTestResults results = new GenTestResults(pathName, asmId);
+                mResults.Add(results);
+                results.ProjectLoadReport = projectLoadReport;
+
+                // Generate source code.
+                ReportProgress("  " + asmId.ToString() + " generate...");
+                IGenerator gen = AssemblerInfo.GetGenerator(asmId);
+                if (gen == null) {
+                    ReportErrMsg("generator unavailable");
+                    ReportProgress("\r\n");
+                    //didFail = true;
+                    continue;
+                }
+                timer.StartTask("Generate Source");
+                gen.Configure(project, workDir, fileName,
+                    AssemblerVersionCache.GetVersion(asmId), settings);
+                List<string> genPathNames = gen.GenerateSource(mWorker);
+                timer.EndTask("Generate Source");
+                if (mWorker.CancellationPending) {
+                    // The generator will stop early if a cancellation is requested.  If we
+                    // don't break here, the compare function will report a failure, which
+                    // isn't too problematic but looks funny.
+                    break;
+                }
+
+                ReportProgress(" verify...");
+                timer.StartTask("Compare Source to Expected");
+                bool match = CompareGeneratedToExpected(pathName, genPathNames);
+                timer.EndTask("Compare Source to Expected");
+                if (match) {
+                    ReportSuccess();
+                    results.GenerateOkay = true;
+                } else {
+                    ReportFailure();
+                    didFail = true;
+
+                    // The fact that it doesn't match the expected sources doesn't mean it's
+                    // invalid.  Go ahead and try to build it.
+                    //continue;
+                }
+
+                // Assemble code.
+                ReportProgress("  " + asmId.ToString() + " assemble...");
+                IAssembler asm = AssemblerInfo.GetAssembler(asmId);
+                if (asm == null) {
+                    ReportErrMsg("assembler unavailable");
+                    ReportProgress("\r\n");
+                    continue;
+                }
+
+                timer.StartTask("Assemble Source");
+                asm.Configure(genPathNames, workDir);
+                AssemblerResults asmResults = asm.RunAssembler(mWorker);
+                timer.EndTask("Assemble Source");
+                if (asmResults == null) {
+                    ReportErrMsg("unable to run assembler");
+                    ReportFailure();
+                    didFail = true;
+                    continue;
+                }
+                if (asmResults.ExitCode != 0) {
+                    ReportErrMsg("assembler returned code=" + asmResults.ExitCode);
+                    ReportFailure();
+                    didFail = true;
+                    continue;
+                }
+
+                results.AsmResults = asmResults;
+
+                ReportProgress(" verify...");
+                timer.StartTask("Compare Binary to Expected");
+                FileInfo fi = new FileInfo(asmResults.OutputPathName);
+                if (fi.Length != project.FileData.Length) {
+                    ReportErrMsg("asm output mismatch: length is " + fi.Length + ", expected " +
+                        project.FileData.Length);
+                    ReportFailure();
+                    didFail = true;
+                    continue;
+                } else if (!FileUtil.CompareBinaryFile(project.FileData, asmResults.OutputPathName,
+                        out int badOffset, out byte badFileVal)) {
+                    ReportErrMsg("asm output mismatch: offset +" + badOffset.ToString("x6") +
+                        " has value $" + badFileVal.ToString("x2") + ", expected $" +
+                        project.FileData[badOffset].ToString("x2"));
+                    ReportFailure();
+                    didFail = true;
+                    continue;
+                }
+                timer.EndTask("Compare Binary to Expected");
+
+                // Victory!
+                results.AssembleOkay = true;
+                ReportSuccess();
+
+                timer.EndTask("Full Test Duration");
+                results.Timer = timer;
+
+                // We don't scrub the directory on success at this point.  We could, but we'd
+                // need to remove only those files associated with the currently assembler.
+                // Otherwise, a failure followed by a success would wipe out the unsuccessful
+                // temporaries.
+            }
+
+            // If something failed, leave the bits around for examination.  Otherwise, try to
+            // remove the directory and all its contents.
+            if (!didFail && !RetainOutput) {
+                ScrubWorkDirectory(workDir, testNum);
+                RemoveWorkDirectory(workDir);
+            }
+
+            project.Cleanup();
+            return !didFail;
+        }
+
+        /// <summary>
+        /// Gets a copy of the AppSettings with a standard set of formatting options (e.g. lower
+        /// case for everything).
+        /// </summary>
+        /// <returns>New app settings object.</returns>
+        private AppSettings CreateNormalizedSettings() {
+            AppSettings settings = AppSettings.Global.GetCopy();
+
+            // Override all asm formatting options.  We can ignore ShiftBeforeAdjust and the
+            // pseudo-op names because those are set by the generators.
+            settings.SetBool(AppSettings.FMT_UPPER_HEX_DIGITS, false);
+            settings.SetBool(AppSettings.FMT_UPPER_OP_MNEMONIC, false);
+            settings.SetBool(AppSettings.FMT_UPPER_PSEUDO_OP_MNEMONIC, false);
+            settings.SetBool(AppSettings.FMT_UPPER_OPERAND_A, true);
+            settings.SetBool(AppSettings.FMT_UPPER_OPERAND_S, true);
+            settings.SetBool(AppSettings.FMT_UPPER_OPERAND_XY, false);
+            settings.SetBool(AppSettings.FMT_ADD_SPACE_FULL_COMMENT, false);
+
+            // Don't show the assembler ident line.  You can make a case for this being
+            // mandatory, since the generated code is only guaranteed to work with the
+            // assembler for which it was targeted, but I expect we'll quickly get to a
+            // place where we don't have to work around assembler bugs, and this will just
+            // become a nuisance.
+            settings.SetBool(AppSettings.SRCGEN_ADD_IDENT_COMMENT, false);
+
+            // Don't break lines with long labels.  That way we can redefine "long"
+            // without breaking our tests.  (This is purely cosmetic.)
+            settings.SetBool(AppSettings.SRCGEN_LONG_LABEL_NEW_LINE, false);
+
+            // This could be on or off.  Off seems less distracting.
+            settings.SetBool(AppSettings.SRCGEN_SHOW_CYCLE_COUNTS, false);
+
+            // Disable label localization.  We want to be able to play with this a bit
+            // without disrupting all the other tests.  Use a test-only feature to enable
+            // it for the localization test.
+            settings.SetBool(AppSettings.SRCGEN_DISABLE_LABEL_LOCALIZATION, true);
+
+            IEnumerator<AssemblerInfo> iter = AssemblerInfo.GetInfoEnumerator();
+            while (iter.MoveNext()) {
+                AssemblerInfo.Id asmId = iter.Current.AssemblerId;
+                AssemblerConfig curConfig =
+                    AssemblerConfig.GetConfig(settings, asmId);
+                AssemblerConfig defConfig =
+                    AssemblerInfo.GetAssembler(asmId).GetDefaultConfig();
+
+                // Merge the two together.  We want the default assembler config for most
+                // things, but the executable path from the current config.
+                defConfig.ExecutablePath = curConfig.ExecutablePath;
+
+                // Write it into the test settings.
+                AssemblerConfig.SetConfig(settings, asmId, defConfig);
+            }
+
+            return settings;
+        }
+
+        /// <summary>
+        /// Applies app setting overrides that were specified in the project settings.
+        /// </summary>
+        private void ApplyProjectSettings(AppSettings settings, DisasmProject project) {
+            // We could probably make this a more general mechanism, but that would strain
+            // things a bit, since we need to know the settings name, bool/int/string, and
+            // desired value.  Easier to just have a set of named features.
+            const string ENABLE_LABEL_LOCALIZATION = "__ENABLE_LABEL_LOCALIZATION";
+            const string ENABLE_LABEL_NEWLINE = "__ENABLE_LABEL_NEWLINE";
+
+            if (project.ProjectProps.ProjectSyms.ContainsKey(ENABLE_LABEL_LOCALIZATION)) {
+                settings.SetBool(AppSettings.SRCGEN_DISABLE_LABEL_LOCALIZATION, false);
+            }
+            if (project.ProjectProps.ProjectSyms.ContainsKey(ENABLE_LABEL_NEWLINE)) {
+                settings.SetBool(AppSettings.SRCGEN_LONG_LABEL_NEW_LINE, true);
+            }
+        }
+
+        private DisasmProject InstantiateProject(string dataPathName,
+                out FileLoadReport projectLoadReport) {
+            DisasmProject project = new DisasmProject();
+            // always use AppDomain sandbox
+
+            projectLoadReport = null;
+
+            int testNum = GetTestNum(dataPathName);
+
+            if (testNum < 2000) {
+                // create new disasm project for data file
+                byte[] fileData;
+                try {
+                    fileData = LoadDataFile(dataPathName);
+                } catch (Exception ex) {
+                    ReportErrMsg(ex.Message);
+                    return null;
+                }
+
+                project.Initialize(fileData.Length);
+                project.PrepForNew(fileData, Path.GetFileName(dataPathName));
+                // no platform symbols to load
+            } else {
+                // deserialize project file, failing if we can't find it
+                string projectPathName = dataPathName + ProjectFile.FILENAME_EXT;
+                if (!ProjectFile.DeserializeFromFile(projectPathName,
+                        project, out projectLoadReport)) {
+                    ReportErrMsg(projectLoadReport.Format());
+                    return null;
+                }
+
+                byte[] fileData;
+                try {
+                    fileData = LoadDataFile(dataPathName);
+                } catch (Exception ex) {
+                    ReportErrMsg(ex.Message);
+                    return null;
+                }
+
+                project.SetFileData(fileData, Path.GetFileName(dataPathName));
+                project.ProjectPathName = projectPathName;
+                project.LoadExternalFiles();
+            }
+
+            TaskTimer genTimer = new TaskTimer();
+            DebugLog genLog = new DebugLog();
+            genLog.SetMinPriority(DebugLog.Priority.Silent);
+            project.Analyze(UndoableChange.ReanalysisScope.CodeAndData, genLog, genTimer);
+
+            return project;
+        }
+
+        /// <summary>
+        /// Loads the test case data file.
+        ///
+        /// Throws an exception on failure.
+        /// </summary>
+        /// <param name="pathName">Full path to test case data file.</param>
+        /// <returns>File contents.</returns>
+        private byte[] LoadDataFile(string pathName) {
+            byte[] fileData;
+
+            using (FileStream fs = File.Open(pathName, FileMode.Open, FileAccess.Read)) {
+                Debug.Assert(fs.Length <= DisasmProject.MAX_DATA_FILE_SIZE);
+                fileData = new byte[fs.Length];
+                int actual = fs.Read(fileData, 0, (int)fs.Length);
+                if (actual != fs.Length) {
+                    // Not expected -- should be able to read the entire file in one shot.
+                    throw new Exception(Res.Strings.OPEN_DATA_PARTIAL_READ);
+                }
+            }
+            return fileData;
+        }
+
+        /// <summary>
+        /// Creates a work directory for the specified test case.  The new directory will be
+        /// created in the same directory as the test, and named after it.
+        ///
+        /// If the directory already exists, the previous contents will be scrubbed.
+        ///
+        /// If the file already exists but isn't a directory, this will fail.
+        /// </summary>
+        /// <param name="pathName">Test case path name.</param>
+        /// <returns>Path of work directory, or null if creation failed.</returns>
+        private string CreateWorkDirectory(string pathName) {
+            string baseDir = Path.GetDirectoryName(pathName);
+            int testNum = GetTestNum(pathName);
+            string workDirName = "tmp" + testNum.ToString();
+            string workDirPath = Path.Combine(baseDir, workDirName);
+            if (Directory.Exists(workDirPath)) {
+                ScrubWorkDirectory(workDirPath, testNum);
+            } else if (File.Exists(workDirPath)) {
+                ReportErrMsg("file '" + workDirPath + "' exists, not directory");
+                return null;
+            } else {
+                try {
+                    Directory.CreateDirectory(workDirPath);
+                } catch (Exception ex) {
+                    ReportErrMsg(ex.Message);
+                    return null;
+                }
+            }
+            return workDirPath;
+        }
+
+        /// <summary>
+        /// Removes the contents of a temporary work directory.  Only files that we believe
+        /// to be products of the generator or assembler are removed.
+        /// </summary>
+        /// <param name="workDir"></param>
+        /// <param name="testNum"></param>
+        private void ScrubWorkDirectory(string workDir, int testNum) {
+            string checkString = testNum.ToString();
+            if (checkString.Length != 4) {
+                Debug.Assert(false);
+                return;
+            }
+
+            foreach (string pathName in Directory.EnumerateFiles(workDir)) {
+                bool doRemove = false;
+                string fileName = Path.GetFileName(pathName);
+                if (fileName.Contains(checkString)) {
+                    doRemove = true;
+                } else {
+                    foreach (string str in sScrubList) {
+                        if (fileName == str) {
+                            doRemove = true;
+                        }
+                    }
+                }
+
+                if (!doRemove) {
+                    ReportErrMsg("not removing '" + fileName + "'");
+                    continue;
+                } else {
+                    try {
+                        File.Delete(pathName);
+                        //Debug.WriteLine("removed " + pathName);
+                    } catch (Exception ex) {
+                        ReportErrMsg("unable to remove '" + fileName + "': " + ex.Message);
+                        // don't stop -- keep trying to remove things
+                    }
+                }
+            }
+        }
+
+        private void RemoveWorkDirectory(string workDir) {
+            try {
+                Directory.Delete(workDir);
+            } catch (Exception ex) {
+                ReportErrMsg("unable to remove work dir: " + ex.Message);
+            }
+        }
+
+        /// <summary>
+        /// Compares each file in genFileNames to the corresponding file in Expected.
+        /// </summary>
+        /// <param name="pathName">Full pathname of test case.</param>
+        /// <param name="genPathNames">List of file names from source generator.</param>
+        /// <returns></returns>
+        private bool CompareGeneratedToExpected(string pathName, List<string> genPathNames) {
+            string expectedDir = Path.Combine(Path.GetDirectoryName(pathName), EXPECTED_DIR_NAME);
+
+            foreach (string path in genPathNames) {
+                string fileName = Path.GetFileName(path);
+                string compareName = Path.Combine(expectedDir, fileName);
+
+                if (!File.Exists(compareName)) {
+                    // File was generated unexpectedly.
+                    ReportErrMsg("file '" + fileName + "' not found in " + EXPECTED_DIR_NAME);
+                    return false;
+                }
+
+                // Compare the file contents as lines of text.  The files may use different
+                // line terminators (e.g. LF vs. CRLF), so we can't use file length as a
+                // factor.
+                if (!FileUtil.CompareTextFiles(path, compareName, out int firstDiffLine,
+                        out string line1, out string line2)) {
+                    ReportErrMsg("file '" + fileName + "' differs on line " + firstDiffLine);
+                    Debug.WriteLine("File #1: " + line1);
+                    Debug.WriteLine("File #2: " + line2);
+                    return false;
+                }
+            }
+
+            // NOTE: to be thorough, we should check to see if a file exists in Expected
+            // that doesn't exist in the work directory.  This is slightly more awkward since
+            // Expected is a big pile of everything, but we should be able to do it by
+            // filtering filenames with the test number.
+
+            return true;
+        }
+    }
+}

SourceGenWPF/Tests/ProgressMessage.cs

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2019 faddenSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+using System.Windows.Media;
+
+namespace SourceGenWPF.Tests {
+    /// <summary>
+    /// Progress message, with colorful text.  This is generated by the worker thread and
+    /// passed to the UI thread.
+    /// </summary>
+    public class ProgressMessage {
+        public string Text { get; private set; }
+        public Color Color { get; private set; }
+        public bool HasColor { get { return Color.A != 0; } }
+
+        public ProgressMessage(string msg) : this(msg, Color.FromArgb(0, 0, 0, 0)) { }
+
+        public ProgressMessage(string msg, Color color) {
+            Text = msg;
+            Color = color;
+        }
+    }
+}

SourceGenWPF/Tests/WpfGui/GenTestRunner.xaml

@@ -0,0 +1,64 @@
+<!--
+Copyright 2019 faddenSoft
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<Window x:Class="SourceGenWPF.Tests.WpfGui.GenTestRunner"
+        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+        xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+        xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+        xmlns:system="clr-namespace:System;assembly=mscorlib"
+        xmlns:local="clr-namespace:SourceGenWPF.Tests.WpfGui"
+        mc:Ignorable="d"
+        Title="Source Generation Test"
+        Icon="/SourceGenWPF;component/Res/SourceGenIcon.ico"
+        Width="640" Height="480" MinWidth="640" MinHeight="480"
+        Closing="Window_Closing">
+
+    <Window.Resources>
+        <system:String x:Key="str_RunTest">Run Test</system:String>
+        <system:String x:Key="str_CancelTest">Cancel</system:String>
+    </Window.Resources>
+
+    <DockPanel Margin="8">
+        <StackPanel DockPanel.Dock="Top" Orientation="Horizontal">
+            <Button Name="runButton" Content="{Binding RunButtonLabel}" Width="120"
+                    FontWeight="Bold" IsEnabled="{Binding IsNotRunning}"
+                    Click="RunCancelButton_Click"/>
+            <CheckBox Content="Retain output" Margin="20,3,0,0"
+                      IsChecked="{Binding IsOutputRetained}"/>
+        </StackPanel>
+
+        <RichTextBox Name="progressRichTextBox" DockPanel.Dock="Top" Height="130" Margin="0,8,0,0"
+                     VerticalScrollBarVisibility="Auto" IsReadOnly="True">
+            <RichTextBox.Resources>
+                <!-- remove excess vertical space between paragraphs -->
+                <Style TargetType="{x:Type Paragraph}">
+                    <Setter Property="Margin" Value="0"/>
+                </Style>
+            </RichTextBox.Resources>
+        </RichTextBox>
+
+        <ComboBox DockPanel.Dock="Top" Name="outputSelectComboBox" Margin="0,20,0,0" Width="360"
+                  HorizontalAlignment="Left"
+                  SelectionChanged="OutputSelectComboBox_SelectedIndexChanged"/>
+
+        <Button DockPanel.Dock="Bottom" Content="Close" Width="70" IsCancel="True"
+                HorizontalAlignment="Right" Margin="0,8,0,0"/>
+
+        <TextBox DockPanel.Dock="Top" Name="outputTextBox" IsReadOnly="True" Margin="0,8,0,0"
+                 VerticalScrollBarVisibility="Auto"/>
+    </DockPanel>
+</Window>

SourceGenWPF/Tests/WpfGui/GenTestRunner.xaml.cs

@@ -0,0 +1,259 @@
+/*
+ * Copyright 2019 faddenSoft
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Documents;
+using System.Windows.Media;
+
+using CommonWPF;
+
+namespace SourceGenWPF.Tests.WpfGui {
+    /// <summary>
+    /// Source generation test runner.
+    /// </summary>
+    public partial class GenTestRunner : Window, INotifyPropertyChanged {
+        private List<GenTest.GenTestResults> mLastResults;
+
+        private BackgroundWorker mWorker;
+
+        private FlowDocument mFlowDoc = new FlowDocument();
+        private Color mDefaultColor;
+
+        public event PropertyChangedEventHandler PropertyChanged;
+        private void OnPropertyChanged([CallerMemberName] string propertyName = "") {
+            PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName));
+        }
+
+        /// <summary>
+        /// True when we're not running.  Used to enable the "run test" button.
+        /// </summary>
+        public bool IsNotRunning {
+            get { return mIsNotRunning; }
+            set {
+                mIsNotRunning = value;
+                OnPropertyChanged();
+            }
+        }
+        private bool mIsNotRunning;
+
+        public bool IsOutputRetained {
+            get { return mIsOutputRetained; }
+            set {
+                mIsOutputRetained = value;
+                OnPropertyChanged();
+            }
+        }
+        private bool mIsOutputRetained;
+
+        public string RunButtonLabel {
+            get { return mRunButtonLabel; }
+            set {
+                mRunButtonLabel = value;
+                OnPropertyChanged();
+            }
+        }
+        private string mRunButtonLabel;
+
+
+        public GenTestRunner(Window owner) {
+            InitializeComponent();
+            Owner = owner;
+            DataContext = this;
+
+            mDefaultColor = ((SolidColorBrush)progressRichTextBox.Foreground).Color;
+
+            // Create and configure the BackgroundWorker.
+            mWorker = new BackgroundWorker();
+            mWorker.WorkerReportsProgress = true;
+            mWorker.WorkerSupportsCancellation = true;
+            mWorker.DoWork += BackgroundWorker_DoWork;
+            mWorker.ProgressChanged += BackgroundWorker_ProgressChanged;
+            mWorker.RunWorkerCompleted += BackgroundWorker_RunWorkerCompleted;
+
+            IsNotRunning = true;
+            RunButtonLabel = (string)FindResource("str_RunTest");
+            progressRichTextBox.Document = mFlowDoc;
+        }
+
+        /// <summary>
+        /// Handles a click on the "run test" button, which becomes a "cancel test" button once
+        /// the test has started.
+        /// </summary>
+        private void RunCancelButton_Click(object sender, RoutedEventArgs e) {
+            if (mWorker.IsBusy) {
+                IsNotRunning = false;
+                mWorker.CancelAsync();
+            } else {
+                ResetDialog();
+                RunButtonLabel = (string)FindResource("str_CancelTest");
+                mWorker.RunWorkerAsync();
+            }
+        }
+
+        /// <summary>
+        /// Cancels the test if the user closes the window.
+        /// </summary>
+        private void Window_Closing(object sender, CancelEventArgs e) {
+            if (mWorker.IsBusy) {
+                mWorker.CancelAsync();
+            }
+        }
+
+        private void ResetDialog() {
+            outputSelectComboBox.Items.Clear();
+            mFlowDoc.Blocks.Clear();
+            outputTextBox.Clear();
+            mLastResults = null;
+        }
+
+        // NOTE: executes on work thread.  DO NOT do any UI work here.  Pass the test
+        // results through e.Result.
+        private void BackgroundWorker_DoWork(object sender, DoWorkEventArgs e) {
+            BackgroundWorker worker = sender as BackgroundWorker;
+
+            GenTest test = new GenTest();
+            test.RetainOutput = IsOutputRetained;   // should be okay to read from work thread
+            List<GenTest.GenTestResults> results = test.Run(worker);
+
+            if (worker.CancellationPending) {
+                e.Cancel = true;
+            } else {
+                e.Result = results;
+            }
+        }
+
+        // Callback that fires when a progress update is made.
+        private void BackgroundWorker_ProgressChanged(object sender, ProgressChangedEventArgs e) {
+            // We get progress from GenTest, and from the IAssembler/IGenerator classes.  This
+            // is necessary to make cancellation work right, and allows us to show the
+            // asm/gen progress messages if we want to.
+            if (e.UserState is ProgressMessage) {
+                ProgressMessage msg = e.UserState as ProgressMessage;
+                if (msg.HasColor) {
+                    progressRichTextBox.AppendText(msg.Text, msg.Color);
+                } else {
+                    // plain foreground text color
+                    progressRichTextBox.AppendText(msg.Text, mDefaultColor);
+                }
+                progressRichTextBox.ScrollToEnd();
+            } else {
+                // Most progress updates have an e.ProgressPercentage value and a blank string.
+                if (!string.IsNullOrEmpty((string)e.UserState)) {
+                    Debug.WriteLine("Sub-progress: " + e.UserState);
+                }
+            }
+        }
+
+        // Callback that fires when execution completes.
+        private void BackgroundWorker_RunWorkerCompleted(object sender,
+                RunWorkerCompletedEventArgs e) {
+            if (e.Cancelled) {
+                Debug.WriteLine("Test halted -- user cancellation");
+            } else if (e.Error != null) {
+                // test harness shouldn't be throwing errors like this
+                Debug.WriteLine("Test failed: " + e.Error.ToString());
+                progressRichTextBox.AppendText("\r\n");
+                progressRichTextBox.AppendText(e.Error.ToString(), mDefaultColor);
+                progressRichTextBox.ScrollToEnd();
+            } else {
+                Debug.WriteLine("Tests complete");
+                mLastResults = e.Result as List<GenTest.GenTestResults>;
+                if (mLastResults != null) {
+                    PopulateOutputSelect();
+                }
+            }
+
+            RunButtonLabel = (string)FindResource("str_RunTest");
+            IsNotRunning = true;
+        }
+
+        private void PopulateOutputSelect() {
+            outputSelectComboBox.Items.Clear();
+            if (mLastResults.Count == 0) {
+                return;
+            }
+
+            foreach (GenTest.GenTestResults results in mLastResults) {
+                outputSelectComboBox.Items.Add(results);
+            }
+
+            // Trigger update.
+            outputSelectComboBox.SelectedIndex = 0;
+        }
+
+        private void OutputSelectComboBox_SelectedIndexChanged(object sender,
+                SelectionChangedEventArgs e) {
+            int sel = outputSelectComboBox.SelectedIndex;
+            if (sel < 0) {
+                // selection has been cleared
+                outputTextBox.Text = string.Empty;
+                return;
+            }
+            if (mLastResults == null && mLastResults.Count <= sel) {
+                Debug.WriteLine("SelIndexChanged to " + sel + ", not available");
+                return;
+            }
+
+            GenTest.GenTestResults results = mLastResults[sel];
+
+            StringBuilder sb = new StringBuilder(512);
+            sb.AppendFormat("Path: {0}\r\n", results.PathName);
+            sb.AppendFormat("Assembler: {0}\r\n", results.AsmId);
+            if (results.ProjectLoadReport != null) {
+                sb.AppendFormat("Project load: {0}\r\n", results.ProjectLoadReport.Format());
+            }
+            if (results.GenerateOkay) {
+                sb.Append("Source gen: OK\r\n");
+            } else {
+                sb.Append("Source gen: FAIL\r\n");
+            }
+            if (results.AssembleOkay) {
+                sb.Append("Asm gen: OK\r\n");
+            } else {
+                sb.Append("Asm gen: FAIL\r\n");
+            }
+            if (results.AsmResults != null) {
+                AsmGen.AssemblerResults asmr = results.AsmResults;
+                sb.AppendFormat("Cmd line: {0}\r\n", asmr.CommandLine);
+                if (!results.AssembleOkay) {
+                    sb.AppendFormat("Exit code: {0}\r\n", asmr.ExitCode);
+                }
+                if (asmr.Stdout != null && asmr.Stdout.Length > 2) {
+                    sb.Append("----- stdout -----\r\n");
+                    sb.Append(asmr.Stdout);
+                    sb.Append("\r\n");
+                }
+                if (asmr.Stderr != null && asmr.Stderr.Length > 2) {
+                    sb.Append("----- stderr -----\r\n");
+                    sb.Append(asmr.Stderr);
+                    sb.Append("\r\n");
+                }
+            }
+            if (results.Timer != null) {
+                sb.Append("\r\n----- task times -----\r\n");
+                sb.Append(results.Timer.DumpToString(string.Empty));
+            }
+
+            outputTextBox.Text = sb.ToString();
+        }
+    }
+}

SourceGenWPF/WpfGui/MainWindow.xaml

@@ -54,6 +54,7 @@ limitations under the License.
                 </RoutedUICommand.InputGestures>
             </RoutedUICommand>
             <RoutedUICommand x:Key="CloseCmd" Text="Close"/>
+            <RoutedUICommand x:Key="DebugSourceGenerationTestsCmd" Text="Source Generation Tests..."/>
             <RoutedUICommand x:Key="EditAddressCmd" Text="Set Address..."/>
             <RoutedUICommand x:Key="EditStatusFlagsCmd" Text="Override Status Flags..."/>
             <RoutedUICommand x:Key="ExitCmd" Text="Exit"/>
@@ -105,6 +106,8 @@ limitations under the License.
                         CanExecute="IsProjectOpen" Executed="AssembleCmd_Executed"/>
         <CommandBinding Command="{StaticResource CloseCmd}"
                         CanExecute="IsProjectOpen" Executed="CloseCmd_Executed"/>
+        <CommandBinding Command="{StaticResource DebugSourceGenerationTestsCmd}"
+                        Executed="DebugSourceGenerationTests_Executed"/>
         <CommandBinding Command="{StaticResource EditAddressCmd}"
                         CanExecute="CanEditAddress" Executed="EditAddressCmd_Executed"/>
         <CommandBinding Command="{StaticResource EditStatusFlagsCmd}"
@@ -221,7 +224,7 @@ limitations under the License.
                 <MenuItem Header="Toggle Comment Rulers"/>
                 <MenuItem Header="Use Keep-Alive Hack"/>
                 <Separator/>
-                <MenuItem Header="Source Generation Tests..."/>
+                <MenuItem Command="{StaticResource DebugSourceGenerationTestsCmd}"/>
             </MenuItem>
         </Menu>
 

SourceGenWPF/WpfGui/MainWindow.xaml.cs

@@ -783,6 +783,10 @@ namespace SourceGenWPF.WpfGui {
             }
         }
 
+        private void DebugSourceGenerationTests_Executed(object sender, ExecutedRoutedEventArgs e) {
+            mMainCtrl.RunSourceGenerationTests();
+        }
+
         private void EditAddressCmd_Executed(object sender, ExecutedRoutedEventArgs e) {
             mMainCtrl.EditAddress();
         }