Added syntax highlighting to Applesoft Viewer, reworked retokenizer

README.md

@@ -7,7 +7,7 @@ This code is being developed using Qt5, with development work being done on Linu
 
 Among other things, the app includes:
 * Code that reads and parses DOS-format disk images at the disk-structure level. (It's just read-only for now, and ProDOS is on the horizon)
-* An Applesoft parser/retokenizer/reformatter which can show code in its normal form or reindented on multiple lines, and can optionally show integers as hex values inline. The retokenizer creates new tokens for strings, variable names, numeric values, DATA payloads, REM remarks, and other things that are nice to have -- and sets the stage to create an analyzer for deeper processing of the code later on.
+* An Applesoft parser/retokenizer/reformatter which can show code in its normal form or reindented on multiple lines with syntax highlighting, and can optionally show integers as hex values inline. The retokenizer creates new tokens for strings, variable names, numeric values, DATA payloads, REM remarks, and other things that are nice to have -- and sets the stage to create an analyzer for deeper processing of the code later on.
 * A binary file processor with a flow-tracing disassembler, which can show hex dumps and disassembly of code and data blocks (it also temporarily has some common labels for well-known addresses hard-coded into the display, though dynamic labeling of symbols is in the works.) It can also display the relocation table for type-R relocatable files.
 * A Text file viewer.
 * A graphics viewer which can show hires screen dumps as monochrome, with NTSC artifacts, or a hybrid mode where each pixel's natural color is shown in-place (with no white color blending.)
@@ -20,7 +20,7 @@ Among other things, the app includes:
 There is also a Hex/Decimal converter utility and some in-app documentation for HRCG control character codes. Mainly stuff that I found useful at the time I was writing it.
 
 In the pipeline are some more useful features (as I get a chance):
-* Syntax Highlighting
+* Syntax Highlighting **(Currently in progress!)**
 * More low-level disk exploration tools (sector-level stuff, comparing DOS versions between disks, etc.)
 * More analysis of AppleSoft programs (Tracing flow control, find addresses of PEEKS, POKES, CALLS, USRs, and &'s)
 * Processing and analysis of Integer Basic files

src/applesoftfile/ApplesoftRetokenizer.cpp

@@ -6,17 +6,25 @@
 
 ApplesoftRetokenizer::ApplesoftRetokenizer()
 {
-
+    m_isParsed = false;
 }
 
 void ApplesoftRetokenizer::setData(QByteArray data)
 {
     m_data = data;
     m_data_end = data.length();
+    m_isParsed = false;
 }
 
 void ApplesoftRetokenizer::parse(quint16 start_address)
 {
+    if (m_isParsed)
+    {
+        qWarning("File is already parsed.  Not reparsing.");
+        return;
+    }
+
+    //TODO:  This could be changed to search for hidden space between applesoft lines
     int idx = 0;
     quint8 val = 0;
     m_retokenized_lines.clear();
@@ -46,11 +54,123 @@ void ApplesoftRetokenizer::parse(quint16 start_address)
         m_retokenized_lines.append(line);
     }
 
-     m_data_end = idx;
+    m_data_end = idx;
 
     if (idx < m_data.length()) {
         qDebug() << QString("%1 byte(s) unaccounted for.").arg(m_data.length() - idx);
     }
+
+    retokenizeLinesForFormatting();
+
+    m_isParsed = true;
+}
+
+void ApplesoftRetokenizer::retokenizeLinesForFormatting()
+{
+    QVector<ApplesoftLine> retLines;
+
+    foreach(ApplesoftLine line, m_retokenized_lines)
+    {
+        int indentlevel = 1;
+     //   quint16 linenum = line.linenum;
+
+        bool firstToken = true;
+        ApplesoftToken previousToken;
+        QMutableVectorIterator<ApplesoftToken> tokenIt(line.tokens);
+        while (tokenIt.hasNext())
+        {
+            ApplesoftToken token = tokenIt.next();
+            bool isFlowTarget = false;
+
+            QString tokenstr = token.getRawPrintableString();
+            if (firstToken)
+            {
+                if (!tokenstr.startsWith(" "))
+                {
+                    ApplesoftToken tmptoken(ApplesoftToken::OptFmtLeadingSpaceTokenValue);
+                    tokenIt.remove();
+                    tokenIt.insert(tmptoken);
+                    tokenIt.insert(token);
+                }
+                firstToken = false;
+            }
+
+            quint16 preTokenId = previousToken.getTokenId();
+            if (preTokenId == ApplesoftToken::ASGoto ||
+                preTokenId == ApplesoftToken::ASGosub ||
+                preTokenId == ApplesoftToken::ASThen)
+            {
+                isFlowTarget = false;
+                if (preTokenId == ApplesoftToken::ASGoto || preTokenId == ApplesoftToken::ASGosub)
+                {
+                    isFlowTarget = true;
+                }
+                else if (preTokenId == ApplesoftToken::ASThen
+                         && token.getTokenId() == ApplesoftToken::IntegerTokenVal)
+                {
+                    isFlowTarget = true;
+                }
+                if (isFlowTarget)
+                {
+                    QPair<quint16,quint16> pair;
+                    pair.first = line.linenum;
+                    pair.second = token.getWordValue();
+                    m_flowTargets.append(pair);
+
+                    ApplesoftToken tmptoken(ApplesoftToken::OptFmtFlagFlowTargetNextTokenValue);
+                    tokenIt.remove();
+                    tokenIt.insert(tmptoken);
+                    tokenIt.insert(token);
+                }
+            }
+
+            if (token.getTokenId() == ApplesoftToken::ASReturn)
+            {
+                ApplesoftToken tmptoken(ApplesoftToken::OptFmtReturnLineBreakTokenValue);
+                tokenIt.insert(tmptoken);
+            }
+
+            if (token.getTokenId() == ':')
+            {
+                ApplesoftToken tmptoken(ApplesoftToken::OptFmtIndentLineBreakTokenValue);
+                tokenIt.insert(tmptoken);
+                for (int ind = 0; ind < indentlevel; ind++)
+                {
+                    ApplesoftToken tmptoken(ApplesoftToken::OptFmtIndentTabTokenValue);
+                    tokenIt.insert(tmptoken);
+                }
+                if (!tokenIt.peekNext().getRawPrintableString().startsWith(" "))
+                {
+                    ApplesoftToken tmptoken(ApplesoftToken::OptFmtIndentSpaceTokenValue);
+                    tokenIt.insert(tmptoken);
+                }
+            }
+            if (token.getTokenId() == ApplesoftToken::ASThen)
+            {
+                indentlevel++;
+                if (tokenIt.peekNext().getTokenId() != ApplesoftToken::IntegerTokenVal)
+                {
+                    ApplesoftToken tmptoken(ApplesoftToken::OptFmtIndentLineBreakTokenValue);
+                    tokenIt.insert(tmptoken);
+                    for (int ind = 0; ind < indentlevel; ind++)
+                    {
+                        ApplesoftToken tmptoken(ApplesoftToken::OptFmtIndentTabTokenValue);
+                        tokenIt.insert(tmptoken);
+                    }
+                    if (!tokenIt.peekNext().getRawPrintableString().startsWith(" "))
+                    {
+                        ApplesoftToken tmptoken(ApplesoftToken::OptFmtIndentSpaceTokenValue);
+                        tokenIt.insert(tmptoken);
+                    }
+                }
+            }
+
+            previousToken = token;
+        }
+        retLines.append(line);
+    }
+
+    m_retokenized_lines = retLines;
 }
 
 void ApplesoftRetokenizer::retokenizeLine(ApplesoftLine &line)
@@ -115,35 +235,35 @@ QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeStrings(QVector<Applesof
         if (token.getTokenId() >= 0x80)
         {
             replacements.append(token);
-         //   continue;
+            //   continue;
         } else
 
-        if (token.getWordValue() == '"')
+            if (token.getWordValue() == '"')
-        {
-            if (!inString)
             {
-                inString = true;
+                if (!inString)
-                buffer.append(token.getWordValue());
+                {
-              //  continue;
+                    inString = true;
-            }
+                    buffer.append(token.getWordValue());
-            else
+                    //  continue;
-            {
+                }
-                buffer.append(token.getWordValue());
+                else
-                ApplesoftToken strtoken(ApplesoftToken::StringTokenVal, buffer);
+                {
-                replacements.append(strtoken);
+                    buffer.append(token.getWordValue());
-                buffer.clear();
+                    ApplesoftToken strtoken(ApplesoftToken::StringTokenVal, buffer);
-                inString = false;
+                    replacements.append(strtoken);
-//                continue;
+                    buffer.clear();
-            }
+                    inString = false;
-        } else
+                    //                continue;
+                }
+            } else
 
-        if (inString)
+                if (inString)
-        {
+                {
-            buffer.append(token.getWordValue());
+                    buffer.append(token.getWordValue());
-            // continue;
+                    // continue;
-        } else
+                } else
 
-        replacements.append(token);
+                    replacements.append(token);
     }
 
     return replacements;
@@ -176,7 +296,7 @@ QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeDataStatements(QVector<A
     }
     if (inData) {
         QVector<ApplesoftToken> dataTokens;
-        dataTokens = processDataPayload(datatokenbuffer);
+        dataTokens = retokenizeDataPayload(datatokenbuffer);
         replacements.append(dataTokens);
         datatokenbuffer.clear();
         inData = false;
@@ -185,7 +305,7 @@ QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeDataStatements(QVector<A
 }
 
 
-QVector<ApplesoftToken> ApplesoftRetokenizer::processDataPayload(QVector<ApplesoftToken>& datatokens)
+QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeDataPayload(QVector<ApplesoftToken>& datatokens)
 {
     QVector<ApplesoftToken> retval;
 
@@ -249,13 +369,13 @@ QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeVariables(QVector<Apples
     }
     QList<QRegularExpressionMatch> matchstack;
     QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
- //   qDebug() << parsestring;
+    //   qDebug() << parsestring;
     while (matches.hasNext()) {
         QRegularExpressionMatch rematch = matches.next();
         matchstack.push_front(rematch);
 
-//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
+        //        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
-//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
+        //                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
     }
 
     foreach(QRegularExpressionMatch rematch, matchstack)
@@ -320,13 +440,13 @@ QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeNumbers(QVector<Applesof
     }
     QList<QRegularExpressionMatch> matchstack;
     QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
-//    qDebug() << parsestring;
+    //    qDebug() << parsestring;
     while (matches.hasNext()) {
         QRegularExpressionMatch rematch = matches.next();
         matchstack.push_front(rematch);
 
-//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
+        //        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
-//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
+        //                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
     }
 
     foreach(QRegularExpressionMatch rematch, matchstack)
@@ -395,28 +515,28 @@ QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeNegativeNumbers(QVector<
         else if (token.getTokenId() == ApplesoftToken::IntVarTokenVal) lastWasInt = true;
         else if (token.getTokenId() == ApplesoftToken::FloatVarTokenVal) lastWasInt = true;
         else if (token.getTokenId() == ')') lastWasInt = true;
-else
+        else
-        if (token.getTokenId() == ApplesoftToken::ASMINUS)
+            if (token.getTokenId() == ApplesoftToken::ASMINUS)
-        {
-            if (!lastWasInt && it.hasNext() && it.peekNext().getTokenId() == ApplesoftToken::IntegerTokenVal)
             {
-                it.remove();
+                if (!lastWasInt && it.hasNext() && it.peekNext().getTokenId() == ApplesoftToken::IntegerTokenVal)
-                token = it.next();
+                {
-                it.remove();
+                    it.remove();
-                int val = token.getUnsignedIntegerValue() * -1;
+                    token = it.next();
-                token.setValue(val);
+                    it.remove();
-                it.insert(token);
+                    int val = token.getUnsignedIntegerValue() * -1;
-                lastWasInt = true;
+                    token.setValue(val);
+                    it.insert(token);
+                    lastWasInt = true;
+                }
+                else
+                {
+                    lastWasInt = false;
+                }
             }
             else
             {
                 lastWasInt = false;
             }
-        }
-        else
-        {
-            lastWasInt = false;
-        }
     }
 
     return tmptokens.toVector();

src/applesoftfile/ApplesoftRetokenizer.h

@@ -5,6 +5,8 @@
 
 #include <QByteArray>
 #include <QVector>
+#include <QList>
+#include <QPair>
 
 class ApplesoftRetokenizer
 {
@@ -18,8 +20,12 @@ public:
 
     QVector<ApplesoftLine> getRetokenizedLines() { return m_retokenized_lines; }
 
+    QList<QPair<quint16, quint16> >  getFlowTargets() const { return m_flowTargets; }
 
 private:
+    void retokenizeLinesForFormatting();
+
+
     void retokenizeLine(ApplesoftLine &line);
     QVector<ApplesoftToken> retokenizeRems(QVector<ApplesoftToken> &datatokens);
     QVector<ApplesoftToken> retokenizeStrings(QVector<ApplesoftToken> &datatokens);
@@ -29,10 +35,16 @@ private:
     QVector<ApplesoftToken> retokenizeNumbers(QVector<ApplesoftToken> &datatokens);
     QVector<ApplesoftToken> retokenizeNegativeNumbers(QVector<ApplesoftToken> &datatokens);
 
+
     QVector<ApplesoftLine> m_retokenized_lines;
     QByteArray m_data;
 
     quint16 m_data_end;
+
+    bool m_isParsed;
+
+    QList<QPair<quint16, quint16> > m_flowTargets;
+
 };
 
 

src/applesoftfile/applesoftfile.cxx

@@ -35,6 +35,8 @@ void ApplesoftFile::setData(QByteArray data)
     m_retokenizer->parse();
     m_data_end = m_retokenizer->getEndOfDataOffset();
     m_lines = m_retokenizer->getRetokenizedLines();
+
+    m_flowTargets = m_retokenizer->getFlowTargets();
 }
 
 QByteArray ApplesoftFile::rawData() {

src/applesoftfile/applesoftfile.h

@@ -33,6 +33,9 @@ private:
     quint16 m_length;
 
     ApplesoftRetokenizer *m_retokenizer;
+
+    QList<QPair<quint16, quint16> > m_flowTargets;
+
 };
 
 

src/applesoftfile/applesoftformatter.cxx

@@ -9,7 +9,6 @@ ApplesoftFormatter::ApplesoftFormatter(QObject *parent) :
     QObject(parent)
 {
     m_file = Q_NULLPTR;
-    m_formattedDocument = QSharedPointer<QTextDocument>(new QTextDocument());
 }
 
 void ApplesoftFormatter::setFile(ApplesoftFile *file)
@@ -21,13 +20,11 @@ void ApplesoftFormatter::setFile(ApplesoftFile *file)
 void ApplesoftFormatter::formatText()
 {
     m_formattedText.clear();
-    m_formattedDocument->clear();
 
     if (!m_file) {
         return;
     }
 
-    QTextCursor cursor(m_formattedDocument.data());
 
     QString formattedText;
     m_flowTargets.clear();
@@ -37,7 +34,6 @@ void ApplesoftFormatter::formatText()
 
         int indentlevel = 1;
         formattedText.append(linestring);
-        cursor.insertText(linestring, ApplesoftToken::defaultTextFormat());
 
 
         QVectorIterator<ApplesoftToken> tokenIt(line.tokens);
@@ -58,7 +54,6 @@ void ApplesoftFormatter::formatText()
                 }
                 firstToken = false;
             }
-            cursor.insertText(tokenstr, token.textFormat());
 
 
             //TODO: Move this to the parser.
@@ -69,12 +64,12 @@ void ApplesoftFormatter::formatText()
             {
                 isFlowTarget = false;
                 if (previousToken.getTokenId() == ApplesoftToken::ASGoto ||
-                    previousToken.getTokenId() == ApplesoftToken::ASGosub)
+                        previousToken.getTokenId() == ApplesoftToken::ASGosub)
                 {
                     isFlowTarget = true;
                 }
                 else if (previousToken.getTokenId() == ApplesoftToken::ASThen &&
-                        token.getTokenId() == ApplesoftToken::IntegerTokenVal)
+                         token.getTokenId() == ApplesoftToken::IntegerTokenVal)
                 {
                     isFlowTarget = true;
                 }
@@ -177,14 +172,244 @@ void ApplesoftFormatter::formatText()
 
             formattedText.append(tokenstr);
             previousToken = token;
-        }
+        } // While tokenIt.hasNext();
 
         formattedText.append("\n");
         if (m_format_options.testFlag(ReindentCode))
         {
-  //          retval.append("\n");
+            //          retval.append("\n");
         }
 
-    }
+    } //foreach line
     m_formattedText = formattedText;
 }
+
+
+void ApplesoftFormatter::newFormatText()
+{
+    m_formattedText.clear();
+
+    QString formattedText;
+
+    foreach (ApplesoftLine line, m_file->getLines())
+    {
+        QString linestring = QString("%1 ").arg(line.linenum,5,10,QChar(' '));
+        formattedText.append(linestring);
+
+        QVectorIterator<ApplesoftToken>tokenIt(line.tokens);
+        while (tokenIt.hasNext())
+        {
+            ApplesoftToken token = tokenIt.next();
+            bool isBranchTarget = false;
+            if (token.isOptFmtToken())
+            {
+                switch (token.getTokenId())
+                {
+
+                case ApplesoftToken::OptFmtFlagFlowTargetNextTokenValue:
+                {
+                    if (m_format_options.testFlag(ShowIntsAsHex))
+                        isBranchTarget = true;
+                    break;
+                }
+
+                case ApplesoftToken::OptFmtIndentLineBreakTokenValue:
+                {
+                    if (m_format_options.testFlag(ReindentCode))
+                        formattedText.append("\n");
+                    break;
+                }
+                case ApplesoftToken::OptFmtIndentSpaceTokenValue:
+                {
+                    if (m_format_options.testFlag(ReindentCode))
+                        formattedText.append(" ");
+                    break;
+                }
+                case ApplesoftToken::OptFmtIndentTabTokenValue:
+                {
+                    if (m_format_options.testFlag(ReindentCode))
+                        formattedText.append("      ");
+                    break;
+                }
+                case ApplesoftToken::OptFmtLeadingSpaceTokenValue:
+                {
+                    formattedText.append(" ");
+                    break;
+                }
+                case ApplesoftToken::OptFmtReturnLineBreakTokenValue:
+                {
+                    if (m_format_options.testFlag(BreakAfterReturn))
+                        formattedText.append("\n");
+                    break;
+                }
+                default:
+                {
+                    break;
+                }
+                }
+            }  // isOptFmt
+            else
+            {
+                QString tokenstr = token.getRawPrintableString();
+
+                if (token.getTokenId() == ApplesoftToken::IntegerTokenVal)
+                {
+                    if (m_format_options.testFlag(ShowIntsAsHex) && !isBranchTarget)
+                    {
+                        quint32 ui32val = token.getUnsignedIntegerValue();
+                        qint32  i32val = token.getIntegerValue();
+                        if ((i32val < 128 && i32val >= -128) || ui32val < 256)
+                        {
+                            quint8 ui8 = ui32val;
+                            tokenstr = HEXPREFIX+uint8ToHex(ui8);
+                        }
+                        else if ((i32val < 32768 && i32val >= -32768) || ui32val < 65536)
+                        {
+                            quint16 ui16 = ui32val;
+                            tokenstr = HEXPREFIX+uint16ToHex(ui16);
+                        }
+                        else
+                        {
+                            tokenstr = HEXPREFIX+uint32ToHex(ui32val);
+                        }
+                    } // isShowIntsAsHex
+                }
+
+                if (m_format_options.testFlag(ShowCtrlChars))
+                {
+                    tokenstr.replace(QChar(0x7f),QChar(0x2401));
+
+                    for (int idx = 1; idx <= 0x1f; idx++) {
+                        if (idx == '\n') continue;
+                        tokenstr.replace(QChar(idx),QChar(idx+0x2400));
+                        //              tokenstr.replace(QChar(idx), QString("<%1>").arg(uint8ToHex(idx)));
+                    }
+                }  // if ShowCtrlChars
+
+                formattedText.append(tokenstr);
+
+            }
+
+        } // while tokenIt.hasNext()
+
+        formattedText.append("\n");
+
+    } // foreach line
+    m_formattedText = formattedText;
+}
+
+void ApplesoftFormatter::formatDocument(QTextDocument *doc)
+{
+    if (!doc) return;
+
+    doc->clear();
+    QTextCursor cursor(doc);
+
+    foreach (ApplesoftLine line, m_file->getLines())
+    {
+        QString linestring = QString("%1 ").arg(line.linenum,5,10,QChar(' '));
+        cursor.insertText(linestring,ApplesoftToken::textFormat(ApplesoftToken::LineNumberTokenVal));
+
+        QVectorIterator<ApplesoftToken>tokenIt(line.tokens);
+        while (tokenIt.hasNext())
+        {
+            ApplesoftToken token = tokenIt.next();
+            bool isBranchTarget = false;
+            if (token.isOptFmtToken())
+            {
+                switch (token.getTokenId())
+                {
+
+                case ApplesoftToken::OptFmtFlagFlowTargetNextTokenValue:
+                {
+                    if (m_format_options.testFlag(ShowIntsAsHex))
+                        isBranchTarget = true;
+                    break;
+                }
+
+                case ApplesoftToken::OptFmtIndentLineBreakTokenValue:
+                {
+                    if (m_format_options.testFlag(ReindentCode))
+                        cursor.insertBlock();
+                    break;
+                }
+                case ApplesoftToken::OptFmtIndentSpaceTokenValue:
+                {
+                    if (m_format_options.testFlag(ReindentCode))
+                        cursor.insertText(" ",ApplesoftToken::defaultTextFormat());
+                    break;
+                }
+                case ApplesoftToken::OptFmtIndentTabTokenValue:
+                {
+                    if (m_format_options.testFlag(ReindentCode))
+                        cursor.insertText("      ",ApplesoftToken::defaultTextFormat());
+                    break;
+                }
+                case ApplesoftToken::OptFmtLeadingSpaceTokenValue:
+                {
+                    cursor.insertText(" ",ApplesoftToken::defaultTextFormat());
+                    break;
+                }
+                case ApplesoftToken::OptFmtReturnLineBreakTokenValue:
+                {
+                    if (m_format_options.testFlag(BreakAfterReturn))
+                        cursor.insertBlock();
+                    break;
+                }
+                default:
+                {
+                    break;
+                }
+                }
+            }  // isOptFmt
+            else
+            {
+                QString tokenstr = token.getRawPrintableString();
+                QTextCharFormat fmt = token.textFormat();
+
+                if (token.getTokenId() == ApplesoftToken::IntegerTokenVal)
+                {
+                    if (m_format_options.testFlag(ShowIntsAsHex) && !isBranchTarget)
+                    {
+                        quint32 ui32val = token.getUnsignedIntegerValue();
+                        qint32  i32val = token.getIntegerValue();
+                        if ((i32val < 128 && i32val >= -128) || ui32val < 256)
+                        {
+                            quint8 ui8 = ui32val;
+                            tokenstr = HEXPREFIX+uint8ToHex(ui8);
+                        }
+                        else if ((i32val < 32768 && i32val >= -32768) || ui32val < 65536)
+                        {
+                            quint16 ui16 = ui32val;
+                            tokenstr = HEXPREFIX+uint16ToHex(ui16);
+                        }
+                        else
+                        {
+                            tokenstr = HEXPREFIX+uint32ToHex(ui32val);
+                        }
+                    } // isShowIntsAsHex
+                }
+
+                if (m_format_options.testFlag(ShowCtrlChars))
+                {
+                    tokenstr.replace(QChar(0x7f),QChar(0x2401));
+
+                    for (int idx = 1; idx <= 0x1f; idx++) {
+                        if (idx == '\n') continue;
+                        tokenstr.replace(QChar(idx),QChar(idx+0x2400));
+                        //              tokenstr.replace(QChar(idx), QString("<%1>").arg(uint8ToHex(idx)));
+                    }
+                }  // if ShowCtrlChars
+
+                //formattedText.append(tokenstr);
+                cursor.insertText(tokenstr,fmt);
+
+            }
+
+        } // while tokenIt.hasNext()
+
+        //formattedText.append("\n");
+        cursor.insertBlock();
+
+    } // foreach line
+}

src/applesoftfile/applesoftformatter.h

@@ -35,11 +35,12 @@ public:
     FormatOptions flags() { return m_format_options; }
 
     void formatText();
-    QSharedPointer<QTextDocument> getFormattedDocument() const { return m_formattedDocument; }
     QString getFormattedText() const { return m_formattedText; }
 
     QList<QPair<quint16, quint16> > flowTargets() const { return m_flowTargets; }
 
+    void newFormatText();
+    void formatDocument(QTextDocument *doc);
 signals:
     void newFile(ApplesoftFile *file);
 
@@ -52,7 +53,6 @@ private:
     ApplesoftFile *m_file;
 
     QString m_formattedText;
-    QSharedPointer<QTextDocument> m_formattedDocument;
 
 };
 

src/applesoftfile/applesofttoken.cxx

@@ -80,6 +80,9 @@ void ApplesoftToken::setTokenId(quint16 id)
     } else if (id == StringAryVarTokenVal) {
         m_token_type = STRING_ARY_VARIABLE_TOKEN;
         m_command_type = NONE;
+    } else if (id >= 0xe000 && id < 0xf000) {
+        m_token_type = OPTIONAL_FORMAT_TOKEN;
+        m_command_type = OPTIONAL_FORMAT;
     }
 }
 
@@ -122,6 +125,8 @@ QString ApplesoftToken::getRawPrintableString() const
         return getStringValue();
     } else if (m_token_id == StringAryVarTokenVal) {
         return getStringValue();
+    } else if (m_token_id >= 0xe000 && m_token_id < 0xf000) {
+        return "";
     } else {
         return "[temp undefined]";
     }
@@ -130,16 +135,55 @@ QString ApplesoftToken::getRawPrintableString() const
 QTextCharFormat ApplesoftToken::defaultTextFormat()
 {
     QTextCharFormat tf;  // Default
-    tf.setFontFamily("Typewriter");
+//    tf.setFont(QFont("Courier"));
-    tf.setFontPointSize(10);
+//    tf.setFontPointSize(10);
-    tf.setForeground(Qt::red);
+    tf.setForeground(Qt::black);
     return tf;
 }
 
 QTextCharFormat ApplesoftToken::textFormat(quint16 tokenType)
 {
-    makeTextCharFormats();
+    QTextCharFormat tf = defaultTextFormat();
-    return m_textcharformats[TCFDefault];
+
+    if (tokenType < 0x80) // Ascii
+    {
+        tf.setForeground(Qt::black);
+    }
+    else if (tokenType < 0x100) // Applesoft Tokens
+    {
+        tf.setForeground(Qt::black);
+    }
+    else if (tokenType == StringTokenVal)
+    {
+        tf.setForeground(Qt::blue);
+        tf.setFontWeight(QFont::Bold);
+    }
+    else if (tokenType == IntegerTokenVal || tokenType == FloatTokenVal)
+    {
+        tf.setForeground(Qt::darkGreen);
+        tf.setFontWeight(QFont::Bold);
+    }
+    else if (tokenType == StringVarTokenVal ||
+             tokenType == StringAryVarTokenVal ||
+             tokenType == IntVarTokenVal ||
+             tokenType == IntAryVarTokenVal ||
+             tokenType == FloatVarTokenVal ||
+             tokenType == FloatAryVarTokenVal)
+    {
+        tf.setFontWeight(QFont::Bold);
+        tf.setForeground(Qt::darkMagenta);
+    }
+    else if (tokenType == RemStringTokenVal)
+    {
+        tf.setForeground(Qt::darkGray);
+        tf.setFontUnderline(true);
+    }
+    else if (tokenType == DataStringTokenVal)
+    {
+        tf.setForeground(Qt::darkRed);
+    }
+
+    return tf;
 }
 
 
@@ -159,9 +203,6 @@ void ApplesoftToken::makeTextCharFormats()
 //    TCFRemString,
 //    TCFUnknown
 
-    QTextCharFormat tf = defaultTextFormat();
-
-    m_textcharformats.insert(TCFDefault, tf);
 
 }
 

src/applesoftfile/applesofttoken.h

@@ -6,6 +6,22 @@
 #include <QMap>
 #include <QTextCharFormat>
 
+enum TextCharFormatType {
+    TCFDefault,
+    TCFCtrlChar,
+    TCFAscii,
+    TCFFunction,
+    TCFOperator,
+    TCFUnusedToken,
+    TCFNumber,
+    TCFString,
+    TCFVariable,
+    TCFDataString,
+    TCFRemString,
+    TCFUnknown
+};
+
+
 class ApplesoftToken
 {
 public:
@@ -26,6 +42,15 @@ public:
     static const quint16 StringVarTokenVal    = 0x109;
     static const quint16 StringAryVarTokenVal = 0x10A;
 
+    static const quint16 OptFmtLeadingSpaceTokenValue       = 0xe000;
+    static const quint16 OptFmtIndentLineBreakTokenValue    = 0xe001;
+    static const quint16 OptFmtIndentTabTokenValue          = 0xe002;
+    static const quint16 OptFmtIndentSpaceTokenValue        = 0xe003;
+    static const quint16 OptFmtFlagFlowTargetNextTokenValue = 0xe004;
+    static const quint16 OptFmtReturnLineBreakTokenValue    = 0xe005;
+
+
+    static const quint16 LineNumberTokenVal   = 0xfffe;
     static const quint16 DefaultTokenVal      = 0xffff;
 
 
@@ -117,7 +142,8 @@ public:
         FLOAT_VARIABLE_TOKEN       = 0xB,
         FLOAT_ARY_VARIABLE_TOKEN   = 0xC,
         STRING_VARIABLE_TOKEN      = 0xD,
-        STRING_ARY_VARIABLE_TOKEN  = 0xE
+        STRING_ARY_VARIABLE_TOKEN  = 0xE,
+        OPTIONAL_FORMAT_TOKEN      = 0xF
     } TokenType;
 
     typedef enum {
@@ -125,7 +151,8 @@ public:
         COMMAND,
         OPERATOR,
         FUNCTION,
-        UNDEFINED_COMMAND
+        UNDEFINED_COMMAND,
+        OPTIONAL_FORMAT
     } CommandType;
 
     ApplesoftToken();
@@ -156,32 +183,20 @@ public:
         return textFormat(m_token_id);
     }
 
-    QTextCharFormat textFormat(quint16 tokentype) ;
+    static QTextCharFormat textFormat(quint16 tokentype) ;
 
-    static QString getStringForToken(quint8 token) {
+    QString getStringForToken(quint8 token) {
         if (m_tokens.size() == 0) { initializeTokenTable(); }
         return m_tokens[token];
     }
 
     static QTextCharFormat defaultTextFormat();
 
+    bool isOptFmtToken() const { return (m_token_id >= 0xe000 && m_token_id < 0xf000); }
+
 private:
     void makeTextCharFormats();
 
-    enum TextCharFormatType {
-        TCFDefault,
-        TCFCtrlChar,
-        TCFAscii,
-        TCFFunction,
-        TCFOperator,
-        TCFUnusedToken,
-        TCFNumber,
-        TCFString,
-        TCFVariable,
-        TCFDataString,
-        TCFRemString,
-        TCFUnknown
-    };
 
     static QMap<quint16, QString> m_tokens;
 
@@ -197,5 +212,4 @@ private:
 };
 
 
-
 #endif // APPLESOFTTOKEN_H

src/ui/viewers/applesoftfileviewer.cxx

@@ -30,6 +30,7 @@ ApplesoftFileViewer::ApplesoftFileViewer(QWidget *parent) :
 
     setIndentCode(settings.value("ASViewer.indentCode",false).toBool(), NoReformat);
     setIntsAsHex(settings.value("ASViewer.intsAsHex",false).toBool(), NoReformat);
+    setBreakAfterReturn(settings.value("ASViewer.breakAfterReturn",false).toBool(), NoReformat);
 }
 
 ApplesoftFileViewer::~ApplesoftFileViewer()
@@ -37,8 +38,6 @@ ApplesoftFileViewer::~ApplesoftFileViewer()
     delete ui;
 }
 
-
-
 bool ApplesoftFileViewer::makeMenuOptions(QMenu *menu)
 {
     QSettings settings;
@@ -59,6 +58,14 @@ bool ApplesoftFileViewer::makeMenuOptions(QMenu *menu)
     connect(action, SIGNAL(toggled(bool)),SLOT(setIndentCode(bool)));
     menu->addAction(action);
 
+    action = new QAction("Blank &Line after RETURNs",menu);
+    action->setCheckable(true);
+    action->setChecked(settings.value("ASViewer.breakAfterReturn",false).toBool());
+    setIndentCode(settings.value("ASViewer.breakAfterReturn",false).toBool(),NoReformat);
+    connect(action, SIGNAL(toggled(bool)), ui->findText,SLOT(clear()));
+    connect(action, SIGNAL(toggled(bool)),SLOT(setBreakAfterReturn(bool)));
+    menu->addAction(action);
+
     menu->addSeparator();
 
     action = new QAction("Show &Variable Explorer...",menu);
@@ -113,6 +120,22 @@ void ApplesoftFileViewer::setIndentCode(bool enabled, ReformatRule reformat)
         reformatText();
 }
 
+void ApplesoftFileViewer::setBreakAfterReturn(bool enabled, ReformatRule reformat)
+{
+    if (enabled)
+    {
+        m_formatter->setFlags(m_formatter->flags() | ApplesoftFormatter::BreakAfterReturn);
+    }
+    else
+    {
+        m_formatter->setFlags(m_formatter->flags() & ~ApplesoftFormatter::BreakAfterReturn);
+    }
+    QSettings settings;
+    settings.setValue("ASViewer.breakAfterReturn",enabled);
+    if (reformat == ForceReformat)
+        reformatText();
+}
+
 void ApplesoftFileViewer::setIntsAsHex(bool enabled, ReformatRule reformat)
 {
     if (enabled)
@@ -131,9 +154,15 @@ void ApplesoftFileViewer::setIntsAsHex(bool enabled, ReformatRule reformat)
 
 void ApplesoftFileViewer::reformatText()
 {
-    m_formatter->formatText();
+    QTextDocument *doc = ui->textArea->document();
-    ui->textArea->setText(m_formatter->getFormattedText());
+
-    qDebug() << m_formatter->flowTargets();
+    m_formatter->formatDocument(doc);
+
+    QTextCursor cursor = ui->textArea->textCursor();
+    cursor.movePosition(QTextCursor::Start,QTextCursor::MoveAnchor);
+    ui->textArea->setTextCursor(cursor);
+
+    //qDebug() << m_formatter->flowTargets();
 }
 
 void ApplesoftFileViewer::setFile(GenericFile *file) {
@@ -151,8 +180,11 @@ void ApplesoftFileViewer::setFile(ApplesoftFile *file) {
     QString title = QString("AppleSoft Viewer: %1").arg(m_file->filename());
     m_title = title;
     setWindowTitle(title);
-    m_formatter->formatText();
+    QTextDocument *doc = ui->textArea->document();
-    ui->textArea->setText(m_formatter->getFormattedText());
+    m_formatter->formatDocument(doc);
+    QTextCursor cursor = ui->textArea->textCursor();
+    cursor.movePosition(QTextCursor::Start,QTextCursor::MoveAnchor);
+    ui->textArea->setTextCursor(cursor);
 }
 
 void ApplesoftFileViewer::setData(QByteArray data)

src/ui/viewers/applesoftfileviewer.h

@@ -48,6 +48,7 @@ protected slots:
     void toggleWordWrap(bool enabled);
     void setIndentCode(bool enabled, ReformatRule reformat = ForceReformat);
     void setIntsAsHex(bool enabled, ReformatRule reformat = ForceReformat);
+    void setBreakAfterReturn(bool enabled, ReformatRule reformat = ForceReformat);
     void launchVarBrowser();
     void reformatText();