Moved retokenization of files from ApplesoftFile to ApplesoftRetokenizer class

AppleSAWS.pro

@@ -65,7 +65,8 @@ SOURCES += \
     src/binaryfile/AssemblerSymbols.cpp \
     src/binaryfile/AssemblerSymbolModel.cpp \
     src/ui/diskexplorer/DiskExplorer.cpp \
-    src/ui/diskexplorer/DiskExplorerMapWidget.cpp
+    src/ui/diskexplorer/DiskExplorerMapWidget.cpp \
+    src/applesoftfile/ApplesoftRetokenizer.cpp
 
 
 HEADERS += \
@@ -113,7 +114,8 @@ HEADERS += \
     src/binaryfile/AssemblerSymbolModel.h \
     src/binaryfile/MemoryUsageMap.h \
     src/ui/diskexplorer/DiskExplorer.h \
-    src/ui/diskexplorer/DiskExplorerMapWidget.h
+    src/ui/diskexplorer/DiskExplorerMapWidget.h \
+    src/applesoftfile/ApplesoftRetokenizer.h
 
 FORMS += \
     src/ui/catalogwidget.ui \

src/applesoftfile/ApplesoftRetokenizer.cpp

@@ -0,0 +1,424 @@
+#include "ApplesoftRetokenizer.h"
+#include "applesofttoken.h"
+
+#include <QDebug>
+#include <QRegularExpression>
+
+ApplesoftRetokenizer::ApplesoftRetokenizer()
+{
+
+}
+
+void ApplesoftRetokenizer::setData(QByteArray data)
+{
+    m_data = data;
+    m_data_end = data.length();
+}
+
+void ApplesoftRetokenizer::parse(quint16 start_address)
+{
+    int idx = 0;
+    quint8 val = 0;
+    m_retokenized_lines.clear();
+
+    quint16 current_address = start_address;
+    while (idx < m_data.length()) {
+        ApplesoftLine line;
+        line.address = current_address;
+        line.next_address = (quint8) m_data[idx] + (((quint8) m_data[idx+1]) *256);
+
+        idx++; idx++;
+        line.linenum = (quint8) m_data[idx] + (((quint8) m_data[idx+1])*256);
+
+        idx++; idx++;
+        if (line.next_address == 0x00) { break; }
+        do {
+            val = m_data[idx++];
+            ApplesoftToken token(val);
+            line.tokens.append(token);
+        } while (val != 0x00);
+
+
+        retokenizeLine(line);
+
+        current_address = line.next_address;
+
+        m_retokenized_lines.append(line);
+    }
+
+     m_data_end = idx;
+
+    if (idx < m_data.length()) {
+        qDebug() << QString("%1 byte(s) unaccounted for.").arg(m_data.length() - idx);
+    }
+}
+
+void ApplesoftRetokenizer::retokenizeLine(ApplesoftLine &line)
+{
+    line.tokens = retokenizeRems(line.tokens);
+    line.tokens = retokenizeStrings(line.tokens);
+    line.tokens = retokenizeDataStatements(line.tokens);
+    line.tokens = retokenizeVariables(line.tokens);
+    line.tokens = retokenizeNumbers(line.tokens);
+    line.tokens = retokenizeNegativeNumbers(line.tokens);
+}
+
+QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeRems(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle REMs
+    ApplesoftToken token;
+    QVector<ApplesoftToken> replacements;
+    QVector<ApplesoftToken> tmptokens = datatokens;
+    QByteArray buffer;
+
+    bool inRem = false;
+
+    while (!tmptokens.isEmpty())
+    {
+        token = tmptokens.takeFirst();
+
+        if (!inRem) {
+            replacements.append(token);
+            if (token.getByteValue() == ApplesoftToken::ASRem)
+            {
+                inRem = true;
+            }
+        }
+        else
+        {
+            buffer.append(token.getByteValue());
+        }
+    }
+    if (inRem) {
+        ApplesoftToken remstrtoken(ApplesoftToken::RemStringTokenVal, buffer);
+        replacements.append(remstrtoken);
+        buffer.clear();
+        inRem = false;
+    }
+
+    return replacements;
+}
+
+QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeStrings(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle Strings
+    QVector<ApplesoftToken> replacements;
+    QVector<ApplesoftToken> tmptokens = datatokens;
+    QString buffer;
+    ApplesoftToken token;
+
+    bool inString = false;
+
+    while (!tmptokens.isEmpty())
+    {
+        token = tmptokens.takeFirst();
+        if (token.getTokenId() >= 0x80)
+        {
+            replacements.append(token);
+         //   continue;
+        } else
+
+        if (token.getWordValue() == '"')
+        {
+            if (!inString)
+            {
+                inString = true;
+                buffer.append(token.getWordValue());
+              //  continue;
+            }
+            else
+            {
+                buffer.append(token.getWordValue());
+                ApplesoftToken strtoken(ApplesoftToken::StringTokenVal, buffer);
+                replacements.append(strtoken);
+                buffer.clear();
+                inString = false;
+//                continue;
+            }
+        } else
+
+        if (inString)
+        {
+            buffer.append(token.getWordValue());
+            // continue;
+        } else
+
+        replacements.append(token);
+    }
+
+    return replacements;
+}
+
+QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeDataStatements(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle DATAs
+    QVector<ApplesoftToken> tmptokens = datatokens;
+    QVector<ApplesoftToken> replacements;
+    ApplesoftToken token;
+
+    QVector<ApplesoftToken> datatokenbuffer;
+    bool inData = false;
+    while (!tmptokens.isEmpty())
+    {
+        token = tmptokens.takeFirst();
+
+        if (!inData) {
+            replacements.append(token);
+            if (token.getTokenId() == ApplesoftToken::ASData)
+            {
+                inData = true;
+            }
+        }
+        else
+        {
+            datatokenbuffer.append(token);
+        }
+    }
+    if (inData) {
+        QVector<ApplesoftToken> dataTokens;
+        dataTokens = processDataPayload(datatokenbuffer);
+        replacements.append(dataTokens);
+        datatokenbuffer.clear();
+        inData = false;
+    }
+    return replacements;
+}
+
+
+QVector<ApplesoftToken> ApplesoftRetokenizer::processDataPayload(QVector<ApplesoftToken>& datatokens)
+{
+    QVector<ApplesoftToken> retval;
+
+    ApplesoftToken token;
+
+    QString stringbuffer;
+
+    while (!datatokens.isEmpty())
+    {
+        token = datatokens.takeFirst();
+        if (token.getTokenId() == ApplesoftToken::StringTokenVal)
+        {
+            ApplesoftToken newToken(ApplesoftToken::DataStringTokenVal, token.getStringValue());
+            retval.append(newToken);
+            continue;
+        }
+        if (token.getWordValue() == ',')
+        {
+            if (!stringbuffer.isEmpty())
+            {
+                ApplesoftToken datastrtoken(ApplesoftToken::DataStringTokenVal, stringbuffer);
+                retval.append(datastrtoken);
+                stringbuffer.clear();
+            }
+            retval.append(token);
+            continue;
+
+        }
+        stringbuffer.append(token.getWordValue());
+    }
+    if (!stringbuffer.isEmpty())
+    {
+        ApplesoftToken datastrtoken(ApplesoftToken::DataStringTokenVal, stringbuffer);
+        retval.append(datastrtoken);
+        stringbuffer.clear();
+    }
+    return retval;
+}
+QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeVariables(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle variable names
+    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
+    ApplesoftToken token;
+
+    QRegularExpression varregexp("[A-Za-z][A-Za-z0-9]*[$%]?\\(?");
+
+    QString parsestring;
+    // Parse the tokens to find assist
+    for (int idx = 0; idx < tmptokens.count();idx++)
+    {
+        token = datatokens.at(idx);
+
+        if (token.getTokenId() < 0x0080 && token.getTokenId() > 0x0000)
+        {
+            parsestring.append(QChar(token.getWordValue()));
+        }
+        else
+        {
+            parsestring.append("_");
+        }
+    }
+    QList<QRegularExpressionMatch> matchstack;
+    QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
+ //   qDebug() << parsestring;
+    while (matches.hasNext()) {
+        QRegularExpressionMatch rematch = matches.next();
+        matchstack.push_front(rematch);
+
+//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
+//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
+    }
+
+    foreach(QRegularExpressionMatch rematch, matchstack)
+    {
+        QString text = rematch.captured(0);
+        int start = rematch.capturedStart();
+        int length = rematch.capturedLength();
+
+        quint16 tokentype = ApplesoftToken::FloatVarTokenVal;
+        if (text.contains("$")) {
+            tokentype = ApplesoftToken::StringVarTokenVal;
+        } else if (text.contains("%")) {
+            tokentype = ApplesoftToken::IntVarTokenVal;
+        }
+
+        if (text.contains("(")) {
+            if (tokentype == ApplesoftToken::FloatVarTokenVal) {
+                tokentype = ApplesoftToken::FloatAryVarTokenVal;
+            } else if (tokentype == ApplesoftToken::StringVarTokenVal) {
+                tokentype = ApplesoftToken::StringAryVarTokenVal;
+            } else {
+                tokentype = ApplesoftToken::IntAryVarTokenVal;
+            }
+        }
+
+        ApplesoftToken vartoken = ApplesoftToken(tokentype,text);
+
+        for (int idx = 0; idx < length; idx++)
+        {
+            tmptokens.removeAt(start);
+        }
+        tmptokens.insert(start,vartoken);
+
+    }
+
+    datatokens = tmptokens.toVector();
+    return datatokens;
+}
+
+QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeNumbers(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle numbers
+    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
+    ApplesoftToken token;
+
+    QRegularExpression varregexp("[0-9]+(\\.[0-9]*)?");
+
+    QString parsestring;
+    // Parse the tokens to find assist
+    for (int idx = 0; idx < tmptokens.count();idx++)
+    {
+        token = datatokens.at(idx);
+
+        if (token.getTokenId() < 0x0080 && token.getTokenId() > 0x0000)
+        {
+            parsestring.append(QChar(token.getWordValue()));
+        }
+        else
+        {
+            parsestring.append("_");
+        }
+    }
+    QList<QRegularExpressionMatch> matchstack;
+    QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
+//    qDebug() << parsestring;
+    while (matches.hasNext()) {
+        QRegularExpressionMatch rematch = matches.next();
+        matchstack.push_front(rematch);
+
+//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
+//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
+    }
+
+    foreach(QRegularExpressionMatch rematch, matchstack)
+    {
+        QString text = rematch.captured(0);
+        int start = rematch.capturedStart();
+        int length = rematch.capturedLength();
+
+        quint16 tokentype = ApplesoftToken::IntegerTokenVal;
+        if (text.contains(".")) {
+            tokentype = ApplesoftToken::FloatTokenVal;
+        }
+
+        ApplesoftToken vartoken = ApplesoftToken(tokentype,text);
+
+        for (int idx = 0; idx < length; idx++)
+        {
+            tmptokens.removeAt(start);
+        }
+        tmptokens.insert(start,vartoken);
+
+    }
+
+    datatokens = tmptokens.toVector();
+    return datatokens;
+}
+
+QVector<ApplesoftToken> ApplesoftRetokenizer::retokenizeNegativeNumbers(QVector<ApplesoftToken>&datatokens)
+{
+    //TODO:  Code to make determination of negative numbers vs. unary minus/math formulas.
+    //  Prefixed '-' tokens for negative numbers should get merged with the integer value token.
+    //  So, need to determine when we're in an expression vs when we're starting a number.
+    //
+    //  A = -1 should retokenize.
+    //  A = - 4 - 1 should retokenize -4
+    //  A = - 4 - - 1 should retokenize -4 and -1
+    //  A = 3 - 1 shoud not retokenize
+    //  A = A - 1 should not
+    //  A = PEEK(123) - 5 should not
+    //  A = 4 * - 1 should
+    //  A = (1 + 2) - 4 should not
+    //  A = (1 + 2) + - 4 should
+    //  A = (1 + 2) - - 4 should
+    //  POKE - 4, 1 should
+    //  PRINT + - 4 should
+    //  PRINT - 4 should
+    //  PRINT + + + - - - 4 should retokenize the last -4.
+    //  A = 1 - - 4 should
+    //  A = 1 - - - 4 should, probably, but it's errorprone to say the least,
+    //      as are any multiple arbitrary +/-'s.  Have to hope for the best here.
+    //      Best bet would be to look at how AppleSoft handles these values.
+    //  A = - 0 is the same as 0
+
+    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
+    ApplesoftToken token;
+
+    QMutableListIterator<ApplesoftToken>it(tmptokens);
+
+    bool lastWasInt = false;
+
+    while (it.hasNext())
+    {
+        token = it.next();
+        if (token.getTokenId() == ApplesoftToken::IntegerTokenVal) lastWasInt = true;
+        else if (token.getTokenId() == ApplesoftToken::FloatTokenVal) lastWasInt = true;
+        else if (token.getTokenId() == ApplesoftToken::IntVarTokenVal) lastWasInt = true;
+        else if (token.getTokenId() == ApplesoftToken::FloatVarTokenVal) lastWasInt = true;
+        else if (token.getTokenId() == ')') lastWasInt = true;
+else
+        if (token.getTokenId() == ApplesoftToken::ASMINUS)
+        {
+            if (!lastWasInt && it.hasNext() && it.peekNext().getTokenId() == ApplesoftToken::IntegerTokenVal)
+            {
+                it.remove();
+                token = it.next();
+                it.remove();
+                int val = token.getUnsignedIntegerValue() * -1;
+                token.setValue(val);
+                it.insert(token);
+                lastWasInt = true;
+            }
+            else
+            {
+                lastWasInt = false;
+            }
+        }
+        else
+        {
+            lastWasInt = false;
+        }
+    }
+
+    return tmptokens.toVector();
+}
+

src/applesoftfile/ApplesoftRetokenizer.h

@@ -0,0 +1,40 @@
+#ifndef APPLESOFTRETOKENIZER_H
+#define APPLESOFTRETOKENIZER_H
+
+#include "applesoftline.h"
+
+#include <QByteArray>
+#include <QVector>
+
+class ApplesoftRetokenizer
+{
+public:
+    ApplesoftRetokenizer();
+
+    void setData(QByteArray data);
+    void parse(quint16 start_address = 0x0801);
+
+    quint16 getEndOfDataOffset() const { return m_data_end; }
+
+    QVector<ApplesoftLine> getRetokenizedLines() { return m_retokenized_lines; }
+
+
+private:
+    void retokenizeLine(ApplesoftLine &line);
+    QVector<ApplesoftToken> retokenizeRems(QVector<ApplesoftToken> &datatokens);
+    QVector<ApplesoftToken> retokenizeStrings(QVector<ApplesoftToken> &datatokens);
+    QVector<ApplesoftToken> retokenizeDataStatements(QVector<ApplesoftToken> &datatokens);
+    QVector<ApplesoftToken> retokenizeDataPayload(QVector<ApplesoftToken> &datatokens);
+    QVector<ApplesoftToken> retokenizeVariables(QVector<ApplesoftToken> &datatokens);
+    QVector<ApplesoftToken> retokenizeNumbers(QVector<ApplesoftToken> &datatokens);
+    QVector<ApplesoftToken> retokenizeNegativeNumbers(QVector<ApplesoftToken> &datatokens);
+
+    QVector<ApplesoftLine> m_retokenized_lines;
+    QByteArray m_data;
+
+    quint16 m_data_end;
+};
+
+
+#endif // APPLESOFTRETOKENIZER_H
+

src/applesoftfile/applesoftfile.cxx

@@ -7,6 +7,9 @@
 
 ApplesoftFile::ApplesoftFile(QByteArray data) : GenericFile(data)
 {
+    m_retokenizer = Q_NULLPTR;
+    m_data_end = data.length();
+
     if (!data.isEmpty())
     {
         setData(data);
@@ -16,13 +19,22 @@ ApplesoftFile::ApplesoftFile(QByteArray data) : GenericFile(data)
 
 void ApplesoftFile::setData(QByteArray data)
 {
+    if (!m_retokenizer)
+    {
+        m_retokenizer = new ApplesoftRetokenizer();
+    }
+
     GenericFile::setData(data);
 
     quint8 addlo = m_data.at(0);
     quint8 addhi = m_data.at(1);
     m_length = addlo + (addhi * 256);
     m_data.remove(0,2);
-    parse();
+
+    m_retokenizer->setData(m_data);
+    m_retokenizer->parse();
+    m_data_end = m_retokenizer->getEndOfDataOffset();
+    m_lines = m_retokenizer->getRetokenizedLines();
 }
 
 QByteArray ApplesoftFile::rawData() {
@@ -33,45 +45,6 @@ QByteArray ApplesoftFile::rawData() {
     return retval;
 }
 
-void ApplesoftFile::parse(quint16 start_address)
-{
-
-    int idx = 0;
-    quint8 val = 0;
-    m_lines.clear();
-
-    quint16 current_address = start_address;
-    while (idx < m_data.length()) {
-        ApplesoftLine line;
-        line.address = current_address;
-        line.next_address = (quint8) m_data[idx] + (((quint8) m_data[idx+1]) *256);
-
-        idx++; idx++;
-        line.linenum = (quint8) m_data[idx] + (((quint8) m_data[idx+1])*256);
-
-        idx++; idx++;
-        if (line.next_address == 0x00) { break; }
-        do {
-            val = m_data[idx++];
-            ApplesoftToken token(val);
-            line.tokens.append(token);
-        } while (val != 0x00);
-
-        Retokenizer ret;
-        ret.retokenize(line);
-
-        current_address = line.next_address;
-
-        m_lines.append(line);
-    }
-
-    m_data_end = idx;
-
-    if (idx < m_data.length()) {
-        qDebug() << QString("%1 byte(s) unaccounted for.").arg(m_data.length() - idx);
-    }
-}
-
 QStringList ApplesoftFile::extraDataHexValues() {
     QStringList retval;
 
@@ -99,376 +72,4 @@ QByteArray ApplesoftFile::extraData()
     return m_data.mid(m_data_end);
 }
 
-void Retokenizer::retokenize(ApplesoftLine &line)
-{
-    line.tokens = retokenizeRems(line.tokens);
-    line.tokens = retokenizeStrings(line.tokens);
-    line.tokens = retokenizeDataStatements(line.tokens);
-    line.tokens = retokenizeVariables(line.tokens);
-    line.tokens = retokenizeNumbers(line.tokens);
-    line.tokens = retokenizeNegativeNumbers(line.tokens);
 
-
-}
-
-QVector<ApplesoftToken> Retokenizer::retokenizeRems(QVector<ApplesoftToken>&datatokens)
-{
-    // Handle REMs
-    ApplesoftToken token;
-    QVector<ApplesoftToken> replacements;
-    QVector<ApplesoftToken> tmptokens = datatokens;
-    QByteArray buffer;
-
-    bool inRem = false;
-
-    while (!tmptokens.isEmpty())
-    {
-        token = tmptokens.takeFirst();
-
-        if (!inRem) {
-            replacements.append(token);
-            if (token.getByteValue() == ApplesoftToken::ASRem)
-            {
-                inRem = true;
-            }
-        }
-        else
-        {
-            buffer.append(token.getByteValue());
-        }
-    }
-    if (inRem) {
-        ApplesoftToken remstrtoken(ApplesoftToken::RemStringTokenVal, buffer);
-        replacements.append(remstrtoken);
-        buffer.clear();
-        inRem = false;
-    }
-
-    return replacements;
-}
-
-QVector<ApplesoftToken> Retokenizer::retokenizeStrings(QVector<ApplesoftToken>&datatokens)
-{
-    // Handle Strings
-    QVector<ApplesoftToken> replacements;
-    QVector<ApplesoftToken> tmptokens = datatokens;
-    QString buffer;
-    ApplesoftToken token;
-
-    bool inString = false;
-
-    while (!tmptokens.isEmpty())
-    {
-        token = tmptokens.takeFirst();
-        if (token.getTokenId() >= 0x80)
-        {
-            replacements.append(token);
-         //   continue;
-        } else
-
-        if (token.getWordValue() == '"')
-        {
-            if (!inString)
-            {
-                inString = true;
-                buffer.append(token.getWordValue());
-              //  continue;
-            }
-            else
-            {
-                buffer.append(token.getWordValue());
-                ApplesoftToken strtoken(ApplesoftToken::StringTokenVal, buffer);
-                replacements.append(strtoken);
-                buffer.clear();
-                inString = false;
-//                continue;
-            }
-        } else
-
-        if (inString)
-        {
-            buffer.append(token.getWordValue());
-            // continue;
-        } else
-
-        replacements.append(token);
-    }
-
-    return replacements;
-}
-
-QVector<ApplesoftToken> Retokenizer::retokenizeDataStatements(QVector<ApplesoftToken>&datatokens)
-{
-    // Handle DATAs
-    QVector<ApplesoftToken> tmptokens = datatokens;
-    QVector<ApplesoftToken> replacements;
-    ApplesoftToken token;
-
-    QVector<ApplesoftToken> datatokenbuffer;
-    bool inData = false;
-    while (!tmptokens.isEmpty())
-    {
-        token = tmptokens.takeFirst();
-
-        if (!inData) {
-            replacements.append(token);
-            if (token.getTokenId() == ApplesoftToken::ASData)
-            {
-                inData = true;
-            }
-        }
-        else
-        {
-            datatokenbuffer.append(token);
-        }
-    }
-    if (inData) {
-        QVector<ApplesoftToken> dataTokens;
-        dataTokens = processDataPayload(datatokenbuffer);
-        replacements.append(dataTokens);
-        datatokenbuffer.clear();
-        inData = false;
-    }
-    return replacements;
-}
-
-
-QVector<ApplesoftToken> Retokenizer::processDataPayload(QVector<ApplesoftToken>& datatokens)
-{
-    QVector<ApplesoftToken> retval;
-
-    ApplesoftToken token;
-
-    QString stringbuffer;
-
-    while (!datatokens.isEmpty())
-    {
-        token = datatokens.takeFirst();
-        if (token.getTokenId() == ApplesoftToken::StringTokenVal)
-        {
-            ApplesoftToken newToken(ApplesoftToken::DataStringTokenVal, token.getStringValue());
-            retval.append(newToken);
-            continue;
-        }
-        if (token.getWordValue() == ',')
-        {
-            if (!stringbuffer.isEmpty())
-            {
-                ApplesoftToken datastrtoken(ApplesoftToken::DataStringTokenVal, stringbuffer);
-                retval.append(datastrtoken);
-                stringbuffer.clear();
-            }
-            retval.append(token);
-            continue;
-
-        }
-        stringbuffer.append(token.getWordValue());
-    }
-    if (!stringbuffer.isEmpty())
-    {
-        ApplesoftToken datastrtoken(ApplesoftToken::DataStringTokenVal, stringbuffer);
-        retval.append(datastrtoken);
-        stringbuffer.clear();
-    }
-    return retval;
-}
-QVector<ApplesoftToken> Retokenizer::retokenizeVariables(QVector<ApplesoftToken>&datatokens)
-{
-    // Handle variable names
-    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
-    ApplesoftToken token;
-
-    QRegularExpression varregexp("[A-Za-z][A-Za-z0-9]*[$%]?\\(?");
-
-    QString parsestring;
-    // Parse the tokens to find assist
-    for (int idx = 0; idx < tmptokens.count();idx++)
-    {
-        token = datatokens.at(idx);
-
-        if (token.getTokenId() < 0x0080 && token.getTokenId() > 0x0000)
-        {
-            parsestring.append(QChar(token.getWordValue()));
-        }
-        else
-        {
-            parsestring.append("_");
-        }
-    }
-    QList<QRegularExpressionMatch> matchstack;
-    QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
- //   qDebug() << parsestring;
-    while (matches.hasNext()) {
-        QRegularExpressionMatch rematch = matches.next();
-        matchstack.push_front(rematch);
-
-//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
-//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
-    }
-
-    foreach(QRegularExpressionMatch rematch, matchstack)
-    {
-        QString text = rematch.captured(0);
-        int start = rematch.capturedStart();
-        int length = rematch.capturedLength();
-
-        quint16 tokentype = ApplesoftToken::FloatVarTokenVal;
-        if (text.contains("$")) {
-            tokentype = ApplesoftToken::StringVarTokenVal;
-        } else if (text.contains("%")) {
-            tokentype = ApplesoftToken::IntVarTokenVal;
-        }
-
-        if (text.contains("(")) {
-            if (tokentype == ApplesoftToken::FloatVarTokenVal) {
-                tokentype = ApplesoftToken::FloatAryVarTokenVal;
-            } else if (tokentype == ApplesoftToken::StringVarTokenVal) {
-                tokentype = ApplesoftToken::StringAryVarTokenVal;
-            } else {
-                tokentype = ApplesoftToken::IntAryVarTokenVal;
-            }
-        }
-
-        ApplesoftToken vartoken = ApplesoftToken(tokentype,text);
-
-        for (int idx = 0; idx < length; idx++)
-        {
-            tmptokens.removeAt(start);
-        }
-        tmptokens.insert(start,vartoken);
-
-    }
-
-    datatokens = tmptokens.toVector();
-    return datatokens;
-}
-
-QVector<ApplesoftToken> Retokenizer::retokenizeNumbers(QVector<ApplesoftToken>&datatokens)
-{
-    // Handle numbers
-    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
-    ApplesoftToken token;
-
-    QRegularExpression varregexp("[0-9]+(\\.[0-9]*)?");
-
-    QString parsestring;
-    // Parse the tokens to find assist
-    for (int idx = 0; idx < tmptokens.count();idx++)
-    {
-        token = datatokens.at(idx);
-
-        if (token.getTokenId() < 0x0080 && token.getTokenId() > 0x0000)
-        {
-            parsestring.append(QChar(token.getWordValue()));
-        }
-        else
-        {
-            parsestring.append("_");
-        }
-    }
-    QList<QRegularExpressionMatch> matchstack;
-    QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
-//    qDebug() << parsestring;
-    while (matches.hasNext()) {
-        QRegularExpressionMatch rematch = matches.next();
-        matchstack.push_front(rematch);
-
-//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
-//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
-    }
-
-    foreach(QRegularExpressionMatch rematch, matchstack)
-    {
-        QString text = rematch.captured(0);
-        int start = rematch.capturedStart();
-        int length = rematch.capturedLength();
-
-        quint16 tokentype = ApplesoftToken::IntegerTokenVal;
-        if (text.contains(".")) {
-            tokentype = ApplesoftToken::FloatTokenVal;
-        }
-
-        ApplesoftToken vartoken = ApplesoftToken(tokentype,text);
-
-        for (int idx = 0; idx < length; idx++)
-        {
-            tmptokens.removeAt(start);
-        }
-        tmptokens.insert(start,vartoken);
-
-    }
-
-    datatokens = tmptokens.toVector();
-    return datatokens;
-}
-
-QVector<ApplesoftToken> Retokenizer::retokenizeNegativeNumbers(QVector<ApplesoftToken>&datatokens)
-{
-    //TODO:  Code to make determination of negative numbers vs. unary minus/math formulas.
-    //  Prefixed '-' tokens for negative numbers should get merged with the integer value token.
-    //  So, need to determine when we're in an expression vs when we're starting a number.
-    //
-    //  A = -1 should retokenize.
-    //  A = - 4 - 1 should retokenize -4
-    //  A = - 4 - - 1 should retokenize -4 and -1
-    //  A = 3 - 1 shoud not retokenize
-    //  A = A - 1 should not
-    //  A = PEEK(123) - 5 should not
-    //  A = 4 * - 1 should
-    //  A = (1 + 2) - 4 should not
-    //  A = (1 + 2) + - 4 should
-    //  A = (1 + 2) - - 4 should
-    //  POKE - 4, 1 should
-    //  PRINT + - 4 should
-    //  PRINT - 4 should
-    //  PRINT + + + - - - 4 should retokenize the last -4.
-    //  A = 1 - - 4 should
-    //  A = 1 - - - 4 should, probably, but it's errorprone to say the least,
-    //      as are any multiple arbitrary +/-'s.  Have to hope for the best here.
-    //      Best bet would be to look at how AppleSoft handles these values.
-    //  A = - 0 is the same as 0
-
-    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
-    ApplesoftToken token;
-
-    QMutableListIterator<ApplesoftToken>it(tmptokens);
-
-    bool lastWasInt = false;
-
-    while (it.hasNext())
-    {
-        token = it.next();
-        if (token.getTokenId() == ApplesoftToken::IntegerTokenVal) lastWasInt = true;
-        else if (token.getTokenId() == ApplesoftToken::FloatTokenVal) lastWasInt = true;
-        else if (token.getTokenId() == ApplesoftToken::IntVarTokenVal) lastWasInt = true;
-        else if (token.getTokenId() == ApplesoftToken::FloatVarTokenVal) lastWasInt = true;
-        else if (token.getTokenId() == ')') lastWasInt = true;
-else
-        if (token.getTokenId() == ApplesoftToken::ASMINUS)
-        {
-            if (!lastWasInt && it.hasNext() && it.peekNext().getTokenId() == ApplesoftToken::IntegerTokenVal)
-            {
-                it.remove();
-                token = it.next();
-                it.remove();
-                int val = token.getUnsignedIntegerValue() * -1;
-                token.setValue(val);
-                it.insert(token);
-                lastWasInt = true;
-            }
-            else
-            {
-                lastWasInt = false;
-            }
-        }
-        else
-        {
-            lastWasInt = false;
-        }
-
-
-
-    }
-
-    return tmptokens.toVector();
-}

src/applesoftfile/applesoftfile.h

@@ -9,6 +9,7 @@
 #include "applesoftline.h"
 #include "genericfile.h"
 #include "applesofttoken.h"
+#include "ApplesoftRetokenizer.h"
 
 
 class ApplesoftFile : public GenericFile
@@ -26,27 +27,14 @@ public:
     QByteArray rawData();
 
 private:
-    void parse(quint16 start_address = 0x0801);
 
     QVector<ApplesoftLine> m_lines;
     int m_data_end;
     quint16 m_length;
+
+    ApplesoftRetokenizer *m_retokenizer;
 };
 
-class Retokenizer {
-public:
-    static void retokenize(ApplesoftLine &line);
-
-protected:
-private:
-    static QVector<ApplesoftToken> retokenizeRems(QVector<ApplesoftToken> &datatokens);
-    static QVector<ApplesoftToken> retokenizeStrings(QVector<ApplesoftToken> &datatokens);
-    static QVector<ApplesoftToken> retokenizeDataStatements(QVector<ApplesoftToken> &datatokens);
-    static QVector<ApplesoftToken> processDataPayload(QVector<ApplesoftToken> &datatokens);
-    static QVector<ApplesoftToken> retokenizeVariables(QVector<ApplesoftToken> &datatokens);
-    static QVector<ApplesoftToken> retokenizeNumbers(QVector<ApplesoftToken> &datatokens);
-    static QVector<ApplesoftToken> retokenizeNegativeNumbers(QVector<ApplesoftToken> &datatokens);
-};
 
 
 #endif // APPLESOFTFILE_H

src/applesoftfile/applesoftformatter.cxx

@@ -1,12 +1,15 @@
 #include "applesoftformatter.h"
 #include "util.h"
+#include <QTextCursor>
+#include <QTextCharFormat>
 
 #define HEXPREFIX "0x"
 
 ApplesoftFormatter::ApplesoftFormatter(QObject *parent) :
     QObject(parent)
 {
-    m_file = 0;
+    m_file = Q_NULLPTR;
+    m_formattedDocument = QSharedPointer<QTextDocument>(new QTextDocument());
 }
 
 void ApplesoftFormatter::setFile(ApplesoftFile *file)
@@ -15,20 +18,27 @@ void ApplesoftFormatter::setFile(ApplesoftFile *file)
     emit newFile(file);
 }
 
-QString ApplesoftFormatter::formatText()
+void ApplesoftFormatter::formatText()
 {
+    m_formattedText.clear();
+    m_formattedDocument->clear();
+
     if (!m_file) {
-        return ("No File");
+        return;
     }
 
-    QString retval;
+    QTextCursor cursor(m_formattedDocument.data());
+
+    QString formattedText;
     m_flowTargets.clear();
 
     foreach (ApplesoftLine line, m_file->getLines()) {
         QString linestring = QString("%1 ").arg(line.linenum,5,10,QChar(' '));
 
         int indentlevel = 1;
-        retval.append(linestring);
+        formattedText.append(linestring);
+        cursor.insertText(linestring, ApplesoftToken::defaultTextFormat());
+
 
         QVectorIterator<ApplesoftToken> tokenIt(line.tokens);
         ApplesoftToken previousToken;
@@ -48,6 +58,8 @@ QString ApplesoftFormatter::formatText()
                 }
                 firstToken = false;
             }
+            cursor.insertText(tokenstr, token.textFormat());
+
 
             //TODO: Move this to the parser.
             //TODO: This doesn't yet handle: ON expr GOTO/GOSUB n1,n2,n3,...
@@ -163,16 +175,16 @@ QString ApplesoftFormatter::formatText()
                 }
             }
 
-            retval.append(tokenstr);
+            formattedText.append(tokenstr);
             previousToken = token;
         }
 
-        retval.append("\n");
+        formattedText.append("\n");
         if (m_format_options.testFlag(ReindentCode))
         {
   //          retval.append("\n");
         }
 
     }
-    return retval;
+    m_formattedText = formattedText;
 }

src/applesoftfile/applesoftformatter.h

@@ -3,6 +3,9 @@
 
 #include <QObject>
 #include <QFlags>
+#include <QTextDocument>
+#include <QSharedPointer>
+
 #include "applesoftfile.h"
 
 class ApplesoftFormatter : public QObject
@@ -31,7 +34,9 @@ public:
     void setFile(ApplesoftFile *file);
     FormatOptions flags() { return m_format_options; }
 
-    QString formatText();
+    void formatText();
+    QSharedPointer<QTextDocument> getFormattedDocument() const { return m_formattedDocument; }
+    QString getFormattedText() const { return m_formattedText; }
 
     QList<QPair<quint16, quint16> > flowTargets() const { return m_flowTargets; }
 
@@ -46,6 +51,9 @@ private:
 
     ApplesoftFile *m_file;
 
+    QString m_formattedText;
+    QSharedPointer<QTextDocument> m_formattedDocument;
+
 };
 
 Q_DECLARE_OPERATORS_FOR_FLAGS(ApplesoftFormatter::FormatOptions)

src/applesoftfile/applesofttoken.cxx

@@ -5,7 +5,7 @@ QMap<quint16, QString> ApplesoftToken::m_tokens = QMap<quint16, QString>();
 ApplesoftToken::ApplesoftToken()
 {
     if (m_tokens.size() == 0) { initializeTokenTable(); }
-    setTokenId(0xFFFF);
+    setTokenId(DefaultTokenVal);
 }
 
 ApplesoftToken::ApplesoftToken(quint16 id)
@@ -88,29 +88,13 @@ void ApplesoftToken::setValue(QVariant value)
     m_payload = value;
 }
 
-QString ApplesoftToken::getHtmlPrintableString() const
-{
-    QString baseval = getRawPrintableString().toHtmlEscaped();
 
-    if (getTokenId() <= 0x7f)
-        return QString("<font color=\"black\">%1</font>").arg(baseval);
-    if (getTokenId() <= 0xff)
-        return QString("<font color=\"red\">%1</font>").arg(baseval);
-    if (getTokenId() == ApplesoftToken::StringTokenVal)
-        return QString("<font color=\"black\">%1</font>").arg(baseval);
-    if (getTokenId() == ApplesoftToken::DataStringTokenVal)
-        return QString("<font color=\"green\">%1</font>").arg(baseval);
-    if (getTokenId() == ApplesoftToken::RemStringTokenVal)
-        return QString("<font color=\"grey\">%1</font>").arg(baseval);
-    if (getTokenId() == ApplesoftToken::IntegerTokenVal || getTokenId() == ApplesoftToken::FloatAryVarTokenVal)
-        return QString("<font color=\"blue\">%1</font>").arg(baseval);
-
-    return QString("<font color=\"orange\">%1</font>").arg(baseval);
-}
 
 QString ApplesoftToken::getRawPrintableString() const
 {
-    if (m_token_id == 0x00) {
+    if (m_token_id == DefaultTokenVal) {
+        return "";
+    } else if (m_token_id == 0x00) {
         return "";
     } else if (m_token_id <= 0x7f) {
         return QString((m_token_id));
@@ -143,6 +127,44 @@ QString ApplesoftToken::getRawPrintableString() const
     }
 }
 
+QTextCharFormat ApplesoftToken::defaultTextFormat()
+{
+    QTextCharFormat tf;  // Default
+    tf.setFontFamily("Typewriter");
+    tf.setFontPointSize(10);
+    tf.setForeground(Qt::red);
+    return tf;
+}
+
+QTextCharFormat ApplesoftToken::textFormat(quint16 tokenType)
+{
+    makeTextCharFormats();
+    return m_textcharformats[TCFDefault];
+}
+
+
+
+void ApplesoftToken::makeTextCharFormats()
+{
+//    TCFDefault,
+//    TCFCtrlChar,
+//    TCFAscii,
+//    TCFFunction,
+//    TCFOperator,
+//    TCFUnusedToken,
+//    TCFNumber,
+//    TCFString,
+//    TCFVariable,
+//    TCFDataString,
+//    TCFRemString,
+//    TCFUnknown
+
+    QTextCharFormat tf = defaultTextFormat();
+
+    m_textcharformats.insert(TCFDefault, tf);
+
+}
+
 void ApplesoftToken::initializeTokenTable()
 {
     m_tokens[ASEnd]      = " END ";     m_tokens[ASFor]     = " FOR ";

src/applesoftfile/applesofttoken.h

@@ -4,6 +4,7 @@
 #include <QString>
 #include <QVariant>
 #include <QMap>
+#include <QTextCharFormat>
 
 class ApplesoftToken
 {
@@ -25,6 +26,8 @@ public:
     static const quint16 StringVarTokenVal    = 0x109;
     static const quint16 StringAryVarTokenVal = 0x10A;
 
+    static const quint16 DefaultTokenVal      = 0xffff;
+
 
 
     static const quint16 ASEnd;      static const quint16 ASFor;
@@ -147,15 +150,38 @@ public:
     CommandType getCommandType() const { return m_command_type; }
 
     QString getRawPrintableString() const;
-    QString getHtmlPrintableString() const;
+
+    QTextCharFormat textFormat()
+    {
+        return textFormat(m_token_id);
+    }
+
+    QTextCharFormat textFormat(quint16 tokentype) ;
 
     static QString getStringForToken(quint8 token) {
         if (m_tokens.size() == 0) { initializeTokenTable(); }
         return m_tokens[token];
     }
 
+    static QTextCharFormat defaultTextFormat();
 
 private:
+    void makeTextCharFormats();
+
+    enum TextCharFormatType {
+        TCFDefault,
+        TCFCtrlChar,
+        TCFAscii,
+        TCFFunction,
+        TCFOperator,
+        TCFUnusedToken,
+        TCFNumber,
+        TCFString,
+        TCFVariable,
+        TCFDataString,
+        TCFRemString,
+        TCFUnknown
+    };
 
     static QMap<quint16, QString> m_tokens;
 
@@ -165,6 +191,8 @@ private:
     TokenType m_token_type;
     CommandType m_command_type;
 
+    QMap<TextCharFormatType,QTextCharFormat> m_textcharformats;
+
     static void initializeTokenTable();
 };
 

src/ui/viewers/applesoftfileviewer.cxx

@@ -131,7 +131,8 @@ void ApplesoftFileViewer::setIntsAsHex(bool enabled, ReformatRule reformat)
 
 void ApplesoftFileViewer::reformatText()
 {
-    ui->textArea->setText(m_formatter->formatText());
+    m_formatter->formatText();
+    ui->textArea->setText(m_formatter->getFormattedText());
     qDebug() << m_formatter->flowTargets();
 }
 
@@ -150,8 +151,8 @@ void ApplesoftFileViewer::setFile(ApplesoftFile *file) {
     QString title = QString("AppleSoft Viewer: %1").arg(m_file->filename());
     m_title = title;
     setWindowTitle(title);
-
+    m_formatter->formatText();
-    ui->textArea->setText(m_formatter->formatText());
+    ui->textArea->setText(m_formatter->getFormattedText());
 }
 
 void ApplesoftFileViewer::setData(QByteArray data)