Now parses strings and numbers into enhanced tokens, as well.

src/applesoftfile/applesoftfile.cxx

@@ -1,5 +1,9 @@
 #include "applesoftfile.h"
 #include <QDebug>
+#include <QRegularExpression>
+#include <QRegularExpressionMatch>
+#include <QRegularExpressionMatchIterator>
+#include <QList>
 
 ApplesoftFile::ApplesoftFile(QByteArray data) : GenericFile(data)
 {
@@ -32,6 +36,7 @@ QByteArray ApplesoftFile::rawData() {
 
 void ApplesoftFile::parse(quint16 start_address)
 {
+
     int idx = 0;
     quint8 val = 0;
 
@@ -96,19 +101,25 @@ QByteArray ApplesoftFile::extraData()
 
 void Retokenizer::retokenize(ApplesoftLine &line)
 {
-    Q_UNUSED(line);
+    line.tokens = retokenizeRems(line.tokens);
+    line.tokens = retokenizeStrings(line.tokens);
+    line.tokens = retokenizeDataStatements(line.tokens);
+    line.tokens = retokenizeVariables(line.tokens);
+    line.tokens = retokenizeNumbers(line.tokens);
 
 
-    QVector<ApplesoftToken> replacements;
+}
-    QVector<ApplesoftToken> tmptokens = line.tokens;
+
+QVector<ApplesoftToken> Retokenizer::retokenizeRems(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle REMs
     ApplesoftToken token;
-
+    QVector<ApplesoftToken> replacements;
+    QVector<ApplesoftToken> tmptokens = datatokens;
     QByteArray buffer;
 
     bool inRem = false;
 
-    // Handle REMs
-
     while (!tmptokens.isEmpty())
     {
         token = tmptokens.takeFirst();
@@ -132,14 +143,17 @@ void Retokenizer::retokenize(ApplesoftLine &line)
         inRem = false;
     }
 
-    line.tokens = replacements;
+    return replacements;
-    replacements.clear();
+}
-
-    // Handle DATAs
 
+QVector<ApplesoftToken> Retokenizer::retokenizeStrings(QVector<ApplesoftToken>&datatokens)
+{
     // Handle Strings
+    QVector<ApplesoftToken> replacements;
+    QVector<ApplesoftToken> tmptokens = datatokens;
+    QString buffer;
+    ApplesoftToken token;
 
-    tmptokens = line.tokens;
     bool inString = false;
 
     while (!tmptokens.isEmpty())
@@ -148,45 +162,241 @@ void Retokenizer::retokenize(ApplesoftLine &line)
         if (token.getTokenId() >= 0x80)
         {
             replacements.append(token);
-            continue;
+         //   continue;
-        }
+        } else
 
-        if (token.getByteValue() == '"')
+        if (token.getWordValue() == '"')
         {
             if (!inString)
             {
                 inString = true;
-                buffer.append(token.getByteValue());
+                buffer.append(token.getWordValue());
-                continue;
+              //  continue;
             }
             else
             {
-                buffer.append(token.getByteValue());
+                buffer.append(token.getWordValue());
                 ApplesoftToken strtoken(ApplesoftToken::StringTokenVal, buffer);
                 replacements.append(strtoken);
                 buffer.clear();
                 inString = false;
-                continue;
+//                continue;
             }
-        }
+        } else
 
         if (inString)
         {
-            buffer.append(token.getByteValue());
+            buffer.append(token.getWordValue());
-            continue;
+            // continue;
-        }
+        } else
 
         replacements.append(token);
     }
 
-    line.tokens = replacements;
+    return replacements;
-
 }
 
-QByteArray Retokenizer::retokenizePart(QByteArray part) {
+QVector<ApplesoftToken> Retokenizer::retokenizeDataStatements(QVector<ApplesoftToken>&datatokens)
-    QByteArray retval;
+{
+    // Handle DATAs
+    QVector<ApplesoftToken> tmptokens = datatokens;
+    QVector<ApplesoftToken> replacements;
+    ApplesoftToken token;
 
-    Q_UNUSED(part);
+    QVector<ApplesoftToken> datatokenbuffer;
+    bool inData = false;
+    while (!tmptokens.isEmpty())
+    {
+        token = tmptokens.takeFirst();
 
+        if (!inData) {
+            replacements.append(token);
+            if (token.getTokenId() == ApplesoftToken::ASData)
+            {
+                inData = true;
+            }
+        }
+        else
+        {
+            datatokenbuffer.append(token);
+        }
+    }
+    if (inData) {
+        QVector<ApplesoftToken> dataTokens;
+        dataTokens = processDataPayload(datatokenbuffer);
+        replacements.append(dataTokens);
+        datatokenbuffer.clear();
+        inData = false;
+    }
+    return replacements;
+}
+
+
+QVector<ApplesoftToken> Retokenizer::processDataPayload(QVector<ApplesoftToken>& datatokens)
+{
+    QVector<ApplesoftToken> retval;
+
+    ApplesoftToken token;
+
+    QString stringbuffer;
+
+    while (!datatokens.isEmpty())
+    {
+        token = datatokens.takeFirst();
+        if (token.getTokenId() == ApplesoftToken::StringTokenVal)
+        {
+            ApplesoftToken newToken(ApplesoftToken::DataStringTokenVal, token.getStringValue());
+            retval.append(newToken);
+            continue;
+        }
+        if (token.getWordValue() == ',')
+        {
+            if (!stringbuffer.isEmpty())
+            {
+                ApplesoftToken datastrtoken(ApplesoftToken::DataStringTokenVal, stringbuffer);
+                retval.append(datastrtoken);
+                stringbuffer.clear();
+            }
+            retval.append(token);
+            continue;
+
+        }
+        stringbuffer.append(token.getWordValue());
+    }
+    if (!stringbuffer.isEmpty())
+    {
+        ApplesoftToken datastrtoken(ApplesoftToken::DataStringTokenVal, stringbuffer);
+        retval.append(datastrtoken);
+        stringbuffer.clear();
+    }
     return retval;
 }
+QVector<ApplesoftToken> Retokenizer::retokenizeVariables(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle variable names
+    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
+    ApplesoftToken token;
+
+    QRegularExpression varregexp("[A-Za-z][A-Za-z0-9]*[$%]?\\(?");
+
+    QString parsestring;
+    // Parse the tokens to find assist
+    for (int idx = 0; idx < tmptokens.count();idx++)
+    {
+        token = datatokens.at(idx);
+
+        if (token.getTokenId() < 0x0080 && token.getTokenId() > 0x0000)
+        {
+            parsestring.append(QChar(token.getWordValue()));
+        }
+        else
+        {
+            parsestring.append("_");
+        }
+    }
+    QList<QRegularExpressionMatch> matchstack;
+    QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
+ //   qDebug() << parsestring;
+    while (matches.hasNext()) {
+        QRegularExpressionMatch rematch = matches.next();
+        matchstack.push_front(rematch);
+
+//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
+//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
+    }
+
+    foreach(QRegularExpressionMatch rematch, matchstack)
+    {
+        QString text = rematch.captured(0);
+        int start = rematch.capturedStart();
+        int length = rematch.capturedLength();
+
+        quint16 tokentype = ApplesoftToken::FloatVarTokenVal;
+        if (text.contains("$")) {
+            tokentype = ApplesoftToken::StringVarTokenVal;
+        } else if (text.contains("%")) {
+            tokentype = ApplesoftToken::IntVarTokenVal;
+        }
+
+        if (text.contains("(")) {
+            if (tokentype == ApplesoftToken::FloatVarTokenVal) {
+                tokentype = ApplesoftToken::FloatAryVarTokenVal;
+            } else if (tokentype == ApplesoftToken::StringVarTokenVal) {
+                tokentype = ApplesoftToken::StringAryVarTokenVal;
+            } else {
+                tokentype = ApplesoftToken::IntAryVarTokenVal;
+            }
+        }
+
+        ApplesoftToken vartoken = ApplesoftToken(tokentype,text);
+
+        for (int idx = 0; idx < length; idx++)
+        {
+            tmptokens.removeAt(start);
+        }
+        tmptokens.insert(start,vartoken);
+
+    }
+
+    datatokens = tmptokens.toVector();
+    return datatokens;
+}
+
+QVector<ApplesoftToken> Retokenizer::retokenizeNumbers(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle numbers
+    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
+    ApplesoftToken token;
+
+    QRegularExpression varregexp("[0-9]+(\\.[0-9]*)?");
+
+    QString parsestring;
+    // Parse the tokens to find assist
+    for (int idx = 0; idx < tmptokens.count();idx++)
+    {
+        token = datatokens.at(idx);
+
+        if (token.getTokenId() < 0x0080 && token.getTokenId() > 0x0000)
+        {
+            parsestring.append(QChar(token.getWordValue()));
+        }
+        else
+        {
+            parsestring.append("_");
+        }
+    }
+    QList<QRegularExpressionMatch> matchstack;
+    QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
+//    qDebug() << parsestring;
+    while (matches.hasNext()) {
+        QRegularExpressionMatch rematch = matches.next();
+        matchstack.push_front(rematch);
+
+//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
+//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
+    }
+
+    foreach(QRegularExpressionMatch rematch, matchstack)
+    {
+        QString text = rematch.captured(0);
+        int start = rematch.capturedStart();
+        int length = rematch.capturedLength();
+
+        quint16 tokentype = ApplesoftToken::IntegerTokenVal;
+        if (text.contains(".")) {
+            tokentype = ApplesoftToken::FloatTokenVal;
+        }
+
+        ApplesoftToken vartoken = ApplesoftToken(tokentype,text);
+
+        for (int idx = 0; idx < length; idx++)
+        {
+            tmptokens.removeAt(start);
+        }
+        tmptokens.insert(start,vartoken);
+
+    }
+
+    datatokens = tmptokens.toVector();
+    return datatokens;
+}

src/applesoftfile/applesoftfile.h

@@ -38,8 +38,15 @@ private:
 class Retokenizer {
 public:
     static void retokenize(ApplesoftLine &line);
+
 private:
-    QByteArray retokenizePart(QByteArray part);
+    static QVector<ApplesoftToken> retokenizeRems(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> retokenizeStrings(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> retokenizeDataStatements(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> processDataPayload(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> retokenizeVariables(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> retokenizeNumbers(QVector<ApplesoftToken> &datatokens);
 };
 
+
 #endif // APPLESOFTFILE_H

src/applesoftfile/applesoftformatter.cxx

@@ -23,45 +23,45 @@ QString ApplesoftFormatter::formatText()
 
     foreach (ApplesoftLine line, m_file->getLines()) {
         QString linestring = QString("%1  ").arg(line.linenum,5,10,QChar(' '));
+
         retval.append(linestring);
         foreach (ApplesoftToken token, line.tokens)
         {
             QString tokenstr = token.getRawPrintableString();
 
             if (m_format_options.testFlag(BreakAfterReturn)) {
-                if (token.getByteValue() == ApplesoftToken::ASReturn)
+                if (token.getTokenId() == ApplesoftToken::ASReturn)
                 {
                     tokenstr += "\n";
                 }
             }
 
-#define DEBUGTOKENS
+#define noDEBUGTOKENS
 #ifdef DEBUGTOKENS
             if (token.getTokenId() >= 0x80)
             {
-                tokenstr = QString(" {%1} ").arg(uint16ToHex(token.getTokenId()));
+                tokenstr = QString("{%1 (%2)}").arg(tokenstr).arg(uint16ToHex(token.getTokenId()));
-  //              tokenstr = " __ ";
+               // tokenstr = " __ ";
-            }
-            else
-            {
-                tokenstr = QString("%1").arg(QChar(token.getByteValue()));
             }
+
 #endif
 
-            if (m_format_options.testFlag(ShowCtrlChars)) {
+   //         if (m_format_options.testFlag(ShowCtrlChars))
+            {
                 tokenstr.replace(QChar(0x7f),QChar(0x2401));
 
                 for (int idx = 1; idx <= 0x1f; idx++) {
                     if (idx == '\n') continue;
-                    if (idx == '\t') continue;
                     tokenstr.replace(QChar(idx),QChar(idx+0x2400));
+      //              tokenstr.replace(QChar(idx), QString("<%1>").arg(uint8ToHex(idx)));
                 }
             }
 
-
             retval.append(tokenstr);
         }
         retval.append("\n");
+
     }
+
     return retval;
 }

src/applesoftfile/applesofttoken.cxx

@@ -89,16 +89,59 @@ void ApplesoftToken::setValue(QVariant value)
     m_payload = value;
 }
 
+QString ApplesoftToken::getHtmlPrintableString()
+{
+    QString baseval = getRawPrintableString().toHtmlEscaped();
+
+    if (getTokenId() <= 0x7f)
+        return QString("<font color=\"black\">%1</font>").arg(baseval);
+    if (getTokenId() <= 0xff)
+        return QString("<font color=\"red\">%1</font>").arg(baseval);
+    if (getTokenId() == ApplesoftToken::StringTokenVal)
+        return QString("<font color=\"black\">%1</font>").arg(baseval);
+    if (getTokenId() == ApplesoftToken::DataStringTokenVal)
+        return QString("<font color=\"green\">%1</font>").arg(baseval);
+    if (getTokenId() == ApplesoftToken::RemStringTokenVal)
+        return QString("<font color=\"grey\">%1</font>").arg(baseval);
+    if (getTokenId() == ApplesoftToken::IntegerTokenVal || getTokenId() == ApplesoftToken::FloatAryVarTokenVal)
+        return QString("<font color=\"blue\">%1</font>").arg(baseval);
+
+
+
+    return QString("<font color=\"orange\">%1</font>").arg(baseval);
+
+}
+
 QString ApplesoftToken::getRawPrintableString()
 {
     if (m_token_id == 0x00) {
         return "";
     } else if (m_token_id <= 0x7f) {
-        return QString(QChar(m_token_id));
+        return QString((m_token_id));
     } else if (m_token_id <= 0xff) {
         return m_tokens[m_token_id];
     } else if (m_token_id == StringTokenVal) {
-        return getByteStringValue();
+        return getStringValue();
+    } else if (m_token_id == RemStringTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == DataStringTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == IntegerTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == FloatTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == IntVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == IntAryVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == FloatVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == FloatAryVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == StringVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == StringAryVarTokenVal) {
+        return getStringValue();
     } else {
         return "[temp undefined]";
     }

src/applesoftfile/applesofttoken.h

@@ -16,6 +16,8 @@ public:
     static const quint16 IntegerTokenVal      = 0x103;
     static const quint16 FloatTokenVal        = 0x104;
 
+
+
     static const quint16 IntVarTokenVal       = 0x105;
     static const quint16 IntAryVarTokenVal    = 0x106;
 
@@ -25,6 +27,8 @@ public:
     static const quint16 StringVarTokenVal    = 0x109;
     static const quint16 StringAryVarTokenVal = 0x10A;
 
+
+
     static const quint16 ASEnd;      static const quint16 ASFor;
     static const quint16 ASNext;     static const quint16 ASData;
     static const quint16 ASInput;    static const quint16 ASDel;
@@ -134,6 +138,7 @@ public:
     QVariant getValue() const { return m_payload; }
 
     QByteArray getByteStringValue() const { return m_payload.toByteArray(); }
+    QString getStringValue() const { return m_payload.toString(); }
     quint16 getWordValue() const { return (quint16) (m_payload.toUInt() & 0xFFFF); }
     quint8 getByteValue() const { return (quint8) (m_payload.toUInt() & 0xFF); }
 
@@ -147,6 +152,7 @@ public:
         return m_tokens[token];
     }
 
+    QString getHtmlPrintableString();
 private:
 
     static QMap<quint16, QString> m_tokens;

src/ui/catalogwidget.cxx

@@ -61,7 +61,7 @@ void CatalogWidget::prepForNewDisk(QString filename, DiskFile *disk)
 
 QString CatalogWidget::createToolTip(FileDescriptiveEntry &fde) {
     QString retval;
-
+qDebug() << AppleString(fde.filename).printable().trimmed();
     retval += AppleString(fde.filename).printable().trimmed() + "\n";
     retval += QString("Type: %1\n").arg(fde.fileType());
     retval += QString("Sectors: %1 (%2 bytes)\n").arg(fde.lengthInSectors).arg(fde.lengthInSectors*256);

src/ui/viewers/applesoftfileviewer.cxx

@@ -12,8 +12,8 @@ ApplesoftFileViewer::ApplesoftFileViewer(QWidget *parent) :
     setWindowTitle(title);
 
     m_formatter = new ApplesoftFormatter(this);
-    m_formatter->setFlags(ApplesoftFormatter::PrettyFlags | ApplesoftFormatter::BreakAfterReturn);
+    //m_formatter->setFlags(ApplesoftFormatter::PrettyFlags | ApplesoftFormatter::BreakAfterReturn);
-
+    m_formatter->setFlags(ApplesoftFormatter::AllFlags);
     connect(ui->findButton,SIGNAL(clicked(bool)), SLOT(findText()));
     m_isFirstFind = true;
     ui->textArea->setUndoRedoEnabled(false);

src/ui/viewers/applesoftfileviewer.ui

@@ -53,8 +53,10 @@
     <widget class="QTextBrowser" name="textArea">
      <property name="font">
       <font>
-       <family>Courier 10 Pitch</family>
+       <family>Misc Fixed</family>
-       <pointsize>10</pointsize>
+       <pointsize>14</pointsize>
+       <weight>75</weight>
+       <bold>true</bold>
       </font>
      </property>
      <property name="acceptDrops">