Now parses strings and numbers into enhanced tokens, as well.

2016-10-12 16:50:49 -05:00 · 2016-10-12 16:50:49 -05:00 · 6125c0523c
parent 9da780252f
commit 6125c0523c
8 changed files with 314 additions and 46 deletions
--- a/src/applesoftfile/applesoftfile.cxx
+++ b/src/applesoftfile/applesoftfile.cxx
@ -1,5 +1,9 @@
 #include "applesoftfile.h"
 #include <QDebug>
+#include <QRegularExpression>
+#include <QRegularExpressionMatch>
+#include <QRegularExpressionMatchIterator>
+#include <QList>

 ApplesoftFile::ApplesoftFile(QByteArray data) : GenericFile(data)
 {
@ -32,6 +36,7 @@ QByteArray ApplesoftFile::rawData() {

 void ApplesoftFile::parse(quint16 start_address)
 {
+
    int idx = 0;
    quint8 val = 0;

@ -96,19 +101,25 @@ QByteArray ApplesoftFile::extraData()

 void Retokenizer::retokenize(ApplesoftLine &line)
 {
-    Q_UNUSED(line);
+    line.tokens = retokenizeRems(line.tokens);
+    line.tokens = retokenizeStrings(line.tokens);
+    line.tokens = retokenizeDataStatements(line.tokens);
+    line.tokens = retokenizeVariables(line.tokens);
+    line.tokens = retokenizeNumbers(line.tokens);


-    QVector<ApplesoftToken> replacements;
-    QVector<ApplesoftToken> tmptokens = line.tokens;
+}
+
+QVector<ApplesoftToken> Retokenizer::retokenizeRems(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle REMs
    ApplesoftToken token;
-
+    QVector<ApplesoftToken> replacements;
+    QVector<ApplesoftToken> tmptokens = datatokens;
    QByteArray buffer;

    bool inRem = false;

-    // Handle REMs
-
    while (!tmptokens.isEmpty())
    {
        token = tmptokens.takeFirst();
@ -132,14 +143,17 @@ void Retokenizer::retokenize(ApplesoftLine &line)
        inRem = false;
    }

-    line.tokens = replacements;
-    replacements.clear();
-
-    // Handle DATAs
+    return replacements;
+}

+QVector<ApplesoftToken> Retokenizer::retokenizeStrings(QVector<ApplesoftToken>&datatokens)
+{
    // Handle Strings
+    QVector<ApplesoftToken> replacements;
+    QVector<ApplesoftToken> tmptokens = datatokens;
+    QString buffer;
+    ApplesoftToken token;

-    tmptokens = line.tokens;
    bool inString = false;

    while (!tmptokens.isEmpty())
@ -148,45 +162,241 @@ void Retokenizer::retokenize(ApplesoftLine &line)
        if (token.getTokenId() >= 0x80)
        {
            replacements.append(token);
-            continue;
-        }
+         //   continue;
+        } else

-        if (token.getByteValue() == '"')
+        if (token.getWordValue() == '"')
        {
            if (!inString)
            {
                inString = true;
-                buffer.append(token.getByteValue());
-                continue;
+                buffer.append(token.getWordValue());
+              //  continue;
            }
            else
            {
-                buffer.append(token.getByteValue());
+                buffer.append(token.getWordValue());
                ApplesoftToken strtoken(ApplesoftToken::StringTokenVal, buffer);
                replacements.append(strtoken);
                buffer.clear();
                inString = false;
-                continue;
+//                continue;
            }
-        }
+        } else

        if (inString)
        {
-            buffer.append(token.getByteValue());
-            continue;
-        }
+            buffer.append(token.getWordValue());
+            // continue;
+        } else

        replacements.append(token);
    }

-    line.tokens = replacements;
-
+    return replacements;
 }

-QByteArray Retokenizer::retokenizePart(QByteArray part) {
-    QByteArray retval;
+QVector<ApplesoftToken> Retokenizer::retokenizeDataStatements(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle DATAs
+    QVector<ApplesoftToken> tmptokens = datatokens;
+    QVector<ApplesoftToken> replacements;
+    ApplesoftToken token;

-    Q_UNUSED(part);
+    QVector<ApplesoftToken> datatokenbuffer;
+    bool inData = false;
+    while (!tmptokens.isEmpty())
+    {
+        token = tmptokens.takeFirst();

+        if (!inData) {
+            replacements.append(token);
+            if (token.getTokenId() == ApplesoftToken::ASData)
+            {
+                inData = true;
+            }
+        }
+        else
+        {
+            datatokenbuffer.append(token);
+        }
+    }
+    if (inData) {
+        QVector<ApplesoftToken> dataTokens;
+        dataTokens = processDataPayload(datatokenbuffer);
+        replacements.append(dataTokens);
+        datatokenbuffer.clear();
+        inData = false;
+    }
+    return replacements;
+}
+
+
+QVector<ApplesoftToken> Retokenizer::processDataPayload(QVector<ApplesoftToken>& datatokens)
+{
+    QVector<ApplesoftToken> retval;
+
+    ApplesoftToken token;
+
+    QString stringbuffer;
+
+    while (!datatokens.isEmpty())
+    {
+        token = datatokens.takeFirst();
+        if (token.getTokenId() == ApplesoftToken::StringTokenVal)
+        {
+            ApplesoftToken newToken(ApplesoftToken::DataStringTokenVal, token.getStringValue());
+            retval.append(newToken);
+            continue;
+        }
+        if (token.getWordValue() == ',')
+        {
+            if (!stringbuffer.isEmpty())
+            {
+                ApplesoftToken datastrtoken(ApplesoftToken::DataStringTokenVal, stringbuffer);
+                retval.append(datastrtoken);
+                stringbuffer.clear();
+            }
+            retval.append(token);
+            continue;
+
+        }
+        stringbuffer.append(token.getWordValue());
+    }
+    if (!stringbuffer.isEmpty())
+    {
+        ApplesoftToken datastrtoken(ApplesoftToken::DataStringTokenVal, stringbuffer);
+        retval.append(datastrtoken);
+        stringbuffer.clear();
+    }
    return retval;
 }
+QVector<ApplesoftToken> Retokenizer::retokenizeVariables(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle variable names
+    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
+    ApplesoftToken token;
+
+    QRegularExpression varregexp("[A-Za-z][A-Za-z0-9]*[$%]?\\(?");
+
+    QString parsestring;
+    // Parse the tokens to find assist
+    for (int idx = 0; idx < tmptokens.count();idx++)
+    {
+        token = datatokens.at(idx);
+
+        if (token.getTokenId() < 0x0080 && token.getTokenId() > 0x0000)
+        {
+            parsestring.append(QChar(token.getWordValue()));
+        }
+        else
+        {
+            parsestring.append("_");
+        }
+    }
+    QList<QRegularExpressionMatch> matchstack;
+    QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
+ //   qDebug() << parsestring;
+    while (matches.hasNext()) {
+        QRegularExpressionMatch rematch = matches.next();
+        matchstack.push_front(rematch);
+
+//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
+//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
+    }
+
+    foreach(QRegularExpressionMatch rematch, matchstack)
+    {
+        QString text = rematch.captured(0);
+        int start = rematch.capturedStart();
+        int length = rematch.capturedLength();
+
+        quint16 tokentype = ApplesoftToken::FloatVarTokenVal;
+        if (text.contains("$")) {
+            tokentype = ApplesoftToken::StringVarTokenVal;
+        } else if (text.contains("%")) {
+            tokentype = ApplesoftToken::IntVarTokenVal;
+        }
+
+        if (text.contains("(")) {
+            if (tokentype == ApplesoftToken::FloatVarTokenVal) {
+                tokentype = ApplesoftToken::FloatAryVarTokenVal;
+            } else if (tokentype == ApplesoftToken::StringVarTokenVal) {
+                tokentype = ApplesoftToken::StringAryVarTokenVal;
+            } else {
+                tokentype = ApplesoftToken::IntAryVarTokenVal;
+            }
+        }
+
+        ApplesoftToken vartoken = ApplesoftToken(tokentype,text);
+
+        for (int idx = 0; idx < length; idx++)
+        {
+            tmptokens.removeAt(start);
+        }
+        tmptokens.insert(start,vartoken);
+
+    }
+
+    datatokens = tmptokens.toVector();
+    return datatokens;
+}
+
+QVector<ApplesoftToken> Retokenizer::retokenizeNumbers(QVector<ApplesoftToken>&datatokens)
+{
+    // Handle numbers
+    QList<ApplesoftToken> tmptokens = QList<ApplesoftToken>::fromVector(datatokens);
+    ApplesoftToken token;
+
+    QRegularExpression varregexp("[0-9]+(\\.[0-9]*)?");
+
+    QString parsestring;
+    // Parse the tokens to find assist
+    for (int idx = 0; idx < tmptokens.count();idx++)
+    {
+        token = datatokens.at(idx);
+
+        if (token.getTokenId() < 0x0080 && token.getTokenId() > 0x0000)
+        {
+            parsestring.append(QChar(token.getWordValue()));
+        }
+        else
+        {
+            parsestring.append("_");
+        }
+    }
+    QList<QRegularExpressionMatch> matchstack;
+    QRegularExpressionMatchIterator matches = varregexp.globalMatch(parsestring);
+//    qDebug() << parsestring;
+    while (matches.hasNext()) {
+        QRegularExpressionMatch rematch = matches.next();
+        matchstack.push_front(rematch);
+
+//        qDebug() << "Capture " << " = " << rematch.capturedTexts() << "From: " << rematch.capturedStart()
+//                 << "To: " << rematch.capturedEnd()-1 << "("<<rematch.capturedLength()<<")";
+    }
+
+    foreach(QRegularExpressionMatch rematch, matchstack)
+    {
+        QString text = rematch.captured(0);
+        int start = rematch.capturedStart();
+        int length = rematch.capturedLength();
+
+        quint16 tokentype = ApplesoftToken::IntegerTokenVal;
+        if (text.contains(".")) {
+            tokentype = ApplesoftToken::FloatTokenVal;
+        }
+
+        ApplesoftToken vartoken = ApplesoftToken(tokentype,text);
+
+        for (int idx = 0; idx < length; idx++)
+        {
+            tmptokens.removeAt(start);
+        }
+        tmptokens.insert(start,vartoken);
+
+    }
+
+    datatokens = tmptokens.toVector();
+    return datatokens;
+}
--- a/src/applesoftfile/applesoftfile.h
+++ b/src/applesoftfile/applesoftfile.h
@ -38,8 +38,15 @@ private:
 class Retokenizer {
 public:
    static void retokenize(ApplesoftLine &line);
+
 private:
-    QByteArray retokenizePart(QByteArray part);
+    static QVector<ApplesoftToken> retokenizeRems(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> retokenizeStrings(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> retokenizeDataStatements(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> processDataPayload(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> retokenizeVariables(QVector<ApplesoftToken> &datatokens);
+    static QVector<ApplesoftToken> retokenizeNumbers(QVector<ApplesoftToken> &datatokens);
 };

+
 #endif // APPLESOFTFILE_H
--- a/src/applesoftfile/applesoftformatter.cxx
+++ b/src/applesoftfile/applesoftformatter.cxx
@ -23,45 +23,45 @@ QString ApplesoftFormatter::formatText()

    foreach (ApplesoftLine line, m_file->getLines()) {
        QString linestring = QString("%1  ").arg(line.linenum,5,10,QChar(' '));
+
        retval.append(linestring);
        foreach (ApplesoftToken token, line.tokens)
        {
            QString tokenstr = token.getRawPrintableString();

            if (m_format_options.testFlag(BreakAfterReturn)) {
-                if (token.getByteValue() == ApplesoftToken::ASReturn)
+                if (token.getTokenId() == ApplesoftToken::ASReturn)
                {
                    tokenstr += "\n";
                }
            }

-#define DEBUGTOKENS
+#define noDEBUGTOKENS
 #ifdef DEBUGTOKENS
            if (token.getTokenId() >= 0x80)
            {
-                tokenstr = QString(" {%1} ").arg(uint16ToHex(token.getTokenId()));
-  //              tokenstr = " __ ";
-            }
-            else
-            {
-                tokenstr = QString("%1").arg(QChar(token.getByteValue()));
+                tokenstr = QString("{%1 (%2)}").arg(tokenstr).arg(uint16ToHex(token.getTokenId()));
+               // tokenstr = " __ ";
            }
+
 #endif

-            if (m_format_options.testFlag(ShowCtrlChars)) {
+   //         if (m_format_options.testFlag(ShowCtrlChars))
+            {
                tokenstr.replace(QChar(0x7f),QChar(0x2401));

                for (int idx = 1; idx <= 0x1f; idx++) {
                    if (idx == '\n') continue;
-                    if (idx == '\t') continue;
                    tokenstr.replace(QChar(idx),QChar(idx+0x2400));
+      //              tokenstr.replace(QChar(idx), QString("<%1>").arg(uint8ToHex(idx)));
                }
            }

-
            retval.append(tokenstr);
        }
        retval.append("\n");
+
    }
+
    return retval;
 }
--- a/src/applesoftfile/applesofttoken.cxx
+++ b/src/applesoftfile/applesofttoken.cxx
@ -89,16 +89,59 @@ void ApplesoftToken::setValue(QVariant value)
    m_payload = value;
 }

+QString ApplesoftToken::getHtmlPrintableString()
+{
+    QString baseval = getRawPrintableString().toHtmlEscaped();
+
+    if (getTokenId() <= 0x7f)
+        return QString("<font color=\"black\">%1</font>").arg(baseval);
+    if (getTokenId() <= 0xff)
+        return QString("<font color=\"red\">%1</font>").arg(baseval);
+    if (getTokenId() == ApplesoftToken::StringTokenVal)
+        return QString("<font color=\"black\">%1</font>").arg(baseval);
+    if (getTokenId() == ApplesoftToken::DataStringTokenVal)
+        return QString("<font color=\"green\">%1</font>").arg(baseval);
+    if (getTokenId() == ApplesoftToken::RemStringTokenVal)
+        return QString("<font color=\"grey\">%1</font>").arg(baseval);
+    if (getTokenId() == ApplesoftToken::IntegerTokenVal || getTokenId() == ApplesoftToken::FloatAryVarTokenVal)
+        return QString("<font color=\"blue\">%1</font>").arg(baseval);
+
+
+
+    return QString("<font color=\"orange\">%1</font>").arg(baseval);
+
+}
+
 QString ApplesoftToken::getRawPrintableString()
 {
    if (m_token_id == 0x00) {
        return "";
    } else if (m_token_id <= 0x7f) {
-        return QString(QChar(m_token_id));
+        return QString((m_token_id));
    } else if (m_token_id <= 0xff) {
        return m_tokens[m_token_id];
    } else if (m_token_id == StringTokenVal) {
-        return getByteStringValue();
+        return getStringValue();
+    } else if (m_token_id == RemStringTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == DataStringTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == IntegerTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == FloatTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == IntVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == IntAryVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == FloatVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == FloatAryVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == StringVarTokenVal) {
+        return getStringValue();
+    } else if (m_token_id == StringAryVarTokenVal) {
+        return getStringValue();
    } else {
        return "[temp undefined]";
    }
--- a/src/applesoftfile/applesofttoken.h
+++ b/src/applesoftfile/applesofttoken.h
@ -16,6 +16,8 @@ public:
    static const quint16 IntegerTokenVal      = 0x103;
    static const quint16 FloatTokenVal        = 0x104;

+
+
    static const quint16 IntVarTokenVal       = 0x105;
    static const quint16 IntAryVarTokenVal    = 0x106;

@ -25,6 +27,8 @@ public:
    static const quint16 StringVarTokenVal    = 0x109;
    static const quint16 StringAryVarTokenVal = 0x10A;

+
+
    static const quint16 ASEnd;      static const quint16 ASFor;
    static const quint16 ASNext;     static const quint16 ASData;
    static const quint16 ASInput;    static const quint16 ASDel;
@ -134,6 +138,7 @@ public:
    QVariant getValue() const { return m_payload; }

    QByteArray getByteStringValue() const { return m_payload.toByteArray(); }
+    QString getStringValue() const { return m_payload.toString(); }
    quint16 getWordValue() const { return (quint16) (m_payload.toUInt() & 0xFFFF); }
    quint8 getByteValue() const { return (quint8) (m_payload.toUInt() & 0xFF); }

@ -147,6 +152,7 @@ public:
        return m_tokens[token];
    }

+    QString getHtmlPrintableString();
 private:

    static QMap<quint16, QString> m_tokens;
--- a/src/ui/catalogwidget.cxx
+++ b/src/ui/catalogwidget.cxx
@ -61,7 +61,7 @@ void CatalogWidget::prepForNewDisk(QString filename, DiskFile *disk)

 QString CatalogWidget::createToolTip(FileDescriptiveEntry &fde) {
    QString retval;
-
+qDebug() << AppleString(fde.filename).printable().trimmed();
    retval += AppleString(fde.filename).printable().trimmed() + "\n";
    retval += QString("Type: %1\n").arg(fde.fileType());
    retval += QString("Sectors: %1 (%2 bytes)\n").arg(fde.lengthInSectors).arg(fde.lengthInSectors*256);
--- a/src/ui/viewers/applesoftfileviewer.cxx
+++ b/src/ui/viewers/applesoftfileviewer.cxx
@ -12,8 +12,8 @@ ApplesoftFileViewer::ApplesoftFileViewer(QWidget *parent) :
    setWindowTitle(title);

    m_formatter = new ApplesoftFormatter(this);
-    m_formatter->setFlags(ApplesoftFormatter::PrettyFlags | ApplesoftFormatter::BreakAfterReturn);
-
+    //m_formatter->setFlags(ApplesoftFormatter::PrettyFlags | ApplesoftFormatter::BreakAfterReturn);
+    m_formatter->setFlags(ApplesoftFormatter::AllFlags);
    connect(ui->findButton,SIGNAL(clicked(bool)), SLOT(findText()));
    m_isFirstFind = true;
    ui->textArea->setUndoRedoEnabled(false);
--- a/src/ui/viewers/applesoftfileviewer.ui
+++ b/src/ui/viewers/applesoftfileviewer.ui
@ -53,8 +53,10 @@
    <widget class="QTextBrowser" name="textArea">
     <property name="font">
      <font>
-       <family>Courier 10 Pitch</family>
-       <pointsize>10</pointsize>
+       <family>Misc Fixed</family>
+       <pointsize>14</pointsize>
+       <weight>75</weight>
+       <bold>true</bold>
      </font>
     </property>
     <property name="acceptDrops">