Changing disassembler to walk code paths

AppleSAWS.pro

@@ -109,7 +109,8 @@ HEADERS += \
     src/ui/widgets/LocationInfoDialog.h \
     src/binaryfile/EntryPoints.h \
     src/binaryfile/AssemblerSymbols.h \
-    src/binaryfile/AssemblerSymbolModel.h
+    src/binaryfile/AssemblerSymbolModel.h \
+    src/binaryfile/MemoryUsageMap.h
 
 FORMS += \
     src/ui/catalogwidget.ui \

src/binaryfile/MemoryUsageMap.h

@@ -0,0 +1,107 @@
+#ifndef MEMORYUSAGEMAP_H
+#define MEMORYUSAGEMAP_H
+
+#include <QFlag>
+#include <QFlags>
+#include <QVector>
+
+enum MemoryUsage {
+    Unknown            = 0x00000000,
+    Data               = 0x00000001,
+    Operation          = 0x00000002,
+  //  OperationArgHi     = 0x00000004,
+  //  OperationArgLo     = 0x00000008,
+    OperationArg       = 0x00000010,
+
+    RefAddressHi       = 0x00000100,
+    RefAddressLo       = 0x00000200,
+    RefAddress         = RefAddressLo | RefAddressHi,
+    ZeroPageRefAddress = 0x00000400,
+
+    InvalidOperation   = 0x00001000,
+    Break              = 0x00002000,
+    UndeterminedJump   = 0x00004000,
+    StopsExecution     = InvalidOperation | Break | UndeterminedJump,
+
+    EntryPointAddr     = 0x00010000,
+    BranchOffsetAddr   = 0x00020000,
+    Jump               = 0x00040000,
+    Return             = 0x00080000,
+};
+Q_DECLARE_FLAGS(MemoryUsages,MemoryUsage)
+
+
+class MemoryUsageMap : public QVector<MemoryUsages>
+{
+
+public:
+    MemoryUsageMap()
+    {
+        fill(MemoryUsage::Unknown,65536);
+    }
+
+    void clearData()
+    {
+        for (int idx = 0; idx < size(); idx++)
+        {
+            (*this)[idx] = MemoryUsage::Unknown;
+        }
+    }
+
+    void merge(const MemoryUsageMap &other)
+    {
+        if (other.size() != size())
+        {
+            qWarning("Mismatched size of MemoryMaps!");
+            return;
+        }
+        for (int idx = 0; idx < size(); idx++)
+        {
+            (*this)[idx] |= other[idx];
+        }
+    }
+
+    QList<quint16> addressesWhichContain(MemoryUsage usage)
+    {
+        QList<quint16> retval;
+        for (int idx = 0; idx < size(); idx++)
+        {
+            if (value(idx).testFlag(usage))
+            {
+                retval.append((quint16) idx);
+            }
+        }
+        return retval;
+    }
+
+private:
+    void clear();
+    void append(const MemoryUsages &);
+    void append(MemoryUsages &&);
+    void append(const QVector<MemoryUsages>);
+    void insert(int,const MemoryUsages &);
+    void insert(int,int,const MemoryUsages &);
+    void move(int,int);
+    void prepend(const MemoryUsages&);
+    void push_back(const MemoryUsages&);
+    void push_back(MemoryUsages&&);
+    void push_front(const MemoryUsages&);
+    void remove(int);
+    void remove(int,int);
+    void removeAll(const MemoryUsages&);
+    void removeAt(int);
+    void removeFirst();
+    void removeLast();
+    bool removeOne(const MemoryUsages&);
+    void reserve(int);
+    void resize(int);
+    void swap(QVector<MemoryUsages>&);
+    MemoryUsages &takeAt(int);
+    MemoryUsages &takeFirst();
+    MemoryUsages &takeLast();
+
+};
+
+Q_DECLARE_OPERATORS_FOR_FLAGS(MemoryUsages)
+
+#endif // MEMORYUSAGEMAP_H

src/binaryfile/disassembler.cxx

@@ -4,58 +4,182 @@
 #include <math.h>
 
 
-QList<DisassembledItem> Disassembler::disassemble(quint16 from, quint16 to) {
+Disassembler::Disassembler(QByteArray memimage)
-    QList<DisassembledItem> retval;
+{
+    m_memimage = memimage, makeOpcodeTable();
+    m_memusagemap.clearData();
+}
 
-    quint16 next = 0;
+QList<DisassembledItem> Disassembler::disassemble(quint16 from, quint16 to,bool processRecursively) {
+    QList<DisassembledItem> retval;
+    qDebug() << "Disassemble: From"<<uint16ToHex(from)<<"to"<<uint16ToHex(to);
+    //#define OLDDISSEM
+#ifdef OLDDISSEM
     for (int idx = from; idx <= to; )
     {
-        DisassembledItem item = disassembleOp(quint16(idx), &next);
+
+        DisassembledItem item;
+        disassembleOp(quint16(idx),item);
         retval.append(item);
-        idx = next ;
+        idx = item.nextContiguousAddress();
-        if (idx > 0xffff || (next < from)) {
+        if (idx > 0xffff || (idx < from)) {
             qDebug() << "Breaking.";
             break;
         }
     }
+#else
+    MemoryUsageMap memuse;
+
+    bool stopping = false;
+    quint16 next = from;
+
+    while (!stopping)
+    {
+        //   quint8 opcode = m_memimage[next];
+        //   qDebug() << "Opcode: " << uint8ToHex(opcode);
+        DisassembledItem item;
+        bool ok = false;
+
+        if (next >= from && next <= to)  //TODO: Remove this to not limit disassembly to program range
+            ok = disassembleOp(next,item,&memuse);
+
+        if (ok)
+        {
+            retval.append(item);
+
+            quint16 flow  = item.nextFlowAddress();
+            qDebug() << uint16ToHex(next) << uint16ToHex(flow);
+            if (item.isBranch())
+            {
+                qDebug() << "Is Branch";
+                if (!m_jumps.contains(item.targetAddress()))
+                {
+                    m_jumps.append(item.targetAddress());
+                }
+            }
+
+            if (item.isJsr() && !item.canNotFollow())
+            {
+                if (item.targetAddress() <= to) //TODO: Remove this to not limit disassembly to program range
+                    if (!m_jumps.contains(item.targetAddress()))
+                    {
+                        m_jumps.append(item.targetAddress());
+                    }
+            }
+
+            if (next <= to) //TODO: Remove this to not limit disassembly to program range
+            {
+                next = flow;
+                stopping = item.stopsProcessing();
+            }
+            else stopping = true;
+        }
+        else
+        {
+            stopping = true; // already processed this address
+        }
+
+        //     if (found.contains(next)) stopping = true;
+        if (next >= to) stopping = true;
+        //        if (stopping) {
+        //            qDebug() << "Stopping.  Stops processing: "
+        //                     << item.stopsProcessing()
+        //                     << ", next>=to: " << (next >= to)
+        //                     << ", alreadyFound: " << ((!ok)?"true":"false");
+        //        }
+    }
+
+    m_memusagemap.merge(memuse);
+
+    if (processRecursively)
+    while (m_jumps.size())
+    {
+        quint16 num = m_jumps.takeFirst();
+        if (!m_memusagemap[num].testFlag(Operation))
+        {
+            if (num >= from && num <= to) // TODO: remove this to not limit disassembly to program range
+            {
+                qDebug() << "Calling recursively to"<<uint16ToHex(num);
+                retval.append(disassemble(num,to,false));
+                qDebug() << "Return from recursive call.";
+            }
+        }
+    }
+
+#endif
+    qSort(retval);
+
+    //    QStringList hexdump;
+    //    foreach (quint16 adr,m_memusagemap.addressesWhichContain(Operation))
+    //    {
+    //        hexdump.append(uint16ToHex(adr));
+    //    }
+    //    qDebug() << "Operations:" << hexdump;
+
+    //    hexdump.clear();
+    //    foreach (quint16 adr,m_memusagemap.addressesWhichContain(OperationArg))
+    //    {
+    //        hexdump.append(uint16ToHex(adr));
+    //    }
+    //    qDebug() << "Operations Args:" << hexdump;
+
 
     return retval;
 }
 
-
+bool Disassembler::disassembleOp(quint16 address, DisassembledItem &retval, MemoryUsageMap *memuse)
-DisassembledItem Disassembler::disassembleOp(quint16 address, quint16 *nextAddress)
 {
-    DisassembledItem retval;
+
+    if (memuse)
+    {
+        if ((*memuse)[address].testFlag(Operation)) return false;
+    }
 
     quint8 opcode = m_memimage[address];
     AssyInstruction op = m_opcodeinfo[opcode];
+    retval.setInstruction(op);
+
+    if (opcode == 0x6C || opcode == 0x7c) // Indirect jumps
+        retval.setCanNotFollow(true);
 
     QString disassemblyLine;
     QString hexValueString;
     QByteArray hexValues;
     hexValues.append(opcode);
 
+    // Prepare Op Code arguments
+
     for (int idx = 1; idx < op.numArgs()+1; idx++) {
         quint8 val = m_memimage[address+idx];
         hexValues.append(val);
     }
-
+    if (memuse)
+    {
+        (*memuse)[address].setFlag(Operation);
+        for (int idx = 1; idx < op.numArgs()+1; idx++)
+        {
+            (*memuse)[address+idx].setFlag(OperationArg);
+        }
+    }
 
     quint16 argval = 0;
-    if (op.numArgs() == 1) { argval = (quint8) hexValues[1]; }
+    if (op.numArgs() == 1)
-    else if (op.numArgs() == 2) { argval = (quint8) hexValues[1] + ((quint8) hexValues[2] * 256); }
+        argval = (quint8) hexValues[1];
+    else if (op.numArgs() == 2)
+        argval = (quint8) hexValues[1] + ((quint8) hexValues[2] * 256);
 
     for (int idx = 0; idx < hexValues.length(); idx++) {
         hexValueString.append(QString("%1 ").arg((quint8) hexValues[idx],2,16,QChar('0')));
     }
     
-    if (nextAddress) {
+    retval.setNextContiguousAddress(address+1+op.numArgs());
-        *nextAddress = address + 1 + op.numArgs();
+    retval.setNextFlowAddress(address+1+op.numArgs());
-    }
 
+    // Disassemble instruction
     switch (op.addressMode()) {
     case AM_InvalidOp: {
         disassemblyLine = op.mnemonic();
+        retval.setIsInvalidOp(true);
         break;
     }
     case AM_Absolute:{
@@ -140,6 +264,7 @@ DisassembledItem Disassembler::disassembleOp(quint16 address, quint16 *nextAddre
     }
     default:{
         disassemblyLine = op.mnemonic();
+        retval.setIsInvalidOp(true);
         qDebug() << "Unhandled Address Mode: " << op.addressMode();
         break;
     }
@@ -149,11 +274,19 @@ DisassembledItem Disassembler::disassembleOp(quint16 address, quint16 *nextAddre
         retval.setTargetAddress(hexValues[2]*256 + hexValues[1]);
     }
 
+    if (opcode == 0x4c)
+    {
+        qDebug() << "JMP: Setting next flow address to"<<uint16ToHex(hexValues[2]*256 + hexValues[1]);
+        retval.setNextFlowAddress(hexValues[2]*256 + hexValues[1]);
+    }
+
+
     retval.setAddress(address);
     retval.setDisassembledString(disassemblyLine.toUpper());
     retval.setHexString(hexValueString.trimmed().toUpper().leftJustified(12,' '));
     retval.setHexValues(hexValues);
-    return retval;
+
+    return true;
 }
 
 void Disassembler::makeOpcodeTable()
@@ -743,6 +876,8 @@ DisassembledItem::DisassembledItem(AssyInstruction instr) {
 
 void DisassembledItem::setInstruction(AssyInstruction instr) {
     m_instruction = instr;
+    //    qDebug() << "Set instruction: " << uint8ToHex(instr.opcode());
+    //    qDebug() << " Copied instr:" << m_instruction.debugStr();
     if (instr.opcode() == 0x20) { m_is_jsr = true; }
     if (instr.opcode() == 0x10) { m_is_branch = true; } // BPL
     if (instr.opcode() == 0x30) { m_is_branch = true; } // BMI
@@ -772,8 +907,13 @@ QString DisassembledItem::disassembledString() {
 
 void DisassembledItem::init() {
     m_address = m_target_address = 0;
+    m_nextContiguousAddress = 0;
+    m_nextFlowAddress = 0;
     m_is_jump = m_is_branch = m_is_jsr = false;
     m_unknown_ta = true;
     m_raw_arg = 0;
     m_has_arg = false;
+    m_canNotFollow = false;
+    m_isInvalidOp = false;
+
 }

src/binaryfile/disassembler.h

@@ -1,9 +1,13 @@
 #ifndef DISASSEMBLER_H
 #define DISASSEMBLER_H
 
+#include "util.h"
+#include "MemoryUsageMap.h"
+
 #include <QByteArray>
 #include <QStringList>
 #include <QHash>
+#include <QDebug>
 
 enum AddressMode {
     AM_InvalidOp,
@@ -39,6 +43,8 @@ public:
 
     quint8 numArgs();
 
+    QString debugStr() { return QString("%1 %2 %3").arg(uint8ToHex(m_opcode)).arg(m_mnemonic).arg(m_addressMode); }
+
 private:
     QString m_mnemonic;
     quint8 m_opcode;
@@ -52,6 +58,7 @@ public:
 
     DisassembledItem(AssyInstruction instr);
 
+    bool operator<(const DisassembledItem &other) const { return (address() < other.address()); }
 
     void setInstruction(AssyInstruction instr);
 
@@ -64,6 +71,8 @@ public:
     void setJsr(bool jsr) { m_is_jsr = jsr; }
     void setTargetAddress(quint16 ta) { m_unknown_ta = false; m_target_address = ta; }
     void setRawArgument(quint16 arg) { m_has_arg = true; m_raw_arg = arg; }
+    void setCanNotFollow(bool canNotFollow) { m_canNotFollow = canNotFollow; }
+    void setIsInvalidOp(bool isInvalid) { m_isInvalidOp = isInvalid; }
 
     AssyInstruction assyInstruction() const { return m_instruction; }
     QString rawDisassembledString() const { return m_disassembly_text; }
@@ -75,8 +84,24 @@ public:
     bool isBranch() const { return m_is_branch; }
     bool isJump() const { return m_is_jump; }
     bool isJsr() const { return m_is_jsr; }
-    bool isReturn() { return (m_instruction.opcode() == 0x60); }
+    bool isReturn() { return (m_instruction.opcode() == 0x60) || m_instruction.opcode() == 0x40; }
     bool isBreak() { return (m_instruction.opcode() == 0x00); }
+    bool isInvalidOp() { return m_isInvalidOp; }
+
+    bool canNotFollow() { return m_canNotFollow; }
+
+    bool stopsProcessing() {
+        if (isReturn()) qDebug() << "Is Return";
+        if (isInvalidOp()) qDebug() << "Is Invalid Op" << uint8ToHex(m_instruction.opcode());
+        if (canNotFollow()) qDebug() << "Can not follow indirect jump";
+        if (isBreak()) qDebug() << "Is Break";
+        return isBreak() || isInvalidOp() || isReturn() || canNotFollow(); }
+
+    quint16 nextContiguousAddress() { return m_nextContiguousAddress; }
+    quint16 nextFlowAddress() { return m_nextFlowAddress; }
+
+    void setNextContiguousAddress(quint16 addr) { m_nextContiguousAddress = addr; }
+    void setNextFlowAddress(quint16 addr) { m_nextFlowAddress = addr; }
 
     quint16 targetAddress() const { return m_target_address; }
     bool hasArg() const { return m_has_arg; }
@@ -89,17 +114,22 @@ private:
     void init();
 
     quint16 m_address;
+    quint16 m_nextContiguousAddress;
+    quint16 m_nextFlowAddress;
+    quint16 m_target_address;
+
     QByteArray m_hexvalues;
     QString m_disassembly_text;
     QString m_hexstring;
     bool m_is_branch;
     bool m_is_jump;
     bool m_is_jsr;
-    quint16 m_target_address;
     AssyInstruction m_instruction;
     bool m_unknown_ta;
     quint16 m_raw_arg;
     bool m_has_arg;
+    bool m_isInvalidOp;
+    bool m_canNotFollow;
 };
 
 
@@ -108,23 +138,30 @@ private:
 class Disassembler
 {
 public:
-    Disassembler(QByteArray memimage) { m_memimage = memimage, makeOpcodeTable(); }
+    Disassembler(QByteArray memimage);
 
     enum ProcessorType {
         P6502,
         P65C02
     };
 
-    QList<DisassembledItem> disassemble(quint16 from, quint16 to);
+    QList<DisassembledItem> disassemble(quint16 from, quint16 to,bool processRecursively = true);
+
+    MemoryUsageMap *memoryUsageMap() { return &m_memusagemap; }
 
 private:
 
-    DisassembledItem disassembleOp(quint16 address, quint16 *nextAddress = 0);
+    bool disassembleOp(quint16 address, DisassembledItem &retval, MemoryUsageMap *memuse = Q_NULLPTR);
     void makeOpcodeTable();
 
     QHash<quint8,AssyInstruction> m_opcodeinfo;
     QByteArray m_memimage;
 
+    QList<quint16> m_jumps;
+ //   QList<quint16> found;
+
+    MemoryUsageMap m_memusagemap;
+
 };
 
 #endif // DISASSEMBLER_H

src/ui/widgets/DisassemblerMetadataDialog.cpp

@@ -112,7 +112,6 @@ void DisassemblerMetadataDialog::handleRemoveSymbolButton()
 
 }
 
-
 void DisassemblerMetadataDialog::processSymbols()
 {
     m_as->doTestData();