Changing disassembler to walk code paths

This commit is contained in:
Mark Long 2016-10-26 11:32:48 -05:00
parent ae7b8c83dd
commit 18ca84b42d
5 changed files with 307 additions and 23 deletions

View File

@ -109,7 +109,8 @@ HEADERS += \
src/ui/widgets/LocationInfoDialog.h \ src/ui/widgets/LocationInfoDialog.h \
src/binaryfile/EntryPoints.h \ src/binaryfile/EntryPoints.h \
src/binaryfile/AssemblerSymbols.h \ src/binaryfile/AssemblerSymbols.h \
src/binaryfile/AssemblerSymbolModel.h src/binaryfile/AssemblerSymbolModel.h \
src/binaryfile/MemoryUsageMap.h
FORMS += \ FORMS += \
src/ui/catalogwidget.ui \ src/ui/catalogwidget.ui \

View File

@ -0,0 +1,107 @@
#ifndef MEMORYUSAGEMAP_H
#define MEMORYUSAGEMAP_H
#include <QFlag>
#include <QFlags>
#include <QVector>
enum MemoryUsage {
Unknown = 0x00000000,
Data = 0x00000001,
Operation = 0x00000002,
// OperationArgHi = 0x00000004,
// OperationArgLo = 0x00000008,
OperationArg = 0x00000010,
RefAddressHi = 0x00000100,
RefAddressLo = 0x00000200,
RefAddress = RefAddressLo | RefAddressHi,
ZeroPageRefAddress = 0x00000400,
InvalidOperation = 0x00001000,
Break = 0x00002000,
UndeterminedJump = 0x00004000,
StopsExecution = InvalidOperation | Break | UndeterminedJump,
EntryPointAddr = 0x00010000,
BranchOffsetAddr = 0x00020000,
Jump = 0x00040000,
Return = 0x00080000,
};
Q_DECLARE_FLAGS(MemoryUsages,MemoryUsage)
class MemoryUsageMap : public QVector<MemoryUsages>
{
public:
MemoryUsageMap()
{
fill(MemoryUsage::Unknown,65536);
}
void clearData()
{
for (int idx = 0; idx < size(); idx++)
{
(*this)[idx] = MemoryUsage::Unknown;
}
}
void merge(const MemoryUsageMap &other)
{
if (other.size() != size())
{
qWarning("Mismatched size of MemoryMaps!");
return;
}
for (int idx = 0; idx < size(); idx++)
{
(*this)[idx] |= other[idx];
}
}
QList<quint16> addressesWhichContain(MemoryUsage usage)
{
QList<quint16> retval;
for (int idx = 0; idx < size(); idx++)
{
if (value(idx).testFlag(usage))
{
retval.append((quint16) idx);
}
}
return retval;
}
private:
void clear();
void append(const MemoryUsages &);
void append(MemoryUsages &&);
void append(const QVector<MemoryUsages>);
void insert(int,const MemoryUsages &);
void insert(int,int,const MemoryUsages &);
void move(int,int);
void prepend(const MemoryUsages&);
void push_back(const MemoryUsages&);
void push_back(MemoryUsages&&);
void push_front(const MemoryUsages&);
void remove(int);
void remove(int,int);
void removeAll(const MemoryUsages&);
void removeAt(int);
void removeFirst();
void removeLast();
bool removeOne(const MemoryUsages&);
void reserve(int);
void resize(int);
void swap(QVector<MemoryUsages>&);
MemoryUsages &takeAt(int);
MemoryUsages &takeFirst();
MemoryUsages &takeLast();
};
Q_DECLARE_OPERATORS_FOR_FLAGS(MemoryUsages)
#endif // MEMORYUSAGEMAP_H

View File

@ -4,58 +4,182 @@
#include <math.h> #include <math.h>
QList<DisassembledItem> Disassembler::disassemble(quint16 from, quint16 to) { Disassembler::Disassembler(QByteArray memimage)
QList<DisassembledItem> retval; {
m_memimage = memimage, makeOpcodeTable();
m_memusagemap.clearData();
}
quint16 next = 0; QList<DisassembledItem> Disassembler::disassemble(quint16 from, quint16 to,bool processRecursively) {
QList<DisassembledItem> retval;
qDebug() << "Disassemble: From"<<uint16ToHex(from)<<"to"<<uint16ToHex(to);
//#define OLDDISSEM
#ifdef OLDDISSEM
for (int idx = from; idx <= to; ) for (int idx = from; idx <= to; )
{ {
DisassembledItem item = disassembleOp(quint16(idx), &next);
DisassembledItem item;
disassembleOp(quint16(idx),item);
retval.append(item); retval.append(item);
idx = next ; idx = item.nextContiguousAddress();
if (idx > 0xffff || (next < from)) { if (idx > 0xffff || (idx < from)) {
qDebug() << "Breaking."; qDebug() << "Breaking.";
break; break;
} }
} }
#else
MemoryUsageMap memuse;
bool stopping = false;
quint16 next = from;
while (!stopping)
{
// quint8 opcode = m_memimage[next];
// qDebug() << "Opcode: " << uint8ToHex(opcode);
DisassembledItem item;
bool ok = false;
if (next >= from && next <= to) //TODO: Remove this to not limit disassembly to program range
ok = disassembleOp(next,item,&memuse);
if (ok)
{
retval.append(item);
quint16 flow = item.nextFlowAddress();
qDebug() << uint16ToHex(next) << uint16ToHex(flow);
if (item.isBranch())
{
qDebug() << "Is Branch";
if (!m_jumps.contains(item.targetAddress()))
{
m_jumps.append(item.targetAddress());
}
}
if (item.isJsr() && !item.canNotFollow())
{
if (item.targetAddress() <= to) //TODO: Remove this to not limit disassembly to program range
if (!m_jumps.contains(item.targetAddress()))
{
m_jumps.append(item.targetAddress());
}
}
if (next <= to) //TODO: Remove this to not limit disassembly to program range
{
next = flow;
stopping = item.stopsProcessing();
}
else stopping = true;
}
else
{
stopping = true; // already processed this address
}
// if (found.contains(next)) stopping = true;
if (next >= to) stopping = true;
// if (stopping) {
// qDebug() << "Stopping. Stops processing: "
// << item.stopsProcessing()
// << ", next>=to: " << (next >= to)
// << ", alreadyFound: " << ((!ok)?"true":"false");
// }
}
m_memusagemap.merge(memuse);
if (processRecursively)
while (m_jumps.size())
{
quint16 num = m_jumps.takeFirst();
if (!m_memusagemap[num].testFlag(Operation))
{
if (num >= from && num <= to) // TODO: remove this to not limit disassembly to program range
{
qDebug() << "Calling recursively to"<<uint16ToHex(num);
retval.append(disassemble(num,to,false));
qDebug() << "Return from recursive call.";
}
}
}
#endif
qSort(retval);
// QStringList hexdump;
// foreach (quint16 adr,m_memusagemap.addressesWhichContain(Operation))
// {
// hexdump.append(uint16ToHex(adr));
// }
// qDebug() << "Operations:" << hexdump;
// hexdump.clear();
// foreach (quint16 adr,m_memusagemap.addressesWhichContain(OperationArg))
// {
// hexdump.append(uint16ToHex(adr));
// }
// qDebug() << "Operations Args:" << hexdump;
return retval; return retval;
} }
bool Disassembler::disassembleOp(quint16 address, DisassembledItem &retval, MemoryUsageMap *memuse)
DisassembledItem Disassembler::disassembleOp(quint16 address, quint16 *nextAddress)
{ {
DisassembledItem retval;
if (memuse)
{
if ((*memuse)[address].testFlag(Operation)) return false;
}
quint8 opcode = m_memimage[address]; quint8 opcode = m_memimage[address];
AssyInstruction op = m_opcodeinfo[opcode]; AssyInstruction op = m_opcodeinfo[opcode];
retval.setInstruction(op);
if (opcode == 0x6C || opcode == 0x7c) // Indirect jumps
retval.setCanNotFollow(true);
QString disassemblyLine; QString disassemblyLine;
QString hexValueString; QString hexValueString;
QByteArray hexValues; QByteArray hexValues;
hexValues.append(opcode); hexValues.append(opcode);
// Prepare Op Code arguments
for (int idx = 1; idx < op.numArgs()+1; idx++) { for (int idx = 1; idx < op.numArgs()+1; idx++) {
quint8 val = m_memimage[address+idx]; quint8 val = m_memimage[address+idx];
hexValues.append(val); hexValues.append(val);
} }
if (memuse)
{
(*memuse)[address].setFlag(Operation);
for (int idx = 1; idx < op.numArgs()+1; idx++)
{
(*memuse)[address+idx].setFlag(OperationArg);
}
}
quint16 argval = 0; quint16 argval = 0;
if (op.numArgs() == 1) { argval = (quint8) hexValues[1]; } if (op.numArgs() == 1)
else if (op.numArgs() == 2) { argval = (quint8) hexValues[1] + ((quint8) hexValues[2] * 256); } argval = (quint8) hexValues[1];
else if (op.numArgs() == 2)
argval = (quint8) hexValues[1] + ((quint8) hexValues[2] * 256);
for (int idx = 0; idx < hexValues.length(); idx++) { for (int idx = 0; idx < hexValues.length(); idx++) {
hexValueString.append(QString("%1 ").arg((quint8) hexValues[idx],2,16,QChar('0'))); hexValueString.append(QString("%1 ").arg((quint8) hexValues[idx],2,16,QChar('0')));
} }
if (nextAddress) { retval.setNextContiguousAddress(address+1+op.numArgs());
*nextAddress = address + 1 + op.numArgs(); retval.setNextFlowAddress(address+1+op.numArgs());
}
// Disassemble instruction
switch (op.addressMode()) { switch (op.addressMode()) {
case AM_InvalidOp: { case AM_InvalidOp: {
disassemblyLine = op.mnemonic(); disassemblyLine = op.mnemonic();
retval.setIsInvalidOp(true);
break; break;
} }
case AM_Absolute:{ case AM_Absolute:{
@ -140,6 +264,7 @@ DisassembledItem Disassembler::disassembleOp(quint16 address, quint16 *nextAddre
} }
default:{ default:{
disassemblyLine = op.mnemonic(); disassemblyLine = op.mnemonic();
retval.setIsInvalidOp(true);
qDebug() << "Unhandled Address Mode: " << op.addressMode(); qDebug() << "Unhandled Address Mode: " << op.addressMode();
break; break;
} }
@ -149,11 +274,19 @@ DisassembledItem Disassembler::disassembleOp(quint16 address, quint16 *nextAddre
retval.setTargetAddress(hexValues[2]*256 + hexValues[1]); retval.setTargetAddress(hexValues[2]*256 + hexValues[1]);
} }
if (opcode == 0x4c)
{
qDebug() << "JMP: Setting next flow address to"<<uint16ToHex(hexValues[2]*256 + hexValues[1]);
retval.setNextFlowAddress(hexValues[2]*256 + hexValues[1]);
}
retval.setAddress(address); retval.setAddress(address);
retval.setDisassembledString(disassemblyLine.toUpper()); retval.setDisassembledString(disassemblyLine.toUpper());
retval.setHexString(hexValueString.trimmed().toUpper().leftJustified(12,' ')); retval.setHexString(hexValueString.trimmed().toUpper().leftJustified(12,' '));
retval.setHexValues(hexValues); retval.setHexValues(hexValues);
return retval;
return true;
} }
void Disassembler::makeOpcodeTable() void Disassembler::makeOpcodeTable()
@ -743,6 +876,8 @@ DisassembledItem::DisassembledItem(AssyInstruction instr) {
void DisassembledItem::setInstruction(AssyInstruction instr) { void DisassembledItem::setInstruction(AssyInstruction instr) {
m_instruction = instr; m_instruction = instr;
// qDebug() << "Set instruction: " << uint8ToHex(instr.opcode());
// qDebug() << " Copied instr:" << m_instruction.debugStr();
if (instr.opcode() == 0x20) { m_is_jsr = true; } if (instr.opcode() == 0x20) { m_is_jsr = true; }
if (instr.opcode() == 0x10) { m_is_branch = true; } // BPL if (instr.opcode() == 0x10) { m_is_branch = true; } // BPL
if (instr.opcode() == 0x30) { m_is_branch = true; } // BMI if (instr.opcode() == 0x30) { m_is_branch = true; } // BMI
@ -772,8 +907,13 @@ QString DisassembledItem::disassembledString() {
void DisassembledItem::init() { void DisassembledItem::init() {
m_address = m_target_address = 0; m_address = m_target_address = 0;
m_nextContiguousAddress = 0;
m_nextFlowAddress = 0;
m_is_jump = m_is_branch = m_is_jsr = false; m_is_jump = m_is_branch = m_is_jsr = false;
m_unknown_ta = true; m_unknown_ta = true;
m_raw_arg = 0; m_raw_arg = 0;
m_has_arg = false; m_has_arg = false;
m_canNotFollow = false;
m_isInvalidOp = false;
} }

View File

@ -1,9 +1,13 @@
#ifndef DISASSEMBLER_H #ifndef DISASSEMBLER_H
#define DISASSEMBLER_H #define DISASSEMBLER_H
#include "util.h"
#include "MemoryUsageMap.h"
#include <QByteArray> #include <QByteArray>
#include <QStringList> #include <QStringList>
#include <QHash> #include <QHash>
#include <QDebug>
enum AddressMode { enum AddressMode {
AM_InvalidOp, AM_InvalidOp,
@ -39,6 +43,8 @@ public:
quint8 numArgs(); quint8 numArgs();
QString debugStr() { return QString("%1 %2 %3").arg(uint8ToHex(m_opcode)).arg(m_mnemonic).arg(m_addressMode); }
private: private:
QString m_mnemonic; QString m_mnemonic;
quint8 m_opcode; quint8 m_opcode;
@ -52,6 +58,7 @@ public:
DisassembledItem(AssyInstruction instr); DisassembledItem(AssyInstruction instr);
bool operator<(const DisassembledItem &other) const { return (address() < other.address()); }
void setInstruction(AssyInstruction instr); void setInstruction(AssyInstruction instr);
@ -64,6 +71,8 @@ public:
void setJsr(bool jsr) { m_is_jsr = jsr; } void setJsr(bool jsr) { m_is_jsr = jsr; }
void setTargetAddress(quint16 ta) { m_unknown_ta = false; m_target_address = ta; } void setTargetAddress(quint16 ta) { m_unknown_ta = false; m_target_address = ta; }
void setRawArgument(quint16 arg) { m_has_arg = true; m_raw_arg = arg; } void setRawArgument(quint16 arg) { m_has_arg = true; m_raw_arg = arg; }
void setCanNotFollow(bool canNotFollow) { m_canNotFollow = canNotFollow; }
void setIsInvalidOp(bool isInvalid) { m_isInvalidOp = isInvalid; }
AssyInstruction assyInstruction() const { return m_instruction; } AssyInstruction assyInstruction() const { return m_instruction; }
QString rawDisassembledString() const { return m_disassembly_text; } QString rawDisassembledString() const { return m_disassembly_text; }
@ -75,8 +84,24 @@ public:
bool isBranch() const { return m_is_branch; } bool isBranch() const { return m_is_branch; }
bool isJump() const { return m_is_jump; } bool isJump() const { return m_is_jump; }
bool isJsr() const { return m_is_jsr; } bool isJsr() const { return m_is_jsr; }
bool isReturn() { return (m_instruction.opcode() == 0x60); } bool isReturn() { return (m_instruction.opcode() == 0x60) || m_instruction.opcode() == 0x40; }
bool isBreak() { return (m_instruction.opcode() == 0x00); } bool isBreak() { return (m_instruction.opcode() == 0x00); }
bool isInvalidOp() { return m_isInvalidOp; }
bool canNotFollow() { return m_canNotFollow; }
bool stopsProcessing() {
if (isReturn()) qDebug() << "Is Return";
if (isInvalidOp()) qDebug() << "Is Invalid Op" << uint8ToHex(m_instruction.opcode());
if (canNotFollow()) qDebug() << "Can not follow indirect jump";
if (isBreak()) qDebug() << "Is Break";
return isBreak() || isInvalidOp() || isReturn() || canNotFollow(); }
quint16 nextContiguousAddress() { return m_nextContiguousAddress; }
quint16 nextFlowAddress() { return m_nextFlowAddress; }
void setNextContiguousAddress(quint16 addr) { m_nextContiguousAddress = addr; }
void setNextFlowAddress(quint16 addr) { m_nextFlowAddress = addr; }
quint16 targetAddress() const { return m_target_address; } quint16 targetAddress() const { return m_target_address; }
bool hasArg() const { return m_has_arg; } bool hasArg() const { return m_has_arg; }
@ -89,17 +114,22 @@ private:
void init(); void init();
quint16 m_address; quint16 m_address;
quint16 m_nextContiguousAddress;
quint16 m_nextFlowAddress;
quint16 m_target_address;
QByteArray m_hexvalues; QByteArray m_hexvalues;
QString m_disassembly_text; QString m_disassembly_text;
QString m_hexstring; QString m_hexstring;
bool m_is_branch; bool m_is_branch;
bool m_is_jump; bool m_is_jump;
bool m_is_jsr; bool m_is_jsr;
quint16 m_target_address;
AssyInstruction m_instruction; AssyInstruction m_instruction;
bool m_unknown_ta; bool m_unknown_ta;
quint16 m_raw_arg; quint16 m_raw_arg;
bool m_has_arg; bool m_has_arg;
bool m_isInvalidOp;
bool m_canNotFollow;
}; };
@ -108,23 +138,30 @@ private:
class Disassembler class Disassembler
{ {
public: public:
Disassembler(QByteArray memimage) { m_memimage = memimage, makeOpcodeTable(); } Disassembler(QByteArray memimage);
enum ProcessorType { enum ProcessorType {
P6502, P6502,
P65C02 P65C02
}; };
QList<DisassembledItem> disassemble(quint16 from, quint16 to); QList<DisassembledItem> disassemble(quint16 from, quint16 to,bool processRecursively = true);
MemoryUsageMap *memoryUsageMap() { return &m_memusagemap; }
private: private:
DisassembledItem disassembleOp(quint16 address, quint16 *nextAddress = 0); bool disassembleOp(quint16 address, DisassembledItem &retval, MemoryUsageMap *memuse = Q_NULLPTR);
void makeOpcodeTable(); void makeOpcodeTable();
QHash<quint8,AssyInstruction> m_opcodeinfo; QHash<quint8,AssyInstruction> m_opcodeinfo;
QByteArray m_memimage; QByteArray m_memimage;
QList<quint16> m_jumps;
// QList<quint16> found;
MemoryUsageMap m_memusagemap;
}; };
#endif // DISASSEMBLER_H #endif // DISASSEMBLER_H

View File

@ -112,7 +112,6 @@ void DisassemblerMetadataDialog::handleRemoveSymbolButton()
} }
void DisassemblerMetadataDialog::processSymbols() void DisassemblerMetadataDialog::processSymbols()
{ {
m_as->doTestData(); m_as->doTestData();