Completely revamp optimization algorithms for safety and ability

examples/hello_commodore.cpp

@@ -24,21 +24,6 @@ static void puts(uint8_t x, uint8_t y, std::string_view str) {
 }
 
 int main() {
-  std::uint8_t x = 0;
+  puts(15, 10, "hello commodore!");
-  std::uint8_t y = 0;
-
-  while (true) {
-    puts(x, y, "hello commodore!");
-    x += 3;
-    ++y;
-    if (x > 26) {
-      x = 0;
-    }
-
-    if (y>25) {
-      y = 0;
-    }
-
-  }
 }
 

include/6502.hpp

@@ -20,15 +20,20 @@ struct mos6502 : ASMLine
     bne,
     bpl,
 
+    cpx,
     cpy,
     cmp,
     clc,
 
     dec,
+    dex,
+    dey,
 
     eor,
 
     inc,
+    inx,
+    iny,
 
     jmp,
     jsr,
@@ -79,12 +84,17 @@ struct mos6502 : ASMLine
     case OpCode::AND:
     case OpCode::asl:
     case OpCode::bit:
+    case OpCode::cpx:
     case OpCode::cpy:
     case OpCode::cmp:
     case OpCode::clc:
     case OpCode::dec:
+    case OpCode::dex:
     case OpCode::eor:
     case OpCode::inc:
+    case OpCode::inx:
+    case OpCode::iny:
+    case OpCode::dey:
     case OpCode::jmp:
     case OpCode::jsr:
     case OpCode::lda:
@@ -124,6 +134,7 @@ struct mos6502 : ASMLine
     case OpCode::bit:
     case OpCode::cmp:
     case OpCode::cpy:
+    case OpCode::cpx:
       return true;
     case OpCode::adc:
     case OpCode::AND:
@@ -136,8 +147,12 @@ struct mos6502 : ASMLine
     case OpCode::bcs:
     case OpCode::clc:
     case OpCode::dec:
+    case OpCode::dex:
     case OpCode::eor:
     case OpCode::inc:
+    case OpCode::inx:
+    case OpCode::iny:
+    case OpCode::dey:
     case OpCode::jmp:
     case OpCode::jsr:
     case OpCode::lda:
@@ -231,6 +246,11 @@ struct mos6502 : ASMLine
     case OpCode::bcc: return "bcc";
     case OpCode::bcs: return "bcs";
     case OpCode::nop: return "nop";
+    case OpCode::inx: return "inx";
+    case OpCode::dex: return "dex";
+    case OpCode::cpx: return "cpx";
+    case OpCode::dey: return "dey";
+    case OpCode::iny: return "iny";
     case OpCode::unknown: return "";
     }
 

include/optimizer.hpp

@@ -3,21 +3,255 @@
 
 #include "6502.hpp"
 #include "personality.hpp"
+#include <span>
 #include <vector>
 
-bool optimize(std::vector<mos6502> &instructions, const Personality &personality)
+
+constexpr bool consume_directives(auto &begin, const auto &end)
 {
-  // return false;
+  if (begin != end && begin->type == ASMLine::Type::Directive) {
+    ++begin;
+    return true;
+  }
+  return false;
+}
 
-  if (instructions.size() < 2) { return false; }
 
-  const auto next_instruction = [&instructions](auto i) {
+constexpr bool consume_labels(auto &begin, const auto &end)
-    do {
+{
-      ++i;
+  if (begin != end && begin->type == ASMLine::Type::Label) {
-    } while (i < instructions.size() && instructions[i].type == ASMLine::Type::Directive);
+    ++begin;
-    return i;
+    return true;
+  }
+  return false;
+}
+
+constexpr bool is_opcode(const mos6502 &op, const auto... opcodes) { return ((op.opcode == opcodes) || ...); }
+
+constexpr bool is_end_of_block(const auto &begin)
+{
+  if (begin->type == ASMLine::Type::Label) { return true; }
+
+  return is_opcode(*begin,
+    mos6502::OpCode::jsr,
+    mos6502::OpCode::jmp,
+    mos6502::OpCode::bcc,
+    mos6502::OpCode::bcs,
+    mos6502::OpCode::beq,
+    mos6502::OpCode::bne,
+    mos6502::OpCode::bpl);
+}
+
+constexpr bool consume_end_of_block(auto &begin, const auto &end)
+{
+  if (begin != end && is_end_of_block(begin)) {
+    ++begin;
+    return true;
+  }
+  return false;
+}
+
+static std::vector<std::span<mos6502>> get_optimizable_blocks(std::vector<mos6502> &statements)
+{
+  std::vector<std::span<mos6502>> blocks;
+
+  auto begin = std::begin(statements);
+  auto end = std::end(statements);
+
+  const auto find_end_of_block = [](auto &find_begin, const auto &find_end) {
+    while (find_begin != find_end) {
+      if (is_end_of_block(find_begin)) { return; }
+      ++find_begin;
+    }
   };
 
+  while (begin != end) {
+    while (consume_end_of_block(begin, end) || consume_directives(begin, end) || consume_labels(begin, end)) {}
+
+    const auto block_start = begin;
+    find_end_of_block(begin, end);
+
+    blocks.emplace_back(block_start, begin);
+  }
+
+  return blocks;
+}
+
+static bool is_virtual_register_op(const mos6502 &op, const Personality &personality)
+{
+  for (int i = 0; i < 32; ++i) {
+    if (personality.get_register(i).value == op.op.value) { return true; }
+  }
+
+  return false;
+}
+
+static bool optimize_dead_tax(std::span<mos6502> &block)
+{
+  for (auto itr = block.begin(); itr != block.end(); ++itr) {
+    if (is_opcode(*itr, mos6502::OpCode::tax, mos6502::OpCode::tsx, mos6502::OpCode::ldx)) {
+      for (auto inner = std::next(itr); inner != block.end(); ++inner) {
+        if (is_opcode(*inner,
+              mos6502::OpCode::txa,
+              mos6502::OpCode::txs,
+              mos6502::OpCode::stx,
+              mos6502::OpCode::inx,
+              mos6502::OpCode::dex,
+              mos6502::OpCode::cpx)) {
+          break;
+        }
+        if (is_opcode(*inner, mos6502::OpCode::tax, mos6502::OpCode::tsx, mos6502::OpCode::ldx)) {
+          // redundant store found
+          *itr = mos6502(ASMLine::Type::Directive, "; removed dead load of X: " + itr->to_string());
+          return true;
+        }
+      }
+    }
+  }
+
+  return false;
+}
+
+bool optimize_dead_sta(std::span<mos6502> &block, const Personality &personality)
+{
+  for (auto itr = block.begin(); itr != block.end(); ++itr) {
+    if (itr->opcode == mos6502::OpCode::sta && is_virtual_register_op(*itr, personality)) {
+      for (auto inner = std::next(itr); inner != block.end(); ++inner) {
+        if (inner->op.value.find('(') != std::string::npos) {
+          // this is an indexed operation, which is risky to optimize a sta around on the virtual registers,
+          // so we'll skip this block
+          break;
+        }
+        if (inner->op.value == itr->op.value) {
+          if (is_opcode(*inner, mos6502::OpCode::sta)) {
+            // redundant store found
+            *itr = mos6502(ASMLine::Type::Directive, "; removed dead store of a: " + itr->to_string());
+            return true;
+          } else {
+            // someone else is operating on us, time to abort
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  return false;
+}
+
+bool optimize_redundant_ldy(std::span<mos6502> &block)
+{
+  for (auto itr = block.begin(); itr != block.end(); ++itr) {
+    if (itr->opcode == mos6502::OpCode::ldy && itr->op.value.starts_with('#')) {
+      for (auto inner = std::next(itr); inner != block.end(); ++inner) {
+        if (is_opcode(*inner,
+              mos6502::OpCode::cpy,
+              mos6502::OpCode::tya,
+              mos6502::OpCode::tay,
+              mos6502::OpCode::sty,
+              mos6502::OpCode::iny,
+              mos6502::OpCode::dey)) {
+          break;// break, these all operate on Y
+        }
+        // we found a matching ldy
+        if (is_opcode(*inner, mos6502::OpCode::ldy)) {
+          // with the same value
+          // note: this operation is only safe because we know that our system only uses Y
+          // for index operations and we don't rely (or even necessarily *want* the changes to N,Z)
+          if (inner->op.value == itr->op.value) {
+            *inner = mos6502(ASMLine::Type::Directive, "; removed redundant ldy: " + inner->to_string());
+            return true;
+          } else {
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  return false;
+}
+
+bool optimize_redundant_lda(std::span<mos6502> &block, const Personality &personality)
+{
+  // look for a literal or virtual register load into A
+  // that is redundant later
+  for (auto itr = block.begin(); itr != block.end(); ++itr) {
+    if (itr->opcode == mos6502::OpCode::lda &&
+      (itr->op.value.starts_with('#')
+          || is_virtual_register_op(*itr, personality))) {
+      for (auto inner = std::next(itr); inner != block.end(); ++inner) {
+        if (is_opcode(*inner,
+              mos6502::OpCode::tay,
+              mos6502::OpCode::tax,
+              mos6502::OpCode::sta,
+              mos6502::OpCode::pha,
+              mos6502::OpCode::nop)) {
+          continue;// OK to skip instructions that don't modify A or change flags
+        }
+        if (inner->type == ASMLine::Type::Directive) {
+          continue;// OK to skip directives
+        }
+        if (is_opcode(*inner, mos6502::OpCode::lda)) {
+          if (inner->op == itr->op) {
+            // we found a matching lda, after an sta, we can remove it
+            *inner = mos6502(ASMLine::Type::Directive, "; removed redundant lda: " + inner->to_string());
+            return true;
+          } else {
+            break;
+          }
+        }
+
+        break;// we can only optimize around tax and comments right now
+      }
+    }
+  }
+
+  return false;
+}
+
+bool optimize_redundant_lda_after_sta(std::span<mos6502> &block)
+{
+  for (auto itr = block.begin(); itr != block.end(); ++itr) {
+    if (itr->opcode == mos6502::OpCode::sta) {
+      for (auto inner = std::next(itr); inner != block.end(); ++inner) {
+        if (is_opcode(*inner,
+              mos6502::OpCode::tax,
+              mos6502::OpCode::tay,
+              mos6502::OpCode::clc,
+              mos6502::OpCode::sec,
+              mos6502::OpCode::sta,
+              mos6502::OpCode::pha,
+              mos6502::OpCode::txs,
+              mos6502::OpCode::php,
+              mos6502::OpCode::sty,
+              mos6502::OpCode::nop)) {
+          continue;// OK to skip instructions that don't modify A or change flags
+        }
+        if (inner->type == ASMLine::Type::Directive) {
+          continue;// OK to skip directives
+        }
+        if (is_opcode(*inner, mos6502::OpCode::lda)) {
+          if (inner->op == itr->op) {
+            // we found a matching lda, after a sta, we can remove it
+            *inner = mos6502(ASMLine::Type::Directive, "; removed redundant lda: " + inner->to_string());
+            return true;
+          } else {
+            break;
+          }
+        }
+
+        break;// we can only optimize around tax and comments right now
+      }
+    }
+  }
+
+  return false;
+}
+
+bool optimize(std::vector<mos6502> &instructions, [[maybe_unused]] const Personality &personality)
+{
+
 
   // remove unused flag-fix-up blocks
   // it might make sense in the future to only insert these if determined they are needed?
@@ -37,153 +271,24 @@ bool optimize(std::vector<mos6502> &instructions, const Personality &personality
     }
   }
 
-
+  // replace use of __zero_reg__ with literal 0
-  // look for redundant load of lda after a tax
-  for (size_t op = 0; op < instructions.size() - 3; ++op) {
-    if (instructions[op].opcode == mos6502::OpCode::sta && instructions[op + 1].opcode == mos6502::OpCode::tax
-        && instructions[op + 2].opcode == mos6502::OpCode::lda
-        && instructions[op].op.value == instructions[op + 2].op.value) {
-      instructions[op + 2] =
-        mos6502(ASMLine::Type::Directive, "; removed redundant lda: " + instructions[op + 2].to_string());
-      return true;
-    }
-  }
-
-  // look for redundant stores to 0-page registers with sta
-  for (size_t op = 0; op < instructions.size(); ++op) {
-    // todo, make sure this is in the register map
-    if (instructions[op].opcode == mos6502::OpCode::sta && instructions[op].op.value.size() == 3) {
-      for (size_t next_op = op + 1; next_op < instructions.size(); ++next_op) {
-        if (instructions[next_op].opcode != mos6502::OpCode::sta
-            && instructions[next_op].op.value == instructions[op].op.value) {
-          // we just found a use of ourselves back, abort the search, there's probably something else going on
-          break;
-        }
-        if (instructions[next_op].opcode == mos6502::OpCode::lda
-            && instructions[next_op].op.value != instructions[op].op.value) {
-          // someone just loaded lda with a different value, so we need to abort!
-          break;
-        }
-        // abort at pla
-        if (instructions[next_op].opcode == mos6502::OpCode::pla) { break; }
-        // abort at jsr
-        if (instructions[next_op].opcode == mos6502::OpCode::jsr) { break; }
-
-        // abort at label
-        if (instructions[next_op].type == ASMLine::Type::Label) { break; }
-
-        if (instructions[next_op].opcode == mos6502::OpCode::sta
-            && instructions[next_op].op.value == instructions[op].op.value) {
-          // looks like we found a redundant store, remove the first one
-          instructions[op] =
-            mos6502(ASMLine::Type::Directive, "; removed redundant sta: " + instructions[op].to_string());
-          return true;
-        }
-      }
-    }
-  }
-
-  for (size_t op = 0; op < instructions.size() - 1; ++op) {
-    // look for a transfer of A -> X immediately followed by LDX
-    if (instructions[op].opcode == mos6502::OpCode::sta) {
-      const auto next_op = next_instruction(op);
-      if (instructions[next_op].opcode == mos6502::OpCode::ldx && instructions[op].op == instructions[next_op].op) {
-        auto last_comment = instructions[next_op].comment;
-        instructions[next_op] = mos6502(mos6502::OpCode::tax);
-        instructions[next_op].comment = last_comment;
-        return true;
-      }
-    }
-  }
-
-  for (size_t op = 0; op < instructions.size() - 1; ++op) {
-    // look for a transfer of A -> X immediately followed by LDX
-    if (instructions[op].opcode == mos6502::OpCode::tax) {
-      const auto next_op = next_instruction(op);
-      if (instructions[next_op].opcode == mos6502::OpCode::ldx) {
-        instructions[op] =
-          mos6502(ASMLine::Type::Directive, "; removed redundant tax: " + instructions[op].to_string());
-        return true;
-      }
-    }
-  }
-
-  for (size_t op = 0; op < instructions.size() - 1; ++op) {
-    // look for a transfer of Y -> A immediately followed by A -> Y
-    if (instructions[op].opcode == mos6502::OpCode::tya) {
-      const auto next_op = next_instruction(op);
-      if (instructions[next_op].opcode == mos6502::OpCode::tay) {
-        instructions[op] =
-          mos6502(ASMLine::Type::Directive, "; removed redundant tay: " + instructions[op].to_string());
-        return true;
-      }
-    }
-  }
-
-  for (size_t op = 0; op < instructions.size() - 1; ++op) {
-    // look for a store A -> loc immediately followed by loc -> A
-    if (instructions[op].opcode == mos6502::OpCode::sta) {
-
-      const auto next = next_instruction(op);
-      if (instructions[next].opcode == mos6502::OpCode::lda && instructions[next].op == instructions[op].op) {
-        instructions[next] =
-          mos6502(ASMLine::Type::Directive, "; removed redundant lda: " + instructions[next].to_string());
-        return true;
-      }
-    }
-  }
-
   for (auto &op : instructions) {
     if (op.type == ASMLine::Type::Instruction && op.op.type == Operand::Type::literal
         && op.op.value == personality.get_register(1).value && op.opcode != mos6502::OpCode::sta) {
       // replace use of zero reg with literal 0
+      const auto old_string = op.to_string();
       op.op.value = "#0";
+      op.comment = "replaced use of register 1 with a literal 0, because of AVR GCC __zero_reg__  ; " + old_string;
     }
   }
 
-  for (size_t op = 0; op < instructions.size() - 1; ++op) {
+  bool block_optimized = false;
-    if (instructions[op].opcode == mos6502::OpCode::ldy && instructions[op].op.type == Operand::Type::literal) {
+  for (auto &block : get_optimizable_blocks(instructions)) {
-      auto op2 = op + 1;
+    block_optimized = block_optimized || optimize_redundant_lda_after_sta(block) || optimize_dead_sta(block, personality) || optimize_dead_tax(block)
+        || optimize_redundant_ldy(block) || optimize_redundant_lda(block, personality);
 
-      while (op2 < instructions.size() && (instructions[op2].type != ASMLine::Type::Label)
-             && (instructions[op2].opcode == mos6502::OpCode::jsr)) {
-        // while inside this label
-        if (instructions[op2].opcode == mos6502::OpCode::ldy) {
-
-          if (instructions[op2].op.value == instructions[op].op.value) {
-            instructions[op2] =
-              mos6502(ASMLine::Type::Directive, "; removed redundant ldy: " + instructions[op2].to_string());
-            return true;
-          } else {
-            // if we ldy with any other value in this block then abort
-            break;
-          }
-        }
-        ++op2;
-      }
-    }
   }
-
+  return block_optimized;
-
-  for (size_t op = 0; op < instructions.size() - 1; ++op) {
-    if (instructions[op].opcode == mos6502::OpCode::lda && instructions[op].op.type == Operand::Type::literal) {
-      const auto operand = instructions[op].op;
-      auto op2 = op + 1;
-      // look for multiple stores of the same value
-      while (
-        op2 < instructions.size()
-        && (instructions[op2].opcode == mos6502::OpCode::sta || instructions[op2].type == ASMLine::Type::Directive)) {
-        ++op2;
-      }
-      if (instructions[op2].opcode == mos6502::OpCode::lda && operand == instructions[op2].op) {
-        instructions[op2] =
-          mos6502(ASMLine::Type::Directive, "; removed redundant lda: " + instructions[op2].to_string());
-        return true;
-      }
-    }
-  }
-
-  return false;
 }
 
 #endif// INC_6502_CPP_OPTIMIZER_HPP