diff --git a/teensy/teensy-keyboard.cpp b/teensy/teensy-keyboard.cpp
index 5f48936..ea20ae5 100644
--- a/teensy/teensy-keyboard.cpp
+++ b/teensy/teensy-keyboard.cpp
@@ -81,6 +81,12 @@ static uint8_t shiftedNumber[] = { '<', // ,
 
 TeensyKeyboard::TeensyKeyboard(VMKeyboard *k) : PhysicalKeyboard(k)
 {
+  // Need to set the rows to be pull-ups early, so the pullups have
+  // time to settle -- otherwise we get a phantom set of keypresses
+  // on startup for all the keys in the first column
+  for (byte i=0; i<ROWS; i++) {
+    pinMode(rowsPins[i], INPUT_PULLUP);
+  }
   keypad.setDebounceTime(10);
   
   leftShiftPressed = false;