From a5ddb51a3b3320b6107f38872af335f4d11628d5 Mon Sep 17 00:00:00 2001 From: Maxim Poliakovski Date: Sun, 16 May 2021 00:53:15 +0200 Subject: [PATCH] ppcmmu: initial TLB implementation for reads. --- cpu/ppc/ppcexceptions.cpp | 3 + cpu/ppc/ppcmmu.cpp | 351 ++++++++++++++++++++++++++++++++++++++ cpu/ppc/ppcmmu.h | 15 ++ cpu/ppc/ppcopcodes.cpp | 3 + 4 files changed, 372 insertions(+) diff --git a/cpu/ppc/ppcexceptions.cpp b/cpu/ppc/ppcexceptions.cpp index 73bf1ff..015da7f 100644 --- a/cpu/ppc/ppcexceptions.cpp +++ b/cpu/ppc/ppcexceptions.cpp @@ -22,6 +22,7 @@ along with this program. If not, see . /** @file Handling of low-level PPC exceptions. */ #include "ppcemu.h" +#include "ppcmmu.h" #include #include #include @@ -109,6 +110,8 @@ jmp_buf exc_env; /* Global exception environment. */ ppc_next_instruction_address |= 0xFFF00000; } + mmu_change_mode(); + longjmp(exc_env, 2); /* return to the main execution loop. */ } diff --git a/cpu/ppc/ppcmmu.cpp b/cpu/ppc/ppcmmu.cpp index 18ccb50..a6a43eb 100644 --- a/cpu/ppc/ppcmmu.cpp +++ b/cpu/ppc/ppcmmu.cpp @@ -135,6 +135,13 @@ public: }; #endif +/** Temporary TLB test variables. */ +bool MemAccessType; // true - memory, false - I/O +uint64_t MemAddr = 0; +MMIODevice *Device = 0; +uint32_t DevOffset = 0; + + /** remember recently used physical memory regions for quicker translation. */ AddressMapEntry last_read_area = {0xFFFFFFFF, 0xFFFFFFFF}; AddressMapEntry last_write_area = {0xFFFFFFFF, 0xFFFFFFFF}; @@ -160,6 +167,17 @@ static inline T read_phys_mem(AddressMapEntry *mru_rgn, uint32_t addr) #ifdef MMU_PROFILING dmem_reads_total++; #endif + + if (!MemAccessType) { + LOG_F(ERROR, "TLB real memory access expected!"); + } + + if ((mru_rgn->mem_ptr + (addr - mru_rgn->start)) != (uint8_t *)MemAddr) { + LOG_F(ERROR, "TLB address mismatch! Expected: 0x%llu, got: 0x%llu", + (uint64_t)(mru_rgn->mem_ptr + (addr - mru_rgn->start)), + (uint64_t)MemAddr); + } + switch(sizeof(T)) { case 1: return *(mru_rgn->mem_ptr + (addr - mru_rgn->start)); @@ -187,6 +205,15 @@ static inline T read_phys_mem(AddressMapEntry *mru_rgn, uint32_t addr) #ifdef MMU_PROFILING iomem_reads_total++; #endif + if (MemAccessType) { + LOG_F(ERROR, "TLB I/O memory access expected!"); + } + + if (mru_rgn->devobj != Device || (addr - mru_rgn->start) != DevOffset) { + LOG_F(ERROR, "TLB MMIO access mismatch! Expected: 0x%X, got: 0x%X", + addr - mru_rgn->start, DevOffset); + } + return (mru_rgn->devobj->read(mru_rgn->start, addr - mru_rgn->start, sizeof(T))); } else { @@ -311,6 +338,43 @@ void dbat_update(uint32_t bat_reg) { } } +/** PowerPC-style block address translation. */ +template +static BATResult ppc_block_address_translation(uint32_t la) +{ + uint32_t pa; // translated physical address + uint8_t prot; // protection bits for the translated address + PPC_BAT_entry *bat_array; + + bool bat_hit = false; + unsigned msr_pr = !!(ppc_state.msr & 0x4000); + + bat_array = (type == BATType::Instruction) ? ibat_array : dbat_array; + + // Format: %XY + // X - supervisor access bit, Y - problem/user access bit + // Those bits are mutually exclusive + unsigned access_bits = ((msr_pr ^ 1) << 1) | msr_pr; + + for (int bat_index = 0; bat_index < 4; bat_index++) { + PPC_BAT_entry* bat_entry = &bat_array[bat_index]; + + if ((bat_entry->access & access_bits) && ((la & bat_entry->hi_mask) == bat_entry->bepi)) { + bat_hit = true; + +#ifdef MMU_PROFILING + bat_transl_total++; +#endif + // logical to physical translation + pa = bat_entry->phys_hi | (la & ~bat_entry->hi_mask); + prot = bat_entry->prot; + break; + } + } + + return BATResult{bat_hit, prot, pa}; +} + static inline uint8_t* calc_pteg_addr(uint32_t hash) { uint32_t sdr1_val, pteg_addr; @@ -672,8 +736,244 @@ static uint32_t mem_grab_unaligned(uint32_t addr, uint32_t size) { return ret; } +#define PAGE_SIZE_BITS 12 +#define TLB_SIZE 4096 +#define TLB2_WAYS 4 +#define TLB_INVALID_TAG 0xFFFFFFFF + +typedef struct TLBEntry { + uint32_t tag; + uint16_t flags; + uint16_t lru_bits; + union { + int64_t host_va_offset; + AddressMapEntry* reg_desc; + }; +} TLBEntry; + +// primary TLB for all MMU modes +static std::array mode1_tlb1; +static std::array mode2_tlb1; +static std::array mode3_tlb1; + +// secondary TLB for all MMU modes +static std::array mode1_tlb2; +static std::array mode2_tlb2; +static std::array mode3_tlb2; + +TLBEntry *pCurTLB1; // current primary TLB +TLBEntry *pCurTLB2; // current secondary TLB + +uint32_t tlb_size_mask = TLB_SIZE - 1; + +// fake TLB entry for handling of unmapped memory accesses +uint64_t UnmappedVal = -1ULL; +TLBEntry UnmappedMem = {TLB_INVALID_TAG, 0, 0, 0}; + +uint8_t MMUMode = {0xFF}; + +void mmu_change_mode() +{ + uint8_t mmu_mode = ((ppc_state.msr >> 3) & 0x2) | ((ppc_state.msr >> 14) & 1); + + if (MMUMode != mmu_mode) { + switch(mmu_mode) { + case 0: // real address mode + pCurTLB1 = &mode1_tlb1[0]; + pCurTLB2 = &mode1_tlb2[0]; + break; + case 2: // supervisor mode with data translation enabled + pCurTLB1 = &mode2_tlb1[0]; + pCurTLB2 = &mode2_tlb2[0]; + break; + case 3: // user mode with data translation enabled + pCurTLB1 = &mode3_tlb1[0]; + pCurTLB2 = &mode3_tlb2[0]; + break; + } + MMUMode = mmu_mode; + } +} + +static TLBEntry* tlb2_target_entry(uint32_t gp_va) +{ + TLBEntry *tlb_entry; + + tlb_entry = &pCurTLB2[((gp_va >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; + + // select the target from invalid blocks first + if (tlb_entry[0].tag == TLB_INVALID_TAG) { + // update LRU bits + tlb_entry[0].lru_bits = 0x3; + tlb_entry[1].lru_bits = 0x2; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + return tlb_entry; + } else if (tlb_entry[1].tag == TLB_INVALID_TAG) { + // update LRU bits + tlb_entry[0].lru_bits = 0x2; + tlb_entry[1].lru_bits = 0x3; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + return &tlb_entry[1]; + } else if (tlb_entry[2].tag == TLB_INVALID_TAG) { + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x3; + tlb_entry[3].lru_bits = 0x2; + return &tlb_entry[2]; + } else if (tlb_entry[3].tag == TLB_INVALID_TAG) { + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x2; + tlb_entry[3].lru_bits = 0x3; + return &tlb_entry[3]; + } else { // no invalid blocks, replace an existing one according with the hLRU policy + if (tlb_entry[0].lru_bits == 0) { + // update LRU bits + tlb_entry[0].lru_bits = 0x3; + tlb_entry[1].lru_bits = 0x2; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + return tlb_entry; + } else if (tlb_entry[1].lru_bits == 0) { + // update LRU bits + tlb_entry[0].lru_bits = 0x2; + tlb_entry[1].lru_bits = 0x3; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + return &tlb_entry[1]; + } else if (tlb_entry[2].lru_bits == 0) { + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x3; + tlb_entry[3].lru_bits = 0x2; + return &tlb_entry[2]; + } else { + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x2; + tlb_entry[3].lru_bits = 0x3; + return &tlb_entry[3]; + } + } +} + +static TLBEntry* tlb2_refill(uint32_t guest_va, int is_write) +{ + uint32_t phys_addr; + TLBEntry *tlb_entry; + + const uint32_t tag = guest_va & ~0xFFFUL; + + // attempt block address translation first + BATResult bat_res = ppc_block_address_translation(guest_va); + if (bat_res.hit) { + // check block protection + if (!bat_res.prot || ((bat_res.prot & 1) && is_write)) { + ppc_state.spr[SPR::DSISR] = 0x08000000 | (is_write << 25); + ppc_state.spr[SPR::DAR] = guest_va; + mmu_exception_handler(Except_Type::EXC_DSI, 0); + } + phys_addr = bat_res.phys; + } else { + // page address translation + phys_addr = page_address_translate(guest_va, false, !!(ppc_state.msr & 0x4000), is_write); + } + + // look up host virtual address + AddressMapEntry* reg_desc = mem_ctrl_instance->find_range(phys_addr); + if (reg_desc) { + // refill the secondary TLB + tlb_entry = tlb2_target_entry(tag); + tlb_entry->tag = tag; + if (reg_desc->type & RT_MMIO) { + tlb_entry->flags = 2; // MMIO region + tlb_entry->reg_desc = reg_desc; + } else { + tlb_entry->flags = 1; // memory region backed by host memory + tlb_entry->host_va_offset = (int64_t)reg_desc->mem_ptr - reg_desc->start; + } + return tlb_entry; + } else { + LOG_F(ERROR, "Read from unmapped memory at 0x%08X!\n", phys_addr); + UnmappedMem.tag = tag; + UnmappedMem.host_va_offset = (int64_t)(&UnmappedVal) - guest_va; + return &UnmappedMem; + } +} + +static inline uint64_t tlb_translate_addr(uint32_t guest_va) +{ + TLBEntry *tlb1_entry, *tlb2_entry; + + const uint32_t tag = guest_va & ~0xFFFUL; + + // look up address in the primary TLB + tlb1_entry = &pCurTLB1[(guest_va >> PAGE_SIZE_BITS) & tlb_size_mask]; + if (tlb1_entry->tag == tag) { // primary TLB hit -> fast path + MemAccessType = true; + MemAddr = tlb1_entry->host_va_offset + guest_va; + return tlb1_entry->host_va_offset + guest_va; + } else { // primary TLB miss -> look up address in the secondary TLB + tlb2_entry = &pCurTLB2[((guest_va >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; + if (tlb2_entry->tag == tag) { + // update LRU bits + tlb2_entry[0].lru_bits = 0x3; + tlb2_entry[1].lru_bits = 0x2; + tlb2_entry[2].lru_bits &= 0x1; + tlb2_entry[3].lru_bits &= 0x1; + } else if (tlb2_entry[1].tag == tag) { + tlb2_entry = &tlb2_entry[1]; + // update LRU bits + tlb2_entry[0].lru_bits = 0x2; + tlb2_entry[1].lru_bits = 0x3; + tlb2_entry[2].lru_bits &= 0x1; + tlb2_entry[3].lru_bits &= 0x1; + } else if (tlb2_entry[2].tag == tag) { + tlb2_entry = &tlb2_entry[2]; + // update LRU bits + tlb2_entry[0].lru_bits &= 0x1; + tlb2_entry[1].lru_bits &= 0x1; + tlb2_entry[2].lru_bits = 0x3; + tlb2_entry[3].lru_bits = 0x2; + } else if (tlb2_entry[3].tag == tag) { + tlb2_entry = &tlb2_entry[3]; + // update LRU bits + tlb2_entry[0].lru_bits &= 0x1; + tlb2_entry[1].lru_bits &= 0x1; + tlb2_entry[2].lru_bits = 0x2; + tlb2_entry[3].lru_bits = 0x3; + } else { // secondary TLB miss -> + // perform full address translation and refill the secondary TLB + tlb2_entry = tlb2_refill(guest_va, 0); + } + + if (tlb2_entry->flags & 1) { // is it a real memory region? + // refill the primary TLB + tlb1_entry->tag = tag; + tlb1_entry->flags = 1; + tlb1_entry->host_va_offset = tlb2_entry->host_va_offset; + MemAccessType = true; + MemAddr = tlb1_entry->host_va_offset + guest_va; + return tlb1_entry->host_va_offset + guest_va; + } else { // an attempt to access a memory-mapped device + MemAccessType = false; + Device = tlb2_entry->reg_desc->devobj; + DevOffset = guest_va - tlb2_entry->reg_desc->start; + return guest_va - tlb2_entry->reg_desc->start; + } + } +} + /** Grab a value from memory into a register */ uint8_t mem_grab_byte(uint32_t addr) { + tlb_translate_addr(addr); + /* data address translation if enabled */ if (ppc_state.msr & 0x10) { addr = ppc_mmu_addr_translate(addr, 0); @@ -683,6 +983,8 @@ uint8_t mem_grab_byte(uint32_t addr) { } uint16_t mem_grab_word(uint32_t addr) { + tlb_translate_addr(addr); + if (addr & 1) { return mem_grab_unaligned(addr, 2); } @@ -696,6 +998,8 @@ uint16_t mem_grab_word(uint32_t addr) { } uint32_t mem_grab_dword(uint32_t addr) { + tlb_translate_addr(addr); + if (addr & 3) { return mem_grab_unaligned(addr, 4); } @@ -709,6 +1013,8 @@ uint32_t mem_grab_dword(uint32_t addr) { } uint64_t mem_grab_qword(uint32_t addr) { + tlb_translate_addr(addr); + if (addr & 7) { LOG_F(ERROR, "SOS! Attempt to read unaligned QWORD at 0x%08X\n", addr); exit(-1); // FIXME! @@ -801,6 +1107,51 @@ uint64_t mem_read_dbg(uint32_t virt_addr, uint32_t size) { void ppc_mmu_init() { mmu_exception_handler = ppc_exception_handler; + // invalidate all TLB entries + for(auto &tlb_el : mode1_tlb1) { + tlb_el.tag = TLB_INVALID_TAG; + tlb_el.flags = 0; + tlb_el.lru_bits = 0; + tlb_el.host_va_offset = 0; + } + + for(auto &tlb_el : mode2_tlb1) { + tlb_el.tag = TLB_INVALID_TAG; + tlb_el.flags = 0; + tlb_el.lru_bits = 0; + tlb_el.host_va_offset = 0; + } + + for(auto &tlb_el : mode3_tlb1) { + tlb_el.tag = TLB_INVALID_TAG; + tlb_el.flags = 0; + tlb_el.lru_bits = 0; + tlb_el.host_va_offset = 0; + } + + for(auto &tlb_el : mode1_tlb2) { + tlb_el.tag = TLB_INVALID_TAG; + tlb_el.flags = 0; + tlb_el.lru_bits = 0; + tlb_el.host_va_offset = 0; + } + + for(auto &tlb_el : mode2_tlb2) { + tlb_el.tag = TLB_INVALID_TAG; + tlb_el.flags = 0; + tlb_el.lru_bits = 0; + tlb_el.host_va_offset = 0; + } + + for(auto &tlb_el : mode3_tlb2) { + tlb_el.tag = TLB_INVALID_TAG; + tlb_el.flags = 0; + tlb_el.lru_bits = 0; + tlb_el.host_va_offset = 0; + } + + mmu_change_mode(); + #ifdef MMU_PROFILING gProfilerObj->register_profile("PPC_MMU", std::unique_ptr(new MMUProfile())); diff --git a/cpu/ppc/ppcmmu.h b/cpu/ppc/ppcmmu.h index d8910da..f132301 100644 --- a/cpu/ppc/ppcmmu.h +++ b/cpu/ppc/ppcmmu.h @@ -40,12 +40,27 @@ typedef struct PPC_BAT_entry { uint32_t bepi; /* copy of Block effective page index */ } PPC_BAT_entry; +/** Block address translation types. */ +enum BATType : int { + Instruction, + Data +}; + +/** Result of the block address translation. */ +typedef struct BATResult { + bool hit; + uint8_t prot; + uint32_t phys; +} BATResult; + extern void ibat_update(uint32_t bat_reg); extern void dbat_update(uint32_t bat_reg); extern uint8_t* mmu_get_dma_mem(uint32_t addr, uint32_t size); +extern void mmu_change_mode(void); + extern void ppc_set_cur_instruction(const uint8_t* ptr); extern void mem_write_byte(uint32_t addr, uint8_t value); extern void mem_write_word(uint32_t addr, uint16_t value); diff --git a/cpu/ppc/ppcopcodes.cpp b/cpu/ppc/ppcopcodes.cpp index 48a8e40..34d7a2c 100644 --- a/cpu/ppc/ppcopcodes.cpp +++ b/cpu/ppc/ppcopcodes.cpp @@ -824,6 +824,7 @@ void dppc_interpreter::ppc_mtmsr() { } reg_s = (ppc_cur_instruction >> 21) & 31; ppc_state.msr = ppc_state.gpr[reg_s]; + mmu_change_mode(); } void dppc_interpreter::ppc_mfspr() { @@ -1278,6 +1279,8 @@ void dppc_interpreter::ppc_rfi() { ppc_state.msr = (new_msr_val | new_srr1_val) & 0xFFFBFFFFUL; ppc_next_instruction_address = ppc_state.spr[SPR::SRR0] & 0xFFFFFFFCUL; + mmu_change_mode(); + grab_return = true; bb_kind = BB_end_kind::BB_RFI; }