From 212cd58f4015c6e5fa60d18b909d586587bace7e Mon Sep 17 00:00:00 2001 From: Maxim Poliakovski Date: Sat, 25 Sep 2021 15:57:35 +0200 Subject: [PATCH] ppcmmu: refactor and clean up. --- cpu/ppc/ppcmmu.cpp | 2191 +++++++++++++++++++++-------------------- cpu/ppc/ppcmmu.h | 19 +- debugger/debugger.cpp | 2 +- 3 files changed, 1117 insertions(+), 1095 deletions(-) diff --git a/cpu/ppc/ppcmmu.cpp b/cpu/ppc/ppcmmu.cpp index 03d23a9..46bfcc7 100644 --- a/cpu/ppc/ppcmmu.cpp +++ b/cpu/ppc/ppcmmu.cpp @@ -22,9 +22,7 @@ along with this program. If not, see . /** @file PowerPC Memory Management Unit emulation. */ /* TODO: - - implement TLB - implement 601-style BATs - - add proper error and exception handling */ #include "ppcmmu.h" @@ -49,7 +47,6 @@ PPC_BAT_entry dbat_array[4] = {{0}}; //#define MMU_PROFILING // uncomment this to enable MMU profiling //#define TLB_PROFILING // uncomment this to enable SoftTLB profiling -/* MMU profiling */ #ifdef MMU_PROFILING /* global variables for lightweight MMU profiling */ @@ -65,6 +62,1103 @@ uint64_t unaligned_writes = 0; // counts unaligned writes uint64_t unaligned_crossp_r = 0; // counts unaligned crosspage reads uint64_t unaligned_crossp_w = 0; // counts unaligned crosspage writes +#endif // MMU_PROFILING + +#ifdef TLB_PROFILING + +/* global variables for lightweight SoftTLB profiling */ +uint64_t num_primary_itlb_hits = 0; // number of hits in the primary ITLB +uint64_t num_secondary_itlb_hits = 0; // number of hits in the secondary ITLB +uint64_t num_itlb_refills = 0; // number of ITLB refills +uint64_t num_primary_dtlb_hits = 0; // number of hits in the primary DTLB +uint64_t num_secondary_dtlb_hits = 0; // number of hits in the secondary DTLB +uint64_t num_dtlb_refills = 0; // number of DTLB refills +uint64_t num_entry_replacements = 0; // number of entry replacements + +#endif // TLB_PROFILING + +/** remember recently used physical memory regions for quicker translation. */ +AddressMapEntry last_read_area = {0xFFFFFFFF, 0xFFFFFFFF}; +AddressMapEntry last_write_area = {0xFFFFFFFF, 0xFFFFFFFF}; +AddressMapEntry last_exec_area = {0xFFFFFFFF, 0xFFFFFFFF}; +AddressMapEntry last_ptab_area = {0xFFFFFFFF, 0xFFFFFFFF}; +AddressMapEntry last_dma_area = {0xFFFFFFFF, 0xFFFFFFFF}; + +void ppc_set_cur_instruction(const uint8_t* ptr) { + ppc_cur_instruction = READ_DWORD_BE_A(ptr); +} + +/** PowerPC-style block address translation. */ +template +static BATResult ppc_block_address_translation(uint32_t la) +{ + uint32_t pa; // translated physical address + uint8_t prot; // protection bits for the translated address + PPC_BAT_entry *bat_array; + + bool bat_hit = false; + unsigned msr_pr = !!(ppc_state.msr & 0x4000); + + bat_array = (type == BATType::IBAT) ? ibat_array : dbat_array; + + // Format: %XY + // X - supervisor access bit, Y - problem/user access bit + // Those bits are mutually exclusive + unsigned access_bits = ((msr_pr ^ 1) << 1) | msr_pr; + + for (int bat_index = 0; bat_index < 4; bat_index++) { + PPC_BAT_entry* bat_entry = &bat_array[bat_index]; + + if ((bat_entry->access & access_bits) && ((la & bat_entry->hi_mask) == bat_entry->bepi)) { + bat_hit = true; + +#ifdef MMU_PROFILING + bat_transl_total++; +#endif + // logical to physical translation + pa = bat_entry->phys_hi | (la & ~bat_entry->hi_mask); + prot = bat_entry->prot; + break; + } + } + + return BATResult{bat_hit, prot, pa}; +} + +static inline uint8_t* calc_pteg_addr(uint32_t hash) +{ + uint32_t sdr1_val, pteg_addr; + + sdr1_val = ppc_state.spr[SPR::SDR1]; + + pteg_addr = sdr1_val & 0xFE000000; + pteg_addr |= (sdr1_val & 0x01FF0000) | (((sdr1_val & 0x1FF) << 16) & ((hash & 0x7FC00) << 6)); + pteg_addr |= (hash & 0x3FF) << 6; + + if (pteg_addr >= last_ptab_area.start && pteg_addr <= last_ptab_area.end) { + return last_ptab_area.mem_ptr + (pteg_addr - last_ptab_area.start); + } else { + AddressMapEntry* entry = mem_ctrl_instance->find_range(pteg_addr); + if (entry && entry->type & (RT_ROM | RT_RAM)) { + last_ptab_area.start = entry->start; + last_ptab_area.end = entry->end; + last_ptab_area.mem_ptr = entry->mem_ptr; + return last_ptab_area.mem_ptr + (pteg_addr - last_ptab_area.start); + } else { + ABORT_F("SOS: no page table region was found at %08X!\n", pteg_addr); + } + } +} + +static bool search_pteg(uint8_t* pteg_addr, uint8_t** ret_pte_addr, uint32_t vsid, + uint16_t page_index, uint8_t pteg_num) +{ + /* construct PTE matching word */ + uint32_t pte_check = 0x80000000 | (vsid << 7) | (pteg_num << 6) | (page_index >> 10); + +#ifdef MMU_INTEGRITY_CHECKS + /* PTEG integrity check that ensures that all matching PTEs have + identical RPN, WIMG and PP bits (PPC PEM 32-bit 7.6.2, rule 5). */ + uint32_t pte_word2_check; + bool match_found = false; + + for (int i = 0; i < 8; i++, pteg_addr += 8) { + if (pte_check == READ_DWORD_BE_A(pteg_addr)) { + if (match_found) { + if ((READ_DWORD_BE_A(pteg_addr) & 0xFFFFF07B) != pte_word2_check) { + ABORT_F("Multiple PTEs with different RPN/WIMG/PP found!\n"); + } + } else { + /* isolate RPN, WIMG and PP fields */ + pte_word2_check = READ_DWORD_BE_A(pteg_addr) & 0xFFFFF07B; + *ret_pte_addr = pteg_addr; + } + } + } +#else + for (int i = 0; i < 8; i++, pteg_addr += 8) { + if (pte_check == READ_DWORD_BE_A(pteg_addr)) { + *ret_pte_addr = pteg_addr; + return true; + } + } +#endif + + return false; +} + +static PATResult page_address_translation(uint32_t la, bool is_instr_fetch, + unsigned msr_pr, int is_write) +{ + uint32_t sr_val, page_index, pteg_hash1, vsid, pte_word2; + unsigned key, pp; + uint8_t* pte_addr; + + sr_val = ppc_state.sr[(la >> 28) & 0x0F]; + if (sr_val & 0x80000000) { + ABORT_F("Direct-store segments not supported, LA=%0xX\n", la); + } + + /* instruction fetch from a no-execute segment will cause ISI exception */ + if ((sr_val & 0x10000000) && is_instr_fetch) { + mmu_exception_handler(Except_Type::EXC_ISI, 0x10000000); + } + + page_index = (la >> 12) & 0xFFFF; + pteg_hash1 = (sr_val & 0x7FFFF) ^ page_index; + vsid = sr_val & 0x0FFFFFF; + + if (!search_pteg(calc_pteg_addr(pteg_hash1), &pte_addr, vsid, page_index, 0)) { + if (!search_pteg(calc_pteg_addr(~pteg_hash1), &pte_addr, vsid, page_index, 1)) { + if (is_instr_fetch) { + mmu_exception_handler(Except_Type::EXC_ISI, 0x40000000); + } else { + ppc_state.spr[SPR::DSISR] = 0x40000000 | (is_write << 25); + ppc_state.spr[SPR::DAR] = la; + mmu_exception_handler(Except_Type::EXC_DSI, 0); + } + } + } + + pte_word2 = READ_DWORD_BE_A(pte_addr + 4); + + key = (((sr_val >> 29) & 1) & msr_pr) | (((sr_val >> 30) & 1) & (msr_pr ^ 1)); + + /* check page access */ + pp = pte_word2 & 3; + + // the following scenarios cause DSI/ISI exception: + // any access with key = 1 and PP = %00 + // write access with key = 1 and PP = %01 + // write access with PP = %11 + if ((key && (!pp || (pp == 1 && is_write))) || (pp == 3 && is_write)) { + if (is_instr_fetch) { + mmu_exception_handler(Except_Type::EXC_ISI, 0x08000000); + } else { + ppc_state.spr[SPR::DSISR] = 0x08000000 | (is_write << 25); + ppc_state.spr[SPR::DAR] = la; + mmu_exception_handler(Except_Type::EXC_DSI, 0); + } + } + + /* update R and C bits */ + /* For simplicity, R is set on each access, C is set only for writes */ + pte_addr[6] |= 0x01; + if (is_write) { + pte_addr[7] |= 0x80; + } + + /* return physical address, access protection and C status */ + return PATResult{ + ((pte_word2 & 0xFFFFF000) | (la & 0x00000FFF)), + static_cast((key << 2) | pp), + static_cast(pte_word2 & 0x80) + }; +} + +/** PowerPC-style MMU instruction address translation. */ +static uint32_t mmu_instr_translation(uint32_t la) +{ + uint32_t pa; /* translated physical address */ + + bool bat_hit = false; + unsigned msr_pr = !!(ppc_state.msr & 0x4000); + + // Format: %XY + // X - supervisor access bit, Y - problem/user access bit + // Those bits are mutually exclusive + unsigned access_bits = ((msr_pr ^ 1) << 1) | msr_pr; + + for (int bat_index = 0; bat_index < 4; bat_index++) { + PPC_BAT_entry* bat_entry = &ibat_array[bat_index]; + + if ((bat_entry->access & access_bits) && ((la & bat_entry->hi_mask) == bat_entry->bepi)) { + bat_hit = true; + +#ifdef MMU_PROFILING + bat_transl_total++; +#endif + + if (!bat_entry->prot) { + mmu_exception_handler(Except_Type::EXC_ISI, 0x08000000); + } + + // logical to physical translation + pa = bat_entry->phys_hi | (la & ~bat_entry->hi_mask); + break; + } + } + + /* page address translation */ + if (!bat_hit) { + PATResult pat_res = page_address_translation(la, true, msr_pr, 0); + pa = pat_res.phys; + +#ifdef MMU_PROFILING + ptab_transl_total++; +#endif + } + + return pa; +} + +/** PowerPC-style MMU data address translation. */ +static uint32_t ppc_mmu_addr_translate(uint32_t la, int is_write) +{ + uint32_t pa; /* translated physical address */ + + bool bat_hit = false; + unsigned msr_pr = !!(ppc_state.msr & 0x4000); + + // Format: %XY + // X - supervisor access bit, Y - problem/user access bit + // Those bits are mutually exclusive + unsigned access_bits = ((msr_pr ^ 1) << 1) | msr_pr; + + for (int bat_index = 0; bat_index < 4; bat_index++) { + PPC_BAT_entry* bat_entry = &dbat_array[bat_index]; + + if ((bat_entry->access & access_bits) && ((la & bat_entry->hi_mask) == bat_entry->bepi)) { + bat_hit = true; + +#ifdef MMU_PROFILING + bat_transl_total++; +#endif + + if (!bat_entry->prot || ((bat_entry->prot & 1) && is_write)) { + ppc_state.spr[SPR::DSISR] = 0x08000000 | (is_write << 25); + ppc_state.spr[SPR::DAR] = la; + mmu_exception_handler(Except_Type::EXC_DSI, 0); + } + + // logical to physical translation + pa = bat_entry->phys_hi | (la & ~bat_entry->hi_mask); + break; + } + } + + /* page address translation */ + if (!bat_hit) { + PATResult pat_res = page_address_translation(la, false, msr_pr, is_write); + pa = pat_res.phys; + +#ifdef MMU_PROFILING + ptab_transl_total++; +#endif + } + + return pa; +} + +uint8_t* mmu_get_dma_mem(uint32_t addr, uint32_t size) +{ + if (addr >= last_dma_area.start && (addr + size) <= last_dma_area.end) { + return last_dma_area.mem_ptr + (addr - last_dma_area.start); + } else { + AddressMapEntry* entry = mem_ctrl_instance->find_range(addr); + if (entry && entry->type & (RT_ROM | RT_RAM)) { + last_dma_area.start = entry->start; + last_dma_area.end = entry->end; + last_dma_area.mem_ptr = entry->mem_ptr; + return last_dma_area.mem_ptr + (addr - last_dma_area.start); + } else { + ABORT_F("SOS: DMA access to unmapped memory %08X!\n", addr); + } + } +} + +// primary ITLB for all MMU modes +static std::array itlb1_mode1; +static std::array itlb1_mode2; +static std::array itlb1_mode3; + +// secondary ITLB for all MMU modes +static std::array itlb2_mode1; +static std::array itlb2_mode2; +static std::array itlb2_mode3; + +// primary DTLB for all MMU modes +static std::array dtlb1_mode1; +static std::array dtlb1_mode2; +static std::array dtlb1_mode3; + +// secondary DTLB for all MMU modes +static std::array dtlb2_mode1; +static std::array dtlb2_mode2; +static std::array dtlb2_mode3; + +TLBEntry *pCurITLB1; // current primary ITLB +TLBEntry *pCurITLB2; // current secondary ITLB +TLBEntry *pCurDTLB1; // current primary DTLB +TLBEntry *pCurDTLB2; // current secondary DTLB + +uint32_t tlb_size_mask = TLB_SIZE - 1; + +// fake TLB entry for handling of unmapped memory accesses +uint64_t UnmappedVal = -1ULL; +TLBEntry UnmappedMem = {TLB_INVALID_TAG, 0, 0, 0}; + +uint8_t CurITLBMode = {0xFF}; // current ITLB mode +uint8_t CurDTLBMode = {0xFF}; // current DTLB mode + +void mmu_change_mode() +{ + uint8_t mmu_mode; + + // switch ITLB tables first + mmu_mode = ((ppc_state.msr >> 4) & 0x2) | ((ppc_state.msr >> 14) & 1); + + if (CurITLBMode != mmu_mode) { + switch(mmu_mode) { + case 0: // real address mode + pCurITLB1 = &dtlb1_mode1[0]; + pCurITLB2 = &dtlb2_mode1[0]; + break; + case 2: // supervisor mode with instruction translation enabled + pCurITLB1 = &dtlb1_mode2[0]; + pCurITLB2 = &dtlb2_mode2[0]; + break; + case 3: // user mode with instruction translation enabled + pCurITLB1 = &dtlb1_mode3[0]; + pCurITLB2 = &dtlb2_mode3[0]; + break; + } + CurITLBMode = mmu_mode; + } + + // then switch DTLB tables + mmu_mode = ((ppc_state.msr >> 3) & 0x2) | ((ppc_state.msr >> 14) & 1); + + if (CurDTLBMode != mmu_mode) { + switch(mmu_mode) { + case 0: // real address mode + pCurDTLB1 = &dtlb1_mode1[0]; + pCurDTLB2 = &dtlb2_mode1[0]; + break; + case 2: // supervisor mode with data translation enabled + pCurDTLB1 = &dtlb1_mode2[0]; + pCurDTLB2 = &dtlb2_mode2[0]; + break; + case 3: // user mode with data translation enabled + pCurDTLB1 = &dtlb1_mode3[0]; + pCurDTLB2 = &dtlb2_mode3[0]; + break; + } + CurDTLBMode = mmu_mode; + } +} + +template +static TLBEntry* tlb2_target_entry(uint32_t gp_va) +{ + TLBEntry *tlb_entry; + + if (tlb_type == TLBType::ITLB) { + tlb_entry = &pCurITLB2[((gp_va >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; + } else { + tlb_entry = &pCurDTLB2[((gp_va >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; + } + + // select the target from invalid blocks first + if (tlb_entry[0].tag == TLB_INVALID_TAG) { + // update LRU bits + tlb_entry[0].lru_bits = 0x3; + tlb_entry[1].lru_bits = 0x2; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + return tlb_entry; + } else if (tlb_entry[1].tag == TLB_INVALID_TAG) { + // update LRU bits + tlb_entry[0].lru_bits = 0x2; + tlb_entry[1].lru_bits = 0x3; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + return &tlb_entry[1]; + } else if (tlb_entry[2].tag == TLB_INVALID_TAG) { + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x3; + tlb_entry[3].lru_bits = 0x2; + return &tlb_entry[2]; + } else if (tlb_entry[3].tag == TLB_INVALID_TAG) { + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x2; + tlb_entry[3].lru_bits = 0x3; + return &tlb_entry[3]; + } else { // no free entries, replace an existing one according with the hLRU policy +#ifdef TLB_PROFILING + num_entry_replacements++; +#endif + if (tlb_entry[0].lru_bits == 0) { + // update LRU bits + tlb_entry[0].lru_bits = 0x3; + tlb_entry[1].lru_bits = 0x2; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + return tlb_entry; + } else if (tlb_entry[1].lru_bits == 0) { + // update LRU bits + tlb_entry[0].lru_bits = 0x2; + tlb_entry[1].lru_bits = 0x3; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + return &tlb_entry[1]; + } else if (tlb_entry[2].lru_bits == 0) { + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x3; + tlb_entry[3].lru_bits = 0x2; + return &tlb_entry[2]; + } else { + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x2; + tlb_entry[3].lru_bits = 0x3; + return &tlb_entry[3]; + } + } +} + +static TLBEntry* itlb2_refill(uint32_t guest_va) +{ + uint32_t phys_addr; + TLBEntry *tlb_entry; + uint16_t flags = 0; + + /* instruction address translation if enabled */ + if (ppc_state.msr & 0x20) { + // attempt block address translation first + BATResult bat_res = ppc_block_address_translation(guest_va); + if (bat_res.hit) { + // check block protection + // only PP = 0 (no access) causes ISI exception + if (!bat_res.prot) { + mmu_exception_handler(Except_Type::EXC_ISI, 0x08000000); + } + phys_addr = bat_res.phys; + flags |= TLBFlags::TLBE_FROM_BAT; // tell the world we come from + } else { + // page address translation + PATResult pat_res = page_address_translation(guest_va, true, + !!(ppc_state.msr & 0x4000), 0); + phys_addr = pat_res.phys; + flags = TLBFlags::TLBE_FROM_PAT; // tell the world we come from + } + } else { // instruction translation disabled + phys_addr = guest_va; + } + + // look up host virtual address + AddressMapEntry* reg_desc = mem_ctrl_instance->find_range(phys_addr); + if (reg_desc) { + if (reg_desc->type & RT_MMIO) { + ABORT_F("Instruction fetch from MMIO region at 0x%08X!\n", phys_addr); + } + // refill the secondary TLB + const uint32_t tag = guest_va & ~0xFFFUL; + tlb_entry = tlb2_target_entry(tag); + tlb_entry->tag = tag; + tlb_entry->flags = flags | TLBFlags::PAGE_MEM; + tlb_entry->host_va_offset = (int64_t)reg_desc->mem_ptr - guest_va + + (phys_addr - reg_desc->start); + } else { + ABORT_F("Instruction fetch from unmapped memory at 0x%08X!\n", phys_addr); + } + + return tlb_entry; +} + +static TLBEntry* dtlb2_refill(uint32_t guest_va, int is_write) +{ + uint32_t phys_addr; + uint16_t flags = 0; + TLBEntry *tlb_entry; + + const uint32_t tag = guest_va & ~0xFFFUL; + + /* data address translation if enabled */ + if (ppc_state.msr & 0x10) { + // attempt block address translation first + BATResult bat_res = ppc_block_address_translation(guest_va); + if (bat_res.hit) { + // check block protection + if (!bat_res.prot || ((bat_res.prot & 1) && is_write)) { + LOG_F(9, "BAT DSI exception in TLB2 refill!"); + LOG_F(9, "Attempt to write to read-only region, LA=0x%08X, PC=0x%08X!", guest_va, ppc_state.pc); + ppc_state.spr[SPR::DSISR] = 0x08000000 | (is_write << 25); + ppc_state.spr[SPR::DAR] = guest_va; + mmu_exception_handler(Except_Type::EXC_DSI, 0); + } + phys_addr = bat_res.phys; + flags = TLBFlags::PTE_SET_C; // prevent PTE.C updates for BAT + flags |= TLBFlags::TLBE_FROM_BAT; // tell the world we come from + if (bat_res.prot == 2) { + flags |= TLBFlags::PAGE_WRITABLE; + } + } else { + // page address translation + PATResult pat_res = page_address_translation(guest_va, false, + !!(ppc_state.msr & 0x4000), is_write); + phys_addr = pat_res.phys; + flags = TLBFlags::TLBE_FROM_PAT; // tell the world we come from + if (pat_res.prot <= 2 || pat_res.prot == 6) { + flags |= TLBFlags::PAGE_WRITABLE; + } + if (is_write || pat_res.pte_c_status) { + // C-bit of the PTE is already set so the TLB logic + // doesn't need to update it anymore + flags |= TLBFlags::PTE_SET_C; + } + } + } else { // data translation disabled + phys_addr = guest_va; + flags = TLBFlags::PTE_SET_C; // no PTE.C updates in real addressing mode + flags |= TLBFlags::PAGE_WRITABLE; // assume physical pages are writable + } + + // look up host virtual address + AddressMapEntry* reg_desc = mem_ctrl_instance->find_range(phys_addr); + if (reg_desc) { + // refill the secondary TLB + tlb_entry = tlb2_target_entry(tag); + tlb_entry->tag = tag; + if (reg_desc->type & RT_MMIO) { // MMIO region + tlb_entry->flags = flags | TLBFlags::PAGE_IO; + tlb_entry->reg_desc = reg_desc; + } else { // memory region backed by host memory + tlb_entry->flags = flags | TLBFlags::PAGE_MEM; + tlb_entry->host_va_offset = (int64_t)reg_desc->mem_ptr - guest_va + + (phys_addr - reg_desc->start); + } + return tlb_entry; + } else { + LOG_F(ERROR, "Read from unmapped memory at 0x%08X!\n", phys_addr); + UnmappedMem.tag = tag; + UnmappedMem.host_va_offset = (int64_t)(&UnmappedVal) - guest_va; + return &UnmappedMem; + } +} + +template +static inline TLBEntry* lookup_secondary_tlb(uint32_t guest_va, uint32_t tag) { + TLBEntry *tlb_entry; + + if (tlb_type == TLBType::ITLB) { + tlb_entry = &pCurITLB2[((guest_va >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; + } else { + tlb_entry = &pCurDTLB2[((guest_va >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; + } + + if (tlb_entry->tag == tag) { + // update LRU bits + tlb_entry[0].lru_bits = 0x3; + tlb_entry[1].lru_bits = 0x2; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + } else if (tlb_entry[1].tag == tag) { + tlb_entry = &tlb_entry[1]; + // update LRU bits + tlb_entry[0].lru_bits = 0x2; + tlb_entry[1].lru_bits = 0x3; + tlb_entry[2].lru_bits &= 0x1; + tlb_entry[3].lru_bits &= 0x1; + } else if (tlb_entry[2].tag == tag) { + tlb_entry = &tlb_entry[2]; + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x3; + tlb_entry[3].lru_bits = 0x2; + } else if (tlb_entry[3].tag == tag) { + tlb_entry = &tlb_entry[3]; + // update LRU bits + tlb_entry[0].lru_bits &= 0x1; + tlb_entry[1].lru_bits &= 0x1; + tlb_entry[2].lru_bits = 0x2; + tlb_entry[3].lru_bits = 0x3; + } else { + return nullptr; + } + return tlb_entry; +} + +uint8_t *mmu_translate_imem(uint32_t vaddr) +{ + TLBEntry *tlb1_entry, *tlb2_entry; + uint8_t *host_va; + +#ifdef MMU_PROFILING + exec_reads_total++; +#endif + + const uint32_t tag = vaddr & ~0xFFFUL; + + // look up guest virtual address in the primary ITLB + tlb1_entry = &pCurITLB1[(vaddr >> PAGE_SIZE_BITS) & tlb_size_mask]; + if (tlb1_entry->tag == tag) { // primary ITLB hit -> fast path +#ifdef TLB_PROFILING + num_primary_itlb_hits++; +#endif + host_va = (uint8_t *)(tlb1_entry->host_va_offset + vaddr); + } else { + // primary ITLB miss -> look up address in the secondary ITLB + tlb2_entry = lookup_secondary_tlb(vaddr, tag); + if (tlb2_entry == nullptr) { +#ifdef TLB_PROFILING + num_itlb_refills++; +#endif + // secondary ITLB miss -> + // perform full address translation and refill the secondary ITLB + tlb2_entry = itlb2_refill(vaddr); + } +#ifdef TLB_PROFILING + else { + num_secondary_itlb_hits++; + } +#endif + // refill the primary ITLB + tlb1_entry->tag = tag; + tlb1_entry->flags = tlb2_entry->flags; + tlb1_entry->host_va_offset = tlb2_entry->host_va_offset; + host_va = (uint8_t *)(tlb1_entry->host_va_offset + vaddr); + } + + ppc_set_cur_instruction(host_va); + + return host_va; +} + +// Forward declarations. +static uint32_t read_unaligned(uint32_t guest_va, uint8_t *host_va, uint32_t size); +static void write_unaligned(uint32_t guest_va, uint8_t *host_va, uint32_t value, + uint32_t size); + +template +inline T mmu_read_vmem(uint32_t guest_va) +{ + TLBEntry *tlb1_entry, *tlb2_entry; + uint8_t *host_va; + + const uint32_t tag = guest_va & ~0xFFFUL; + + // look up guest virtual address in the primary TLB + tlb1_entry = &pCurDTLB1[(guest_va >> PAGE_SIZE_BITS) & tlb_size_mask]; + if (tlb1_entry->tag == tag) { // primary TLB hit -> fast path +#ifdef TLB_PROFILING + num_primary_dtlb_hits++; +#endif + host_va = (uint8_t *)(tlb1_entry->host_va_offset + guest_va); + } else { + // primary TLB miss -> look up address in the secondary TLB + tlb2_entry = lookup_secondary_tlb(guest_va, tag); + if (tlb2_entry == nullptr) { +#ifdef TLB_PROFILING + num_dtlb_refills++; +#endif + // secondary TLB miss -> + // perform full address translation and refill the secondary TLB + tlb2_entry = dtlb2_refill(guest_va, 0); + } +#ifdef TLB_PROFILING + else { + num_secondary_dtlb_hits++; + } +#endif + + if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region? + // refill the primary TLB + tlb1_entry->tag = tag; + tlb1_entry->flags = tlb2_entry->flags; + tlb1_entry->host_va_offset = tlb2_entry->host_va_offset; + host_va = (uint8_t *)(tlb1_entry->host_va_offset + guest_va); + } else { // otherwise, it's an access to a memory-mapped device +#ifdef MMU_PROFILING + iomem_reads_total++; +#endif + return ( + tlb2_entry->reg_desc->devobj->read(tlb2_entry->reg_desc->start, + guest_va - tlb2_entry->reg_desc->start, + sizeof(T)) + ); + } + } + +#ifdef MMU_PROFILING + dmem_reads_total++; +#endif + + // handle unaligned memory accesses + if (sizeof(T) > 1 && (guest_va & (sizeof(T) - 1))) { + return read_unaligned(guest_va, host_va, sizeof(T)); + } + + // handle aligned memory accesses + switch(sizeof(T)) { + case 1: + return *host_va; + case 2: + return READ_WORD_BE_A(host_va); + case 4: + return READ_DWORD_BE_A(host_va); + case 8: + return READ_QWORD_BE_A(host_va); + } +} + +// explicitely instantiate all required mmu_read_vmem variants +template uint8_t mmu_read_vmem(uint32_t guest_va); +template uint16_t mmu_read_vmem(uint32_t guest_va); +template uint32_t mmu_read_vmem(uint32_t guest_va); +template uint64_t mmu_read_vmem(uint32_t guest_va); + +template +inline void mmu_write_vmem(uint32_t guest_va, T value) +{ + TLBEntry *tlb1_entry, *tlb2_entry; + uint8_t *host_va; + + const uint32_t tag = guest_va & ~0xFFFUL; + + // look up guest virtual address in the primary TLB + tlb1_entry = &pCurDTLB1[(guest_va >> PAGE_SIZE_BITS) & tlb_size_mask]; + if (tlb1_entry->tag == tag) { // primary TLB hit -> fast path +#ifdef TLB_PROFILING + num_primary_dtlb_hits++; +#endif + if (!(tlb1_entry->flags & TLBFlags::PAGE_WRITABLE)) { + ppc_state.spr[SPR::DSISR] = 0x08000000 | (1 << 25); + ppc_state.spr[SPR::DAR] = guest_va; + mmu_exception_handler(Except_Type::EXC_DSI, 0); + } + if (!(tlb1_entry->flags & TLBFlags::PTE_SET_C)) { + // perform full page address translation to update PTE.C bit + PATResult pat_res = page_address_translation(guest_va, false, + !!(ppc_state.msr & 0x4000), true); + tlb1_entry->flags |= TLBFlags::PTE_SET_C; + + // don't forget to update the secondary TLB as well + tlb2_entry = lookup_secondary_tlb(guest_va, tag); + if (tlb2_entry != nullptr) { + tlb2_entry->flags |= TLBFlags::PTE_SET_C; + } + } + host_va = (uint8_t *)(tlb1_entry->host_va_offset + guest_va); + } else { + // primary TLB miss -> look up address in the secondary TLB + tlb2_entry = lookup_secondary_tlb(guest_va, tag); + if (tlb2_entry == nullptr) { +#ifdef TLB_PROFILING + num_dtlb_refills++; +#endif + // secondary TLB miss -> + // perform full address translation and refill the secondary TLB + tlb2_entry = dtlb2_refill(guest_va, 1); + } +#ifdef TLB_PROFILING + else { + num_secondary_dtlb_hits++; + } +#endif + + if (!(tlb2_entry->flags & TLBFlags::PAGE_WRITABLE)) { + ppc_state.spr[SPR::DSISR] = 0x08000000 | (1 << 25); + ppc_state.spr[SPR::DAR] = guest_va; + mmu_exception_handler(Except_Type::EXC_DSI, 0); + } + + if (!(tlb2_entry->flags & TLBFlags::PTE_SET_C)) { + // perform full page address translation to update PTE.C bit + PATResult pat_res = page_address_translation(guest_va, false, + !!(ppc_state.msr & 0x4000), true); + tlb2_entry->flags |= TLBFlags::PTE_SET_C; + } + + if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region? + // refill the primary TLB + tlb1_entry->tag = tag; + tlb1_entry->flags = tlb2_entry->flags; + tlb1_entry->host_va_offset = tlb2_entry->host_va_offset; + host_va = (uint8_t *)(tlb1_entry->host_va_offset + guest_va); + } else { // otherwise, it's an access to a memory-mapped device +#ifdef MMU_PROFILING + iomem_writes_total++; +#endif + tlb2_entry->reg_desc->devobj->write(tlb2_entry->reg_desc->start, + guest_va - tlb2_entry->reg_desc->start, + value, sizeof(T)); + return; + } + } + +#ifdef MMU_PROFILING + dmem_writes_total++; +#endif + + // handle unaligned memory accesses + if (sizeof(T) > 1 && (guest_va & (sizeof(T) - 1))) { + write_unaligned(guest_va, host_va, value, sizeof(T)); + return; + } + + // handle aligned memory accesses + switch(sizeof(T)) { + case 1: + *host_va = value; + break; + case 2: + WRITE_WORD_BE_A(host_va, value); + break; + case 4: + WRITE_DWORD_BE_A(host_va, value); + break; + case 8: + WRITE_QWORD_BE_A(host_va, value); + break; + } +} + +// explicitely instantiate all required mmu_write_vmem variants +template void mmu_write_vmem(uint32_t guest_va, uint8_t value); +template void mmu_write_vmem(uint32_t guest_va, uint16_t value); +template void mmu_write_vmem(uint32_t guest_va, uint32_t value); +template void mmu_write_vmem(uint32_t guest_va, uint64_t value); + +static uint32_t read_unaligned(uint32_t guest_va, uint8_t *host_va, uint32_t size) +{ + uint32_t result = 0; + + // is it a misaligned cross-page read? + if (((guest_va & 0xFFF) + size) > 0x1000) { +#ifdef MMU_PROFILING + unaligned_crossp_r++; +#endif + // Break such a memory access into multiple, bytewise accesses. + // Because such accesses suffer a performance penalty, they will be + // presumably very rare so don't waste time optimizing the code below. + for (int i = 0; i < size; guest_va++, i++) { + result = (result << 8) | mmu_read_vmem(guest_va); + } + } else { +#ifdef MMU_PROFILING + unaligned_reads++; +#endif + switch(size) { + case 2: + return READ_WORD_BE_U(host_va); + case 4: + return READ_DWORD_BE_U(host_va); + case 8: // FIXME: should we raise alignment exception here? + return READ_QWORD_BE_U(host_va); + } + } + return result; +} + +static void write_unaligned(uint32_t guest_va, uint8_t *host_va, uint32_t value, + uint32_t size) +{ + // is it a misaligned cross-page write? + if (((guest_va & 0xFFF) + size) > 0x1000) { +#ifdef MMU_PROFILING + unaligned_crossp_w++; +#endif + // Break such a memory access into multiple, bytewise accesses. + // Because such accesses suffer a performance penalty, they will be + // presumably very rare so don't waste time optimizing the code below. + + uint32_t shift = (size - 1) * 8; + + for (int i = 0; i < size; shift -= 8, guest_va++, i++) { + mmu_write_vmem(guest_va, (value >> shift) & 0xFF); + } + } else { +#ifdef MMU_PROFILING + unaligned_writes++; +#endif + switch(size) { + case 2: + WRITE_WORD_BE_U(host_va, value); + break; + case 4: + WRITE_DWORD_BE_U(host_va, value); + break; + case 8: // FIXME: should we raise alignment exception here? + WRITE_QWORD_BE_U(host_va, value); + break; + } + } +} + +void tlb_flush_entry(uint32_t ea) +{ + TLBEntry *tlb_entry, *tlb1, *tlb2; + + const uint32_t tag = ea & ~0xFFFUL; + + for (int m = 0; m < 6; m++) { + switch (m) { + case 0: + tlb1 = &itlb1_mode1[0]; + tlb2 = &itlb2_mode1[0]; + break; + case 1: + tlb1 = &itlb1_mode2[0]; + tlb2 = &itlb2_mode2[0]; + break; + case 2: + tlb1 = &itlb1_mode3[0]; + tlb2 = &itlb2_mode3[0]; + break; + case 3: + tlb1 = &dtlb1_mode1[0]; + tlb2 = &dtlb2_mode1[0]; + break; + case 4: + tlb1 = &dtlb1_mode2[0]; + tlb2 = &dtlb2_mode2[0]; + break; + case 5: + tlb1 = &dtlb1_mode3[0]; + tlb2 = &dtlb2_mode3[0]; + break; + } + + // flush primary TLB + tlb_entry = &tlb1[(ea >> PAGE_SIZE_BITS) & tlb_size_mask]; + if (tlb_entry->tag == tag) { + tlb_entry->tag = TLB_INVALID_TAG; + //LOG_F(INFO, "Invalidated primary TLB entry at 0x%X", ea); + } + + // flush secondary TLB + tlb_entry = &tlb2[((ea >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; + for (int i = 0; i < TLB2_WAYS; i++) { + if (tlb_entry[i].tag == tag) { + tlb_entry[i].tag = TLB_INVALID_TAG; + //LOG_F(INFO, "Invalidated secondary TLB entry at 0x%X", ea); + } + } + } +} + +void tlb_flush_entries(TLBFlags type) +{ + int i; + + // Flush BAT entries from the primary TLBs + for (i = 0; i < TLB_SIZE; i++) { + if (dtlb1_mode2[i].flags & type) { + dtlb1_mode2[i].tag = TLB_INVALID_TAG; + } + + if (dtlb1_mode3[i].flags & type) { + dtlb1_mode3[i].tag = TLB_INVALID_TAG; + } + } + + // Flush BAT entries from the secondary TLBs + for (i = 0; i < TLB_SIZE * TLB2_WAYS; i++) { + if (dtlb2_mode2[i].flags & type) { + dtlb2_mode2[i].tag = TLB_INVALID_TAG; + } + + if (dtlb2_mode3[i].flags & type) { + dtlb2_mode3[i].tag = TLB_INVALID_TAG; + } + } +} + +bool gTLBFlushBatEntries = false; +bool gTLBFlushPatEntries = false; + +void tlb_flush_bat_entries() +{ + if (!gTLBFlushBatEntries) + return; + + tlb_flush_entries(TLBE_FROM_BAT); + + gTLBFlushBatEntries = false; +} + +void tlb_flush_pat_entries() +{ + if (!gTLBFlushPatEntries) + return; + + tlb_flush_entries(TLBE_FROM_PAT); + + gTLBFlushPatEntries = false; +} + +void ibat_update(uint32_t bat_reg) +{ + int upper_reg_num; + uint32_t bl, hi_mask; + PPC_BAT_entry* bat_entry; + + upper_reg_num = bat_reg & 0xFFFFFFFE; + + if (ppc_state.spr[upper_reg_num] & 3) { // is that BAT pair valid? + bat_entry = &ibat_array[(bat_reg - 528) >> 1]; + bl = (ppc_state.spr[upper_reg_num] >> 2) & 0x7FF; + hi_mask = ~((bl << 17) | 0x1FFFF); + + bat_entry->access = ppc_state.spr[upper_reg_num] & 3; + bat_entry->prot = ppc_state.spr[upper_reg_num + 1] & 3; + bat_entry->hi_mask = hi_mask; + bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask; + bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask; + + if (!gTLBFlushBatEntries) { + gTLBFlushBatEntries = true; + add_ctx_sync_action(&tlb_flush_bat_entries); + } + } +} + +void dbat_update(uint32_t bat_reg) +{ + int upper_reg_num; + uint32_t bl, hi_mask; + PPC_BAT_entry* bat_entry; + + upper_reg_num = bat_reg & 0xFFFFFFFE; + + if (ppc_state.spr[upper_reg_num] & 3) { // is that BAT pair valid? + bat_entry = &dbat_array[(bat_reg - 536) >> 1]; + bl = (ppc_state.spr[upper_reg_num] >> 2) & 0x7FF; + hi_mask = ~((bl << 17) | 0x1FFFF); + + bat_entry->access = ppc_state.spr[upper_reg_num] & 3; + bat_entry->prot = ppc_state.spr[upper_reg_num + 1] & 3; + bat_entry->hi_mask = hi_mask; + bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask; + bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask; + + if (!gTLBFlushBatEntries) { + gTLBFlushBatEntries = true; + add_ctx_sync_action(&tlb_flush_bat_entries); + } + } +} + +void mmu_pat_ctx_changed() +{ + if (!gTLBFlushPatEntries) { + gTLBFlushPatEntries = true; + add_ctx_sync_action(&tlb_flush_pat_entries); + } +} + +/* MMU profiling. */ +#ifdef MMU_PROFILING + #include "utils/profiler.h" #include @@ -139,15 +1233,6 @@ public: /* SoftTLB profiling. */ #ifdef TLB_PROFILING -/* global variables for lightweight SoftTLB profiling */ -uint64_t num_primary_itlb_hits = 0; // number of hits in the primary ITLB -uint64_t num_secondary_itlb_hits = 0; // number of hits in the secondary ITLB -uint64_t num_itlb_refills = 0; // number of ITLB refills -uint64_t num_primary_dtlb_hits = 0; // number of hits in the primary DTLB -uint64_t num_secondary_dtlb_hits = 0; // number of hits in the secondary DTLB -uint64_t num_dtlb_refills = 0; // number of DTLB refills -uint64_t num_entry_replacements = 0; // number of entry replacements - #include "utils/profiler.h" #include @@ -196,15 +1281,8 @@ public: }; #endif - -/** remember recently used physical memory regions for quicker translation. */ -AddressMapEntry last_read_area = {0xFFFFFFFF, 0xFFFFFFFF}; -AddressMapEntry last_write_area = {0xFFFFFFFF, 0xFFFFFFFF}; -AddressMapEntry last_exec_area = {0xFFFFFFFF, 0xFFFFFFFF}; -AddressMapEntry last_ptab_area = {0xFFFFFFFF, 0xFFFFFFFF}; -AddressMapEntry last_dma_area = {0xFFFFFFFF, 0xFFFFFFFF}; - - +//=================== Old and slow code. Kept for reference ================= +#if 0 template static inline T read_phys_mem(AddressMapEntry *mru_rgn, uint32_t addr) { @@ -316,349 +1394,6 @@ static inline void write_phys_mem(AddressMapEntry *mru_rgn, uint32_t addr, T val } } -uint8_t* mmu_get_dma_mem(uint32_t addr, uint32_t size) { - if (addr >= last_dma_area.start && (addr + size) <= last_dma_area.end) { - return last_dma_area.mem_ptr + (addr - last_dma_area.start); - } else { - AddressMapEntry* entry = mem_ctrl_instance->find_range(addr); - if (entry && entry->type & (RT_ROM | RT_RAM)) { - last_dma_area.start = entry->start; - last_dma_area.end = entry->end; - last_dma_area.mem_ptr = entry->mem_ptr; - return last_dma_area.mem_ptr + (addr - last_dma_area.start); - } else { - ABORT_F("SOS: DMA access to unmapped memory %08X!\n", addr); - } - } -} - -void ppc_set_cur_instruction(const uint8_t* ptr) { - ppc_cur_instruction = READ_DWORD_BE_A(ptr); -} - -bool gTLBFlushBatEntries = false; -bool gTLBFlushPatEntries = false; - -// Forward declarations. -void tlb_flush_bat_entries(); -void tlb_flush_pat_entries(); - -void ibat_update(uint32_t bat_reg) { - int upper_reg_num; - uint32_t bl, hi_mask; - PPC_BAT_entry* bat_entry; - - upper_reg_num = bat_reg & 0xFFFFFFFE; - - if (ppc_state.spr[upper_reg_num] & 3) { // is that BAT pair valid? - bat_entry = &ibat_array[(bat_reg - 528) >> 1]; - bl = (ppc_state.spr[upper_reg_num] >> 2) & 0x7FF; - hi_mask = ~((bl << 17) | 0x1FFFF); - - bat_entry->access = ppc_state.spr[upper_reg_num] & 3; - bat_entry->prot = ppc_state.spr[upper_reg_num + 1] & 3; - bat_entry->hi_mask = hi_mask; - bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask; - bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask; - - if (!gTLBFlushBatEntries) { - gTLBFlushBatEntries = true; - add_ctx_sync_action(&tlb_flush_bat_entries); - } - } -} - -void dbat_update(uint32_t bat_reg) { - int upper_reg_num; - uint32_t bl, hi_mask; - PPC_BAT_entry* bat_entry; - - upper_reg_num = bat_reg & 0xFFFFFFFE; - - if (ppc_state.spr[upper_reg_num] & 3) { // is that BAT pair valid? - bat_entry = &dbat_array[(bat_reg - 536) >> 1]; - bl = (ppc_state.spr[upper_reg_num] >> 2) & 0x7FF; - hi_mask = ~((bl << 17) | 0x1FFFF); - - bat_entry->access = ppc_state.spr[upper_reg_num] & 3; - bat_entry->prot = ppc_state.spr[upper_reg_num + 1] & 3; - bat_entry->hi_mask = hi_mask; - bat_entry->phys_hi = ppc_state.spr[upper_reg_num + 1] & hi_mask; - bat_entry->bepi = ppc_state.spr[upper_reg_num] & hi_mask; - - if (!gTLBFlushBatEntries) { - gTLBFlushBatEntries = true; - add_ctx_sync_action(&tlb_flush_bat_entries); - } - } -} - -void mmu_pat_ctx_changed() -{ - if (!gTLBFlushPatEntries) { - gTLBFlushPatEntries = true; - add_ctx_sync_action(&tlb_flush_pat_entries); - } -} - -/** PowerPC-style block address translation. */ -template -static BATResult ppc_block_address_translation(uint32_t la) -{ - uint32_t pa; // translated physical address - uint8_t prot; // protection bits for the translated address - PPC_BAT_entry *bat_array; - - bool bat_hit = false; - unsigned msr_pr = !!(ppc_state.msr & 0x4000); - - bat_array = (type == BATType::IBAT) ? ibat_array : dbat_array; - - // Format: %XY - // X - supervisor access bit, Y - problem/user access bit - // Those bits are mutually exclusive - unsigned access_bits = ((msr_pr ^ 1) << 1) | msr_pr; - - for (int bat_index = 0; bat_index < 4; bat_index++) { - PPC_BAT_entry* bat_entry = &bat_array[bat_index]; - - if ((bat_entry->access & access_bits) && ((la & bat_entry->hi_mask) == bat_entry->bepi)) { - bat_hit = true; - -#ifdef MMU_PROFILING - bat_transl_total++; -#endif - // logical to physical translation - pa = bat_entry->phys_hi | (la & ~bat_entry->hi_mask); - prot = bat_entry->prot; - break; - } - } - - return BATResult{bat_hit, prot, pa}; -} - -static inline uint8_t* calc_pteg_addr(uint32_t hash) { - uint32_t sdr1_val, pteg_addr; - - sdr1_val = ppc_state.spr[SPR::SDR1]; - - pteg_addr = sdr1_val & 0xFE000000; - pteg_addr |= (sdr1_val & 0x01FF0000) | (((sdr1_val & 0x1FF) << 16) & ((hash & 0x7FC00) << 6)); - pteg_addr |= (hash & 0x3FF) << 6; - - if (pteg_addr >= last_ptab_area.start && pteg_addr <= last_ptab_area.end) { - return last_ptab_area.mem_ptr + (pteg_addr - last_ptab_area.start); - } else { - AddressMapEntry* entry = mem_ctrl_instance->find_range(pteg_addr); - if (entry && entry->type & (RT_ROM | RT_RAM)) { - last_ptab_area.start = entry->start; - last_ptab_area.end = entry->end; - last_ptab_area.mem_ptr = entry->mem_ptr; - return last_ptab_area.mem_ptr + (pteg_addr - last_ptab_area.start); - } else { - ABORT_F("SOS: no page table region was found at %08X!\n", pteg_addr); - } - } -} - -static bool search_pteg( - uint8_t* pteg_addr, uint8_t** ret_pte_addr, uint32_t vsid, uint16_t page_index, uint8_t pteg_num) { - /* construct PTE matching word */ - uint32_t pte_check = 0x80000000 | (vsid << 7) | (pteg_num << 6) | (page_index >> 10); - -#ifdef MMU_INTEGRITY_CHECKS - /* PTEG integrity check that ensures that all matching PTEs have - identical RPN, WIMG and PP bits (PPC PEM 32-bit 7.6.2, rule 5). */ - uint32_t pte_word2_check; - bool match_found = false; - - for (int i = 0; i < 8; i++, pteg_addr += 8) { - if (pte_check == READ_DWORD_BE_A(pteg_addr)) { - if (match_found) { - if ((READ_DWORD_BE_A(pteg_addr) & 0xFFFFF07B) != pte_word2_check) { - ABORT_F("Multiple PTEs with different RPN/WIMG/PP found!\n"); - } - } else { - /* isolate RPN, WIMG and PP fields */ - pte_word2_check = READ_DWORD_BE_A(pteg_addr) & 0xFFFFF07B; - *ret_pte_addr = pteg_addr; - } - } - } -#else - for (int i = 0; i < 8; i++, pteg_addr += 8) { - if (pte_check == READ_DWORD_BE_A(pteg_addr)) { - *ret_pte_addr = pteg_addr; - return true; - } - } -#endif - - return false; -} - -static PATResult page_address_translation(uint32_t la, bool is_instr_fetch, - unsigned msr_pr, int is_write) -{ - uint32_t sr_val, page_index, pteg_hash1, vsid, pte_word2; - unsigned key, pp; - uint8_t* pte_addr; - - sr_val = ppc_state.sr[(la >> 28) & 0x0F]; - if (sr_val & 0x80000000) { - ABORT_F("Direct-store segments not supported, LA=%0xX\n", la); - } - - /* instruction fetch from a no-execute segment will cause ISI exception */ - if ((sr_val & 0x10000000) && is_instr_fetch) { - mmu_exception_handler(Except_Type::EXC_ISI, 0x10000000); - } - - page_index = (la >> 12) & 0xFFFF; - pteg_hash1 = (sr_val & 0x7FFFF) ^ page_index; - vsid = sr_val & 0x0FFFFFF; - - if (!search_pteg(calc_pteg_addr(pteg_hash1), &pte_addr, vsid, page_index, 0)) { - if (!search_pteg(calc_pteg_addr(~pteg_hash1), &pte_addr, vsid, page_index, 1)) { - if (is_instr_fetch) { - mmu_exception_handler(Except_Type::EXC_ISI, 0x40000000); - } else { - ppc_state.spr[SPR::DSISR] = 0x40000000 | (is_write << 25); - ppc_state.spr[SPR::DAR] = la; - mmu_exception_handler(Except_Type::EXC_DSI, 0); - } - } - } - - pte_word2 = READ_DWORD_BE_A(pte_addr + 4); - - key = (((sr_val >> 29) & 1) & msr_pr) | (((sr_val >> 30) & 1) & (msr_pr ^ 1)); - - /* check page access */ - pp = pte_word2 & 3; - - // the following scenarios cause DSI/ISI exception: - // any access with key = 1 and PP = %00 - // write access with key = 1 and PP = %01 - // write access with PP = %11 - if ((key && (!pp || (pp == 1 && is_write))) || (pp == 3 && is_write)) { - if (is_instr_fetch) { - mmu_exception_handler(Except_Type::EXC_ISI, 0x08000000); - } else { - ppc_state.spr[SPR::DSISR] = 0x08000000 | (is_write << 25); - ppc_state.spr[SPR::DAR] = la; - mmu_exception_handler(Except_Type::EXC_DSI, 0); - } - } - - /* update R and C bits */ - /* For simplicity, R is set on each access, C is set only for writes */ - pte_addr[6] |= 0x01; - if (is_write) { - pte_addr[7] |= 0x80; - } - - /* return physical address, access protection and C status */ - return PATResult{ - ((pte_word2 & 0xFFFFF000) | (la & 0x00000FFF)), - static_cast((key << 2) | pp), - static_cast(pte_word2 & 0x80) - }; -} - -/** PowerPC-style MMU instruction address translation. */ -static uint32_t mmu_instr_translation(uint32_t la) { - uint32_t pa; /* translated physical address */ - - bool bat_hit = false; - unsigned msr_pr = !!(ppc_state.msr & 0x4000); - - // Format: %XY - // X - supervisor access bit, Y - problem/user access bit - // Those bits are mutually exclusive - unsigned access_bits = ((msr_pr ^ 1) << 1) | msr_pr; - - for (int bat_index = 0; bat_index < 4; bat_index++) { - PPC_BAT_entry* bat_entry = &ibat_array[bat_index]; - - if ((bat_entry->access & access_bits) && ((la & bat_entry->hi_mask) == bat_entry->bepi)) { - bat_hit = true; - -#ifdef MMU_PROFILING - bat_transl_total++; -#endif - - if (!bat_entry->prot) { - mmu_exception_handler(Except_Type::EXC_ISI, 0x08000000); - } - - // logical to physical translation - pa = bat_entry->phys_hi | (la & ~bat_entry->hi_mask); - break; - } - } - - /* page address translation */ - if (!bat_hit) { - PATResult pat_res = page_address_translation(la, true, msr_pr, 0); - pa = pat_res.phys; - -#ifdef MMU_PROFILING - ptab_transl_total++; -#endif - } - - return pa; -} - -/** PowerPC-style MMU data address translation. */ -static uint32_t ppc_mmu_addr_translate(uint32_t la, int is_write) { - uint32_t pa; /* translated physical address */ - - bool bat_hit = false; - unsigned msr_pr = !!(ppc_state.msr & 0x4000); - - // Format: %XY - // X - supervisor access bit, Y - problem/user access bit - // Those bits are mutually exclusive - unsigned access_bits = ((msr_pr ^ 1) << 1) | msr_pr; - - for (int bat_index = 0; bat_index < 4; bat_index++) { - PPC_BAT_entry* bat_entry = &dbat_array[bat_index]; - - if ((bat_entry->access & access_bits) && ((la & bat_entry->hi_mask) == bat_entry->bepi)) { - bat_hit = true; - -#ifdef MMU_PROFILING - bat_transl_total++; -#endif - - if (!bat_entry->prot || ((bat_entry->prot & 1) && is_write)) { - ppc_state.spr[SPR::DSISR] = 0x08000000 | (is_write << 25); - ppc_state.spr[SPR::DAR] = la; - mmu_exception_handler(Except_Type::EXC_DSI, 0); - } - - // logical to physical translation - pa = bat_entry->phys_hi | (la & ~bat_entry->hi_mask); - break; - } - } - - /* page address translation */ - if (!bat_hit) { - PATResult pat_res = page_address_translation(la, false, msr_pr, is_write); - pa = pat_res.phys; - -#ifdef MMU_PROFILING - ptab_transl_total++; -#endif - } - - return pa; -} - static void mem_write_unaligned(uint32_t addr, uint32_t value, uint32_t size) { #ifdef MMU_DEBUG LOG_F(WARNING, "Attempt to write unaligned %d bytes to 0x%08X\n", size, addr); @@ -703,377 +1438,6 @@ static void mem_write_unaligned(uint32_t addr, uint32_t value, uint32_t size) { } } -// primary ITLB for all MMU modes -static std::array itlb1_mode1; -static std::array itlb1_mode2; -static std::array itlb1_mode3; - -// secondary ITLB for all MMU modes -static std::array itlb2_mode1; -static std::array itlb2_mode2; -static std::array itlb2_mode3; - -// primary DTLB for all MMU modes -static std::array dtlb1_mode1; -static std::array dtlb1_mode2; -static std::array dtlb1_mode3; - -// secondary DTLB for all MMU modes -static std::array dtlb2_mode1; -static std::array dtlb2_mode2; -static std::array dtlb2_mode3; - -TLBEntry *pCurITLB1; // current primary ITLB -TLBEntry *pCurITLB2; // current secondary ITLB -TLBEntry *pCurDTLB1; // current primary DTLB -TLBEntry *pCurDTLB2; // current secondary DTLB - -uint32_t tlb_size_mask = TLB_SIZE - 1; - -// fake TLB entry for handling of unmapped memory accesses -uint64_t UnmappedVal = -1ULL; -TLBEntry UnmappedMem = {TLB_INVALID_TAG, 0, 0, 0}; - -uint8_t CurITLBMode = {0xFF}; // current ITLB mode -uint8_t CurDTLBMode = {0xFF}; // current DTLB mode - -void mmu_change_mode() -{ - uint8_t mmu_mode; - - // switch ITLB tables first - mmu_mode = ((ppc_state.msr >> 4) & 0x2) | ((ppc_state.msr >> 14) & 1); - - if (CurITLBMode != mmu_mode) { - switch(mmu_mode) { - case 0: // real address mode - pCurITLB1 = &dtlb1_mode1[0]; - pCurITLB2 = &dtlb2_mode1[0]; - break; - case 2: // supervisor mode with instruction translation enabled - pCurITLB1 = &dtlb1_mode2[0]; - pCurITLB2 = &dtlb2_mode2[0]; - break; - case 3: // user mode with instruction translation enabled - pCurITLB1 = &dtlb1_mode3[0]; - pCurITLB2 = &dtlb2_mode3[0]; - break; - } - CurITLBMode = mmu_mode; - } - - // then switch DTLB tables - mmu_mode = ((ppc_state.msr >> 3) & 0x2) | ((ppc_state.msr >> 14) & 1); - - if (CurDTLBMode != mmu_mode) { - switch(mmu_mode) { - case 0: // real address mode - pCurDTLB1 = &dtlb1_mode1[0]; - pCurDTLB2 = &dtlb2_mode1[0]; - break; - case 2: // supervisor mode with data translation enabled - pCurDTLB1 = &dtlb1_mode2[0]; - pCurDTLB2 = &dtlb2_mode2[0]; - break; - case 3: // user mode with data translation enabled - pCurDTLB1 = &dtlb1_mode3[0]; - pCurDTLB2 = &dtlb2_mode3[0]; - break; - } - CurDTLBMode = mmu_mode; - } -} - -static TLBEntry* tlb2_target_entry(uint32_t gp_va) -{ - TLBEntry *tlb_entry; - - tlb_entry = &pCurDTLB2[((gp_va >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; - - // select the target from invalid blocks first - if (tlb_entry[0].tag == TLB_INVALID_TAG) { - // update LRU bits - tlb_entry[0].lru_bits = 0x3; - tlb_entry[1].lru_bits = 0x2; - tlb_entry[2].lru_bits &= 0x1; - tlb_entry[3].lru_bits &= 0x1; - return tlb_entry; - } else if (tlb_entry[1].tag == TLB_INVALID_TAG) { - // update LRU bits - tlb_entry[0].lru_bits = 0x2; - tlb_entry[1].lru_bits = 0x3; - tlb_entry[2].lru_bits &= 0x1; - tlb_entry[3].lru_bits &= 0x1; - return &tlb_entry[1]; - } else if (tlb_entry[2].tag == TLB_INVALID_TAG) { - // update LRU bits - tlb_entry[0].lru_bits &= 0x1; - tlb_entry[1].lru_bits &= 0x1; - tlb_entry[2].lru_bits = 0x3; - tlb_entry[3].lru_bits = 0x2; - return &tlb_entry[2]; - } else if (tlb_entry[3].tag == TLB_INVALID_TAG) { - // update LRU bits - tlb_entry[0].lru_bits &= 0x1; - tlb_entry[1].lru_bits &= 0x1; - tlb_entry[2].lru_bits = 0x2; - tlb_entry[3].lru_bits = 0x3; - return &tlb_entry[3]; - } else { // no free entries, replace an existing one according with the hLRU policy -#ifdef TLB_PROFILING - num_entry_replacements++; -#endif - if (tlb_entry[0].lru_bits == 0) { - // update LRU bits - tlb_entry[0].lru_bits = 0x3; - tlb_entry[1].lru_bits = 0x2; - tlb_entry[2].lru_bits &= 0x1; - tlb_entry[3].lru_bits &= 0x1; - return tlb_entry; - } else if (tlb_entry[1].lru_bits == 0) { - // update LRU bits - tlb_entry[0].lru_bits = 0x2; - tlb_entry[1].lru_bits = 0x3; - tlb_entry[2].lru_bits &= 0x1; - tlb_entry[3].lru_bits &= 0x1; - return &tlb_entry[1]; - } else if (tlb_entry[2].lru_bits == 0) { - // update LRU bits - tlb_entry[0].lru_bits &= 0x1; - tlb_entry[1].lru_bits &= 0x1; - tlb_entry[2].lru_bits = 0x3; - tlb_entry[3].lru_bits = 0x2; - return &tlb_entry[2]; - } else { - // update LRU bits - tlb_entry[0].lru_bits &= 0x1; - tlb_entry[1].lru_bits &= 0x1; - tlb_entry[2].lru_bits = 0x2; - tlb_entry[3].lru_bits = 0x3; - return &tlb_entry[3]; - } - } -} - -static TLBEntry* itlb2_refill(uint32_t guest_va) -{ - uint32_t phys_addr; - TLBEntry *tlb_entry; - uint16_t flags = 0; - - /* instruction address translation if enabled */ - if (ppc_state.msr & 0x20) { - // attempt block address translation first - BATResult bat_res = ppc_block_address_translation(guest_va); - if (bat_res.hit) { - // check block protection - // only PP = 0 (no access) causes ISI exception - if (!bat_res.prot) { - mmu_exception_handler(Except_Type::EXC_ISI, 0x08000000); - } - phys_addr = bat_res.phys; - flags |= TLBFlags::TLBE_FROM_BAT; // tell the world we come from - } else { - // page address translation - PATResult pat_res = page_address_translation(guest_va, true, - !!(ppc_state.msr & 0x4000), 0); - phys_addr = pat_res.phys; - flags = TLBFlags::TLBE_FROM_PAT; // tell the world we come from - } - } else { // instruction translation disabled - phys_addr = guest_va; - } - - // look up host virtual address - AddressMapEntry* reg_desc = mem_ctrl_instance->find_range(phys_addr); - if (reg_desc) { - if (reg_desc->type & RT_MMIO) { - ABORT_F("Instruction fetch from MMIO region at 0x%08X!\n", phys_addr); - } - // refill the secondary TLB - const uint32_t tag = guest_va & ~0xFFFUL; - tlb_entry = tlb2_target_entry(tag); - tlb_entry->tag = tag; - tlb_entry->flags = flags | TLBFlags::PAGE_MEM; - tlb_entry->host_va_offset = (int64_t)reg_desc->mem_ptr - guest_va + - (phys_addr - reg_desc->start); - } else { - ABORT_F("Instruction fetch from unmapped memory at 0x%08X!\n", phys_addr); - } - - return tlb_entry; -} - -static TLBEntry* dtlb2_refill(uint32_t guest_va, int is_write) -{ - uint32_t phys_addr; - uint16_t flags = 0; - TLBEntry *tlb_entry; - - const uint32_t tag = guest_va & ~0xFFFUL; - - /* data address translation if enabled */ - if (ppc_state.msr & 0x10) { - // attempt block address translation first - BATResult bat_res = ppc_block_address_translation(guest_va); - if (bat_res.hit) { - // check block protection - if (!bat_res.prot || ((bat_res.prot & 1) && is_write)) { - LOG_F(9, "BAT DSI exception in TLB2 refill!"); - LOG_F(9, "Attempt to write to read-only region, LA=0x%08X, PC=0x%08X!", guest_va, ppc_state.pc); - ppc_state.spr[SPR::DSISR] = 0x08000000 | (is_write << 25); - ppc_state.spr[SPR::DAR] = guest_va; - mmu_exception_handler(Except_Type::EXC_DSI, 0); - } - phys_addr = bat_res.phys; - flags = TLBFlags::PTE_SET_C; // prevent PTE.C updates for BAT - flags |= TLBFlags::TLBE_FROM_BAT; // tell the world we come from - if (bat_res.prot == 2) { - flags |= TLBFlags::PAGE_WRITABLE; - } - } else { - // page address translation - PATResult pat_res = page_address_translation(guest_va, false, - !!(ppc_state.msr & 0x4000), is_write); - phys_addr = pat_res.phys; - flags = TLBFlags::TLBE_FROM_PAT; // tell the world we come from - if (pat_res.prot <= 2 || pat_res.prot == 6) { - flags |= TLBFlags::PAGE_WRITABLE; - } - if (is_write || pat_res.pte_c_status) { - // C-bit of the PTE is already set so the TLB logic - // doesn't need to update it anymore - flags |= TLBFlags::PTE_SET_C; - } - } - } else { // data translation disabled - phys_addr = guest_va; - flags = TLBFlags::PTE_SET_C; // no PTE.C updates in real addressing mode - flags |= TLBFlags::PAGE_WRITABLE; // assume physical pages are writable - } - - // look up host virtual address - AddressMapEntry* reg_desc = mem_ctrl_instance->find_range(phys_addr); - if (reg_desc) { - // refill the secondary TLB - tlb_entry = tlb2_target_entry(tag); - tlb_entry->tag = tag; - if (reg_desc->type & RT_MMIO) { // MMIO region - tlb_entry->flags = flags | TLBFlags::PAGE_IO; - tlb_entry->reg_desc = reg_desc; - } else { // memory region backed by host memory - tlb_entry->flags = flags | TLBFlags::PAGE_MEM; - tlb_entry->host_va_offset = (int64_t)reg_desc->mem_ptr - guest_va + - (phys_addr - reg_desc->start); - } - return tlb_entry; - } else { - LOG_F(ERROR, "Read from unmapped memory at 0x%08X!\n", phys_addr); - UnmappedMem.tag = tag; - UnmappedMem.host_va_offset = (int64_t)(&UnmappedVal) - guest_va; - return &UnmappedMem; - } -} - -void tlb_flush_entry(uint32_t ea) -{ - TLBEntry *tlb_entry, *tlb1, *tlb2; - - const uint32_t tag = ea & ~0xFFFUL; - - for (int m = 0; m < 6; m++) { - switch (m) { - case 0: - tlb1 = &itlb1_mode1[0]; - tlb2 = &itlb2_mode1[0]; - break; - case 1: - tlb1 = &itlb1_mode2[0]; - tlb2 = &itlb2_mode2[0]; - break; - case 2: - tlb1 = &itlb1_mode3[0]; - tlb2 = &itlb2_mode3[0]; - break; - case 3: - tlb1 = &dtlb1_mode1[0]; - tlb2 = &dtlb2_mode1[0]; - break; - case 4: - tlb1 = &dtlb1_mode2[0]; - tlb2 = &dtlb2_mode2[0]; - break; - case 5: - tlb1 = &dtlb1_mode3[0]; - tlb2 = &dtlb2_mode3[0]; - break; - } - - // flush primary TLB - tlb_entry = &tlb1[(ea >> PAGE_SIZE_BITS) & tlb_size_mask]; - if (tlb_entry->tag == tag) { - tlb_entry->tag = TLB_INVALID_TAG; - //LOG_F(INFO, "Invalidated primary TLB entry at 0x%X", ea); - } - - // flush secondary TLB - tlb_entry = &tlb2[((ea >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; - for (int i = 0; i < TLB2_WAYS; i++) { - if (tlb_entry[i].tag == tag) { - tlb_entry[i].tag = TLB_INVALID_TAG; - //LOG_F(INFO, "Invalidated secondary TLB entry at 0x%X", ea); - } - } - } -} - -void tlb_flush_entries(TLBFlags type) -{ - int i; - - // Flush BAT entries from the primary TLBs - for (i = 0; i < TLB_SIZE; i++) { - if (dtlb1_mode2[i].flags & type) { - dtlb1_mode2[i].tag = TLB_INVALID_TAG; - } - - if (dtlb1_mode3[i].flags & type) { - dtlb1_mode3[i].tag = TLB_INVALID_TAG; - } - } - - // Flush BAT entries from the secondary TLBs - for (i = 0; i < TLB_SIZE * TLB2_WAYS; i++) { - if (dtlb2_mode2[i].flags & type) { - dtlb2_mode2[i].tag = TLB_INVALID_TAG; - } - - if (dtlb2_mode3[i].flags & type) { - dtlb2_mode3[i].tag = TLB_INVALID_TAG; - } - } -} - -void tlb_flush_bat_entries() -{ - if (!gTLBFlushBatEntries) - return; - - tlb_flush_entries(TLBE_FROM_BAT); - - gTLBFlushBatEntries = false; -} - -void tlb_flush_pat_entries() -{ - if (!gTLBFlushPatEntries) - return; - - tlb_flush_entries(TLBE_FROM_PAT); - - gTLBFlushPatEntries = false; -} - static inline uint64_t tlb_translate_addr(uint32_t guest_va) { TLBEntry *tlb1_entry, *tlb2_entry; @@ -1181,306 +1545,6 @@ static uint32_t mem_grab_unaligned(uint32_t addr, uint32_t size) { return ret; } -template -static inline TLBEntry* lookup_secondary_tlb(uint32_t guest_va, uint32_t tag) { - TLBEntry *tlb_entry; - - if (tlb_type == TLBType::ITLB) { - tlb_entry = &pCurITLB2[((guest_va >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; - } else { - tlb_entry = &pCurDTLB2[((guest_va >> PAGE_SIZE_BITS) & tlb_size_mask) * TLB2_WAYS]; - } - - if (tlb_entry->tag == tag) { - // update LRU bits - tlb_entry[0].lru_bits = 0x3; - tlb_entry[1].lru_bits = 0x2; - tlb_entry[2].lru_bits &= 0x1; - tlb_entry[3].lru_bits &= 0x1; - } else if (tlb_entry[1].tag == tag) { - tlb_entry = &tlb_entry[1]; - // update LRU bits - tlb_entry[0].lru_bits = 0x2; - tlb_entry[1].lru_bits = 0x3; - tlb_entry[2].lru_bits &= 0x1; - tlb_entry[3].lru_bits &= 0x1; - } else if (tlb_entry[2].tag == tag) { - tlb_entry = &tlb_entry[2]; - // update LRU bits - tlb_entry[0].lru_bits &= 0x1; - tlb_entry[1].lru_bits &= 0x1; - tlb_entry[2].lru_bits = 0x3; - tlb_entry[3].lru_bits = 0x2; - } else if (tlb_entry[3].tag == tag) { - tlb_entry = &tlb_entry[3]; - // update LRU bits - tlb_entry[0].lru_bits &= 0x1; - tlb_entry[1].lru_bits &= 0x1; - tlb_entry[2].lru_bits = 0x2; - tlb_entry[3].lru_bits = 0x3; - } else { - return nullptr; - } - return tlb_entry; -} - -// Forward declarations. -static uint32_t read_unaligned(uint32_t guest_va, uint8_t *host_va, uint32_t size); -static void write_unaligned(uint32_t guest_va, uint8_t *host_va, uint32_t value, - uint32_t size); - -template -inline T mmu_read_vmem(uint32_t guest_va) { - TLBEntry *tlb1_entry, *tlb2_entry; - uint8_t *host_va; - - const uint32_t tag = guest_va & ~0xFFFUL; - - // look up guest virtual address in the primary TLB - tlb1_entry = &pCurDTLB1[(guest_va >> PAGE_SIZE_BITS) & tlb_size_mask]; - if (tlb1_entry->tag == tag) { // primary TLB hit -> fast path -#ifdef TLB_PROFILING - num_primary_dtlb_hits++; -#endif - host_va = (uint8_t *)(tlb1_entry->host_va_offset + guest_va); - } else { - // primary TLB miss -> look up address in the secondary TLB - tlb2_entry = lookup_secondary_tlb(guest_va, tag); - if (tlb2_entry == nullptr) { -#ifdef TLB_PROFILING - num_dtlb_refills++; -#endif - // secondary TLB miss -> - // perform full address translation and refill the secondary TLB - tlb2_entry = dtlb2_refill(guest_va, 0); - } -#ifdef TLB_PROFILING - else { - num_secondary_dtlb_hits++; - } -#endif - - if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region? - // refill the primary TLB - tlb1_entry->tag = tag; - tlb1_entry->flags = tlb2_entry->flags; - tlb1_entry->host_va_offset = tlb2_entry->host_va_offset; - host_va = (uint8_t *)(tlb1_entry->host_va_offset + guest_va); - } else { // otherwise, it's an access to a memory-mapped device -#ifdef MMU_PROFILING - iomem_reads_total++; -#endif - return ( - tlb2_entry->reg_desc->devobj->read(tlb2_entry->reg_desc->start, - guest_va - tlb2_entry->reg_desc->start, sizeof(T)) - ); - } - } - -#ifdef MMU_PROFILING - dmem_reads_total++; -#endif - - // handle unaligned memory accesses - if (sizeof(T) > 1 && (guest_va & (sizeof(T) - 1))) { - return read_unaligned(guest_va, host_va, sizeof(T)); - } - - // handle aligned memory accesses - switch(sizeof(T)) { - case 1: - return *host_va; - case 2: - return READ_WORD_BE_A(host_va); - case 4: - return READ_DWORD_BE_A(host_va); - case 8: - return READ_QWORD_BE_A(host_va); - } -} - -// explicitely instantiate all required mmu_read_vmem variants -template uint8_t mmu_read_vmem(uint32_t guest_va); -template uint16_t mmu_read_vmem(uint32_t guest_va); -template uint32_t mmu_read_vmem(uint32_t guest_va); -template uint64_t mmu_read_vmem(uint32_t guest_va); - -template -inline void mmu_write_vmem(uint32_t guest_va, T value) { - TLBEntry *tlb1_entry, *tlb2_entry; - uint8_t *host_va; - - const uint32_t tag = guest_va & ~0xFFFUL; - - // look up guest virtual address in the primary TLB - tlb1_entry = &pCurDTLB1[(guest_va >> PAGE_SIZE_BITS) & tlb_size_mask]; - if (tlb1_entry->tag == tag) { // primary TLB hit -> fast path -#ifdef TLB_PROFILING - num_primary_dtlb_hits++; -#endif - if (!(tlb1_entry->flags & TLBFlags::PAGE_WRITABLE)) { - ppc_state.spr[SPR::DSISR] = 0x08000000 | (1 << 25); - ppc_state.spr[SPR::DAR] = guest_va; - mmu_exception_handler(Except_Type::EXC_DSI, 0); - } - if (!(tlb1_entry->flags & TLBFlags::PTE_SET_C)) { - // perform full page address translation to update PTE.C bit - PATResult pat_res = page_address_translation(guest_va, false, - !!(ppc_state.msr & 0x4000), true); - tlb1_entry->flags |= TLBFlags::PTE_SET_C; - - // don't forget to update the secondary TLB as well - tlb2_entry = lookup_secondary_tlb(guest_va, tag); - if (tlb2_entry != nullptr) { - tlb2_entry->flags |= TLBFlags::PTE_SET_C; - } - } - host_va = (uint8_t *)(tlb1_entry->host_va_offset + guest_va); - } else { - // primary TLB miss -> look up address in the secondary TLB - tlb2_entry = lookup_secondary_tlb(guest_va, tag); - if (tlb2_entry == nullptr) { -#ifdef TLB_PROFILING - num_dtlb_refills++; -#endif - // secondary TLB miss -> - // perform full address translation and refill the secondary TLB - tlb2_entry = dtlb2_refill(guest_va, 1); - } -#ifdef TLB_PROFILING - else { - num_secondary_dtlb_hits++; - } -#endif - - if (!(tlb2_entry->flags & TLBFlags::PAGE_WRITABLE)) { - ppc_state.spr[SPR::DSISR] = 0x08000000 | (1 << 25); - ppc_state.spr[SPR::DAR] = guest_va; - mmu_exception_handler(Except_Type::EXC_DSI, 0); - } - - if (!(tlb2_entry->flags & TLBFlags::PTE_SET_C)) { - // perform full page address translation to update PTE.C bit - PATResult pat_res = page_address_translation(guest_va, false, - !!(ppc_state.msr & 0x4000), true); - tlb2_entry->flags |= TLBFlags::PTE_SET_C; - } - - if (tlb2_entry->flags & TLBFlags::PAGE_MEM) { // is it a real memory region? - // refill the primary TLB - tlb1_entry->tag = tag; - tlb1_entry->flags = tlb2_entry->flags; - tlb1_entry->host_va_offset = tlb2_entry->host_va_offset; - host_va = (uint8_t *)(tlb1_entry->host_va_offset + guest_va); - } else { // otherwise, it's an access to a memory-mapped device -#ifdef MMU_PROFILING - iomem_writes_total++; -#endif - tlb2_entry->reg_desc->devobj->write(tlb2_entry->reg_desc->start, - guest_va - tlb2_entry->reg_desc->start, value, sizeof(T)); - return; - } - } - -#ifdef MMU_PROFILING - dmem_writes_total++; -#endif - - // handle unaligned memory accesses - if (sizeof(T) > 1 && (guest_va & (sizeof(T) - 1))) { - write_unaligned(guest_va, host_va, value, sizeof(T)); - return; - } - - // handle aligned memory accesses - switch(sizeof(T)) { - case 1: - *host_va = value; - break; - case 2: - WRITE_WORD_BE_A(host_va, value); - break; - case 4: - WRITE_DWORD_BE_A(host_va, value); - break; - case 8: - WRITE_QWORD_BE_A(host_va, value); - break; - } -} - -// explicitely instantiate all required mmu_write_vmem variants -template void mmu_write_vmem(uint32_t guest_va, uint8_t value); -template void mmu_write_vmem(uint32_t guest_va, uint16_t value); -template void mmu_write_vmem(uint32_t guest_va, uint32_t value); -template void mmu_write_vmem(uint32_t guest_va, uint64_t value); - -static uint32_t read_unaligned(uint32_t guest_va, uint8_t *host_va, uint32_t size) -{ - uint32_t result = 0; - - // is it a misaligned cross-page read? - if (((guest_va & 0xFFF) + size) > 0x1000) { -#ifdef MMU_PROFILING - unaligned_crossp_r++; -#endif - // Break such a memory access into multiple, bytewise accesses. - // Because such accesses suffer a performance penalty, they will be - // presumably very rare so don't waste time optimizing the code below. - for (int i = 0; i < size; guest_va++, i++) { - result = (result << 8) | mmu_read_vmem(guest_va); - } - } else { -#ifdef MMU_PROFILING - unaligned_reads++; -#endif - switch(size) { - case 2: - return READ_WORD_BE_U(host_va); - case 4: - return READ_DWORD_BE_U(host_va); - case 8: // FIXME: should we raise alignment exception here? - return READ_QWORD_BE_U(host_va); - } - } - return result; -} - -static void write_unaligned(uint32_t guest_va, uint8_t *host_va, uint32_t value, - uint32_t size) -{ - // is it a misaligned cross-page write? - if (((guest_va & 0xFFF) + size) > 0x1000) { -#ifdef MMU_PROFILING - unaligned_crossp_w++; -#endif - // Break such a memory access into multiple, bytewise accesses. - // Because such accesses suffer a performance penalty, they will be - // presumably very rare so don't waste time optimizing the code below. - - uint32_t shift = (size - 1) * 8; - - for (int i = 0; i < size; shift -= 8, guest_va++, i++) { - mmu_write_vmem(guest_va, (value >> shift) & 0xFF); - } - } else { -#ifdef MMU_PROFILING - unaligned_writes++; -#endif - switch(size) { - case 2: - WRITE_WORD_BE_U(host_va, value); - break; - case 4: - WRITE_DWORD_BE_U(host_va, value); - break; - case 8: // FIXME: should we raise alignment exception here? - WRITE_QWORD_BE_U(host_va, value); - break; - } - } -} - void mem_write_byte(uint32_t addr, uint8_t value) { mmu_write_vmem(addr, value); @@ -1596,52 +1660,6 @@ uint64_t mem_grab_qword(uint32_t addr) { return read_phys_mem(&last_read_area, addr); } -uint8_t *mmu_translate_imem(uint32_t vaddr) -{ - TLBEntry *tlb1_entry, *tlb2_entry; - uint8_t *host_va; - -#ifdef MMU_PROFILING - exec_reads_total++; -#endif - - const uint32_t tag = vaddr & ~0xFFFUL; - - // look up guest virtual address in the primary ITLB - tlb1_entry = &pCurITLB1[(vaddr >> PAGE_SIZE_BITS) & tlb_size_mask]; - if (tlb1_entry->tag == tag) { // primary ITLB hit -> fast path -#ifdef TLB_PROFILING - num_primary_itlb_hits++; -#endif - host_va = (uint8_t *)(tlb1_entry->host_va_offset + vaddr); - } else { - // primary ITLB miss -> look up address in the secondary ITLB - tlb2_entry = lookup_secondary_tlb(vaddr, tag); - if (tlb2_entry == nullptr) { -#ifdef TLB_PROFILING - num_itlb_refills++; -#endif - // secondary ITLB miss -> - // perform full address translation and refill the secondary ITLB - tlb2_entry = itlb2_refill(vaddr); - } -#ifdef TLB_PROFILING - else { - num_secondary_itlb_hits++; - } -#endif - // refill the primary ITLB - tlb1_entry->tag = tag; - tlb1_entry->flags = tlb2_entry->flags; - tlb1_entry->host_va_offset = tlb2_entry->host_va_offset; - host_va = (uint8_t *)(tlb1_entry->host_va_offset + vaddr); - } - - ppc_set_cur_instruction(host_va); - - return host_va; -} - uint8_t* quickinstruction_translate(uint32_t addr) { uint8_t* real_addr; @@ -1672,6 +1690,7 @@ uint8_t* quickinstruction_translate(uint32_t addr) { return real_addr; } +#endif uint64_t mem_read_dbg(uint32_t virt_addr, uint32_t size) { uint32_t save_dsisr, save_dar; @@ -1685,19 +1704,19 @@ uint64_t mem_read_dbg(uint32_t virt_addr, uint32_t size) { try { switch (size) { case 1: - ret_val = mem_grab_byte(virt_addr); + ret_val = mmu_read_vmem(virt_addr); break; case 2: - ret_val = mem_grab_word(virt_addr); + ret_val = mmu_read_vmem(virt_addr); break; case 4: - ret_val = mem_grab_dword(virt_addr); + ret_val = mmu_read_vmem(virt_addr); break; case 8: - ret_val = mem_grab_qword(virt_addr); + ret_val = mmu_read_vmem(virt_addr); break; default: - ret_val = mem_grab_byte(virt_addr); + ret_val = mmu_read_vmem(virt_addr); } } catch (std::invalid_argument& exc) { /* restore MMU-related CPU state */ diff --git a/cpu/ppc/ppcmmu.h b/cpu/ppc/ppcmmu.h index b9dc5fa..649e927 100644 --- a/cpu/ppc/ppcmmu.h +++ b/cpu/ppc/ppcmmu.h @@ -101,6 +101,16 @@ extern void mmu_pat_ctx_changed(); extern void tlb_flush_entry(uint32_t ea); extern void ppc_set_cur_instruction(const uint8_t* ptr); +extern uint64_t mem_read_dbg(uint32_t virt_addr, uint32_t size); +uint8_t *mmu_translate_imem(uint32_t vaddr); + +template +extern T mmu_read_vmem(uint32_t guest_va); +template +extern void mmu_write_vmem(uint32_t guest_va, T value); + +//====================== Deprecated calls ========================= +#if 0 extern void mem_write_byte(uint32_t addr, uint8_t value); extern void mem_write_word(uint32_t addr, uint16_t value); extern void mem_write_dword(uint32_t addr, uint32_t value); @@ -109,14 +119,7 @@ extern uint8_t mem_grab_byte(uint32_t addr); extern uint16_t mem_grab_word(uint32_t addr); extern uint32_t mem_grab_dword(uint32_t addr); extern uint64_t mem_grab_qword(uint32_t addr); -extern uint64_t mem_read_dbg(uint32_t virt_addr, uint32_t size); extern uint8_t* quickinstruction_translate(uint32_t address_grab); - -uint8_t *mmu_translate_imem(uint32_t vaddr); - -template -extern T mmu_read_vmem(uint32_t guest_va); -template -extern void mmu_write_vmem(uint32_t guest_va, T value); +#endif #endif // PPCMMU_H diff --git a/debugger/debugger.cpp b/debugger/debugger.cpp index 983d824..3af759b 100644 --- a/debugger/debugger.cpp +++ b/debugger/debugger.cpp @@ -151,7 +151,7 @@ void exec_single_68k() /* calculate address of the current opcode table entry as follows: get_word(68k_PC) * entry_size + table_base */ - cur_instr_tab_entry = mem_grab_word(cur_68k_pc) * 8 + emu_table_virt; + cur_instr_tab_entry = mmu_read_vmem(cur_68k_pc) * 8 + emu_table_virt; /* grab the PPC PC too */ reg = "PC";