/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- * vim: set ts=8 sts=4 et sw=4 tw=99: * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "jit/shared/CodeGenerator-shared-inl.h" #include "mozilla/DebugOnly.h" #include "jit/CompactBuffer.h" #include "jit/IonCaches.h" #include "jit/JitcodeMap.h" #include "jit/JitSpewer.h" #include "jit/MacroAssembler.h" #include "jit/MIR.h" #include "jit/MIRGenerator.h" #include "jit/OptimizationTracking.h" #include "js/Conversions.h" #include "vm/TraceLogging.h" #include "jit/JitFrames-inl.h" #include "jit/MacroAssembler-inl.h" using namespace js; using namespace js::jit; using mozilla::BitwiseCast; using mozilla::DebugOnly; namespace js { namespace jit { MacroAssembler& CodeGeneratorShared::ensureMasm(MacroAssembler* masmArg) { if (masmArg) return *masmArg; maybeMasm_.emplace(); return *maybeMasm_; } CodeGeneratorShared::CodeGeneratorShared(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masmArg) : maybeMasm_(), masm(ensureMasm(masmArg)), gen(gen), graph(*graph), current(nullptr), snapshots_(), recovers_(), deoptTable_(nullptr), #ifdef DEBUG pushedArgs_(0), #endif lastOsiPointOffset_(0), safepoints_(graph->totalSlotCount(), (gen->info().nargs() + 1) * sizeof(Value)), returnLabel_(), stubSpace_(), nativeToBytecodeMap_(nullptr), nativeToBytecodeMapSize_(0), nativeToBytecodeTableOffset_(0), nativeToBytecodeNumRegions_(0), nativeToBytecodeScriptList_(nullptr), nativeToBytecodeScriptListLength_(0), trackedOptimizationsMap_(nullptr), trackedOptimizationsMapSize_(0), trackedOptimizationsRegionTableOffset_(0), trackedOptimizationsTypesTableOffset_(0), trackedOptimizationsAttemptsTableOffset_(0), osrEntryOffset_(0), skipArgCheckEntryOffset_(0), #ifdef CHECK_OSIPOINT_REGISTERS checkOsiPointRegisters(JitOptions.checkOsiPointRegisters), #endif frameDepth_(graph->paddedLocalSlotsSize() + graph->argumentsSize()), frameInitialAdjustment_(0) { if (gen->isProfilerInstrumentationEnabled()) masm.enableProfilingInstrumentation(); if (gen->compilingAsmJS()) { // Since asm.js uses the system ABI which does not necessarily use a // regular array where all slots are sizeof(Value), it maintains the max // argument stack depth separately. MOZ_ASSERT(graph->argumentSlotCount() == 0); frameDepth_ += gen->maxAsmJSStackArgBytes(); if (gen->usesSimd()) { // If the function uses any SIMD then we may need to insert padding // so that local slots are aligned for SIMD. frameInitialAdjustment_ = ComputeByteAlignment(sizeof(AsmJSFrame), AsmJSStackAlignment); frameDepth_ += frameInitialAdjustment_; // Keep the stack aligned. Some SIMD sequences build values on the // stack and need the stack aligned. frameDepth_ += ComputeByteAlignment(sizeof(AsmJSFrame) + frameDepth_, AsmJSStackAlignment); } else if (gen->performsCall()) { // An MAsmJSCall does not align the stack pointer at calls sites but // instead relies on the a priori stack adjustment. This must be the // last adjustment of frameDepth_. frameDepth_ += ComputeByteAlignment(sizeof(AsmJSFrame) + frameDepth_, AsmJSStackAlignment); } // FrameSizeClass is only used for bailing, which cannot happen in // asm.js code. frameClass_ = FrameSizeClass::None(); } else { frameClass_ = FrameSizeClass::FromDepth(frameDepth_); } } bool CodeGeneratorShared::generatePrologue() { MOZ_ASSERT(masm.framePushed() == 0); MOZ_ASSERT(!gen->compilingAsmJS()); #ifdef JS_USE_LINK_REGISTER masm.pushReturnAddress(); #endif // If profiling, save the current frame pointer to a per-thread global field. if (isProfilerInstrumentationEnabled()) masm.profilerEnterFrame(masm.getStackPointer(), CallTempReg0); // Ensure that the Ion frame is properly aligned. masm.assertStackAlignment(JitStackAlignment, 0); // Note that this automatically sets MacroAssembler::framePushed(). masm.reserveStack(frameSize()); masm.checkStackAlignment(); emitTracelogIonStart(); return true; } bool CodeGeneratorShared::generateEpilogue() { MOZ_ASSERT(!gen->compilingAsmJS()); masm.bind(&returnLabel_); emitTracelogIonStop(); masm.freeStack(frameSize()); MOZ_ASSERT(masm.framePushed() == 0); // If profiling, reset the per-thread global lastJitFrame to point to // the previous frame. if (isProfilerInstrumentationEnabled()) masm.profilerExitFrame(); masm.ret(); // On systems that use a constant pool, this is a good time to emit. masm.flushBuffer(); return true; } bool CodeGeneratorShared::generateOutOfLineCode() { for (size_t i = 0; i < outOfLineCode_.length(); i++) { // Add native => bytecode mapping entries for OOL sites. // Not enabled on asm.js yet since asm doesn't contain bytecode mappings. if (!gen->compilingAsmJS()) { if (!addNativeToBytecodeEntry(outOfLineCode_[i]->bytecodeSite())) return false; } if (!gen->alloc().ensureBallast()) return false; JitSpew(JitSpew_Codegen, "# Emitting out of line code"); masm.setFramePushed(outOfLineCode_[i]->framePushed()); lastPC_ = outOfLineCode_[i]->pc(); outOfLineCode_[i]->bind(&masm); outOfLineCode_[i]->generate(this); } return !masm.oom(); } void CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code, const MInstruction* mir) { MOZ_ASSERT(mir); addOutOfLineCode(code, mir->trackedSite()); } void CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code, const BytecodeSite* site) { code->setFramePushed(masm.framePushed()); code->setBytecodeSite(site); MOZ_ASSERT_IF(!gen->compilingAsmJS(), code->script()->containsPC(code->pc())); masm.propagateOOM(outOfLineCode_.append(code)); } bool CodeGeneratorShared::addNativeToBytecodeEntry(const BytecodeSite* site) { // Skip the table entirely if profiling is not enabled. if (!isProfilerInstrumentationEnabled()) return true; // Fails early if the last added instruction caused the macro assembler to // run out of memory as continuity assumption below do not hold. if (masm.oom()) return false; MOZ_ASSERT(site); MOZ_ASSERT(site->tree()); MOZ_ASSERT(site->pc()); InlineScriptTree* tree = site->tree(); jsbytecode* pc = site->pc(); uint32_t nativeOffset = masm.currentOffset(); MOZ_ASSERT_IF(nativeToBytecodeList_.empty(), nativeOffset == 0); if (!nativeToBytecodeList_.empty()) { size_t lastIdx = nativeToBytecodeList_.length() - 1; NativeToBytecode& lastEntry = nativeToBytecodeList_[lastIdx]; MOZ_ASSERT(nativeOffset >= lastEntry.nativeOffset.offset()); // If the new entry is for the same inlineScriptTree and same // bytecodeOffset, but the nativeOffset has changed, do nothing. // The same site just generated some more code. if (lastEntry.tree == tree && lastEntry.pc == pc) { JitSpew(JitSpew_Profiling, " => In-place update [%u-%u]", lastEntry.nativeOffset.offset(), nativeOffset); return true; } // If the new entry is for the same native offset, then update the // previous entry with the new bytecode site, since the previous // bytecode site did not generate any native code. if (lastEntry.nativeOffset.offset() == nativeOffset) { lastEntry.tree = tree; lastEntry.pc = pc; JitSpew(JitSpew_Profiling, " => Overwriting zero-length native region."); // This overwrite might have made the entry merge-able with a // previous one. If so, merge it. if (lastIdx > 0) { NativeToBytecode& nextToLastEntry = nativeToBytecodeList_[lastIdx - 1]; if (nextToLastEntry.tree == lastEntry.tree && nextToLastEntry.pc == lastEntry.pc) { JitSpew(JitSpew_Profiling, " => Merging with previous region"); nativeToBytecodeList_.erase(&lastEntry); } } dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1); return true; } } // Otherwise, some native code was generated for the previous bytecode site. // Add a new entry for code that is about to be generated. NativeToBytecode entry; entry.nativeOffset = CodeOffset(nativeOffset); entry.tree = tree; entry.pc = pc; if (!nativeToBytecodeList_.append(entry)) return false; JitSpew(JitSpew_Profiling, " => Push new entry."); dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1); return true; } void CodeGeneratorShared::dumpNativeToBytecodeEntries() { #ifdef JS_JITSPEW InlineScriptTree* topTree = gen->info().inlineScriptTree(); JitSpewStart(JitSpew_Profiling, "Native To Bytecode Entries for %s:%d\n", topTree->script()->filename(), topTree->script()->lineno()); for (unsigned i = 0; i < nativeToBytecodeList_.length(); i++) dumpNativeToBytecodeEntry(i); #endif } void CodeGeneratorShared::dumpNativeToBytecodeEntry(uint32_t idx) { #ifdef JS_JITSPEW NativeToBytecode& ref = nativeToBytecodeList_[idx]; InlineScriptTree* tree = ref.tree; JSScript* script = tree->script(); uint32_t nativeOffset = ref.nativeOffset.offset(); unsigned nativeDelta = 0; unsigned pcDelta = 0; if (idx + 1 < nativeToBytecodeList_.length()) { NativeToBytecode* nextRef = &ref + 1; nativeDelta = nextRef->nativeOffset.offset() - nativeOffset; if (nextRef->tree == ref.tree) pcDelta = nextRef->pc - ref.pc; } JitSpewStart(JitSpew_Profiling, " %08x [+%-6d] => %-6d [%-4d] {%-10s} (%s:%d", ref.nativeOffset.offset(), nativeDelta, ref.pc - script->code(), pcDelta, CodeName[JSOp(*ref.pc)], script->filename(), script->lineno()); for (tree = tree->caller(); tree; tree = tree->caller()) { JitSpewCont(JitSpew_Profiling, " <= %s:%d", tree->script()->filename(), tree->script()->lineno()); } JitSpewCont(JitSpew_Profiling, ")"); JitSpewFin(JitSpew_Profiling); #endif } bool CodeGeneratorShared::addTrackedOptimizationsEntry(const TrackedOptimizations* optimizations) { if (!isOptimizationTrackingEnabled()) return true; MOZ_ASSERT(optimizations); uint32_t nativeOffset = masm.currentOffset(); if (!trackedOptimizations_.empty()) { NativeToTrackedOptimizations& lastEntry = trackedOptimizations_.back(); MOZ_ASSERT_IF(!masm.oom(), nativeOffset >= lastEntry.endOffset.offset()); // If we're still generating code for the same set of optimizations, // we are done. if (lastEntry.optimizations == optimizations) return true; } // If we're generating code for a new set of optimizations, add a new // entry. NativeToTrackedOptimizations entry; entry.startOffset = CodeOffset(nativeOffset); entry.endOffset = CodeOffset(nativeOffset); entry.optimizations = optimizations; return trackedOptimizations_.append(entry); } void CodeGeneratorShared::extendTrackedOptimizationsEntry(const TrackedOptimizations* optimizations) { if (!isOptimizationTrackingEnabled()) return; uint32_t nativeOffset = masm.currentOffset(); NativeToTrackedOptimizations& entry = trackedOptimizations_.back(); MOZ_ASSERT(entry.optimizations == optimizations); MOZ_ASSERT_IF(!masm.oom(), nativeOffset >= entry.endOffset.offset()); entry.endOffset = CodeOffset(nativeOffset); // If we generated no code, remove the last entry. if (nativeOffset == entry.startOffset.offset()) trackedOptimizations_.popBack(); } // see OffsetOfFrameSlot static inline int32_t ToStackIndex(LAllocation* a) { if (a->isStackSlot()) { MOZ_ASSERT(a->toStackSlot()->slot() >= 1); return a->toStackSlot()->slot(); } return -int32_t(sizeof(JitFrameLayout) + a->toArgument()->index()); } void CodeGeneratorShared::encodeAllocation(LSnapshot* snapshot, MDefinition* mir, uint32_t* allocIndex) { if (mir->isBox()) mir = mir->toBox()->getOperand(0); MIRType type = mir->isRecoveredOnBailout() ? MIRType_None : mir->isUnused() ? MIRType_MagicOptimizedOut : mir->type(); RValueAllocation alloc; switch (type) { case MIRType_None: { MOZ_ASSERT(mir->isRecoveredOnBailout()); uint32_t index = 0; LRecoverInfo* recoverInfo = snapshot->recoverInfo(); MNode** it = recoverInfo->begin(); MNode** end = recoverInfo->end(); while (it != end && mir != *it) { ++it; ++index; } // This MDefinition is recovered, thus it should be listed in the // LRecoverInfo. MOZ_ASSERT(it != end && mir == *it); // Lambda should have a default value readable for iterating over the // inner frames. if (mir->isLambda()) { MConstant* constant = mir->toLambda()->functionOperand(); uint32_t cstIndex; masm.propagateOOM(graph.addConstantToPool(constant->value(), &cstIndex)); alloc = RValueAllocation::RecoverInstruction(index, cstIndex); break; } alloc = RValueAllocation::RecoverInstruction(index); break; } case MIRType_Undefined: alloc = RValueAllocation::Undefined(); break; case MIRType_Null: alloc = RValueAllocation::Null(); break; case MIRType_Int32: case MIRType_String: case MIRType_Symbol: case MIRType_Object: case MIRType_ObjectOrNull: case MIRType_Boolean: case MIRType_Double: { LAllocation* payload = snapshot->payloadOfSlot(*allocIndex); if (payload->isConstant()) { MConstant* constant = mir->toConstant(); uint32_t index; masm.propagateOOM(graph.addConstantToPool(constant->value(), &index)); alloc = RValueAllocation::ConstantPool(index); break; } JSValueType valueType = (type == MIRType_ObjectOrNull) ? JSVAL_TYPE_OBJECT : ValueTypeFromMIRType(type); MOZ_ASSERT(payload->isMemory() || payload->isRegister()); if (payload->isMemory()) alloc = RValueAllocation::Typed(valueType, ToStackIndex(payload)); else if (payload->isGeneralReg()) alloc = RValueAllocation::Typed(valueType, ToRegister(payload)); else if (payload->isFloatReg()) alloc = RValueAllocation::Double(ToFloatRegister(payload)); break; } case MIRType_Float32: case MIRType_Int32x4: case MIRType_Float32x4: { LAllocation* payload = snapshot->payloadOfSlot(*allocIndex); if (payload->isConstant()) { MConstant* constant = mir->toConstant(); uint32_t index; masm.propagateOOM(graph.addConstantToPool(constant->value(), &index)); alloc = RValueAllocation::ConstantPool(index); break; } MOZ_ASSERT(payload->isMemory() || payload->isFloatReg()); if (payload->isFloatReg()) alloc = RValueAllocation::AnyFloat(ToFloatRegister(payload)); else alloc = RValueAllocation::AnyFloat(ToStackIndex(payload)); break; } case MIRType_MagicOptimizedArguments: case MIRType_MagicOptimizedOut: case MIRType_MagicUninitializedLexical: { uint32_t index; Value v = MagicValue(type == MIRType_MagicOptimizedArguments ? JS_OPTIMIZED_ARGUMENTS : (type == MIRType_MagicOptimizedOut ? JS_OPTIMIZED_OUT : JS_UNINITIALIZED_LEXICAL)); masm.propagateOOM(graph.addConstantToPool(v, &index)); alloc = RValueAllocation::ConstantPool(index); break; } default: { MOZ_ASSERT(mir->type() == MIRType_Value); LAllocation* payload = snapshot->payloadOfSlot(*allocIndex); #ifdef JS_NUNBOX32 LAllocation* type = snapshot->typeOfSlot(*allocIndex); if (type->isRegister()) { if (payload->isRegister()) alloc = RValueAllocation::Untyped(ToRegister(type), ToRegister(payload)); else alloc = RValueAllocation::Untyped(ToRegister(type), ToStackIndex(payload)); } else { if (payload->isRegister()) alloc = RValueAllocation::Untyped(ToStackIndex(type), ToRegister(payload)); else alloc = RValueAllocation::Untyped(ToStackIndex(type), ToStackIndex(payload)); } #elif JS_PUNBOX64 if (payload->isRegister()) alloc = RValueAllocation::Untyped(ToRegister(payload)); else alloc = RValueAllocation::Untyped(ToStackIndex(payload)); #endif break; } } // This set an extra bit as part of the RValueAllocation, such that we know // that recover instruction have to be executed without wrapping the // instruction in a no-op recover instruction. if (mir->isIncompleteObject()) alloc.setNeedSideEffect(); snapshots_.add(alloc); *allocIndex += mir->isRecoveredOnBailout() ? 0 : 1; } void CodeGeneratorShared::encode(LRecoverInfo* recover) { if (recover->recoverOffset() != INVALID_RECOVER_OFFSET) return; uint32_t numInstructions = recover->numInstructions(); JitSpew(JitSpew_IonSnapshots, "Encoding LRecoverInfo %p (frameCount %u, instructions %u)", (void*)recover, recover->mir()->frameCount(), numInstructions); MResumePoint::Mode mode = recover->mir()->mode(); MOZ_ASSERT(mode != MResumePoint::Outer); bool resumeAfter = (mode == MResumePoint::ResumeAfter); RecoverOffset offset = recovers_.startRecover(numInstructions, resumeAfter); for (MNode* insn : *recover) recovers_.writeInstruction(insn); recovers_.endRecover(); recover->setRecoverOffset(offset); masm.propagateOOM(!recovers_.oom()); } void CodeGeneratorShared::encode(LSnapshot* snapshot) { if (snapshot->snapshotOffset() != INVALID_SNAPSHOT_OFFSET) return; LRecoverInfo* recoverInfo = snapshot->recoverInfo(); encode(recoverInfo); RecoverOffset recoverOffset = recoverInfo->recoverOffset(); MOZ_ASSERT(recoverOffset != INVALID_RECOVER_OFFSET); JitSpew(JitSpew_IonSnapshots, "Encoding LSnapshot %p (LRecover %p)", (void*)snapshot, (void*) recoverInfo); SnapshotOffset offset = snapshots_.startSnapshot(recoverOffset, snapshot->bailoutKind()); #ifdef TRACK_SNAPSHOTS uint32_t pcOpcode = 0; uint32_t lirOpcode = 0; uint32_t lirId = 0; uint32_t mirOpcode = 0; uint32_t mirId = 0; if (LNode* ins = instruction()) { lirOpcode = ins->op(); lirId = ins->id(); if (ins->mirRaw()) { mirOpcode = ins->mirRaw()->op(); mirId = ins->mirRaw()->id(); if (ins->mirRaw()->trackedPc()) pcOpcode = *ins->mirRaw()->trackedPc(); } } snapshots_.trackSnapshot(pcOpcode, mirOpcode, mirId, lirOpcode, lirId); #endif uint32_t allocIndex = 0; for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) { DebugOnly allocWritten = snapshots_.allocWritten(); encodeAllocation(snapshot, *it, &allocIndex); MOZ_ASSERT_IF(!snapshots_.oom(), allocWritten + 1 == snapshots_.allocWritten()); } MOZ_ASSERT(allocIndex == snapshot->numSlots()); snapshots_.endSnapshot(); snapshot->setSnapshotOffset(offset); masm.propagateOOM(!snapshots_.oom()); } bool CodeGeneratorShared::assignBailoutId(LSnapshot* snapshot) { MOZ_ASSERT(snapshot->snapshotOffset() != INVALID_SNAPSHOT_OFFSET); // Can we not use bailout tables at all? if (!deoptTable_) return false; MOZ_ASSERT(frameClass_ != FrameSizeClass::None()); if (snapshot->bailoutId() != INVALID_BAILOUT_ID) return true; // Is the bailout table full? if (bailouts_.length() >= BAILOUT_TABLE_SIZE) return false; unsigned bailoutId = bailouts_.length(); snapshot->setBailoutId(bailoutId); JitSpew(JitSpew_IonSnapshots, "Assigned snapshot bailout id %u", bailoutId); return bailouts_.append(snapshot->snapshotOffset()); } bool CodeGeneratorShared::encodeSafepoints() { for (SafepointIndex& index : safepointIndices_) { LSafepoint* safepoint = index.safepoint(); if (!safepoint->encoded()) safepoints_.encode(safepoint); index.resolve(); } return !safepoints_.oom(); } bool CodeGeneratorShared::createNativeToBytecodeScriptList(JSContext* cx) { js::Vector scriptList; InlineScriptTree* tree = gen->info().inlineScriptTree(); for (;;) { // Add script from current tree. bool found = false; for (uint32_t i = 0; i < scriptList.length(); i++) { if (scriptList[i] == tree->script()) { found = true; break; } } if (!found) { if (!scriptList.append(tree->script())) return false; } // Process rest of tree // If children exist, emit children. if (tree->hasChildren()) { tree = tree->firstChild(); continue; } // Otherwise, find the first tree up the chain (including this one) // that contains a next sibling. while (!tree->hasNextCallee() && tree->hasCaller()) tree = tree->caller(); // If we found a sibling, use it. if (tree->hasNextCallee()) { tree = tree->nextCallee(); continue; } // Otherwise, we must have reached the top without finding any siblings. MOZ_ASSERT(tree->isOutermostCaller()); break; } // Allocate array for list. JSScript** data = cx->runtime()->pod_malloc(scriptList.length()); if (!data) return false; for (uint32_t i = 0; i < scriptList.length(); i++) data[i] = scriptList[i]; // Success. nativeToBytecodeScriptListLength_ = scriptList.length(); nativeToBytecodeScriptList_ = data; return true; } bool CodeGeneratorShared::generateCompactNativeToBytecodeMap(JSContext* cx, JitCode* code) { MOZ_ASSERT(nativeToBytecodeScriptListLength_ == 0); MOZ_ASSERT(nativeToBytecodeScriptList_ == nullptr); MOZ_ASSERT(nativeToBytecodeMap_ == nullptr); MOZ_ASSERT(nativeToBytecodeMapSize_ == 0); MOZ_ASSERT(nativeToBytecodeTableOffset_ == 0); MOZ_ASSERT(nativeToBytecodeNumRegions_ == 0); if (!createNativeToBytecodeScriptList(cx)) return false; MOZ_ASSERT(nativeToBytecodeScriptListLength_ > 0); MOZ_ASSERT(nativeToBytecodeScriptList_ != nullptr); CompactBufferWriter writer; uint32_t tableOffset = 0; uint32_t numRegions = 0; if (!JitcodeIonTable::WriteIonTable( writer, nativeToBytecodeScriptList_, nativeToBytecodeScriptListLength_, &nativeToBytecodeList_[0], &nativeToBytecodeList_[0] + nativeToBytecodeList_.length(), &tableOffset, &numRegions)) { js_free(nativeToBytecodeScriptList_); return false; } MOZ_ASSERT(tableOffset > 0); MOZ_ASSERT(numRegions > 0); // Writer is done, copy it to sized buffer. uint8_t* data = cx->runtime()->pod_malloc(writer.length()); if (!data) { js_free(nativeToBytecodeScriptList_); return false; } memcpy(data, writer.buffer(), writer.length()); nativeToBytecodeMap_ = data; nativeToBytecodeMapSize_ = writer.length(); nativeToBytecodeTableOffset_ = tableOffset; nativeToBytecodeNumRegions_ = numRegions; verifyCompactNativeToBytecodeMap(code); JitSpew(JitSpew_Profiling, "Compact Native To Bytecode Map [%p-%p]", data, data + nativeToBytecodeMapSize_); return true; } void CodeGeneratorShared::verifyCompactNativeToBytecodeMap(JitCode* code) { #ifdef DEBUG MOZ_ASSERT(nativeToBytecodeScriptListLength_ > 0); MOZ_ASSERT(nativeToBytecodeScriptList_ != nullptr); MOZ_ASSERT(nativeToBytecodeMap_ != nullptr); MOZ_ASSERT(nativeToBytecodeMapSize_ > 0); MOZ_ASSERT(nativeToBytecodeTableOffset_ > 0); MOZ_ASSERT(nativeToBytecodeNumRegions_ > 0); // The pointer to the table must be 4-byte aligned const uint8_t* tablePtr = nativeToBytecodeMap_ + nativeToBytecodeTableOffset_; MOZ_ASSERT(uintptr_t(tablePtr) % sizeof(uint32_t) == 0); // Verify that numRegions was encoded correctly. const JitcodeIonTable* ionTable = reinterpret_cast(tablePtr); MOZ_ASSERT(ionTable->numRegions() == nativeToBytecodeNumRegions_); // Region offset for first region should be at the start of the payload region. // Since the offsets are backward from the start of the table, the first entry // backoffset should be equal to the forward table offset from the start of the // allocated data. MOZ_ASSERT(ionTable->regionOffset(0) == nativeToBytecodeTableOffset_); // Verify each region. for (uint32_t i = 0; i < ionTable->numRegions(); i++) { // Back-offset must point into the payload region preceding the table, not before it. MOZ_ASSERT(ionTable->regionOffset(i) <= nativeToBytecodeTableOffset_); // Back-offset must point to a later area in the payload region than previous // back-offset. This means that back-offsets decrease monotonically. MOZ_ASSERT_IF(i > 0, ionTable->regionOffset(i) < ionTable->regionOffset(i - 1)); JitcodeRegionEntry entry = ionTable->regionEntry(i); // Ensure native code offset for region falls within jitcode. MOZ_ASSERT(entry.nativeOffset() <= code->instructionsSize()); // Read out script/pc stack and verify. JitcodeRegionEntry::ScriptPcIterator scriptPcIter = entry.scriptPcIterator(); while (scriptPcIter.hasMore()) { uint32_t scriptIdx = 0, pcOffset = 0; scriptPcIter.readNext(&scriptIdx, &pcOffset); // Ensure scriptIdx refers to a valid script in the list. MOZ_ASSERT(scriptIdx < nativeToBytecodeScriptListLength_); JSScript* script = nativeToBytecodeScriptList_[scriptIdx]; // Ensure pcOffset falls within the script. MOZ_ASSERT(pcOffset < script->length()); } // Obtain the original nativeOffset and pcOffset and script. uint32_t curNativeOffset = entry.nativeOffset(); JSScript* script = nullptr; uint32_t curPcOffset = 0; { uint32_t scriptIdx = 0; scriptPcIter.reset(); scriptPcIter.readNext(&scriptIdx, &curPcOffset); script = nativeToBytecodeScriptList_[scriptIdx]; } // Read out nativeDeltas and pcDeltas and verify. JitcodeRegionEntry::DeltaIterator deltaIter = entry.deltaIterator(); while (deltaIter.hasMore()) { uint32_t nativeDelta = 0; int32_t pcDelta = 0; deltaIter.readNext(&nativeDelta, &pcDelta); curNativeOffset += nativeDelta; curPcOffset = uint32_t(int32_t(curPcOffset) + pcDelta); // Ensure that nativeOffset still falls within jitcode after delta. MOZ_ASSERT(curNativeOffset <= code->instructionsSize()); // Ensure that pcOffset still falls within bytecode after delta. MOZ_ASSERT(curPcOffset < script->length()); } } #endif // DEBUG } bool CodeGeneratorShared::generateCompactTrackedOptimizationsMap(JSContext* cx, JitCode* code, IonTrackedTypeVector* allTypes) { MOZ_ASSERT(trackedOptimizationsMap_ == nullptr); MOZ_ASSERT(trackedOptimizationsMapSize_ == 0); MOZ_ASSERT(trackedOptimizationsRegionTableOffset_ == 0); MOZ_ASSERT(trackedOptimizationsTypesTableOffset_ == 0); MOZ_ASSERT(trackedOptimizationsAttemptsTableOffset_ == 0); if (trackedOptimizations_.empty()) return true; UniqueTrackedOptimizations unique(cx); if (!unique.init()) return false; // Iterate through all entries to deduplicate their optimization attempts. for (size_t i = 0; i < trackedOptimizations_.length(); i++) { NativeToTrackedOptimizations& entry = trackedOptimizations_[i]; if (!unique.add(entry.optimizations)) return false; } // Sort the unique optimization attempts by frequency to stabilize the // attempts' indices in the compact table we will write later. if (!unique.sortByFrequency(cx)) return false; // Write out the ranges and the table. CompactBufferWriter writer; uint32_t numRegions; uint32_t regionTableOffset; uint32_t typesTableOffset; uint32_t attemptsTableOffset; if (!WriteIonTrackedOptimizationsTable(cx, writer, trackedOptimizations_.begin(), trackedOptimizations_.end(), unique, &numRegions, ®ionTableOffset, &typesTableOffset, &attemptsTableOffset, allTypes)) { return false; } MOZ_ASSERT(regionTableOffset > 0); MOZ_ASSERT(typesTableOffset > 0); MOZ_ASSERT(attemptsTableOffset > 0); MOZ_ASSERT(typesTableOffset > regionTableOffset); MOZ_ASSERT(attemptsTableOffset > typesTableOffset); // Copy over the table out of the writer's buffer. uint8_t* data = cx->runtime()->pod_malloc(writer.length()); if (!data) return false; memcpy(data, writer.buffer(), writer.length()); trackedOptimizationsMap_ = data; trackedOptimizationsMapSize_ = writer.length(); trackedOptimizationsRegionTableOffset_ = regionTableOffset; trackedOptimizationsTypesTableOffset_ = typesTableOffset; trackedOptimizationsAttemptsTableOffset_ = attemptsTableOffset; verifyCompactTrackedOptimizationsMap(code, numRegions, unique, allTypes); JitSpew(JitSpew_OptimizationTracking, "== Compact Native To Optimizations Map [%p-%p] size %u", data, data + trackedOptimizationsMapSize_, trackedOptimizationsMapSize_); JitSpew(JitSpew_OptimizationTracking, " with type list of length %u, size %u", allTypes->length(), allTypes->length() * sizeof(IonTrackedTypeWithAddendum)); return true; } #ifdef DEBUG class ReadTempAttemptsVectorOp : public JS::ForEachTrackedOptimizationAttemptOp { TempOptimizationAttemptsVector* attempts_; bool oom_; public: explicit ReadTempAttemptsVectorOp(TempOptimizationAttemptsVector* attempts) : attempts_(attempts), oom_(false) { } bool oom() { return oom_; } void operator()(JS::TrackedStrategy strategy, JS::TrackedOutcome outcome) override { if (!attempts_->append(OptimizationAttempt(strategy, outcome))) oom_ = true; } }; struct ReadTempTypeInfoVectorOp : public IonTrackedOptimizationsTypeInfo::ForEachOp { TempAllocator& alloc_; TempOptimizationTypeInfoVector* types_; TempTypeList accTypes_; bool oom_; public: ReadTempTypeInfoVectorOp(TempAllocator& alloc, TempOptimizationTypeInfoVector* types) : alloc_(alloc), types_(types), accTypes_(alloc), oom_(false) { } bool oom() { return oom_; } void readType(const IonTrackedTypeWithAddendum& tracked) override { if (!accTypes_.append(tracked.type)) oom_ = true; } void operator()(JS::TrackedTypeSite site, MIRType mirType) override { OptimizationTypeInfo ty(alloc_, site, mirType); for (uint32_t i = 0; i < accTypes_.length(); i++) { if (!ty.trackType(accTypes_[i])) oom_ = true; } if (!types_->append(mozilla::Move(ty))) oom_ = true; accTypes_.clear(); } }; #endif // DEBUG void CodeGeneratorShared::verifyCompactTrackedOptimizationsMap(JitCode* code, uint32_t numRegions, const UniqueTrackedOptimizations& unique, const IonTrackedTypeVector* allTypes) { #ifdef DEBUG MOZ_ASSERT(trackedOptimizationsMap_ != nullptr); MOZ_ASSERT(trackedOptimizationsMapSize_ > 0); MOZ_ASSERT(trackedOptimizationsRegionTableOffset_ > 0); MOZ_ASSERT(trackedOptimizationsTypesTableOffset_ > 0); MOZ_ASSERT(trackedOptimizationsAttemptsTableOffset_ > 0); // Table pointers must all be 4-byte aligned. const uint8_t* regionTableAddr = trackedOptimizationsMap_ + trackedOptimizationsRegionTableOffset_; const uint8_t* typesTableAddr = trackedOptimizationsMap_ + trackedOptimizationsTypesTableOffset_; const uint8_t* attemptsTableAddr = trackedOptimizationsMap_ + trackedOptimizationsAttemptsTableOffset_; MOZ_ASSERT(uintptr_t(regionTableAddr) % sizeof(uint32_t) == 0); MOZ_ASSERT(uintptr_t(typesTableAddr) % sizeof(uint32_t) == 0); MOZ_ASSERT(uintptr_t(attemptsTableAddr) % sizeof(uint32_t) == 0); // Assert that the number of entries matches up for the tables. const IonTrackedOptimizationsRegionTable* regionTable = (const IonTrackedOptimizationsRegionTable*) regionTableAddr; MOZ_ASSERT(regionTable->numEntries() == numRegions); const IonTrackedOptimizationsTypesTable* typesTable = (const IonTrackedOptimizationsTypesTable*) typesTableAddr; MOZ_ASSERT(typesTable->numEntries() == unique.count()); const IonTrackedOptimizationsAttemptsTable* attemptsTable = (const IonTrackedOptimizationsAttemptsTable*) attemptsTableAddr; MOZ_ASSERT(attemptsTable->numEntries() == unique.count()); // Verify each region. uint32_t trackedIdx = 0; for (uint32_t regionIdx = 0; regionIdx < regionTable->numEntries(); regionIdx++) { // Check reverse offsets are within bounds. MOZ_ASSERT(regionTable->entryOffset(regionIdx) <= trackedOptimizationsRegionTableOffset_); MOZ_ASSERT_IF(regionIdx > 0, regionTable->entryOffset(regionIdx) < regionTable->entryOffset(regionIdx - 1)); IonTrackedOptimizationsRegion region = regionTable->entry(regionIdx); // Check the region range is covered by jitcode. MOZ_ASSERT(region.startOffset() <= code->instructionsSize()); MOZ_ASSERT(region.endOffset() <= code->instructionsSize()); IonTrackedOptimizationsRegion::RangeIterator iter = region.ranges(); while (iter.more()) { // Assert that the offsets are correctly decoded from the delta. uint32_t startOffset, endOffset; uint8_t index; iter.readNext(&startOffset, &endOffset, &index); NativeToTrackedOptimizations& entry = trackedOptimizations_[trackedIdx++]; MOZ_ASSERT(startOffset == entry.startOffset.offset()); MOZ_ASSERT(endOffset == entry.endOffset.offset()); MOZ_ASSERT(index == unique.indexOf(entry.optimizations)); // Assert that the type info and attempts vectors are correctly // decoded. This is disabled for now if the types table might // contain nursery pointers, in which case the types might not // match, see bug 1175761. if (!code->runtimeFromMainThread()->gc.storeBuffer.cancelIonCompilations()) { IonTrackedOptimizationsTypeInfo typeInfo = typesTable->entry(index); TempOptimizationTypeInfoVector tvec(alloc()); ReadTempTypeInfoVectorOp top(alloc(), &tvec); typeInfo.forEach(top, allTypes); MOZ_ASSERT_IF(!top.oom(), entry.optimizations->matchTypes(tvec)); } IonTrackedOptimizationsAttempts attempts = attemptsTable->entry(index); TempOptimizationAttemptsVector avec(alloc()); ReadTempAttemptsVectorOp aop(&avec); attempts.forEach(aop); MOZ_ASSERT_IF(!aop.oom(), entry.optimizations->matchAttempts(avec)); } } #endif } void CodeGeneratorShared::markSafepoint(LInstruction* ins) { markSafepointAt(masm.currentOffset(), ins); } void CodeGeneratorShared::markSafepointAt(uint32_t offset, LInstruction* ins) { MOZ_ASSERT_IF(!safepointIndices_.empty() && !masm.oom(), offset - safepointIndices_.back().displacement() >= sizeof(uint32_t)); masm.propagateOOM(safepointIndices_.append(SafepointIndex(offset, ins->safepoint()))); } void CodeGeneratorShared::ensureOsiSpace() { // For a refresher, an invalidation point is of the form: // 1: call // 2: ... // 3: // // The four bytes *before* instruction 2 are overwritten with an offset. // Callers must ensure that the instruction itself has enough bytes to // support this. // // The bytes *at* instruction 3 are overwritten with an invalidation jump. // jump. These bytes may be in a completely different IR sequence, but // represent the join point of the call out of the function. // // At points where we want to ensure that invalidation won't corrupt an // important instruction, we make sure to pad with nops. if (masm.currentOffset() - lastOsiPointOffset_ < Assembler::PatchWrite_NearCallSize()) { int32_t paddingSize = Assembler::PatchWrite_NearCallSize(); paddingSize -= masm.currentOffset() - lastOsiPointOffset_; for (int32_t i = 0; i < paddingSize; ++i) masm.nop(); } MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() - lastOsiPointOffset_ >= Assembler::PatchWrite_NearCallSize()); lastOsiPointOffset_ = masm.currentOffset(); } uint32_t CodeGeneratorShared::markOsiPoint(LOsiPoint* ins) { encode(ins->snapshot()); ensureOsiSpace(); uint32_t offset = masm.currentOffset(); SnapshotOffset so = ins->snapshot()->snapshotOffset(); masm.propagateOOM(osiIndices_.append(OsiIndex(offset, so))); return offset; } #ifdef CHECK_OSIPOINT_REGISTERS template static void HandleRegisterDump(Op op, MacroAssembler& masm, LiveRegisterSet liveRegs, Register activation, Register scratch) { const size_t baseOffset = JitActivation::offsetOfRegs(); // Handle live GPRs. for (GeneralRegisterIterator iter(liveRegs.gprs()); iter.more(); iter++) { Register reg = *iter; Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg)); if (reg == activation) { // To use the original value of the activation register (that's // now on top of the stack), we need the scratch register. masm.push(scratch); masm.loadPtr(Address(masm.getStackPointer(), sizeof(uintptr_t)), scratch); op(scratch, dump); masm.pop(scratch); } else { op(reg, dump); } } // Handle live FPRs. for (FloatRegisterIterator iter(liveRegs.fpus()); iter.more(); iter++) { FloatRegister reg = *iter; Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg)); op(reg, dump); } } class StoreOp { MacroAssembler& masm; public: explicit StoreOp(MacroAssembler& masm) : masm(masm) {} void operator()(Register reg, Address dump) { masm.storePtr(reg, dump); } void operator()(FloatRegister reg, Address dump) { if (reg.isDouble()) masm.storeDouble(reg, dump); else if (reg.isSingle()) masm.storeFloat32(reg, dump); #if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) else if (reg.isSimd128()) masm.storeUnalignedFloat32x4(reg, dump); #endif else MOZ_CRASH("Unexpected register type."); } }; static void StoreAllLiveRegs(MacroAssembler& masm, LiveRegisterSet liveRegs) { // Store a copy of all live registers before performing the call. // When we reach the OsiPoint, we can use this to check nothing // modified them in the meantime. // Load pointer to the JitActivation in a scratch register. AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All()); Register scratch = allRegs.takeAny(); masm.push(scratch); masm.loadJitActivation(scratch); Address checkRegs(scratch, JitActivation::offsetOfCheckRegs()); masm.add32(Imm32(1), checkRegs); StoreOp op(masm); HandleRegisterDump(op, masm, liveRegs, scratch, allRegs.getAny()); masm.pop(scratch); } class VerifyOp { MacroAssembler& masm; Label* failure_; public: VerifyOp(MacroAssembler& masm, Label* failure) : masm(masm), failure_(failure) {} void operator()(Register reg, Address dump) { masm.branchPtr(Assembler::NotEqual, dump, reg, failure_); } void operator()(FloatRegister reg, Address dump) { FloatRegister scratch; if (reg.isDouble()) { scratch = ScratchDoubleReg; masm.loadDouble(dump, scratch); masm.branchDouble(Assembler::DoubleNotEqual, scratch, reg, failure_); } else if (reg.isSingle()) { scratch = ScratchFloat32Reg; masm.loadFloat32(dump, scratch); masm.branchFloat(Assembler::DoubleNotEqual, scratch, reg, failure_); } // :TODO: (Bug 1133745) Add support to verify SIMD registers. } }; void CodeGeneratorShared::verifyOsiPointRegs(LSafepoint* safepoint) { // Ensure the live registers stored by callVM did not change between // the call and this OsiPoint. Try-catch relies on this invariant. // Load pointer to the JitActivation in a scratch register. AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All()); Register scratch = allRegs.takeAny(); masm.push(scratch); masm.loadJitActivation(scratch); // If we should not check registers (because the instruction did not call // into the VM, or a GC happened), we're done. Label failure, done; Address checkRegs(scratch, JitActivation::offsetOfCheckRegs()); masm.branch32(Assembler::Equal, checkRegs, Imm32(0), &done); // Having more than one VM function call made in one visit function at // runtime is a sec-ciritcal error, because if we conservatively assume that // one of the function call can re-enter Ion, then the invalidation process // will potentially add a call at a random location, by patching the code // before the return address. masm.branch32(Assembler::NotEqual, checkRegs, Imm32(1), &failure); // Set checkRegs to 0, so that we don't try to verify registers after we // return from this script to the caller. masm.store32(Imm32(0), checkRegs); // Ignore clobbered registers. Some instructions (like LValueToInt32) modify // temps after calling into the VM. This is fine because no other // instructions (including this OsiPoint) will depend on them. Also // backtracking can also use the same register for an input and an output. // These are marked as clobbered and shouldn't get checked. LiveRegisterSet liveRegs; liveRegs.set() = RegisterSet::Intersect(safepoint->liveRegs().set(), RegisterSet::Not(safepoint->clobberedRegs().set())); VerifyOp op(masm, &failure); HandleRegisterDump(op, masm, liveRegs, scratch, allRegs.getAny()); masm.jump(&done); // Do not profile the callWithABI that occurs below. This is to avoid a // rare corner case that occurs when profiling interacts with itself: // // When slow profiling assertions are turned on, FunctionBoundary ops // (which update the profiler pseudo-stack) may emit a callVM, which // forces them to have an osi point associated with them. The // FunctionBoundary for inline function entry is added to the caller's // graph with a PC from the caller's code, but during codegen it modifies // SPS instrumentation to add the callee as the current top-most script. // When codegen gets to the OSIPoint, and the callWithABI below is // emitted, the codegen thinks that the current frame is the callee, but // the PC it's using from the OSIPoint refers to the caller. This causes // the profiler instrumentation of the callWithABI below to ASSERT, since // the script and pc are mismatched. To avoid this, we simply omit // instrumentation for these callWithABIs. // Any live register captured by a safepoint (other than temp registers) // must remain unchanged between the call and the OsiPoint instruction. masm.bind(&failure); masm.assumeUnreachable("Modified registers between VM call and OsiPoint"); masm.bind(&done); masm.pop(scratch); } bool CodeGeneratorShared::shouldVerifyOsiPointRegs(LSafepoint* safepoint) { if (!checkOsiPointRegisters) return false; if (safepoint->liveRegs().emptyGeneral() && safepoint->liveRegs().emptyFloat()) return false; // No registers to check. return true; } void CodeGeneratorShared::resetOsiPointRegs(LSafepoint* safepoint) { if (!shouldVerifyOsiPointRegs(safepoint)) return; // Set checkRegs to 0. If we perform a VM call, the instruction // will set it to 1. AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All()); Register scratch = allRegs.takeAny(); masm.push(scratch); masm.loadJitActivation(scratch); Address checkRegs(scratch, JitActivation::offsetOfCheckRegs()); masm.store32(Imm32(0), checkRegs); masm.pop(scratch); } #endif // Before doing any call to Cpp, you should ensure that volatile // registers are evicted by the register allocator. void CodeGeneratorShared::callVM(const VMFunction& fun, LInstruction* ins, const Register* dynStack) { // If we're calling a function with an out parameter type of double, make // sure we have an FPU. MOZ_ASSERT_IF(fun.outParam == Type_Double, GetJitContext()->runtime->jitSupportsFloatingPoint()); #ifdef DEBUG if (ins->mirRaw()) { MOZ_ASSERT(ins->mirRaw()->isInstruction()); MInstruction* mir = ins->mirRaw()->toInstruction(); MOZ_ASSERT_IF(mir->needsResumePoint(), mir->resumePoint()); } #endif #ifdef JS_TRACE_LOGGING emitTracelogStartEvent(TraceLogger_VM); #endif // Stack is: // ... frame ... // [args] #ifdef DEBUG MOZ_ASSERT(pushedArgs_ == fun.explicitArgs); pushedArgs_ = 0; #endif // Get the wrapper of the VM function. JitCode* wrapper = gen->jitRuntime()->getVMWrapper(fun); if (!wrapper) { masm.setOOM(); return; } #ifdef CHECK_OSIPOINT_REGISTERS if (shouldVerifyOsiPointRegs(ins->safepoint())) StoreAllLiveRegs(masm, ins->safepoint()->liveRegs()); #endif // Push an exit frame descriptor. If |dynStack| is a valid pointer to a // register, then its value is added to the value of the |framePushed()| to // fill the frame descriptor. if (dynStack) { masm.addPtr(Imm32(masm.framePushed()), *dynStack); masm.makeFrameDescriptor(*dynStack, JitFrame_IonJS); masm.Push(*dynStack); // descriptor } else { masm.pushStaticFrameDescriptor(JitFrame_IonJS); } // Call the wrapper function. The wrapper is in charge to unwind the stack // when returning from the call. Failures are handled with exceptions based // on the return value of the C functions. To guard the outcome of the // returned value, use another LIR instruction. ensureOsiSpace(); uint32_t callOffset = masm.callJit(wrapper); markSafepointAt(callOffset, ins); // Remove rest of the frame left on the stack. We remove the return address // which is implicitly poped when returning. int framePop = sizeof(ExitFrameLayout) - sizeof(void*); // Pop arguments from framePushed. masm.implicitPop(fun.explicitStackSlots() * sizeof(void*) + framePop); // Stack is: // ... frame ... #ifdef JS_TRACE_LOGGING emitTracelogStopEvent(TraceLogger_VM); #endif } class OutOfLineTruncateSlow : public OutOfLineCodeBase { FloatRegister src_; Register dest_; bool needFloat32Conversion_; public: OutOfLineTruncateSlow(FloatRegister src, Register dest, bool needFloat32Conversion = false) : src_(src), dest_(dest), needFloat32Conversion_(needFloat32Conversion) { } void accept(CodeGeneratorShared* codegen) { codegen->visitOutOfLineTruncateSlow(this); } FloatRegister src() const { return src_; } Register dest() const { return dest_; } bool needFloat32Conversion() const { return needFloat32Conversion_; } }; OutOfLineCode* CodeGeneratorShared::oolTruncateDouble(FloatRegister src, Register dest, MInstruction* mir) { OutOfLineTruncateSlow* ool = new(alloc()) OutOfLineTruncateSlow(src, dest); addOutOfLineCode(ool, mir); return ool; } void CodeGeneratorShared::emitTruncateDouble(FloatRegister src, Register dest, MInstruction* mir) { OutOfLineCode* ool = oolTruncateDouble(src, dest, mir); masm.branchTruncateDouble(src, dest, ool->entry()); masm.bind(ool->rejoin()); } void CodeGeneratorShared::emitTruncateFloat32(FloatRegister src, Register dest, MInstruction* mir) { OutOfLineTruncateSlow* ool = new(alloc()) OutOfLineTruncateSlow(src, dest, true); addOutOfLineCode(ool, mir); masm.branchTruncateFloat32(src, dest, ool->entry()); masm.bind(ool->rejoin()); } void CodeGeneratorShared::visitOutOfLineTruncateSlow(OutOfLineTruncateSlow* ool) { FloatRegister src = ool->src(); Register dest = ool->dest(); saveVolatile(dest); #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) if (ool->needFloat32Conversion()) { masm.convertFloat32ToDouble(src, ScratchDoubleReg); src = ScratchDoubleReg; } #else FloatRegister srcSingle = src.asSingle(); if (ool->needFloat32Conversion()) { MOZ_ASSERT(src.isSingle()); masm.push(src); masm.convertFloat32ToDouble(src, src); src = src.asDouble(); } #endif masm.setupUnalignedABICall(dest); masm.passABIArg(src, MoveOp::DOUBLE); if (gen->compilingAsmJS()) masm.callWithABI(wasm::SymbolicAddress::ToInt32); else masm.callWithABI(BitwiseCast(JS::ToInt32)); masm.storeCallResult(dest); #if !defined(JS_CODEGEN_ARM) && !defined(JS_CODEGEN_ARM64) if (ool->needFloat32Conversion()) masm.pop(srcSingle); #endif restoreVolatile(dest); masm.jump(ool->rejoin()); } bool CodeGeneratorShared::omitOverRecursedCheck() const { // If the current function makes no calls (which means it isn't recursive) // and it uses only a small amount of stack space, it doesn't need a // stack overflow check. Note that the actual number here is somewhat // arbitrary, and codegen actually uses small bounded amounts of // additional stack space in some cases too. return frameSize() < 64 && !gen->performsCall(); } void CodeGeneratorShared::emitAsmJSCall(LAsmJSCall* ins) { MAsmJSCall* mir = ins->mir(); if (mir->spIncrement()) masm.freeStack(mir->spIncrement()); MOZ_ASSERT((sizeof(AsmJSFrame) + masm.framePushed()) % AsmJSStackAlignment == 0); #ifdef DEBUG static_assert(AsmJSStackAlignment >= ABIStackAlignment && AsmJSStackAlignment % ABIStackAlignment == 0, "The asm.js stack alignment should subsume the ABI-required alignment"); Label ok; masm.branchTestStackPtr(Assembler::Zero, Imm32(AsmJSStackAlignment - 1), &ok); masm.breakpoint(); masm.bind(&ok); #endif MAsmJSCall::Callee callee = mir->callee(); switch (callee.which()) { case MAsmJSCall::Callee::Internal: masm.call(mir->desc(), callee.internal()); break; case MAsmJSCall::Callee::Dynamic: masm.call(mir->desc(), ToRegister(ins->getOperand(mir->dynamicCalleeOperandIndex()))); break; case MAsmJSCall::Callee::Builtin: masm.call(BuiltinToImmediate(callee.builtin())); break; } if (mir->spIncrement()) masm.reserveStack(mir->spIncrement()); } void CodeGeneratorShared::emitPreBarrier(Register base, const LAllocation* index) { if (index->isConstant()) { Address address(base, ToInt32(index) * sizeof(Value)); masm.patchableCallPreBarrier(address, MIRType_Value); } else { BaseIndex address(base, ToRegister(index), TimesEight); masm.patchableCallPreBarrier(address, MIRType_Value); } } void CodeGeneratorShared::emitPreBarrier(Address address) { masm.patchableCallPreBarrier(address, MIRType_Value); } Label* CodeGeneratorShared::labelForBackedgeWithImplicitCheck(MBasicBlock* mir) { // If this is a loop backedge to a loop header with an implicit interrupt // check, use a patchable jump. Skip this search if compiling without a // script for asm.js, as there will be no interrupt check instruction. // Due to critical edge unsplitting there may no longer be unique loop // backedges, so just look for any edge going to an earlier block in RPO. if (!gen->compilingAsmJS() && mir->isLoopHeader() && mir->id() <= current->mir()->id()) { for (LInstructionIterator iter = mir->lir()->begin(); iter != mir->lir()->end(); iter++) { if (iter->isMoveGroup()) { // Continue searching for an interrupt check. } else if (iter->isInterruptCheckImplicit()) { return iter->toInterruptCheckImplicit()->oolEntry(); } else { // The interrupt check should be the first instruction in the // loop header other than the initial label and move groups. MOZ_ASSERT(iter->isInterruptCheck()); return nullptr; } } } return nullptr; } void CodeGeneratorShared::jumpToBlock(MBasicBlock* mir) { // Skip past trivial blocks. mir = skipTrivialBlocks(mir); // No jump necessary if we can fall through to the next block. if (isNextBlock(mir->lir())) return; if (Label* oolEntry = labelForBackedgeWithImplicitCheck(mir)) { // Note: the backedge is initially a jump to the next instruction. // It will be patched to the target block's label during link(). RepatchLabel rejoin; CodeOffsetJump backedge = masm.backedgeJump(&rejoin, mir->lir()->label()); masm.bind(&rejoin); masm.propagateOOM(patchableBackedges_.append(PatchableBackedgeInfo(backedge, mir->lir()->label(), oolEntry))); } else { masm.jump(mir->lir()->label()); } } // This function is not used for MIPS/MIPS64. MIPS has branchToBlock. #if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) && !defined(JS_CODEGEN_OSX_PPC) void CodeGeneratorShared::jumpToBlock(MBasicBlock* mir, Assembler::Condition cond) { // Skip past trivial blocks. mir = skipTrivialBlocks(mir); if (Label* oolEntry = labelForBackedgeWithImplicitCheck(mir)) { // Note: the backedge is initially a jump to the next instruction. // It will be patched to the target block's label during link(). RepatchLabel rejoin; CodeOffsetJump backedge = masm.jumpWithPatch(&rejoin, cond, mir->lir()->label()); masm.bind(&rejoin); masm.propagateOOM(patchableBackedges_.append(PatchableBackedgeInfo(backedge, mir->lir()->label(), oolEntry))); } else { masm.j(cond, mir->lir()->label()); } } #endif MOZ_WARN_UNUSED_RESULT bool CodeGeneratorShared::addCacheLocations(const CacheLocationList& locs, size_t* numLocs, size_t* curIndex) { size_t firstIndex = runtimeData_.length(); size_t numLocations = 0; for (CacheLocationList::iterator iter = locs.begin(); iter != locs.end(); iter++) { // allocateData() ensures that sizeof(CacheLocation) is word-aligned. // If this changes, we will need to pad to ensure alignment. if (!allocateData(sizeof(CacheLocation), curIndex)) return false; new (&runtimeData_[*curIndex]) CacheLocation(iter->pc, iter->script); numLocations++; } MOZ_ASSERT(numLocations != 0); *numLocs = numLocations; *curIndex = firstIndex; return true; } ReciprocalMulConstants CodeGeneratorShared::computeDivisionConstants(uint32_t d, int maxLog) { MOZ_ASSERT(maxLog >= 2 && maxLog <= 32); // In what follows, 0 < d < 2^maxLog and d is not a power of 2. MOZ_ASSERT(d < (uint64_t(1) << maxLog) && (d & (d - 1)) != 0); // Speeding up division by non power-of-2 constants is possible by // calculating, during compilation, a value M such that high-order // bits of M*n correspond to the result of the division of n by d. // No value of M can serve this purpose for arbitrarily big values // of n but, for optimizing integer division, we're just concerned // with values of n whose absolute value is bounded (by fitting in // an integer type, say). With this in mind, we'll find a constant // M as above that works for -2^maxLog <= n < 2^maxLog; maxLog can // then be 31 for signed division or 32 for unsigned division. // // The original presentation of this technique appears in Hacker's // Delight, a book by Henry S. Warren, Jr.. A proof of correctness // for our version follows; we'll denote maxLog by L in the proof, // for conciseness. // // Formally, for |d| < 2^L, we'll compute two magic values M and s // in the ranges 0 <= M < 2^(L+1) and 0 <= s <= L such that // (M * n) >> (32 + s) = floor(n/d) if 0 <= n < 2^L // (M * n) >> (32 + s) = ceil(n/d) - 1 if -2^L <= n < 0. // // Define p = 32 + s, M = ceil(2^p/d), and assume that s satisfies // M - 2^p/d <= 2^(p-L)/d. (1) // (Observe that p = CeilLog32(d) + L satisfies this, as the right // side of (1) is at least one in this case). Then, // // a) If p <= CeilLog32(d) + L, then M < 2^(L+1) - 1. // Proof: Indeed, M is monotone in p and, for p equal to the above // value, the bounds 2^L > d >= 2^(p-L-1) + 1 readily imply that // 2^p / d < 2^p/(d - 1) * (d - 1)/d // <= 2^(L+1) * (1 - 1/d) < 2^(L+1) - 2. // The claim follows by applying the ceiling function. // // b) For any 0 <= n < 2^L, floor(Mn/2^p) = floor(n/d). // Proof: Put x = floor(Mn/2^p); it's the unique integer for which // Mn/2^p - 1 < x <= Mn/2^p. (2) // Using M >= 2^p/d on the LHS and (1) on the RHS, we get // n/d - 1 < x <= n/d + n/(2^L d) < n/d + 1/d. // Since x is an integer, it's not in the interval (n/d, (n+1)/d), // and so n/d - 1 < x <= n/d, which implies x = floor(n/d). // // c) For any -2^L <= n < 0, floor(Mn/2^p) + 1 = ceil(n/d). // Proof: The proof is similar. Equation (2) holds as above. Using // M > 2^p/d (d isn't a power of 2) on the RHS and (1) on the LHS, // n/d + n/(2^L d) - 1 < x < n/d. // Using n >= -2^L and summing 1, // n/d - 1/d < x + 1 < n/d + 1. // Since x + 1 is an integer, this implies n/d <= x + 1 < n/d + 1. // In other words, x + 1 = ceil(n/d). // // Condition (1) isn't necessary for the existence of M and s with // the properties above. Hacker's Delight provides a slightly less // restrictive condition when d >= 196611, at the cost of a 3-page // proof of correctness, for the case L = 31. // // Note that, since d*M - 2^p = d - (2^p)%d, (1) can be written as // 2^(p-L) >= d - (2^p)%d. // In order to avoid overflow in the (2^p) % d calculation, we can // compute it as (2^p-1) % d + 1, where 2^p-1 can then be computed // without overflow as UINT64_MAX >> (64-p). // We now compute the least p >= 32 with the property above... int32_t p = 32; while ((uint64_t(1) << (p-maxLog)) + (UINT64_MAX >> (64-p)) % d + 1 < d) p++; // ...and the corresponding M. For either the signed (L=31) or the // unsigned (L=32) case, this value can be too large (cf. item a). // Codegen can still multiply by M by multiplying by (M - 2^L) and // adjusting the value afterwards, if this is the case. ReciprocalMulConstants rmc; rmc.multiplier = (UINT64_MAX >> (64-p))/d + 1; rmc.shiftAmount = p - 32; return rmc; } #ifdef JS_TRACE_LOGGING void CodeGeneratorShared::emitTracelogScript(bool isStart) { if (!TraceLogTextIdEnabled(TraceLogger_Scripts)) return; Label done; AllocatableRegisterSet regs(RegisterSet::Volatile()); Register logger = regs.takeAnyGeneral(); Register script = regs.takeAnyGeneral(); masm.Push(logger); CodeOffset patchLogger = masm.movWithPatch(ImmPtr(nullptr), logger); masm.propagateOOM(patchableTraceLoggers_.append(patchLogger)); Address enabledAddress(logger, TraceLoggerThread::offsetOfEnabled()); masm.branch32(Assembler::Equal, enabledAddress, Imm32(0), &done); masm.Push(script); CodeOffset patchScript = masm.movWithPatch(ImmWord(0), script); masm.propagateOOM(patchableTLScripts_.append(patchScript)); if (isStart) masm.tracelogStartId(logger, script); else masm.tracelogStopId(logger, script); masm.Pop(script); masm.bind(&done); masm.Pop(logger); } void CodeGeneratorShared::emitTracelogTree(bool isStart, uint32_t textId) { if (!TraceLogTextIdEnabled(textId)) return; Label done; AllocatableRegisterSet regs(RegisterSet::Volatile()); Register logger = regs.takeAnyGeneral(); masm.Push(logger); CodeOffset patchLocation = masm.movWithPatch(ImmPtr(nullptr), logger); masm.propagateOOM(patchableTraceLoggers_.append(patchLocation)); Address enabledAddress(logger, TraceLoggerThread::offsetOfEnabled()); masm.branch32(Assembler::Equal, enabledAddress, Imm32(0), &done); if (isStart) masm.tracelogStartId(logger, textId); else masm.tracelogStopId(logger, textId); masm.bind(&done); masm.Pop(logger); } #endif } // namespace jit } // namespace js