tenfourfox/intl/uconv/nsUTF8ToUnicode.cpp
Cameron Kaiser c9b2922b70 hello FPR
2017-04-19 00:56:45 -07:00

459 lines
14 KiB
C++

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsUCSupport.h"
#include "nsUTF8ToUnicode.h"
#include "mozilla/CheckedInt.h"
#include "mozilla/SSE.h"
#include "nsCharTraits.h"
#include <algorithm>
#ifdef TENFOURFOX_VMX
#include <altivec.h>
#endif
#define UNICODE_BYTE_ORDER_MARK 0xFEFF
static char16_t* EmitSurrogatePair(uint32_t ucs4, char16_t* aDest)
{
NS_ASSERTION(ucs4 > 0xFFFF, "Should be a supplementary character");
ucs4 -= 0x00010000;
*aDest++ = 0xD800 | (0x000003FF & (ucs4 >> 10));
*aDest++ = 0xDC00 | (0x000003FF & ucs4);
return aDest;
}
//----------------------------------------------------------------------
// Class nsUTF8ToUnicode [implementation]
nsUTF8ToUnicode::nsUTF8ToUnicode()
: nsBasicDecoderSupport()
{
Reset();
}
//----------------------------------------------------------------------
// Subclassing of nsTableDecoderSupport class [implementation]
/**
* Normally the maximum length of the output of the UTF8 decoder in UTF16
* code units is the same as the length of the input in UTF8 code units,
* since 1-byte, 2-byte and 3-byte UTF-8 sequences decode to a single
* UTF-16 character, and 4-byte UTF-8 sequences decode to a surrogate pair.
*
* However, there is an edge case where the output can be longer than the
* input: if the previous buffer ended with an incomplete multi-byte
* sequence and this buffer does not begin with a valid continuation
* byte, we will return NS_ERROR_ILLEGAL_INPUT and the caller may insert a
* replacement character in the output buffer which corresponds to no
* character in the input buffer. So in the worst case the destination
* will need to be one code unit longer than the source.
* See bug 301797.
*/
NS_IMETHODIMP nsUTF8ToUnicode::GetMaxLength(const char * aSrc,
int32_t aSrcLength,
int32_t * aDestLength)
{
mozilla::CheckedInt32 length = aSrcLength;
length += 1;
if (!length.isValid()) {
return NS_ERROR_OUT_OF_MEMORY;
}
*aDestLength = length.value();
return NS_OK;
}
//----------------------------------------------------------------------
// Subclassing of nsBasicDecoderSupport class [implementation]
NS_IMETHODIMP nsUTF8ToUnicode::Reset()
{
mUcs4 = 0; // cached Unicode character
mState = 0; // cached expected number of octets after the current octet
// until the beginning of the next UTF8 character sequence
mBytes = 1; // cached expected number of octets in the current sequence
mFirst = true;
return NS_OK;
}
//----------------------------------------------------------------------
// Subclassing of nsBasicDecoderSupport class [implementation]
// Fast ASCII -> UTF16 inner loop implementations
//
// Convert_ascii_run will update src and dst to the new values, and
// len must be the maximum number ascii chars that it would be valid
// to take from src and place into dst. (That is, the minimum of the
// number of bytes left in src and the number of unichars available in
// dst.)
#ifdef TENFOURFOX_VMX
// Safe to use with aligned and unaligned addresses
#define LoadUnaligned(return, index, target, MSQ, LSQ, mask) \
{ \
LSQ = vec_ldl(index + 15, target); \
return = vec_perm(MSQ, LSQ, mask); \
}
// Based broadly on the SSE2 version -- Cameron Kaiser
// Unaligned version by Tobias Netzel
static inline void
Convert_ascii_run(const char *&src, char16_t *&dst, int32_t len) {
int32_t i = 0;
int32_t alignLen =
std::min(len, int32_t(((-NS_PTR_TO_UINT32(dst)) & 0xf) / sizeof(char16_t)));
// Not worth doing if < 16 bytes.
if (len - alignLen > 15) {
// Align destination to a 16-byte boundary.
for (; i < alignLen; i++) {
if (src[i] & 0x80U) {
src += i;
dst += i;
len -= i;
return;
}
dst[i] = static_cast<unsigned char>(src[i]);
}
// Watch the offset. If we are at a point where our dst and
// src are both 16-byte aligned, use aligned stores. Let
// failure fall through to the bottom to catch byte-by-byte
// moves. This is not as efficient as the SSE2 version because
// unaligned stores are much more difficult on AltiVec.
// However, OS X usually gives us allocations that are aligned.
int32_t maxIndex = len - 17;
register vector unsigned char in, out1, out2;
register vector unsigned char zeroes = vec_splat_u8(0);
register const vector unsigned char bit7 = vec_sl( vec_splat_u8( 1 ), vec_splat_u8( 7 ) );
int32_t ourDstOffset = i * 2;
if ((NS_PTR_TO_UINT32(&src[i]) & 15) == 0) {
// Aligned version (typical path)
for (; i < maxIndex; i += 16, ourDstOffset += 32) {
in = vec_ldl(i, (unsigned char *)src);
if (vec_any_ge(in, bit7)) break; // NOT RETURN!
// Convert, essentially, char to short.
out1 = vec_mergeh(zeroes, in);
out2 = vec_mergel(zeroes, in);
vec_st(out1, ourDstOffset, (unsigned char *)dst);
vec_st(out2, ourDstOffset + 16, (unsigned char *)dst);
}
}
else {
// Unaligned version
register vector unsigned char mask = vec_lvsl(i, (unsigned char *)src);
register vector unsigned char vector1 = vec_ldl(i, (unsigned char *)src);
register vector unsigned char vector2;
for (; i < maxIndex; i += 16, ourDstOffset += 32) {
LoadUnaligned(in, i, (unsigned char *)src, vector1, vector2, mask);
if (vec_any_ge(in, bit7)) break; // NOT RETURN!
// Convert, essentially, char to short.
out1 = vec_mergeh(zeroes, in);
out2 = vec_mergel(zeroes, in);
vec_st(out1, ourDstOffset, (unsigned char *)dst);
vec_st(out2, ourDstOffset + 16, (unsigned char *)dst);
vector1 = vector2;
}
}
// Don't bother with half moves.
}
// finish off byte by byte
for (; i < len && (src[i] & 0x80U) == 0; i++) {
dst[i] = static_cast<unsigned char>(src[i]);
}
src += i;
dst += i;
len -= i;
}
#endif
#if defined(__arm__) || defined(_M_ARM)
// on ARM, do extra work to avoid byte/halfword reads/writes by
// reading/writing a word at a time for as long as we can
static inline void
Convert_ascii_run (const char *&src,
char16_t *&dst,
int32_t len)
{
const uint32_t *src32;
uint32_t *dst32;
// with some alignments, we'd never actually break out of the slow loop, so
// check and do the faster slow loop
if ((((NS_PTR_TO_UINT32(dst) & 3) == 0) && ((NS_PTR_TO_UINT32(src) & 1) == 0)) ||
(((NS_PTR_TO_UINT32(dst) & 3) == 2) && ((NS_PTR_TO_UINT32(src) & 1) == 1)))
{
while (((NS_PTR_TO_UINT32(src) & 3) ||
(NS_PTR_TO_UINT32(dst) & 3)) &&
len > 0)
{
if (*src & 0x80U)
return;
*dst++ = (char16_t) *src++;
len--;
}
} else {
goto finish;
}
// then go 4 bytes at a time
src32 = (const uint32_t*) src;
dst32 = (uint32_t*) dst;
while (len > 4) {
uint32_t in = *src32++;
if (in & 0x80808080U) {
src32--;
break;
}
*dst32++ = ((in & 0x000000ff) >> 0) | ((in & 0x0000ff00) << 8);
*dst32++ = ((in & 0x00ff0000) >> 16) | ((in & 0xff000000) >> 8);
len -= 4;
}
src = (const char *) src32;
dst = (char16_t *) dst32;
finish:
while (len-- > 0 && (*src & 0x80U) == 0) {
*dst++ = (char16_t) *src++;
}
}
#else
#ifdef MOZILLA_MAY_SUPPORT_SSE2
namespace mozilla {
namespace SSE2 {
void Convert_ascii_run(const char *&src, char16_t *&dst, int32_t len);
} // namespace SSE2
} // namespace mozilla
#endif
#ifndef TENFOURFOX_VMX
static inline void
Convert_ascii_run (const char *&src,
char16_t *&dst,
int32_t len)
{
#ifdef MOZILLA_MAY_SUPPORT_SSE2
if (mozilla::supports_sse2()) {
mozilla::SSE2::Convert_ascii_run(src, dst, len);
return;
}
#endif
while (len-- > 0 && (*src & 0x80U) == 0) {
*dst++ = (char16_t) *src++;
}
}
#endif
#endif
NS_IMETHODIMP nsUTF8ToUnicode::Convert(const char * aSrc,
int32_t * aSrcLength,
char16_t * aDest,
int32_t * aDestLength)
{
uint32_t aSrcLen = (uint32_t) (*aSrcLength);
uint32_t aDestLen = (uint32_t) (*aDestLength);
const char *in, *inend;
inend = aSrc + aSrcLen;
char16_t *out, *outend;
outend = aDest + aDestLen;
nsresult res = NS_OK; // conversion result
out = aDest;
if (mState == 0xFF) {
// Emit supplementary character left over from previous iteration. It is
// caller's responsibility to keep a sufficient buffer.
if (aDestLen < 2) {
*aSrcLength = *aDestLength = 0;
return NS_OK_UDEC_MOREOUTPUT;
}
out = EmitSurrogatePair(mUcs4, out);
mUcs4 = 0;
mState = 0;
mBytes = 1;
mFirst = false;
}
// alias these locally for speed
int32_t mUcs4 = this->mUcs4;
uint8_t mState = this->mState;
uint8_t mBytes = this->mBytes;
bool mFirst = this->mFirst;
// Set mFirst to false now so we don't have to every time through the ASCII
// branch within the loop.
if (mFirst && aSrcLen && (0 == (0x80 & (*aSrc))))
mFirst = false;
for (in = aSrc; ((in < inend) && (out < outend)); ++in) {
uint8_t c = *in;
if (0 == mState) {
// When mState is zero we expect either a US-ASCII character or a
// multi-octet sequence.
if (c < 0x80) { // 00..7F
int32_t max_loops = std::min(inend - in, outend - out);
Convert_ascii_run(in, out, max_loops);
--in; // match the rest of the cases
mBytes = 1;
} else if (c < 0xC2) { // C0/C1
// Overlong 2 octet sequence
if (mErrBehavior == kOnError_Signal) {
res = NS_ERROR_ILLEGAL_INPUT;
break;
}
*out++ = UCS2_REPLACEMENT_CHAR;
mFirst = false;
} else if (c < 0xE0) { // C2..DF
// First octet of 2 octet sequence
mUcs4 = c;
mUcs4 = (mUcs4 & 0x1F) << 6;
mState = 1;
mBytes = 2;
} else if (c < 0xF0) { // E0..EF
// First octet of 3 octet sequence
mUcs4 = c;
mUcs4 = (mUcs4 & 0x0F) << 12;
mState = 2;
mBytes = 3;
} else if (c < 0xF5) { // F0..F4
// First octet of 4 octet sequence
mUcs4 = c;
mUcs4 = (mUcs4 & 0x07) << 18;
mState = 3;
mBytes = 4;
} else { // F5..FF
/* Current octet is neither in the US-ASCII range nor a legal first
* octet of a multi-octet sequence.
*/
if (mErrBehavior == kOnError_Signal) {
/* Return an error condition. Caller is responsible for flushing and
* refilling the buffer and resetting state.
*/
res = NS_ERROR_ILLEGAL_INPUT;
break;
}
*out++ = UCS2_REPLACEMENT_CHAR;
mFirst = false;
}
} else {
// When mState is non-zero, we expect a continuation of the multi-octet
// sequence
if (0x80 == (0xC0 & c)) {
if (mState > 1) {
// If we are here, all possibilities are:
// mState == 2 && mBytes == 3 ||
// mState == 2 && mBytes == 4 ||
// mState == 3 && mBytes == 4
if ((mBytes == 3 && ((!mUcs4 && c < 0xA0) || // E0 80..9F
(mUcs4 == 0xD000 && c > 0x9F))) || // ED A0..BF
(mState == 3 && ((!mUcs4 && c < 0x90) || // F0 80..8F
(mUcs4 == 0x100000 && c > 0x8F)))) {// F4 90..BF
// illegal sequences or sequences converted into illegal ranges.
in--;
if (mErrBehavior == kOnError_Signal) {
res = NS_ERROR_ILLEGAL_INPUT;
break;
}
*out++ = UCS2_REPLACEMENT_CHAR;
mState = 0;
mFirst = false;
continue;
}
}
// Legal continuation.
uint32_t shift = (mState - 1) * 6;
uint32_t tmp = c;
tmp = (tmp & 0x0000003FL) << shift;
mUcs4 |= tmp;
if (0 == --mState) {
/* End of the multi-octet sequence. mUcs4 now contains the final
* Unicode codepoint to be output
*/
if (mUcs4 > 0xFFFF) {
// mUcs4 is in the range 0x10000 - 0x10FFFF. Output a UTF-16 pair
if (out + 2 > outend) {
// insufficient space left in the buffer. Keep mUcs4 for the
// next iteration.
mState = 0xFF;
++in;
res = NS_OK_UDEC_MOREOUTPUT;
break;
}
out = EmitSurrogatePair(mUcs4, out);
} else if (UNICODE_BYTE_ORDER_MARK != mUcs4 || !mFirst) {
// Don't output the BOM only if it is the first character
*out++ = mUcs4;
}
//initialize UTF8 cache
mUcs4 = 0;
mState = 0;
mBytes = 1;
mFirst = false;
}
} else {
/* ((0xC0 & c != 0x80) && (mState != 0))
*
* Incomplete multi-octet sequence. Unconsume this
* octet and return an error condition. Caller is responsible
* for flushing and refilling the buffer and resetting state.
*/
in--;
if (mErrBehavior == kOnError_Signal) {
res = NS_ERROR_ILLEGAL_INPUT;
break;
}
*out++ = UCS2_REPLACEMENT_CHAR;
mState = 0;
mFirst = false;
}
}
}
// output not finished, output buffer too short
if ((NS_OK == res) && (in < inend) && (out >= outend))
res = NS_OK_UDEC_MOREOUTPUT;
// last UCS4 is incomplete, make sure the caller
// returns with properly aligned continuation of the buffer
if ((NS_OK == res) && (mState != 0))
res = NS_OK_UDEC_MOREINPUT;
*aSrcLength = in - aSrc;
*aDestLength = out - aDest;
this->mUcs4 = mUcs4;
this->mState = mState;
this->mBytes = mBytes;
this->mFirst = mFirst;
return(res);
}