mirror of
https://github.com/classilla/tenfourfox.git
synced 2024-09-10 11:54:40 +00:00
1234 lines
29 KiB
C++
1234 lines
29 KiB
C++
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
|
|
/* vim: set ts=2 sw=2 et tw=78: */
|
|
/* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
|
|
|
|
//#define __INCREMENTAL 1
|
|
|
|
#include "mozilla/DebugOnly.h"
|
|
|
|
#include "nsScanner.h"
|
|
#include "nsDebug.h"
|
|
#include "nsReadableUtils.h"
|
|
#include "nsIInputStream.h"
|
|
#include "nsIFile.h"
|
|
#include "nsUTF8Utils.h" // for LossyConvertEncoding
|
|
#include "nsCRT.h"
|
|
#include "nsParser.h"
|
|
#include "nsCharsetSource.h"
|
|
|
|
#include "mozilla/dom/EncodingUtils.h"
|
|
|
|
using mozilla::dom::EncodingUtils;
|
|
|
|
// We replace NUL characters with this character.
|
|
static char16_t sInvalid = UCS2_REPLACEMENT_CHAR;
|
|
|
|
nsReadEndCondition::nsReadEndCondition(const char16_t* aTerminateChars) :
|
|
mChars(aTerminateChars), mFilter(char16_t(~0)) // All bits set
|
|
{
|
|
// Build filter that will be used to filter out characters with
|
|
// bits that none of the terminal chars have. This works very well
|
|
// because terminal chars often have only the last 4-6 bits set and
|
|
// normal ascii letters have bit 7 set. Other letters have even higher
|
|
// bits set.
|
|
|
|
// Calculate filter
|
|
const char16_t *current = aTerminateChars;
|
|
char16_t terminalChar = *current;
|
|
while (terminalChar) {
|
|
mFilter &= ~terminalChar;
|
|
++current;
|
|
terminalChar = *current;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Use this constructor if you want i/o to be based on
|
|
* a single string you hand in during construction.
|
|
* This short cut was added for Javascript.
|
|
*
|
|
* @update gess 5/12/98
|
|
* @param aMode represents the parser mode (nav, other)
|
|
* @return
|
|
*/
|
|
nsScanner::nsScanner(const nsAString& anHTMLString)
|
|
{
|
|
MOZ_COUNT_CTOR(nsScanner);
|
|
|
|
mSlidingBuffer = nullptr;
|
|
mCountRemaining = 0;
|
|
mFirstNonWhitespacePosition = -1;
|
|
if (AppendToBuffer(anHTMLString)) {
|
|
mSlidingBuffer->BeginReading(mCurrentPosition);
|
|
} else {
|
|
/* XXX see hack below, re: bug 182067 */
|
|
memset(&mCurrentPosition, 0, sizeof(mCurrentPosition));
|
|
mEndPosition = mCurrentPosition;
|
|
}
|
|
mMarkPosition = mCurrentPosition;
|
|
mIncremental = false;
|
|
mUnicodeDecoder = 0;
|
|
mCharsetSource = kCharsetUninitialized;
|
|
mHasInvalidCharacter = false;
|
|
mReplacementCharacter = char16_t(0x0);
|
|
}
|
|
|
|
/**
|
|
* Use this constructor if you want i/o to be based on strings
|
|
* the scanner receives. If you pass a null filename, you
|
|
* can still provide data to the scanner via append.
|
|
*/
|
|
nsScanner::nsScanner(nsString& aFilename, bool aCreateStream)
|
|
: mFilename(aFilename)
|
|
{
|
|
MOZ_COUNT_CTOR(nsScanner);
|
|
NS_ASSERTION(!aCreateStream, "This is always true.");
|
|
|
|
mSlidingBuffer = nullptr;
|
|
|
|
// XXX This is a big hack. We need to initialize the iterators to something.
|
|
// What matters is that mCurrentPosition == mEndPosition, so that our methods
|
|
// believe that we are at EOF (see bug 182067). We null out mCurrentPosition
|
|
// so that we have some hope of catching null pointer dereferences associated
|
|
// with this hack. --darin
|
|
memset(&mCurrentPosition, 0, sizeof(mCurrentPosition));
|
|
mMarkPosition = mCurrentPosition;
|
|
mEndPosition = mCurrentPosition;
|
|
|
|
mIncremental = true;
|
|
mFirstNonWhitespacePosition = -1;
|
|
mCountRemaining = 0;
|
|
|
|
mUnicodeDecoder = 0;
|
|
mCharsetSource = kCharsetUninitialized;
|
|
mHasInvalidCharacter = false;
|
|
mReplacementCharacter = char16_t(0x0);
|
|
// XML defaults to UTF-8 and about:blank is UTF-8, too.
|
|
SetDocumentCharset(NS_LITERAL_CSTRING("UTF-8"), kCharsetFromDocTypeDefault);
|
|
}
|
|
|
|
nsresult nsScanner::SetDocumentCharset(const nsACString& aCharset , int32_t aSource)
|
|
{
|
|
if (aSource < mCharsetSource) // priority is lower than the current one
|
|
return NS_OK;
|
|
|
|
mCharsetSource = aSource;
|
|
|
|
nsCString charsetName;
|
|
mozilla::DebugOnly<bool> valid =
|
|
EncodingUtils::FindEncodingForLabel(aCharset, charsetName);
|
|
MOZ_ASSERT(valid, "Should never call with a bogus aCharset.");
|
|
|
|
if (!mCharset.IsEmpty() && charsetName.Equals(mCharset)) {
|
|
return NS_OK; // no difference, don't change it
|
|
}
|
|
|
|
// different, need to change it
|
|
|
|
mCharset.Assign(charsetName);
|
|
|
|
mUnicodeDecoder = EncodingUtils::DecoderForEncoding(mCharset);
|
|
mUnicodeDecoder->SetInputErrorBehavior(nsIUnicodeDecoder::kOnError_Signal);
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
|
|
/**
|
|
* default destructor
|
|
*
|
|
* @update gess 3/25/98
|
|
* @param
|
|
* @return
|
|
*/
|
|
nsScanner::~nsScanner() {
|
|
|
|
delete mSlidingBuffer;
|
|
|
|
MOZ_COUNT_DTOR(nsScanner);
|
|
}
|
|
|
|
/**
|
|
* Resets current offset position of input stream to marked position.
|
|
* This allows us to back up to this point if the need should arise,
|
|
* such as when tokenization gets interrupted.
|
|
* NOTE: IT IS REALLY BAD FORM TO CALL RELEASE WITHOUT CALLING MARK FIRST!
|
|
*
|
|
* @update gess 5/12/98
|
|
* @param
|
|
* @return
|
|
*/
|
|
void nsScanner::RewindToMark(void){
|
|
if (mSlidingBuffer) {
|
|
mCountRemaining += (Distance(mMarkPosition, mCurrentPosition));
|
|
mCurrentPosition = mMarkPosition;
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
* Records current offset position in input stream. This allows us
|
|
* to back up to this point if the need should arise, such as when
|
|
* tokenization gets interrupted.
|
|
*
|
|
* @update gess 7/29/98
|
|
* @param
|
|
* @return
|
|
*/
|
|
int32_t nsScanner::Mark() {
|
|
int32_t distance = 0;
|
|
if (mSlidingBuffer) {
|
|
nsScannerIterator oldStart;
|
|
mSlidingBuffer->BeginReading(oldStart);
|
|
|
|
distance = Distance(oldStart, mCurrentPosition);
|
|
|
|
mSlidingBuffer->DiscardPrefix(mCurrentPosition);
|
|
mSlidingBuffer->BeginReading(mCurrentPosition);
|
|
mMarkPosition = mCurrentPosition;
|
|
}
|
|
|
|
return distance;
|
|
}
|
|
|
|
/**
|
|
* Insert data to our underlying input buffer as
|
|
* if it were read from an input stream.
|
|
*
|
|
* @update harishd 01/12/99
|
|
* @return error code
|
|
*/
|
|
bool nsScanner::UngetReadable(const nsAString& aBuffer) {
|
|
if (!mSlidingBuffer) {
|
|
return false;
|
|
}
|
|
|
|
mSlidingBuffer->UngetReadable(aBuffer,mCurrentPosition);
|
|
mSlidingBuffer->BeginReading(mCurrentPosition); // Insertion invalidated our iterators
|
|
mSlidingBuffer->EndReading(mEndPosition);
|
|
|
|
uint32_t length = aBuffer.Length();
|
|
mCountRemaining += length; // Ref. bug 117441
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Append data to our underlying input buffer as
|
|
* if it were read from an input stream.
|
|
*
|
|
* @update gess4/3/98
|
|
* @return error code
|
|
*/
|
|
nsresult nsScanner::Append(const nsAString& aBuffer) {
|
|
if (!AppendToBuffer(aBuffer))
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
return NS_OK;
|
|
}
|
|
|
|
/**
|
|
*
|
|
*
|
|
* @update gess 5/21/98
|
|
* @param
|
|
* @return
|
|
*/
|
|
nsresult nsScanner::Append(const char* aBuffer, uint32_t aLen,
|
|
nsIRequest *aRequest)
|
|
{
|
|
nsresult res = NS_OK;
|
|
if (mUnicodeDecoder) {
|
|
int32_t unicharBufLen = 0;
|
|
|
|
nsresult rv = mUnicodeDecoder->GetMaxLength(aBuffer, aLen, &unicharBufLen);
|
|
if (NS_WARN_IF(NS_FAILED(rv))) {
|
|
return rv;
|
|
}
|
|
|
|
nsScannerString::Buffer* buffer = nsScannerString::AllocBuffer(unicharBufLen + 1);
|
|
NS_ENSURE_TRUE(buffer,NS_ERROR_OUT_OF_MEMORY);
|
|
char16_t *unichars = buffer->DataStart();
|
|
|
|
int32_t totalChars = 0;
|
|
int32_t unicharLength = unicharBufLen;
|
|
int32_t errorPos = -1;
|
|
|
|
do {
|
|
int32_t srcLength = aLen;
|
|
res = mUnicodeDecoder->Convert(aBuffer, &srcLength, unichars, &unicharLength);
|
|
|
|
totalChars += unicharLength;
|
|
// Continuation of failure case
|
|
if(NS_FAILED(res)) {
|
|
// if we failed, we consume one byte, replace it with the replacement
|
|
// character and try the conversion again.
|
|
|
|
// This is only needed because some decoders don't follow the
|
|
// nsIUnicodeDecoder contract: they return a failure when *aDestLength
|
|
// is 0 rather than the correct NS_OK_UDEC_MOREOUTPUT. See bug 244177
|
|
if ((unichars + unicharLength) >= buffer->DataEnd()) {
|
|
NS_ERROR("Unexpected end of destination buffer");
|
|
break;
|
|
}
|
|
|
|
if (mReplacementCharacter == 0x0 && errorPos == -1) {
|
|
errorPos = totalChars;
|
|
}
|
|
unichars[unicharLength++] = mReplacementCharacter == 0x0 ?
|
|
mUnicodeDecoder->GetCharacterForUnMapped() :
|
|
mReplacementCharacter;
|
|
|
|
unichars = unichars + unicharLength;
|
|
unicharLength = unicharBufLen - (++totalChars);
|
|
|
|
mUnicodeDecoder->Reset();
|
|
|
|
if(((uint32_t) (srcLength + 1)) > aLen) {
|
|
srcLength = aLen;
|
|
}
|
|
else {
|
|
++srcLength;
|
|
}
|
|
|
|
aBuffer += srcLength;
|
|
aLen -= srcLength;
|
|
}
|
|
} while (NS_FAILED(res) && (aLen > 0));
|
|
|
|
buffer->SetDataLength(totalChars);
|
|
// Don't propagate return code of unicode decoder
|
|
// since it doesn't reflect on our success or failure
|
|
// - Ref. bug 87110
|
|
res = NS_OK;
|
|
if (!AppendToBuffer(buffer, aRequest, errorPos))
|
|
res = NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
else {
|
|
NS_WARNING("No decoder found.");
|
|
res = NS_ERROR_FAILURE;
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
/**
|
|
* retrieve next char from scanners internal input stream
|
|
*
|
|
* @update gess 3/25/98
|
|
* @param
|
|
* @return error code reflecting read status
|
|
*/
|
|
nsresult nsScanner::GetChar(char16_t& aChar) {
|
|
if (!mSlidingBuffer || mCurrentPosition == mEndPosition) {
|
|
aChar = 0;
|
|
return kEOF;
|
|
}
|
|
|
|
aChar = *mCurrentPosition++;
|
|
--mCountRemaining;
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
|
|
/**
|
|
* peek ahead to consume next char from scanner's internal
|
|
* input buffer
|
|
*
|
|
* @update gess 3/25/98
|
|
* @param
|
|
* @return
|
|
*/
|
|
nsresult nsScanner::Peek(char16_t& aChar, uint32_t aOffset) {
|
|
aChar = 0;
|
|
|
|
if (!mSlidingBuffer || mCurrentPosition == mEndPosition) {
|
|
return kEOF;
|
|
}
|
|
|
|
if (aOffset > 0) {
|
|
if (mCountRemaining <= aOffset)
|
|
return kEOF;
|
|
|
|
nsScannerIterator pos = mCurrentPosition;
|
|
pos.advance(aOffset);
|
|
aChar=*pos;
|
|
}
|
|
else {
|
|
aChar=*mCurrentPosition;
|
|
}
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
nsresult nsScanner::Peek(nsAString& aStr, int32_t aNumChars, int32_t aOffset)
|
|
{
|
|
if (!mSlidingBuffer || mCurrentPosition == mEndPosition) {
|
|
return kEOF;
|
|
}
|
|
|
|
nsScannerIterator start, end;
|
|
|
|
start = mCurrentPosition;
|
|
|
|
if ((int32_t)mCountRemaining <= aOffset) {
|
|
return kEOF;
|
|
}
|
|
|
|
if (aOffset > 0) {
|
|
start.advance(aOffset);
|
|
}
|
|
|
|
if (mCountRemaining < uint32_t(aNumChars + aOffset)) {
|
|
end = mEndPosition;
|
|
}
|
|
else {
|
|
end = start;
|
|
end.advance(aNumChars);
|
|
}
|
|
|
|
if (!CopyUnicodeTo(start, end, aStr)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
|
|
return NS_OK;
|
|
}
|
|
|
|
|
|
/**
|
|
* Skip whitespace on scanner input stream
|
|
*
|
|
* @update gess 3/25/98
|
|
* @param
|
|
* @return error status
|
|
*/
|
|
nsresult nsScanner::SkipWhitespace(int32_t& aNewlinesSkipped) {
|
|
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
char16_t theChar = 0;
|
|
nsresult result = Peek(theChar);
|
|
|
|
if (NS_FAILED(result)) {
|
|
return result;
|
|
}
|
|
|
|
nsScannerIterator current = mCurrentPosition;
|
|
bool done = false;
|
|
bool skipped = false;
|
|
|
|
while (!done && current != mEndPosition) {
|
|
switch(theChar) {
|
|
case '\n':
|
|
case '\r': ++aNewlinesSkipped;
|
|
case ' ' :
|
|
case '\t':
|
|
{
|
|
skipped = true;
|
|
char16_t thePrevChar = theChar;
|
|
theChar = (++current != mEndPosition) ? *current : '\0';
|
|
if ((thePrevChar == '\r' && theChar == '\n') ||
|
|
(thePrevChar == '\n' && theChar == '\r')) {
|
|
theChar = (++current != mEndPosition) ? *current : '\0'; // CRLF == LFCR => LF
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
done = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (skipped) {
|
|
SetPosition(current);
|
|
if (current == mEndPosition) {
|
|
result = kEOF;
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Skip over chars as long as they equal given char
|
|
*
|
|
* @update gess 3/25/98
|
|
* @param
|
|
* @return error code
|
|
*/
|
|
nsresult nsScanner::SkipOver(char16_t aSkipChar){
|
|
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
char16_t ch=0;
|
|
nsresult result=NS_OK;
|
|
|
|
while(NS_OK==result) {
|
|
result=Peek(ch);
|
|
if(NS_OK == result) {
|
|
if(ch!=aSkipChar) {
|
|
break;
|
|
}
|
|
GetChar(ch);
|
|
}
|
|
else break;
|
|
} //while
|
|
return result;
|
|
|
|
}
|
|
|
|
#if 0
|
|
void DoErrTest(nsString& aString) {
|
|
int32_t pos=aString.FindChar(0);
|
|
if(kNotFound<pos) {
|
|
if(aString.Length()-1!=pos) {
|
|
}
|
|
}
|
|
}
|
|
|
|
void DoErrTest(nsCString& aString) {
|
|
int32_t pos=aString.FindChar(0);
|
|
if(kNotFound<pos) {
|
|
if(aString.Length()-1!=pos) {
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
|
|
/**
|
|
* Consume characters until you run into space, a '<', a '>', or a '/'.
|
|
*
|
|
* @param aString - receives new data from stream
|
|
* @return error code
|
|
*/
|
|
nsresult nsScanner::ReadTagIdentifier(nsScannerSharedSubstring& aString) {
|
|
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
char16_t theChar=0;
|
|
nsresult result=Peek(theChar);
|
|
nsScannerIterator current, end;
|
|
bool found=false;
|
|
|
|
current = mCurrentPosition;
|
|
end = mEndPosition;
|
|
|
|
// Loop until we find an illegal character. Everything is then appended
|
|
// later.
|
|
while(current != end && !found) {
|
|
theChar=*current;
|
|
|
|
switch(theChar) {
|
|
case '\n':
|
|
case '\r':
|
|
case ' ' :
|
|
case '\t':
|
|
case '\v':
|
|
case '\f':
|
|
case '<':
|
|
case '>':
|
|
case '/':
|
|
found = true;
|
|
break;
|
|
|
|
case '\0':
|
|
ReplaceCharacter(current, sInvalid);
|
|
break;
|
|
|
|
default:
|
|
break;
|
|
}
|
|
|
|
if (!found) {
|
|
++current;
|
|
}
|
|
}
|
|
|
|
// Don't bother appending nothing.
|
|
if (current != mCurrentPosition) {
|
|
if (!AppendUnicodeTo(mCurrentPosition, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
}
|
|
|
|
SetPosition(current);
|
|
if (current == end) {
|
|
result = kEOF;
|
|
}
|
|
|
|
//DoErrTest(aString);
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Consume characters until you run into a char that's not valid in an
|
|
* entity name
|
|
*
|
|
* @param aString - receives new data from stream
|
|
* @return error code
|
|
*/
|
|
nsresult nsScanner::ReadEntityIdentifier(nsString& aString) {
|
|
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
char16_t theChar=0;
|
|
nsresult result=Peek(theChar);
|
|
nsScannerIterator origin, current, end;
|
|
bool found=false;
|
|
|
|
origin = mCurrentPosition;
|
|
current = mCurrentPosition;
|
|
end = mEndPosition;
|
|
|
|
while(current != end) {
|
|
|
|
theChar=*current;
|
|
if(theChar) {
|
|
found=false;
|
|
switch(theChar) {
|
|
case '_':
|
|
case '-':
|
|
case '.':
|
|
// Don't allow ':' in entity names. See bug 23791
|
|
found = true;
|
|
break;
|
|
default:
|
|
found = ('a'<=theChar && theChar<='z') ||
|
|
('A'<=theChar && theChar<='Z') ||
|
|
('0'<=theChar && theChar<='9');
|
|
break;
|
|
}
|
|
|
|
if(!found) {
|
|
if (!AppendUnicodeTo(mCurrentPosition, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
++current;
|
|
}
|
|
|
|
SetPosition(current);
|
|
if (current == end) {
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
return kEOF;
|
|
}
|
|
|
|
//DoErrTest(aString);
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Consume digits
|
|
*
|
|
* @param aString - should contain digits
|
|
* @return error code
|
|
*/
|
|
nsresult nsScanner::ReadNumber(nsString& aString,int32_t aBase) {
|
|
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
NS_ASSERTION(aBase == 10 || aBase == 16,"base value not supported");
|
|
|
|
char16_t theChar=0;
|
|
nsresult result=Peek(theChar);
|
|
nsScannerIterator origin, current, end;
|
|
|
|
origin = mCurrentPosition;
|
|
current = origin;
|
|
end = mEndPosition;
|
|
|
|
bool done = false;
|
|
while(current != end) {
|
|
theChar=*current;
|
|
if(theChar) {
|
|
done = (theChar < '0' || theChar > '9') &&
|
|
((aBase == 16)? (theChar < 'A' || theChar > 'F') &&
|
|
(theChar < 'a' || theChar > 'f')
|
|
:true);
|
|
if(done) {
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
++current;
|
|
}
|
|
|
|
SetPosition(current);
|
|
if (current == end) {
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
return kEOF;
|
|
}
|
|
|
|
//DoErrTest(aString);
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Consume characters until you find the terminal char
|
|
*
|
|
* @update gess 3/25/98
|
|
* @param aString receives new data from stream
|
|
* @param addTerminal tells us whether to append terminal to aString
|
|
* @return error code
|
|
*/
|
|
nsresult nsScanner::ReadWhitespace(nsScannerSharedSubstring& aString,
|
|
int32_t& aNewlinesSkipped,
|
|
bool& aHaveCR) {
|
|
|
|
aHaveCR = false;
|
|
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
char16_t theChar = 0;
|
|
nsresult result = Peek(theChar);
|
|
|
|
if (NS_FAILED(result)) {
|
|
return result;
|
|
}
|
|
|
|
nsScannerIterator origin, current, end;
|
|
bool done = false;
|
|
|
|
origin = mCurrentPosition;
|
|
current = origin;
|
|
end = mEndPosition;
|
|
|
|
bool haveCR = false;
|
|
|
|
while(!done && current != end) {
|
|
switch(theChar) {
|
|
case '\n':
|
|
case '\r':
|
|
{
|
|
++aNewlinesSkipped;
|
|
char16_t thePrevChar = theChar;
|
|
theChar = (++current != end) ? *current : '\0';
|
|
if ((thePrevChar == '\r' && theChar == '\n') ||
|
|
(thePrevChar == '\n' && theChar == '\r')) {
|
|
theChar = (++current != end) ? *current : '\0'; // CRLF == LFCR => LF
|
|
haveCR = true;
|
|
} else if (thePrevChar == '\r') {
|
|
// Lone CR becomes CRLF; callers should know to remove extra CRs
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
aString.writable().Append(char16_t('\n'));
|
|
origin = current;
|
|
haveCR = true;
|
|
}
|
|
}
|
|
break;
|
|
case ' ' :
|
|
case '\t':
|
|
theChar = (++current != end) ? *current : '\0';
|
|
break;
|
|
default:
|
|
done = true;
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
SetPosition(current);
|
|
if (current == end) {
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
result = kEOF;
|
|
}
|
|
|
|
aHaveCR = haveCR;
|
|
return result;
|
|
}
|
|
|
|
//XXXbz callers of this have to manage their lone '\r' themselves if they want
|
|
//it to work. Good thing they're all in view-source and it deals.
|
|
nsresult nsScanner::ReadWhitespace(nsScannerIterator& aStart,
|
|
nsScannerIterator& aEnd,
|
|
int32_t& aNewlinesSkipped) {
|
|
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
char16_t theChar = 0;
|
|
nsresult result = Peek(theChar);
|
|
|
|
if (NS_FAILED(result)) {
|
|
return result;
|
|
}
|
|
|
|
nsScannerIterator origin, current, end;
|
|
bool done = false;
|
|
|
|
origin = mCurrentPosition;
|
|
current = origin;
|
|
end = mEndPosition;
|
|
|
|
while(!done && current != end) {
|
|
switch(theChar) {
|
|
case '\n':
|
|
case '\r': ++aNewlinesSkipped;
|
|
case ' ' :
|
|
case '\t':
|
|
{
|
|
char16_t thePrevChar = theChar;
|
|
theChar = (++current != end) ? *current : '\0';
|
|
if ((thePrevChar == '\r' && theChar == '\n') ||
|
|
(thePrevChar == '\n' && theChar == '\r')) {
|
|
theChar = (++current != end) ? *current : '\0'; // CRLF == LFCR => LF
|
|
}
|
|
}
|
|
break;
|
|
default:
|
|
done = true;
|
|
aStart = origin;
|
|
aEnd = current;
|
|
break;
|
|
}
|
|
}
|
|
|
|
SetPosition(current);
|
|
if (current == end) {
|
|
aStart = origin;
|
|
aEnd = current;
|
|
result = kEOF;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Consume characters until you encounter one contained in given
|
|
* input set.
|
|
*
|
|
* @update gess 3/25/98
|
|
* @param aString will contain the result of this method
|
|
* @param aTerminalSet is an ordered string that contains
|
|
* the set of INVALID characters
|
|
* @return error code
|
|
*/
|
|
nsresult nsScanner::ReadUntil(nsAString& aString,
|
|
const nsReadEndCondition& aEndCondition,
|
|
bool addTerminal)
|
|
{
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
nsScannerIterator origin, current;
|
|
const char16_t* setstart = aEndCondition.mChars;
|
|
const char16_t* setcurrent;
|
|
|
|
origin = mCurrentPosition;
|
|
current = origin;
|
|
|
|
char16_t theChar=0;
|
|
nsresult result=Peek(theChar);
|
|
|
|
if (NS_FAILED(result)) {
|
|
return result;
|
|
}
|
|
|
|
while (current != mEndPosition) {
|
|
theChar = *current;
|
|
if (theChar == '\0') {
|
|
ReplaceCharacter(current, sInvalid);
|
|
theChar = sInvalid;
|
|
}
|
|
|
|
// Filter out completely wrong characters
|
|
// Check if all bits are in the required area
|
|
if(!(theChar & aEndCondition.mFilter)) {
|
|
// They were. Do a thorough check.
|
|
|
|
setcurrent = setstart;
|
|
while (*setcurrent) {
|
|
if (*setcurrent == theChar) {
|
|
if(addTerminal)
|
|
++current;
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
SetPosition(current);
|
|
|
|
//DoErrTest(aString);
|
|
|
|
return NS_OK;
|
|
}
|
|
++setcurrent;
|
|
}
|
|
}
|
|
|
|
++current;
|
|
}
|
|
|
|
// If we are here, we didn't find any terminator in the string and
|
|
// current = mEndPosition
|
|
SetPosition(current);
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
return kEOF;
|
|
}
|
|
|
|
nsresult nsScanner::ReadUntil(nsScannerSharedSubstring& aString,
|
|
const nsReadEndCondition& aEndCondition,
|
|
bool addTerminal)
|
|
{
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
nsScannerIterator origin, current;
|
|
const char16_t* setstart = aEndCondition.mChars;
|
|
const char16_t* setcurrent;
|
|
|
|
origin = mCurrentPosition;
|
|
current = origin;
|
|
|
|
char16_t theChar=0;
|
|
nsresult result=Peek(theChar);
|
|
|
|
if (NS_FAILED(result)) {
|
|
return result;
|
|
}
|
|
|
|
while (current != mEndPosition) {
|
|
theChar = *current;
|
|
if (theChar == '\0') {
|
|
ReplaceCharacter(current, sInvalid);
|
|
theChar = sInvalid;
|
|
}
|
|
|
|
// Filter out completely wrong characters
|
|
// Check if all bits are in the required area
|
|
if(!(theChar & aEndCondition.mFilter)) {
|
|
// They were. Do a thorough check.
|
|
|
|
setcurrent = setstart;
|
|
while (*setcurrent) {
|
|
if (*setcurrent == theChar) {
|
|
if(addTerminal)
|
|
++current;
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
SetPosition(current);
|
|
|
|
//DoErrTest(aString);
|
|
|
|
return NS_OK;
|
|
}
|
|
++setcurrent;
|
|
}
|
|
}
|
|
|
|
++current;
|
|
}
|
|
|
|
// If we are here, we didn't find any terminator in the string and
|
|
// current = mEndPosition
|
|
SetPosition(current);
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
return kEOF;
|
|
}
|
|
|
|
nsresult nsScanner::ReadUntil(nsScannerIterator& aStart,
|
|
nsScannerIterator& aEnd,
|
|
const nsReadEndCondition &aEndCondition,
|
|
bool addTerminal)
|
|
{
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
nsScannerIterator origin, current;
|
|
const char16_t* setstart = aEndCondition.mChars;
|
|
const char16_t* setcurrent;
|
|
|
|
origin = mCurrentPosition;
|
|
current = origin;
|
|
|
|
char16_t theChar=0;
|
|
nsresult result=Peek(theChar);
|
|
|
|
if (NS_FAILED(result)) {
|
|
aStart = aEnd = current;
|
|
return result;
|
|
}
|
|
|
|
while (current != mEndPosition) {
|
|
theChar = *current;
|
|
if (theChar == '\0') {
|
|
ReplaceCharacter(current, sInvalid);
|
|
theChar = sInvalid;
|
|
}
|
|
|
|
// Filter out completely wrong characters
|
|
// Check if all bits are in the required area
|
|
if(!(theChar & aEndCondition.mFilter)) {
|
|
// They were. Do a thorough check.
|
|
setcurrent = setstart;
|
|
while (*setcurrent) {
|
|
if (*setcurrent == theChar) {
|
|
if(addTerminal)
|
|
++current;
|
|
aStart = origin;
|
|
aEnd = current;
|
|
SetPosition(current);
|
|
|
|
return NS_OK;
|
|
}
|
|
++setcurrent;
|
|
}
|
|
}
|
|
|
|
++current;
|
|
}
|
|
|
|
// If we are here, we didn't find any terminator in the string and
|
|
// current = mEndPosition
|
|
SetPosition(current);
|
|
aStart = origin;
|
|
aEnd = current;
|
|
return kEOF;
|
|
}
|
|
|
|
/**
|
|
* Consumes chars until you see the given terminalChar
|
|
*
|
|
* @update gess 3/25/98
|
|
* @param
|
|
* @return error code
|
|
*/
|
|
nsresult nsScanner::ReadUntil(nsAString& aString,
|
|
char16_t aTerminalChar,
|
|
bool addTerminal)
|
|
{
|
|
if (!mSlidingBuffer) {
|
|
return kEOF;
|
|
}
|
|
|
|
nsScannerIterator origin, current;
|
|
|
|
origin = mCurrentPosition;
|
|
current = origin;
|
|
|
|
char16_t theChar;
|
|
nsresult result = Peek(theChar);
|
|
|
|
if (NS_FAILED(result)) {
|
|
return result;
|
|
}
|
|
|
|
while (current != mEndPosition) {
|
|
theChar = *current;
|
|
if (theChar == '\0') {
|
|
ReplaceCharacter(current, sInvalid);
|
|
theChar = sInvalid;
|
|
}
|
|
|
|
if (aTerminalChar == theChar) {
|
|
if(addTerminal)
|
|
++current;
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
SetPosition(current);
|
|
return NS_OK;
|
|
}
|
|
++current;
|
|
}
|
|
|
|
// If we are here, we didn't find any terminator in the string and
|
|
// current = mEndPosition
|
|
if (!AppendUnicodeTo(origin, current, aString)) {
|
|
return NS_ERROR_OUT_OF_MEMORY;
|
|
}
|
|
SetPosition(current);
|
|
return kEOF;
|
|
|
|
}
|
|
|
|
void nsScanner::BindSubstring(nsScannerSubstring& aSubstring, const nsScannerIterator& aStart, const nsScannerIterator& aEnd)
|
|
{
|
|
aSubstring.Rebind(*mSlidingBuffer, aStart, aEnd);
|
|
}
|
|
|
|
void nsScanner::CurrentPosition(nsScannerIterator& aPosition)
|
|
{
|
|
aPosition = mCurrentPosition;
|
|
}
|
|
|
|
void nsScanner::EndReading(nsScannerIterator& aPosition)
|
|
{
|
|
aPosition = mEndPosition;
|
|
}
|
|
|
|
void nsScanner::SetPosition(nsScannerIterator& aPosition, bool aTerminate, bool aReverse)
|
|
{
|
|
if (mSlidingBuffer) {
|
|
#ifdef DEBUG
|
|
uint32_t origRemaining = mCountRemaining;
|
|
#endif
|
|
|
|
if (aReverse) {
|
|
mCountRemaining += (Distance(aPosition, mCurrentPosition));
|
|
}
|
|
else {
|
|
mCountRemaining -= (Distance(mCurrentPosition, aPosition));
|
|
}
|
|
|
|
NS_ASSERTION((mCountRemaining >= origRemaining && aReverse) ||
|
|
(mCountRemaining <= origRemaining && !aReverse),
|
|
"Improper use of nsScanner::SetPosition. Make sure to set the"
|
|
" aReverse parameter correctly");
|
|
|
|
mCurrentPosition = aPosition;
|
|
if (aTerminate && (mCurrentPosition == mEndPosition)) {
|
|
mMarkPosition = mCurrentPosition;
|
|
mSlidingBuffer->DiscardPrefix(mCurrentPosition);
|
|
}
|
|
}
|
|
}
|
|
|
|
void nsScanner::ReplaceCharacter(nsScannerIterator& aPosition,
|
|
char16_t aChar)
|
|
{
|
|
if (mSlidingBuffer) {
|
|
mSlidingBuffer->ReplaceCharacter(aPosition, aChar);
|
|
}
|
|
}
|
|
|
|
bool nsScanner::AppendToBuffer(nsScannerString::Buffer* aBuf,
|
|
nsIRequest *aRequest,
|
|
int32_t aErrorPos)
|
|
{
|
|
uint32_t countRemaining = mCountRemaining;
|
|
if (!mSlidingBuffer) {
|
|
mSlidingBuffer = new nsScannerString(aBuf);
|
|
if (!mSlidingBuffer)
|
|
return false;
|
|
mSlidingBuffer->BeginReading(mCurrentPosition);
|
|
mMarkPosition = mCurrentPosition;
|
|
mSlidingBuffer->EndReading(mEndPosition);
|
|
mCountRemaining = aBuf->DataLength();
|
|
}
|
|
else {
|
|
mSlidingBuffer->AppendBuffer(aBuf);
|
|
if (mCurrentPosition == mEndPosition) {
|
|
mSlidingBuffer->BeginReading(mCurrentPosition);
|
|
}
|
|
mSlidingBuffer->EndReading(mEndPosition);
|
|
mCountRemaining += aBuf->DataLength();
|
|
}
|
|
|
|
if (aErrorPos != -1 && !mHasInvalidCharacter) {
|
|
mHasInvalidCharacter = true;
|
|
mFirstInvalidPosition = mCurrentPosition;
|
|
mFirstInvalidPosition.advance(countRemaining + aErrorPos);
|
|
}
|
|
|
|
if (mFirstNonWhitespacePosition == -1) {
|
|
nsScannerIterator iter(mCurrentPosition);
|
|
nsScannerIterator end(mEndPosition);
|
|
|
|
while (iter != end) {
|
|
if (!nsCRT::IsAsciiSpace(*iter)) {
|
|
mFirstNonWhitespacePosition = Distance(mCurrentPosition, iter);
|
|
|
|
break;
|
|
}
|
|
|
|
++iter;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* call this to copy bytes out of the scanner that have not yet been consumed
|
|
* by the tokenization process.
|
|
*
|
|
* @update gess 5/12/98
|
|
* @param aCopyBuffer is where the scanner buffer will be copied to
|
|
* @return true if OK or false on OOM
|
|
*/
|
|
bool nsScanner::CopyUnusedData(nsString& aCopyBuffer) {
|
|
if (!mSlidingBuffer) {
|
|
aCopyBuffer.Truncate();
|
|
return true;
|
|
}
|
|
|
|
nsScannerIterator start, end;
|
|
start = mCurrentPosition;
|
|
end = mEndPosition;
|
|
|
|
return CopyUnicodeTo(start, end, aCopyBuffer);
|
|
}
|
|
|
|
/**
|
|
* Retrieve the name of the file that the scanner is reading from.
|
|
* In some cases, it's just a given name, because the scanner isn't
|
|
* really reading from a file.
|
|
*
|
|
* @update gess 5/12/98
|
|
* @return
|
|
*/
|
|
nsString& nsScanner::GetFilename(void) {
|
|
return mFilename;
|
|
}
|
|
|
|
/**
|
|
* Conduct self test. Actually, selftesting for this class
|
|
* occurs in the parser selftest.
|
|
*
|
|
* @update gess 3/25/98
|
|
* @param
|
|
* @return
|
|
*/
|
|
|
|
void nsScanner::SelfTest(void) {
|
|
#ifdef _DEBUG
|
|
#endif
|
|
}
|
|
|
|
void nsScanner::OverrideReplacementCharacter(char16_t aReplacementCharacter)
|
|
{
|
|
mReplacementCharacter = aReplacementCharacter;
|
|
|
|
if (mHasInvalidCharacter) {
|
|
ReplaceCharacter(mFirstInvalidPosition, mReplacementCharacter);
|
|
}
|
|
}
|
|
|