diff -ur icu.org/source/common/utext.cpp icu/source/common/utext.cpp --- icu.org/source/common/utext.cpp 2016-06-15 20:58:17.000000000 +0200 +++ icu/source/common/utext.cpp 2017-04-21 16:38:15.993398034 +0200 @@ -847,9 +847,15 @@ //------------------------------------------------------------------------------ // Chunk size. -// Must be less than 85, because of byte mapping from UChar indexes to native indexes. -// Worst case is three native bytes to one UChar. (Supplemenaries are 4 native bytes -// to two UChars.) +// Must be less than 42 (256/6), because of byte mapping from UChar indexes to native indexes. +// Worst case there are six UTF-8 bytes per UChar. +// obsolete 6 byte form fd + 5 trails maps to fffd +// obsolete 5 byte form fc + 4 trails maps to fffd +// non-shortest 4 byte forms maps to fffd +// normal supplementaries map to a pair of utf-16, two utf8 bytes per utf-16 unit +// mapToUChars array size must allow for the worst case, 6. +// This could be brought down to 4, by treating fd and fc as pure illegal, +// rather than obsolete lead bytes. But that is not compatible with the utf-8 access macros. // enum { UTF8_TEXT_CHUNK_SIZE=32 }; @@ -867,6 +873,15 @@ // pair. Doing this is simpler than checking for the edge case. // +// erAck: older MSVC used on libreoffice-5-3 and 5-2 bails out with +// error C2070: 'unknown': illegal sizeof operand +// for sizeof(UTF8Buf::mapToUChars) +// so have an ugly workaround: +// First define a macro of the original size expression, so a follow-up patch +// on the original code would fail.. +#define UGLY_MAPTOUCHARS_SIZE (UTF8_TEXT_CHUNK_SIZE*6+6) +#define UGLY_SIZEOF_MAPTOUCHARS (sizeof(uint8_t)*(UGLY_MAPTOUCHARS_SIZE)) + struct UTF8Buf { int32_t bufNativeStart; // Native index of first char in UChar buf int32_t bufNativeLimit; // Native index following last char in buf. @@ -889,7 +904,7 @@ // Requires two extra slots, // one for a supplementary starting in the last normal position, // and one for an entry for the buffer limit position. - uint8_t mapToUChars[UTF8_TEXT_CHUNK_SIZE*3+6]; // Map native offset from bufNativeStart to + uint8_t mapToUChars[UGLY_MAPTOUCHARS_SIZE]; // Map native offset from bufNativeStart to // correspoding offset in filled part of buf. int32_t align; }; @@ -1032,6 +1047,7 @@ // Requested index is in this buffer. u8b = (UTF8Buf *)ut->p; // the current buffer mapIndex = ix - u8b->toUCharsMapStart; + U_ASSERT(mapIndex < (int32_t)UGLY_SIZEOF_MAPTOUCHARS); ut->chunkOffset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; return TRUE; @@ -1298,6 +1314,10 @@ // Can only do this if the incoming index is somewhere in the interior of the string. // If index is at the end, there is no character there to look at. if (ix != ut->b) { + // Note: this function will only move the index back if it is on a trail byte + // and there is a preceding lead byte and the sequence from the lead + // through this trail could be part of a valid UTF-8 sequence + // Otherwise the index remains unchanged. U8_SET_CP_START(s8, 0, ix); } @@ -1311,7 +1331,10 @@ UChar *buf = u8b->buf; uint8_t *mapToNative = u8b->mapToNative; uint8_t *mapToUChars = u8b->mapToUChars; - int32_t toUCharsMapStart = ix - (UTF8_TEXT_CHUNK_SIZE*3 + 1); + int32_t toUCharsMapStart = ix - UGLY_SIZEOF_MAPTOUCHARS + 1; + // Note that toUCharsMapStart can be negative. Happens when the remaining + // text from current position to the beginning is less than the buffer size. + // + 1 because mapToUChars must have a slot at the end for the bufNativeLimit entry. int32_t destIx = UTF8_TEXT_CHUNK_SIZE+2; // Start in the overflow region // at end of buffer to leave room // for a surrogate pair at the @@ -1338,6 +1361,7 @@ if (c<0x80) { // Special case ASCII range for speed. buf[destIx] = (UChar)c; + U_ASSERT(toUCharsMapStart <= srcIx); mapToUChars[srcIx - toUCharsMapStart] = (uint8_t)destIx; mapToNative[destIx] = (uint8_t)(srcIx - toUCharsMapStart); } else { @@ -1367,6 +1391,7 @@ do { mapToUChars[sIx-- - toUCharsMapStart] = (uint8_t)destIx; } while (sIx >= srcIx); + U_ASSERT(toUCharsMapStart <= (srcIx+1)); // Set native indexing limit to be the current position. // We are processing a non-ascii, non-native-indexing char now; @@ -1541,6 +1566,7 @@ U_ASSERT(index>=ut->chunkNativeStart+ut->nativeIndexingLimit); U_ASSERT(index<=ut->chunkNativeLimit); int32_t mapIndex = index - u8b->toUCharsMapStart; + U_ASSERT(mapIndex < (int32_t)UGLY_SIZEOF_MAPTOUCHARS); int32_t offset = u8b->mapToUChars[mapIndex] - u8b->bufStartIdx; U_ASSERT(offset>=0 && offset<=ut->chunkLength); return offset; diff -ur icu.org/source/test/intltest/utxttest.cpp icu/source/test/intltest/utxttest.cpp --- icu.org/source/test/intltest/utxttest.cpp 2016-06-15 20:58:17.000000000 +0200 +++ icu/source/test/intltest/utxttest.cpp 2017-04-21 16:14:57.383814739 +0200 @@ -67,6 +67,8 @@ if (exec) Ticket10983(); break; case 7: name = "Ticket12130"; if (exec) Ticket12130(); break; + case 8: name = "Ticket12888"; + if (exec) Ticket12888(); break; default: name = ""; break; } } @@ -1583,3 +1585,63 @@ } utext_close(&ut); } + +// Ticket 12888: bad handling of illegal utf-8 containing many instances of the archaic, now illegal, +// six byte utf-8 forms. Original implementation had an assumption that +// there would be at most three utf-8 bytes per UTF-16 code unit. +// The five and six byte sequences map to a single replacement character. + +void UTextTest::Ticket12888() { + const char *badString = + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80" + "\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80\xfd\x80\x80\x80\x80\x80"; + + UErrorCode status = U_ZERO_ERROR; + LocalUTextPointer ut(utext_openUTF8(NULL, badString, -1, &status)); + TEST_SUCCESS(status); + for (;;) { + UChar32 c = utext_next32(ut.getAlias()); + if (c == U_SENTINEL) { + break; + } + } + int32_t endIdx = utext_getNativeIndex(ut.getAlias()); + if (endIdx != (int32_t)strlen(badString)) { + errln("%s:%d expected=%d, actual=%d", __FILE__, __LINE__, strlen(badString), endIdx); + return; + } + + for (int32_t prevIndex = endIdx; prevIndex>0;) { + UChar32 c = utext_previous32(ut.getAlias()); + int32_t currentIndex = utext_getNativeIndex(ut.getAlias()); + if (c != 0xfffd) { + errln("%s:%d (expected, actual, index) = (%d, %d, %d)\n", + __FILE__, __LINE__, 0xfffd, c, currentIndex); + break; + } + if (currentIndex != prevIndex - 6) { + errln("%s:%d: wrong index. Expected, actual = %d, %d", + __FILE__, __LINE__, prevIndex - 6, currentIndex); + break; + } + prevIndex = currentIndex; + } +} diff -ur icu.org/source/test/intltest/utxttest.h icu/source/test/intltest/utxttest.h --- icu.org/source/test/intltest/utxttest.h 2016-06-15 20:58:17.000000000 +0200 +++ icu/source/test/intltest/utxttest.h 2017-04-21 16:14:57.383814739 +0200 @@ -38,6 +38,7 @@ void Ticket10562(); void Ticket10983(); void Ticket12130(); + void Ticket12888(); private: struct m { // Map between native indices & code points.