diff options
-rw-r--r-- | sc/inc/stringutil.hxx | 3 | ||||
-rw-r--r-- | sc/source/core/tool/stringutil.cxx | 150 | ||||
-rw-r--r-- | sc/source/ui/docshell/datastream.cxx | 370 | ||||
-rw-r--r-- | sc/source/ui/inc/datastream.hxx | 44 | ||||
-rw-r--r-- | sc/source/ui/miscdlgs/datastreamdlg.cxx | 5 |
5 files changed, 399 insertions, 173 deletions
diff --git a/sc/inc/stringutil.hxx b/sc/inc/stringutil.hxx index 08e5c1ed0314..d27500763033 100644 --- a/sc/inc/stringutil.hxx +++ b/sc/inc/stringutil.hxx @@ -126,6 +126,9 @@ public: static bool parseSimpleNumber( const OUString& rStr, sal_Unicode dsep, sal_Unicode gsep, double& rVal); + static bool parseSimpleNumber( + const char* p, size_t n, char dsep, char gsep, double& rVal); + static sal_Int32 SC_DLLPUBLIC GetQuotedTokenCount(const OUString &rIn, const OUString& rQuotedPairs, sal_Unicode cTok = ';' ); static OUString SC_DLLPUBLIC GetQuotedToken(const OUString &rIn, sal_Int32 nToken, const OUString& rQuotedPairs, sal_Unicode cTok, sal_Int32& rIndex ); diff --git a/sc/source/core/tool/stringutil.cxx b/sc/source/core/tool/stringutil.cxx index e711bcba40d4..5bdc2c2fe752 100644 --- a/sc/source/core/tool/stringutil.cxx +++ b/sc/source/core/tool/stringutil.cxx @@ -18,11 +18,13 @@ */ #include "stringutil.hxx" -#include "rtl/ustrbuf.hxx" -#include "rtl/math.hxx" #include "global.hxx" #include "svl/zforlist.hxx" +#include <rtl/ustrbuf.hxx> +#include <rtl/strbuf.hxx> +#include <rtl/math.hxx> + ScSetStringParam::ScSetStringParam() : mpNumFormatter(NULL), mbDetectNumberFormat(true), @@ -194,6 +196,150 @@ bool ScStringUtil::parseSimpleNumber( return true; } +bool ScStringUtil::parseSimpleNumber( + const char* p, size_t n, char dsep, char gsep, double& rVal) +{ + // Actually almost the entire pre-check is unnecessary and we could call + // rtl::math::stringToDouble() just after having exchanged ascii space with + // non-breaking space, if it wasn't for check of grouped digits. The NaN + // and Inf cases that are accepted by stringToDouble() could be detected + // using rtl::math::isFinite() on the result. + + /* TODO: The grouped digits check isn't even valid for locales that do not + * group in thousands ... e.g. Indian locales. But that's something also + * the number scanner doesn't implement yet, only the formatter. */ + + OStringBuffer aBuf; + + size_t i = 0; + const char* pLast = p + (n-1); + sal_Int32 nPosDSep = -1, nPosGSep = -1; + sal_uInt32 nDigitCount = 0; + sal_Int32 nPosExponent = -1; + + // Skip preceding spaces. + for (i = 0; i < n; ++i, ++p) + { + char c = *p; + if (c != ' ') + // first non-space character. Exit. + break; + } + + if (i == n) + // the whole string is space. Fail. + return false; + + n -= i; // Subtract the length of the preceding spaces. + + // Determine the last non-space character. + for (; p != pLast; --pLast, --n) + { + char c = *pLast; + if (c != ' ') + // Non space character. Exit. + break; + } + + for (i = 0; i < n; ++i, ++p) + { + char c = *p; + + if ('0' <= c && c <= '9') + { + // this is a digit. + aBuf.append(c); + ++nDigitCount; + } + else if (c == dsep) + { + // this is a decimal separator. + + if (nPosDSep >= 0) + // a second decimal separator -> not a valid number. + return false; + + if (nPosGSep >= 0 && i - nPosGSep != 4) + // the number has a group separator and the decimal sep is not + // positioned correctly. + return false; + + nPosDSep = i; + nPosGSep = -1; + aBuf.append(c); + nDigitCount = 0; + } + else if (c == gsep) + { + // this is a group (thousand) separator. + + if (i == 0) + // not allowed as the first character. + return false; + + if (nPosDSep >= 0) + // not allowed after the decimal separator. + return false; + + if (nPosGSep >= 0 && nDigitCount != 3) + // must be exactly 3 digits since the last group separator. + return false; + + if (nPosExponent >= 0) + // not allowed in exponent. + return false; + + nPosGSep = i; + nDigitCount = 0; + } + else if (c == '-' || c == '+') + { + // A sign must be the first character if it's given, or immediately + // follow the exponent character if present. + if (i == 0 || (nPosExponent >= 0 && i == static_cast<size_t>(nPosExponent+1))) + aBuf.append(c); + else + return false; + } + else if (c == 'E' || c == 'e') + { + // this is an exponent designator. + + if (nPosExponent >= 0) + // Only one exponent allowed. + return false; + + if (nPosGSep >= 0 && nDigitCount != 3) + // must be exactly 3 digits since the last group separator. + return false; + + aBuf.append(c); + nPosExponent = i; + nPosDSep = -1; + nPosGSep = -1; + nDigitCount = 0; + } + else + return false; + } + + // finished parsing the number. + + if (nPosGSep >= 0 && nDigitCount != 3) + // must be exactly 3 digits since the last group separator. + return false; + + rtl_math_ConversionStatus eStatus = rtl_math_ConversionStatus_Ok; + sal_Int32 nParseEnd = 0; + OString aString( aBuf.makeStringAndClear()); + rVal = ::rtl::math::stringToDouble( aString, dsep, gsep, &eStatus, &nParseEnd); + if (eStatus != rtl_math_ConversionStatus_Ok || nParseEnd < aString.getLength()) + // Not a valid number or not entire string consumed. + return false; + + return true; +} + sal_Int32 ScStringUtil::GetQuotedTokenCount(const OUString &rIn, const OUString& rQuotedPairs, sal_Unicode cTok ) { assert( !(rQuotedPairs.getLength()%2) ); diff --git a/sc/source/ui/docshell/datastream.cxx b/sc/source/ui/docshell/datastream.cxx index c7f52e5e16fc..4c0deac37e77 100644 --- a/sc/source/ui/docshell/datastream.cxx +++ b/sc/source/ui/docshell/datastream.cxx @@ -45,88 +45,161 @@ inline double getNow() return static_cast<double>(now.Seconds) + static_cast<double>(now.Nanosec) / 1000000000.0; } -namespace datastreams { +#if ENABLE_ORCUS -class CallerThread : public salhelper::Thread +class CSVHandler { - DataStream *mpDataStream; + DataStream::Line& mrLine; + size_t mnColCount; + size_t mnCols; + const char* mpLineHead; + public: - osl::Condition maStart; - bool mbTerminate; + CSVHandler( DataStream::Line& rLine, size_t nColCount ) : + mrLine(rLine), mnColCount(nColCount), mnCols(0), mpLineHead(rLine.maLine.getStr()) {} - CallerThread(DataStream *pData): - Thread("CallerThread") - ,mpDataStream(pData) - ,mbTerminate(false) - {} + void begin_parse() {} + void end_parse() {} + void begin_row() {} + void end_row() {} -private: - virtual void execute() + void cell(const char* p, size_t n) { - while (!mbTerminate) + if (mnCols >= mnColCount) + return; + + DataStream::Cell aCell; + if (ScStringUtil::parseSimpleNumber(p, n, '.', ',', aCell.mfValue)) + { + aCell.mbValue = true; + } + else { - // wait for a small amount of time, so that - // painting methods have a chance to be called. - // And also to make UI more responsive. - TimeValue const aTime = {0, 1000}; - maStart.wait(); - maStart.reset(); - if (!mbTerminate) - while (mpDataStream->ImportData()) - wait(aTime); - }; + aCell.mbValue = false; + aCell.maStr.Pos = std::distance(mpLineHead, p); + aCell.maStr.Size = n; + } + mrLine.maCells.push_back(aCell); + + ++mnCols; } }; +#endif + +namespace datastreams { + +void emptyLineQueue( std::queue<DataStream::LinesType*>& rQueue ) +{ + while (!rQueue.empty()) + { + delete rQueue.front(); + rQueue.pop(); + } +} + class ReaderThread : public salhelper::Thread { SvStream *mpStream; + size_t mnColCount; + bool mbTerminate; + osl::Mutex maMtxTerminate; + + std::queue<DataStream::LinesType*> maPendingLines; + std::queue<DataStream::LinesType*> maUsedLines; + osl::Mutex maMtxLines; + + osl::Condition maCondReadStream; + osl::Condition maCondConsume; + +#if ENABLE_ORCUS + orcus::csv_parser_config maConfig; +#endif + public: - bool mbTerminateReading; - osl::Condition maProduceResume; - osl::Condition maConsumeResume; - osl::Mutex maLinesProtector; - std::queue<LinesList* > maPendingLines; - std::queue<LinesList* > maUsedLines; - - ReaderThread(SvStream *pData): - Thread("ReaderThread") - ,mpStream(pData) - ,mbTerminateReading(false) + + ReaderThread(SvStream *pData, size_t nColCount): + Thread("ReaderThread"), + mpStream(pData), + mnColCount(nColCount), + mbTerminate(false) { +#if ENABLE_ORCUS + maConfig.delimiters.push_back(','); + maConfig.text_qualifier = '"'; +#endif } virtual ~ReaderThread() { delete mpStream; - while (!maPendingLines.empty()) - { - delete maPendingLines.front(); - maPendingLines.pop(); - } - while (!maUsedLines.empty()) - { - delete maUsedLines.front(); - maUsedLines.pop(); - } + emptyLineQueue(maPendingLines); + emptyLineQueue(maUsedLines); + } + + bool isTerminateRequested() + { + osl::MutexGuard aGuard(maMtxTerminate); + return mbTerminate; + } + + void requestTerminate() + { + osl::MutexGuard aGuard(maMtxTerminate); + mbTerminate = true; } void endThread() { - mbTerminateReading = true; - maProduceResume.set(); - join(); + requestTerminate(); + maCondReadStream.set(); + } + + void waitForNewLines() + { + maCondConsume.wait(); + maCondConsume.reset(); + } + + DataStream::LinesType* popNewLines() + { + DataStream::LinesType* pLines = maPendingLines.front(); + maPendingLines.pop(); + return pLines; + } + + void resumeReadStream() + { + if (maPendingLines.size() <= 4) + maCondReadStream.set(); // start producer again + } + + bool hasNewLines() + { + return !maPendingLines.empty(); + } + + void pushUsedLines( DataStream::LinesType* pLines ) + { + maUsedLines.push(pLines); + } + + osl::Mutex& getLinesMutex() + { + return maMtxLines; } private: virtual void execute() SAL_OVERRIDE { - while (!mbTerminateReading) + while (!isTerminateRequested()) { - LinesList *pLines = 0; - osl::ResettableMutexGuard aGuard(maLinesProtector); + DataStream::LinesType* pLines = NULL; + osl::ResettableMutexGuard aGuard(maMtxLines); + if (!maUsedLines.empty()) { + // Re-use lines from previous runs. pLines = maUsedLines.front(); maUsedLines.pop(); aGuard.clear(); // unlock @@ -134,28 +207,54 @@ private: else { aGuard.clear(); // unlock - pLines = new LinesList(10); + pLines = new DataStream::LinesType(10); + } + + // Read & store new lines from stream. + for (size_t i = 0, n = pLines->size(); i < n; ++i) + { + DataStream::Line& rLine = (*pLines)[i]; + rLine.maCells.clear(); + mpStream->ReadLine(rLine.maLine); +#if ENABLE_ORCUS + CSVHandler aHdl(rLine, mnColCount); + orcus::csv_parser<CSVHandler> parser(rLine.maLine.getStr(), rLine.maLine.getLength(), aHdl, maConfig); + parser.parse(); +#endif } - for (size_t i = 0; i < pLines->size(); ++i) - mpStream->ReadLine( pLines->at(i) ); + aGuard.reset(); // lock - while (!mbTerminateReading && maPendingLines.size() >= 8) - { // pause reading for a bit + while (!isTerminateRequested() && maPendingLines.size() >= 8) + { + // pause reading for a bit aGuard.clear(); // unlock - maProduceResume.wait(); - maProduceResume.reset(); + maCondReadStream.wait(); + maCondReadStream.reset(); aGuard.reset(); // lock } maPendingLines.push(pLines); - maConsumeResume.set(); + maCondConsume.set(); if (!mpStream->good()) - mbTerminateReading = true; + requestTerminate(); } } }; } +DataStream::Cell::Cell() : mfValue(0.0), mbValue(true) {} + +DataStream::Cell::Cell( const Cell& r ) : mbValue(r.mbValue) +{ + if (r.mbValue) + mfValue = r.mfValue; + else + { + maStr.Pos = r.maStr.Pos; + maStr.Size = r.maStr.Size; + } +} + void DataStream::MakeToolbarVisible() { css::uno::Reference< css::frame::XFrame > xFrame = @@ -219,8 +318,8 @@ DataStream::DataStream(ScDocShell *pShell, const OUString& rURL, const ScRange& mfLastRefreshTime(0.0), mnCurRow(0) { - mxThread = new datastreams::CallerThread( this ); - mxThread->launch(); + maImportTimer.SetTimeout(0); + maImportTimer.SetTimeoutHdl( LINK(this, DataStream, ImportTimerHdl) ); Decode(rURL, rRange, nLimit, eMove, nSettings); } @@ -229,35 +328,36 @@ DataStream::~DataStream() { if (mbRunning) StopImport(); - mxThread->mbTerminate = true; - mxThread->maStart.set(); - mxThread->join(); + if (mxReaderThread.is()) + { mxReaderThread->endThread(); + mxReaderThread->join(); + } delete mpLines; } -OString DataStream::ConsumeLine() +DataStream::Line DataStream::ConsumeLine() { if (!mpLines || mnLinesCount >= mpLines->size()) { mnLinesCount = 0; - if (mxReaderThread->mbTerminateReading) - return OString(); - osl::ResettableMutexGuard aGuard(mxReaderThread->maLinesProtector); + if (mxReaderThread->isTerminateRequested()) + return Line(); + + osl::ResettableMutexGuard aGuard(mxReaderThread->getLinesMutex()); if (mpLines) - mxReaderThread->maUsedLines.push(mpLines); - while (mxReaderThread->maPendingLines.empty()) + mxReaderThread->pushUsedLines(mpLines); + + while (!mxReaderThread->hasNewLines()) { aGuard.clear(); // unlock - mxReaderThread->maConsumeResume.wait(); - mxReaderThread->maConsumeResume.reset(); + mxReaderThread->waitForNewLines(); aGuard.reset(); // lock } - mpLines = mxReaderThread->maPendingLines.front(); - mxReaderThread->maPendingLines.pop(); - if (mxReaderThread->maPendingLines.size() <= 4) - mxReaderThread->maProduceResume.set(); // start producer again + + mpLines = mxReaderThread->popNewLines(); + mxReaderThread->resumeReadStream(); } return mpLines->at(mnLinesCount++); } @@ -327,12 +427,13 @@ void DataStream::StartImport() pStream = new SvScriptStream(msURL); else pStream = new SvFileStream(msURL, STREAM_READ); - mxReaderThread = new datastreams::ReaderThread( pStream ); + mxReaderThread = new datastreams::ReaderThread(pStream, maStartRange.aEnd.Col() - maStartRange.aStart.Col() + 1); mxReaderThread->launch(); } mbRunning = true; maDocAccess.reset(); - mxThread->maStart.set(); + + maImportTimer.Start(); } void DataStream::StopImport() @@ -342,6 +443,7 @@ void DataStream::StopImport() mbRunning = false; Refresh(); + maImportTimer.Stop(); } void DataStream::SetRefreshOnEmptyLine( bool bVal ) @@ -397,82 +499,10 @@ void DataStream::MoveData() #if ENABLE_ORCUS -namespace { - -struct StrVal -{ - ScAddress maPos; - OUString maStr; - - StrVal( const ScAddress& rPos, const OUString& rStr ) : maPos(rPos), maStr(rStr) {} -}; - -struct NumVal -{ - ScAddress maPos; - double mfVal; - - NumVal( const ScAddress& rPos, double fVal ) : maPos(rPos), mfVal(fVal) {} -}; - -typedef std::vector<StrVal> StrValArray; -typedef std::vector<NumVal> NumValArray; - -/** - * This handler handles a single line CSV input. - */ -class CSVHandler -{ - ScAddress maPos; - SCROW mnRow; - SCCOL mnCol; - SCCOL mnEndCol; - SCTAB mnTab; - - StrValArray maStrs; - NumValArray maNums; - -public: - CSVHandler( const ScAddress& rPos, SCCOL nEndCol ) : maPos(rPos), mnEndCol(nEndCol) {} - - void begin_parse() {} - void end_parse() {} - void begin_row() {} - void end_row() {} - - void cell(const char* p, size_t n) - { - if (maPos.Col() <= mnEndCol) - { - OUString aStr(p, n, RTL_TEXTENCODING_UTF8); - double fVal; - if (ScStringUtil::parseSimpleNumber(aStr, '.', ',', fVal)) - maNums.push_back(NumVal(maPos, fVal)); - else - maStrs.push_back(StrVal(maPos, aStr)); - } - maPos.IncCol(); - } - - const StrValArray& getStrs() const { return maStrs; } - const NumValArray& getNums() const { return maNums; } -}; - -} - void DataStream::Text2Doc() { - OString aLine = ConsumeLine(); - orcus::csv_parser_config aConfig; - aConfig.delimiters.push_back(','); - aConfig.text_qualifier = '"'; - CSVHandler aHdl(ScAddress(maStartRange.aStart.Col(), mnCurRow, maStartRange.aStart.Tab()), maStartRange.aEnd.Col()); - orcus::csv_parser<CSVHandler> parser(aLine.getStr(), aLine.getLength(), aHdl, aConfig); - parser.parse(); - - const StrValArray& rStrs = aHdl.getStrs(); - const NumValArray& rNums = aHdl.getNums(); - if (rStrs.empty() && rNums.empty() && mbRefreshOnEmptyLine) + Line aLine = ConsumeLine(); + if (aLine.maCells.empty() && mbRefreshOnEmptyLine) { // Empty line detected. Trigger refresh and discard it. Refresh(); @@ -481,15 +511,24 @@ void DataStream::Text2Doc() MoveData(); { - StrValArray::const_iterator it = rStrs.begin(), itEnd = rStrs.end(); - for (; it != itEnd; ++it) - maDocAccess.setStringCell(it->maPos, it->maStr); - } - - { - NumValArray::const_iterator it = rNums.begin(), itEnd = rNums.end(); - for (; it != itEnd; ++it) - maDocAccess.setNumericCell(it->maPos, it->mfVal); + std::vector<Cell>::const_iterator it = aLine.maCells.begin(), itEnd = aLine.maCells.end(); + SCCOL nCol = maStartRange.aStart.Col(); + const char* pLineHead = aLine.maLine.getStr(); + for (; it != itEnd; ++it, ++nCol) + { + const Cell& rCell = *it; + if (rCell.mbValue) + { + maDocAccess.setNumericCell( + ScAddress(nCol, mnCurRow, maStartRange.aStart.Tab()), rCell.mfValue); + } + else + { + maDocAccess.setStringCell( + ScAddress(nCol, mnCurRow, maStartRange.aStart.Tab()), + OUString(pLineHead+rCell.maStr.Pos, rCell.maStr.Size, RTL_TEXTENCODING_UTF8)); + } + } } if (meMove == NO_MOVE) @@ -514,7 +553,6 @@ void DataStream::Text2Doc() {} bool DataStream::ImportData() { - SolarMutexGuard aGuard; if (!mbValuesInLine) // We no longer support this mode. To be deleted later. return false; @@ -526,6 +564,14 @@ bool DataStream::ImportData() return mbRunning; } +IMPL_LINK_NOARG(DataStream, ImportTimerHdl) +{ + if (ImportData()) + maImportTimer.Start(); + + return 0; +} + } // namespace sc /* vim:set shiftwidth=4 softtabstop=4 expandtab: */ diff --git a/sc/source/ui/inc/datastream.hxx b/sc/source/ui/inc/datastream.hxx index 5a4d8cd444b7..5600a09c1eb4 100644 --- a/sc/source/ui/inc/datastream.hxx +++ b/sc/source/ui/inc/datastream.hxx @@ -14,6 +14,7 @@ #include <rtl/ref.hxx> #include <rtl/ustring.hxx> +#include <vcl/timer.hxx> #include <address.hxx> #include <boost/noncopyable.hpp> @@ -33,15 +34,37 @@ namespace datastreams { class ReaderThread; } -typedef std::vector<OString> LinesList; class DataStream : boost::noncopyable { - OString ConsumeLine(); - void MoveData(); - void Text2Doc(); - public: + struct Cell + { + struct Str + { + size_t Pos; + size_t Size; + }; + + union + { + Str maStr; + double mfValue; + }; + + bool mbValue; + + Cell(); + Cell( const Cell& r ); + }; + + struct Line + { + OString maLine; + std::vector<Cell> maCells; + }; + typedef std::vector<Line> LinesType; + enum MoveType { NO_MOVE, RANGE_DOWN, MOVE_DOWN, MOVE_UP }; enum { SCRIPT_STREAM = 1, VALUES_IN_LINE = 2 }; @@ -75,8 +98,13 @@ public: void SetRefreshOnEmptyLine( bool bVal ); private: + Line ConsumeLine(); + void MoveData(); + void Text2Doc(); void Refresh(); + DECL_LINK( ImportTimerHdl, void* ); + private: ScDocShell* mpDocShell; ScDocument* mpDoc; @@ -89,14 +117,16 @@ private: bool mbRunning; bool mbValuesInLine; bool mbRefreshOnEmptyLine; - LinesList* mpLines; + LinesType* mpLines; size_t mnLinesCount; size_t mnLinesSinceRefresh; double mfLastRefreshTime; SCROW mnCurRow; ScRange maStartRange; ScRange maEndRange; - rtl::Reference<datastreams::CallerThread> mxThread; + + Timer maImportTimer; + rtl::Reference<datastreams::ReaderThread> mxReaderThread; }; diff --git a/sc/source/ui/miscdlgs/datastreamdlg.cxx b/sc/source/ui/miscdlgs/datastreamdlg.cxx index 8a3bee5a813a..b0cc2b0ba6ed 100644 --- a/sc/source/ui/miscdlgs/datastreamdlg.cxx +++ b/sc/source/ui/miscdlgs/datastreamdlg.cxx @@ -100,7 +100,7 @@ ScRange DataStreamDlg::GetStartRange() OUString aStr = m_pEdRange->GetText(); ScDocument* pDoc = mpDocShell->GetDocument(); ScRange aRange; - sal_uInt16 nRes = aRange.Parse(aStr, pDoc); + sal_uInt16 nRes = aRange.Parse(aStr, pDoc, pDoc->GetAddressConvention()); if ((nRes & SCA_VALID) != SCA_VALID || !aRange.IsValid()) { // Invalid range. @@ -118,11 +118,12 @@ ScRange DataStreamDlg::GetStartRange() void DataStreamDlg::Init( const DataStream& rStrm ) { m_pCbUrl->SetText(rStrm.GetURL()); + ScDocument* pDoc = mpDocShell->GetDocument(); ScRange aRange = rStrm.GetRange(); ScRange aTopRange = aRange; aTopRange.aEnd.SetRow(aTopRange.aStart.Row()); - OUString aStr = aTopRange.Format(SCA_VALID); + OUString aStr = aTopRange.Format(SCR_ABS_3D, pDoc, pDoc->GetAddressConvention()); m_pEdRange->SetText(aStr); SCROW nRows = aRange.aEnd.Row() - aRange.aStart.Row() + 1; |