1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
* Version: MPL 1.1 / GPLv3+ / LGPLv3+
*
* The contents of this file are subject to the Mozilla Public License Version
* 1.1 (the "License"); you may not use this file except in compliance with
* the License or as specified alternatively below. You may obtain a copy of
* the License at http://www.mozilla.org/MPL/
*
* Software distributed under the License is distributed on an "AS IS" basis,
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
* for the specific language governing rights and limitations under the
* License.
*
* Major Contributor(s):
* Copyright (C) 2012 Gert van Valkenhoef <g.h.m.van.valkenhoef@rug.nl>
* (initial developer)
*
* All Rights Reserved.
*
* For minor contributions see the git repository.
*
* Alternatively, the contents of this file may be used under the terms of
* either the GNU General Public License Version 3 or later (the "GPLv3+"), or
* the GNU Lesser General Public License Version 3 or later (the "LGPLv3+"),
* in which case the provisions of the GPLv3+ or the LGPLv3+ are applicable
* instead of those above.
*/
#include <helpcompiler/HelpIndexer.hxx>
#include <rtl/string.hxx>
#include <rtl/uri.hxx>
#include <rtl/ustrbuf.hxx>
#include <osl/file.hxx>
#include <osl/thread.h>
#include <boost/scoped_ptr.hpp>
#include <algorithm>
#include "LuceneHelper.hxx"
using namespace lucene::document;
HelpIndexer::HelpIndexer(rtl::OUString const &lang, rtl::OUString const &module,
rtl::OUString const &srcDir, rtl::OUString const &outDir)
: d_lang(lang), d_module(module)
{
d_indexDir = rtl::OUStringBuffer(outDir).append('/').
append(module).appendAscii(RTL_CONSTASCII_STRINGPARAM(".idxl")).toString();
d_captionDir = srcDir + rtl::OUString("/caption");
d_contentDir = srcDir + rtl::OUString("/content");
}
bool HelpIndexer::indexDocuments()
{
if (!scanForFiles())
return false;
try
{
rtl::OUString sLang = d_lang.getToken(0, '-');
bool bUseCJK = sLang == "ja" || sLang == "ko" || sLang == "zh";
// Construct the analyzer appropriate for the given language
boost::scoped_ptr<lucene::analysis::Analyzer> analyzer;
if (bUseCJK)
analyzer.reset(new lucene::analysis::LanguageBasedAnalyzer(L"cjk"));
else
analyzer.reset(new lucene::analysis::standard::StandardAnalyzer());
rtl::OUString ustrSystemPath;
osl::File::getSystemPathFromFileURL(d_indexDir, ustrSystemPath);
rtl::OString indexDirStr = rtl::OUStringToOString(ustrSystemPath, osl_getThreadTextEncoding());
lucene::index::IndexWriter writer(indexDirStr.getStr(), analyzer.get(), true);
//Double limit of tokens allowed, otherwise we'll get a too-many-tokens
//exception for ja help. Could alternative ignore the exception and get
//truncated results as per java-Lucene apparently
writer.setMaxFieldLength(lucene::index::IndexWriter::DEFAULT_MAX_FIELD_LENGTH*2);
// Index the identified help files
Document doc;
for (std::set<rtl::OUString>::iterator i = d_files.begin(); i != d_files.end(); ++i) {
helpDocument(*i, &doc);
writer.addDocument(&doc);
doc.clear();
}
writer.optimize();
// Optimize the index
writer.optimize();
}
catch (CLuceneError &e)
{
d_error = rtl::OUString::createFromAscii(e.what());
return false;
}
return true;
}
rtl::OUString const & HelpIndexer::getErrorMessage() {
return d_error;
}
bool HelpIndexer::scanForFiles() {
if (!scanForFiles(d_contentDir)) {
return false;
}
if (!scanForFiles(d_captionDir)) {
return false;
}
return true;
}
bool HelpIndexer::scanForFiles(rtl::OUString const & path) {
osl::Directory dir(path);
if (osl::FileBase::E_None != dir.open()) {
d_error = rtl::OUString("Error reading directory ") + path;
return true;
}
osl::DirectoryItem item;
osl::FileStatus fileStatus(osl_FileStatus_Mask_FileName | osl_FileStatus_Mask_Type);
while (dir.getNextItem(item) == osl::FileBase::E_None) {
item.getFileStatus(fileStatus);
if (fileStatus.getFileType() == osl::FileStatus::Regular) {
d_files.insert(fileStatus.getFileName());
}
}
return true;
}
bool HelpIndexer::helpDocument(rtl::OUString const & fileName, Document *doc) {
// Add the help path as an indexed, untokenized field.
rtl::OUString path = rtl::OUString("#HLP#") +
d_module + rtl::OUString("/") + fileName;
std::vector<TCHAR> aPath(OUStringToTCHARVec(path));
doc->add(*_CLNEW Field(_T("path"), &aPath[0], Field::STORE_YES | Field::INDEX_UNTOKENIZED));
rtl::OUString sEscapedFileName =
rtl::Uri::encode(fileName,
rtl_UriCharClassUric, rtl_UriEncodeIgnoreEscapes, RTL_TEXTENCODING_UTF8);
// Add the caption as a field.
rtl::OUString captionPath = d_captionDir + rtl::OUString("/") + sEscapedFileName;
doc->add(*_CLNEW Field(_T("caption"), helpFileReader(captionPath), Field::STORE_NO | Field::INDEX_TOKENIZED));
// Add the content as a field.
rtl::OUString contentPath = d_contentDir + rtl::OUString("/") + sEscapedFileName;
doc->add(*_CLNEW Field(_T("content"), helpFileReader(contentPath), Field::STORE_NO | Field::INDEX_TOKENIZED));
return true;
}
lucene::util::Reader *HelpIndexer::helpFileReader(rtl::OUString const & path) {
osl::File file(path);
if (osl::FileBase::E_None == file.open(osl_File_OpenFlag_Read)) {
file.close();
rtl::OUString ustrSystemPath;
osl::File::getSystemPathFromFileURL(path, ustrSystemPath);
rtl::OString pathStr = rtl::OUStringToOString(ustrSystemPath, osl_getThreadTextEncoding());
return _CLNEW lucene::util::FileReader(pathStr.getStr(), "UTF-8");
} else {
return _CLNEW lucene::util::StringReader(L"");
}
}
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
|
