summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xhelpcontent2/help-to-wiki.py17
-rwxr-xr-xhelpcontent2/to-wiki/wikiconv2.py106
-rwxr-xr-xhelpcontent2/upload-wiki.pl1
3 files changed, 89 insertions, 35 deletions
diff --git a/helpcontent2/help-to-wiki.py b/helpcontent2/help-to-wiki.py
index ee4e62869..be86f626e 100755
--- a/helpcontent2/help-to-wiki.py
+++ b/helpcontent2/help-to-wiki.py
@@ -77,6 +77,21 @@ create_wiki_dirs()
print "Generating the titles..."
os.system( "python to-wiki/getalltitles.py source/text > alltitles.csv" )
-wikiconv2.convert(generate_redirects, args)
+try:
+ sdf_path = args[0]
+except:
+ sdf_path = '../../l10n/l10n/source'
+ sys.stderr.write('Path to the .sdf files not provided, using "%s"\n'% sdf_path)
+
+# [16:26:45] <kendy> sophi, timar: Obviously, I am testing with Czech ;-)
+# [16:27:05] <timar> kendy: HUngarian :)
+# [16:27:25] <kendy> timar: :-)
+# [16:27:26] <timar> kendy: Slovenian for Martin
+# [16:28:00] <timar> kendy: and German, Italian, French, Spanish (with large user base)
+# [16:28:13] <sophi> kendy: catalan
+# [16:28:38] <sophi> kendy: also japanese and russian
+# [16:29:48] <sophi> kendy: ans I would say vi too, so you have all kind of script/language and good communities behind ;)
+for lang in ['', 'cs']:
+ wikiconv2.convert(generate_redirects, lang, '%s/%s/localize.sdf'% (sdf_path, lang))
# vim:set shiftwidth=4 softtabstop=4 expandtab:
diff --git a/helpcontent2/to-wiki/wikiconv2.py b/helpcontent2/to-wiki/wikiconv2.py
index 6f5f80761..2b3dd5df5 100755
--- a/helpcontent2/to-wiki/wikiconv2.py
+++ b/helpcontent2/to-wiki/wikiconv2.py
@@ -1,7 +1,6 @@
#!/usr/bin/env python
-import sys
-import thread, threading, time
+import os, sys, thread, threading, time
import xml.parsers.expat
import codecs
from threading import Thread
@@ -9,7 +8,7 @@ from threading import Thread
root="source/"
max_threads = 25
-titles = [[]]
+titles = []
# map of id -> localized text
localization_data = {}
@@ -120,6 +119,8 @@ replace_text_list = \
]
def load_hid_lst():
+ global hid_lst
+ hid_lst = {}
file = codecs.open("helpers/hid.lst", "r", "utf-8")
for line in file:
ids = line.strip().split(" ")
@@ -174,6 +175,8 @@ def escape_equals_sign(text):
return t
def load_localization_data(sdf_file):
+ global localization_data
+ localization_data = {}
try:
file = codecs.open(sdf_file, "r", "utf-8")
except:
@@ -295,7 +298,9 @@ class ElementBase:
# embed part of another file into current structure
def embed_href(self, parent_parser, fname, id):
# parse another xhp
- parser = XhpParser('source/' + fname, False, parent_parser.current_app, parent_parser.wiki_page_name)
+ parser = XhpParser('source/' + fname, False, \
+ parent_parser.current_app, parent_parser.wiki_page_name, \
+ parent_parser.lang)
var = parser.get_variable(id)
if var != None:
@@ -603,9 +608,19 @@ class Ignore(ElementBase):
def __init__(self, attrs, parent, element_name):
ElementBase.__init__(self, element_name, parent)
-class Title(TextElementBase):
+class OrigTitle(TextElementBase):
def __init__(self, attrs, parent):
+ TextElementBase.__init__(self, attrs, parent, 'title', '{{OrigLang|', '}}\n\n', True)
+
+class Title(TextElementBase):
+ def __init__(self, attrs, parent, localized_title):
TextElementBase.__init__(self, attrs, parent, 'title', '{{Lang|', '}}\n\n', True)
+ self.localized_title = localized_title
+
+ def get_all(self):
+ if self.localized_title != '':
+ self.text = self.localized_title
+ return TextElementBase.get_all(self)
class Topic(ElementBase):
def __init__(self, attrs, parent):
@@ -613,7 +628,10 @@ class Topic(ElementBase):
def start_element(self, parser, name, attrs):
if name == 'title':
- self.parse_child(Title(attrs, self))
+ if parser.lang == '':
+ self.parse_child(OrigTitle(attrs, self))
+ else:
+ self.parse_child(Title(attrs, self, get_localized_text(parser.filename, 'tit')))
elif name == 'filename':
self.parse_child(Ignore(attrs, self, name))
else:
@@ -709,7 +727,7 @@ class Sort(ElementBase):
return ElementBase.get_all(self)
class Link(ElementBase):
- def __init__(self, attrs, parent):
+ def __init__(self, attrs, parent, lang):
ElementBase.__init__(self, 'link', parent)
self.link = attrs['href']
@@ -721,6 +739,7 @@ class Link(ElementBase):
self.default_name = self.lname
self.lname = get_link_filename(self.link, self.lname)
self.wikitext = ""
+ self.lang = lang
def char_data(self, parser, data):
self.wikitext = self.wikitext + data
@@ -732,6 +751,8 @@ class Link(ElementBase):
self.wikitext = replace_text(self.wikitext)
if self.link.find("http") >= 0:
text = "["+self.link+" "+self.wikitext+"]"
+ elif self.lang != '':
+ text = '[[%s/%s|%s]]'% (self.lname, self.lang, self.wikitext)
else:
text = "[["+self.lname+"|"+self.wikitext+"]]"
return text
@@ -948,7 +969,7 @@ class Paragraph(ElementBase):
elif name == 'item':
self.parse_child(Item(attrs, self))
elif name == 'link':
- self.parse_child(Link(attrs, self))
+ self.parse_child(Link(attrs, self, parser.lang))
elif name == 'localized':
# we ignore this tag, it is added arbitrary for the paragraphs
# that come from .sdf files
@@ -1066,12 +1087,13 @@ class TableContentParagraph(Paragraph):
self.role = 'tablecontent'
class ParserBase:
- def __init__(self, filename, follow_embed, embedding_app, current_app, wiki_page_name, head_object, buffer):
+ def __init__(self, filename, follow_embed, embedding_app, current_app, wiki_page_name, lang, head_object, buffer):
self.filename = filename
self.follow_embed = follow_embed
self.embedding_app = embedding_app
self.current_app = current_app
self.wiki_page_name = wiki_page_name
+ self.lang = lang
self.head_obj = head_object
p = xml.parsers.expat.ParserCreate()
@@ -1107,7 +1129,7 @@ class ParserBase:
# parse the localized text
text = u'<?xml version="1.0" encoding="UTF-8"?><localized>' + localized_text + '</localized>'
ParserBase(self.filename, self.follow_embed, self.embedding_app, \
- self.current_app, self.wiki_page_name, \
+ self.current_app, self.wiki_page_name, self.lang, \
paragraph, text.encode('utf-8'))
# add it to the overall structure
obj.objects.append(paragraph)
@@ -1132,7 +1154,7 @@ class ParserBase:
self.parse_localized_paragraph(Paragraph(attrs, obj), attrs, obj)
class XhpParser(ParserBase):
- def __init__(self, filename, follow_embed, embedding_app, wiki_page_name):
+ def __init__(self, filename, follow_embed, embedding_app, wiki_page_name, lang):
# we want to ignore the 1st level="1" heading, because in most of the
# cases, it is the only level="1" heading in the file, and it is the
# same as the page title
@@ -1156,10 +1178,12 @@ class XhpParser(ParserBase):
buf = file.read()
file.close()
- ParserBase.__init__(self, filename, follow_embed, embedding_app, current_app, wiki_page_name, XhpFile(), buf.encode('utf-8'))
+ ParserBase.__init__(self, filename, follow_embed, embedding_app,
+ current_app, wiki_page_name, lang, XhpFile(), buf.encode('utf-8'))
def loadallfiles(filename):
global titles
+ titles = []
file = codecs.open(filename, "r", "utf-8")
for line in file:
title = line.split(";", 2)
@@ -1167,14 +1191,15 @@ def loadallfiles(filename):
file.close()
class WikiConverter(Thread):
- def __init__(self, inputfile, wiki_page_name, outputfile):
+ def __init__(self, inputfile, wiki_page_name, lang, outputfile):
Thread.__init__(self)
self.inputfile = inputfile
self.wiki_page_name = wiki_page_name
+ self.lang = lang
self.outputfile = outputfile
def run(self):
- parser = XhpParser(self.inputfile, True, '', self.wiki_page_name)
+ parser = XhpParser(self.inputfile, True, '', self.wiki_page_name, self.lang)
file = codecs.open(self.outputfile, "wb", "utf-8")
file.write(parser.get_all())
file.close()
@@ -1226,37 +1251,50 @@ def write_redirects():
write_link(r, target)
# Main Function
-def convert(generate_redirects, localizations):
- print "Generating the wiki itself..."
+def convert(generate_redirects, lang, sdf_file):
+ if lang == '':
+ print 'Generating the wiki main pages...'
+ else:
+ print 'Generating the wiki pages for language %s...'% lang
load_hid_lst()
loadallfiles("alltitles.csv")
- if len(localizations) > 0:
- sys.stderr.write('Using localizations from "%s"\n'% localizations[0])
- if not load_localization_data(localizations[0]):
+ if lang != '':
+ sys.stderr.write('Using localizations from "%s"\n'% sdf_file)
+ if not load_localization_data(sdf_file):
return
for title in titles:
while threading.active_count() > max_threads:
time.sleep(0.001)
- outfile = ""
- infile = ""
- if len(title) > 1:
- outfile = "wiki/"+title[1].strip()
- infile = title[0].strip()
+ infile = title[0].strip()
+ wikiname = title[1].strip()
+ articledir = 'wiki/' + wikiname
+ try:
+ os.mkdir(articledir)
+ except:
+ pass
+
+ outfile = ''
+ if lang != '':
+ wikiname = '%s/%s'% (wikiname, lang)
+ outfile = '%s/%s'% (articledir, lang)
+ else:
+ outfile = '%s/MAIN'% articledir
+
+ try:
+ file = open(outfile, 'r')
+ except:
try:
- file = open(outfile,"r")
+ wiki = WikiConverter(infile, wikiname, lang, outfile)
+ wiki.start()
+ continue
except:
- try:
- wiki = WikiConverter(infile,title[1].strip(),outfile)
- wiki.start()
- continue
- except:
- print 'Failed to convert "%s" into "%s".\n'% \
- (title[1].strip(), outfile)
- print "Warning: Skipping: "+infile+" > "+outfile
- file.close()
+ print 'Failed to convert "%s" into "%s".\n'% \
+ (infile, outfile)
+ sys.stderr.write('Warning: Skipping: %s > %s\n'% (infile, outfile))
+ file.close()
# wait for everyone to finish
while threading.active_count() > 1:
diff --git a/helpcontent2/upload-wiki.pl b/helpcontent2/upload-wiki.pl
index 85d5201a2..d5b5f556d 100755
--- a/helpcontent2/upload-wiki.pl
+++ b/helpcontent2/upload-wiki.pl
@@ -112,6 +112,7 @@ sub upload_article {
my $pagename = $File::Find::name;
$pagename =~ s/^wiki\///;
+ $pagename =~ s/\/MAIN$//;
# pages starting with lowercase 's' are redirects
if ( $pagename =~ /^s/ ) {