From a6ba1faa80df6729c22459d44d643c7ecd08e41e Mon Sep 17 00:00:00 2001 From: reger Date: Fri, 3 Jun 2016 01:46:30 +0200 Subject: [PATCH] introduce a translation edit servlet Translator_p.html YaCy's UI text translation This is the 1st rudimentary approach to support the translatio utilities. It allows currently to edit untranslated text and save it in a local translation file in the DATA/LOCALE directory. + refactor Translator (less static's) to leverage on class overrides and support garbage collection for this 1 time routine + adjust TranslatorXliff to check for local translations in DATA/LOCALE, this includes storing manually downloaded translation files in DATA as well (to keep default untouched) + on 1st call of Translator_p a master tanslation file is generated, checking the supported languages for missing translation text (later this masterfile is planned to part of the distribution, to harmonize translation key text between the languages) Outlook: the local modifications (possibly as translation fragments instead of complete file) to be shared with maintainer using xlif features. --- htroot/ConfigBasic.java | 4 +- htroot/ConfigLanguage_p.html | 1 + htroot/ConfigLanguage_p.java | 10 +- htroot/Translator_p.html | 53 ++++++++ htroot/Translator_p.java | 127 ++++++++++++++++++ source/net/yacy/crawler/CrawlSwitchboard.java | 4 +- source/net/yacy/data/Translator.java | 16 +-- .../translation/CreateTranslationMasters.java | 8 +- .../translation/ListNonTranslatedFiles.java | 2 +- .../yacy/utils/translation/TranslateAll.java | 2 +- .../TranslateAllToOneLanguage.java | 2 +- .../utils/translation/TranslatorXliff.java | 28 +++- source/net/yacy/yacy.java | 4 +- .../translation/TranslatorXliffTest.java | 2 +- 14 files changed, 230 insertions(+), 33 deletions(-) create mode 100644 htroot/Translator_p.html create mode 100644 htroot/Translator_p.java diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.java index 8c8d79ade..378ee2e07 100644 --- a/htroot/ConfigBasic.java +++ b/htroot/ConfigBasic.java @@ -36,7 +36,6 @@ import java.util.regex.Pattern; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.data.Translator; import net.yacy.data.WorkTables; import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.workflow.InstantBusyThread; @@ -46,6 +45,7 @@ import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.server.http.HTTPDFileHandler; +import net.yacy.utils.translation.TranslatorXliff; import net.yacy.utils.upnp.UPnPMappingType; import net.yacy.utils.upnp.UPnP; @@ -85,7 +85,7 @@ public class ConfigBasic { // language settings if (post != null && post.containsKey("language") && !lang.equals(post.get("language", "default")) && - (Translator.changeLang(env, langPath, post.get("language", "default") + ".lng"))) { + (new TranslatorXliff().changeLang(env, langPath, post.get("language", "default") + ".lng"))) { prop.put("changedLanguage", "1"); } diff --git a/htroot/ConfigLanguage_p.html b/htroot/ConfigLanguage_p.html index 3ddc8a644..d0cca65c6 100644 --- a/htroot/ConfigLanguage_p.html +++ b/htroot/ConfigLanguage_p.html @@ -57,6 +57,7 @@

Make sure that you only download data from trustworthy sources. The new language file might overwrite existing data if a file of the same name exists already.

+

Simple Editor to add untranslated text

#(status)# ::

Unable to get URL: #[url]#

diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index 0ec64ea82..6dc4d7a97 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -82,7 +82,7 @@ public class ConfigLanguage_p { * directory traversal attacks! */ if (langFiles.contains(selectedLanguage) || selectedLanguage.startsWith("default")) { - Translator.changeLang(env, langPath, selectedLanguage); + new TranslatorXliff().changeLang(env, langPath, selectedLanguage); } //delete language file @@ -105,7 +105,8 @@ public class ConfigLanguage_p { final DigestURL u = new DigestURL(url); it = FileUtils.strings(u.get(ClientIdentification.yacyInternetCrawlerAgent, null, null)); try { - File langFile = new File(langPath, u.getFileName()); + TranslatorXliff tx = new TranslatorXliff(); + File langFile = tx.getScratchFile(new File(langPath, u.getFileName())); final OutputStreamWriter bw = new OutputStreamWriter(new FileOutputStream(langFile), StandardCharsets.UTF_8.name()); while (it.hasNext()) { @@ -116,14 +117,13 @@ public class ConfigLanguage_p { // convert downloaded xliff to internal lng file final String ext = Files.getFileExtension(langFile.getName()); if (ext.equalsIgnoreCase("xlf") || ext.equalsIgnoreCase("xliff")) { - TranslatorXliff tx = new TranslatorXliff(); - Map> lng = TranslatorXliff.loadTranslationsListsFromXliff(langFile); + Map> lng = tx.loadTranslationsListsFromXliff(langFile); langFile = new File(langPath, Files.getNameWithoutExtension(langFile.getName())+".lng"); tx.saveAsLngFile(null, langFile, lng); } if (post.containsKey("use_lang") && "on".equals(post.get("use_lang"))) { - Translator.changeLang(env, langPath, langFile.getName()); + tx.changeLang(env, langPath, langFile.getName()); } } catch (final IOException e) { prop.put("status", "2");//error saving the language file diff --git a/htroot/Translator_p.html b/htroot/Translator_p.html new file mode 100644 index 000000000..772fe221c --- /dev/null +++ b/htroot/Translator_p.html @@ -0,0 +1,53 @@ + + + + YaCy '#[clientname]#': Translation Editor + #%env/templates/metas.template%# + + + + + #%env/templates/header.template%# + #%env/templates/submenuDesign.template%# +

Translation Editor

+ +

Translate untranslated text of the user interface (current language). The modified translation file is stored in DATA/LOCALE directory

+ +
+ + + +

Target Language: #[targetlang]#

#[errmsg]#

+
+ + view it + + + + + + #{textlist}# + + + + + #{/textlist}# +
Source TextTranslated Text ( #[targetlang]# )
+ + + + #(filteruntranslated)#::#(/filteruntranslated)# +
+ + + +
+
+ + #%env/templates/footer.template%# + + diff --git a/htroot/Translator_p.java b/htroot/Translator_p.java new file mode 100644 index 000000000..a8aa5b9a0 --- /dev/null +++ b/htroot/Translator_p.java @@ -0,0 +1,127 @@ + +/** + * Translator_p + * Copyright 2012 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany + * First released 14.09.2011 at http://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see . + */ +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.search.Switchboard; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; +import net.yacy.server.servletProperties; +import net.yacy.utils.translation.CreateTranslationMasters; + +public class Translator_p { + + public static servletProperties respond(@SuppressWarnings("unused") final RequestHeader requestHeader, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { + try { + final servletProperties prop = new servletProperties(); + final Switchboard sb = (Switchboard) env; + + String langcfg = env.getConfig("locale.language", "default"); + prop.put("targetlang", langcfg); + if ("default".equals(langcfg)) { + prop.put("errmsg", "activate a different language"); + return prop; + } else { + prop.put("errmsg", ""); + } + + File lngfile = new File("locales", langcfg + ".lng"); + CreateTranslationMasters ctm = new CreateTranslationMasters(/*new File ("locales","master.lng.xlf")*/); + + File masterxlf = new File("locales", "master.lng.xlf"); + if (!masterxlf.exists()) ctm.createMasterTranslationLists(masterxlf); + Map> origTrans = ctm.joinMasterTranslationLists(masterxlf, lngfile); + int i = 0; + if (origTrans.size() > 0) { + String filename = origTrans.keySet().iterator().next(); + if (post != null && post.containsKey("sourcefile")) { + filename = post.get("sourcefile", filename); + } + + Iterator filenameit = origTrans.keySet().iterator(); + + while (filenameit.hasNext()) { + String tmp = filenameit.next(); + prop.put("filelist_" + i + "_filename", tmp); + prop.put("filelist_" + i + "_selected", tmp.equals(filename)); + i++; + } + prop.put("filelist", i); + + prop.add("sourcefile", filename); + Map origTextList = origTrans.get(filename); + + i = 0; + boolean filteruntranslated = false; + int textlistid = -1; + if (post != null) { + filteruntranslated = post.getBoolean("filteruntranslated"); + if (filteruntranslated) { + prop.put("filter.checked", 1); + } else { + prop.put("filter.checked", 0); + } + textlistid = post.getInt("approve", -1); + } + boolean changed = false; + for (String sourcetext : origTextList.keySet()) { + String targettxt = origTextList.get(sourcetext); + if (targettxt == null || targettxt.isEmpty()) { + prop.put("textlist_" + i + "_filteruntranslated", true); + } else if (filteruntranslated) { + continue; + } + if (i == textlistid && post != null) { + String t = post.get("targettxt" + Integer.toString(textlistid)); + // correct common partial html markup (part of text identification for words also used as html parameter) + if (!t.isEmpty()) { + if (sourcetext.startsWith(">") && !t.startsWith(">")) t=">"+t; + if (sourcetext.endsWith("<") && !t.endsWith("<")) t=t+"<"; + } + targettxt = t; + origTextList.replace(sourcetext, targettxt); + changed = true; + } + prop.putHTML("textlist_" + i + "_sourcetxt", sourcetext); + prop.putHTML("textlist_" + i + "_targettxt", targettxt); + prop.put("textlist_" + i + "_tokenid", Integer.toString(i)); + prop.put("textlist_" + i + "_filteruntranslated_tokenid", Integer.toString(i)); + //prop.put("textlist_" + i +"_filteruntranslated", filteruntranslated); + i++; + } + if (post != null && post.containsKey("savetranslationlist")) { + changed = true; + } + if (changed) { + ctm.saveAsLngFile(langcfg, ctm.getScratchFile(lngfile), origTrans); + } + } + prop.put("textlist", i); + return prop; + } catch (IOException ex) { + ConcurrentLog.logException(ex); + } + return null; + } +} diff --git a/source/net/yacy/crawler/CrawlSwitchboard.java b/source/net/yacy/crawler/CrawlSwitchboard.java index 9e3b18081..f6e5c1619 100644 --- a/source/net/yacy/crawler/CrawlSwitchboard.java +++ b/source/net/yacy/crawler/CrawlSwitchboard.java @@ -521,8 +521,8 @@ public final class CrawlSwitchboard { CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE), -1, true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow, - false, - false, + false, // indexText + false, // indexMedia true, false, -1, false, true, CrawlProfile.MATCH_NEVER_STRING, diff --git a/source/net/yacy/data/Translator.java b/source/net/yacy/data/Translator.java index 97507c091..6f87bd79f 100644 --- a/source/net/yacy/data/Translator.java +++ b/source/net/yacy/data/Translator.java @@ -107,7 +107,7 @@ public class Translator { * @param translationFile the File, which contains the Lists * @return a HashMap, which contains for each File a HashMap with translations. */ - public static Map> loadTranslationsLists(final File translationFile) { + public Map> loadTranslationsLists(final File translationFile) { final Map> lists = new HashMap>(); //list of translationLists for different files. Map translationList = new LinkedHashMap(); //current Translation Table (maintaining input order) @@ -186,7 +186,7 @@ public class Translator { return true; } - public static boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final File translationFile, final String extensions){ + public boolean translateFiles(final File sourceDir, final File destDir, final File baseDir, final File translationFile, final String extensions){ return translateFiles(sourceDir, destDir, baseDir, loadTranslationsLists(translationFile), extensions); } @@ -219,7 +219,7 @@ public class Translator { return true; } - public static boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){ + public boolean translateFilesRecursive(final File sourceDir, final File destDir, final File translationFile, final String extensions, final String notdir){ final List dirList=FileUtils.getDirsRecursive(sourceDir, notdir); dirList.add(sourceDir); for (final File file : dirList) { @@ -248,7 +248,7 @@ public class Translator { return map; } - public static boolean changeLang(final serverSwitch env, final File langPath, final String lang) { + public boolean changeLang(final serverSwitch env, final File langPath, final String lang) { boolean ret = false; if ("default".equals(lang) || "default.lng".equals(lang)) { @@ -257,14 +257,10 @@ public class Translator { } else { final String htRootPath = env.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT); final File sourceDir = new File(env.getAppPath(), htRootPath); - final File destDir = new File(env.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut - // .lng - //File destDir = new File(env.getRootPath(), htRootPath + "/locale/" + lang.substring(0, lang.length() - 4));// cut - // .lng + final File destDir = new File(env.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang.substring(0, lang.length() - 4));// cut .lng final File translationFile = new File(langPath, lang); - //if (translator.translateFiles(sourceDir, destDir, translationFile, "html")) { - if (Translator.translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) { + if (translateFilesRecursive(sourceDir, destDir, translationFile, "html,template,inc", "locale")) { env.setConfig("locale.language", lang.substring(0, lang.length() - 4)); Formatter.setLocale(env.getConfig("locale.language", "en")); try { diff --git a/source/net/yacy/utils/translation/CreateTranslationMasters.java b/source/net/yacy/utils/translation/CreateTranslationMasters.java index 67aca5df1..187984447 100644 --- a/source/net/yacy/utils/translation/CreateTranslationMasters.java +++ b/source/net/yacy/utils/translation/CreateTranslationMasters.java @@ -89,7 +89,7 @@ public class CreateTranslationMasters extends TranslatorXliff { public void createMasterTranslationLists(File masterOutputFile) throws IOException { Map> xliffTrans; if (masterOutputFile.exists()) // if file exists, conserve existing master content (may be updated by external tool) - xliffTrans = TranslatorXliff.loadTranslationsListsFromXliff(masterOutputFile); + xliffTrans = loadTranslationsListsFromXliff(masterOutputFile); else xliffTrans = new TreeMap>(); @@ -97,7 +97,7 @@ public class CreateTranslationMasters extends TranslatorXliff { for (String filename : lngFiles) { // load translation list ConcurrentLog.info("TRANSLATOR", "include translation file " + filename); - Map> origTrans = Translator.loadTranslationsLists(new File("locales", filename)); + Map> origTrans = loadTranslationsLists(new File("locales", filename)); for (String transfilename : origTrans.keySet()) { // get translation filename File checkfile = new File("htroot", transfilename); @@ -154,10 +154,10 @@ public class CreateTranslationMasters extends TranslatorXliff { public Map> joinMasterTranslationLists(File xlifmaster, File lngfile) throws IOException { final String filename = lngfile.getName(); - Map> xliffTrans = TranslatorXliff.loadTranslationsListsFromXliff(xlifmaster); + Map> xliffTrans = loadTranslationsListsFromXliff(xlifmaster); // load translation list System.out.println("join into master translation file " + filename); - Map> origTrans = Translator.loadTranslationsLists(lngfile); + Map> origTrans = loadTranslationsLists(lngfile); for (String transfilename : origTrans.keySet()) { // get translation filename // compare translation list diff --git a/source/net/yacy/utils/translation/ListNonTranslatedFiles.java b/source/net/yacy/utils/translation/ListNonTranslatedFiles.java index 69d317ced..075a93f88 100755 --- a/source/net/yacy/utils/translation/ListNonTranslatedFiles.java +++ b/source/net/yacy/utils/translation/ListNonTranslatedFiles.java @@ -87,7 +87,7 @@ public class ListNonTranslatedFiles extends TranslatorUtil { + translationFile); try { - Set translatedRelativePaths = Translator.loadTranslationsLists(translationFile).keySet(); + Set translatedRelativePaths = new Translator().loadTranslationsLists(translationFile).keySet(); List srcFiles = FileUtils.getFilesRecursive(sourceDir, excludedDir, fileFilter); diff --git a/source/net/yacy/utils/translation/TranslateAll.java b/source/net/yacy/utils/translation/TranslateAll.java index e78f70b5d..f5162aece 100755 --- a/source/net/yacy/utils/translation/TranslateAll.java +++ b/source/net/yacy/utils/translation/TranslateAll.java @@ -86,7 +86,7 @@ public class TranslateAll extends TranslatorUtil { File localeDestDir = new File(destDir, localeCode); localeDestDir.mkdirs(); - Translator.translateFilesRecursive(sourceDir, localeDestDir, + new Translator().translateFilesRecursive(sourceDir, localeDestDir, translationFile, extensions, "locale"); } } diff --git a/source/net/yacy/utils/translation/TranslateAllToOneLanguage.java b/source/net/yacy/utils/translation/TranslateAllToOneLanguage.java index e12f28e31..f6a2d00f5 100755 --- a/source/net/yacy/utils/translation/TranslateAllToOneLanguage.java +++ b/source/net/yacy/utils/translation/TranslateAllToOneLanguage.java @@ -66,7 +66,7 @@ public class TranslateAllToOneLanguage extends TranslatorUtil { + translationFile); try { - Translator.translateFilesRecursive(sourceDir, destDir, + new Translator().translateFilesRecursive(sourceDir, destDir, translationFile, extensions, "locale"); } finally { ConcurrentLog.shutdown(); diff --git a/source/net/yacy/utils/translation/TranslatorXliff.java b/source/net/yacy/utils/translation/TranslatorXliff.java index dbef668f3..ddfd45453 100644 --- a/source/net/yacy/utils/translation/TranslatorXliff.java +++ b/source/net/yacy/utils/translation/TranslatorXliff.java @@ -44,6 +44,7 @@ import javax.xml.bind.Unmarshaller; import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.Translator; +import net.yacy.search.Switchboard; import org.oasis.xliff.core_12.Body; import org.oasis.xliff.core_12.Target; @@ -66,7 +67,7 @@ public class TranslatorXliff extends Translator { * @return a HashMap, which contains for each File a HashMap with * translations. */ - public static Map> loadTranslationsListsFromXliff(final File xliffFile) { + public Map> loadTranslationsListsFromXliff(final File xliffFile) { final Map> lngLists = new TreeMap>(); //list of translationLists for different files. /** @@ -144,11 +145,13 @@ public class TranslatorXliff extends Translator { * @param xliffFile * @return translatio map */ - public static Map> loadTranslationsLists(final File xliffFile) { + @Override + public Map> loadTranslationsLists(final File xliffFile) { + File locallng = getScratchFile(xliffFile); if (xliffFile.getName().toLowerCase().endsWith(".xlf") || xliffFile.getName().toLowerCase().endsWith(".xliff")) { - return loadTranslationsListsFromXliff(xliffFile); + return locallng.exists() ? loadTranslationsListsFromXliff(locallng) : loadTranslationsListsFromXliff(xliffFile); } else { - return Translator.loadTranslationsLists(xliffFile); + return locallng.exists() ? super.loadTranslationsLists(locallng) : super.loadTranslationsLists(xliffFile); } } @@ -317,4 +320,21 @@ public class TranslatorXliff extends Translator { } return s; } + + /** + * Get the path to a work/scratch file in the DATA/LOCALE directory with the + * same name as given in the langPath + * + * @param langFile the path with filename to the language file + * @return a path to DATA/LOCALE/langFile.filename() + */ + public File getScratchFile(final File langFile) { + if (Switchboard.getSwitchboard() != null) { // for debug and testing were switchboard is null + File f = Switchboard.getSwitchboard().getDataPath("locale.translated_html", "DATA/LOCALE"); + f = new File(f.getParentFile(), langFile.getName()); + return f; + } else { + return langFile; + } + } } diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 361f18c54..4e26f6dc4 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -46,7 +46,6 @@ import net.yacy.cora.protocol.TimeoutRequest; import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.sorting.Array; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.Translator; import net.yacy.gui.YaCyApp; import net.yacy.gui.framework.Browser; import net.yacy.http.Jetty9HttpServerImpl; @@ -67,6 +66,7 @@ import net.yacy.cora.protocol.ConnectionInfo; import net.yacy.crawler.retrieval.Response; import net.yacy.peers.Seed; import net.yacy.server.serverSwitch; +import net.yacy.utils.translation.TranslatorXliff; /** @@ -362,7 +362,7 @@ public final class yacy { if (currentRev == null || !currentRev.equals(sb.getConfig(Seed.VERSION, ""))) try { //is this another version?! final File sourceDir = new File(sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT)); final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang); - if (Translator.translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it + if (new TranslatorXliff().translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it //write the new Versionnumber final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version")))); bw.write(sb.getConfig(Seed.VERSION, "Error getting Version")); diff --git a/test/java/net/yacy/utils/translation/TranslatorXliffTest.java b/test/java/net/yacy/utils/translation/TranslatorXliffTest.java index 6534e4852..bc5a3ecdb 100644 --- a/test/java/net/yacy/utils/translation/TranslatorXliffTest.java +++ b/test/java/net/yacy/utils/translation/TranslatorXliffTest.java @@ -31,7 +31,7 @@ public class TranslatorXliffTest { for (String filename : lngFiles) { // load translation list System.out.println("Test translation file " + filename); - Map> origTrans = Translator.loadTranslationsLists(new File("locales", filename)); + Map> origTrans = new Translator().loadTranslationsLists(new File("locales", filename)); TranslatorXliff txlif = new TranslatorXliff(); // save as xliff file