From: Felix Dörre Date: Thu, 2 Jun 2016 08:57:49 +0000 (+0200) Subject: Adding source to collect all localisation texts X-Git-Url: https://code.wpia.club/?p=gigi.git;a=commitdiff_plain;h=425e5d8df90bf098df444b26f39fed409bfa75b9;ds=sidebyside Adding source to collect all localisation texts Change-Id: I90632cf4ffaf10bfdc42ec70f8531c3357702866 --- diff --git a/.classpath b/.classpath index cab9630e..0a18b583 100644 --- a/.classpath +++ b/.classpath @@ -7,7 +7,7 @@ - + diff --git a/build.xml b/build.xml index 20c6f496..39854122 100644 --- a/build.xml +++ b/build.xml @@ -30,6 +30,9 @@ + + + @@ -101,8 +104,26 @@ includeantruntime="false" source="${source}" target="${target}"> + + + + + + + + + + + + + + + + + diff --git a/util-testing/org/cacert/gigi/localisation/FileIterable.java b/util-testing/org/cacert/gigi/localisation/FileIterable.java new file mode 100644 index 00000000..3c23c7f4 --- /dev/null +++ b/util-testing/org/cacert/gigi/localisation/FileIterable.java @@ -0,0 +1,50 @@ +package org.cacert.gigi.localisation; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOError; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Iterator; + +public class FileIterable implements Iterable { + File f; + + public FileIterable(File f) { + this.f = f; + } + + @Override + public Iterator iterator() { + try { + return new Iterator() { + BufferedReader br = new BufferedReader(new InputStreamReader( + new FileInputStream(f), "UTF-8")); + String s = null; + private void getNext() { + if (s == null) { + try { + s = br.readLine(); + } catch (IOException e) { + throw new IOError(e); + } + } + } + @Override + public boolean hasNext() { + getNext(); + return s != null; + } + + @Override + public String next() { + String out = s; + s = null; + return out; + } + }; + } catch (IOException e) { + throw new IOError(e); + } + } +} diff --git a/util-testing/org/cacert/gigi/localisation/TaintSource.java b/util-testing/org/cacert/gigi/localisation/TaintSource.java new file mode 100644 index 00000000..84380e9e --- /dev/null +++ b/util-testing/org/cacert/gigi/localisation/TaintSource.java @@ -0,0 +1,119 @@ +package org.cacert.gigi.localisation; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.eclipse.jdt.internal.compiler.lookup.MethodBinding; + +public class TaintSource { + private String pack, cls, meth; + private int tgt; + private TaintSource maskOnly; + + public TaintSource(String pack, String cls, String meth, int tgt) { + this(pack, cls, meth, tgt, null); + } + public TaintSource(String pack, String cls, String meth, int tgt, + TaintSource maskOnly) { + this.pack = pack; + this.cls = cls; + this.meth = meth; + this.tgt = tgt; + this.maskOnly = maskOnly; + + } + public TaintSource(MethodBinding mb) { + pack = new String(mb.declaringClass.qualifiedPackageName()); + cls = new String(mb.declaringClass.qualifiedSourceName()); + meth = new String(mb.readableName()); + } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((cls == null) ? 0 : cls.hashCode()); + result = prime * result + ((meth == null) ? 0 : meth.hashCode()); + result = prime * result + ((pack == null) ? 0 : pack.hashCode()); + return result; + } + @Override + public String toString() { + StringBuffer res = new StringBuffer(); + res.append("new TaintSource("); + res.append("\"" + pack + "\","); + res.append("\"" + cls + "\","); + res.append("\"" + meth + "\",0);"); + return res.toString(); + } + public String toConfLine() { + return pack + " " + cls + "." + meth + "," + tgt + + (maskOnly == null ? "" : "=>" + maskOnly.toConfLine()); + } + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + TaintSource other = (TaintSource) obj; + if (cls == null) { + if (other.cls != null) { + return false; + } + } else if (!cls.equals(other.cls)) { + return false; + } + if (pack == null) { + if (other.pack != null) { + return false; + } + } else if (!pack.equals(other.pack)) { + return false; + } + if (meth == null) { + if (other.meth != null) { + return false; + } + } else if (!meth.equals(other.meth)) { + return false; + } + return true; + } + public static TaintSource parseTaint(String confline) { + // Pattern matches "Taint-lines" + // first part is package name up to space (may not include space or equals sign) + // second part is Class name [with inner class name] (may not include "=" but may include ".") + // third part is method name including params (may not include "=" or ".") + // fourth is index of tainted argument (seperated by "," from the rest) + Pattern p = Pattern + .compile("^([^= ]*) ([^=]*)\\.([^=.]*\\([^)]*\\)),([0-9]+)"); + Matcher m = p.matcher(confline); + if (!m.find()) { + throw new Error(confline); + } + String pack = m.group(1); + String cls = m.group(2); + String meth = m.group(3); + int tgt = Integer.parseInt(m.group(4)); + TaintSource mask = null; + if (m.end() != confline.length()) { + String s = confline.substring(m.end(), m.end() + 2); + if (!s.equals("=>")) { + throw new Error("malformed"); + } + mask = parseTaint(confline.substring(m.end() + 2)); + } + return new TaintSource(pack, cls, meth, tgt, mask); + } + public TaintSource getMaskOnly() { + return maskOnly; + } + public int getTgt() { + return tgt; + } + +} diff --git a/util-testing/org/cacert/gigi/localisation/TranslationCollectingVisitor.java b/util-testing/org/cacert/gigi/localisation/TranslationCollectingVisitor.java new file mode 100644 index 00000000..472789d8 --- /dev/null +++ b/util-testing/org/cacert/gigi/localisation/TranslationCollectingVisitor.java @@ -0,0 +1,206 @@ +package org.cacert.gigi.localisation; +import java.lang.reflect.Method; + +import org.eclipse.jdt.internal.compiler.ASTVisitor; +import org.eclipse.jdt.internal.compiler.ast.AllocationExpression; +import org.eclipse.jdt.internal.compiler.ast.CompilationUnitDeclaration; +import org.eclipse.jdt.internal.compiler.ast.ConditionalExpression; +import org.eclipse.jdt.internal.compiler.ast.ConstructorDeclaration; +import org.eclipse.jdt.internal.compiler.ast.ExplicitConstructorCall; +import org.eclipse.jdt.internal.compiler.ast.Expression; +import org.eclipse.jdt.internal.compiler.ast.MessageSend; +import org.eclipse.jdt.internal.compiler.ast.MethodDeclaration; +import org.eclipse.jdt.internal.compiler.ast.NullLiteral; +import org.eclipse.jdt.internal.compiler.ast.StringLiteral; +import org.eclipse.jdt.internal.compiler.lookup.BlockScope; +import org.eclipse.jdt.internal.compiler.lookup.ClassScope; +import org.eclipse.jdt.internal.compiler.lookup.MethodBinding; +import org.eclipse.jdt.internal.compiler.util.Util; + +public final class TranslationCollectingVisitor extends ASTVisitor { + MethodBinding cm; + private CompilationUnitDeclaration unit; + TaintSource[] ts; + private TranslationCollector translationCollector; + + public TranslationCollectingVisitor(CompilationUnitDeclaration unit, + TaintSource[] target, TranslationCollector c) { + this.unit = unit; + ts = target; + this.translationCollector = c; + } + @Override + public boolean visit(MethodDeclaration methodDeclaration, + org.eclipse.jdt.internal.compiler.lookup.ClassScope scope) { + cm = methodDeclaration.binding; + return true; + } + @Override + public void endVisit(MethodDeclaration methodDeclaration, + org.eclipse.jdt.internal.compiler.lookup.ClassScope scope) { + cm = null; + } + @Override + public boolean visit(ConstructorDeclaration constructorDeclaration, + ClassScope scope) { + cm = constructorDeclaration.binding; + return super.visit(constructorDeclaration, scope); + } + @Override + public void endVisit(ConstructorDeclaration constructorDeclaration, + ClassScope scope) { + cm = null; + } + @Override + public boolean visit(AllocationExpression allocationExpression, + BlockScope scope) { + TaintSource test = new TaintSource(allocationExpression.binding); + for (TaintSource taintSource : ts) { + if (taintSource.equals(test)) { + check(null, scope, + allocationExpression.arguments[taintSource.getTgt()], + allocationExpression.toString()); + return true; + } + } + return super.visit(allocationExpression, scope); + } + @Override + public boolean visit(ExplicitConstructorCall explicitConstructor, + BlockScope scope) { + + TaintSource t = new TaintSource(explicitConstructor.binding); + + for (TaintSource t0 : ts) { + if (t0.equals(t)) { + Expression[] ags = explicitConstructor.arguments; + if (ags == null) { + System.out.println(explicitConstructor); + return true; + } + Expression e = ags[t0.getTgt()]; + check(null, scope, e, explicitConstructor.toString()); + break; + } + } + return super.visit(explicitConstructor, scope); + } + + @Override + public boolean visit( + org.eclipse.jdt.internal.compiler.ast.MessageSend call, + org.eclipse.jdt.internal.compiler.lookup.BlockScope scope) { + if (call.binding == null) { + System.out.println("Unbound:" + call + " in " + call.sourceStart()); + return true; + } + //System.out.println("Message"); + TaintSource t = new TaintSource(call.binding); + + for (TaintSource t0 : ts) { + if (t0.equals(t)) { + Expression[] ags = call.arguments; + if (ags == null) { + System.out.println(call); + return true; + } + Expression e = ags[t0.getTgt()]; + check(call, scope, e, call.toString()); + break; + } + } + return true; + } + private void check(org.eclipse.jdt.internal.compiler.ast.MessageSend call, + org.eclipse.jdt.internal.compiler.lookup.BlockScope scope, + Expression e, String caller) { + if (e instanceof StringLiteral) { + int[] lineEnds = null; + int lineNumber = Util.getLineNumber( + e.sourceStart, + lineEnds = unit.compilationResult + .getLineSeparatorPositions(), 0, + lineEnds.length - 1); + + String content = new String(((StringLiteral) e).source()); + translationCollector.add( + content, + new String(unit.compilationResult.fileName) + .substring(translationCollector.base + .getAbsolutePath().length() + 1) + + ":" + lineNumber); + return; + } + + if (e instanceof NullLiteral) { + return; + } + + if (e instanceof MessageSend) { + MessageSend m2 = (MessageSend) e; + TaintSource ts = new TaintSource(m2.binding); + if (ts.equals(new TaintSource("org.cacert.gigi.pages", "Page", + "getTitle()", 0))) { + return; + } + if (m2.receiver.resolvedType.isCompatibleWith(scope + .getJavaLangEnum())) { + testEnum(m2.receiver, m2.binding); + System.out.println("ENUM-SRC: !" + m2.receiver); + } + } + if (e.resolvedType.isCompatibleWith(scope.getJavaLangEnum())) { + // TODO ? + System.out.println("ENUM-Not-Hanled"); + } + + TaintSource b = cm == null ? null : new TaintSource(cm); + for (TaintSource taintSource : ts) { + if (taintSource.equals(b) + || (taintSource.getMaskOnly() != null && taintSource + .getMaskOnly().equals(b))) { + return; + } + } + if (e instanceof ConditionalExpression) { + check(call, scope, ((ConditionalExpression) e).valueIfFalse, caller); + check(call, scope, ((ConditionalExpression) e).valueIfTrue, caller); + return; + } + + System.out.println(); + + System.out.println(new String(scope.enclosingClassScope() + .referenceType().compilationResult.fileName)); + System.out.println("Cannot Handle: " + e + " in " + + (call == null ? "constructor" : call.sourceStart) + " => " + + caller); + System.out.println(e.getClass()); + System.out.println("To ignore: " + b.toConfLine()); + } + private void testEnum(Expression e, MethodBinding binding) { + if (binding.parameters.length != 0) { + System.out.println("ERROR: meth"); + return; + } + System.out.println(e.resolvedType.getClass()); + String s2 = new String(e.resolvedType.qualifiedPackageName()) + + "." + + (new String(e.resolvedType.qualifiedSourceName()).replace( + '.', '$')); + try { + Class c = Class.forName(s2); + Enum[] e1 = (Enum[]) c.getMethod("values").invoke(null); + Method m = c.getMethod(new String(binding.selector)); + for (int j = 0; j < e1.length; j++) { + System.out.println(m.invoke(e1[j])); + } + } catch (ClassNotFoundException e1) { + e1.printStackTrace(); + } catch (ReflectiveOperationException e1) { + e1.printStackTrace(); + } + System.out.println("ENUM-done: " + e + "!"); + return; + } +} diff --git a/util-testing/org/cacert/gigi/localisation/TranslationCollector.java b/util-testing/org/cacert/gigi/localisation/TranslationCollector.java new file mode 100644 index 00000000..2b8ca7e7 --- /dev/null +++ b/util-testing/org/cacert/gigi/localisation/TranslationCollector.java @@ -0,0 +1,267 @@ +package org.cacert.gigi.localisation; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.TreeSet; + +import org.cacert.gigi.output.template.Template; +import org.eclipse.jdt.core.compiler.CategorizedProblem; +import org.eclipse.jdt.internal.compiler.CompilationResult; +import org.eclipse.jdt.internal.compiler.ast.CompilationUnitDeclaration; +import org.eclipse.jdt.internal.compiler.ast.TypeDeclaration; +import org.eclipse.jdt.internal.compiler.batch.CompilationUnit; +import org.eclipse.jdt.internal.compiler.batch.FileSystem; +import org.eclipse.jdt.internal.compiler.batch.Main; +import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants; +import org.eclipse.jdt.internal.compiler.env.AccessRestriction; +import org.eclipse.jdt.internal.compiler.env.IBinaryType; +import org.eclipse.jdt.internal.compiler.env.ICompilationUnit; +import org.eclipse.jdt.internal.compiler.env.ISourceType; +import org.eclipse.jdt.internal.compiler.impl.CompilerOptions; +import org.eclipse.jdt.internal.compiler.impl.ITypeRequestor; +import org.eclipse.jdt.internal.compiler.lookup.LookupEnvironment; +import org.eclipse.jdt.internal.compiler.lookup.PackageBinding; +import org.eclipse.jdt.internal.compiler.parser.Parser; +import org.eclipse.jdt.internal.compiler.problem.ProblemReporter; + +public class TranslationCollector { + static class TranslationEntry implements Comparable { + String text; + String occur1; + List occur; + public TranslationEntry(String text, String occur) { + this.text = text; + occur1 = occur; + } + public List getOccur() { + if (occur == null) { + return Arrays.asList(occur1); + } + return occur; + } + public void add(String t) { + if (occur == null) { + occur = new ArrayList<>(Arrays.asList(occur1)); + } + occur.add(t); + } + @Override + public int compareTo(TranslationEntry o) { + int i = occur1.compareTo(o.occur1); + if (i != 0) { + return i; + } + + return text.compareTo(o.text); + } + } + + private HashMap translations = new HashMap<>(); + + public final File base; + + public TranslationCollector(File base, File conf) { + this.base = base; + taint = new LinkedList<>(); + for (String s : new FileIterable(conf)) { + taint.add(TaintSource.parseTaint(s)); + } + } + public void run(File out) throws IOException { + scanTemplates(); + scanCode(taint); + + System.out + .println("Total Translatable Strings: " + translations.size()); + TreeSet trs = new TreeSet<>(translations.values()); + writePOFile(out, trs); + + } + + public void add(String text, String line) { + TranslationEntry i = translations.get(text); + if (i == null) { + translations.put(text, new TranslationEntry(text, line)); + return; + } + i.add(line); + } + + private void scanCode(LinkedList taint) throws Error { + PrintWriter out = new PrintWriter(System.out); + Main m = new Main(out, out, false, null, null); + File[] fs = recurse( + new File(new File(new File(base, "src"), "org"), "cacert"), + new LinkedList(), ".java").toArray(new File[0]); + String[] t = new String[fs.length + 3]; + t[0] = "-cp"; + t[1] = new File(base, "bin").getAbsolutePath(); + t[2] = "-7"; + for (int i = 0; i < fs.length; i++) { + t[i + 3] = fs[i].getAbsolutePath(); + } + m.configure(t); + FileSystem environment = m.getLibraryAccess(); + CompilerOptions compilerOptions = new CompilerOptions(m.options);//new HashMap<>());//m.options); + compilerOptions.performMethodsFullRecovery = false; + compilerOptions.performStatementsRecovery = false; + //check + compilerOptions.sourceLevel = ClassFileConstants.JDK1_7; + compilerOptions.complianceLevel = ClassFileConstants.JDK1_7; + compilerOptions.originalComplianceLevel = ClassFileConstants.JDK1_7; + + ProblemReporter pr = new ProblemReporter(m.getHandlingPolicy(), + compilerOptions, m.getProblemFactory()); + ITypeRequestor tr = new ITypeRequestor() { + + @Override + public void accept(ISourceType[] sourceType, + PackageBinding packageBinding, + AccessRestriction accessRestriction) { + throw new IllegalStateException("source type not implemented"); + } + + @Override + public void accept(IBinaryType binaryType, + PackageBinding packageBinding, + AccessRestriction accessRestriction) { + le.createBinaryTypeFrom(binaryType, packageBinding, + accessRestriction); + } + + @Override + public void accept(ICompilationUnit unit, + AccessRestriction accessRestriction) { + throw new IllegalStateException( + "compilation unit not implemented"); + } + }; + le = new LookupEnvironment(tr, compilerOptions, pr, environment); + Parser parser = new Parser(pr, + compilerOptions.parseLiteralExpressionsAsConstants); + CompilationUnit[] sourceUnits = m.getCompilationUnits(); + CompilationUnitDeclaration[] parsedUnits = new CompilationUnitDeclaration[sourceUnits.length]; + for (int i = 0; i < parsedUnits.length; i++) { + + CompilationResult unitResult = new CompilationResult( + sourceUnits[i], i, parsedUnits.length, + compilerOptions.maxProblemsPerUnit); + CompilationUnitDeclaration parsedUnit = parser.parse( + sourceUnits[i], unitResult); + le.buildTypeBindings(parsedUnit, null /*no access restriction*/); + parsedUnits[i] = parsedUnit; + } + le.completeTypeBindings(); + for (int i = 0; i < parsedUnits.length; i++) { + CompilationUnitDeclaration parsedUnit = parsedUnits[i]; + + parser.getMethodBodies(parsedUnit); + parsedUnit.scope.faultInTypes(); + parsedUnit.scope.verifyMethods(le.methodVerifier()); + parsedUnit.resolve(); + } + for (int i = 0; i < parsedUnits.length; i++) { + CompilationUnitDeclaration parsedUnit = parsedUnits[i]; + if (parsedUnit.compilationResult.problems != null) { + int err = 0; + for (int c = 0; c < parsedUnit.compilationResult.problemCount; c++) { + CategorizedProblem problem = parsedUnit.compilationResult.problems[c]; + if (problem.isError()) { + err++; + } + if (OUTPUT_WARNINGS || problem.isError()) { + System.out.println(problem); + StringBuilder prob = new StringBuilder(); + prob.append(parsedUnit.compilationResult.fileName); + prob.append(":"); + prob.append(problem.getSourceLineNumber()); + System.out.println(prob.toString()); + } + } + if (err > 0) { + throw new Error(); + } + } + + if (parsedUnit.types == null) { + System.out.println("No types"); + + } else { + TranslationCollectingVisitor v = new TranslationCollectingVisitor( + parsedUnit, + taint.toArray(new TaintSource[taint.size()]), this); + for (TypeDeclaration td : parsedUnit.types) { + td.traverse(v, td.scope); + } + } + parsedUnits[i] = parsedUnit; + } + } + private void scanTemplates() throws UnsupportedEncodingException, + FileNotFoundException { + File[] ts = recurse( + new File(new File(new File(base, "src"), "org"), "cacert"), + new LinkedList(), ".templ").toArray(new File[0]); + for (File file : ts) { + Template t = new Template(new InputStreamReader( + new FileInputStream(file), "UTF-8")); + LinkedList i = new LinkedList(); + t.addTranslations(i); + for (String string : i) { + add(string, + file.getAbsolutePath().substring( + base.getAbsolutePath().length() + 1) + + ":0"); + } + } + } + + static LookupEnvironment le; + private static final boolean OUTPUT_WARNINGS = false; + + private LinkedList taint; + public static void main(String[] args) throws IOException { + new TranslationCollector(new File(args[1]), new File(args[0])) + .run(new File(args[2])); + } + + public static void writePOFile(File target, + Collection strings) throws IOException { + PrintWriter out = new PrintWriter(target); + for (TranslationEntry s : strings) { + out.print("#:"); + for (String st : s.getOccur()) { + out.print(" " + st); + } + out.println(); + out.println("msgid \"" + + s.text.replace("\\", "\\\\").replace("\"", "\\\"") + "\""); + out.println("msgstr \"\""); + out.println(); + } + out.close(); + } + + private static List recurse(File file, List toAdd, String pt) { + if (file.isDirectory()) { + for (File f : file.listFiles()) { + recurse(f, toAdd, pt); + } + } else { + if (file.getName().endsWith(pt)) { + toAdd.add(file); + } + } + return toAdd; + } +} diff --git a/util-testing/org/cacert/gigi/localisation/conf.txt b/util-testing/org/cacert/gigi/localisation/conf.txt new file mode 100644 index 00000000..f185e20f --- /dev/null +++ b/util-testing/org/cacert/gigi/localisation/conf.txt @@ -0,0 +1,19 @@ +org.cacert.gigi.localisation Language.getTranslation(String),0 +org.cacert.gigi.pages Page.translate(ServletRequest, String),1 +org.cacert.gigi.output.template Form.outputError(PrintWriter, ServletRequest, String, Object[]),2 +org.cacert.gigi Gigi.MenuBuilder.putPage(String, Page, String),2 +org.cacert.gigi Gigi.MenuBuilder.getMenu(String),0 +org.cacert.gigi GigiApiException.GigiApiException(String),0=>org.cacert.gigi GigiApiException.formatPlain(PrintWriter),0 +org.cacert.gigi.output Menu.Menu(String),0=>org.cacert.gigi.output Menu.output(PrintWriter, Language, Map),0 +org.cacert.gigi.output SimpleMenuItem.SimpleMenuItem(String,String),1=>org.cacert.gigi.output SimpleMenuItem.output(PrintWriter, Language, Map),0 +org.cacert.gigi.dbObjects Digest.Digest(String),0 +org.cacert.gigi.dbObjects Certificate.CertificateStatus.CertificateStatus(String),0 +org.cacert.gigi.pages Page.Page(String),0 +org.cacert.gigi.pages OneFormPage.OneFormPage(String, Class),0 +org.cacert.gigi.pages StaticPage.StaticPage(String, InputStream),0 +org.cacert.gigi.output.template SprintfCommand.SprintfCommand(String, List),0=>org.cacert.gigi.output.template SprintfCommand.output(PrintWriter, Language, Map),0 +org.cacert.gigi.output.template SprintfCom---invalid---mand.SprintfCommand(String),0=>org.cacert.gigi.output.template Template.parseCommand(String),0 +org.cacert.gigi.output.template TranslateCommand.TranslateCommand(String),0=>org.cacert.gigi.output.template TranslateCommand.output(PrintWriter, Language, Map),0 +org.cacert.gigi.pages.account.domain DomainOverview.DomainOverview(String),0 +org.cacert.gigi.dbObjects Group.Group(String),0 +org.cacert.gigi.output.template SprintfCommand.createSimple(String, String[]),0