Adding source to collect all localisation texts
authorFelix Dörre <felix@dogcraft.de>
Thu, 2 Jun 2016 08:57:49 +0000 (10:57 +0200)
committerFelix Dörre <felix@dogcraft.de>
Mon, 6 Jun 2016 15:18:44 +0000 (17:18 +0200)
Change-Id: I90632cf4ffaf10bfdc42ec70f8531c3357702866

.classpath
build.xml
util-testing/org/cacert/gigi/localisation/FileIterable.java [new file with mode: 0644]
util-testing/org/cacert/gigi/localisation/TaintSource.java [new file with mode: 0644]
util-testing/org/cacert/gigi/localisation/TranslationCollectingVisitor.java [new file with mode: 0644]
util-testing/org/cacert/gigi/localisation/TranslationCollector.java [new file with mode: 0644]
util-testing/org/cacert/gigi/localisation/conf.txt [new file with mode: 0644]

index cab9630e50879b23c4ea0c49a44b2121e484d828..0a18b58341bd98d5ef7dbad0422e88a808efbe65 100644 (file)
@@ -7,7 +7,7 @@
        <classpathentry kind="src" path="src"/>
        <classpathentry kind="src" path="util"/>
        <classpathentry kind="src" path="tests"/>
-       <classpathentry kind="src" path="util-testing"/>
+       <classpathentry excluding="org/cacert/gigi/locatisation/|org/cacert/gigi/localisation/" kind="src" path="util-testing"/>
        <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
        <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
        <classpathentry kind="con" path="org.eclipse.jdt.USER_LIBRARY/MySQL"/>
index 20c6f496c25d2df87373d4c5c52104d733a49cfb..3985412208147b097fcd58a24f117fc1d6769ebb 100644 (file)
--- a/build.xml
+++ b/build.xml
@@ -30,6 +30,9 @@
                <path refid="JUnit 4.libraryclasspath" />
                <pathelement location="${sqlconnector}" />
        </path>
+       <path id="cacert-gigi.test.classpath.jdt">
+               <pathelement location="${jdt}" />
+       </path>
        <target name="init">
                <mkdir dir="bin" />
                <mkdir dir="binutil" />
                        includeantruntime="false" source="${source}" target="${target}">
                        <compilerarg value="-XDignore.symbol.file"/>
                        <src path="util-testing" />
+                       <exclude name="org/cacert/gigi/localisation/**"/>
+                       <classpath refid="cacert-gigi.classpath" />
+               </javac>
+       </target>
+       <target depends="init, build-project" name="build-testing-l10n">
+               <javac encoding="UTF-8" debug="true" debuglevel="${debuglevel}" destdir="binutil-testing"
+                       includeantruntime="false" source="${source}" target="${target}">
+                       <compilerarg value="-XDignore.symbol.file"/>
+                       <src path="util-testing" />
+                       <include name="org/cacert/gigi/localisation/**"/>
                        <classpath refid="cacert-gigi.classpath" />
+                       <classpath refid="cacert-gigi.test.classpath.jdt" />
                </javac>
+               <java classname="org.cacert.gigi.localisation.TranslationCollector">
+                       <arg value="util-testing/org/cacert/gigi/localisation/conf.txt"/>
+                       <arg value="."/>
+                       <arg value="messages.po"/>
+                       <classpath refid="cacert-gigi.test.classpath" />
+                       <classpath refid="cacert-gigi.test.classpath.jdt" />
+               </java>
        </target>
 
        <target name="native">
diff --git a/util-testing/org/cacert/gigi/localisation/FileIterable.java b/util-testing/org/cacert/gigi/localisation/FileIterable.java
new file mode 100644 (file)
index 0000000..3c23c7f
--- /dev/null
@@ -0,0 +1,50 @@
+package org.cacert.gigi.localisation;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOError;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Iterator;
+
+public class FileIterable implements Iterable<String> {
+       File f;
+
+       public FileIterable(File f) {
+               this.f = f;
+       }
+
+       @Override
+       public Iterator<String> iterator() {
+               try {
+                       return new Iterator<String>() {
+                               BufferedReader br = new BufferedReader(new InputStreamReader(
+                                               new FileInputStream(f), "UTF-8"));
+                               String s = null;
+                               private void getNext() {
+                                       if (s == null) {
+                                               try {
+                                                       s = br.readLine();
+                                               } catch (IOException e) {
+                                                       throw new IOError(e);
+                                               }
+                                       }
+                               }
+                               @Override
+                               public boolean hasNext() {
+                                       getNext();
+                                       return s != null;
+                               }
+
+                               @Override
+                               public String next() {
+                                       String out = s;
+                                       s = null;
+                                       return out;
+                               }
+                       };
+               } catch (IOException e) {
+                       throw new IOError(e);
+               }
+       }
+}
diff --git a/util-testing/org/cacert/gigi/localisation/TaintSource.java b/util-testing/org/cacert/gigi/localisation/TaintSource.java
new file mode 100644 (file)
index 0000000..84380e9
--- /dev/null
@@ -0,0 +1,119 @@
+package org.cacert.gigi.localisation;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.eclipse.jdt.internal.compiler.lookup.MethodBinding;
+
+public class TaintSource {
+       private String pack, cls, meth;
+       private int tgt;
+       private TaintSource maskOnly;
+
+       public TaintSource(String pack, String cls, String meth, int tgt) {
+               this(pack, cls, meth, tgt, null);
+       }
+       public TaintSource(String pack, String cls, String meth, int tgt,
+                       TaintSource maskOnly) {
+               this.pack = pack;
+               this.cls = cls;
+               this.meth = meth;
+               this.tgt = tgt;
+               this.maskOnly = maskOnly;
+
+       }
+       public TaintSource(MethodBinding mb) {
+               pack = new String(mb.declaringClass.qualifiedPackageName());
+               cls = new String(mb.declaringClass.qualifiedSourceName());
+               meth = new String(mb.readableName());
+       }
+       @Override
+       public int hashCode() {
+               final int prime = 31;
+               int result = 1;
+               result = prime * result + ((cls == null) ? 0 : cls.hashCode());
+               result = prime * result + ((meth == null) ? 0 : meth.hashCode());
+               result = prime * result + ((pack == null) ? 0 : pack.hashCode());
+               return result;
+       }
+       @Override
+       public String toString() {
+               StringBuffer res = new StringBuffer();
+               res.append("new TaintSource(");
+               res.append("\"" + pack + "\",");
+               res.append("\"" + cls + "\",");
+               res.append("\"" + meth + "\",0);");
+               return res.toString();
+       }
+       public String toConfLine() {
+               return pack + " " + cls + "." + meth + "," + tgt
+                               + (maskOnly == null ? "" : "=>" + maskOnly.toConfLine());
+       }
+       @Override
+       public boolean equals(Object obj) {
+               if (this == obj) {
+                       return true;
+               }
+               if (obj == null) {
+                       return false;
+               }
+               if (getClass() != obj.getClass()) {
+                       return false;
+               }
+               TaintSource other = (TaintSource) obj;
+               if (cls == null) {
+                       if (other.cls != null) {
+                               return false;
+                       }
+               } else if (!cls.equals(other.cls)) {
+                       return false;
+               }
+               if (pack == null) {
+                       if (other.pack != null) {
+                               return false;
+                       }
+               } else if (!pack.equals(other.pack)) {
+                       return false;
+               }
+               if (meth == null) {
+                       if (other.meth != null) {
+                               return false;
+                       }
+               } else if (!meth.equals(other.meth)) {
+                       return false;
+               }
+               return true;
+       }
+       public static TaintSource parseTaint(String confline) {
+               // Pattern matches "Taint-lines"
+               // first part is package name up to space (may not include space or equals sign)
+               // second part is Class name [with inner class name] (may not include "=" but may include ".")
+               // third part is method name including params (may not include "=" or ".")
+               // fourth is index of tainted argument (seperated by "," from the rest)
+               Pattern p = Pattern
+                               .compile("^([^= ]*) ([^=]*)\\.([^=.]*\\([^)]*\\)),([0-9]+)");
+               Matcher m = p.matcher(confline);
+               if (!m.find()) {
+                       throw new Error(confline);
+               }
+               String pack = m.group(1);
+               String cls = m.group(2);
+               String meth = m.group(3);
+               int tgt = Integer.parseInt(m.group(4));
+               TaintSource mask = null;
+               if (m.end() != confline.length()) {
+                       String s = confline.substring(m.end(), m.end() + 2);
+                       if (!s.equals("=>")) {
+                               throw new Error("malformed");
+                       }
+                       mask = parseTaint(confline.substring(m.end() + 2));
+               }
+               return new TaintSource(pack, cls, meth, tgt, mask);
+       }
+       public TaintSource getMaskOnly() {
+               return maskOnly;
+       }
+       public int getTgt() {
+               return tgt;
+       }
+
+}
diff --git a/util-testing/org/cacert/gigi/localisation/TranslationCollectingVisitor.java b/util-testing/org/cacert/gigi/localisation/TranslationCollectingVisitor.java
new file mode 100644 (file)
index 0000000..472789d
--- /dev/null
@@ -0,0 +1,206 @@
+package org.cacert.gigi.localisation;
+import java.lang.reflect.Method;
+
+import org.eclipse.jdt.internal.compiler.ASTVisitor;
+import org.eclipse.jdt.internal.compiler.ast.AllocationExpression;
+import org.eclipse.jdt.internal.compiler.ast.CompilationUnitDeclaration;
+import org.eclipse.jdt.internal.compiler.ast.ConditionalExpression;
+import org.eclipse.jdt.internal.compiler.ast.ConstructorDeclaration;
+import org.eclipse.jdt.internal.compiler.ast.ExplicitConstructorCall;
+import org.eclipse.jdt.internal.compiler.ast.Expression;
+import org.eclipse.jdt.internal.compiler.ast.MessageSend;
+import org.eclipse.jdt.internal.compiler.ast.MethodDeclaration;
+import org.eclipse.jdt.internal.compiler.ast.NullLiteral;
+import org.eclipse.jdt.internal.compiler.ast.StringLiteral;
+import org.eclipse.jdt.internal.compiler.lookup.BlockScope;
+import org.eclipse.jdt.internal.compiler.lookup.ClassScope;
+import org.eclipse.jdt.internal.compiler.lookup.MethodBinding;
+import org.eclipse.jdt.internal.compiler.util.Util;
+
+public final class TranslationCollectingVisitor extends ASTVisitor {
+       MethodBinding cm;
+       private CompilationUnitDeclaration unit;
+       TaintSource[] ts;
+       private TranslationCollector translationCollector;
+
+       public TranslationCollectingVisitor(CompilationUnitDeclaration unit,
+                       TaintSource[] target, TranslationCollector c) {
+               this.unit = unit;
+               ts = target;
+               this.translationCollector = c;
+       }
+       @Override
+       public boolean visit(MethodDeclaration methodDeclaration,
+                       org.eclipse.jdt.internal.compiler.lookup.ClassScope scope) {
+               cm = methodDeclaration.binding;
+               return true;
+       }
+       @Override
+       public void endVisit(MethodDeclaration methodDeclaration,
+                       org.eclipse.jdt.internal.compiler.lookup.ClassScope scope) {
+               cm = null;
+       }
+       @Override
+       public boolean visit(ConstructorDeclaration constructorDeclaration,
+                       ClassScope scope) {
+               cm = constructorDeclaration.binding;
+               return super.visit(constructorDeclaration, scope);
+       }
+       @Override
+       public void endVisit(ConstructorDeclaration constructorDeclaration,
+                       ClassScope scope) {
+               cm = null;
+       }
+       @Override
+       public boolean visit(AllocationExpression allocationExpression,
+                       BlockScope scope) {
+               TaintSource test = new TaintSource(allocationExpression.binding);
+               for (TaintSource taintSource : ts) {
+                       if (taintSource.equals(test)) {
+                               check(null, scope,
+                                               allocationExpression.arguments[taintSource.getTgt()],
+                                               allocationExpression.toString());
+                               return true;
+                       }
+               }
+               return super.visit(allocationExpression, scope);
+       }
+       @Override
+       public boolean visit(ExplicitConstructorCall explicitConstructor,
+                       BlockScope scope) {
+
+               TaintSource t = new TaintSource(explicitConstructor.binding);
+
+               for (TaintSource t0 : ts) {
+                       if (t0.equals(t)) {
+                               Expression[] ags = explicitConstructor.arguments;
+                               if (ags == null) {
+                                       System.out.println(explicitConstructor);
+                                       return true;
+                               }
+                               Expression e = ags[t0.getTgt()];
+                               check(null, scope, e, explicitConstructor.toString());
+                               break;
+                       }
+               }
+               return super.visit(explicitConstructor, scope);
+       }
+
+       @Override
+       public boolean visit(
+                       org.eclipse.jdt.internal.compiler.ast.MessageSend call,
+                       org.eclipse.jdt.internal.compiler.lookup.BlockScope scope) {
+               if (call.binding == null) {
+                       System.out.println("Unbound:" + call + " in " + call.sourceStart());
+                       return true;
+               }
+               //System.out.println("Message");
+               TaintSource t = new TaintSource(call.binding);
+
+               for (TaintSource t0 : ts) {
+                       if (t0.equals(t)) {
+                               Expression[] ags = call.arguments;
+                               if (ags == null) {
+                                       System.out.println(call);
+                                       return true;
+                               }
+                               Expression e = ags[t0.getTgt()];
+                               check(call, scope, e, call.toString());
+                               break;
+                       }
+               }
+               return true;
+       }
+       private void check(org.eclipse.jdt.internal.compiler.ast.MessageSend call,
+                       org.eclipse.jdt.internal.compiler.lookup.BlockScope scope,
+                       Expression e, String caller) {
+               if (e instanceof StringLiteral) {
+                       int[] lineEnds = null;
+                       int lineNumber = Util.getLineNumber(
+                                       e.sourceStart,
+                                       lineEnds = unit.compilationResult
+                                                       .getLineSeparatorPositions(), 0,
+                                       lineEnds.length - 1);
+
+                       String content = new String(((StringLiteral) e).source());
+                       translationCollector.add(
+                                       content,
+                                       new String(unit.compilationResult.fileName)
+                                                       .substring(translationCollector.base
+                                                                       .getAbsolutePath().length() + 1)
+                                                       + ":" + lineNumber);
+                       return;
+               }
+
+               if (e instanceof NullLiteral) {
+                       return;
+               }
+
+               if (e instanceof MessageSend) {
+                       MessageSend m2 = (MessageSend) e;
+                       TaintSource ts = new TaintSource(m2.binding);
+                       if (ts.equals(new TaintSource("org.cacert.gigi.pages", "Page",
+                                       "getTitle()", 0))) {
+                               return;
+                       }
+                       if (m2.receiver.resolvedType.isCompatibleWith(scope
+                                       .getJavaLangEnum())) {
+                               testEnum(m2.receiver, m2.binding);
+                               System.out.println("ENUM-SRC: !" + m2.receiver);
+                       }
+               }
+               if (e.resolvedType.isCompatibleWith(scope.getJavaLangEnum())) {
+                       // TODO ?
+                       System.out.println("ENUM-Not-Hanled");
+               }
+
+               TaintSource b = cm == null ? null : new TaintSource(cm);
+               for (TaintSource taintSource : ts) {
+                       if (taintSource.equals(b)
+                                       || (taintSource.getMaskOnly() != null && taintSource
+                                                       .getMaskOnly().equals(b))) {
+                               return;
+                       }
+               }
+               if (e instanceof ConditionalExpression) {
+                       check(call, scope, ((ConditionalExpression) e).valueIfFalse, caller);
+                       check(call, scope, ((ConditionalExpression) e).valueIfTrue, caller);
+                       return;
+               }
+
+               System.out.println();
+
+               System.out.println(new String(scope.enclosingClassScope()
+                               .referenceType().compilationResult.fileName));
+               System.out.println("Cannot Handle: " + e + " in "
+                               + (call == null ? "constructor" : call.sourceStart) + " => "
+                               + caller);
+               System.out.println(e.getClass());
+               System.out.println("To ignore: " + b.toConfLine());
+       }
+       private void testEnum(Expression e, MethodBinding binding) {
+               if (binding.parameters.length != 0) {
+                       System.out.println("ERROR: meth");
+                       return;
+               }
+               System.out.println(e.resolvedType.getClass());
+               String s2 = new String(e.resolvedType.qualifiedPackageName())
+                               + "."
+                               + (new String(e.resolvedType.qualifiedSourceName()).replace(
+                                               '.', '$'));
+               try {
+                       Class<?> c = Class.forName(s2);
+                       Enum<?>[] e1 = (Enum[]) c.getMethod("values").invoke(null);
+                       Method m = c.getMethod(new String(binding.selector));
+                       for (int j = 0; j < e1.length; j++) {
+                               System.out.println(m.invoke(e1[j]));
+                       }
+               } catch (ClassNotFoundException e1) {
+                       e1.printStackTrace();
+               } catch (ReflectiveOperationException e1) {
+                       e1.printStackTrace();
+               }
+               System.out.println("ENUM-done: " + e + "!");
+               return;
+       }
+}
diff --git a/util-testing/org/cacert/gigi/localisation/TranslationCollector.java b/util-testing/org/cacert/gigi/localisation/TranslationCollector.java
new file mode 100644 (file)
index 0000000..2b8ca7e
--- /dev/null
@@ -0,0 +1,267 @@
+package org.cacert.gigi.localisation;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.TreeSet;
+
+import org.cacert.gigi.output.template.Template;
+import org.eclipse.jdt.core.compiler.CategorizedProblem;
+import org.eclipse.jdt.internal.compiler.CompilationResult;
+import org.eclipse.jdt.internal.compiler.ast.CompilationUnitDeclaration;
+import org.eclipse.jdt.internal.compiler.ast.TypeDeclaration;
+import org.eclipse.jdt.internal.compiler.batch.CompilationUnit;
+import org.eclipse.jdt.internal.compiler.batch.FileSystem;
+import org.eclipse.jdt.internal.compiler.batch.Main;
+import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants;
+import org.eclipse.jdt.internal.compiler.env.AccessRestriction;
+import org.eclipse.jdt.internal.compiler.env.IBinaryType;
+import org.eclipse.jdt.internal.compiler.env.ICompilationUnit;
+import org.eclipse.jdt.internal.compiler.env.ISourceType;
+import org.eclipse.jdt.internal.compiler.impl.CompilerOptions;
+import org.eclipse.jdt.internal.compiler.impl.ITypeRequestor;
+import org.eclipse.jdt.internal.compiler.lookup.LookupEnvironment;
+import org.eclipse.jdt.internal.compiler.lookup.PackageBinding;
+import org.eclipse.jdt.internal.compiler.parser.Parser;
+import org.eclipse.jdt.internal.compiler.problem.ProblemReporter;
+
+public class TranslationCollector {
+       static class TranslationEntry implements Comparable<TranslationEntry> {
+               String text;
+               String occur1;
+               List<String> occur;
+               public TranslationEntry(String text, String occur) {
+                       this.text = text;
+                       occur1 = occur;
+               }
+               public List<String> getOccur() {
+                       if (occur == null) {
+                               return Arrays.asList(occur1);
+                       }
+                       return occur;
+               }
+               public void add(String t) {
+                       if (occur == null) {
+                               occur = new ArrayList<>(Arrays.asList(occur1));
+                       }
+                       occur.add(t);
+               }
+               @Override
+               public int compareTo(TranslationEntry o) {
+                       int i = occur1.compareTo(o.occur1);
+                       if (i != 0) {
+                               return i;
+                       }
+
+                       return text.compareTo(o.text);
+               }
+       }
+
+       private HashMap<String, TranslationEntry> translations = new HashMap<>();
+
+       public final File base;
+
+       public TranslationCollector(File base, File conf) {
+               this.base = base;
+               taint = new LinkedList<>();
+               for (String s : new FileIterable(conf)) {
+                       taint.add(TaintSource.parseTaint(s));
+               }
+       }
+       public void run(File out) throws IOException {
+               scanTemplates();
+               scanCode(taint);
+
+               System.out
+                               .println("Total Translatable Strings: " + translations.size());
+               TreeSet<TranslationEntry> trs = new TreeSet<>(translations.values());
+               writePOFile(out, trs);
+
+       }
+
+       public void add(String text, String line) {
+               TranslationEntry i = translations.get(text);
+               if (i == null) {
+                       translations.put(text, new TranslationEntry(text, line));
+                       return;
+               }
+               i.add(line);
+       }
+
+       private void scanCode(LinkedList<TaintSource> taint) throws Error {
+               PrintWriter out = new PrintWriter(System.out);
+               Main m = new Main(out, out, false, null, null);
+               File[] fs = recurse(
+                               new File(new File(new File(base, "src"), "org"), "cacert"),
+                               new LinkedList<File>(), ".java").toArray(new File[0]);
+               String[] t = new String[fs.length + 3];
+               t[0] = "-cp";
+               t[1] = new File(base, "bin").getAbsolutePath();
+               t[2] = "-7";
+               for (int i = 0; i < fs.length; i++) {
+                       t[i + 3] = fs[i].getAbsolutePath();
+               }
+               m.configure(t);
+               FileSystem environment = m.getLibraryAccess();
+               CompilerOptions compilerOptions = new CompilerOptions(m.options);//new HashMap<>());//m.options);
+               compilerOptions.performMethodsFullRecovery = false;
+               compilerOptions.performStatementsRecovery = false;
+               //check
+               compilerOptions.sourceLevel = ClassFileConstants.JDK1_7;
+               compilerOptions.complianceLevel = ClassFileConstants.JDK1_7;
+               compilerOptions.originalComplianceLevel = ClassFileConstants.JDK1_7;
+
+               ProblemReporter pr = new ProblemReporter(m.getHandlingPolicy(),
+                               compilerOptions, m.getProblemFactory());
+               ITypeRequestor tr = new ITypeRequestor() {
+
+                       @Override
+                       public void accept(ISourceType[] sourceType,
+                                       PackageBinding packageBinding,
+                                       AccessRestriction accessRestriction) {
+                               throw new IllegalStateException("source type not implemented");
+                       }
+
+                       @Override
+                       public void accept(IBinaryType binaryType,
+                                       PackageBinding packageBinding,
+                                       AccessRestriction accessRestriction) {
+                               le.createBinaryTypeFrom(binaryType, packageBinding,
+                                               accessRestriction);
+                       }
+
+                       @Override
+                       public void accept(ICompilationUnit unit,
+                                       AccessRestriction accessRestriction) {
+                               throw new IllegalStateException(
+                                               "compilation unit not implemented");
+                       }
+               };
+               le = new LookupEnvironment(tr, compilerOptions, pr, environment);
+               Parser parser = new Parser(pr,
+                               compilerOptions.parseLiteralExpressionsAsConstants);
+               CompilationUnit[] sourceUnits = m.getCompilationUnits();
+               CompilationUnitDeclaration[] parsedUnits = new CompilationUnitDeclaration[sourceUnits.length];
+               for (int i = 0; i < parsedUnits.length; i++) {
+
+                       CompilationResult unitResult = new CompilationResult(
+                                       sourceUnits[i], i, parsedUnits.length,
+                                       compilerOptions.maxProblemsPerUnit);
+                       CompilationUnitDeclaration parsedUnit = parser.parse(
+                                       sourceUnits[i], unitResult);
+                       le.buildTypeBindings(parsedUnit, null /*no access restriction*/);
+                       parsedUnits[i] = parsedUnit;
+               }
+               le.completeTypeBindings();
+               for (int i = 0; i < parsedUnits.length; i++) {
+                       CompilationUnitDeclaration parsedUnit = parsedUnits[i];
+
+                       parser.getMethodBodies(parsedUnit);
+                       parsedUnit.scope.faultInTypes();
+                       parsedUnit.scope.verifyMethods(le.methodVerifier());
+                       parsedUnit.resolve();
+               }
+               for (int i = 0; i < parsedUnits.length; i++) {
+                       CompilationUnitDeclaration parsedUnit = parsedUnits[i];
+                       if (parsedUnit.compilationResult.problems != null) {
+                               int err = 0;
+                               for (int c = 0; c < parsedUnit.compilationResult.problemCount; c++) {
+                                       CategorizedProblem problem = parsedUnit.compilationResult.problems[c];
+                                       if (problem.isError()) {
+                                               err++;
+                                       }
+                                       if (OUTPUT_WARNINGS || problem.isError()) {
+                                               System.out.println(problem);
+                                               StringBuilder prob = new StringBuilder();
+                                               prob.append(parsedUnit.compilationResult.fileName);
+                                               prob.append(":");
+                                               prob.append(problem.getSourceLineNumber());
+                                               System.out.println(prob.toString());
+                                       }
+                               }
+                               if (err > 0) {
+                                       throw new Error();
+                               }
+                       }
+
+                       if (parsedUnit.types == null) {
+                               System.out.println("No types");
+
+                       } else {
+                               TranslationCollectingVisitor v = new TranslationCollectingVisitor(
+                                               parsedUnit,
+                                               taint.toArray(new TaintSource[taint.size()]), this);
+                               for (TypeDeclaration td : parsedUnit.types) {
+                                       td.traverse(v, td.scope);
+                               }
+                       }
+                       parsedUnits[i] = parsedUnit;
+               }
+       }
+       private void scanTemplates() throws UnsupportedEncodingException,
+                       FileNotFoundException {
+               File[] ts = recurse(
+                               new File(new File(new File(base, "src"), "org"), "cacert"),
+                               new LinkedList<File>(), ".templ").toArray(new File[0]);
+               for (File file : ts) {
+                       Template t = new Template(new InputStreamReader(
+                                       new FileInputStream(file), "UTF-8"));
+                       LinkedList<String> i = new LinkedList<String>();
+                       t.addTranslations(i);
+                       for (String string : i) {
+                               add(string,
+                                               file.getAbsolutePath().substring(
+                                                               base.getAbsolutePath().length() + 1)
+                                                               + ":0");
+                       }
+               }
+       }
+
+       static LookupEnvironment le;
+       private static final boolean OUTPUT_WARNINGS = false;
+
+       private LinkedList<TaintSource> taint;
+       public static void main(String[] args) throws IOException {
+               new TranslationCollector(new File(args[1]), new File(args[0]))
+                               .run(new File(args[2]));
+       }
+
+       public static void writePOFile(File target,
+                       Collection<TranslationEntry> strings) throws IOException {
+               PrintWriter out = new PrintWriter(target);
+               for (TranslationEntry s : strings) {
+                       out.print("#:");
+                       for (String st : s.getOccur()) {
+                               out.print(" " + st);
+                       }
+                       out.println();
+                       out.println("msgid \""
+                                       + s.text.replace("\\", "\\\\").replace("\"", "\\\"") + "\"");
+                       out.println("msgstr \"\"");
+                       out.println();
+               }
+               out.close();
+       }
+
+       private static List<File> recurse(File file, List<File> toAdd, String pt) {
+               if (file.isDirectory()) {
+                       for (File f : file.listFiles()) {
+                               recurse(f, toAdd, pt);
+                       }
+               } else {
+                       if (file.getName().endsWith(pt)) {
+                               toAdd.add(file);
+                       }
+               }
+               return toAdd;
+       }
+}
diff --git a/util-testing/org/cacert/gigi/localisation/conf.txt b/util-testing/org/cacert/gigi/localisation/conf.txt
new file mode 100644 (file)
index 0000000..f185e20
--- /dev/null
@@ -0,0 +1,19 @@
+org.cacert.gigi.localisation Language.getTranslation(String),0
+org.cacert.gigi.pages Page.translate(ServletRequest, String),1
+org.cacert.gigi.output.template Form.outputError(PrintWriter, ServletRequest, String, Object[]),2
+org.cacert.gigi Gigi.MenuBuilder.putPage(String, Page, String),2
+org.cacert.gigi Gigi.MenuBuilder.getMenu(String),0
+org.cacert.gigi GigiApiException.GigiApiException(String),0=>org.cacert.gigi GigiApiException.formatPlain(PrintWriter),0
+org.cacert.gigi.output Menu.Menu(String),0=>org.cacert.gigi.output Menu.output(PrintWriter, Language, Map),0
+org.cacert.gigi.output SimpleMenuItem.SimpleMenuItem(String,String),1=>org.cacert.gigi.output SimpleMenuItem.output(PrintWriter, Language, Map),0
+org.cacert.gigi.dbObjects Digest.Digest(String),0
+org.cacert.gigi.dbObjects Certificate.CertificateStatus.CertificateStatus(String),0
+org.cacert.gigi.pages Page.Page(String),0
+org.cacert.gigi.pages OneFormPage.OneFormPage(String, Class),0
+org.cacert.gigi.pages StaticPage.StaticPage(String, InputStream),0
+org.cacert.gigi.output.template SprintfCommand.SprintfCommand(String, List),0=>org.cacert.gigi.output.template SprintfCommand.output(PrintWriter, Language, Map),0
+org.cacert.gigi.output.template SprintfCom---invalid---mand.SprintfCommand(String),0=>org.cacert.gigi.output.template Template.parseCommand(String),0
+org.cacert.gigi.output.template TranslateCommand.TranslateCommand(String),0=>org.cacert.gigi.output.template TranslateCommand.output(PrintWriter, Language, Map),0
+org.cacert.gigi.pages.account.domain DomainOverview.DomainOverview(String),0
+org.cacert.gigi.dbObjects Group.Group(String),0
+org.cacert.gigi.output.template SprintfCommand.createSimple(String, String[]),0