Adding source to collect all localisation texts
[gigi.git] / util-testing / org / cacert / gigi / localisation / TranslationCollector.java
1 package org.cacert.gigi.localisation;
2 import java.io.File;
3 import java.io.FileInputStream;
4 import java.io.FileNotFoundException;
5 import java.io.IOException;
6 import java.io.InputStreamReader;
7 import java.io.PrintWriter;
8 import java.io.UnsupportedEncodingException;
9 import java.util.ArrayList;
10 import java.util.Arrays;
11 import java.util.Collection;
12 import java.util.HashMap;
13 import java.util.LinkedList;
14 import java.util.List;
15 import java.util.TreeSet;
16
17 import org.cacert.gigi.output.template.Template;
18 import org.eclipse.jdt.core.compiler.CategorizedProblem;
19 import org.eclipse.jdt.internal.compiler.CompilationResult;
20 import org.eclipse.jdt.internal.compiler.ast.CompilationUnitDeclaration;
21 import org.eclipse.jdt.internal.compiler.ast.TypeDeclaration;
22 import org.eclipse.jdt.internal.compiler.batch.CompilationUnit;
23 import org.eclipse.jdt.internal.compiler.batch.FileSystem;
24 import org.eclipse.jdt.internal.compiler.batch.Main;
25 import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants;
26 import org.eclipse.jdt.internal.compiler.env.AccessRestriction;
27 import org.eclipse.jdt.internal.compiler.env.IBinaryType;
28 import org.eclipse.jdt.internal.compiler.env.ICompilationUnit;
29 import org.eclipse.jdt.internal.compiler.env.ISourceType;
30 import org.eclipse.jdt.internal.compiler.impl.CompilerOptions;
31 import org.eclipse.jdt.internal.compiler.impl.ITypeRequestor;
32 import org.eclipse.jdt.internal.compiler.lookup.LookupEnvironment;
33 import org.eclipse.jdt.internal.compiler.lookup.PackageBinding;
34 import org.eclipse.jdt.internal.compiler.parser.Parser;
35 import org.eclipse.jdt.internal.compiler.problem.ProblemReporter;
36
37 public class TranslationCollector {
38         static class TranslationEntry implements Comparable<TranslationEntry> {
39                 String text;
40                 String occur1;
41                 List<String> occur;
42                 public TranslationEntry(String text, String occur) {
43                         this.text = text;
44                         occur1 = occur;
45                 }
46                 public List<String> getOccur() {
47                         if (occur == null) {
48                                 return Arrays.asList(occur1);
49                         }
50                         return occur;
51                 }
52                 public void add(String t) {
53                         if (occur == null) {
54                                 occur = new ArrayList<>(Arrays.asList(occur1));
55                         }
56                         occur.add(t);
57                 }
58                 @Override
59                 public int compareTo(TranslationEntry o) {
60                         int i = occur1.compareTo(o.occur1);
61                         if (i != 0) {
62                                 return i;
63                         }
64
65                         return text.compareTo(o.text);
66                 }
67         }
68
69         private HashMap<String, TranslationEntry> translations = new HashMap<>();
70
71         public final File base;
72
73         public TranslationCollector(File base, File conf) {
74                 this.base = base;
75                 taint = new LinkedList<>();
76                 for (String s : new FileIterable(conf)) {
77                         taint.add(TaintSource.parseTaint(s));
78                 }
79         }
80         public void run(File out) throws IOException {
81                 scanTemplates();
82                 scanCode(taint);
83
84                 System.out
85                                 .println("Total Translatable Strings: " + translations.size());
86                 TreeSet<TranslationEntry> trs = new TreeSet<>(translations.values());
87                 writePOFile(out, trs);
88
89         }
90
91         public void add(String text, String line) {
92                 TranslationEntry i = translations.get(text);
93                 if (i == null) {
94                         translations.put(text, new TranslationEntry(text, line));
95                         return;
96                 }
97                 i.add(line);
98         }
99
100         private void scanCode(LinkedList<TaintSource> taint) throws Error {
101                 PrintWriter out = new PrintWriter(System.out);
102                 Main m = new Main(out, out, false, null, null);
103                 File[] fs = recurse(
104                                 new File(new File(new File(base, "src"), "org"), "cacert"),
105                                 new LinkedList<File>(), ".java").toArray(new File[0]);
106                 String[] t = new String[fs.length + 3];
107                 t[0] = "-cp";
108                 t[1] = new File(base, "bin").getAbsolutePath();
109                 t[2] = "-7";
110                 for (int i = 0; i < fs.length; i++) {
111                         t[i + 3] = fs[i].getAbsolutePath();
112                 }
113                 m.configure(t);
114                 FileSystem environment = m.getLibraryAccess();
115                 CompilerOptions compilerOptions = new CompilerOptions(m.options);//new HashMap<>());//m.options);
116                 compilerOptions.performMethodsFullRecovery = false;
117                 compilerOptions.performStatementsRecovery = false;
118                 //check
119                 compilerOptions.sourceLevel = ClassFileConstants.JDK1_7;
120                 compilerOptions.complianceLevel = ClassFileConstants.JDK1_7;
121                 compilerOptions.originalComplianceLevel = ClassFileConstants.JDK1_7;
122
123                 ProblemReporter pr = new ProblemReporter(m.getHandlingPolicy(),
124                                 compilerOptions, m.getProblemFactory());
125                 ITypeRequestor tr = new ITypeRequestor() {
126
127                         @Override
128                         public void accept(ISourceType[] sourceType,
129                                         PackageBinding packageBinding,
130                                         AccessRestriction accessRestriction) {
131                                 throw new IllegalStateException("source type not implemented");
132                         }
133
134                         @Override
135                         public void accept(IBinaryType binaryType,
136                                         PackageBinding packageBinding,
137                                         AccessRestriction accessRestriction) {
138                                 le.createBinaryTypeFrom(binaryType, packageBinding,
139                                                 accessRestriction);
140                         }
141
142                         @Override
143                         public void accept(ICompilationUnit unit,
144                                         AccessRestriction accessRestriction) {
145                                 throw new IllegalStateException(
146                                                 "compilation unit not implemented");
147                         }
148                 };
149                 le = new LookupEnvironment(tr, compilerOptions, pr, environment);
150                 Parser parser = new Parser(pr,
151                                 compilerOptions.parseLiteralExpressionsAsConstants);
152                 CompilationUnit[] sourceUnits = m.getCompilationUnits();
153                 CompilationUnitDeclaration[] parsedUnits = new CompilationUnitDeclaration[sourceUnits.length];
154                 for (int i = 0; i < parsedUnits.length; i++) {
155
156                         CompilationResult unitResult = new CompilationResult(
157                                         sourceUnits[i], i, parsedUnits.length,
158                                         compilerOptions.maxProblemsPerUnit);
159                         CompilationUnitDeclaration parsedUnit = parser.parse(
160                                         sourceUnits[i], unitResult);
161                         le.buildTypeBindings(parsedUnit, null /*no access restriction*/);
162                         parsedUnits[i] = parsedUnit;
163                 }
164                 le.completeTypeBindings();
165                 for (int i = 0; i < parsedUnits.length; i++) {
166                         CompilationUnitDeclaration parsedUnit = parsedUnits[i];
167
168                         parser.getMethodBodies(parsedUnit);
169                         parsedUnit.scope.faultInTypes();
170                         parsedUnit.scope.verifyMethods(le.methodVerifier());
171                         parsedUnit.resolve();
172                 }
173                 for (int i = 0; i < parsedUnits.length; i++) {
174                         CompilationUnitDeclaration parsedUnit = parsedUnits[i];
175                         if (parsedUnit.compilationResult.problems != null) {
176                                 int err = 0;
177                                 for (int c = 0; c < parsedUnit.compilationResult.problemCount; c++) {
178                                         CategorizedProblem problem = parsedUnit.compilationResult.problems[c];
179                                         if (problem.isError()) {
180                                                 err++;
181                                         }
182                                         if (OUTPUT_WARNINGS || problem.isError()) {
183                                                 System.out.println(problem);
184                                                 StringBuilder prob = new StringBuilder();
185                                                 prob.append(parsedUnit.compilationResult.fileName);
186                                                 prob.append(":");
187                                                 prob.append(problem.getSourceLineNumber());
188                                                 System.out.println(prob.toString());
189                                         }
190                                 }
191                                 if (err > 0) {
192                                         throw new Error();
193                                 }
194                         }
195
196                         if (parsedUnit.types == null) {
197                                 System.out.println("No types");
198
199                         } else {
200                                 TranslationCollectingVisitor v = new TranslationCollectingVisitor(
201                                                 parsedUnit,
202                                                 taint.toArray(new TaintSource[taint.size()]), this);
203                                 for (TypeDeclaration td : parsedUnit.types) {
204                                         td.traverse(v, td.scope);
205                                 }
206                         }
207                         parsedUnits[i] = parsedUnit;
208                 }
209         }
210         private void scanTemplates() throws UnsupportedEncodingException,
211                         FileNotFoundException {
212                 File[] ts = recurse(
213                                 new File(new File(new File(base, "src"), "org"), "cacert"),
214                                 new LinkedList<File>(), ".templ").toArray(new File[0]);
215                 for (File file : ts) {
216                         Template t = new Template(new InputStreamReader(
217                                         new FileInputStream(file), "UTF-8"));
218                         LinkedList<String> i = new LinkedList<String>();
219                         t.addTranslations(i);
220                         for (String string : i) {
221                                 add(string,
222                                                 file.getAbsolutePath().substring(
223                                                                 base.getAbsolutePath().length() + 1)
224                                                                 + ":0");
225                         }
226                 }
227         }
228
229         static LookupEnvironment le;
230         private static final boolean OUTPUT_WARNINGS = false;
231
232         private LinkedList<TaintSource> taint;
233         public static void main(String[] args) throws IOException {
234                 new TranslationCollector(new File(args[1]), new File(args[0]))
235                                 .run(new File(args[2]));
236         }
237
238         public static void writePOFile(File target,
239                         Collection<TranslationEntry> strings) throws IOException {
240                 PrintWriter out = new PrintWriter(target);
241                 for (TranslationEntry s : strings) {
242                         out.print("#:");
243                         for (String st : s.getOccur()) {
244                                 out.print(" " + st);
245                         }
246                         out.println();
247                         out.println("msgid \""
248                                         + s.text.replace("\\", "\\\\").replace("\"", "\\\"") + "\"");
249                         out.println("msgstr \"\"");
250                         out.println();
251                 }
252                 out.close();
253         }
254
255         private static List<File> recurse(File file, List<File> toAdd, String pt) {
256                 if (file.isDirectory()) {
257                         for (File f : file.listFiles()) {
258                                 recurse(f, toAdd, pt);
259                         }
260                 } else {
261                         if (file.getName().endsWith(pt)) {
262                                 toAdd.add(file);
263                         }
264                 }
265                 return toAdd;
266         }
267 }