From 753aae17442cbcdfcbce2d720b7b5dfd13918294 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Felix=20D=C3=B6rre?= Date: Sun, 22 Nov 2015 19:54:28 +0100 Subject: [PATCH] add: high financial value assessment and move domain verification to own DomainAssessment util fixes #71 Change-Id: I1ee74958d240d6ceca05d1eda61ba83e4f3d9a9c --- .gitignore | 3 + config/gigi.properties.template | 2 + doc/scripts/gigi | 3 + src/org/cacert/gigi/Gigi.java | 2 + src/org/cacert/gigi/dbObjects/CPS.properties | 3 - src/org/cacert/gigi/dbObjects/Domain.java | 83 +-------- .../cacert/gigi/util/DomainAssessment.java | 157 ++++++++++++++++++ src/org/cacert/gigi/util/idn_enabled.dat | 35 ++++ tests/org/cacert/gigi/DomainVerification.java | 102 ++++++------ .../cacert/gigi/pages/account/TestDomain.java | 10 ++ .../cacert/gigi/testUtils/ConfiguredTest.java | 27 ++- .../cacert/gigi/testUtils/ManagedTest.java | 21 ++- .../gigi/util/HighFinancialValueFetcher.java | 41 +++++ 13 files changed, 346 insertions(+), 143 deletions(-) delete mode 100644 src/org/cacert/gigi/dbObjects/CPS.properties create mode 100644 src/org/cacert/gigi/util/DomainAssessment.java create mode 100644 src/org/cacert/gigi/util/idn_enabled.dat create mode 100644 util/org/cacert/gigi/util/HighFinancialValueFetcher.java diff --git a/.gitignore b/.gitignore index 6746097e..6ab3fcd4 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ static.tar.gz /src/org/cacert/gigi/util/effective_tld_names.dat /Gigi.MF +#testing /testKeypair +/financial.dat + /signer diff --git a/config/gigi.properties.template b/config/gigi.properties.template index 6df75cb6..a6406a1e 100644 --- a/config/gigi.properties.template +++ b/config/gigi.properties.template @@ -14,3 +14,5 @@ sql.url=jdbc:postgresql://localhost/cacert #sql.url=jdbc:mysql://localhost:3306/cacert sql.user= sql.password= + +highFinancialValue=/path/to/alexa/list \ No newline at end of file diff --git a/doc/scripts/gigi b/doc/scripts/gigi index 14bccc43..f96d1930 100755 --- a/doc/scripts/gigi +++ b/doc/scripts/gigi @@ -32,6 +32,9 @@ then elif [ "$1" == "reset-database" ] then java -cp $JDBC_DRIVER:$GIGI_EXEC org.cacert.gigi.util.DatabaseManager +elif [ "$1" == "fetch-alexa" ] +then + java -cp $JDBC_DRIVER:$GIGI_EXEC org.cacert.gigi.util.HighFinancialValueFetcher $2 $3 elif [ "$1" == "fetch-locales" ] then if [ "$2" == "" ]; then diff --git a/src/org/cacert/gigi/Gigi.java b/src/org/cacert/gigi/Gigi.java index b6cb3d7f..58e7b11d 100644 --- a/src/org/cacert/gigi/Gigi.java +++ b/src/org/cacert/gigi/Gigi.java @@ -69,6 +69,7 @@ import org.cacert.gigi.pages.wot.MyPoints; import org.cacert.gigi.pages.wot.RequestTTPPage; import org.cacert.gigi.ping.PingerDaemon; import org.cacert.gigi.util.AuthorizationContext; +import org.cacert.gigi.util.DomainAssessment; import org.cacert.gigi.util.ServerConstants; public final class Gigi extends HttpServlet { @@ -235,6 +236,7 @@ public final class Gigi extends HttpServlet { } testing = conf.getProperty("testing") != null; instance = this; + DomainAssessment.init(conf); DatabaseConnection.init(conf); this.truststore = truststore; pinger = new PingerDaemon(truststore); diff --git a/src/org/cacert/gigi/dbObjects/CPS.properties b/src/org/cacert/gigi/dbObjects/CPS.properties deleted file mode 100644 index 87acaed7..00000000 --- a/src/org/cacert/gigi/dbObjects/CPS.properties +++ /dev/null @@ -1,3 +0,0 @@ -# from http://www.cacert.org/policy/CertificationPracticeStatement.php#p3.1.7 on 07.11.2014 -IDN-enabled=ac,ar,at,biz,br,cat,ch,cl,cn,de,dk,es,fi,gr,hu,info,io,ir,is,jp,kr,li,lt,museum,no,org,pl,pr,se,sh,th,tm,tw,vn -# from https://data.iana.org/TLD/tlds-alpha-by-domain.txt diff --git a/src/org/cacert/gigi/dbObjects/Domain.java b/src/org/cacert/gigi/dbObjects/Domain.java index fa6a6d0e..ba8aacff 100644 --- a/src/org/cacert/gigi/dbObjects/Domain.java +++ b/src/org/cacert/gigi/dbObjects/Domain.java @@ -1,20 +1,13 @@ package org.cacert.gigi.dbObjects; -import java.io.IOException; -import java.io.InputStream; -import java.net.IDN; -import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Properties; -import java.util.Set; import org.cacert.gigi.GigiApiException; import org.cacert.gigi.database.GigiPreparedStatement; import org.cacert.gigi.database.GigiResultSet; -import org.cacert.gigi.util.PublicSuffixes; +import org.cacert.gigi.util.DomainAssessment; public class Domain implements IdCachable, Verifyable { @@ -24,18 +17,6 @@ public class Domain implements IdCachable, Verifyable { private int id; - private static final Set IDNEnabledTLDs; - - static { - Properties CPS = new Properties(); - try (InputStream resourceAsStream = Domain.class.getResourceAsStream("CPS.properties")) { - CPS.load(resourceAsStream); - IDNEnabledTLDs = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(CPS.getProperty("IDN-enabled").split(",")))); - } catch (IOException e) { - throw new Error(e); - } - } - private Domain(int id) { try (GigiPreparedStatement ps = new GigiPreparedStatement("SELECT `memid`, `domain` FROM `domains` WHERE `id`=? AND `deleted` IS NULL")) { ps.setInt(1, id); @@ -53,73 +34,13 @@ public class Domain implements IdCachable, Verifyable { public Domain(User actor, CertificateOwner owner, String suffix) throws GigiApiException { suffix = suffix.toLowerCase(); synchronized (Domain.class) { - checkCertifyableDomain(suffix, actor.isInGroup(Group.CODESIGNING)); + DomainAssessment.checkCertifiableDomain(suffix, actor.isInGroup(Group.CODESIGNING)); this.owner = owner; this.suffix = suffix; insert(); } } - public static void checkCertifyableDomain(String s, boolean hasPunycodeRight) throws GigiApiException { - String[] parts = s.split("\\.", -1); - if (parts.length < 2) { - throw new GigiApiException("Domain does not contain '.'."); - } - for (int i = parts.length - 1; i >= 0; i--) { - if ( !isVaildDomainPart(parts[i], hasPunycodeRight)) { - throw new GigiApiException("Syntax error in Domain"); - } - } - String publicSuffix = PublicSuffixes.getInstance().getRegistrablePart(s); - if ( !s.equals(publicSuffix)) { - throw new GigiApiException("You may only register a domain with exactly one lable before the public suffix."); - } - if (("." + s).matches("(\\.[0-9]*)*")) { - // This is not reached because we currently have no TLD that is - // numbers only. But who knows.. - // Better safe than sorry. - throw new GigiApiException("IP Addresses are not allowed"); - } - checkPunycode(parts[0], s.substring(parts[0].length() + 1)); - } - - private static void checkPunycode(String label, String domainContext) throws GigiApiException { - if (label.charAt(2) != '-' || label.charAt(3) != '-') { - return; // is no punycode - } - if ( !IDNEnabledTLDs.contains(domainContext)) { - throw new GigiApiException("Punycode label could not be positively verified."); - } - if ( !label.startsWith("xn--")) { - throw new GigiApiException("Unknown ACE prefix."); - } - try { - String unicode = IDN.toUnicode(label); - if (unicode.startsWith("xn--")) { - throw new GigiApiException("Punycode label could not be positively verified."); - } - } catch (IllegalArgumentException e) { - throw new GigiApiException("Punycode label could not be positively verified."); - } - } - - public static boolean isVaildDomainPart(String s, boolean allowPunycode) { - if ( !s.matches("[a-z0-9-]+")) { - return false; - } - if (s.charAt(0) == '-' || s.charAt(s.length() - 1) == '-') { - return false; - } - if (s.length() > 63) { - return false; - } - boolean canBePunycode = s.length() >= 4 && s.charAt(2) == '-' && s.charAt(3) == '-'; - if (canBePunycode && !allowPunycode) { - return false; - } - return true; - } - private static void checkInsert(String suffix) throws GigiApiException { try (GigiPreparedStatement ps = new GigiPreparedStatement("SELECT 1 FROM `domains` WHERE (`domain`=? OR (CONCAT('.', `domain`)=RIGHT(?,LENGTH(`domain`)+1) OR RIGHT(`domain`,LENGTH(?)+1)=CONCAT('.',?))) AND `deleted` IS NULL")) { ps.setString(1, suffix); diff --git a/src/org/cacert/gigi/util/DomainAssessment.java b/src/org/cacert/gigi/util/DomainAssessment.java new file mode 100644 index 00000000..5070e5cb --- /dev/null +++ b/src/org/cacert/gigi/util/DomainAssessment.java @@ -0,0 +1,157 @@ +package org.cacert.gigi.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.IDN; +import java.net.URL; +import java.util.Collections; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +import org.cacert.gigi.GigiApiException; + +public class DomainAssessment { + + private static class DomainSet { + + private final Set data; + + public DomainSet(URL u) { + this(openStream(u)); + } + + private static Reader openStream(URL u) { + try { + return new InputStreamReader(u.openStream(), "UTF-8"); + } catch (IOException e) { + throw new Error(e); + } + } + + public DomainSet(Reader r) { + HashSet contents = new HashSet<>(); + try { + BufferedReader br = new BufferedReader(r); + String line; + while ((line = br.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + if (line.isEmpty()) { + continue; + } + contents.add(line); + } + } catch (IOException e) { + throw new Error(e); + } + data = Collections.unmodifiableSet(contents); + } + + public boolean contains(String suffix) { + while (suffix.contains(".")) { + if (data.contains(suffix)) { + return true; + } + suffix = suffix.substring(suffix.indexOf('.') + 1); + } + return data.contains(suffix); + } + } + + private static DomainSet financial; + + private static final DomainSet idn_enabled = new DomainSet(DomainAssessment.class.getResource("idn_enabled.dat")); + + public static boolean isHighFinancialValue(String suffix) { + return financial.contains(suffix); + } + + public static void checkCertifiableDomain(String domain, boolean hasPunycodeRight) throws GigiApiException { + if (isHighFinancialValue(domain)) { + throw new GigiApiException("Domain blocked for automatic adding."); + } + String[] parts = domain.split("\\.", -1); + if (parts.length < 2) { + throw new GigiApiException("Domain does not contain '.'."); + } + boolean neededPunycode = false; + for (int i = parts.length - 1; i >= 0; i--) { + if ( !isValidDomainPart(parts[i])) { + throw new GigiApiException("Syntax error in Domain."); + } + boolean canBePunycode = parts[i].length() >= 4 && parts[i].charAt(2) == '-' && parts[i].charAt(3) == '-'; + if (canBePunycode) { + if ( !hasPunycodeRight) { + throw new GigiApiException("Punycode domain without specific right."); + } + punycodeDecode(parts[i]); + neededPunycode = true; + } + + } + if (neededPunycode && !idn_enabled.contains(domain)) { + throw new GigiApiException("Punycode not allowed under this TLD."); + } + + String publicSuffix = PublicSuffixes.getInstance().getRegistrablePart(domain); + if ( !domain.equals(publicSuffix)) { + throw new GigiApiException("You may only register a domain with exactly one label before the public suffix."); + } + + if (("." + domain).matches("(\\.[0-9]*)*")) { + // This is not reached because we currently have no TLD that is + // numbers only. But who knows.. + // Better safe than sorry. + throw new GigiApiException("IP Addresses are not allowed."); + } + } + + private static String punycodeDecode(String label) throws GigiApiException { + if (label.charAt(2) != '-' || label.charAt(3) != '-') { + return label; // is no punycode + } + if ( !label.startsWith("xn--")) { + throw new GigiApiException("Unknown ACE prefix."); + } + try { + String unicode = IDN.toUnicode(label); + if (unicode.startsWith("xn--")) { + throw new GigiApiException("Punycode label could not be positively verified."); + } + return unicode; + } catch (IllegalArgumentException e) { + throw new GigiApiException("Punycode label could not be positively verified."); + } + } + + public static boolean isValidDomainPart(String s) { + if ( !s.matches("[a-z0-9-]+")) { + return false; + } + if (s.charAt(0) == '-' || s.charAt(s.length() - 1) == '-') { + return false; + } + if (s.length() > 63) { + return false; + } + return true; + } + + public static void init(Properties conf) { + String financialName = conf.getProperty("highFinancialValue"); + if (financialName == null) { + throw new Error("No property highFinancialValue was configured"); + } + try { + financial = new DomainSet(new InputStreamReader(new FileInputStream(new File(financialName)), "UTF-8")); + } catch (IOException e) { + throw new Error(e); + } + } +} diff --git a/src/org/cacert/gigi/util/idn_enabled.dat b/src/org/cacert/gigi/util/idn_enabled.dat new file mode 100644 index 00000000..82a67727 --- /dev/null +++ b/src/org/cacert/gigi/util/idn_enabled.dat @@ -0,0 +1,35 @@ +ac +ar +at +biz +br +cat +ch +cl +cn +de +dk +es +fi +gr +hu +info +io +ir +is +jp +kr +li +lt +museum +no +org +pl +pr +se +sh +th +tm +tw +vn +# from https://data.iana.org/TLD/tlds-alpha-by-domain.txt diff --git a/tests/org/cacert/gigi/DomainVerification.java b/tests/org/cacert/gigi/DomainVerification.java index cd31f8ef..1ce02be5 100644 --- a/tests/org/cacert/gigi/DomainVerification.java +++ b/tests/org/cacert/gigi/DomainVerification.java @@ -2,76 +2,82 @@ package org.cacert.gigi; import static org.junit.Assert.*; -import org.cacert.gigi.dbObjects.Domain; +import org.cacert.gigi.testUtils.ConfiguredTest; +import org.cacert.gigi.util.DomainAssessment; import org.junit.Test; -public class DomainVerification { +public class DomainVerification extends ConfiguredTest { @Test public void testDomainPart() { - assertTrue(Domain.isVaildDomainPart("cacert", false)); - assertTrue(Domain.isVaildDomainPart("de", false)); - assertTrue(Domain.isVaildDomainPart("ha2-a", false)); - assertTrue(Domain.isVaildDomainPart("ha2--a", false)); - assertTrue(Domain.isVaildDomainPart("h--a", false)); - assertFalse(Domain.isVaildDomainPart("xn--bla", false)); - assertFalse(Domain.isVaildDomainPart("-xnbla", false)); - assertFalse(Domain.isVaildDomainPart("xnbla-", false)); - assertFalse(Domain.isVaildDomainPart("", false)); - assertTrue(Domain.isVaildDomainPart("2xnbla", false)); - assertTrue(Domain.isVaildDomainPart("xnbla2", false)); - assertTrue(Domain.isVaildDomainPart("123", false)); - assertTrue(Domain.isVaildDomainPart("abcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxy1234567890123", false)); - assertFalse(Domain.isVaildDomainPart("abcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxy12345678901234", false)); + assertTrue(DomainAssessment.isValidDomainPart("cacert")); + assertTrue(DomainAssessment.isValidDomainPart("de")); + assertTrue(DomainAssessment.isValidDomainPart("ha2-a")); + assertTrue(DomainAssessment.isValidDomainPart("ha2--a")); + assertTrue(DomainAssessment.isValidDomainPart("h--a")); + assertFalse(DomainAssessment.isValidDomainPart("-xnbla")); + assertFalse(DomainAssessment.isValidDomainPart("xnbla-")); + assertFalse(DomainAssessment.isValidDomainPart("")); + assertTrue(DomainAssessment.isValidDomainPart("2xnbla")); + assertTrue(DomainAssessment.isValidDomainPart("xnbla2")); + assertTrue(DomainAssessment.isValidDomainPart("123")); + assertTrue(DomainAssessment.isValidDomainPart("abcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxy1234567890123")); + assertFalse(DomainAssessment.isValidDomainPart("abcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxy12345678901234")); } @Test - public void testDomainCertifyable() { - isCertifyableDomain(true, "cacert.org", false); - isCertifyableDomain(true, "cacert.de", false); - isCertifyableDomain(true, "cacert.org", false); - isCertifyableDomain(true, "cacert.org", false); - isCertifyableDomain(true, "1234.org", false); - isCertifyableDomain(false, "a.cacert.org", true); - isCertifyableDomain(false, "gigi.local", true); - isCertifyableDomain(false, "org", true); - isCertifyableDomain(false, "'a.org", true); - isCertifyableDomain(false, ".org", true); - isCertifyableDomain(false, ".org.", true); + public void testDomainCertifiable() { + isCertifiableDomain(true, "cacert.org", false); + isCertifiableDomain(true, "cacert.de", false); + isCertifiableDomain(true, "cacert.org", false); + isCertifiableDomain(true, "cacert.org", false); + isCertifiableDomain(true, "1234.org", false); + isCertifiableDomain(false, "a.cacert.org", true); + isCertifiableDomain(false, "gigi.local", true); + isCertifiableDomain(false, "org", true); + isCertifiableDomain(false, "'a.org", true); + isCertifiableDomain(false, ".org", true); + isCertifiableDomain(false, ".org.", true); // non-real-punycode - isCertifyableDomain(true, "xna-ae.de", false); - isCertifyableDomain(true, "xn-aae.de", false); + isCertifiableDomain(true, "xna-ae.de", false); + isCertifiableDomain(true, "xn-aae.de", false); // illegal punycode: // illegal ace prefix - isCertifyableDomain(false, "aa--b.com", true); - isCertifyableDomain(false, "xm--ae-a.de", true); + isCertifiableDomain(false, "aa--b.com", true); + isCertifiableDomain(false, "xm--ae-a.de", true); // illegal punycode content - isCertifyableDomain(false, "xn--ae-a.com", true); - isCertifyableDomain(false, "xn--ae.de", true); - isCertifyableDomain(false, "xn--ae-a.org", true); - isCertifyableDomain(false, "xn--ae-a.de", true); + isCertifiableDomain(false, "xn--ae-a.com", true); + isCertifiableDomain(false, "xn--ae.de", true); + isCertifiableDomain(false, "xn--ae-a.org", true); + isCertifiableDomain(false, "xn--ae-a.de", true); // valid punycode requires permission - isCertifyableDomain(true, "xn--4ca0bs.de", true); - isCertifyableDomain(false, "xn--4ca0bs.de", false); - isCertifyableDomain(true, "xn--a-zfa9cya.de", true); - isCertifyableDomain(false, "xn--a-zfa9cya.de", false); + isCertifiableDomain(true, "xn--4ca0bs.de", true); + isCertifiableDomain(false, "xn--4ca0bs.de", false); + isCertifiableDomain(true, "xn--a-zfa9cya.de", true); + isCertifiableDomain(false, "xn--a-zfa9cya.de", false); // valid punycode does not help under .com - isCertifyableDomain(false, "xn--a-zfa9cya.com", true); - isCertifyableDomain(true, "zfa9cya.com", true); + isCertifiableDomain(false, "xn--a-zfa9cya.com", true); + isCertifiableDomain(true, "zfa9cya.com", true); - isCertifyableDomain(false, "127.0.0.1", false); - isCertifyableDomain(false, "::1", false); - isCertifyableDomain(false, "127.0.0.1", true); - isCertifyableDomain(false, "::1", true); + isCertifiableDomain(false, "127.0.0.1", false); + isCertifiableDomain(false, "::1", false); + isCertifiableDomain(false, "127.0.0.1", true); + isCertifiableDomain(false, "::1", true); } - private void isCertifyableDomain(boolean b, String string, boolean puny) { + @Test + public void testFinancial() { + isCertifiableDomain(false, "google.com", true); + isCertifiableDomain(false, "twitter.com", true); + } + + private void isCertifiableDomain(boolean b, String string, boolean puny) { try { - Domain.checkCertifyableDomain(string, puny); + DomainAssessment.checkCertifiableDomain(string, puny); assertTrue(b); } catch (GigiApiException e) { assertFalse(e.getMessage(), b); diff --git a/tests/org/cacert/gigi/pages/account/TestDomain.java b/tests/org/cacert/gigi/pages/account/TestDomain.java index 1a3a5a08..770c90e8 100644 --- a/tests/org/cacert/gigi/pages/account/TestDomain.java +++ b/tests/org/cacert/gigi/pages/account/TestDomain.java @@ -19,6 +19,16 @@ public class TestDomain extends ClientTest { assertNotNull(addDomain(cookie, uniq + ".de")); } + @Test + public void testInvalid() throws IOException { + assertNotNull(addDomain(cookie, uniq + ".invalid")); + } + + @Test + public void testHighFinancialValue() throws IOException { + assertNotNull(addDomain(cookie, "google.com")); + } + public static String addDomain(String session, String domain) throws IOException { return executeBasicWebInteraction(session, DomainOverview.PATH, "adddomain&newdomain=" + URLEncoder.encode(domain, "UTF-8"), 1); } diff --git a/tests/org/cacert/gigi/testUtils/ConfiguredTest.java b/tests/org/cacert/gigi/testUtils/ConfiguredTest.java index 90b34682..9010a597 100644 --- a/tests/org/cacert/gigi/testUtils/ConfiguredTest.java +++ b/tests/org/cacert/gigi/testUtils/ConfiguredTest.java @@ -17,6 +17,7 @@ import java.util.regex.Pattern; import org.cacert.gigi.database.DatabaseConnection; import org.cacert.gigi.database.DatabaseConnection.Link; +import org.cacert.gigi.util.DomainAssessment; import org.cacert.gigi.util.PEM; import org.junit.BeforeClass; @@ -39,12 +40,18 @@ public abstract class ConfiguredTest { private static boolean envInited = false; @BeforeClass - public static void initEnvironment() throws IOException { + public static void initEnvironmentHook() throws IOException { + initEnvironment(); + } + + public static Properties initEnvironment() throws IOException { TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + Properties props = generateProps(); if (envInited) { - return; + return props; } envInited = true; + DomainAssessment.init(props); try (FileInputStream inStream = new FileInputStream("config/test.properties")) { testProps.load(inStream); } @@ -56,6 +63,22 @@ public abstract class ConfiguredTest { throw new Error(e); } } + return props; + + } + + private static Properties generateProps() throws Error { + Properties mainProps = new Properties(); + File out = new File("financial.dat"); + if ( !out.exists()) { + try (FileOutputStream fos = new FileOutputStream(out)) { + fos.write("google.com\ntwitter.com\n".getBytes("UTF-8")); + } catch (IOException e) { + throw new Error(e); + } + } + mainProps.setProperty("highFinancialValue", out.getAbsolutePath()); + return mainProps; } public static KeyPair generateKeypair() throws GeneralSecurityException { diff --git a/tests/org/cacert/gigi/testUtils/ManagedTest.java b/tests/org/cacert/gigi/testUtils/ManagedTest.java index f379b3f2..6fa11a91 100644 --- a/tests/org/cacert/gigi/testUtils/ManagedTest.java +++ b/tests/org/cacert/gigi/testUtils/ManagedTest.java @@ -97,13 +97,17 @@ public class ManagedTest extends ConfiguredTest { } @BeforeClass - public static void initEnvironment() { + public static void initEnvironmentHook() { + initEnvironment(); + } + + public static Properties initEnvironment() { try { - ConfiguredTest.initEnvironment(); + Properties mainProps = ConfiguredTest.initEnvironment(); purgeDatabase(); String type = testProps.getProperty("type"); - Properties mainProps = generateMainProps(); + generateMainProps(mainProps); ServerConstants.init(mainProps); if (type.equals("local")) { url = testProps.getProperty("name.www") + ":" + testProps.getProperty("serverPort.https"); @@ -113,7 +117,7 @@ public class ManagedTest extends ConfiguredTest { if (testProps.getProperty("withSigner", "false").equals("true")) { SimpleSigner.runSigner(); } - return; + return mainProps; } url = testProps.getProperty("name.www") + ":" + testProps.getProperty("serverPort.https"); gigi = Runtime.getRuntime().exec(testProps.getProperty("java")); @@ -151,12 +155,13 @@ public class ManagedTest extends ConfiguredTest { ter = new TestEmailReceiver(new InetSocketAddress("localhost", 8473)); ter.start(); SimpleSigner.runSigner(); + return mainProps; } catch (IOException e) { throw new Error(e); } catch (SQLException e1) { - e1.printStackTrace(); + throw new Error(e1); } catch (InterruptedException e) { - e.printStackTrace(); + throw new Error(e); } } @@ -187,8 +192,7 @@ public class ManagedTest extends ConfiguredTest { u.openConnection().getHeaderField("Location"); } - private static Properties generateMainProps() { - Properties mainProps = new Properties(); + private static void generateMainProps(Properties mainProps) { mainProps.setProperty("testrunner", "true"); mainProps.setProperty("host", "127.0.0.1"); mainProps.setProperty("name.secure", testProps.getProperty("name.secure")); @@ -205,7 +209,6 @@ public class ManagedTest extends ConfiguredTest { mainProps.setProperty("sql.user", testProps.getProperty("sql.user")); mainProps.setProperty("sql.password", testProps.getProperty("sql.password")); mainProps.setProperty("testing", "true"); - return mainProps; } @AfterClass diff --git a/util/org/cacert/gigi/util/HighFinancialValueFetcher.java b/util/org/cacert/gigi/util/HighFinancialValueFetcher.java new file mode 100644 index 00000000..c05c7ca6 --- /dev/null +++ b/util/org/cacert/gigi/util/HighFinancialValueFetcher.java @@ -0,0 +1,41 @@ +package org.cacert.gigi.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.net.URL; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class HighFinancialValueFetcher { + + public static void main(String[] args) throws IOException { + int max = 1000; + if (args.length > 1) { + max = Integer.parseInt(args[1]); + } + PrintWriter fos = new PrintWriter(new File(args[0]), "UTF-8"); + ZipInputStream zis = new ZipInputStream(new URL("https://s3.amazonaws.com/alexa-static/top-1m.csv.zip").openStream()); + ZipEntry ze; + outer: + while ((ze = zis.getNextEntry()) != null) { + System.out.println(ze.getName()); + BufferedReader br = new BufferedReader(new InputStreamReader(zis, "UTF-8")); + String line; + while ((line = br.readLine()) != null) { + String[] parts = line.split(","); + int i = Integer.parseInt(parts[0]); + if (i > max) { + zis.close(); + break outer; + } + fos.println(parts[1]); + System.out.println(line); + } + } + fos.close(); + } + +} -- 2.39.2