/src/org/cacert/gigi/util/effective_tld_names.dat
/Gigi.MF
+#testing
/testKeypair
+/financial.dat
+
/signer
#sql.url=jdbc:mysql://localhost:3306/cacert
sql.user=
sql.password=
+
+highFinancialValue=/path/to/alexa/list
\ No newline at end of file
elif [ "$1" == "reset-database" ]
then
java -cp $JDBC_DRIVER:$GIGI_EXEC org.cacert.gigi.util.DatabaseManager
+elif [ "$1" == "fetch-alexa" ]
+then
+ java -cp $JDBC_DRIVER:$GIGI_EXEC org.cacert.gigi.util.HighFinancialValueFetcher $2 $3
elif [ "$1" == "fetch-locales" ]
then
if [ "$2" == "" ]; then
import org.cacert.gigi.pages.wot.RequestTTPPage;
import org.cacert.gigi.ping.PingerDaemon;
import org.cacert.gigi.util.AuthorizationContext;
+import org.cacert.gigi.util.DomainAssessment;
import org.cacert.gigi.util.ServerConstants;
public final class Gigi extends HttpServlet {
}
testing = conf.getProperty("testing") != null;
instance = this;
+ DomainAssessment.init(conf);
DatabaseConnection.init(conf);
this.truststore = truststore;
pinger = new PingerDaemon(truststore);
+++ /dev/null
-# from http://www.cacert.org/policy/CertificationPracticeStatement.php#p3.1.7 on 07.11.2014
-IDN-enabled=ac,ar,at,biz,br,cat,ch,cl,cn,de,dk,es,fi,gr,hu,info,io,ir,is,jp,kr,li,lt,museum,no,org,pl,pr,se,sh,th,tm,tw,vn
-# from https://data.iana.org/TLD/tlds-alpha-by-domain.txt
package org.cacert.gigi.dbObjects;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.IDN;
-import java.util.Arrays;
import java.util.Collections;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
-import java.util.Properties;
-import java.util.Set;
import org.cacert.gigi.GigiApiException;
import org.cacert.gigi.database.GigiPreparedStatement;
import org.cacert.gigi.database.GigiResultSet;
-import org.cacert.gigi.util.PublicSuffixes;
+import org.cacert.gigi.util.DomainAssessment;
public class Domain implements IdCachable, Verifyable {
private int id;
- private static final Set<String> IDNEnabledTLDs;
-
- static {
- Properties CPS = new Properties();
- try (InputStream resourceAsStream = Domain.class.getResourceAsStream("CPS.properties")) {
- CPS.load(resourceAsStream);
- IDNEnabledTLDs = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(CPS.getProperty("IDN-enabled").split(","))));
- } catch (IOException e) {
- throw new Error(e);
- }
- }
-
private Domain(int id) {
try (GigiPreparedStatement ps = new GigiPreparedStatement("SELECT `memid`, `domain` FROM `domains` WHERE `id`=? AND `deleted` IS NULL")) {
ps.setInt(1, id);
public Domain(User actor, CertificateOwner owner, String suffix) throws GigiApiException {
suffix = suffix.toLowerCase();
synchronized (Domain.class) {
- checkCertifyableDomain(suffix, actor.isInGroup(Group.CODESIGNING));
+ DomainAssessment.checkCertifiableDomain(suffix, actor.isInGroup(Group.CODESIGNING));
this.owner = owner;
this.suffix = suffix;
insert();
}
}
- public static void checkCertifyableDomain(String s, boolean hasPunycodeRight) throws GigiApiException {
- String[] parts = s.split("\\.", -1);
- if (parts.length < 2) {
- throw new GigiApiException("Domain does not contain '.'.");
- }
- for (int i = parts.length - 1; i >= 0; i--) {
- if ( !isVaildDomainPart(parts[i], hasPunycodeRight)) {
- throw new GigiApiException("Syntax error in Domain");
- }
- }
- String publicSuffix = PublicSuffixes.getInstance().getRegistrablePart(s);
- if ( !s.equals(publicSuffix)) {
- throw new GigiApiException("You may only register a domain with exactly one lable before the public suffix.");
- }
- if (("." + s).matches("(\\.[0-9]*)*")) {
- // This is not reached because we currently have no TLD that is
- // numbers only. But who knows..
- // Better safe than sorry.
- throw new GigiApiException("IP Addresses are not allowed");
- }
- checkPunycode(parts[0], s.substring(parts[0].length() + 1));
- }
-
- private static void checkPunycode(String label, String domainContext) throws GigiApiException {
- if (label.charAt(2) != '-' || label.charAt(3) != '-') {
- return; // is no punycode
- }
- if ( !IDNEnabledTLDs.contains(domainContext)) {
- throw new GigiApiException("Punycode label could not be positively verified.");
- }
- if ( !label.startsWith("xn--")) {
- throw new GigiApiException("Unknown ACE prefix.");
- }
- try {
- String unicode = IDN.toUnicode(label);
- if (unicode.startsWith("xn--")) {
- throw new GigiApiException("Punycode label could not be positively verified.");
- }
- } catch (IllegalArgumentException e) {
- throw new GigiApiException("Punycode label could not be positively verified.");
- }
- }
-
- public static boolean isVaildDomainPart(String s, boolean allowPunycode) {
- if ( !s.matches("[a-z0-9-]+")) {
- return false;
- }
- if (s.charAt(0) == '-' || s.charAt(s.length() - 1) == '-') {
- return false;
- }
- if (s.length() > 63) {
- return false;
- }
- boolean canBePunycode = s.length() >= 4 && s.charAt(2) == '-' && s.charAt(3) == '-';
- if (canBePunycode && !allowPunycode) {
- return false;
- }
- return true;
- }
-
private static void checkInsert(String suffix) throws GigiApiException {
try (GigiPreparedStatement ps = new GigiPreparedStatement("SELECT 1 FROM `domains` WHERE (`domain`=? OR (CONCAT('.', `domain`)=RIGHT(?,LENGTH(`domain`)+1) OR RIGHT(`domain`,LENGTH(?)+1)=CONCAT('.',?))) AND `deleted` IS NULL")) {
ps.setString(1, suffix);
--- /dev/null
+package org.cacert.gigi.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.IDN;
+import java.net.URL;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+import org.cacert.gigi.GigiApiException;
+
+public class DomainAssessment {
+
+ private static class DomainSet {
+
+ private final Set<String> data;
+
+ public DomainSet(URL u) {
+ this(openStream(u));
+ }
+
+ private static Reader openStream(URL u) {
+ try {
+ return new InputStreamReader(u.openStream(), "UTF-8");
+ } catch (IOException e) {
+ throw new Error(e);
+ }
+ }
+
+ public DomainSet(Reader r) {
+ HashSet<String> contents = new HashSet<>();
+ try {
+ BufferedReader br = new BufferedReader(r);
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("#")) {
+ continue;
+ }
+ if (line.isEmpty()) {
+ continue;
+ }
+ contents.add(line);
+ }
+ } catch (IOException e) {
+ throw new Error(e);
+ }
+ data = Collections.unmodifiableSet(contents);
+ }
+
+ public boolean contains(String suffix) {
+ while (suffix.contains(".")) {
+ if (data.contains(suffix)) {
+ return true;
+ }
+ suffix = suffix.substring(suffix.indexOf('.') + 1);
+ }
+ return data.contains(suffix);
+ }
+ }
+
+ private static DomainSet financial;
+
+ private static final DomainSet idn_enabled = new DomainSet(DomainAssessment.class.getResource("idn_enabled.dat"));
+
+ public static boolean isHighFinancialValue(String suffix) {
+ return financial.contains(suffix);
+ }
+
+ public static void checkCertifiableDomain(String domain, boolean hasPunycodeRight) throws GigiApiException {
+ if (isHighFinancialValue(domain)) {
+ throw new GigiApiException("Domain blocked for automatic adding.");
+ }
+ String[] parts = domain.split("\\.", -1);
+ if (parts.length < 2) {
+ throw new GigiApiException("Domain does not contain '.'.");
+ }
+ boolean neededPunycode = false;
+ for (int i = parts.length - 1; i >= 0; i--) {
+ if ( !isValidDomainPart(parts[i])) {
+ throw new GigiApiException("Syntax error in Domain.");
+ }
+ boolean canBePunycode = parts[i].length() >= 4 && parts[i].charAt(2) == '-' && parts[i].charAt(3) == '-';
+ if (canBePunycode) {
+ if ( !hasPunycodeRight) {
+ throw new GigiApiException("Punycode domain without specific right.");
+ }
+ punycodeDecode(parts[i]);
+ neededPunycode = true;
+ }
+
+ }
+ if (neededPunycode && !idn_enabled.contains(domain)) {
+ throw new GigiApiException("Punycode not allowed under this TLD.");
+ }
+
+ String publicSuffix = PublicSuffixes.getInstance().getRegistrablePart(domain);
+ if ( !domain.equals(publicSuffix)) {
+ throw new GigiApiException("You may only register a domain with exactly one label before the public suffix.");
+ }
+
+ if (("." + domain).matches("(\\.[0-9]*)*")) {
+ // This is not reached because we currently have no TLD that is
+ // numbers only. But who knows..
+ // Better safe than sorry.
+ throw new GigiApiException("IP Addresses are not allowed.");
+ }
+ }
+
+ private static String punycodeDecode(String label) throws GigiApiException {
+ if (label.charAt(2) != '-' || label.charAt(3) != '-') {
+ return label; // is no punycode
+ }
+ if ( !label.startsWith("xn--")) {
+ throw new GigiApiException("Unknown ACE prefix.");
+ }
+ try {
+ String unicode = IDN.toUnicode(label);
+ if (unicode.startsWith("xn--")) {
+ throw new GigiApiException("Punycode label could not be positively verified.");
+ }
+ return unicode;
+ } catch (IllegalArgumentException e) {
+ throw new GigiApiException("Punycode label could not be positively verified.");
+ }
+ }
+
+ public static boolean isValidDomainPart(String s) {
+ if ( !s.matches("[a-z0-9-]+")) {
+ return false;
+ }
+ if (s.charAt(0) == '-' || s.charAt(s.length() - 1) == '-') {
+ return false;
+ }
+ if (s.length() > 63) {
+ return false;
+ }
+ return true;
+ }
+
+ public static void init(Properties conf) {
+ String financialName = conf.getProperty("highFinancialValue");
+ if (financialName == null) {
+ throw new Error("No property highFinancialValue was configured");
+ }
+ try {
+ financial = new DomainSet(new InputStreamReader(new FileInputStream(new File(financialName)), "UTF-8"));
+ } catch (IOException e) {
+ throw new Error(e);
+ }
+ }
+}
--- /dev/null
+ac
+ar
+at
+biz
+br
+cat
+ch
+cl
+cn
+de
+dk
+es
+fi
+gr
+hu
+info
+io
+ir
+is
+jp
+kr
+li
+lt
+museum
+no
+org
+pl
+pr
+se
+sh
+th
+tm
+tw
+vn
+# from https://data.iana.org/TLD/tlds-alpha-by-domain.txt
import static org.junit.Assert.*;
-import org.cacert.gigi.dbObjects.Domain;
+import org.cacert.gigi.testUtils.ConfiguredTest;
+import org.cacert.gigi.util.DomainAssessment;
import org.junit.Test;
-public class DomainVerification {
+public class DomainVerification extends ConfiguredTest {
@Test
public void testDomainPart() {
- assertTrue(Domain.isVaildDomainPart("cacert", false));
- assertTrue(Domain.isVaildDomainPart("de", false));
- assertTrue(Domain.isVaildDomainPart("ha2-a", false));
- assertTrue(Domain.isVaildDomainPart("ha2--a", false));
- assertTrue(Domain.isVaildDomainPart("h--a", false));
- assertFalse(Domain.isVaildDomainPart("xn--bla", false));
- assertFalse(Domain.isVaildDomainPart("-xnbla", false));
- assertFalse(Domain.isVaildDomainPart("xnbla-", false));
- assertFalse(Domain.isVaildDomainPart("", false));
- assertTrue(Domain.isVaildDomainPart("2xnbla", false));
- assertTrue(Domain.isVaildDomainPart("xnbla2", false));
- assertTrue(Domain.isVaildDomainPart("123", false));
- assertTrue(Domain.isVaildDomainPart("abcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxy1234567890123", false));
- assertFalse(Domain.isVaildDomainPart("abcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxy12345678901234", false));
+ assertTrue(DomainAssessment.isValidDomainPart("cacert"));
+ assertTrue(DomainAssessment.isValidDomainPart("de"));
+ assertTrue(DomainAssessment.isValidDomainPart("ha2-a"));
+ assertTrue(DomainAssessment.isValidDomainPart("ha2--a"));
+ assertTrue(DomainAssessment.isValidDomainPart("h--a"));
+ assertFalse(DomainAssessment.isValidDomainPart("-xnbla"));
+ assertFalse(DomainAssessment.isValidDomainPart("xnbla-"));
+ assertFalse(DomainAssessment.isValidDomainPart(""));
+ assertTrue(DomainAssessment.isValidDomainPart("2xnbla"));
+ assertTrue(DomainAssessment.isValidDomainPart("xnbla2"));
+ assertTrue(DomainAssessment.isValidDomainPart("123"));
+ assertTrue(DomainAssessment.isValidDomainPart("abcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxy1234567890123"));
+ assertFalse(DomainAssessment.isValidDomainPart("abcdefghijklmnopqrstuvwxyabcdefghijklmnopqrstuvwxy12345678901234"));
}
@Test
- public void testDomainCertifyable() {
- isCertifyableDomain(true, "cacert.org", false);
- isCertifyableDomain(true, "cacert.de", false);
- isCertifyableDomain(true, "cacert.org", false);
- isCertifyableDomain(true, "cacert.org", false);
- isCertifyableDomain(true, "1234.org", false);
- isCertifyableDomain(false, "a.cacert.org", true);
- isCertifyableDomain(false, "gigi.local", true);
- isCertifyableDomain(false, "org", true);
- isCertifyableDomain(false, "'a.org", true);
- isCertifyableDomain(false, ".org", true);
- isCertifyableDomain(false, ".org.", true);
+ public void testDomainCertifiable() {
+ isCertifiableDomain(true, "cacert.org", false);
+ isCertifiableDomain(true, "cacert.de", false);
+ isCertifiableDomain(true, "cacert.org", false);
+ isCertifiableDomain(true, "cacert.org", false);
+ isCertifiableDomain(true, "1234.org", false);
+ isCertifiableDomain(false, "a.cacert.org", true);
+ isCertifiableDomain(false, "gigi.local", true);
+ isCertifiableDomain(false, "org", true);
+ isCertifiableDomain(false, "'a.org", true);
+ isCertifiableDomain(false, ".org", true);
+ isCertifiableDomain(false, ".org.", true);
// non-real-punycode
- isCertifyableDomain(true, "xna-ae.de", false);
- isCertifyableDomain(true, "xn-aae.de", false);
+ isCertifiableDomain(true, "xna-ae.de", false);
+ isCertifiableDomain(true, "xn-aae.de", false);
// illegal punycode:
// illegal ace prefix
- isCertifyableDomain(false, "aa--b.com", true);
- isCertifyableDomain(false, "xm--ae-a.de", true);
+ isCertifiableDomain(false, "aa--b.com", true);
+ isCertifiableDomain(false, "xm--ae-a.de", true);
// illegal punycode content
- isCertifyableDomain(false, "xn--ae-a.com", true);
- isCertifyableDomain(false, "xn--ae.de", true);
- isCertifyableDomain(false, "xn--ae-a.org", true);
- isCertifyableDomain(false, "xn--ae-a.de", true);
+ isCertifiableDomain(false, "xn--ae-a.com", true);
+ isCertifiableDomain(false, "xn--ae.de", true);
+ isCertifiableDomain(false, "xn--ae-a.org", true);
+ isCertifiableDomain(false, "xn--ae-a.de", true);
// valid punycode requires permission
- isCertifyableDomain(true, "xn--4ca0bs.de", true);
- isCertifyableDomain(false, "xn--4ca0bs.de", false);
- isCertifyableDomain(true, "xn--a-zfa9cya.de", true);
- isCertifyableDomain(false, "xn--a-zfa9cya.de", false);
+ isCertifiableDomain(true, "xn--4ca0bs.de", true);
+ isCertifiableDomain(false, "xn--4ca0bs.de", false);
+ isCertifiableDomain(true, "xn--a-zfa9cya.de", true);
+ isCertifiableDomain(false, "xn--a-zfa9cya.de", false);
// valid punycode does not help under .com
- isCertifyableDomain(false, "xn--a-zfa9cya.com", true);
- isCertifyableDomain(true, "zfa9cya.com", true);
+ isCertifiableDomain(false, "xn--a-zfa9cya.com", true);
+ isCertifiableDomain(true, "zfa9cya.com", true);
- isCertifyableDomain(false, "127.0.0.1", false);
- isCertifyableDomain(false, "::1", false);
- isCertifyableDomain(false, "127.0.0.1", true);
- isCertifyableDomain(false, "::1", true);
+ isCertifiableDomain(false, "127.0.0.1", false);
+ isCertifiableDomain(false, "::1", false);
+ isCertifiableDomain(false, "127.0.0.1", true);
+ isCertifiableDomain(false, "::1", true);
}
- private void isCertifyableDomain(boolean b, String string, boolean puny) {
+ @Test
+ public void testFinancial() {
+ isCertifiableDomain(false, "google.com", true);
+ isCertifiableDomain(false, "twitter.com", true);
+ }
+
+ private void isCertifiableDomain(boolean b, String string, boolean puny) {
try {
- Domain.checkCertifyableDomain(string, puny);
+ DomainAssessment.checkCertifiableDomain(string, puny);
assertTrue(b);
} catch (GigiApiException e) {
assertFalse(e.getMessage(), b);
assertNotNull(addDomain(cookie, uniq + ".de"));
}
+ @Test
+ public void testInvalid() throws IOException {
+ assertNotNull(addDomain(cookie, uniq + ".invalid"));
+ }
+
+ @Test
+ public void testHighFinancialValue() throws IOException {
+ assertNotNull(addDomain(cookie, "google.com"));
+ }
+
public static String addDomain(String session, String domain) throws IOException {
return executeBasicWebInteraction(session, DomainOverview.PATH, "adddomain&newdomain=" + URLEncoder.encode(domain, "UTF-8"), 1);
}
import org.cacert.gigi.database.DatabaseConnection;
import org.cacert.gigi.database.DatabaseConnection.Link;
+import org.cacert.gigi.util.DomainAssessment;
import org.cacert.gigi.util.PEM;
import org.junit.BeforeClass;
private static boolean envInited = false;
@BeforeClass
- public static void initEnvironment() throws IOException {
+ public static void initEnvironmentHook() throws IOException {
+ initEnvironment();
+ }
+
+ public static Properties initEnvironment() throws IOException {
TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
+ Properties props = generateProps();
if (envInited) {
- return;
+ return props;
}
envInited = true;
+ DomainAssessment.init(props);
try (FileInputStream inStream = new FileInputStream("config/test.properties")) {
testProps.load(inStream);
}
throw new Error(e);
}
}
+ return props;
+
+ }
+
+ private static Properties generateProps() throws Error {
+ Properties mainProps = new Properties();
+ File out = new File("financial.dat");
+ if ( !out.exists()) {
+ try (FileOutputStream fos = new FileOutputStream(out)) {
+ fos.write("google.com\ntwitter.com\n".getBytes("UTF-8"));
+ } catch (IOException e) {
+ throw new Error(e);
+ }
+ }
+ mainProps.setProperty("highFinancialValue", out.getAbsolutePath());
+ return mainProps;
}
public static KeyPair generateKeypair() throws GeneralSecurityException {
}
@BeforeClass
- public static void initEnvironment() {
+ public static void initEnvironmentHook() {
+ initEnvironment();
+ }
+
+ public static Properties initEnvironment() {
try {
- ConfiguredTest.initEnvironment();
+ Properties mainProps = ConfiguredTest.initEnvironment();
purgeDatabase();
String type = testProps.getProperty("type");
- Properties mainProps = generateMainProps();
+ generateMainProps(mainProps);
ServerConstants.init(mainProps);
if (type.equals("local")) {
url = testProps.getProperty("name.www") + ":" + testProps.getProperty("serverPort.https");
if (testProps.getProperty("withSigner", "false").equals("true")) {
SimpleSigner.runSigner();
}
- return;
+ return mainProps;
}
url = testProps.getProperty("name.www") + ":" + testProps.getProperty("serverPort.https");
gigi = Runtime.getRuntime().exec(testProps.getProperty("java"));
ter = new TestEmailReceiver(new InetSocketAddress("localhost", 8473));
ter.start();
SimpleSigner.runSigner();
+ return mainProps;
} catch (IOException e) {
throw new Error(e);
} catch (SQLException e1) {
- e1.printStackTrace();
+ throw new Error(e1);
} catch (InterruptedException e) {
- e.printStackTrace();
+ throw new Error(e);
}
}
u.openConnection().getHeaderField("Location");
}
- private static Properties generateMainProps() {
- Properties mainProps = new Properties();
+ private static void generateMainProps(Properties mainProps) {
mainProps.setProperty("testrunner", "true");
mainProps.setProperty("host", "127.0.0.1");
mainProps.setProperty("name.secure", testProps.getProperty("name.secure"));
mainProps.setProperty("sql.user", testProps.getProperty("sql.user"));
mainProps.setProperty("sql.password", testProps.getProperty("sql.password"));
mainProps.setProperty("testing", "true");
- return mainProps;
}
@AfterClass
--- /dev/null
+package org.cacert.gigi.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+public class HighFinancialValueFetcher {
+
+ public static void main(String[] args) throws IOException {
+ int max = 1000;
+ if (args.length > 1) {
+ max = Integer.parseInt(args[1]);
+ }
+ PrintWriter fos = new PrintWriter(new File(args[0]), "UTF-8");
+ ZipInputStream zis = new ZipInputStream(new URL("https://s3.amazonaws.com/alexa-static/top-1m.csv.zip").openStream());
+ ZipEntry ze;
+ outer:
+ while ((ze = zis.getNextEntry()) != null) {
+ System.out.println(ze.getName());
+ BufferedReader br = new BufferedReader(new InputStreamReader(zis, "UTF-8"));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ int i = Integer.parseInt(parts[0]);
+ if (i > max) {
+ zis.close();
+ break outer;
+ }
+ fos.println(parts[1]);
+ System.out.println(line);
+ }
+ }
+ fos.close();
+ }
+
+}
projects / gigi.git / commitdiff