diff --git a/source/net/yacy/cora/document/id/DigestURL.java b/source/net/yacy/cora/document/id/DigestURL.java index 6b5f60d3b..93b7f00a2 100644 --- a/source/net/yacy/cora/document/id/DigestURL.java +++ b/source/net/yacy/cora/document/id/DigestURL.java @@ -221,7 +221,7 @@ public class DigestURL extends MultiProtocolURL implements Serializable { /** * calculated YaCy-Hash of this URI * - * @note needs DNS lookup to check if the addresses domain is local + * @note needs DNS lookup to check if the addresses domain is local (when the top-level domain part in the host name is unknown) * that causes that this method may be very slow * * @return hash diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index 15cb43fa3..8e98029e8 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -478,21 +478,28 @@ public class Domains { "ZW=Zimbabwe", "YT=Mayotte" }; - private static final String[] TLD_Generic = { - "COM=US Commercial", + + private static final String[] TLD_Sponsored = { "AERO=The air-transport industry", - "ARPA=operationally-critical infrastructural identifier spaces", - "BIZ=Business", "COOP=cooperative associations", - "INFO=", "JOBS=human resource managers", - "MOBI=mobile products and services", "MUSEUM=Museums", - "NAME=Individuals", - "PRO=Credentialed professionals", "TEL=Published contact data", "TRAVEL=The travel industry", "INT=International", + }; + + private static final String[] TLD_Infrastructure = { + "ARPA=operationally-critical infrastructural identifier spaces", + }; + + private static final String[] TLD_GenericRestricted = { + "BIZ=Business", + "NAME=Individuals", + "PRO=Credentialed professionals", + }; + + private static final String[] TLD_OpenNIC = { // domains from the OpenNIC project, http://www.opennicproject.org, see also http://wiki.opennic.glue/OpenNICNamespaces "GLUE=OpenNIC Internal Architectural use", "BBS=OpenNIC Bulletin Board System servers", @@ -658,7 +665,10 @@ public class Domains { ccSLD_TLD.addAll(Arrays.asList(ccSLD_TLD_list)); } - private static Map TLDID = new ConcurrentHashMap(32); + /** + * Map top-level domains (lower caes) to TLD category identifiers. + */ + private static Map TLDID = new ConcurrentHashMap(); //private static HashMap TLDName = new HashMap(); private static void insertTLDProps(final String[] TLDList, final int id) { @@ -697,8 +707,21 @@ public class Domains { insertTLDProps(TLD_MiddleEastWestAsia, TLD_MiddleEastWestAsia_ID); insertTLDProps(TLD_NorthAmericaOceania, TLD_NorthAmericaOceania_ID); insertTLDProps(TLD_Africa, TLD_Africa_ID); - insertTLDProps(TLD_Generic, TLD_Generic_ID); - // the id=7 is used to flag local addresses + for(GenericTLD tld : GenericTLD.values()) { + TLDID.put(tld.getDomainName(), TLD_Generic_ID); + } + /* + * IANA lists the following top-level domains in other catetories than 'generic' but we + * still associate them with YaCy's TLD_Generic_ID otherwise the URLs hash would + * be modified + */ + insertTLDProps(TLD_GenericRestricted, TLD_Generic_ID); + insertTLDProps(TLD_Infrastructure, TLD_Generic_ID); + insertTLDProps(TLD_Sponsored, TLD_Generic_ID); + + insertTLDProps(TLD_OpenNIC, TLD_Generic_ID); + + // the id=7 (TLD_Local_ID) is used to flag local addresses } private static KeyList globalHosts = null; @@ -1113,6 +1136,7 @@ public class Domains { public static int getDomainID(final String host, final InetAddress hostaddress) { if (host == null || host.isEmpty()) return TLD_Local_ID; final int p = host.lastIndexOf('.'); + // TODO (must be careful as this would change URL hash generation) : lower case the TLD part before checking its category id, as the TLDID map contains lower cased TLDs as keys */ final String tld = (p > 0) ? host.substring(p + 1) : ""; final Integer i = TLDID.get(tld); if (i != null) return i.intValue(); @@ -1182,6 +1206,7 @@ public class Domains { // check simply if the tld in the host is a known tld final int p = host.lastIndexOf('.'); + // TODO (must be careful as this would change URL hash generation) : lower case the TLD part before checking its category id, as the TLDID map contains lower cased TLDs as keys */ final String tld = (p > 0) ? host.substring(p + 1) : ""; final Integer i = TLDID.get(tld); if (i != null) return false; @@ -1195,7 +1220,8 @@ public class Domains { private static boolean isLocal(final InetAddress a) { final boolean localp = noLocalCheck || // DO NOT REMOVE THIS! it is correct to return true if the check is off - a == null || + a == null || // TODO returning true here after dns resolution failed can make hash generation inconsistent on some hosts + // (hash is marked with TLD_LOCAL_ID when host name is not found within timeout, but then is marked again with TLD_Generic when the host name is found within timeout on another request) a.isAnyLocalAddress() || a.isLinkLocalAddress() || a.isLoopbackAddress() || @@ -1313,4 +1339,4 @@ public class Domains { System.out.println("Intranet IP: " + b); } } -} +} \ No newline at end of file diff --git a/source/net/yacy/cora/protocol/GenericTLD.java b/source/net/yacy/cora/protocol/GenericTLD.java new file mode 100644 index 000000000..07fe19418 --- /dev/null +++ b/source/net/yacy/cora/protocol/GenericTLD.java @@ -0,0 +1,1291 @@ +// GenericTLD.java +// ----------------------- +// part of YaCy +// Copyright 2017 by luccioman; https://github.com/luccioman +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.cora.protocol; + +import java.util.Locale; + +/** + * Enumeration of generic top-level domains (gTLD). Please use the + * {@link #getDomainName()} function to get a consistent domain name value, + * notably for IDNs (Internationalized Domain Names) that contain characters + * that can not be used as part of a Java enum name. + * + * @see IANA Root Zone Database + * reference + */ +public enum GenericTLD { + + aaa("American Automobile Association, Inc."), + aarp("AARP"), + abarth("Fiat Chrysler Automobiles N.V."), + abb("ABB Ltd"), + abbott("Abbott Laboratories, Inc."), + abbvie("AbbVie Inc."), + abc("Disney Enterprises, Inc."), + able("Able Inc."), + abogado("Top Level Domain Holdings Limited"), + abudhabi("Abu Dhabi Systems and Information Centre"), + academy("Half Oaks, LLC"), + accenture("Accenture plc"), + accountant("dot Accountant Limited"), + accountants("Knob Town, LLC"), + aco("ACO Severin Ahlmann GmbH & Co. KG"), + active("Active Network, LLC"), + actor("United TLD Holdco Ltd."), + adac("Allgemeiner Deutscher Automobil-Club e.V. (ADAC)"), + ads("Charleston Road Registry Inc."), + adult("ICM Registry AD LLC"), + aeg("Aktiebolaget Electrolux"), + aetna("Aetna Life Insurance Company"), + afamilycompany("Johnson Shareholdings, Inc."), + afl("Australian Football League"), + africa("ZA Central Registry NPC trading as Registry.Africa"), + agakhan("Fondation Aga Khan (Aga Khan Foundation)"), + agency("Steel Falls, LLC"), + aig("American International Group, Inc."), + aigo("aigo Digital Technology Co,Ltd."), + airbus("Airbus S.A.S."), + airforce("United TLD Holdco Ltd."), + airtel("Bharti Airtel Limited"), + akdn("Fondation Aga Khan (Aga Khan Foundation)"), + alfaromeo("Fiat Chrysler Automobiles N.V."), + alibaba("Alibaba Group Holding Limited"), + alipay("Alibaba Group Holding Limited"), + allfinanz("Allfinanz Deutsche Vermögensberatung Aktiengesellschaft"), + allstate("Allstate Fire and Casualty Insurance Company"), + ally("Ally Financial Inc."), + alsace("REGION GRAND EST"), + alstom("ALSTOM"), + americanexpress("American Express Travel Related Services Company, Inc."), + americanfamily("AmFam, Inc."), + amex("American Express Travel Related Services Company, Inc."), + amfam("AmFam, Inc."), + amica("Amica Mutual Insurance Company"), + amsterdam("Gemeente Amsterdam"), + analytics("Campus IP LLC"), + android("Charleston Road Registry Inc."), + anquan("QIHOO 360 TECHNOLOGY CO. LTD."), + anz("Australia and New Zealand Banking Group Limited"), + aol("AOL Inc."), + apartments("June Maple, LLC"), + app("Charleston Road Registry Inc."), + apple("Apple Inc."), + aquarelle("Aquarelle.com"), + arab("League of Arab States"), + aramco("Aramco Services Company"), + archi("STARTING DOT LIMITED"), + army("United TLD Holdco Ltd."), + art("UK Creative Ideas Limited"), + arte("Association Relative à la Télévision Européenne G.E.I.E."), + asda("Wal-Mart Stores, Inc."), + associates("Baxter Hill, LLC"), + athleta("The Gap, Inc."), + attorney("United TLD Holdco, Ltd"), + auction("United TLD HoldCo, Ltd."), + audi("AUDI Aktiengesellschaft"), + audible("Amazon Registry Services, Inc."), + audio("Uniregistry, Corp."), + auspost("Australian Postal Corporation"), + author("Amazon Registry Services, Inc."), + auto("Cars Registry Limited"), + autos("DERAutos, LLC"), + avianca("Aerovias del Continente Americano S.A. Avianca"), + aws("Amazon Registry Services, Inc."), + axa("AXA SA"), + azure("Microsoft Corporation"), + baby("Johnson & Johnson Services, Inc."), + baidu("Baidu, Inc."), + banamex("Citigroup Inc."), + bananarepublic("The Gap, Inc."), + band("United TLD Holdco, Ltd"), + bank("fTLD Registry Services, LLC"), + bar("Punto 2012 Sociedad Anonima Promotora de Inversion de Capital Variable"), + barcelona("Municipi de Barcelona"), + barclaycard("Barclays Bank PLC"), + barclays("Barclays Bank PLC"), + barefoot("Gallo Vineyards, Inc."), + bargains("Half Hallow, LLC"), + baseball("MLB Advanced Media DH, LLC"), + basketball("Fédération Internationale de Basketball (FIBA)"), + bauhaus("Werkhaus GmbH"), + bayern("Bayern Connect GmbH"), + bbc("British Broadcasting Corporation"), + bbt("BB&T Corporation"), + bbva("BANCO BILBAO VIZCAYA ARGENTARIA, S.A."), + bcg("The Boston Consulting Group, Inc."), + bcn("Municipi de Barcelona"), + beats("Beats Electronics, LLC"), + beauty("L'Oréal"), + beer("Top Level Domain Holdings Limited"), + bentley("Bentley Motors Limited"), + berlin("dotBERLIN GmbH & Co. KG"), + best("BestTLD Pty Ltd"), + bestbuy("BBY Solutions, Inc."), + bet("Afilias plc"), + bharti("Bharti Enterprises (Holding) Private Limited"), + bible("American Bible Society"), + bid("dot Bid Limited"), + bike("Grand Hollow, LLC"), + bing("Microsoft Corporation"), + bingo("Sand Cedar, LLC"), + bio("STARTING DOT LIMITED"), + black("Afilias plc"), + blackfriday("Uniregistry, Corp."), + blanco("BLANCO GmbH + Co KG"), + blockbuster("Dish DBS Corporation"), + blog("Knock Knock WHOIS There, LLC"), + bloomberg("Bloomberg IP Holdings LLC"), + blue("Afilias plc"), + bms("Bristol-Myers Squibb Company"), + bmw("Bayerische Motoren Werke Aktiengesellschaft"), + bnl("Banca Nazionale del Lavoro"), + bnpparibas("BNP Paribas"), + boats("DERBoats, LLC"), + boehringer("Boehringer Ingelheim International GmbH"), + bofa("Bank of America Corporation"), + bom("Núcleo de Informação e Coordenação do Ponto BR - NIC.br"), + bond("Bond University Limited"), + boo("Charleston Road Registry Inc."), + book("Amazon Registry Services, Inc."), + booking("Booking.com B.V."), + boots("THE BOOTS COMPANY PLC"), + bosch("Robert Bosch GMBH"), + bostik("Bostik SA"), + boston("Boston TLD Management, LLC"), + bot("Amazon Registry Services, Inc."), + boutique("Over Galley, LLC"), + box("NS1 Limited"), + bradesco("Banco Bradesco S.A."), + bridgestone("Bridgestone Corporation"), + broadway("Celebrate Broadway, Inc."), + broker("DOTBROKER REGISTRY LTD"), + brother("Brother Industries, Ltd."), + brussels("DNS.be vzw"), + budapest("Top Level Domain Holdings Limited"), + bugatti("Bugatti International SA"), + build("Plan Bee LLC"), + builders("Atomic Madison, LLC"), + business("Spring Cross, LLC"), + buy("Amazon Registry Services, INC"), + buzz("DOTSTRATEGY CO."), + bzh("Association www.bzh"), + cab("Half Sunset, LLC"), + cafe("Pioneer Canyon, LLC"), + cal("Charleston Road Registry Inc."), + call("Amazon Registry Services, Inc."), + calvinklein("PVH gTLD Holdings LLC"), + cam("AC Webconnecting Holding B.V."), + camera("Atomic Maple, LLC"), + camp("Delta Dynamite, LLC"), + cancerresearch("Australian Cancer Research Foundation"), + canon("Canon Inc."), + capetown("ZA Central Registry NPC trading as ZA Central Registry"), + capital("Delta Mill, LLC"), + capitalone("Capital One Financial Corporation"), + car("Cars Registry Limited"), + caravan("Caravan International, Inc."), + cards("Foggy Hollow, LLC"), + care("Goose Cross, LLC"), + career("dotCareer LLC"), + careers("Wild Corner, LLC"), + cars("Cars Registry Limited"), + cartier("Richemont DNS Inc."), + casa("Top Level Domain Holdings Limited"), + CASE("CNH Industrial N.V."), + caseih("CNH Industrial N.V."), + cash("Delta Lake, LLC"), + casino("Binky Sky, LLC"), + catering("New Falls. LLC"), + catholic("Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)"), + cba("COMMONWEALTH BANK OF AUSTRALIA"), + cbn("The Christian Broadcasting Network, Inc."), + cbre("CBRE, Inc."), + cbs("CBS Domains Inc."), + ceb("The Corporate Executive Board Company"), + center("Tin Mill, LLC"), + ceo("CEOTLD Pty Ltd"), + cern("European Organization for Nuclear Research (\"CERN\")"), + cfa("CFA Institute"), + cfd("DOTCFD REGISTRY LTD"), + chanel("Chanel International B.V."), + channel("Charleston Road Registry Inc."), + chase("JPMorgan Chase Bank, National Association"), + chat("Sand Fields, LLC"), + cheap("Sand Cover, LLC"), + chintai("CHINTAI Corporation"), + chloe("Not assigned"), + christmas("Uniregistry, Corp."), + chrome("Charleston Road Registry Inc."), + chrysler("FCA US LLC."), + church("Holly Fileds, LLC"), + cipriani("Hotel Cipriani Srl"), + circle("Amazon Registry Services, Inc."), + cisco("Cisco Technology, Inc."), + citadel("Citadel Domain LLC"), + citi("Citigroup Inc."), + citic("CITIC Group Corporation"), + city("Snow Sky, LLC"), + cityeats("Lifestyle Domain Holdings, Inc."), + claims("Black Corner, LLC"), + cleaning("Fox Shadow, LLC"), + click("Uniregistry, Corp."), + clinic("Goose Park, LLC"), + clinique("The Estée Lauder Companies Inc."), + clothing("Steel Lake, LLC"), + cloud("ARUBA PEC S.p.A."), + club(".CLUB DOMAINS, LLC"), + clubmed("Club Méditerranée S.A."), + coach("Koko Island, LLC"), + codes("Puff Willow, LLC"), + coffee("Trixy Cover, LLC"), + college("XYZ.COM LLC"), + cologne("NetCologne Gesellschaft für Telekommunikation mbH"), + com("VeriSign Global Registry Services"), + comcast("Comcast IP Holdings I, LLC"), + commbank("COMMONWEALTH BANK OF AUSTRALIA"), + community("Fox Orchard, LLC"), + company("Silver Avenue, LLC"), + compare("iSelect Ltd"), + computer("Pine Mill, LLC"), + comsec("VeriSign, Inc."), + condos("Pine House, LLC"), + construction("Fox Dynamite, LLC"), + consulting("United TLD Holdco, LTD."), + contact("Top Level Spectrum, Inc."), + contractors("Magic Woods, LLC"), + cooking("Top Level Domain Holdings Limited"), + cookingchannel("Lifestyle Domain Holdings, Inc."), + cool("Koko Lake, LLC"), + corsica("Collectivité Territoriale de Corse"), + country("Top Level Domain Holdings Limited"), + coupon("Amazon Registry Services, Inc."), + coupons("Black Island, LLC"), + courses("OPEN UNIVERSITIES AUSTRALIA PTY LTD"), + credit("Snow Shadow, LLC"), + creditcard("Binky Frostbite, LLC"), + creditunion("CUNA Performance Resources, LLC"), + cricket("dot Cricket Limited"), + crown("Crown Equipment Corporation"), + crs("Federated Co-operatives Limited"), + cruise("Viking River Cruises (Bermuda) Ltd."), + cruises("Spring Way, LLC"), + csc("Alliance-One Services, Inc."), + cuisinella("SALM S.A.S."), + cymru("Nominet UK"), + cyou("Beijing Gamease Age Digital Technology Co., Ltd."), + dabur("Dabur India Limited"), + dad("Charleston Road Registry Inc."), + dance("United TLD Holdco Ltd."), + data("Dish DBS Corporation"), + date("dot Date Limited"), + dating("Pine Fest, LLC"), + datsun("NISSAN MOTOR CO., LTD."), + day("Charleston Road Registry Inc."), + dclk("Charleston Road Registry Inc."), + dds("Minds + Machines Group Limited"), + deal("Amazon Registry Services, Inc."), + dealer("Dealer Dot Com, Inc."), + deals("Sand Sunset, LLC"), + degree("United TLD Holdco, Ltd"), + delivery("Steel Station, LLC"), + dell("Dell Inc."), + deloitte("Deloitte Touche Tohmatsu"), + delta("Delta Air Lines, Inc."), + democrat("United TLD Holdco Ltd."), + dental("Tin Birch, LLC"), + dentist("United TLD Holdco, Ltd"), + desi("Desi Networks LLC"), + design("Top Level Design, LLC"), + dev("Charleston Road Registry Inc."), + dhl("Deutsche Post AG"), + diamonds("John Edge, LLC"), + diet("Uniregistry, Corp."), + digital("Dash Park, LLC"), + direct("Half Trail, LLC"), + directory("Extra Madison, LLC"), + discount("Holly Hill, LLC"), + discover("Discover Financial Services"), + dish("Dish DBS Corporation"), + diy("Lifestyle Domain Holdings, Inc."), + dnp("Dai Nippon Printing Co., Ltd."), + docs("Charleston Road Registry Inc."), + doctor("Brice Trail, LLC"), + dodge("FCA US LLC."), + dog("Koko Mill, LLC"), + doha("Communications Regulatory Authority (CRA)"), + domains("Sugar Cross, LLC"), + doosan("Retired"), + dot("Dish DBS Corporation"), + download("dot Support Limited"), + drive("Charleston Road Registry Inc."), + dtv("Dish DBS Corporation"), + dubai("Dubai Smart Government Department"), + duck("Johnson Shareholdings, Inc."), + dunlop("The Goodyear Tire & Rubber Company"), + duns("The Dun & Bradstreet Corporation"), + dupont("E. I. du Pont de Nemours and Company"), + durban("ZA Central Registry NPC trading as ZA Central Registry"), + dvag("Deutsche Vermögensberatung Aktiengesellschaft DVAG"), + dvr("Hughes Satellite Systems Corporation"), + earth("Interlink Co., Ltd."), + eat("Charleston Road Registry Inc."), + eco("Big Room Inc."), + edeka("EDEKA Verband kaufmännischer Genossenschaften e.V."), + education("Brice Way, LLC"), + email("Spring Madison, LLC"), + emerck("Merck KGaA"), + energy("Binky Birch, LLC"), + engineer("United TLD Holdco Ltd."), + engineering("Romeo Canyon"), + enterprises("Snow Oaks, LLC"), + epost("Deutsche Post AG"), + epson("Seiko Epson Corporation"), + equipment("Corn Station, LLC"), + ericsson("Telefonaktiebolaget L M Ericsson"), + erni("ERNI Group Holding AG"), + esq("Charleston Road Registry Inc."), + estate("Trixy Park, LLC"), + esurance("Esurance Insurance Company"), + etisalat("Emirates Telecommunications Corporation (trading as Etisalat)"), + eurovision("European Broadcasting Union (EBU)"), + eus("Puntueus Fundazioa"), + events("Pioneer Maple, LLC"), + everbank("EverBank"), + exchange("Spring Falls, LLC"), + expert("Magic Pass, LLC"), + exposed("Victor Beach, LLC"), + express("Sea Sunset, LLC"), + extraspace("Extra Space Storage LLC"), + fage("Fage International S.A."), + fail("Atomic Pipe, LLC"), + fairwinds("FairWinds Partners, LLC"), + faith("dot Faith Limited"), + family("United TLD Holdco Ltd."), + fan("Asiamix Digital Ltd"), + fans("Asiamix Digital Limited"), + farm("Just Maple, LLC"), + farmers("Farmers Insurance Exchange"), + fashion("Top Level Domain Holdings Limited"), + fast("Amazon Registry Services, Inc."), + fedex("Federal Express Corporation"), + feedback("Top Level Spectrum, Inc."), + ferrari("Fiat Chrysler Automobiles N.V."), + ferrero("Ferrero Trading Lux S.A."), + fiat("Fiat Chrysler Automobiles N.V."), + fidelity("Fidelity Brokerage Services LLC"), + fido("Rogers Communications Canada Inc."), + film("Motion Picture Domain Registry Pty Ltd"), + FINAL("Núcleo de Informação e Coordenação do Ponto BR - NIC.br"), + finance("Cotton Cypress, LLC"), + financial("Just Cover, LLC"), + fire("Amazon Registry Services, Inc."), + firestone("Bridgestone Licensing Services, Inc."), + firmdale("Firmdale Holdings Limited"), + fish("Fox Woods, LLC"), + fishing("Top Level Domain Holdings Limited"), + fit("Minds + Machines Group Limited"), + fitness("Brice Orchard, LLC"), + flickr("Yahoo! Domain Services Inc."), + flights("Fox Station, LLC"), + flir("FLIR Systems, Inc."), + florist("Half Cypress, LLC"), + flowers("Uniregistry, Corp."), + flsmidth("Retired"), + fly("Charleston Road Registry Inc."), + foo("Charleston Road Registry Inc."), + food("Lifestyle Domain Holdings, Inc."), + foodnetwork("Lifestyle Domain Holdings, Inc."), + football("Foggy Farms, LLC"), + ford("Ford Motor Company"), + forex("DOTFOREX REGISTRY LTD"), + forsale("United TLD Holdco, LLC"), + forum("Fegistry, LLC"), + foundation("John Dale, LLC"), + fox("FOX Registry, LLC"), + free("Amazon Registry Services, Inc."), + fresenius("Fresenius Immobilien-Verwaltungs-GmbH"), + frl("FRLregistry B.V."), + frogans("OP3FT"), + frontdoor("Lifestyle Domain Holdings, Inc."), + frontier("Frontier Communications Corporation"), + ftr("Frontier Communications Corporation"), + fujitsu("Fujitsu Limited"), + fujixerox("Xerox DNHC LLC"), + fun("DotSpace, Inc."), + fund("John Castle, LLC"), + furniture("Lone Fields, LLC"), + futbol("United TLD Holdco, Ltd."), + fyi("Silver Tigers, LLC"), + gal("Asociación puntoGAL"), + gallery("Sugar House, LLC"), + gallo("Gallo Vineyards, Inc."), + gallup("Gallup, Inc."), + game("Uniregistry, Corp."), + games("United TLD Holdco Ltd."), + gap("The Gap, Inc."), + garden("Top Level Domain Holdings Limited"), + gbiz("Charleston Road Registry Inc."), + gdn("Joint Stock Company \"Navigation-information systems\""), + gea("GEA Group Aktiengesellschaft"), + gent("Combell nv"), + genting("Resorts World Inc. Pte. Ltd."), + george("Wal-Mart Stores, Inc."), + ggee("GMO Internet, Inc."), + gift("Uniregistry, Corp."), + gifts("Goose Sky, LLC"), + gives("United TLD Holdco Ltd."), + giving("Giving Limited"), + glade("Johnson Shareholdings, Inc."), + glass("Black Cover, LLC"), + gle("Charleston Road Registry Inc."), + global("Dot Global Domain Registry Limited"), + globo("Globo Comunicação e Participações S.A"), + gmail("Charleston Road Registry Inc."), + gmbh("Extra Dynamite, LLC"), + gmo("GMO Internet, Inc."), + gmx("1&1 Mail & Media GmbH"), + godaddy("Go Daddy East, LLC"), + gold("June Edge, LLC"), + goldpoint("YODOBASHI CAMERA CO.,LTD."), + golf("Lone Falls, LLC"), + goo("NTT Resonant Inc."), + goodhands("Allstate Fire and Casualty Insurance Company"), + goodyear("The Goodyear Tire & Rubber Company"), + goog("Charleston Road Registry Inc."), + google("Charleston Road Registry Inc."), + gop("Republican State Leadership Committee, Inc."), + got("Amazon Registry Services, Inc."), + grainger("Grainger Registry Services, LLC"), + graphics("Over Madison, LLC"), + gratis("Pioneer Tigers, LLC"), + green("DotGreen Registry Limited"), + gripe("Corn Sunset, LLC"), + grocery("Wal-Mart Stores, Inc."), + group("Romeo Town, LLC"), + guardian("The Guardian Life Insurance Company of America"), + gucci("Guccio Gucci S.p.a."), + guge("Charleston Road Registry Inc."), + guide("Snow Moon, LLC"), + guitars("Uniregistry, Corp."), + guru("Pioneer Cypress, LLC"), + hair("L'Oreal"), + hamburg("Hamburg Top-Level-Domain GmbH"), + hangout("Charleston Road Registry Inc."), + haus("United TLD Holdco, LTD."), + hbo("HBO Registry Services, Inc."), + hdfc("HOUSING DEVELOPMENT FINANCE CORPORATION LIMITED"), + hdfcbank("HDFC Bank Limited"), + health("DotHealth, LLC"), + healthcare("Silver Glen, LLC"), + help("Uniregistry, Corp."), + helsinki("City of Helsinki"), + here("Charleston Road Registry Inc."), + hermes("Hermes International"), + hgtv("Lifestyle Domain Holdings, Inc."), + hiphop("Uniregistry, Corp."), + hisamitsu("Hisamitsu Pharmaceutical Co.,Inc."), + hitachi("Hitachi, Ltd."), + hiv("Uniregistry, Corp."), + hkt("PCCW-HKT DataCom Services Limited"), + hockey("Half Willow, LLC"), + holdings("John Madison, LLC"), + holiday("Goose Woods, LLC"), + homedepot("Home Depot Product Authority, LLC"), + homegoods("The TJX Companies, Inc."), + homes("DERHomes, LLC"), + homesense("The TJX Companies, Inc."), + honda("Honda Motor Co., Ltd."), + honeywell("Honeywell GTLD LLC"), + horse("Top Level Domain Holdings Limited"), + hospital("Ruby Pike, LLC"), + host("DotHost Inc."), + hosting("Uniregistry, Corp."), + hot("Amazon Registry Services, Inc."), + hoteles("Travel Reservations SRL"), + hotels("Booking.com B.V."), + hotmail("Microsoft Corporation"), + house("Sugar Park, LLC"), + how("Charleston Road Registry Inc."), + hsbc("HSBC Global Services (UK) Limited"), + htc("Not assigned"), + hughes("Hughes Satellite Systems Corporation"), + hyatt("Hyatt GTLD, L.L.C."), + hyundai("Hyundai Motor Company"), + ibm("International Business Machines Corporation"), + icbc("Industrial and Commercial Bank of China Limited"), + ice("IntercontinentalExchange, Inc."), + icu("One.com A/S"), + ieee("IEEE Global LLC"), + ifm("ifm electronic gmbh"), + iinet("Retired"), + ikano("Ikano S.A."), + imamat("Fondation Aga Khan (Aga Khan Foundation)"), + imdb("Amazon Registry Services, Inc."), + immo("Auburn Bloom, LLC"), + immobilien("United TLD Holdco Ltd."), + industries("Outer House, LLC"), + infiniti("NISSAN MOTOR CO., LTD."), + info("Afilias Limited"), + ing("Charleston Road Registry Inc."), + ink("Top Level Design, LLC"), + institute("Outer Maple, LLC"), + insurance("fTLD Registry Services LLC"), + insure("Pioneer Willow, LLC"), + intel("Intel Corporation"), + international("Wild Way, LLC"), + intuit("Intuit Administrative Services, Inc."), + investments("Holly Glen, LLC"), + ipiranga("Ipiranga Produtos de Petroleo S.A."), + irish("Tin Mill LLC"), + iselect("iSelect Ltd"), + ismaili("Fondation Aga Khan (Aga Khan Foundation)"), + ist("Istanbul Metropolitan Municipality"), + istanbul("Istanbul Metropolitan Municipality"), + itau("Itau Unibanco Holding S.A."), + itv("ITV Services Limited"), + iveco("CNH Industrial N.V."), + iwc("Richemont DNS Inc."), + jaguar("Jaguar Land Rover Ltd"), + java("Oracle Corporation"), + jcb("JCB Co., Ltd."), + jcp("JCP Media, Inc."), + jeep("FCA US LLC."), + jetzt("Wild Frostbite, LLC"), + jewelry("Wild Bloom, LLC"), + jio("Affinity Names, Inc."), + jlc("Richemont DNS Inc."), + jll("Jones Lang LaSalle Incorporated"), + jmp("Matrix IP LLC"), + jnj("Johnson & Johnson Services, Inc."), + joburg("ZA Central Registry NPC trading as ZA Central Registry"), + jot("Amazon Registry Services, Inc."), + joy("Amazon Registry Services, Inc."), + jpmorgan("JPMorgan Chase Bank, National Association"), + jprs("Japan Registry Services Co., Ltd."), + juegos("Uniregistry, Corp."), + juniper("JUNIPER NETWORKS, INC."), + kaufen("United TLD Holdco Ltd."), + kddi("KDDI CORPORATION"), + kerryhotels("Kerry Trading Co. Limited"), + kerrylogistics("Kerry Trading Co. Limited"), + kerryproperties("Kerry Trading Co. Limited"), + kfh("Kuwait Finance House"), + kia("KIA MOTORS CORPORATION"), + kim("Afilias plc"), + kinder("Ferrero Trading Lux S.A."), + kindle("Amazon Registry Services, Inc."), + kitchen("Just Goodbye, LLC"), + kiwi("DOT KIWI LIMITED"), + koeln("NetCologne Gesellschaft für Telekommunikation mbH"), + komatsu("Komatsu Ltd."), + kosher("Kosher Marketing Assets LLC"), + kpmg("KPMG International Cooperative (KPMG International Genossenschaft)"), + kpn("Koninklijke KPN N.V."), + krd("KRG Department of Information Technology"), + kred("KredTLD Pty Ltd"), + kuokgroup("Kerry Trading Co. Limited"), + kyoto("Academic Institution: Kyoto Jyoho Gakuen"), + lacaixa("CAIXA D'ESTALVIS I PENSIONS DE BARCELONA"), + ladbrokes("LADBROKES INTERNATIONAL PLC"), + lamborghini("Automobili Lamborghini S.p.A."), + lamer("The Estée Lauder Companies Inc."), + lancaster("LANCASTER"), + lancia("Fiat Chrysler Automobiles N.V."), + lancome("L'Oréal"), + land("Pine Moon, LLC"), + landrover("Jaguar Land Rover Ltd"), + lanxess("LANXESS Corporation"), + lasalle("Jones Lang LaSalle Incorporated"), + lat("ECOM-LAC Federación de Latinoamérica y el Caribe para Internet y el Comercio Electrónico"), + latino("Dish DBS Corporation"), + latrobe("La Trobe University"), + law("Minds + Machines Group Limited"), + lawyer("United TLD Holdco, Ltd"), + lds("IRI Domain Management, LLC"), + lease("Victor Trail, LLC"), + leclerc("A.C.D. LEC Association des Centres Distributeurs Edouard Leclerc"), + lefrak("LeFrak Organization, Inc."), + legal("Blue Falls, LLC"), + lego("LEGO Juris A/S"), + lexus("TOYOTA MOTOR CORPORATION"), + lgbt("Afilias plc"), + liaison("Liaison Technologies, Incorporated"), + lidl("Schwarz Domains und Services GmbH & Co. KG"), + life("Trixy Oaks, LLC"), + lifeinsurance("American Council of Life Insurers"), + lifestyle("Lifestyle Domain Holdings, Inc."), + lighting("John McCook, LLC"), + like("Amazon Registry Services, Inc."), + lilly("Eli Lilly and Company"), + limited("Big Fest, LLC"), + limo("Hidden Frostbite, LLC"), + lincoln("Ford Motor Company"), + linde("Linde Aktiengesellschaft"), + link("Uniregistry, Corp."), + lipsy("Lipsy Ltd"), + live("United TLD Holdco Ltd."), + living("Lifestyle Domain Holdings, Inc."), + lixil("LIXIL Group Corporation"), + loan("dot Loan Limited"), + loans("June Woods, LLC"), + locker("Dish DBS Corporation"), + locus("Locus Analytics LLC"), + loft("Annco, Inc."), + lol("Uniregistry, Corp."), + london("Dot London Domains Limited"), + lotte("Lotte Holdings Co., Ltd."), + lotto("Afilias plc"), + love("Merchant Law Group LLP"), + lpl("LPL Holdings, Inc."), + lplfinancial("LPL Holdings, Inc."), + ltd("Over Corner, LLC"), + ltda("InterNetX Corp."), + lundbeck("H. Lundbeck A/S"), + lupin("LUPIN LIMITED"), + luxe("Top Level Domain Holdings Limited"), + luxury("Luxury Partners LLC"), + macys("Macys, Inc."), + madrid("Comunidad de Madrid"), + maif("Mutuelle Assurance Instituteur France (MAIF)"), + maison("Victor Frostbite, LLC"), + makeup("L'Oréal"), + man("MAN SE"), + management("John Goodbye, LLC"), + mango("PUNTO FA S.L."), + map("Charleston Road Registry Inc."), + market("United TLD Holdco, Ltd"), + marketing("Fern Pass, LLC"), + markets("DOTMARKETS REGISTRY LTD"), + marriott("Marriott Worldwide Corporation"), + marshalls("The TJX Companies, Inc."), + maserati("Fiat Chrysler Automobiles N.V."), + mattel("Mattel Sites, Inc."), + mba("Lone Hollow, LLC"), + mcd("Not assigned"), + mcdonalds("Not assigned"), + mckinsey("McKinsey Holdings, Inc."), + med("Medistry LLC"), + media("Grand Glen, LLC"), + meet("Charleston Road Registry Inc."), + melbourne("The Crown in right of the State of Victoria, represented by its Department of State Development, Business and Innovation"), + meme("Charleston Road Registry Inc."), + memorial("Dog Beach, LLC"), + men("Exclusive Registry Limited"), + menu("Wedding TLD2, LLC"), + meo("MEO Serviços de Comunicações e Multimédia, S.A."), + merckmsd("MSD Registry Holdings, Inc."), + metlife("MetLife Services and Solutions, LLC"), + miami("Top Level Domain Holdings Limited"), + microsoft("Microsoft Corporation"), + mini("Bayerische Motoren Werke Aktiengesellschaft"), + mint("Intuit Administrative Services, Inc."), + mit("Massachusetts Institute of Technology"), + mitsubishi("Mitsubishi Corporation"), + mlb("MLB Advanced Media DH, LLC"), + mls("The Canadian Real Estate Association"), + mma("MMA IARD"), + mobi("Afilias Technologies Limited dba dotMobi"), + mobile("Dish DBS Corporation"), + mobily("GreenTech Consultancy Company W.L.L."), + moda("United TLD Holdco Ltd."), + moe("Interlink Co., Ltd."), + moi("Amazon Registry Services, Inc."), + mom("Uniregistry, Corp."), + monash("Monash University"), + money("Outer McCook, LLC"), + monster("Monster Worldwide, Inc."), + montblanc("Not assigned"), + mopar("FCA US LLC."), + mormon("IRI Domain Management, LLC (\"Applicant\")"), + mortgage("United TLD Holdco, Ltd"), + moscow("Foundation for Assistance for Internet Technologies and Infrastructure Development (FAITID)"), + moto("Motorola Trademark Holdings, LLC"), + motorcycles("DERMotorcycles, LLC"), + mov("Charleston Road Registry Inc."), + movie("New Frostbite, LLC"), + movistar("Telefónica S.A."), + msd("MSD Registry Holdings, Inc."), + mtn("MTN Dubai Limited"), + mtpc("Retired"), + mtr("MTR Corporation Limited"), + mutual("Northwestern Mutual MU TLD Registry, LLC"), + mutuelle("Retired"), + nab("National Australia Bank Limited"), + nadex("Nadex Domains, Inc"), + nagoya("GMO Registry, Inc."), + nationwide("Nationwide Mutual Insurance Company"), + natura("NATURA COSMÉTICOS S.A."), + navy("United TLD Holdco Ltd."), + nba("NBA REGISTRY, LLC"), + nec("NEC Corporation"), + // keep the following TLD commented : it is already listed in Domains.TLD_NorthAmericaOceania and adding it here would modify related URLs hash generation + // net("VeriSign Global Registry Services"), + netbank("COMMONWEALTH BANK OF AUSTRALIA"), + netflix("Netflix, Inc."), + network("Trixy Manor, LLC"), + neustar("NeuStar, Inc."), + NEW("Charleston Road Registry Inc."), + newholland("CNH Industrial N.V."), + news("United TLD Holdco Ltd."), + next("Next plc"), + nextdirect("Next plc"), + nexus("Charleston Road Registry Inc."), + nfl("NFL Reg Ops LLC"), + ngo("Public Interest Registry"), + nhk("Japan Broadcasting Corporation (NHK)"), + nico("DWANGO Co., Ltd."), + nike("NIKE, Inc."), + nikon("NIKON CORPORATION"), + ninja("United TLD Holdco Ltd."), + nissan("NISSAN MOTOR CO., LTD."), + nissay("Nippon Life Insurance Company"), + nokia("Nokia Corporation"), + northwesternmutual("Northwestern Mutual Registry, LLC"), + norton("Symantec Corporation"), + now("Amazon Registry Services, Inc."), + nowruz("Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti."), + nowtv("Starbucks (HK) Limited"), + nra("NRA Holdings Company, INC."), + nrw("Minds + Machines GmbH"), + ntt("NIPPON TELEGRAPH AND TELEPHONE CORPORATION"), + nyc("The City of New York by and through the New York City Department of Information Technology & Telecommunications"), + obi("OBI Group Holding SE & Co. KGaA"), + observer("Top Level Spectrum, Inc."), + off("Johnson Shareholdings, Inc."), + office("Microsoft Corporation"), + okinawa("BRregistry, Inc."), + olayan("Crescent Holding GmbH"), + olayangroup("Crescent Holding GmbH"), + oldnavy("The Gap, Inc."), + ollo("Dish DBS Corporation"), + omega("The Swatch Group Ltd"), + one("One.com A/S"), + ong("Public Interest Registry"), + onl("I-REGISTRY Ltd., Niederlassung Deutschland"), + online("DotOnline Inc."), + onyourside("Nationwide Mutual Insurance Company"), + ooo("INFIBEAM INCORPORATION LIMITED"), + open("American Express Travel Related Services Company, Inc."), + oracle("Oracle Corporation"), + orange("Orange Brand Services Limited"), + // keep the following TLD commented : it is already listed in Domains.TLD_NorthAmericaOceania and adding it here would modify related URLs hash generation + //org("Public Interest Registry (PIR)"), + organic("Afilias plc"), + orientexpress("Retired"), + origins("The Estée Lauder Companies Inc."), + osaka("Osaka Registry Co., Ltd."), + otsuka("Otsuka Holdings Co., Ltd."), + ott("Dish DBS Corporation"), + ovh("OVH SAS"), + page("Charleston Road Registry Inc."), + pamperedchef("Not assigned"), + panasonic("Panasonic Corporation"), + panerai("Richemont DNS Inc."), + paris("City of Paris"), + pars("Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti."), + partners("Magic Glen, LLC"), + parts("Sea Goodbye, LLC"), + party("Blue Sky Registry Limited"), + passagens("Travel Reservations SRL"), + pay("Amazon Registry Services, Inc."), + pccw("PCCW Enterprises Limited"), + pet("Afilias plc"), + pfizer("Pfizer Inc."), + pharmacy("National Association of Boards of Pharmacy"), + phd("Charleston Road Registry Inc."), + philips("Koninklijke Philips N.V."), + phone("Dish DBS Corporation"), + photo("Uniregistry, Corp."), + photography("Sugar Glen, LLC"), + photos("Sea Corner, LLC"), + physio("PhysBiz Pty Ltd"), + piaget("Richemont DNS Inc."), + pics("Uniregistry, Corp."), + pictet("Pictet Europe S.A."), + pictures("Foggy Sky, LLC"), + pid("Top Level Spectrum, Inc."), + pin("Amazon Registry Services, Inc."), + ping("Ping Registry Provider, Inc."), + pink("Afilias plc"), + pioneer("Pioneer Corporation"), + pizza("Foggy Moon, LLC"), + place("Snow Galley, LLC"), + play("Charleston Road Registry Inc."), + playstation("Sony Computer Entertainment Inc."), + plumbing("Spring Tigers, LLC"), + plus("Sugar Mill, LLC"), + pnc("PNC Domain Co., LLC"), + pohl("Deutsche Vermögensberatung Aktiengesellschaft DVAG"), + poker("Afilias plc"), + politie("Politie Nederland"), + porn("ICM Registry PN LLC"), + pramerica("Prudential Financial, Inc."), + praxi("Praxi S.p.A."), + press("DotPress Inc."), + prime("Amazon Registry Services, Inc."), + prod("Charleston Road Registry Inc."), + productions("Magic Birch, LLC"), + prof("Charleston Road Registry Inc."), + progressive("Progressive Casualty Insurance Company"), + promo("Afilias plc"), + properties("Big Pass, LLC"), + property("Uniregistry, Corp."), + protection("XYZ.COM LLC"), + pru("Prudential Financial, Inc."), + prudential("Prudential Financial, Inc."), + pub("United TLD Holdco Ltd."), + pwc("PricewaterhouseCoopers LLP"), + qpon("dotCOOL, Inc."), + quebec("PointQuébec Inc"), + quest("Quest ION Limited"), + qvc("QVC, Inc."), + racing("Premier Registry Limited"), + radio("European Broadcasting Union (EBU)"), + raid("Johnson Shareholdings, Inc."), + read("Amazon Registry Services, Inc."), + realestate("dotRealEstate LLC"), + realtor("Real Estate Domains LLC"), + realty("Fegistry, LLC"), + recipes("Grand Island, LLC"), + red("Afilias plc"), + redstone("Redstone Haute Couture Co., Ltd."), + redumbrella("Travelers TLD, LLC"), + rehab("United TLD Holdco Ltd."), + reise("Foggy Way, LLC"), + reisen("New Cypress, LLC"), + reit("National Association of Real Estate Investment Trusts, Inc."), + reliance("Reliance Industries Limited"), + ren("Beijing Qianxiang Wangjing Technology Development Co., Ltd."), + rent("XYZ.COM LLC"), + rentals("Big Hollow,LLC"), + repair("Lone Sunset, LLC"), + report("Binky Glen, LLC"), + republican("United TLD Holdco Ltd."), + rest("Punto 2012 Sociedad Anonima Promotora de Inversion de Capital Variable"), + restaurant("Snow Avenue, LLC"), + review("dot Review Limited"), + reviews("United TLD Holdco, Ltd."), + rexroth("Robert Bosch GMBH"), + rich("I-REGISTRY Ltd., Niederlassung Deutschland"), + richardli("Pacific Century Asset Management (HK) Limited"), + ricoh("Ricoh Company, Ltd."), + rightathome("Johnson Shareholdings, Inc."), + ril("Reliance Industries Limited"), + rio("Empresa Municipal de Informática SA - IPLANRIO"), + rip("United TLD Holdco Ltd."), + rmit("Royal Melbourne Institute of Technology"), + rocher("Ferrero Trading Lux S.A."), + rocks("United TLD Holdco, LTD."), + rodeo("Top Level Domain Holdings Limited"), + rogers("Rogers Communications Canada Inc."), + room("Amazon Registry Services, Inc."), + rsvp("Charleston Road Registry Inc."), + rugby("World Rugby Strategic Developments Limited"), + ruhr("regiodot GmbH & Co. KG"), + run("Snow Park, LLC"), + rwe("RWE AG"), + ryukyu("BRregistry, Inc."), + saarland("dotSaarland GmbH"), + safe("Amazon Registry Services, Inc."), + safety("Safety Registry Services, LLC."), + sakura("SAKURA Internet Inc."), + sale("United TLD Holdco, Ltd"), + salon("Outer Orchard, LLC"), + samsclub("Wal-Mart Stores, Inc."), + samsung("SAMSUNG SDS CO., LTD"), + sandvik("Sandvik AB"), + sandvikcoromant("Sandvik AB"), + sanofi("Sanofi"), + sap("SAP AG"), + sapo("MEO Serviços de Comunicações e Multimédia, S.A."), + sarl("Delta Orchard, LLC"), + sas("Research IP LLC"), + save("Amazon Registry Services, Inc."), + saxo("Saxo Bank A/S"), + sbi("STATE BANK OF INDIA"), + sbs("SPECIAL BROADCASTING SERVICE CORPORATION"), + sca("SVENSKA CELLULOSA AKTIEBOLAGET SCA (publ)"), + scb("The Siam Commercial Bank Public Company Limited (\"SCB\")"), + schaeffler("Schaeffler Technologies AG & Co. KG"), + schmidt("SALM S.A.S."), + scholarships("Scholarships.com, LLC"), + school("Little Galley, LLC"), + schule("Outer Moon, LLC"), + schwarz("Schwarz Domains und Services GmbH & Co. KG"), + science("dot Science Limited"), + scjohnson("Johnson Shareholdings, Inc."), + scor("SCOR SE"), + scot("Dot Scot Registry Limited"), + search("Charleston Road Registry Inc."), + seat("SEAT, S.A. (Sociedad Unipersonal)"), + secure("Amazon Registry Services, Inc."), + security("XYZ.COM LLC"), + seek("Seek Limited"), + select("iSelect Ltd"), + sener("Sener Ingeniería y Sistemas, S.A."), + services("Fox Castle, LLC"), + ses("SES"), + seven("Seven West Media Ltd"), + sew("SEW-EURODRIVE GmbH & Co KG"), + sex("ICM Registry SX LLC"), + sexy("Uniregistry, Corp."), + sfr("Societe Francaise du Radiotelephone - SFR"), + shangrila("Shangri‐La International Hotel Management Limited"), + sharp("Sharp Corporation"), + shaw("Shaw Cablesystems G.P."), + shell("Shell Information Technology International Inc"), + shia("Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti."), + shiksha("Afilias plc"), + shoes("Binky Galley, LLC"), + shop("GMO Registry, Inc."), + shopping("Over Keep, LLC"), + shouji("QIHOO 360 TECHNOLOGY CO. LTD."), + show("Snow Beach, LLC"), + showtime("CBS Domains Inc."), + shriram("Shriram Capital Ltd."), + silk("Amazon Registry Services, Inc."), + sina("Sina Corporation"), + singles("Fern Madison, LLC"), + site("DotSite Inc."), + ski("STARTING DOT LIMITED"), + skin("L'Oréal"), + sky("Sky International AG"), + skype("Microsoft Corporation"), + sling("Hughes Satellite Systems Corporation"), + smart("Smart Communications, Inc. (SMART)"), + smile("Amazon Registry Services, Inc."), + sncf("SNCF (Société Nationale des Chemins de fer Francais)"), + soccer("Foggy Shadow, LLC"), + social("United TLD Holdco Ltd."), + softbank("SoftBank Group Corp."), + software("United TLD Holdco, Ltd"), + sohu("Sohu.com Limited"), + solar("Ruby Town, LLC"), + solutions("Silver Cover, LLC"), + song("Amazon Registry Services, Inc."), + sony("Sony Corporation"), + soy("Charleston Road Registry Inc."), + space("DotSpace Inc."), + spiegel("SPIEGEL-Verlag Rudolf Augstein GmbH & Co. KG"), + spot("Amazon Registry Services, Inc."), + spreadbetting("DOTSPREADBETTING REGISTRY LTD"), + srl("InterNetX Corp."), + srt("FCA US LLC."), + stada("STADA Arzneimittel AG"), + staples("Staples, Inc."), + star("Star India Private Limited"), + starhub("StarHub Limited"), + statebank("STATE BANK OF INDIA"), + statefarm("State Farm Mutual Automobile Insurance Company"), + statoil("Statoil ASA"), + stc("Saudi Telecom Company"), + stcgroup("Saudi Telecom Company"), + stockholm("Stockholms kommun"), + storage("XYZ.COM LLC"), + store("DotStore Inc."), + stream("dot Stream Limited"), + studio("United TLD Holdco Ltd."), + study("OPEN UNIVERSITIES AUSTRALIA PTY LTD"), + style("Binky Moon, LLC"), + sucks("Vox Populi Registry Ltd."), + supplies("Atomic Fields, LLC"), + supply("Half Falls, LLC"), + support("Grand Orchard, LLC"), + surf("Top Level Domain Holdings Limited"), + surgery("Tin Avenue, LLC"), + suzuki("SUZUKI MOTOR CORPORATION"), + swatch("The Swatch Group Ltd"), + swiftcover("Swiftcover Insurance Services Limited"), + swiss("Swiss Confederation"), + sydney("State of New South Wales, Department of Premier and Cabinet"), + symantec("Symantec Corporation"), + systems("Dash Cypress, LLC"), + tab("Tabcorp Holdings Limited"), + taipei("Taipei City Government"), + talk("Amazon Registry Services, Inc."), + taobao("Alibaba Group Holding Limited"), + target("Target Domain Holdings, LLC"), + tatamotors("Tata Motors Ltd"), + tatar("Limited Liability Company \"Coordination Center of Regional Domain of Tatarstan Republic\""), + tattoo("Uniregistry, Corp."), + tax("Storm Orchard, LLC"), + taxi("Pine Falls, LLC"), + tci("Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti."), + tdk("TDK Corporation"), + team("Atomic Lake, LLC"), + tech("Dot Tech LLC"), + technology("Auburn Falls, LLC"), + telecity("TelecityGroup International Limited"), + telefonica("Telefónica S.A."), + temasek("Temasek Holdings (Private) Limited"), + tennis("Cotton Bloom, LLC"), + teva("Teva Pharmaceutical Industries Limited"), + thd("Home Depot Product Authority, LLC"), + theater("Blue Tigers, LLC"), + theatre("XYZ.COM LLC"), + tiaa("Teachers Insurance and Annuity Association of America"), + tickets("Accent Media Limited"), + tienda("Victor Manor, LLC"), + tiffany("Tiffany and Company"), + tips("Corn Willow, LLC"), + tires("Dog Edge, LLC"), + tirol("punkt Tirol GmbH"), + tjmaxx("The TJX Companies, Inc."), + tjx("The TJX Companies, Inc."), + tkmaxx("The TJX Companies, Inc."), + tmall("Alibaba Group Holding Limited"), + today("Pearl Woods, LLC"), + tokyo("GMO Registry, Inc."), + tools("Pioneer North, LLC"), + top("Jiangsu Bangning Science & Technology Co.,Ltd."), + toray("Toray Industries, Inc."), + toshiba("TOSHIBA Corporation"), + total("Total SA"), + tours("Sugar Station, LLC"), + town("Koko Moon, LLC"), + toyota("TOYOTA MOTOR CORPORATION"), + toys("Pioneer Orchard, LLC"), + trade("Elite Registry Limited"), + trading("DOTTRADING REGISTRY LTD"), + training("Wild Willow, LLC"), + travelchannel("Lifestyle Domain Holdings, Inc."), + travelers("Travelers TLD, LLC"), + travelersinsurance("Travelers TLD, LLC"), + trust("Artemis Internet Inc"), + trv("Travelers TLD, LLC"), + tube("Latin American Telecom LLC"), + tui("TUI AG"), + tunes("Amazon Registry Services, Inc."), + tushu("Amazon Registry Services, Inc."), + tvs("T V SUNDRAM IYENGAR & SONS PRIVATE LIMITED"), + ubank("National Australia Bank Limited"), + ubs("UBS AG"), + uconnect("FCA US LLC."), + unicom("China United Network Communications Corporation Limited"), + university("Little Station, LLC"), + uno("Dot Latin LLC"), + uol("UBN INTERNET LTDA."), + ups("UPS Market Driver, Inc."), + vacations("Atomic Tigers, LLC"), + vana("Lifestyle Domain Holdings, Inc."), + vanguard("The Vanguard Group, Inc."), + vegas("Dot Vegas, Inc."), + ventures("Binky Lake, LLC"), + verisign("VeriSign, Inc."), + versicherung("TLD-BOX Registrydienstleistungen GmbH"), + vet("United TLD Holdco, Ltd"), + viajes("Black Madison, LLC"), + video("United TLD Holdco, Ltd"), + vig("VIENNA INSURANCE GROUP AG Wiener Versicherung Gruppe"), + viking("Viking River Cruises (Bermuda) Ltd."), + villas("New Sky, LLC"), + vin("Holly Shadow, LLC"), + vip("Minds + Machines Group Limited"), + virgin("Virgin Enterprises Limited"), + visa("Visa Worldwide Pte. Limited"), + vision("Koko Station, LLC"), + vista("Vistaprint Limited"), + vistaprint("Vistaprint Limited"), + viva("Saudi Telecom Company"), + vivo("Telefonica Brasil S.A."), + vlaanderen("DNS.be vzw"), + vodka("Top Level Domain Holdings Limited"), + volkswagen("Volkswagen Group of America Inc."), + volvo("Volvo Holding Sverige Aktiebolag"), + vote("Monolith Registry LLC"), + voting("Valuetainment Corp."), + voto("Monolith Registry LLC"), + voyage("Ruby House, LLC"), + vuelos("Travel Reservations SRL"), + wales("Nominet UK"), + walmart("Wal-Mart Stores, Inc."), + walter("Sandvik AB"), + wang("Zodiac Wang Limited"), + wanggou("Amazon Registry Services, Inc."), + warman("Weir Group IP Limited"), + watch("Sand Shadow, LLC"), + watches("Richemont DNS Inc."), + weather("International Business Machines Corporation"), + weatherchannel("International Business Machines Corporation"), + webcam("dot Webcam Limited"), + weber("Saint-Gobain Weber SA"), + website("DotWebsite Inc."), + wed("Atgron, Inc."), + wedding("Top Level Domain Holdings Limited"), + weibo("Sina Corporation"), + weir("Weir Group IP Limited"), + whoswho("Who's Who Registry"), + wien("punkt.wien GmbH"), + wiki("Top Level Design, LLC"), + williamhill("William Hill Organization Limited"), + win("First Registry Limited"), + windows("Microsoft Corporation"), + wine("June Station, LLC"), + winners("The TJX Companies, Inc."), + wme("William Morris Endeavor Entertainment, LLC"), + wolterskluwer("Wolters Kluwer N.V."), + woodside("Woodside Petroleum Limited"), + work("Top Level Domain Holdings Limited"), + works("Little Dynamite, LLC"), + world("Bitter Fields, LLC"), + wow("Amazon Registry Services, Inc."), + wtc("World Trade Centers Association, Inc."), + wtf("Hidden Way, LLC"), + xbox("Microsoft Corporation"), + xerox("Xerox DNHC LLC"), + xfinity("Comcast IP Holdings I, LLC"), + xihuan("QIHOO 360 TECHNOLOGY CO. LTD."), + xin("Elegant Leader Limited"), + XN__11B4C3D("VeriSign Sarl"), + XN__1CK2E1B("Amazon Registry Services, Inc."), + XN__1QQW23A("Guangzhou YU Wei Information Technology Co., Ltd."), + XN__30RR7Y("Excellent First Limited"), + XN__3BST00M("Eagle Horizon Limited"), + XN__3DS443G("TLD REGISTRY LIMITED"), + XN__3OQ18VL8PN36A("Volkswagen (China) Investment Co., Ltd."), + XN__3PXU8K("VeriSign Sarl"), + XN__42C2D9A("VeriSign Sarl"), + XN__45Q11C("Zodiac Gemini Ltd"), + XN__4GBRIM("Suhub Electronic Establishment"), + XN__55QW42G("China Organizational Name Administration Center"), + XN__55QX5D("Computer Network Information Center of Chinese Academy of Sciences (China Internet Network Information Center)"), + XN__5SU34J936BGSG("Shangri‐La International Hotel Management Limited"), + XN__5TZM5G("Global Website TLD Asia Limited"), + XN__6FRZ82G("Afilias plc"), + XN__6QQ986B3XL("Tycoon Treasure Limited"), + XN__80ADXHKS("Foundation for Assistance for Internet Technologies and Infrastructure Development (FAITID)"), + XN__80AQECDR1A("Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)"), + XN__80ASEHDB("CORE Association"), + XN__80ASWG("CORE Association"), + XN__8Y0A063A("China United Network Communications Corporation Limited"), + XN__9DBQ2A("VeriSign Sarl"), + XN__9ET52U("RISE VICTORY LIMITED"), + XN__9KRT00A("Sina Corporation"), + XN__B4W605FERD("Temasek Holdings (Private) Limited"), + XN__BCK1B9A5DRE4C("Amazon Registry Services, Inc."), + XN__C1AVG("Public Interest Registry"), + XN__C2BR7G("VeriSign Sarl"), + XN__CCK2B3B("Amazon Registry Services, Inc."), + XN__CG4BKI("SAMSUNG SDS CO., LTD"), + XN__CZR694B("HU YI GLOBAL INFORMATION RESOURCES(HOLDING) COMPANY.HONGKONG LIMITED"), + XN__CZRS0T("Wild Island, LLC"), + XN__CZRU2D("Zodiac Aquarius Limited"), + XN__D1ACJ3B("The Foundation for Network Initiatives “The Smart Internet”"), + XN__ECKVDTC9D("Amazon Registry Services, Inc."), + XN__EFVY88H("Xinhua News Agency Guangdong Branch 新华通讯社广东分社"), + XN__ESTV75G("Industrial and Commercial Bank of China Limited"), + XN__FCT429K("Amazon Registry Services, Inc."), + XN__FHBEI("VeriSign Sarl"), + XN__FIQ228C5HS("TLD REGISTRY LIMITED"), + XN__FIQ64B("CITIC Group Corporation"), + XN__FJQ720A("Will Bloom, LLC"), + XN__FLW351E("Charleston Road Registry Inc."), + XN__FZYS8D69UVGM("PCCW Enterprises Limited"), + XN__G2XX48C("Minds + Machines Group Limited"), + XN__GCKR3F0F("Amazon Registry Services, Inc."), + XN__GK3AT1E("Amazon Registry Services, Inc."), + XN__HXT814E("Zodiac Taurus Ltd."), + XN__I1B6B1A6A2E("Public Interest Registry"), + XN__IMR513N("HU YI GLOBAL INFORMATION RESOURCES (HOLDING) COMPANY. HONGKONG LIMITED"), + XN__IO0A7I("Computer Network Information Center of Chinese Academy of Sciences (China Internet Network Information Center)"), + XN__J1AEF("VeriSign Sarl"), + XN__JLQ61U9W7B("Nokia Corporation"), + XN__JVR189M("Amazon Registry Services, Inc."), + XN__KCRX77D1X4A("Koninklijke Philips N.V."), + XN__KPU716F("Richemont DNS Inc."), + XN__KPUT3I("Beijing RITT-Net Technology Development Co., Ltd"), + XN__MGBA3A3EJT("Aramco Services Company"), + XN__MGBA7C0BBN0A("Crescent Holding GmbH"), + XN__MGBAAKC7DVF("Emirates Telecommunications Corporation (trading as Etisalat)"), + XN__MGBAB2BD("CORE Association"), + XN__MGBB9FBPOB("GreenTech Consultancy Company W.L.L."), + XN__MGBCA7DZDO("Abu Dhabi Systems and Information Centre"), + XN__MGBI4ECEXP("Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)"), + XN__MGBT3DHD("Asia Green IT System Bilgisayar San. ve Tic. Ltd. Sti."), + XN__MK1BU44C("VeriSign Sarl"), + XN__MXTQ1M("Net-Chinese Co., Ltd."), + XN__NGBC5AZD("International Domain Registry Pty. Ltd."), + XN__NGBE9E0A("Kuwait Finance House"), + XN__NGBRX("League of Arab States"), + XN__NQV7F("Public Interest Registry"), + XN__NQV7FS00EMA("Public Interest Registry"), + XN__NYQY26A("Stable Tone Limited"), + XN__P1ACF("Rusnames Limited"), + XN__PBT977C("Richemont DNS Inc."), + XN__PSSY2U("VeriSign Sarl"), + XN__Q9JYB4C("Charleston Road Registry Inc."), + XN__QCKA1PMC("Charleston Road Registry Inc."), + XN__RHQV96G("Stable Tone Limited"), + XN__ROVU88B("Amazon Registry Services, Inc."), + XN__SES554G("KNET Co., Ltd"), + XN__T60B56A("VeriSign Sarl"), + XN__TCKWE("VeriSign Sarl"), + XN__TIQ49XQYJ("Pontificium Consilium de Comunicationibus Socialibus (PCCS) (Pontifical Council for Social Communication)"), + XN__UNUP4Y("Spring Fields, LLC"), + XN__VERMGENSBERATER_CTB("Deutsche Vermögensberatung Aktiengesellschaft DVAG"), + XN__VERMGENSBERATUNG_PWB("Deutsche Vermögensberatung Aktiengesellschaft DVAG"), + XN__VHQUV("Dash McCook, LLC"), + XN__VUQ861B("Beijing Tele-info Network Technology Co., Ltd."), + XN__W4R85EL8FHU5DNRA("Kerry Trading Co. Limited"), + XN__W4RS40L("Kerry Trading Co. Limited"), + XN__XHQ521B("Guangzhou YU Wei Information Technology Co., Ltd."), + XN__ZFR164B ("China Organizational Name Administration Center"), + xperia("Sony Mobile Communications AB"), + xyz("XYZ.COM LLC"), + yachts("DERYachts, LLC"), + yahoo("Yahoo! Domain Services Inc."), + yamaxun("Amazon Registry Services, Inc."), + yandex("YANDEX, LLC"), + yodobashi("YODOBASHI CAMERA CO.,LTD."), + yoga("Top Level Domain Holdings Limited"), + yokohama("GMO Registry, Inc."), + you("Amazon Registry Services, Inc."), + youtube("Charleston Road Registry Inc."), + yun("QIHOO 360 TECHNOLOGY CO. LTD."), + zappos("Amazon Registry Services, Inc."), + zara("Industria de Diseño Textil, S.A. (INDITEX, S.A.)"), + zero("Amazon Registry Services, Inc."), + zip("Charleston Road Registry Inc."), + zippo("Zadco Company"), + zone("Outer Falls, LLC"), + zuerich("Kanton Zürich (Canton of Zurich)"); + + /** The TLD manager's name */ + private final String tldManager; + + private GenericTLD(final String tldManager) { + this.tldManager = tldManager; + } + + /** + * @return the name of the organization which manages this top-level domain name + */ + public String getTldManager() { + return this.tldManager; + } + + /** + * @return the lower cased, Punycode encoded domain name of this enumeration instance + */ + public String getDomainName() { + return this.toString().toLowerCase(Locale.ROOT).replace('_', '-'); + } + +} diff --git a/test/java/net/yacy/cora/document/id/DigestURLHashPerfTest.java b/test/java/net/yacy/cora/document/id/DigestURLHashPerfTest.java new file mode 100644 index 000000000..b2df145e4 --- /dev/null +++ b/test/java/net/yacy/cora/document/id/DigestURLHashPerfTest.java @@ -0,0 +1,114 @@ +// DigestURLHashPerfTest.java +// ----------------------- +// part of YaCy +// Copyright 2017 by luccioman; https://github.com/luccioman +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.cora.document.id; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.net.MalformedURLException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.io.output.NullOutputStream; + +import net.yacy.cora.document.encoding.ASCII; +import net.yacy.cora.protocol.Domains; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.kelondro.util.FileUtils; + +/** + * Testing DigestURL hash generation performances + */ +public class DigestURLHashPerfTest { + + /** + * Run and measure the {@link DigestURL#hash()} method on a list of urls + * provided in a given file (one URL per line). When an output file path is + * provided, generated hashes are written to it. + * + * @param args + * parameters + * @throws IOException + */ + public static void main(final String[] args) throws IOException { + if (args.length < 1) { + System.out.println("Usage : java DigestURLHashPerfTest [outputFilePath]"); + return; + } + + final File inFile = new File(args[0]); + final List urls = FileUtils.getListArray(inFile); + + System.out.println(urls.size() + " URLs loaded from " + inFile.getAbsolutePath()); + + try (OutputStream outStream = args.length >= 2 ? new FileOutputStream(args[1]) : new NullOutputStream(); + OutputStreamWriter writer = new OutputStreamWriter(outStream, StandardCharsets.UTF_8.name()); + BufferedWriter out = new BufferedWriter(writer);) { + + if (args.length >= 2) { + System.out.println("Writing URL hashes to " + args[1]); + } + byte[] hash; + DigestURL url; + long beginTime = System.nanoTime(), time, minTime = Long.MAX_VALUE, maxTime = 0, meanTime = 0, + totalTime = 0; + int step = 0; + for (final String urlStr : urls) { + try { + url = new DigestURL(urlStr); + beginTime = System.nanoTime(); + hash = url.hash(); + time = System.nanoTime() - beginTime; + minTime = Math.min(minTime, time); + maxTime = Math.max(maxTime, time); + totalTime += time; + out.write(ASCII.String(hash)); + out.newLine(); + step++; + } catch (final MalformedURLException e) { + e.printStackTrace(); + } + } + if (step > 0) { + meanTime = totalTime / step; + } else { + meanTime = totalTime; + } + + System.out.println("Hash generation total time (ms) : " + TimeUnit.NANOSECONDS.toMillis(totalTime) + " on " + + step + " urls."); + System.out.println("Render mean time (ms) : " + TimeUnit.NANOSECONDS.toMillis(meanTime)); + System.out.println("Render min time (ms) : " + TimeUnit.NANOSECONDS.toMillis(minTime)); + System.out.println("Render max time (ms) : " + TimeUnit.NANOSECONDS.toMillis(maxTime)); + } finally { + try { + Domains.close(); + } finally { + ConcurrentLog.shutdown(); + } + } + + } + +}