From 7f31762cb6261806542cc6d1188ca07db98a6950 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 23 May 2017 20:04:38 -0400 Subject: [PATCH 1/5] net: add an internal subnet for representing unresolved hostnames We currently do two resolves for dns seeds: one for the results, and one to serve in addrman as the source for those addresses. There's no requirement that the source hostname resolves to the stored identifier, only that the mapping is unique. So rather than incurring the second lookup, combine a private subnet with a hash of the hostname. The resulting v6 ip is guaranteed not to be publicy routable, and has only a negligible chance of colliding with a user's internal network (which would be of no consequence anyway). --- src/net.cpp | 2 +- src/netaddress.cpp | 48 ++++++++++++++++++++++++++++++++------ src/netaddress.h | 8 +++++++ src/rpc/net.cpp | 2 +- src/test/netbase_tests.cpp | 13 +++++++++++ 5 files changed, 64 insertions(+), 9 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index 75d1719e8..e10016d98 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -240,7 +240,7 @@ bool RemoveLocal(const CService& addr) /** Make a particular network entirely off-limits (no automatic connects to it) */ void SetLimited(enum Network net, bool fLimited) { - if (net == NET_UNROUTABLE) + if (net == NET_UNROUTABLE || net == NET_INTERNAL) return; LOCK(cs_mapLocalHost); vfLimited[net] = fLimited; diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 34a702986..89f257c64 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -15,6 +15,9 @@ static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; static const unsigned char pchOnionCat[] = {0xFD,0x87,0xD8,0x7E,0xEB,0x43}; +// 0xFD + sha256("bitcoin")[0:5] +static const unsigned char g_internal_prefix[] = { 0xFD, 0x6B, 0x88, 0xC0, 0x87, 0x24 }; + void CNetAddr::Init() { memset(ip, 0, sizeof(ip)); @@ -42,6 +45,18 @@ void CNetAddr::SetRaw(Network network, const uint8_t *ip_in) } } +bool CNetAddr::SetInternal(const std::string &name) +{ + if (name.empty()) { + return false; + } + unsigned char hash[32] = {}; + CSHA256().Write((const unsigned char*)name.data(), name.size()).Finalize(hash); + memcpy(ip, g_internal_prefix, sizeof(g_internal_prefix)); + memcpy(ip + sizeof(g_internal_prefix), hash, sizeof(ip) - sizeof(g_internal_prefix)); + return true; +} + bool CNetAddr::SetSpecial(const std::string &strName) { if (strName.size()>6 && strName.substr(strName.size() - 6, 6) == ".onion") { @@ -84,7 +99,7 @@ bool CNetAddr::IsIPv4() const bool CNetAddr::IsIPv6() const { - return (!IsIPv4() && !IsTor()); + return (!IsIPv4() && !IsTor() && !IsInternal()); } bool CNetAddr::IsRFC1918() const @@ -199,6 +214,9 @@ bool CNetAddr::IsValid() const if (IsRFC3849()) return false; + if (IsInternal()) + return false; + if (IsIPv4()) { // INADDR_NONE @@ -217,11 +235,19 @@ bool CNetAddr::IsValid() const bool CNetAddr::IsRoutable() const { - return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || (IsRFC4193() && !IsTor()) || IsRFC4843() || IsLocal()); + return IsValid() && !(IsRFC1918() || IsRFC2544() || IsRFC3927() || IsRFC4862() || IsRFC6598() || IsRFC5737() || (IsRFC4193() && !IsTor()) || IsRFC4843() || IsLocal() || IsInternal()); +} + +bool CNetAddr::IsInternal() const +{ + return memcmp(ip, g_internal_prefix, sizeof(g_internal_prefix)) == 0; } enum Network CNetAddr::GetNetwork() const { + if (IsInternal()) + return NET_INTERNAL; + if (!IsRoutable()) return NET_UNROUTABLE; @@ -238,6 +264,8 @@ std::string CNetAddr::ToStringIP() const { if (IsTor()) return EncodeBase32(&ip[6], 10) + ".onion"; + if (IsInternal()) + return EncodeBase32(ip + sizeof(g_internal_prefix), sizeof(ip) - sizeof(g_internal_prefix)) + ".internal"; CService serv(*this, 0); struct sockaddr_storage sockaddr; socklen_t socklen = sizeof(sockaddr); @@ -305,9 +333,15 @@ std::vector CNetAddr::GetGroup() const nClass = 255; nBits = 0; } - - // all unroutable addresses belong to the same group - if (!IsRoutable()) + // all internal-usage addresses get their own group + if (IsInternal()) + { + nClass = NET_INTERNAL; + nStartByte = sizeof(g_internal_prefix); + nBits = (sizeof(ip) - sizeof(g_internal_prefix)) * 8; + } + // all other unroutable addresses belong to the same group + else if (!IsRoutable()) { nClass = NET_UNROUTABLE; nBits = 0; @@ -393,7 +427,7 @@ int CNetAddr::GetReachabilityFrom(const CNetAddr *paddrPartner) const REACH_PRIVATE }; - if (!IsRoutable()) + if (!IsRoutable() || IsInternal()) return REACH_UNREACHABLE; int ourNet = GetExtNetwork(this); @@ -552,7 +586,7 @@ std::string CService::ToStringPort() const std::string CService::ToStringIPPort() const { - if (IsIPv4() || IsTor()) { + if (IsIPv4() || IsTor() || IsInternal()) { return ToStringIP() + ":" + ToStringPort(); } else { return "[" + ToStringIP() + "]:" + ToStringPort(); diff --git a/src/netaddress.h b/src/netaddress.h index fbc4d1a65..80716600d 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -22,6 +22,7 @@ enum Network NET_IPV4, NET_IPV6, NET_TOR, + NET_INTERNAL, NET_MAX, }; @@ -45,6 +46,12 @@ class CNetAddr */ void SetRaw(Network network, const uint8_t *data); + /** + * Transform an arbitrary string into a non-routable ipv6 address. + * Useful for mapping resolved addresses back to their source. + */ + bool SetInternal(const std::string& name); + bool SetSpecial(const std::string &strName); // for Tor addresses bool IsIPv4() const; // IPv4 mapped address (::FFFF:0:0/96, 0.0.0.0/0) bool IsIPv6() const; // IPv6 address (not mapped IPv4, not Tor) @@ -64,6 +71,7 @@ class CNetAddr bool IsTor() const; bool IsLocal() const; bool IsRoutable() const; + bool IsInternal() const; bool IsValid() const; enum Network GetNetwork() const; std::string ToString() const; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 10bf99eb3..ab4ecbcf1 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -397,7 +397,7 @@ static UniValue GetNetworksInfo() for(int n=0; n(n); - if(network == NET_UNROUTABLE) + if(network == NET_UNROUTABLE || network == NET_INTERNAL) continue; proxyType proxy; UniValue obj(UniValue::VOBJ); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index e4b4b8572..94a3dac4b 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -25,6 +25,13 @@ static CSubNet ResolveSubNet(const char* subnet) return ret; } +static CNetAddr CreateInternal(const char* host) +{ + CNetAddr addr; + addr.SetInternal(host); + return addr; +} + BOOST_AUTO_TEST_CASE(netbase_networks) { BOOST_CHECK(ResolveIP("127.0.0.1").GetNetwork() == NET_UNROUTABLE); @@ -32,6 +39,7 @@ BOOST_AUTO_TEST_CASE(netbase_networks) BOOST_CHECK(ResolveIP("8.8.8.8").GetNetwork() == NET_IPV4); BOOST_CHECK(ResolveIP("2001::8888").GetNetwork() == NET_IPV6); BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetNetwork() == NET_TOR); + BOOST_CHECK(CreateInternal("foo.com").GetNetwork() == NET_INTERNAL); } @@ -58,6 +66,8 @@ BOOST_AUTO_TEST_CASE(netbase_properties) BOOST_CHECK(ResolveIP("8.8.8.8").IsRoutable()); BOOST_CHECK(ResolveIP("2001::1").IsRoutable()); BOOST_CHECK(ResolveIP("127.0.0.1").IsValid()); + BOOST_CHECK(CreateInternal("FD6B:88C0:8724:edb1:8e4:3588:e546:35ca").IsInternal()); + BOOST_CHECK(CreateInternal("bar.com").IsInternal()); } @@ -281,6 +291,9 @@ BOOST_AUTO_TEST_CASE(netbase_getgroup) BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup() == std::vector({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup() == std::vector({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 + // baz.net sha256 hash: 12929400eb4607c4ac075f087167e75286b179c693eb059a01774b864e8fe505 + std::vector internal_group = {NET_INTERNAL, 0x12, 0x92, 0x94, 0x00, 0xeb, 0x46, 0x07, 0xc4, 0xac, 0x07}; + BOOST_CHECK(CreateInternal("baz.net").GetGroup() == internal_group); } BOOST_AUTO_TEST_SUITE_END() From 6d0bd5b73d14517b349bb07656a18b2acb0d5c45 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 13 Jun 2017 17:26:50 -0400 Subject: [PATCH 2/5] net: do not allow resolving to an internal address In order to prevent mixups, our internal range is never allowed as a resolve result. This means that no user-provided string will ever be confused with an internal address. --- src/netbase.cpp | 9 +++++++-- src/test/netbase_tests.cpp | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/netbase.cpp b/src/netbase.cpp index 32557dd17..a23f92e1e 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -108,17 +108,22 @@ bool static LookupIntern(const char *pszName, std::vector& vIP, unsign struct addrinfo *aiTrav = aiRes; while (aiTrav != NULL && (nMaxSolutions == 0 || vIP.size() < nMaxSolutions)) { + CNetAddr resolved; if (aiTrav->ai_family == AF_INET) { assert(aiTrav->ai_addrlen >= sizeof(sockaddr_in)); - vIP.push_back(CNetAddr(((struct sockaddr_in*)(aiTrav->ai_addr))->sin_addr)); + resolved = CNetAddr(((struct sockaddr_in*)(aiTrav->ai_addr))->sin_addr); } if (aiTrav->ai_family == AF_INET6) { assert(aiTrav->ai_addrlen >= sizeof(sockaddr_in6)); struct sockaddr_in6* s6 = (struct sockaddr_in6*) aiTrav->ai_addr; - vIP.push_back(CNetAddr(s6->sin6_addr, s6->sin6_scope_id)); + resolved = CNetAddr(s6->sin6_addr, s6->sin6_scope_id); + } + /* Never allow resolving to an internal address. Consider any such result invalid */ + if (!resolved.IsInternal()) { + vIP.push_back(resolved); } aiTrav = aiTrav->ai_next; diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 94a3dac4b..b45a7fcc5 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -113,6 +113,11 @@ BOOST_AUTO_TEST_CASE(netbase_lookupnumeric) BOOST_CHECK(TestParse("[::]:8333", "[::]:8333")); BOOST_CHECK(TestParse("[127.0.0.1]", "127.0.0.1:65535")); BOOST_CHECK(TestParse(":::", "[::]:0")); + + // verify that an internal address fails to resolve + BOOST_CHECK(TestParse("[fd6b:88c0:8724:1:2:3:4:5]", "[::]:0")); + // and that a one-off resolves correctly + BOOST_CHECK(TestParse("[fd6c:88c0:8724:1:2:3:4:5]", "[fd6c:88c0:8724:1:2:3:4:5]:65535")); } BOOST_AUTO_TEST_CASE(onioncat_test) From 6cdc488e3604267b1e115da5c50ffb970cd30084 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Tue, 23 May 2017 20:48:08 -0400 Subject: [PATCH 3/5] net: switch to dummy internal ip for dns seed source This addresss the TODO to avoid resolving twice. --- src/chainparams.cpp | 20 ++++++++++---------- src/chainparams.h | 4 ++-- src/net.cpp | 20 +++++++------------- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 3b42c5fb2..0eeaee70e 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -124,12 +124,12 @@ public: assert(genesis.hashMerkleRoot == uint256S("0x4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")); // Note that of those with the service bits flag, most only support a subset of possible options - vSeeds.push_back(CDNSSeedData("bitcoin.sipa.be", "seed.bitcoin.sipa.be", true)); // Pieter Wuille, only supports x1, x5, x9, and xd - vSeeds.push_back(CDNSSeedData("bluematt.me", "dnsseed.bluematt.me", true)); // Matt Corallo, only supports x9 - vSeeds.push_back(CDNSSeedData("dashjr.org", "dnsseed.bitcoin.dashjr.org")); // Luke Dashjr - vSeeds.push_back(CDNSSeedData("bitcoinstats.com", "seed.bitcoinstats.com", true)); // Christian Decker, supports x1 - xf - vSeeds.push_back(CDNSSeedData("bitcoin.jonasschnelli.ch", "seed.bitcoin.jonasschnelli.ch", true)); // Jonas Schnelli, only supports x1, x5, x9, and xd - vSeeds.push_back(CDNSSeedData("petertodd.org", "seed.btc.petertodd.org", true)); // Peter Todd, only supports x1, x5, x9, and xd + vSeeds.emplace_back("seed.bitcoin.sipa.be", true); // Pieter Wuille, only supports x1, x5, x9, and xd + vSeeds.emplace_back("dnsseed.bluematt.me", true); // Matt Corallo, only supports x9 + vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org"); // Luke Dashjr + vSeeds.emplace_back("seed.bitcoinstats.com", true); // Christian Decker, supports x1 - xf + vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch", true); // Jonas Schnelli, only supports x1, x5, x9, and xd + vSeeds.emplace_back("seed.btc.petertodd.org", true); // Peter Todd, only supports x1, x5, x9, and xd base58Prefixes[PUBKEY_ADDRESS] = std::vector(1,0); base58Prefixes[SCRIPT_ADDRESS] = std::vector(1,5); @@ -225,10 +225,10 @@ public: vFixedSeeds.clear(); vSeeds.clear(); // nodes with support for servicebits filtering should be at the top - vSeeds.push_back(CDNSSeedData("testnetbitcoin.jonasschnelli.ch", "testnet-seed.bitcoin.jonasschnelli.ch", true)); - vSeeds.push_back(CDNSSeedData("petertodd.org", "seed.tbtc.petertodd.org", true)); - vSeeds.push_back(CDNSSeedData("bluematt.me", "testnet-seed.bluematt.me")); - vSeeds.push_back(CDNSSeedData("bitcoin.schildbach.de", "testnet-seed.bitcoin.schildbach.de")); + vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch", true); + vSeeds.emplace_back("seed.tbtc.petertodd.org", true); + vSeeds.emplace_back("testnet-seed.bluematt.me"); + vSeeds.emplace_back("testnet-seed.bitcoin.schildbach.de"); base58Prefixes[PUBKEY_ADDRESS] = std::vector(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector(1,196); diff --git a/src/chainparams.h b/src/chainparams.h index a2f136171..16175171a 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -15,9 +15,9 @@ #include struct CDNSSeedData { - std::string name, host; + std::string host; bool supportsServiceBitsFiltering; - CDNSSeedData(const std::string &strName, const std::string &strHost, bool supportsServiceBitsFilteringIn = false) : name(strName), host(strHost), supportsServiceBitsFiltering(supportsServiceBitsFilteringIn) {} + CDNSSeedData(const std::string &strHost, bool supportsServiceBitsFilteringIn = false) : host(strHost), supportsServiceBitsFiltering(supportsServiceBitsFilteringIn) {} }; struct SeedSpec6 { diff --git a/src/net.cpp b/src/net.cpp index e10016d98..0fcf9e0ab 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1604,7 +1604,12 @@ void CConnman::ThreadDNSAddressSeed() std::vector vIPs; std::vector vAdd; ServiceFlags requiredServiceBits = nRelevantServices; - if (LookupHost(GetDNSHost(seed, &requiredServiceBits).c_str(), vIPs, 0, true)) + std::string host = GetDNSHost(seed, &requiredServiceBits); + CNetAddr resolveSource; + if (!resolveSource.SetInternal(host)) { + continue; + } + if (LookupHost(host.c_str(), vIPs, 0, true)) { BOOST_FOREACH(const CNetAddr& ip, vIPs) { @@ -1614,18 +1619,7 @@ void CConnman::ThreadDNSAddressSeed() vAdd.push_back(addr); found++; } - } - if (interruptNet) { - return; - } - // TODO: The seed name resolve may fail, yielding an IP of [::], which results in - // addrman assigning the same source to results from different seeds. - // This should switch to a hard-coded stable dummy IP for each seed name, so that the - // resolve is not required at all. - if (!vIPs.empty()) { - CService seedSource; - Lookup(seed.name.c_str(), seedSource, 0, true); - addrman.Add(vAdd, seedSource); + addrman.Add(vAdd, resolveSource); } } } From d5c7c1cfe3090775bf30dee531d511801926c18b Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Wed, 21 Jun 2017 15:45:20 -0400 Subject: [PATCH 4/5] net: use an internal address for fixed seeds --- src/net.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net.cpp b/src/net.cpp index 0fcf9e0ab..cd9e0d05d 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -1718,7 +1718,7 @@ void CConnman::ThreadOpenConnections() if (!done) { LogPrintf("Adding fixed seed nodes as DNS doesn't seem to be available.\n"); CNetAddr local; - LookupHost("127.0.0.1", local, false); + local.SetInternal("fixedseeds"); addrman.Add(convertSeed6(Params().FixedSeeds()), local); done = true; } From c1be28536467e90ce75eaa7d8c338f6485c4bee5 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Thu, 22 Jun 2017 15:13:58 -0400 Subject: [PATCH 5/5] chainparams: make supported service bits option explicit --- src/chainparams.cpp | 6 +++--- src/chainparams.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 0eeaee70e..dc4d2621e 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -126,7 +126,7 @@ public: // Note that of those with the service bits flag, most only support a subset of possible options vSeeds.emplace_back("seed.bitcoin.sipa.be", true); // Pieter Wuille, only supports x1, x5, x9, and xd vSeeds.emplace_back("dnsseed.bluematt.me", true); // Matt Corallo, only supports x9 - vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org"); // Luke Dashjr + vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org", false); // Luke Dashjr vSeeds.emplace_back("seed.bitcoinstats.com", true); // Christian Decker, supports x1 - xf vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch", true); // Jonas Schnelli, only supports x1, x5, x9, and xd vSeeds.emplace_back("seed.btc.petertodd.org", true); // Peter Todd, only supports x1, x5, x9, and xd @@ -227,8 +227,8 @@ public: // nodes with support for servicebits filtering should be at the top vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch", true); vSeeds.emplace_back("seed.tbtc.petertodd.org", true); - vSeeds.emplace_back("testnet-seed.bluematt.me"); - vSeeds.emplace_back("testnet-seed.bitcoin.schildbach.de"); + vSeeds.emplace_back("testnet-seed.bluematt.me", false); + vSeeds.emplace_back("testnet-seed.bitcoin.schildbach.de", false); base58Prefixes[PUBKEY_ADDRESS] = std::vector(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector(1,196); diff --git a/src/chainparams.h b/src/chainparams.h index 16175171a..f55ae4cf7 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -17,7 +17,7 @@ struct CDNSSeedData { std::string host; bool supportsServiceBitsFiltering; - CDNSSeedData(const std::string &strHost, bool supportsServiceBitsFilteringIn = false) : host(strHost), supportsServiceBitsFiltering(supportsServiceBitsFilteringIn) {} + CDNSSeedData(const std::string &strHost, bool supportsServiceBitsFilteringIn) : host(strHost), supportsServiceBitsFiltering(supportsServiceBitsFilteringIn) {} }; struct SeedSpec6 {