Merge pull request #1198 from jgarzik/addrman

Replace BDB-managed addr.dat with bitcoin-managed peers.dat

src/db.cpp

@@ -164,8 +164,6 @@ void CDB::Close()
     unsigned int nMinutes = 0;
     if (fReadOnly)
         nMinutes = 1;
-    if (strFile == "addr.dat")
-        nMinutes = 2;
     if (strFile == "blkindex.dat")
         nMinutes = 2;
     if (strFile == "blkindex.dat" && IsInitialBlockDownload())
@@ -310,7 +308,7 @@ void DBFlush(bool fShutdown)
                 CloseDb(strFile);
                 printf("%s checkpoint\n", strFile.c_str());
                 dbenv.txn_checkpoint(0, 0, 0);
-                if ((strFile != "blkindex.dat" && strFile != "addr.dat") || fDetachDB) {
+                if (strFile != "blkindex.dat" || fDetachDB) {
                     printf("%s detach\n", strFile.c_str());
                     dbenv.lsn_reset(strFile.c_str(), 0);
                 }
@@ -737,65 +735,96 @@ bool CTxDB::LoadBlockIndex()
 // CAddrDB
 //
 
-bool CAddrDB::WriteAddrman(const CAddrMan& addrman)
+
+CAddrDB::CAddrDB()
 {
-    return Write(string("addrman"), addrman);
+    pathAddr = GetDataDir() / "peers.dat";
 }
 
-bool CAddrDB::LoadAddresses()
+bool CAddrDB::Write(const CAddrMan& addr)
 {
-    if (Read(string("addrman"), addrman))
-    {
-        printf("Loaded %i addresses\n", addrman.size());
-        return true;
+    // Generate random temporary filename
+    unsigned short randv = 0;
+    RAND_bytes((unsigned char *)&randv, sizeof(randv));
+    std::string tmpfn = strprintf("peers.dat.%04x", randv);
+
+    // serialize addresses, checksum data up to that point, then append csum
+    CDataStream ssPeers(SER_DISK, CLIENT_VERSION);
+    ssPeers << FLATDATA(pchMessageStart);
+    ssPeers << addr;
+    uint256 hash = Hash(ssPeers.begin(), ssPeers.end());
+    ssPeers << hash;
+
+    // open temp output file, and associate with CAutoFile
+    boost::filesystem::path pathTmp = GetDataDir() / tmpfn;
+    FILE *file = fopen(pathTmp.string().c_str(), "wb");
+    CAutoFile fileout = CAutoFile(file, SER_DISK, CLIENT_VERSION);
+    if (!fileout)
+        return error("CAddrman::Write() : open failed");
+
+    // Write and commit header, data
+    try {
+        fileout << ssPeers;
     }
-    
-    // Read pre-0.6 addr records
+    catch (std::exception &e) {
+        return error("CAddrman::Write() : I/O error");
+    }
+    FileCommit(fileout);
+    fileout.fclose();
 
-    vector<CAddress> vAddr;
-    vector<vector<unsigned char> > vDelete;
+    // replace existing peers.dat, if any, with new peers.dat.XXXX
+    if (!RenameOver(pathTmp, pathAddr))
+        return error("CAddrman::Write() : Rename-into-place failed");
 
-    // Get cursor
-    Dbc* pcursor = GetCursor();
-    if (!pcursor)
-        return false;
-
-    loop
-    {
-        // Read next record
-        CDataStream ssKey(SER_DISK, CLIENT_VERSION);
-        CDataStream ssValue(SER_DISK, CLIENT_VERSION);
-        int ret = ReadAtCursor(pcursor, ssKey, ssValue);
-        if (ret == DB_NOTFOUND)
-            break;
-        else if (ret != 0)
-            return false;
+    return true;
+}
 
-        // Unserialize
-        string strType;
-        ssKey >> strType;
-        if (strType == "addr")
-        {
-            CAddress addr;
-            ssValue >> addr;
-            vAddr.push_back(addr);
-        }
+bool CAddrDB::Read(CAddrMan& addr)
+{
+    // open input file, and associate with CAutoFile
+    FILE *file = fopen(pathAddr.string().c_str(), "rb");
+    CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION);
+    if (!filein)
+        return error("CAddrman::Read() : open failed");
+
+    // use file size to size memory buffer
+    int fileSize = GetFilesize(filein);
+    int dataSize = fileSize - sizeof(uint256);
+    vector<unsigned char> vchData;
+    vchData.resize(dataSize);
+    uint256 hashIn;
+
+    // read data and checksum from file
+    try {
+        filein.read((char *)&vchData[0], dataSize);
+        filein >> hashIn;
     }
-    pcursor->close();
+    catch (std::exception &e) {
+        return error("CAddrman::Read() 2 : I/O error or stream data corrupted");
+    }
+    filein.fclose();
 
-    addrman.Add(vAddr, CNetAddr("0.0.0.0"));
-    printf("Loaded %i addresses\n", addrman.size());
+    CDataStream ssPeers(vchData, SER_DISK, CLIENT_VERSION);
 
-    // Note: old records left; we ran into hangs-on-startup
-    // bugs for some users who (we think) were running after
-    // an unclean shutdown.
+    // verify stored checksum matches input data
+    uint256 hashTmp = Hash(ssPeers.begin(), ssPeers.end());
+    if (hashIn != hashTmp)
+        return error("CAddrman::Read() : checksum mismatch; data corrupted");
 
-    return true;
-}
+    // de-serialize address data
+    unsigned char pchMsgTmp[4];
+    try {
+        ssPeers >> FLATDATA(pchMsgTmp);
+        ssPeers >> addr;
+    }
+    catch (std::exception &e) {
+        return error("CAddrman::Read() : I/O error or stream data corrupted");
+    }
 
-bool LoadAddresses()
-{
-    return CAddrDB("cr+").LoadAddresses();
-}
+    // finally, verify the network matches ours
+    if (memcmp(pchMsgTmp, pchMessageStart, sizeof(pchMsgTmp)))
+        return error("CAddrman::Read() : invalid network magic number");
 
+    return true;
+}
 

src/db.h

@@ -296,20 +296,15 @@ public:
 
 
 
-/** Access to the (IP) address database (addr.dat) */
-class CAddrDB : public CDB
+/** Access to the (IP) address database (peers.dat) */
+class CAddrDB
 {
-public:
-    CAddrDB(const char* pszMode="r+") : CDB("addr.dat", pszMode) { }
 private:
-    CAddrDB(const CAddrDB&);
-    void operator=(const CAddrDB&);
+    boost::filesystem::path pathAddr;
 public:
-    bool WriteAddrman(const CAddrMan& addr);
-    bool LoadAddresses();
+    CAddrDB();
+    bool Write(const CAddrMan& addr);
+    bool Read(CAddrMan& addr);
 };
 
-bool LoadAddresses();
-
-
 #endif // BITCOIN_DB_H

src/init.cpp

@@ -371,9 +371,15 @@ bool AppInit2()
     InitMessage(_("Loading addresses..."));
     printf("Loading addresses...\n");
     nStart = GetTimeMillis();
-    if (!LoadAddresses())
-        strErrors << _("Error loading addr.dat") << "\n";
-    printf(" addresses   %15"PRI64d"ms\n", GetTimeMillis() - nStart);
+
+    {
+        CAddrDB adb;
+        if (!adb.Read(addrman))
+            printf("Invalid or missing peers.dat; recreating\n");
+    }
+
+    printf("Loaded %i addresses from peers.dat  %"PRI64d"ms\n",
+           addrman.size(), GetTimeMillis() - nStart);
 
     InitMessage(_("Loading block index..."));
     printf("Loading block index...\n");

src/main.h

@@ -956,13 +956,7 @@ public:
         // Flush stdio buffers and commit to disk before returning
         fflush(fileout);
         if (!IsInitialBlockDownload() || (nBestHeight+1) % 500 == 0)
-        {
-#ifdef WIN32
-            _commit(_fileno(fileout));
-#else
-            fsync(fileno(fileout));
-#endif
-        }
+            FileCommit(fileout);
 
         return true;
     }

src/net.cpp

@@ -1278,8 +1278,13 @@ unsigned int pnSeed[] =
 
 void DumpAddresses()
 {
+    int64 nStart = GetTimeMillis();
+
     CAddrDB adb;
-    adb.WriteAddrman(addrman);
+    adb.Write(addrman);
+
+    printf("Flushed %d addresses to peers.dat  %"PRI64d"ms\n",
+           addrman.size(), GetTimeMillis() - nStart);
 }
 
 void ThreadDumpAddress2(void* parg)

src/net.h

@@ -19,7 +19,6 @@
 #include "protocol.h"
 #include "addrman.h"
 
-class CAddrDB;
 class CRequestTracker;
 class CNode;
 class CBlockIndex;

src/util.cpp

@@ -910,6 +910,27 @@ void CreatePidFile(const boost::filesystem::path &path, pid_t pid)
     }
 }
 
+bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest)
+{
+#ifdef WIN32
+    return MoveFileEx(src.string().c_str(), dest.string().c_str(),
+                      MOVEFILE_REPLACE_EXISTING);
+#else
+    int rc = std::rename(src.string().c_str(), dest.string().c_str());
+    return (rc == 0);
+#endif /* WIN32 */
+}
+
+void FileCommit(FILE *fileout)
+{
+    fflush(fileout);                // harmless if redundantly called
+#ifdef WIN32
+    _commit(_fileno(fileout));
+#else
+    fsync(fileno(fileout));
+#endif
+}
+
 int GetFilesize(FILE* file)
 {
     int nSavePos = ftell(file);

src/util.h

@@ -152,7 +152,9 @@ std::string EncodeBase64(const std::string& str);
 void ParseParameters(int argc, const char*const argv[]);
 bool WildcardMatch(const char* psz, const char* mask);
 bool WildcardMatch(const std::string& str, const std::string& mask);
+void FileCommit(FILE *fileout);
 int GetFilesize(FILE* file);
+bool RenameOver(boost::filesystem::path src, boost::filesystem::path dest);
 boost::filesystem::path GetDefaultDataDir();
 const boost::filesystem::path &GetDataDir(bool fNetSpecific = true);
 boost::filesystem::path GetConfigFile();