twisterarmy
/
twister-core
mirror of https://github.com/twisterarmy/twister-core.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge branch 'dbenv' into tmp 

Conflicts:
	src/db.cpp
miguelfreitas
Jeff Garzik 13 years ago committed by Jeff Garzik
parent
5d0f7c4f47 ffe8b77a61
commit
0134a1c08c
5 changed files with 166 additions and 139 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 182
      src/db.cpp
  2. 84
      src/db.h
  3. 6
      src/init.cpp
  4. 1
      src/main.cpp
  5. 32
      src/walletdb.cpp

182
src/db.cpp
Unescape View File

@ -26,14 +26,9 @@ unsigned int nWalletDBUpdated;
// CDB // CDB
// //
CCriticalSection cs_db; CDBEnv bitdb;
static bool fDbEnvInit = false;
bool fDetachDB = false; void CDBEnv::EnvShutdown()
DbEnv dbenv(0);
map<string, int> mapFileUseCount;
static map<string, Db*> mapDb;
static void EnvShutdown()
{ {
if (!fDbEnvInit) if (!fDbEnvInit)
return; return;
@ -50,21 +45,76 @@ static void EnvShutdown()
DbEnv(0).remove(GetDataDir().string().c_str(), 0); DbEnv(0).remove(GetDataDir().string().c_str(), 0);
} }
class CDBInit CDBEnv::CDBEnv() : dbenv(0)
{ {
public:
CDBInit()
{
}
~CDBInit()
{
EnvShutdown();
}
} }
instance_of_cdbinit;
CDBEnv::~CDBEnv()
{
EnvShutdown();
}
void CDBEnv::Close()
{
EnvShutdown();
}
bool CDBEnv::Open(boost::filesystem::path pathEnv_)
{
if (fDbEnvInit)
return true;
if (fShutdown)
return false;
pathEnv = pathEnv_;
filesystem::path pathDataDir = pathEnv;
filesystem::path pathLogDir = pathDataDir / "database";
filesystem::create_directory(pathLogDir);
filesystem::path pathErrorFile = pathDataDir / "db.log";
printf("dbenv.open LogDir=%s ErrorFile=%s\n", pathLogDir.string().c_str(), pathErrorFile.string().c_str());
unsigned int nEnvFlags = 0;
if (GetBoolArg("-privdb", true))
nEnvFlags |= DB_PRIVATE;
int nDbCache = GetArg("-dbcache", 25);
dbenv.set_lg_dir(pathLogDir.string().c_str());
dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1);
dbenv.set_lg_bsize(1048576);
dbenv.set_lg_max(10485760);
dbenv.set_lk_max_locks(10000);
dbenv.set_lk_max_objects(10000);
dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug
dbenv.set_flags(DB_AUTO_COMMIT, 1);
dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1);
dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1);
int ret = dbenv.open(pathDataDir.string().c_str(),
DB_CREATE |
DB_INIT_LOCK |
DB_INIT_LOG |
DB_INIT_MPOOL |
DB_INIT_TXN |
DB_THREAD |
DB_RECOVER |
nEnvFlags,
S_IRUSR | S_IWUSR);
if (ret > 0)
return error("CDB() : error %d opening database environment", ret);
fDbEnvInit = true;
return true;
}
void CDBEnv::CheckpointLSN(std::string strFile)
{
dbenv.txn_checkpoint(0, 0, 0);
dbenv.lsn_reset(strFile.c_str(), 0);
}
CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL) CDB::CDB(const char *pszFile, const char* pszMode) :
pdb(NULL), activeTxn(NULL)
{ {
int ret; int ret;
if (pszFile == NULL) if (pszFile == NULL)
@ -76,54 +126,17 @@ CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL)
if (fCreate) if (fCreate)
nFlags |= DB_CREATE; nFlags |= DB_CREATE;
unsigned int nEnvFlags = 0;
if (GetBoolArg("-privdb", true))
nEnvFlags |= DB_PRIVATE;
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
if (!fDbEnvInit) if (!bitdb.Open(GetDataDir()))
{ throw runtime_error("env open failed");
if (fShutdown)
return;
filesystem::path pathDataDir = GetDataDir();
filesystem::path pathLogDir = pathDataDir / "database";
filesystem::create_directory(pathLogDir);
filesystem::path pathErrorFile = pathDataDir / "db.log";
printf("dbenv.open LogDir=%s ErrorFile=%s\n", pathLogDir.string().c_str(), pathErrorFile.string().c_str());
int nDbCache = GetArg("-dbcache", 25);
dbenv.set_lg_dir(pathLogDir.string().c_str());
dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1);
dbenv.set_lg_bsize(1048576);
dbenv.set_lg_max(10485760);
dbenv.set_lk_max_locks(10000);
dbenv.set_lk_max_objects(10000);
dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug
dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1);
dbenv.set_flags(DB_AUTO_COMMIT, 1);
dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1);
ret = dbenv.open(pathDataDir.string().c_str(),
DB_CREATE |
DB_INIT_LOCK |
DB_INIT_LOG |
DB_INIT_MPOOL |
DB_INIT_TXN |
DB_THREAD |
DB_RECOVER |
nEnvFlags,
S_IRUSR | S_IWUSR);
if (ret > 0)
throw runtime_error(strprintf("CDB() : error %d opening database environment", ret));
fDbEnvInit = true;
}
strFile = pszFile; strFile = pszFile;
++mapFileUseCount[strFile]; ++bitdb.mapFileUseCount[strFile];
pdb = mapDb[strFile]; pdb = bitdb.mapDb[strFile];
if (pdb == NULL) if (pdb == NULL)
{ {
pdb = new Db(&dbenv, 0); pdb = new Db(&bitdb.dbenv, 0);
ret = pdb->open(NULL, // Txn pointer ret = pdb->open(NULL, // Txn pointer
pszFile, // Filename pszFile, // Filename
@ -137,8 +150,8 @@ CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL)
delete pdb; delete pdb;
pdb = NULL; pdb = NULL;
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
--mapFileUseCount[strFile]; --bitdb.mapFileUseCount[strFile];
} }
strFile = ""; strFile = "";
throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", pszFile, ret)); throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", pszFile, ret));
@ -152,7 +165,7 @@ CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL)
fReadOnly = fTmp; fReadOnly = fTmp;
} }
mapDb[strFile] = pdb; bitdb.mapDb[strFile] = pdb;
} }
} }
} }
@ -161,9 +174,9 @@ void CDB::Close()
{ {
if (!pdb) if (!pdb)
return; return;
if (!vTxn.empty()) if (activeTxn)
vTxn.front()->abort(); activeTxn->abort();
vTxn.clear(); activeTxn = NULL;
pdb = NULL; pdb = NULL;
// Flush database activity from memory pool to disk log // Flush database activity from memory pool to disk log
@ -175,15 +188,15 @@ void CDB::Close()
if (strFile == "blkindex.dat" && IsInitialBlockDownload()) if (strFile == "blkindex.dat" && IsInitialBlockDownload())
nMinutes = 5; nMinutes = 5;
dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100)*1024 : 0, nMinutes, 0); bitdb.dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100)*1024 : 0, nMinutes, 0);
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
--mapFileUseCount[strFile]; --bitdb.mapFileUseCount[strFile];
} }
} }
void CloseDb(const string& strFile) void CDBEnv::CloseDb(const string& strFile)
{ {
{ {
LOCK(cs_db); LOCK(cs_db);
@ -203,21 +216,20 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip)
while (!fShutdown) while (!fShutdown)
{ {
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
if (!mapFileUseCount.count(strFile) || mapFileUseCount[strFile] == 0) if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0)
{ {
// Flush log data to the dat file // Flush log data to the dat file
CloseDb(strFile); bitdb.CloseDb(strFile);
dbenv.txn_checkpoint(0, 0, 0); bitdb.CheckpointLSN(strFile);
dbenv.lsn_reset(strFile.c_str(), 0); bitdb.mapFileUseCount.erase(strFile);
mapFileUseCount.erase(strFile);
bool fSuccess = true; bool fSuccess = true;
printf("Rewriting %s...\n", strFile.c_str()); printf("Rewriting %s...\n", strFile.c_str());
string strFileRes = strFile + ".rewrite"; string strFileRes = strFile + ".rewrite";
{ // surround usage of db with extra {} { // surround usage of db with extra {}
CDB db(strFile.c_str(), "r"); CDB db(strFile.c_str(), "r");
Db* pdbCopy = new Db(&dbenv, 0); Db* pdbCopy = new Db(&bitdb.dbenv, 0);
int ret = pdbCopy->open(NULL, // Txn pointer int ret = pdbCopy->open(NULL, // Txn pointer
strFileRes.c_str(), // Filename strFileRes.c_str(), // Filename
@ -267,7 +279,7 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip)
if (fSuccess) if (fSuccess)
{ {
db.Close(); db.Close();
CloseDb(strFile); bitdb.CloseDb(strFile);
if (pdbCopy->close(0)) if (pdbCopy->close(0))
fSuccess = false; fSuccess = false;
delete pdbCopy; delete pdbCopy;
@ -275,10 +287,10 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip)
} }
if (fSuccess) if (fSuccess)
{ {
Db dbA(&dbenv, 0); Db dbA(&bitdb.dbenv, 0);
if (dbA.remove(strFile.c_str(), NULL, 0)) if (dbA.remove(strFile.c_str(), NULL, 0))
fSuccess = false; fSuccess = false;
Db dbB(&dbenv, 0); Db dbB(&bitdb.dbenv, 0);
if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0)) if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
fSuccess = false; fSuccess = false;
} }
@ -293,12 +305,12 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip)
} }
void DBFlush(bool fShutdown) void CDBEnv::Flush(bool fShutdown)
{ {
int64 nStart = GetTimeMillis(); int64 nStart = GetTimeMillis();
// Flush log data to the actual data file // Flush log data to the actual data file
// on all files that are not in use // on all files that are not in use
printf("DBFlush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started"); printf("Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started");
if (!fDbEnvInit) if (!fDbEnvInit)
return; return;
{ {
@ -332,7 +344,7 @@ void DBFlush(bool fShutdown)
if (mapFileUseCount.empty()) if (mapFileUseCount.empty())
{ {
dbenv.log_archive(&listp, DB_ARCH_REMOVE); dbenv.log_archive(&listp, DB_ARCH_REMOVE);
EnvShutdown(); Close();
} }
} }
} }

84
src/db.h
Unescape View File

@ -25,21 +25,56 @@ class CWallet;
class CWalletTx; class CWalletTx;
extern unsigned int nWalletDBUpdated; extern unsigned int nWalletDBUpdated;
extern bool fDetachDB;
extern DbEnv dbenv;
extern void DBFlush(bool fShutdown);
void ThreadFlushWalletDB(void* parg); void ThreadFlushWalletDB(void* parg);
bool BackupWallet(const CWallet& wallet, const std::string& strDest); bool BackupWallet(const CWallet& wallet, const std::string& strDest);
class CDBEnv
{
private:
bool fDetachDB;
bool fDbEnvInit;
boost::filesystem::path pathEnv;
void EnvShutdown();
public:
mutable CCriticalSection cs_db;
DbEnv dbenv;
std::map<std::string, int> mapFileUseCount;
std::map<std::string, Db*> mapDb;
CDBEnv();
~CDBEnv();
bool Open(boost::filesystem::path pathEnv_);
void Close();
void Flush(bool fShutdown);
void CheckpointLSN(std::string strFile);
void SetDetach(bool fDetachDB_) { fDetachDB = fDetachDB_; }
void CloseDb(const std::string& strFile);
DbTxn *TxnBegin(int flags=DB_TXN_WRITE_NOSYNC)
{
DbTxn* ptxn = NULL;
int ret = dbenv.txn_begin(NULL, &ptxn, flags);
if (!ptxn || ret != 0)
return NULL;
return ptxn;
}
};
extern CDBEnv bitdb;
/** RAII class that provides access to a Berkeley database */ /** RAII class that provides access to a Berkeley database */
class CDB class CDB
{ {
protected: protected:
Db* pdb; Db* pdb;
std::string strFile; std::string strFile;
std::vector<DbTxn*> vTxn; DbTxn *activeTxn;
bool fReadOnly; bool fReadOnly;
explicit CDB(const char* pszFile, const char* pszMode="r+"); explicit CDB(const char* pszFile, const char* pszMode="r+");
@ -66,7 +101,7 @@ protected:
// Read // Read
Dbt datValue; Dbt datValue;
datValue.set_flags(DB_DBT_MALLOC); datValue.set_flags(DB_DBT_MALLOC);
int ret = pdb->get(GetTxn(), &datKey, &datValue, 0); int ret = pdb->get(activeTxn, &datKey, &datValue, 0);
memset(datKey.get_data(), 0, datKey.get_size()); memset(datKey.get_data(), 0, datKey.get_size());
if (datValue.get_data() == NULL) if (datValue.get_data() == NULL)
return false; return false;
@ -107,7 +142,7 @@ protected:
Dbt datValue(&ssValue[0], ssValue.size()); Dbt datValue(&ssValue[0], ssValue.size());
// Write // Write
int ret = pdb->put(GetTxn(), &datKey, &datValue, (fOverwrite ? 0 : DB_NOOVERWRITE)); int ret = pdb->put(activeTxn, &datKey, &datValue, (fOverwrite ? 0 : DB_NOOVERWRITE));
// Clear memory in case it was a private key // Clear memory in case it was a private key
memset(datKey.get_data(), 0, datKey.get_size()); memset(datKey.get_data(), 0, datKey.get_size());
@ -130,7 +165,7 @@ protected:
Dbt datKey(&ssKey[0], ssKey.size()); Dbt datKey(&ssKey[0], ssKey.size());
// Erase // Erase
int ret = pdb->del(GetTxn(), &datKey, 0); int ret = pdb->del(activeTxn, &datKey, 0);
// Clear memory // Clear memory
memset(datKey.get_data(), 0, datKey.get_size()); memset(datKey.get_data(), 0, datKey.get_size());
@ -150,7 +185,7 @@ protected:
Dbt datKey(&ssKey[0], ssKey.size()); Dbt datKey(&ssKey[0], ssKey.size());
// Exists // Exists
int ret = pdb->exists(GetTxn(), &datKey, 0); int ret = pdb->exists(activeTxn, &datKey, 0);
// Clear memory // Clear memory
memset(datKey.get_data(), 0, datKey.get_size()); memset(datKey.get_data(), 0, datKey.get_size());
@ -207,46 +242,33 @@ protected:
return 0; return 0;
} }
DbTxn* GetTxn()
{
if (!vTxn.empty())
return vTxn.back();
else
return NULL;
}
public: public:
bool TxnBegin() bool TxnBegin()
{ {
if (!pdb) if (!pdb || activeTxn)
return false; return false;
DbTxn* ptxn = NULL; DbTxn* ptxn = bitdb.TxnBegin();
int ret = dbenv.txn_begin(GetTxn(), &ptxn, DB_TXN_WRITE_NOSYNC); if (!ptxn)
if (!ptxn || ret != 0)
return false; return false;
vTxn.push_back(ptxn); activeTxn = ptxn;
return true; return true;
} }
bool TxnCommit() bool TxnCommit()
{ {
if (!pdb) if (!pdb || !activeTxn)
return false; return false;
if (vTxn.empty()) int ret = activeTxn->commit(0);
return false; activeTxn = NULL;
int ret = vTxn.back()->commit(0);
vTxn.pop_back();
return (ret == 0); return (ret == 0);
} }
bool TxnAbort() bool TxnAbort()
{ {
if (!pdb) if (!pdb || !activeTxn)
return false;
if (vTxn.empty())
return false; return false;
int ret = vTxn.back()->abort(); int ret = activeTxn->abort();
vTxn.pop_back(); activeTxn = NULL;
return (ret == 0); return (ret == 0);
} }

6
src/init.cpp
Unescape View File

@ -56,9 +56,9 @@ void Shutdown(void* parg)
{ {
fShutdown = true; fShutdown = true;
nTransactionsUpdated++; nTransactionsUpdated++;
DBFlush(false); bitdb.Flush(false);
StopNode(); StopNode();
DBFlush(true); bitdb.Flush(true);
boost::filesystem::remove(GetPidFile()); boost::filesystem::remove(GetPidFile());
UnregisterWallet(pwalletMain); UnregisterWallet(pwalletMain);
delete pwalletMain; delete pwalletMain;
@ -335,7 +335,7 @@ bool AppInit2()
// ********************************************************* Step 3: parameter-to-internal-flags // ********************************************************* Step 3: parameter-to-internal-flags
fDebug = GetBoolArg("-debug"); fDebug = GetBoolArg("-debug");
fDetachDB = GetBoolArg("-detachdb", false); bitdb.SetDetach(GetBoolArg("-detachdb", false));
#if !defined(WIN32) && !defined(QT_GUI) #if !defined(WIN32) && !defined(QT_GUI)
fDaemon = GetBoolArg("-daemon"); fDaemon = GetBoolArg("-daemon");

1
src/main.cpp
Unescape View File

@ -1463,7 +1463,6 @@ bool static Reorganize(CTxDB& txdb, CBlockIndex* pindexNew)
if (!block.ConnectBlock(txdb, pindex)) if (!block.ConnectBlock(txdb, pindex))
{ {
// Invalid block // Invalid block
txdb.TxnAbort();
return error("Reorganize() : ConnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str()); return error("Reorganize() : ConnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str());
} }

32
src/walletdb.cpp
Unescape View File

@ -13,10 +13,6 @@ using namespace boost;
static uint64 nAccountingEntryNumber = 0; static uint64 nAccountingEntryNumber = 0;
extern CCriticalSection cs_db;
extern map<string, int> mapFileUseCount;
extern void CloseDb(const string& strFile);
// //
// CWalletDB // CWalletDB
// //
@ -350,13 +346,13 @@ void ThreadFlushWalletDB(void* parg)
if (nLastFlushed != nWalletDBUpdated && GetTime() - nLastWalletUpdate >= 2) if (nLastFlushed != nWalletDBUpdated && GetTime() - nLastWalletUpdate >= 2)
{ {
TRY_LOCK(cs_db,lockDb); TRY_LOCK(bitdb.cs_db,lockDb);
if (lockDb) if (lockDb)
{ {
// Don't do this if any databases are in use // Don't do this if any databases are in use
int nRefCount = 0; int nRefCount = 0;
map<string, int>::iterator mi = mapFileUseCount.begin(); map<string, int>::iterator mi = bitdb.mapFileUseCount.begin();
while (mi != mapFileUseCount.end()) while (mi != bitdb.mapFileUseCount.end())
{ {
nRefCount += (*mi).second; nRefCount += (*mi).second;
mi++; mi++;
@ -364,19 +360,18 @@ void ThreadFlushWalletDB(void* parg)
if (nRefCount == 0 && !fShutdown) if (nRefCount == 0 && !fShutdown)
{ {
map<string, int>::iterator mi = mapFileUseCount.find(strFile); map<string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile);
if (mi != mapFileUseCount.end()) if (mi != bitdb.mapFileUseCount.end())
{ {
printf("Flushing wallet.dat\n"); printf("Flushing wallet.dat\n");
nLastFlushed = nWalletDBUpdated; nLastFlushed = nWalletDBUpdated;
int64 nStart = GetTimeMillis(); int64 nStart = GetTimeMillis();
// Flush wallet.dat so it's self contained // Flush wallet.dat so it's self contained
CloseDb(strFile); bitdb.CloseDb(strFile);
dbenv.txn_checkpoint(0, 0, 0); bitdb.CheckpointLSN(strFile);
dbenv.lsn_reset(strFile.c_str(), 0);
mapFileUseCount.erase(mi++); bitdb.mapFileUseCount.erase(mi++);
printf("Flushed wallet.dat %"PRI64d"ms\n", GetTimeMillis() - nStart); printf("Flushed wallet.dat %"PRI64d"ms\n", GetTimeMillis() - nStart);
} }
} }
@ -392,14 +387,13 @@ bool BackupWallet(const CWallet& wallet, const string& strDest)
while (!fShutdown) while (!fShutdown)
{ {
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
if (!mapFileUseCount.count(wallet.strWalletFile) || mapFileUseCount[wallet.strWalletFile] == 0) if (!bitdb.mapFileUseCount.count(wallet.strWalletFile) || bitdb.mapFileUseCount[wallet.strWalletFile] == 0)
{ {
// Flush log data to the dat file // Flush log data to the dat file
CloseDb(wallet.strWalletFile); bitdb.CloseDb(wallet.strWalletFile);
dbenv.txn_checkpoint(0, 0, 0); bitdb.CheckpointLSN(wallet.strWalletFile);
dbenv.lsn_reset(wallet.strWalletFile.c_str(), 0); bitdb.mapFileUseCount.erase(wallet.strWalletFile);
mapFileUseCount.erase(wallet.strWalletFile);
// Copy wallet.dat // Copy wallet.dat
filesystem::path pathSrc = GetDataDir() / wallet.strWalletFile; filesystem::path pathSrc = GetDataDir() / wallet.strWalletFile;

Write Preview
Loading…
Cancel
Save