twisterarmy/twister-core
1
0
Fork 0
mirror of https://github.com/twisterarmy/twister-core.git synced 2025-01-11 07:17:53 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'dbenv' into tmp

Browse Source 
Conflicts:
	src/db.cpp
This commit is contained in:
Jeff Garzik 2012-05-22 17:45:00 -04:00 committed by Jeff Garzik
commit 0134a1c08c
5 changed files with 165 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
src/db.cpp
View File

@ -26,14 +26,9 @@ unsigned int nWalletDBUpdated;
// CDB // CDB
// //
CCriticalSection cs_db; CDBEnv bitdb;
static bool fDbEnvInit = false;
bool fDetachDB = false;
DbEnv dbenv(0);
map<string, int> mapFileUseCount;
static map<string, Db*> mapDb;
static void EnvShutdown() void CDBEnv::EnvShutdown()
{ {
if (!fDbEnvInit) if (!fDbEnvInit)
return; return;
@ -50,21 +45,76 @@ static void EnvShutdown()
DbEnv(0).remove(GetDataDir().string().c_str(), 0); DbEnv(0).remove(GetDataDir().string().c_str(), 0);
} }
class CDBInit CDBEnv::CDBEnv() : dbenv(0)
{ {
public:
CDBInit()
{
}
~CDBInit()
{
EnvShutdown();
}
} }
instance_of_cdbinit;
CDBEnv::~CDBEnv()
{
EnvShutdown();
}
void CDBEnv::Close()
{
EnvShutdown();
}
bool CDBEnv::Open(boost::filesystem::path pathEnv_)
{
if (fDbEnvInit)
return true;
if (fShutdown)
return false;
pathEnv = pathEnv_;
filesystem::path pathDataDir = pathEnv;
filesystem::path pathLogDir = pathDataDir / "database";
filesystem::create_directory(pathLogDir);
filesystem::path pathErrorFile = pathDataDir / "db.log";
printf("dbenv.open LogDir=%s ErrorFile=%s\n", pathLogDir.string().c_str(), pathErrorFile.string().c_str());
unsigned int nEnvFlags = 0;
if (GetBoolArg("-privdb", true))
nEnvFlags |= DB_PRIVATE;
int nDbCache = GetArg("-dbcache", 25);
dbenv.set_lg_dir(pathLogDir.string().c_str());
dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1);
dbenv.set_lg_bsize(1048576);
dbenv.set_lg_max(10485760);
dbenv.set_lk_max_locks(10000);
dbenv.set_lk_max_objects(10000);
dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug
dbenv.set_flags(DB_AUTO_COMMIT, 1);
dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1);
dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1);
int ret = dbenv.open(pathDataDir.string().c_str(),
DB_CREATE |
DB_INIT_LOCK |
DB_INIT_LOG |
DB_INIT_MPOOL |
DB_INIT_TXN |
DB_THREAD |
DB_RECOVER |
nEnvFlags,
S_IRUSR | S_IWUSR);
if (ret > 0)
return error("CDB() : error %d opening database environment", ret);
fDbEnvInit = true;
return true;
}
void CDBEnv::CheckpointLSN(std::string strFile)
{
dbenv.txn_checkpoint(0, 0, 0);
dbenv.lsn_reset(strFile.c_str(), 0);
}
CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL) CDB::CDB(const char *pszFile, const char* pszMode) :
pdb(NULL), activeTxn(NULL)
{ {
int ret; int ret;
if (pszFile == NULL) if (pszFile == NULL)
@ -76,54 +126,17 @@ CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL)
if (fCreate) if (fCreate)
nFlags |= DB_CREATE; nFlags |= DB_CREATE;
unsigned int nEnvFlags = 0;
if (GetBoolArg("-privdb", true))
nEnvFlags |= DB_PRIVATE;
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
if (!fDbEnvInit) if (!bitdb.Open(GetDataDir()))
{ throw runtime_error("env open failed");
if (fShutdown)
return;
filesystem::path pathDataDir = GetDataDir();
filesystem::path pathLogDir = pathDataDir / "database";
filesystem::create_directory(pathLogDir);
filesystem::path pathErrorFile = pathDataDir / "db.log";
printf("dbenv.open LogDir=%s ErrorFile=%s\n", pathLogDir.string().c_str(), pathErrorFile.string().c_str());
int nDbCache = GetArg("-dbcache", 25);
dbenv.set_lg_dir(pathLogDir.string().c_str());
dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1);
dbenv.set_lg_bsize(1048576);
dbenv.set_lg_max(10485760);
dbenv.set_lk_max_locks(10000);
dbenv.set_lk_max_objects(10000);
dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug
dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1);
dbenv.set_flags(DB_AUTO_COMMIT, 1);
dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1);
ret = dbenv.open(pathDataDir.string().c_str(),
DB_CREATE |
DB_INIT_LOCK |
DB_INIT_LOG |
DB_INIT_MPOOL |
DB_INIT_TXN |
DB_THREAD |
DB_RECOVER |
nEnvFlags,
S_IRUSR | S_IWUSR);
if (ret > 0)
throw runtime_error(strprintf("CDB() : error %d opening database environment", ret));
fDbEnvInit = true;
}
strFile = pszFile; strFile = pszFile;
++mapFileUseCount[strFile]; ++bitdb.mapFileUseCount[strFile];
pdb = mapDb[strFile]; pdb = bitdb.mapDb[strFile];
if (pdb == NULL) if (pdb == NULL)
{ {
pdb = new Db(&dbenv, 0); pdb = new Db(&bitdb.dbenv, 0);
ret = pdb->open(NULL, // Txn pointer ret = pdb->open(NULL, // Txn pointer
pszFile, // Filename pszFile, // Filename
@ -137,8 +150,8 @@ CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL)
delete pdb; delete pdb;
pdb = NULL; pdb = NULL;
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
--mapFileUseCount[strFile]; --bitdb.mapFileUseCount[strFile];
} }
strFile = ""; strFile = "";
throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", pszFile, ret)); throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", pszFile, ret));
@ -152,7 +165,7 @@ CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL)
fReadOnly = fTmp; fReadOnly = fTmp;
} }
mapDb[strFile] = pdb; bitdb.mapDb[strFile] = pdb;
} }
} }
} }
@ -161,9 +174,9 @@ void CDB::Close()
{ {
if (!pdb) if (!pdb)
return; return;
if (!vTxn.empty()) if (activeTxn)
vTxn.front()->abort(); activeTxn->abort();
vTxn.clear(); activeTxn = NULL;
pdb = NULL; pdb = NULL;
// Flush database activity from memory pool to disk log // Flush database activity from memory pool to disk log
@ -175,15 +188,15 @@ void CDB::Close()
if (strFile == "blkindex.dat" && IsInitialBlockDownload()) if (strFile == "blkindex.dat" && IsInitialBlockDownload())
nMinutes = 5; nMinutes = 5;
dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100)*1024 : 0, nMinutes, 0); bitdb.dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100)*1024 : 0, nMinutes, 0);
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
--mapFileUseCount[strFile]; --bitdb.mapFileUseCount[strFile];
} }
} }
void CloseDb(const string& strFile) void CDBEnv::CloseDb(const string& strFile)
{ {
{ {
LOCK(cs_db); LOCK(cs_db);
@ -203,21 +216,20 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip)
while (!fShutdown) while (!fShutdown)
{ {
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
if (!mapFileUseCount.count(strFile) || mapFileUseCount[strFile] == 0) if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0)
{ {
// Flush log data to the dat file // Flush log data to the dat file
CloseDb(strFile); bitdb.CloseDb(strFile);
dbenv.txn_checkpoint(0, 0, 0); bitdb.CheckpointLSN(strFile);
dbenv.lsn_reset(strFile.c_str(), 0); bitdb.mapFileUseCount.erase(strFile);
mapFileUseCount.erase(strFile);
bool fSuccess = true; bool fSuccess = true;
printf("Rewriting %s...\n", strFile.c_str()); printf("Rewriting %s...\n", strFile.c_str());
string strFileRes = strFile + ".rewrite"; string strFileRes = strFile + ".rewrite";
{ // surround usage of db with extra {} { // surround usage of db with extra {}
CDB db(strFile.c_str(), "r"); CDB db(strFile.c_str(), "r");
Db* pdbCopy = new Db(&dbenv, 0); Db* pdbCopy = new Db(&bitdb.dbenv, 0);
int ret = pdbCopy->open(NULL, // Txn pointer int ret = pdbCopy->open(NULL, // Txn pointer
strFileRes.c_str(), // Filename strFileRes.c_str(), // Filename
@ -267,7 +279,7 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip)
if (fSuccess) if (fSuccess)
{ {
db.Close(); db.Close();
CloseDb(strFile); bitdb.CloseDb(strFile);
if (pdbCopy->close(0)) if (pdbCopy->close(0))
fSuccess = false; fSuccess = false;
delete pdbCopy; delete pdbCopy;
@ -275,10 +287,10 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip)
} }
if (fSuccess) if (fSuccess)
{ {
Db dbA(&dbenv, 0); Db dbA(&bitdb.dbenv, 0);
if (dbA.remove(strFile.c_str(), NULL, 0)) if (dbA.remove(strFile.c_str(), NULL, 0))
fSuccess = false; fSuccess = false;
Db dbB(&dbenv, 0); Db dbB(&bitdb.dbenv, 0);
if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0)) if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
fSuccess = false; fSuccess = false;
} }
@ -293,12 +305,12 @@ bool CDB::Rewrite(const string& strFile, const char* pszSkip)
} }
void DBFlush(bool fShutdown) void CDBEnv::Flush(bool fShutdown)
{ {
int64 nStart = GetTimeMillis(); int64 nStart = GetTimeMillis();
// Flush log data to the actual data file // Flush log data to the actual data file
// on all files that are not in use // on all files that are not in use
printf("DBFlush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started"); printf("Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started");
if (!fDbEnvInit) if (!fDbEnvInit)
return; return;
{ {
@ -332,7 +344,7 @@ void DBFlush(bool fShutdown)
if (mapFileUseCount.empty()) if (mapFileUseCount.empty())
{ {
dbenv.log_archive(&listp, DB_ARCH_REMOVE); dbenv.log_archive(&listp, DB_ARCH_REMOVE);
EnvShutdown(); Close();
} }
} }
} }

84
src/db.h
View File

@ -25,21 +25,56 @@ class CWallet;
class CWalletTx; class CWalletTx;
extern unsigned int nWalletDBUpdated; extern unsigned int nWalletDBUpdated;
extern bool fDetachDB;
extern DbEnv dbenv;
extern void DBFlush(bool fShutdown);
void ThreadFlushWalletDB(void* parg); void ThreadFlushWalletDB(void* parg);
bool BackupWallet(const CWallet& wallet, const std::string& strDest); bool BackupWallet(const CWallet& wallet, const std::string& strDest);
class CDBEnv
{
private:
bool fDetachDB;
bool fDbEnvInit;
boost::filesystem::path pathEnv;
void EnvShutdown();
public:
mutable CCriticalSection cs_db;
DbEnv dbenv;
std::map<std::string, int> mapFileUseCount;
std::map<std::string, Db*> mapDb;
CDBEnv();
~CDBEnv();
bool Open(boost::filesystem::path pathEnv_);
void Close();
void Flush(bool fShutdown);
void CheckpointLSN(std::string strFile);
void SetDetach(bool fDetachDB_) { fDetachDB = fDetachDB_; }
void CloseDb(const std::string& strFile);
DbTxn *TxnBegin(int flags=DB_TXN_WRITE_NOSYNC)
{
DbTxn* ptxn = NULL;
int ret = dbenv.txn_begin(NULL, &ptxn, flags);
if (!ptxn || ret != 0)
return NULL;
return ptxn;
}
};
extern CDBEnv bitdb;
/** RAII class that provides access to a Berkeley database */ /** RAII class that provides access to a Berkeley database */
class CDB class CDB
{ {
protected: protected:
Db* pdb; Db* pdb;
std::string strFile; std::string strFile;
std::vector<DbTxn*> vTxn; DbTxn *activeTxn;
bool fReadOnly; bool fReadOnly;
explicit CDB(const char* pszFile, const char* pszMode="r+"); explicit CDB(const char* pszFile, const char* pszMode="r+");
@ -66,7 +101,7 @@ protected:
// Read // Read
Dbt datValue; Dbt datValue;
datValue.set_flags(DB_DBT_MALLOC); datValue.set_flags(DB_DBT_MALLOC);
int ret = pdb->get(GetTxn(), &datKey, &datValue, 0); int ret = pdb->get(activeTxn, &datKey, &datValue, 0);
memset(datKey.get_data(), 0, datKey.get_size()); memset(datKey.get_data(), 0, datKey.get_size());
if (datValue.get_data() == NULL) if (datValue.get_data() == NULL)
return false; return false;
@ -107,7 +142,7 @@ protected:
Dbt datValue(&ssValue[0], ssValue.size()); Dbt datValue(&ssValue[0], ssValue.size());
// Write // Write
int ret = pdb->put(GetTxn(), &datKey, &datValue, (fOverwrite ? 0 : DB_NOOVERWRITE)); int ret = pdb->put(activeTxn, &datKey, &datValue, (fOverwrite ? 0 : DB_NOOVERWRITE));
// Clear memory in case it was a private key // Clear memory in case it was a private key
memset(datKey.get_data(), 0, datKey.get_size()); memset(datKey.get_data(), 0, datKey.get_size());
@ -130,7 +165,7 @@ protected:
Dbt datKey(&ssKey[0], ssKey.size()); Dbt datKey(&ssKey[0], ssKey.size());
// Erase // Erase
int ret = pdb->del(GetTxn(), &datKey, 0); int ret = pdb->del(activeTxn, &datKey, 0);
// Clear memory // Clear memory
memset(datKey.get_data(), 0, datKey.get_size()); memset(datKey.get_data(), 0, datKey.get_size());
@ -150,7 +185,7 @@ protected:
Dbt datKey(&ssKey[0], ssKey.size()); Dbt datKey(&ssKey[0], ssKey.size());
// Exists // Exists
int ret = pdb->exists(GetTxn(), &datKey, 0); int ret = pdb->exists(activeTxn, &datKey, 0);
// Clear memory // Clear memory
memset(datKey.get_data(), 0, datKey.get_size()); memset(datKey.get_data(), 0, datKey.get_size());
@ -207,46 +242,33 @@ protected:
return 0; return 0;
} }
DbTxn* GetTxn()
{
if (!vTxn.empty())
return vTxn.back();
else
return NULL;
}
public: public:
bool TxnBegin() bool TxnBegin()
{ {
if (!pdb) if (!pdb || activeTxn)
return false; return false;
DbTxn* ptxn = NULL; DbTxn* ptxn = bitdb.TxnBegin();
int ret = dbenv.txn_begin(GetTxn(), &ptxn, DB_TXN_WRITE_NOSYNC); if (!ptxn)
if (!ptxn || ret != 0)
return false; return false;
vTxn.push_back(ptxn); activeTxn = ptxn;
return true; return true;
} }
bool TxnCommit() bool TxnCommit()
{ {
if (!pdb) if (!pdb || !activeTxn)
return false; return false;
if (vTxn.empty()) int ret = activeTxn->commit(0);
return false; activeTxn = NULL;
int ret = vTxn.back()->commit(0);
vTxn.pop_back();
return (ret == 0); return (ret == 0);
} }
bool TxnAbort() bool TxnAbort()
{ {
if (!pdb) if (!pdb || !activeTxn)
return false; return false;
if (vTxn.empty()) int ret = activeTxn->abort();
return false; activeTxn = NULL;
int ret = vTxn.back()->abort();
vTxn.pop_back();
return (ret == 0); return (ret == 0);
} }

6
src/init.cpp
View File

@ -56,9 +56,9 @@ void Shutdown(void* parg)
{ {
fShutdown = true; fShutdown = true;
nTransactionsUpdated++; nTransactionsUpdated++;
DBFlush(false); bitdb.Flush(false);
StopNode(); StopNode();
DBFlush(true); bitdb.Flush(true);
boost::filesystem::remove(GetPidFile()); boost::filesystem::remove(GetPidFile());
UnregisterWallet(pwalletMain); UnregisterWallet(pwalletMain);
delete pwalletMain; delete pwalletMain;
@ -335,7 +335,7 @@ bool AppInit2()
// ********************************************************* Step 3: parameter-to-internal-flags // ********************************************************* Step 3: parameter-to-internal-flags
fDebug = GetBoolArg("-debug"); fDebug = GetBoolArg("-debug");
fDetachDB = GetBoolArg("-detachdb", false); bitdb.SetDetach(GetBoolArg("-detachdb", false));
#if !defined(WIN32) && !defined(QT_GUI) #if !defined(WIN32) && !defined(QT_GUI)
fDaemon = GetBoolArg("-daemon"); fDaemon = GetBoolArg("-daemon");

1
src/main.cpp
View File

@ -1463,7 +1463,6 @@ bool static Reorganize(CTxDB& txdb, CBlockIndex* pindexNew)
if (!block.ConnectBlock(txdb, pindex)) if (!block.ConnectBlock(txdb, pindex))
{ {
// Invalid block // Invalid block
txdb.TxnAbort();
return error("Reorganize() : ConnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str()); return error("Reorganize() : ConnectBlock %s failed", pindex->GetBlockHash().ToString().substr(0,20).c_str());
} }

32
src/walletdb.cpp
View File

@ -13,10 +13,6 @@ using namespace boost;
static uint64 nAccountingEntryNumber = 0; static uint64 nAccountingEntryNumber = 0;
extern CCriticalSection cs_db;
extern map<string, int> mapFileUseCount;
extern void CloseDb(const string& strFile);
// //
// CWalletDB // CWalletDB
// //
@ -350,13 +346,13 @@ void ThreadFlushWalletDB(void* parg)
if (nLastFlushed != nWalletDBUpdated && GetTime() - nLastWalletUpdate >= 2) if (nLastFlushed != nWalletDBUpdated && GetTime() - nLastWalletUpdate >= 2)
{ {
TRY_LOCK(cs_db,lockDb); TRY_LOCK(bitdb.cs_db,lockDb);
if (lockDb) if (lockDb)
{ {
// Don't do this if any databases are in use // Don't do this if any databases are in use
int nRefCount = 0; int nRefCount = 0;
map<string, int>::iterator mi = mapFileUseCount.begin(); map<string, int>::iterator mi = bitdb.mapFileUseCount.begin();
while (mi != mapFileUseCount.end()) while (mi != bitdb.mapFileUseCount.end())
{ {
nRefCount += (*mi).second; nRefCount += (*mi).second;
mi++; mi++;
@ -364,19 +360,18 @@ void ThreadFlushWalletDB(void* parg)
if (nRefCount == 0 && !fShutdown) if (nRefCount == 0 && !fShutdown)
{ {
map<string, int>::iterator mi = mapFileUseCount.find(strFile); map<string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile);
if (mi != mapFileUseCount.end()) if (mi != bitdb.mapFileUseCount.end())
{ {
printf("Flushing wallet.dat\n"); printf("Flushing wallet.dat\n");
nLastFlushed = nWalletDBUpdated; nLastFlushed = nWalletDBUpdated;
int64 nStart = GetTimeMillis(); int64 nStart = GetTimeMillis();
// Flush wallet.dat so it's self contained // Flush wallet.dat so it's self contained
CloseDb(strFile); bitdb.CloseDb(strFile);
dbenv.txn_checkpoint(0, 0, 0); bitdb.CheckpointLSN(strFile);
dbenv.lsn_reset(strFile.c_str(), 0);
mapFileUseCount.erase(mi++); bitdb.mapFileUseCount.erase(mi++);
printf("Flushed wallet.dat %"PRI64d"ms\n", GetTimeMillis() - nStart); printf("Flushed wallet.dat %"PRI64d"ms\n", GetTimeMillis() - nStart);
} }
} }
@ -392,14 +387,13 @@ bool BackupWallet(const CWallet& wallet, const string& strDest)
while (!fShutdown) while (!fShutdown)
{ {
{ {
LOCK(cs_db); LOCK(bitdb.cs_db);
if (!mapFileUseCount.count(wallet.strWalletFile) || mapFileUseCount[wallet.strWalletFile] == 0) if (!bitdb.mapFileUseCount.count(wallet.strWalletFile) || bitdb.mapFileUseCount[wallet.strWalletFile] == 0)
{ {
// Flush log data to the dat file // Flush log data to the dat file
CloseDb(wallet.strWalletFile); bitdb.CloseDb(wallet.strWalletFile);
dbenv.txn_checkpoint(0, 0, 0); bitdb.CheckpointLSN(wallet.strWalletFile);
dbenv.lsn_reset(wallet.strWalletFile.c_str(), 0); bitdb.mapFileUseCount.erase(wallet.strWalletFile);
mapFileUseCount.erase(wallet.strWalletFile);
// Copy wallet.dat // Copy wallet.dat
filesystem::path pathSrc = GetDataDir() / wallet.strWalletFile; filesystem::path pathSrc = GetDataDir() / wallet.strWalletFile;