d4708
/
qBittorrent
mirror of https://github.com/d47081/qBittorrent.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Don't save the GeoIP nodes beforehand. 

It saves a lot of memory and isn't much faster.
adaptive-webui-19844
sledgehammer999 10 years ago
parent
787b824d90
commit
ef5b3b90c3
2 changed files with 377 additions and 447 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 388
      src/core/net/private/geoipdatabase.cpp
  2. 48
      src/core/net/private/geoipdatabase.h

388
src/core/net/private/geoipdatabase.cpp
Unescape View File

@ -27,12 +27,8 @@ @@ -27,12 +27,8 @@
*/
#include <QDebug>
#include <QCoreApplication>
#include <QVariant>
#include <QHash>
#include <QList>
#include <QScopedArrayPointer>
#include <QScopedPointer>
#include <QHostAddress>
#include <QDateTime>
#include <QFile>
@ -40,28 +36,15 @@ @@ -40,28 +36,15 @@
#include "core/types.h"
#include "geoipdatabase.h"
struct Node
{
quint32 left;
quint32 right;
};
struct GeoIPData
{
// Metadata
quint16 ipVersion;
quint16 recordSize;
quint32 nodeCount;
QDateTime buildEpoch;
// Search data
QList<Node> index;
QHash<quint32, QString> countries;
};
namespace
{
const quint32 __ENDIAN_TEST__ = 0x00000001;
const bool __IS_LITTLE_ENDIAN__ = (reinterpret_cast<const uchar *>(&__ENDIAN_TEST__)[0] == 0x01);
const int MAX_FILE_SIZE = 10485760; // 10MB
const char DB_TYPE[] = "GeoLite2-Country";
const quint32 MAX_METADATA_SIZE = 131072; // 128KB
const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
const char DATA_SECTION_SEPARATOR[16] = { 0 };
enum class DataType
{
@ -83,89 +66,60 @@ namespace @@ -83,89 +66,60 @@ namespace
Float = 15
};
struct DataFieldDescriptor
{
#if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
Q_IPV6ADDR createMappedAddress(quint32 ip4);
#endif
}
struct DataFieldDescriptor
{
DataType fieldType;
union
{
quint32 fieldSize;
quint32 offset; // Pointer
};
};
const int MAX_FILE_SIZE = 10485760; // 10MB
const char DB_TYPE[] = "GeoLite2-Country";
const quint32 MAX_METADATA_SIZE = 131072; // 128KB
const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
const char DATA_SECTION_SEPARATOR[16] = { 0 };
#if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
Q_IPV6ADDR createMappedAddress(quint32 ip4);
#endif
};
class Loader
{
Q_DECLARE_TR_FUNCTIONS(GeoIPDatabase)
public:
GeoIPData *load(const QString &filename);
GeoIPData *load(const QByteArray &data);
QString error() const;
private:
bool parseMetadata(const QVariantHash &metadata);
bool loadDB();
QVariantHash readMetadata();
QVariant readDataField(quint32 &offset);
bool readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out);
void fromBigEndian(uchar *buf, quint32 len);
QVariant readMapValue(quint32 &offset, quint32 count);
QVariant readArrayValue(quint32 &offset, quint32 count);
template<typename T>
QVariant readPlainValue(quint32 &offset, quint8 len)
{
T value = 0;
const uchar *const data = m_data + offset;
const quint32 availSize = m_size - offset;
GeoIPDatabase::GeoIPDatabase(quint32 size)
: m_ipVersion(0)
, m_recordSize(0)
, m_nodeCount(0)
, m_nodeSize(0)
, m_indexSize(0)
, m_recordBytes(0)
, m_size(size)
, m_data(new uchar[size])
{
}
if ((len > 0) && (len <= sizeof(T) && (availSize >= len))) {
// copy input data to last 'len' bytes of 'value'
uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len);
memcpy(dst, data, len);
fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T));
offset += len;
GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
{
GeoIPDatabase *db = 0;
QFile file(filename);
if (file.size() > MAX_FILE_SIZE) {
error = tr("Unsupported database file size.");
return 0;
}
return QVariant::fromValue(value);
if (!file.open(QFile::ReadOnly)) {
error = file.errorString();
return 0;
}
private:
const uchar *m_data;
quint32 m_size;
QString m_error;
GeoIPData *m_geoIPData;
};
}
// GeoIPDatabase
db = new GeoIPDatabase(file.size());
GeoIPDatabase::GeoIPDatabase(GeoIPData *geoIPData)
: m_geoIPData(geoIPData)
{
}
if (file.read((char *)db->m_data, db->m_size) != db->m_size) {
error = file.errorString();
delete db;
return 0;
}
GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
{
GeoIPDatabase *db = 0;
Loader loader;
GeoIPData *geoIPData = loader.load(filename);
if (!geoIPData)
error = loader.error();
else
db = new GeoIPDatabase(geoIPData);
if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error)) {
delete db;
return 0;
}
return db;
}
@ -173,20 +127,26 @@ GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error) @@ -173,20 +127,26 @@ GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error)
{
GeoIPDatabase *db = 0;
if (data.size() > MAX_FILE_SIZE) {
error = tr("Unsupported database file size.");
return 0;
}
Loader loader;
GeoIPData *geoIPData = loader.load(data);
if (!geoIPData)
error = loader.error();
else
db = new GeoIPDatabase(geoIPData);
db = new GeoIPDatabase(data.size());
memcpy((char *)db->m_data, data.constData(), db->m_size);
if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error)) {
delete db;
return 0;
}
return db;
}
GeoIPDatabase::~GeoIPDatabase()
{
delete m_geoIPData;
delete [] m_data;
}
QString GeoIPDatabase::type() const
@ -196,12 +156,12 @@ QString GeoIPDatabase::type() const @@ -196,12 +156,12 @@ QString GeoIPDatabase::type() const
quint16 GeoIPDatabase::ipVersion() const
{
return m_geoIPData->ipVersion;
return m_ipVersion;
}
QDateTime GeoIPDatabase::buildEpoch() const
{
return m_geoIPData->buildEpoch;
return m_buildEpoch;
}
QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
@ -213,100 +173,66 @@ QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const @@ -213,100 +173,66 @@ QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
#else
Q_IPV6ADDR addr = hostAddr.toIPv6Address();
#endif
const quint32 nodeCount = static_cast<quint32>(m_geoIPData->index.size());
Node node = m_geoIPData->index[0];
const uchar *ptr = m_data;
for (int i = 0; i < 16; ++i) {
for (int j = 0; j < 8; ++j) {
bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1);
quint32 id = (right ? node.right : node.left);
if (id == nodeCount)
return QString();
else if (id > nodeCount)
return m_geoIPData->countries[id];
else
node = m_geoIPData->index[id];
}
}
// Interpret the left/right record as number
if (right)
ptr += m_recordBytes;
return QString();
}
namespace
{
// Loader
quint32 id = 0;
uchar *idPtr = reinterpret_cast<uchar *>(&id);
memcpy(&idPtr[4 - m_recordBytes], ptr, m_recordBytes);
fromBigEndian(idPtr, 4);
GeoIPData *Loader::load(const QString &filename)
{
QFile file(filename);
if (file.size() > MAX_FILE_SIZE) {
m_error = tr("Unsupported database file size.");
return 0;
if (id == m_nodeCount) {
return QString();
}
if (!file.open(QFile::ReadOnly)) {
m_error = file.errorString();
return 0;
else if (id > m_nodeCount) {
QString country = m_countries.value(id);
if (country.isEmpty()) {
const quint32 offset = id - m_nodeCount - sizeof(DATA_SECTION_SEPARATOR);
quint32 tmp = offset + m_indexSize + sizeof(DATA_SECTION_SEPARATOR);
QVariant val = readDataField(tmp);
if (val.userType() == QMetaType::QVariantHash) {
country = val.toHash()["country"].toHash()["iso_code"].toString();
m_countries[id] = country;
}
m_size = file.size();
QScopedArrayPointer<uchar> data(new uchar[m_size]);
m_data = data.data();
if (file.read((char *)m_data, m_size) != m_size) {
m_error = file.errorString();
return 0;
}
QScopedPointer<GeoIPData> geoIPData(new GeoIPData);
m_geoIPData = geoIPData.data();
if (!parseMetadata(readMetadata()) || !loadDB())
return 0;
return geoIPData.take();
return country;
}
GeoIPData *Loader::load(const QByteArray &data)
{
if (data.size() > MAX_FILE_SIZE) {
m_error = tr("Unsupported database file size.");
return 0;
else {
ptr = m_data + (id * m_nodeSize);
}
m_size = data.size();
m_data = reinterpret_cast<const uchar *>(data.constData());
QScopedPointer<GeoIPData> geoIPData(new GeoIPData);
m_geoIPData = geoIPData.data();
if (!parseMetadata(readMetadata()) || !loadDB())
return 0;
return geoIPData.take();
}
QString Loader::error() const
{
return m_error;
}
return QString();
}
#define CHECK_METADATA_REQ(key, type) \
if (!metadata.contains(#key)) { \
m_error = errMsgNotFound.arg(#key); \
if (!metadata.contains(#key)) { \
error = errMsgNotFound.arg(#key); \
return false; \
} \
else if (metadata.value(#key).userType() != QMetaType::type) { \
m_error = errMsgInvalid.arg(#key); \
} \
else if (metadata.value(#key).userType() != QMetaType::type) { \
error = errMsgInvalid.arg(#key); \
return false; \
}
}
#define CHECK_METADATA_OPT(key, type) \
if (metadata.contains(#key)) { \
if (metadata.contains(#key)) { \
if (metadata.value(#key).userType() != QMetaType::type) { \
m_error = errMsgInvalid.arg(#key); \
error = errMsgInvalid.arg(#key); \
return false; \
} \
}
}
bool Loader::parseMetadata(const QVariantHash &metadata)
{
bool GeoIPDatabase::parseMetadata(const QVariantHash &metadata, QString &error)
{
const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
@ -317,102 +243,63 @@ namespace @@ -317,102 +243,63 @@ namespace
uint versionMajor = metadata.value("binary_format_major_version").toUInt();
uint versionMinor = metadata.value("binary_format_minor_version").toUInt();
if (versionMajor != 2) {
m_error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
return false;
}
CHECK_METADATA_REQ(ip_version, UShort);
m_geoIPData->ipVersion = metadata.value("ip_version").value<quint16>();
if (m_geoIPData->ipVersion != 6) {
m_error = tr("Unsupported IP version: %1").arg(m_geoIPData->ipVersion);
m_ipVersion = metadata.value("ip_version").value<quint16>();
if (m_ipVersion != 6) {
error = tr("Unsupported IP version: %1").arg(m_ipVersion);
return false;
}
CHECK_METADATA_REQ(record_size, UShort);
m_geoIPData->recordSize = metadata.value("record_size").value<quint16>();
if (m_geoIPData->recordSize != 24) {
m_error = tr("Unsupported record size: %1").arg(m_geoIPData->recordSize);
m_recordSize = metadata.value("record_size").value<quint16>();
if (m_recordSize != 24) {
error = tr("Unsupported record size: %1").arg(m_recordSize);
return false;
}
m_nodeSize = m_recordSize / 4;
m_recordBytes = m_nodeSize / 2;
CHECK_METADATA_REQ(node_count, UInt);
m_geoIPData->nodeCount = metadata.value("node_count").value<quint32>();
m_nodeCount = metadata.value("node_count").value<quint32>();
m_indexSize = m_nodeCount * m_nodeSize;
CHECK_METADATA_REQ(database_type, QString);
QString dbType = metadata.value("database_type").toString();
if (dbType != DB_TYPE) {
m_error = tr("Invalid database type: %1").arg(dbType);
error = tr("Invalid database type: %1").arg(dbType);
return false;
}
CHECK_METADATA_REQ(build_epoch, ULongLong);
m_geoIPData->buildEpoch = QDateTime::fromTime_t(metadata.value("build_epoch").toULongLong());
m_buildEpoch = QDateTime::fromTime_t(metadata.value("build_epoch").toULongLong());
CHECK_METADATA_OPT(languages, QVariantList);
CHECK_METADATA_OPT(description, QVariantHash);
return true;
}
}
bool Loader::loadDB()
{
bool GeoIPDatabase::loadDB(QString &error) const
{
qDebug() << "Parsing MaxMindDB index tree...";
const int nodeSize = m_geoIPData->recordSize / 4; // in bytes
const int indexSize = m_geoIPData->nodeCount * nodeSize;
const int nodeSize = m_recordSize / 4; // in bytes
const int indexSize = m_nodeCount * nodeSize;
if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
|| (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0)) {
m_error = tr("Database corrupted: no data section found.");
error = tr("Database corrupted: no data section found.");
return false;
}
m_geoIPData->index.reserve(m_geoIPData->nodeCount);
const int recordBytes = nodeSize / 2;
const uchar *ptr = m_data;
bool left = true;
Node node;
for (quint32 i = 0; i < (2 * m_geoIPData->nodeCount); ++i) {
quint32 id = 0;
uchar *idPtr = reinterpret_cast<uchar *>(&id);
memcpy(&idPtr[4 - recordBytes], ptr, recordBytes);
fromBigEndian(idPtr, 4);
if ((id > m_geoIPData->nodeCount) && !m_geoIPData->countries.contains(id)) {
const quint32 offset = id - m_geoIPData->nodeCount - sizeof(DATA_SECTION_SEPARATOR);
quint32 tmp = offset + indexSize + sizeof(DATA_SECTION_SEPARATOR);
QVariant val = readDataField(tmp);
if (val.userType() == QMetaType::QVariantHash) {
m_geoIPData->countries[id] = val.toHash()["country"].toHash()["iso_code"].toString();
}
else if (val.userType() == QVariant::Invalid) {
m_error = tr("Database corrupted: invalid data type at DATA@%1").arg(offset, 8, 16, QLatin1Char('0'));
return false;
}
else {
m_error = tr("Invalid database: unsupported data type at DATA@%1").arg(offset, 8, 16, QLatin1Char('0'));
return false;
}
}
if (left) {
node.left = id;
}
else {
node.right = id;
m_geoIPData->index << node;
}
left = !left;
ptr += recordBytes;
}
return true;
}
}
QVariantHash Loader::readMetadata()
{
QVariantHash GeoIPDatabase::readMetadata() const
{
const char *ptr = reinterpret_cast<const char *>(m_data);
quint32 size = m_size;
if (m_size > MAX_METADATA_SIZE) {
@ -427,16 +314,15 @@ namespace @@ -427,16 +314,15 @@ namespace
index += (m_size - MAX_METADATA_SIZE); // from begin of all data
quint32 offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
QVariant metadata = readDataField(offset);
m_size = index; // truncate m_size to not contain metadata section
if (metadata.userType() == QMetaType::QVariantHash)
return metadata.toHash();
}
return QVariantHash();
}
}
QVariant Loader::readDataField(quint32 &offset)
{
QVariant GeoIPDatabase::readDataField(quint32 &offset) const
{
DataFieldDescriptor descr;
if (!readDataFieldDescriptor(offset, descr))
return QVariant();
@ -446,7 +332,7 @@ namespace @@ -446,7 +332,7 @@ namespace
if (descr.fieldType == DataType::Pointer) {
usePointer = true;
// convert offset from data section to global
locOffset = descr.offset + (m_geoIPData->nodeCount * m_geoIPData->recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
locOffset = descr.offset + (m_nodeCount * m_recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
if (!readDataFieldDescriptor(locOffset, descr))
return QVariant();
}
@ -511,10 +397,10 @@ namespace @@ -511,10 +397,10 @@ namespace
if (!usePointer)
offset = locOffset;
return fieldValue;
}
}
bool Loader::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out)
{
bool GeoIPDatabase::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const
{
const uchar *dataPtr = m_data + offset;
int availSize = m_size - offset;
if (availSize < 1) return false;
@ -566,16 +452,16 @@ namespace @@ -566,16 +452,16 @@ namespace
}
return true;
}
}
void Loader::fromBigEndian(uchar *buf, quint32 len)
{
void GeoIPDatabase::fromBigEndian(uchar *buf, quint32 len) const
{
if (__IS_LITTLE_ENDIAN__)
std::reverse(buf, buf + len);
}
}
QVariant Loader::readMapValue(quint32 &offset, quint32 count)
{
QVariant GeoIPDatabase::readMapValue(quint32 &offset, quint32 count) const
{
QVariantHash map;
for (quint32 i = 0; i < count; ++i) {
@ -592,10 +478,10 @@ namespace @@ -592,10 +478,10 @@ namespace
}
return map;
}
}
QVariant Loader::readArrayValue(quint32 &offset, quint32 count)
{
QVariant GeoIPDatabase::readArrayValue(quint32 &offset, quint32 count) const
{
QVariantList array;
for (quint32 i = 0; i < count; ++i) {
@ -607,8 +493,10 @@ namespace @@ -607,8 +493,10 @@ namespace
}
return array;
}
}
namespace
{
#if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
Q_IPV6ADDR createMappedAddress(quint32 ip4)
{

48
src/core/net/private/geoipdatabase.h
Unescape View File

@ -30,16 +30,19 @@ @@ -30,16 +30,19 @@
#define GEOIPDATABASE_H
#include <QtGlobal>
#include <QCoreApplication>
class QHostAddress;
class QString;
class QByteArray;
class QDateTime;
struct GeoIPData;
struct DataFieldDescriptor;
class GeoIPDatabase
{
Q_DECLARE_TR_FUNCTIONS(GeoIPDatabase)
public:
static GeoIPDatabase *load(const QString &filename, QString &error);
static GeoIPDatabase *load(const QByteArray &data, QString &error);
@ -52,9 +55,48 @@ public: @@ -52,9 +55,48 @@ public:
QString lookup(const QHostAddress &hostAddr) const;
private:
GeoIPDatabase(GeoIPData *geoIPData);
GeoIPDatabase(quint32 size);
bool parseMetadata(const QVariantHash &metadata, QString &error);
bool loadDB(QString &error) const;
QVariantHash readMetadata() const;
QVariant readDataField(quint32 &offset) const;
bool readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const;
void fromBigEndian(uchar *buf, quint32 len) const;
QVariant readMapValue(quint32 &offset, quint32 count) const;
QVariant readArrayValue(quint32 &offset, quint32 count) const;
template<typename T>
QVariant readPlainValue(quint32 &offset, quint8 len) const
{
T value = 0;
const uchar *const data = m_data + offset;
const quint32 availSize = m_size - offset;
if ((len > 0) && (len <= sizeof(T) && (availSize >= len))) {
// copy input data to last 'len' bytes of 'value'
uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len);
memcpy(dst, data, len);
fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T));
offset += len;
}
return QVariant::fromValue(value);
}
GeoIPData *m_geoIPData;
// Metadata
quint16 m_ipVersion;
quint16 m_recordSize;
quint32 m_nodeCount;
int m_nodeSize;
int m_indexSize;
int m_recordBytes;
QDateTime m_buildEpoch;
// Search data
mutable QHash<quint32, QString> m_countries;
quint32 m_size;
const uchar *m_data;
};
#endif // GEOIPDATABASE_H

Write Preview
Loading…
Cancel
Save