Browse Source

Don't save the GeoIP nodes beforehand.

It saves a lot of memory and isn't much faster.
adaptive-webui-19844
sledgehammer999 10 years ago
parent
commit
ef5b3b90c3
  1. 388
      src/core/net/private/geoipdatabase.cpp
  2. 48
      src/core/net/private/geoipdatabase.h

388
src/core/net/private/geoipdatabase.cpp

@ -27,12 +27,8 @@
*/ */
#include <QDebug> #include <QDebug>
#include <QCoreApplication>
#include <QVariant> #include <QVariant>
#include <QHash> #include <QHash>
#include <QList>
#include <QScopedArrayPointer>
#include <QScopedPointer>
#include <QHostAddress> #include <QHostAddress>
#include <QDateTime> #include <QDateTime>
#include <QFile> #include <QFile>
@ -40,28 +36,15 @@
#include "core/types.h" #include "core/types.h"
#include "geoipdatabase.h" #include "geoipdatabase.h"
struct Node
{
quint32 left;
quint32 right;
};
struct GeoIPData
{
// Metadata
quint16 ipVersion;
quint16 recordSize;
quint32 nodeCount;
QDateTime buildEpoch;
// Search data
QList<Node> index;
QHash<quint32, QString> countries;
};
namespace namespace
{ {
const quint32 __ENDIAN_TEST__ = 0x00000001; const quint32 __ENDIAN_TEST__ = 0x00000001;
const bool __IS_LITTLE_ENDIAN__ = (reinterpret_cast<const uchar *>(&__ENDIAN_TEST__)[0] == 0x01); const bool __IS_LITTLE_ENDIAN__ = (reinterpret_cast<const uchar *>(&__ENDIAN_TEST__)[0] == 0x01);
const int MAX_FILE_SIZE = 10485760; // 10MB
const char DB_TYPE[] = "GeoLite2-Country";
const quint32 MAX_METADATA_SIZE = 131072; // 128KB
const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
const char DATA_SECTION_SEPARATOR[16] = { 0 };
enum class DataType enum class DataType
{ {
@ -83,89 +66,60 @@ namespace
Float = 15 Float = 15
}; };
struct DataFieldDescriptor #if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
{ Q_IPV6ADDR createMappedAddress(quint32 ip4);
#endif
}
struct DataFieldDescriptor
{
DataType fieldType; DataType fieldType;
union union
{ {
quint32 fieldSize; quint32 fieldSize;
quint32 offset; // Pointer quint32 offset; // Pointer
}; };
}; };
const int MAX_FILE_SIZE = 10485760; // 10MB
const char DB_TYPE[] = "GeoLite2-Country";
const quint32 MAX_METADATA_SIZE = 131072; // 128KB
const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
const char DATA_SECTION_SEPARATOR[16] = { 0 };
#if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
Q_IPV6ADDR createMappedAddress(quint32 ip4);
#endif
class Loader GeoIPDatabase::GeoIPDatabase(quint32 size)
{ : m_ipVersion(0)
Q_DECLARE_TR_FUNCTIONS(GeoIPDatabase) , m_recordSize(0)
, m_nodeCount(0)
public: , m_nodeSize(0)
GeoIPData *load(const QString &filename); , m_indexSize(0)
GeoIPData *load(const QByteArray &data); , m_recordBytes(0)
QString error() const; , m_size(size)
, m_data(new uchar[size])
private: {
bool parseMetadata(const QVariantHash &metadata); }
bool loadDB();
QVariantHash readMetadata();
QVariant readDataField(quint32 &offset);
bool readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out);
void fromBigEndian(uchar *buf, quint32 len);
QVariant readMapValue(quint32 &offset, quint32 count);
QVariant readArrayValue(quint32 &offset, quint32 count);
template<typename T>
QVariant readPlainValue(quint32 &offset, quint8 len)
{
T value = 0;
const uchar *const data = m_data + offset;
const quint32 availSize = m_size - offset;
if ((len > 0) && (len <= sizeof(T) && (availSize >= len))) { GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
// copy input data to last 'len' bytes of 'value' {
uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len); GeoIPDatabase *db = 0;
memcpy(dst, data, len); QFile file(filename);
fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T)); if (file.size() > MAX_FILE_SIZE) {
offset += len; error = tr("Unsupported database file size.");
return 0;
} }
return QVariant::fromValue(value); if (!file.open(QFile::ReadOnly)) {
error = file.errorString();
return 0;
} }
private: db = new GeoIPDatabase(file.size());
const uchar *m_data;
quint32 m_size;
QString m_error;
GeoIPData *m_geoIPData;
};
}
// GeoIPDatabase
GeoIPDatabase::GeoIPDatabase(GeoIPData *geoIPData) if (file.read((char *)db->m_data, db->m_size) != db->m_size) {
: m_geoIPData(geoIPData) error = file.errorString();
{ delete db;
} return 0;
}
GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
{
GeoIPDatabase *db = 0;
Loader loader; if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error)) {
GeoIPData *geoIPData = loader.load(filename); delete db;
if (!geoIPData) return 0;
error = loader.error(); }
else
db = new GeoIPDatabase(geoIPData);
return db; return db;
} }
@ -173,20 +127,26 @@ GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error) GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error)
{ {
GeoIPDatabase *db = 0; GeoIPDatabase *db = 0;
if (data.size() > MAX_FILE_SIZE) {
error = tr("Unsupported database file size.");
return 0;
}
Loader loader; db = new GeoIPDatabase(data.size());
GeoIPData *geoIPData = loader.load(data);
if (!geoIPData) memcpy((char *)db->m_data, data.constData(), db->m_size);
error = loader.error();
else if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error)) {
db = new GeoIPDatabase(geoIPData); delete db;
return 0;
}
return db; return db;
} }
GeoIPDatabase::~GeoIPDatabase() GeoIPDatabase::~GeoIPDatabase()
{ {
delete m_geoIPData; delete [] m_data;
} }
QString GeoIPDatabase::type() const QString GeoIPDatabase::type() const
@ -196,12 +156,12 @@ QString GeoIPDatabase::type() const
quint16 GeoIPDatabase::ipVersion() const quint16 GeoIPDatabase::ipVersion() const
{ {
return m_geoIPData->ipVersion; return m_ipVersion;
} }
QDateTime GeoIPDatabase::buildEpoch() const QDateTime GeoIPDatabase::buildEpoch() const
{ {
return m_geoIPData->buildEpoch; return m_buildEpoch;
} }
QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
@ -213,100 +173,66 @@ QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
#else #else
Q_IPV6ADDR addr = hostAddr.toIPv6Address(); Q_IPV6ADDR addr = hostAddr.toIPv6Address();
#endif #endif
const quint32 nodeCount = static_cast<quint32>(m_geoIPData->index.size());
Node node = m_geoIPData->index[0]; const uchar *ptr = m_data;
for (int i = 0; i < 16; ++i) { for (int i = 0; i < 16; ++i) {
for (int j = 0; j < 8; ++j) { for (int j = 0; j < 8; ++j) {
bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1); bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1);
quint32 id = (right ? node.right : node.left); // Interpret the left/right record as number
if (id == nodeCount) if (right)
return QString(); ptr += m_recordBytes;
else if (id > nodeCount)
return m_geoIPData->countries[id];
else
node = m_geoIPData->index[id];
}
}
return QString(); quint32 id = 0;
} uchar *idPtr = reinterpret_cast<uchar *>(&id);
memcpy(&idPtr[4 - m_recordBytes], ptr, m_recordBytes);
namespace fromBigEndian(idPtr, 4);
{
// Loader
GeoIPData *Loader::load(const QString &filename) if (id == m_nodeCount) {
{ return QString();
QFile file(filename);
if (file.size() > MAX_FILE_SIZE) {
m_error = tr("Unsupported database file size.");
return 0;
} }
else if (id > m_nodeCount) {
if (!file.open(QFile::ReadOnly)) { QString country = m_countries.value(id);
m_error = file.errorString(); if (country.isEmpty()) {
return 0; const quint32 offset = id - m_nodeCount - sizeof(DATA_SECTION_SEPARATOR);
quint32 tmp = offset + m_indexSize + sizeof(DATA_SECTION_SEPARATOR);
QVariant val = readDataField(tmp);
if (val.userType() == QMetaType::QVariantHash) {
country = val.toHash()["country"].toHash()["iso_code"].toString();
m_countries[id] = country;
} }
m_size = file.size();
QScopedArrayPointer<uchar> data(new uchar[m_size]);
m_data = data.data();
if (file.read((char *)m_data, m_size) != m_size) {
m_error = file.errorString();
return 0;
} }
return country;
QScopedPointer<GeoIPData> geoIPData(new GeoIPData);
m_geoIPData = geoIPData.data();
if (!parseMetadata(readMetadata()) || !loadDB())
return 0;
return geoIPData.take();
} }
else {
GeoIPData *Loader::load(const QByteArray &data) ptr = m_data + (id * m_nodeSize);
{
if (data.size() > MAX_FILE_SIZE) {
m_error = tr("Unsupported database file size.");
return 0;
} }
m_size = data.size();
m_data = reinterpret_cast<const uchar *>(data.constData());
QScopedPointer<GeoIPData> geoIPData(new GeoIPData);
m_geoIPData = geoIPData.data();
if (!parseMetadata(readMetadata()) || !loadDB())
return 0;
return geoIPData.take();
} }
QString Loader::error() const
{
return m_error;
} }
return QString();
}
#define CHECK_METADATA_REQ(key, type) \ #define CHECK_METADATA_REQ(key, type) \
if (!metadata.contains(#key)) { \ if (!metadata.contains(#key)) { \
m_error = errMsgNotFound.arg(#key); \ error = errMsgNotFound.arg(#key); \
return false; \ return false; \
} \ } \
else if (metadata.value(#key).userType() != QMetaType::type) { \ else if (metadata.value(#key).userType() != QMetaType::type) { \
m_error = errMsgInvalid.arg(#key); \ error = errMsgInvalid.arg(#key); \
return false; \ return false; \
} }
#define CHECK_METADATA_OPT(key, type) \ #define CHECK_METADATA_OPT(key, type) \
if (metadata.contains(#key)) { \ if (metadata.contains(#key)) { \
if (metadata.value(#key).userType() != QMetaType::type) { \ if (metadata.value(#key).userType() != QMetaType::type) { \
m_error = errMsgInvalid.arg(#key); \ error = errMsgInvalid.arg(#key); \
return false; \ return false; \
} \ } \
} }
bool Loader::parseMetadata(const QVariantHash &metadata) bool GeoIPDatabase::parseMetadata(const QVariantHash &metadata, QString &error)
{ {
const QString errMsgNotFound = tr("Metadata error: '%1' entry not found."); const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type."); const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
@ -317,102 +243,63 @@ namespace
uint versionMajor = metadata.value("binary_format_major_version").toUInt(); uint versionMajor = metadata.value("binary_format_major_version").toUInt();
uint versionMinor = metadata.value("binary_format_minor_version").toUInt(); uint versionMinor = metadata.value("binary_format_minor_version").toUInt();
if (versionMajor != 2) { if (versionMajor != 2) {
m_error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor); error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
return false; return false;
} }
CHECK_METADATA_REQ(ip_version, UShort); CHECK_METADATA_REQ(ip_version, UShort);
m_geoIPData->ipVersion = metadata.value("ip_version").value<quint16>(); m_ipVersion = metadata.value("ip_version").value<quint16>();
if (m_geoIPData->ipVersion != 6) { if (m_ipVersion != 6) {
m_error = tr("Unsupported IP version: %1").arg(m_geoIPData->ipVersion); error = tr("Unsupported IP version: %1").arg(m_ipVersion);
return false; return false;
} }
CHECK_METADATA_REQ(record_size, UShort); CHECK_METADATA_REQ(record_size, UShort);
m_geoIPData->recordSize = metadata.value("record_size").value<quint16>(); m_recordSize = metadata.value("record_size").value<quint16>();
if (m_geoIPData->recordSize != 24) { if (m_recordSize != 24) {
m_error = tr("Unsupported record size: %1").arg(m_geoIPData->recordSize); error = tr("Unsupported record size: %1").arg(m_recordSize);
return false; return false;
} }
m_nodeSize = m_recordSize / 4;
m_recordBytes = m_nodeSize / 2;
CHECK_METADATA_REQ(node_count, UInt); CHECK_METADATA_REQ(node_count, UInt);
m_geoIPData->nodeCount = metadata.value("node_count").value<quint32>(); m_nodeCount = metadata.value("node_count").value<quint32>();
m_indexSize = m_nodeCount * m_nodeSize;
CHECK_METADATA_REQ(database_type, QString); CHECK_METADATA_REQ(database_type, QString);
QString dbType = metadata.value("database_type").toString(); QString dbType = metadata.value("database_type").toString();
if (dbType != DB_TYPE) { if (dbType != DB_TYPE) {
m_error = tr("Invalid database type: %1").arg(dbType); error = tr("Invalid database type: %1").arg(dbType);
return false; return false;
} }
CHECK_METADATA_REQ(build_epoch, ULongLong); CHECK_METADATA_REQ(build_epoch, ULongLong);
m_geoIPData->buildEpoch = QDateTime::fromTime_t(metadata.value("build_epoch").toULongLong()); m_buildEpoch = QDateTime::fromTime_t(metadata.value("build_epoch").toULongLong());
CHECK_METADATA_OPT(languages, QVariantList); CHECK_METADATA_OPT(languages, QVariantList);
CHECK_METADATA_OPT(description, QVariantHash); CHECK_METADATA_OPT(description, QVariantHash);
return true; return true;
} }
bool Loader::loadDB() bool GeoIPDatabase::loadDB(QString &error) const
{ {
qDebug() << "Parsing MaxMindDB index tree..."; qDebug() << "Parsing MaxMindDB index tree...";
const int nodeSize = m_geoIPData->recordSize / 4; // in bytes const int nodeSize = m_recordSize / 4; // in bytes
const int indexSize = m_geoIPData->nodeCount * nodeSize; const int indexSize = m_nodeCount * nodeSize;
if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR))) if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
|| (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0)) { || (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0)) {
m_error = tr("Database corrupted: no data section found."); error = tr("Database corrupted: no data section found.");
return false; return false;
} }
m_geoIPData->index.reserve(m_geoIPData->nodeCount);
const int recordBytes = nodeSize / 2;
const uchar *ptr = m_data;
bool left = true;
Node node;
for (quint32 i = 0; i < (2 * m_geoIPData->nodeCount); ++i) {
quint32 id = 0;
uchar *idPtr = reinterpret_cast<uchar *>(&id);
memcpy(&idPtr[4 - recordBytes], ptr, recordBytes);
fromBigEndian(idPtr, 4);
if ((id > m_geoIPData->nodeCount) && !m_geoIPData->countries.contains(id)) {
const quint32 offset = id - m_geoIPData->nodeCount - sizeof(DATA_SECTION_SEPARATOR);
quint32 tmp = offset + indexSize + sizeof(DATA_SECTION_SEPARATOR);
QVariant val = readDataField(tmp);
if (val.userType() == QMetaType::QVariantHash) {
m_geoIPData->countries[id] = val.toHash()["country"].toHash()["iso_code"].toString();
}
else if (val.userType() == QVariant::Invalid) {
m_error = tr("Database corrupted: invalid data type at DATA@%1").arg(offset, 8, 16, QLatin1Char('0'));
return false;
}
else {
m_error = tr("Invalid database: unsupported data type at DATA@%1").arg(offset, 8, 16, QLatin1Char('0'));
return false;
}
}
if (left) {
node.left = id;
}
else {
node.right = id;
m_geoIPData->index << node;
}
left = !left;
ptr += recordBytes;
}
return true; return true;
} }
QVariantHash Loader::readMetadata() QVariantHash GeoIPDatabase::readMetadata() const
{ {
const char *ptr = reinterpret_cast<const char *>(m_data); const char *ptr = reinterpret_cast<const char *>(m_data);
quint32 size = m_size; quint32 size = m_size;
if (m_size > MAX_METADATA_SIZE) { if (m_size > MAX_METADATA_SIZE) {
@ -427,16 +314,15 @@ namespace
index += (m_size - MAX_METADATA_SIZE); // from begin of all data index += (m_size - MAX_METADATA_SIZE); // from begin of all data
quint32 offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK)); quint32 offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
QVariant metadata = readDataField(offset); QVariant metadata = readDataField(offset);
m_size = index; // truncate m_size to not contain metadata section
if (metadata.userType() == QMetaType::QVariantHash) if (metadata.userType() == QMetaType::QVariantHash)
return metadata.toHash(); return metadata.toHash();
} }
return QVariantHash(); return QVariantHash();
} }
QVariant Loader::readDataField(quint32 &offset) QVariant GeoIPDatabase::readDataField(quint32 &offset) const
{ {
DataFieldDescriptor descr; DataFieldDescriptor descr;
if (!readDataFieldDescriptor(offset, descr)) if (!readDataFieldDescriptor(offset, descr))
return QVariant(); return QVariant();
@ -446,7 +332,7 @@ namespace
if (descr.fieldType == DataType::Pointer) { if (descr.fieldType == DataType::Pointer) {
usePointer = true; usePointer = true;
// convert offset from data section to global // convert offset from data section to global
locOffset = descr.offset + (m_geoIPData->nodeCount * m_geoIPData->recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR); locOffset = descr.offset + (m_nodeCount * m_recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
if (!readDataFieldDescriptor(locOffset, descr)) if (!readDataFieldDescriptor(locOffset, descr))
return QVariant(); return QVariant();
} }
@ -511,10 +397,10 @@ namespace
if (!usePointer) if (!usePointer)
offset = locOffset; offset = locOffset;
return fieldValue; return fieldValue;
} }
bool Loader::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) bool GeoIPDatabase::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const
{ {
const uchar *dataPtr = m_data + offset; const uchar *dataPtr = m_data + offset;
int availSize = m_size - offset; int availSize = m_size - offset;
if (availSize < 1) return false; if (availSize < 1) return false;
@ -566,16 +452,16 @@ namespace
} }
return true; return true;
} }
void Loader::fromBigEndian(uchar *buf, quint32 len) void GeoIPDatabase::fromBigEndian(uchar *buf, quint32 len) const
{ {
if (__IS_LITTLE_ENDIAN__) if (__IS_LITTLE_ENDIAN__)
std::reverse(buf, buf + len); std::reverse(buf, buf + len);
} }
QVariant Loader::readMapValue(quint32 &offset, quint32 count) QVariant GeoIPDatabase::readMapValue(quint32 &offset, quint32 count) const
{ {
QVariantHash map; QVariantHash map;
for (quint32 i = 0; i < count; ++i) { for (quint32 i = 0; i < count; ++i) {
@ -592,10 +478,10 @@ namespace
} }
return map; return map;
} }
QVariant Loader::readArrayValue(quint32 &offset, quint32 count) QVariant GeoIPDatabase::readArrayValue(quint32 &offset, quint32 count) const
{ {
QVariantList array; QVariantList array;
for (quint32 i = 0; i < count; ++i) { for (quint32 i = 0; i < count; ++i) {
@ -607,8 +493,10 @@ namespace
} }
return array; return array;
} }
namespace
{
#if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0)) #if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
Q_IPV6ADDR createMappedAddress(quint32 ip4) Q_IPV6ADDR createMappedAddress(quint32 ip4)
{ {

48
src/core/net/private/geoipdatabase.h

@ -30,16 +30,19 @@
#define GEOIPDATABASE_H #define GEOIPDATABASE_H
#include <QtGlobal> #include <QtGlobal>
#include <QCoreApplication>
class QHostAddress; class QHostAddress;
class QString; class QString;
class QByteArray; class QByteArray;
class QDateTime; class QDateTime;
struct GeoIPData; struct DataFieldDescriptor;
class GeoIPDatabase class GeoIPDatabase
{ {
Q_DECLARE_TR_FUNCTIONS(GeoIPDatabase)
public: public:
static GeoIPDatabase *load(const QString &filename, QString &error); static GeoIPDatabase *load(const QString &filename, QString &error);
static GeoIPDatabase *load(const QByteArray &data, QString &error); static GeoIPDatabase *load(const QByteArray &data, QString &error);
@ -52,9 +55,48 @@ public:
QString lookup(const QHostAddress &hostAddr) const; QString lookup(const QHostAddress &hostAddr) const;
private: private:
GeoIPDatabase(GeoIPData *geoIPData); GeoIPDatabase(quint32 size);
bool parseMetadata(const QVariantHash &metadata, QString &error);
bool loadDB(QString &error) const;
QVariantHash readMetadata() const;
QVariant readDataField(quint32 &offset) const;
bool readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const;
void fromBigEndian(uchar *buf, quint32 len) const;
QVariant readMapValue(quint32 &offset, quint32 count) const;
QVariant readArrayValue(quint32 &offset, quint32 count) const;
template<typename T>
QVariant readPlainValue(quint32 &offset, quint8 len) const
{
T value = 0;
const uchar *const data = m_data + offset;
const quint32 availSize = m_size - offset;
if ((len > 0) && (len <= sizeof(T) && (availSize >= len))) {
// copy input data to last 'len' bytes of 'value'
uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len);
memcpy(dst, data, len);
fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T));
offset += len;
}
return QVariant::fromValue(value);
}
GeoIPData *m_geoIPData; // Metadata
quint16 m_ipVersion;
quint16 m_recordSize;
quint32 m_nodeCount;
int m_nodeSize;
int m_indexSize;
int m_recordBytes;
QDateTime m_buildEpoch;
// Search data
mutable QHash<quint32, QString> m_countries;
quint32 m_size;
const uchar *m_data;
}; };
#endif // GEOIPDATABASE_H #endif // GEOIPDATABASE_H

Loading…
Cancel
Save