d4708
/
qBittorrent
mirror of https://github.com/d47081/qBittorrent.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Don't save the GeoIP nodes beforehand. 

It saves a lot of memory and isn't much faster.
adaptive-webui-19844
sledgehammer999 10 years ago
parent
787b824d90
commit
ef5b3b90c3
2 changed files with 377 additions and 447 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 776
      src/core/net/private/geoipdatabase.cpp
  2. 48
      src/core/net/private/geoipdatabase.h

776
src/core/net/private/geoipdatabase.cpp
Unescape View File

@ -27,12 +27,8 @@
*/ */
#include <QDebug> #include <QDebug>
#include <QCoreApplication>
#include <QVariant> #include <QVariant>
#include <QHash> #include <QHash>
#include <QList>
#include <QScopedArrayPointer>
#include <QScopedPointer>
#include <QHostAddress> #include <QHostAddress>
#include <QDateTime> #include <QDateTime>
#include <QFile> #include <QFile>
@ -40,28 +36,15 @@
#include "core/types.h" #include "core/types.h"
#include "geoipdatabase.h" #include "geoipdatabase.h"
struct Node
{
quint32 left;
quint32 right;
};
struct GeoIPData
{
// Metadata
quint16 ipVersion;
quint16 recordSize;
quint32 nodeCount;
QDateTime buildEpoch;
// Search data
QList<Node> index;
QHash<quint32, QString> countries;
};
namespace namespace
{ {
const quint32 __ENDIAN_TEST__ = 0x00000001; const quint32 __ENDIAN_TEST__ = 0x00000001;
const bool __IS_LITTLE_ENDIAN__ = (reinterpret_cast<const uchar *>(&__ENDIAN_TEST__)[0] == 0x01); const bool __IS_LITTLE_ENDIAN__ = (reinterpret_cast<const uchar *>(&__ENDIAN_TEST__)[0] == 0x01);
const int MAX_FILE_SIZE = 10485760; // 10MB
const char DB_TYPE[] = "GeoLite2-Country";
const quint32 MAX_METADATA_SIZE = 131072; // 128KB
const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
const char DATA_SECTION_SEPARATOR[16] = { 0 };
enum class DataType enum class DataType
{ {
@ -83,89 +66,60 @@ namespace
Float = 15 Float = 15
}; };
struct DataFieldDescriptor
{
DataType fieldType;
union
{
quint32 fieldSize;
quint32 offset; // Pointer
};
};
const int MAX_FILE_SIZE = 10485760; // 10MB
const char DB_TYPE[] = "GeoLite2-Country";
const quint32 MAX_METADATA_SIZE = 131072; // 128KB
const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
const char DATA_SECTION_SEPARATOR[16] = { 0 };
#if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0)) #if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
Q_IPV6ADDR createMappedAddress(quint32 ip4); Q_IPV6ADDR createMappedAddress(quint32 ip4);
#endif #endif
}
class Loader struct DataFieldDescriptor
{
DataType fieldType;
union
{ {
Q_DECLARE_TR_FUNCTIONS(GeoIPDatabase) quint32 fieldSize;
quint32 offset; // Pointer
public:
GeoIPData *load(const QString &filename);
GeoIPData *load(const QByteArray &data);
QString error() const;
private:
bool parseMetadata(const QVariantHash &metadata);
bool loadDB();
QVariantHash readMetadata();
QVariant readDataField(quint32 &offset);
bool readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out);
void fromBigEndian(uchar *buf, quint32 len);
QVariant readMapValue(quint32 &offset, quint32 count);
QVariant readArrayValue(quint32 &offset, quint32 count);
template<typename T>
QVariant readPlainValue(quint32 &offset, quint8 len)
{
T value = 0;
const uchar *const data = m_data + offset;
const quint32 availSize = m_size - offset;
if ((len > 0) && (len <= sizeof(T) && (availSize >= len))) {
// copy input data to last 'len' bytes of 'value'
uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len);
memcpy(dst, data, len);
fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T));
offset += len;
}
return QVariant::fromValue(value);
}
private:
const uchar *m_data;
quint32 m_size;
QString m_error;
GeoIPData *m_geoIPData;
}; };
} };
// GeoIPDatabase
GeoIPDatabase::GeoIPDatabase(GeoIPData *geoIPData) GeoIPDatabase::GeoIPDatabase(quint32 size)
: m_geoIPData(geoIPData) : m_ipVersion(0)
, m_recordSize(0)
, m_nodeCount(0)
, m_nodeSize(0)
, m_indexSize(0)
, m_recordBytes(0)
, m_size(size)
, m_data(new uchar[size])
{ {
} }
GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error) GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
{ {
GeoIPDatabase *db = 0; GeoIPDatabase *db = 0;
QFile file(filename);
if (file.size() > MAX_FILE_SIZE) {
error = tr("Unsupported database file size.");
return 0;
}
Loader loader; if (!file.open(QFile::ReadOnly)) {
GeoIPData *geoIPData = loader.load(filename); error = file.errorString();
if (!geoIPData) return 0;
error = loader.error(); }
else
db = new GeoIPDatabase(geoIPData); db = new GeoIPDatabase(file.size());
if (file.read((char *)db->m_data, db->m_size) != db->m_size) {
error = file.errorString();
delete db;
return 0;
}
if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error)) {
delete db;
return 0;
}
return db; return db;
} }
@ -173,20 +127,26 @@ GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error) GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error)
{ {
GeoIPDatabase *db = 0; GeoIPDatabase *db = 0;
if (data.size() > MAX_FILE_SIZE) {
error = tr("Unsupported database file size.");
return 0;
}
db = new GeoIPDatabase(data.size());
Loader loader; memcpy((char *)db->m_data, data.constData(), db->m_size);
GeoIPData *geoIPData = loader.load(data);
if (!geoIPData) if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error)) {
error = loader.error(); delete db;
else return 0;
db = new GeoIPDatabase(geoIPData); }
return db; return db;
} }
GeoIPDatabase::~GeoIPDatabase() GeoIPDatabase::~GeoIPDatabase()
{ {
delete m_geoIPData; delete [] m_data;
} }
QString GeoIPDatabase::type() const QString GeoIPDatabase::type() const
@ -196,12 +156,12 @@ QString GeoIPDatabase::type() const
quint16 GeoIPDatabase::ipVersion() const quint16 GeoIPDatabase::ipVersion() const
{ {
return m_geoIPData->ipVersion; return m_ipVersion;
} }
QDateTime GeoIPDatabase::buildEpoch() const QDateTime GeoIPDatabase::buildEpoch() const
{ {
return m_geoIPData->buildEpoch; return m_buildEpoch;
} }
QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
@ -213,402 +173,330 @@ QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
#else #else
Q_IPV6ADDR addr = hostAddr.toIPv6Address(); Q_IPV6ADDR addr = hostAddr.toIPv6Address();
#endif #endif
const quint32 nodeCount = static_cast<quint32>(m_geoIPData->index.size());
Node node = m_geoIPData->index[0]; const uchar *ptr = m_data;
for (int i = 0; i < 16; ++i) { for (int i = 0; i < 16; ++i) {
for (int j = 0; j < 8; ++j) { for (int j = 0; j < 8; ++j) {
bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1); bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1);
quint32 id = (right ? node.right : node.left); // Interpret the left/right record as number
if (id == nodeCount) if (right)
ptr += m_recordBytes;
quint32 id = 0;
uchar *idPtr = reinterpret_cast<uchar *>(&id);
memcpy(&idPtr[4 - m_recordBytes], ptr, m_recordBytes);
fromBigEndian(idPtr, 4);
if (id == m_nodeCount) {
return QString(); return QString();
else if (id > nodeCount) }
return m_geoIPData->countries[id]; else if (id > m_nodeCount) {
else QString country = m_countries.value(id);
node = m_geoIPData->index[id]; if (country.isEmpty()) {
const quint32 offset = id - m_nodeCount - sizeof(DATA_SECTION_SEPARATOR);
quint32 tmp = offset + m_indexSize + sizeof(DATA_SECTION_SEPARATOR);
QVariant val = readDataField(tmp);
if (val.userType() == QMetaType::QVariantHash) {
country = val.toHash()["country"].toHash()["iso_code"].toString();
m_countries[id] = country;
}
}
return country;
}
else {
ptr = m_data + (id * m_nodeSize);
}
} }
} }
return QString(); return QString();
} }
namespace #define CHECK_METADATA_REQ(key, type) \
{ if (!metadata.contains(#key)) { \
// Loader error = errMsgNotFound.arg(#key); \
return false; \
GeoIPData *Loader::load(const QString &filename) } \
{ else if (metadata.value(#key).userType() != QMetaType::type) { \
QFile file(filename); error = errMsgInvalid.arg(#key); \
if (file.size() > MAX_FILE_SIZE) { return false; \
m_error = tr("Unsupported database file size."); }
return 0;
}
if (!file.open(QFile::ReadOnly)) {
m_error = file.errorString();
return 0;
}
m_size = file.size();
QScopedArrayPointer<uchar> data(new uchar[m_size]);
m_data = data.data();
if (file.read((char *)m_data, m_size) != m_size) {
m_error = file.errorString();
return 0;
}
QScopedPointer<GeoIPData> geoIPData(new GeoIPData);
m_geoIPData = geoIPData.data();
if (!parseMetadata(readMetadata()) || !loadDB())
return 0;
return geoIPData.take();
}
GeoIPData *Loader::load(const QByteArray &data)
{
if (data.size() > MAX_FILE_SIZE) {
m_error = tr("Unsupported database file size.");
return 0;
}
m_size = data.size();
m_data = reinterpret_cast<const uchar *>(data.constData());
QScopedPointer<GeoIPData> geoIPData(new GeoIPData); #define CHECK_METADATA_OPT(key, type) \
m_geoIPData = geoIPData.data(); if (metadata.contains(#key)) { \
if (!parseMetadata(readMetadata()) || !loadDB()) if (metadata.value(#key).userType() != QMetaType::type) { \
return 0; error = errMsgInvalid.arg(#key); \
return false; \
} \
}
return geoIPData.take(); bool GeoIPDatabase::parseMetadata(const QVariantHash &metadata, QString &error)
{
const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
qDebug() << "Parsing MaxMindDB metadata...";
CHECK_METADATA_REQ(binary_format_major_version, UShort);
CHECK_METADATA_REQ(binary_format_minor_version, UShort);
uint versionMajor = metadata.value("binary_format_major_version").toUInt();
uint versionMinor = metadata.value("binary_format_minor_version").toUInt();
if (versionMajor != 2) {
error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
return false;
} }
QString Loader::error() const CHECK_METADATA_REQ(ip_version, UShort);
{ m_ipVersion = metadata.value("ip_version").value<quint16>();
return m_error; if (m_ipVersion != 6) {
error = tr("Unsupported IP version: %1").arg(m_ipVersion);
return false;
} }
#define CHECK_METADATA_REQ(key, type) \ CHECK_METADATA_REQ(record_size, UShort);
if (!metadata.contains(#key)) { \ m_recordSize = metadata.value("record_size").value<quint16>();
m_error = errMsgNotFound.arg(#key); \ if (m_recordSize != 24) {
return false; \ error = tr("Unsupported record size: %1").arg(m_recordSize);
} \ return false;
else if (metadata.value(#key).userType() != QMetaType::type) { \
m_error = errMsgInvalid.arg(#key); \
return false; \
} }
m_nodeSize = m_recordSize / 4;
#define CHECK_METADATA_OPT(key, type) \ m_recordBytes = m_nodeSize / 2;
if (metadata.contains(#key)) { \
if (metadata.value(#key).userType() != QMetaType::type) { \ CHECK_METADATA_REQ(node_count, UInt);
m_error = errMsgInvalid.arg(#key); \ m_nodeCount = metadata.value("node_count").value<quint32>();
return false; \ m_indexSize = m_nodeCount * m_nodeSize;
} \
CHECK_METADATA_REQ(database_type, QString);
QString dbType = metadata.value("database_type").toString();
if (dbType != DB_TYPE) {
error = tr("Invalid database type: %1").arg(dbType);
return false;
} }
bool Loader::parseMetadata(const QVariantHash &metadata) CHECK_METADATA_REQ(build_epoch, ULongLong);
{ m_buildEpoch = QDateTime::fromTime_t(metadata.value("build_epoch").toULongLong());
const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
qDebug() << "Parsing MaxMindDB metadata...";
CHECK_METADATA_REQ(binary_format_major_version, UShort);
CHECK_METADATA_REQ(binary_format_minor_version, UShort);
uint versionMajor = metadata.value("binary_format_major_version").toUInt();
uint versionMinor = metadata.value("binary_format_minor_version").toUInt();
if (versionMajor != 2) {
m_error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
return false;
}
CHECK_METADATA_REQ(ip_version, UShort);
m_geoIPData->ipVersion = metadata.value("ip_version").value<quint16>();
if (m_geoIPData->ipVersion != 6) {
m_error = tr("Unsupported IP version: %1").arg(m_geoIPData->ipVersion);
return false;
}
CHECK_METADATA_REQ(record_size, UShort); CHECK_METADATA_OPT(languages, QVariantList);
m_geoIPData->recordSize = metadata.value("record_size").value<quint16>(); CHECK_METADATA_OPT(description, QVariantHash);
if (m_geoIPData->recordSize != 24) {
m_error = tr("Unsupported record size: %1").arg(m_geoIPData->recordSize);
return false;
}
CHECK_METADATA_REQ(node_count, UInt); return true;
m_geoIPData->nodeCount = metadata.value("node_count").value<quint32>(); }
CHECK_METADATA_REQ(database_type, QString);
QString dbType = metadata.value("database_type").toString();
if (dbType != DB_TYPE) {
m_error = tr("Invalid database type: %1").arg(dbType);
return false;
}
CHECK_METADATA_REQ(build_epoch, ULongLong);
m_geoIPData->buildEpoch = QDateTime::fromTime_t(metadata.value("build_epoch").toULongLong());
CHECK_METADATA_OPT(languages, QVariantList);
CHECK_METADATA_OPT(description, QVariantHash);
return true; bool GeoIPDatabase::loadDB(QString &error) const
{
qDebug() << "Parsing MaxMindDB index tree...";
const int nodeSize = m_recordSize / 4; // in bytes
const int indexSize = m_nodeCount * nodeSize;
if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
|| (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0)) {
error = tr("Database corrupted: no data section found.");
return false;
} }
bool Loader::loadDB() return true;
{ }
qDebug() << "Parsing MaxMindDB index tree...";
const int nodeSize = m_geoIPData->recordSize / 4; // in bytes
const int indexSize = m_geoIPData->nodeCount * nodeSize;
if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
|| (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0)) {
m_error = tr("Database corrupted: no data section found.");
return false;
}
m_geoIPData->index.reserve(m_geoIPData->nodeCount);
const int recordBytes = nodeSize / 2;
const uchar *ptr = m_data;
bool left = true;
Node node;
for (quint32 i = 0; i < (2 * m_geoIPData->nodeCount); ++i) {
quint32 id = 0;
uchar *idPtr = reinterpret_cast<uchar *>(&id);
memcpy(&idPtr[4 - recordBytes], ptr, recordBytes);
fromBigEndian(idPtr, 4);
if ((id > m_geoIPData->nodeCount) && !m_geoIPData->countries.contains(id)) {
const quint32 offset = id - m_geoIPData->nodeCount - sizeof(DATA_SECTION_SEPARATOR);
quint32 tmp = offset + indexSize + sizeof(DATA_SECTION_SEPARATOR);
QVariant val = readDataField(tmp);
if (val.userType() == QMetaType::QVariantHash) {
m_geoIPData->countries[id] = val.toHash()["country"].toHash()["iso_code"].toString();
}
else if (val.userType() == QVariant::Invalid) {
m_error = tr("Database corrupted: invalid data type at DATA@%1").arg(offset, 8, 16, QLatin1Char('0'));
return false;
}
else {
m_error = tr("Invalid database: unsupported data type at DATA@%1").arg(offset, 8, 16, QLatin1Char('0'));
return false;
}
}
if (left) {
node.left = id;
}
else {
node.right = id;
m_geoIPData->index << node;
}
left = !left;
ptr += recordBytes;
}
return true; QVariantHash GeoIPDatabase::readMetadata() const
{
const char *ptr = reinterpret_cast<const char *>(m_data);
quint32 size = m_size;
if (m_size > MAX_METADATA_SIZE) {
ptr += m_size - MAX_METADATA_SIZE;
size = MAX_METADATA_SIZE;
} }
QVariantHash Loader::readMetadata() const QByteArray data = QByteArray::fromRawData(ptr, size);
{ int index = data.lastIndexOf(METADATA_BEGIN_MARK);
const char *ptr = reinterpret_cast<const char *>(m_data); if (index >= 0) {
quint32 size = m_size; if (m_size > MAX_METADATA_SIZE)
if (m_size > MAX_METADATA_SIZE) { index += (m_size - MAX_METADATA_SIZE); // from begin of all data
ptr += m_size - MAX_METADATA_SIZE; quint32 offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
size = MAX_METADATA_SIZE; QVariant metadata = readDataField(offset);
} if (metadata.userType() == QMetaType::QVariantHash)
return metadata.toHash();
const QByteArray data = QByteArray::fromRawData(ptr, size);
int index = data.lastIndexOf(METADATA_BEGIN_MARK);
if (index >= 0) {
if (m_size > MAX_METADATA_SIZE)
index += (m_size - MAX_METADATA_SIZE); // from begin of all data
quint32 offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
QVariant metadata = readDataField(offset);
m_size = index; // truncate m_size to not contain metadata section
if (metadata.userType() == QMetaType::QVariantHash)
return metadata.toHash();
}
return QVariantHash();
} }
QVariant Loader::readDataField(quint32 &offset) return QVariantHash();
{ }
DataFieldDescriptor descr;
if (!readDataFieldDescriptor(offset, descr)) QVariant GeoIPDatabase::readDataField(quint32 &offset) const
{
DataFieldDescriptor descr;
if (!readDataFieldDescriptor(offset, descr))
return QVariant();
quint32 locOffset = offset;
bool usePointer = false;
if (descr.fieldType == DataType::Pointer) {
usePointer = true;
// convert offset from data section to global
locOffset = descr.offset + (m_nodeCount * m_recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
if (!readDataFieldDescriptor(locOffset, descr))
return QVariant(); return QVariant();
}
quint32 locOffset = offset; QVariant fieldValue;
bool usePointer = false; switch (descr.fieldType) {
if (descr.fieldType == DataType::Pointer) { case DataType::Pointer:
usePointer = true; qDebug() << "* Illegal Pointer using";
// convert offset from data section to global break;
locOffset = descr.offset + (m_geoIPData->nodeCount * m_geoIPData->recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR); case DataType::String:
if (!readDataFieldDescriptor(locOffset, descr)) fieldValue = QString::fromUtf8(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
return QVariant(); locOffset += descr.fieldSize;
} break;
case DataType::Double:
if (descr.fieldSize == 8)
fieldValue = readPlainValue<double>(locOffset, descr.fieldSize);
else
qDebug() << "* Invalid field size for type: Double";
break;
case DataType::Bytes:
fieldValue = QByteArray(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
locOffset += descr.fieldSize;
break;
case DataType::Integer16:
fieldValue = readPlainValue<quint16>(locOffset, descr.fieldSize);
break;
case DataType::Integer32:
fieldValue = readPlainValue<quint32>(locOffset, descr.fieldSize);
break;
case DataType::Map:
fieldValue = readMapValue(locOffset, descr.fieldSize);
break;
case DataType::SignedInteger32:
fieldValue = readPlainValue<qint32>(locOffset, descr.fieldSize);
break;
case DataType::Integer64:
fieldValue = readPlainValue<quint64>(locOffset, descr.fieldSize);
break;
case DataType::Integer128:
qDebug() << "* Unsupported data type: Integer128";
break;
case DataType::Array:
fieldValue = readArrayValue(locOffset, descr.fieldSize);
break;
case DataType::DataCacheContainer:
qDebug() << "* Unsupported data type: DataCacheContainer";
break;
case DataType::EndMarker:
qDebug() << "* Unsupported data type: EndMarker";
break;
case DataType::Boolean:
fieldValue = QVariant::fromValue(static_cast<bool>(descr.fieldSize));
break;
case DataType::Float:
if (descr.fieldSize == 4)
fieldValue = readPlainValue<float>(locOffset, descr.fieldSize);
else
qDebug() << "* Invalid field size for type: Float";
break;
}
QVariant fieldValue; if (!usePointer)
switch (descr.fieldType) { offset = locOffset;
case DataType::Pointer: return fieldValue;
qDebug() << "* Illegal Pointer using"; }
break;
case DataType::String:
fieldValue = QString::fromUtf8(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
locOffset += descr.fieldSize;
break;
case DataType::Double:
if (descr.fieldSize == 8)
fieldValue = readPlainValue<double>(locOffset, descr.fieldSize);
else
qDebug() << "* Invalid field size for type: Double";
break;
case DataType::Bytes:
fieldValue = QByteArray(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
locOffset += descr.fieldSize;
break;
case DataType::Integer16:
fieldValue = readPlainValue<quint16>(locOffset, descr.fieldSize);
break;
case DataType::Integer32:
fieldValue = readPlainValue<quint32>(locOffset, descr.fieldSize);
break;
case DataType::Map:
fieldValue = readMapValue(locOffset, descr.fieldSize);
break;
case DataType::SignedInteger32:
fieldValue = readPlainValue<qint32>(locOffset, descr.fieldSize);
break;
case DataType::Integer64:
fieldValue = readPlainValue<quint64>(locOffset, descr.fieldSize);
break;
case DataType::Integer128:
qDebug() << "* Unsupported data type: Integer128";
break;
case DataType::Array:
fieldValue = readArrayValue(locOffset, descr.fieldSize);
break;
case DataType::DataCacheContainer:
qDebug() << "* Unsupported data type: DataCacheContainer";
break;
case DataType::EndMarker:
qDebug() << "* Unsupported data type: EndMarker";
break;
case DataType::Boolean:
fieldValue = QVariant::fromValue(static_cast<bool>(descr.fieldSize));
break;
case DataType::Float:
if (descr.fieldSize == 4)
fieldValue = readPlainValue<float>(locOffset, descr.fieldSize);
else
qDebug() << "* Invalid field size for type: Float";
break;
}
if (!usePointer) bool GeoIPDatabase::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const
offset = locOffset; {
return fieldValue; const uchar *dataPtr = m_data + offset;
int availSize = m_size - offset;
if (availSize < 1) return false;
out.fieldType = static_cast<DataType>((dataPtr[0] & 0xE0) >> 5);
if (out.fieldType == DataType::Pointer) {
int size = ((dataPtr[0] & 0x18) >> 3);
if (availSize < (size + 2)) return false;
if (size == 0)
out.offset = ((dataPtr[0] & 0x07) << 8) + dataPtr[1];
else if (size == 1)
out.offset = ((dataPtr[0] & 0x07) << 16) + (dataPtr[1] << 8) + dataPtr[2] + 2048;
else if (size == 2)
out.offset = ((dataPtr[0] & 0x07) << 24) + (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 526336;
else if (size == 3)
out.offset = (dataPtr[1] << 24) + (dataPtr[2] << 16) + (dataPtr[3] << 8) + dataPtr[4];
offset += size + 2;
return true;
} }
bool Loader::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) out.fieldSize = dataPtr[0] & 0x1F;
{ if (out.fieldSize <= 28) {
const uchar *dataPtr = m_data + offset; if (out.fieldType == DataType::Unknown) {
int availSize = m_size - offset; out.fieldType = static_cast<DataType>(dataPtr[1] + 7);
if (availSize < 1) return false; if ((out.fieldType <= DataType::Map) || (out.fieldType > DataType::Float) || (availSize < 3))
return false;
out.fieldType = static_cast<DataType>((dataPtr[0] & 0xE0) >> 5);
if (out.fieldType == DataType::Pointer) {
int size = ((dataPtr[0] & 0x18) >> 3);
if (availSize < (size + 2)) return false;
if (size == 0)
out.offset = ((dataPtr[0] & 0x07) << 8) + dataPtr[1];
else if (size == 1)
out.offset = ((dataPtr[0] & 0x07) << 16) + (dataPtr[1] << 8) + dataPtr[2] + 2048;
else if (size == 2)
out.offset = ((dataPtr[0] & 0x07) << 24) + (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 526336;
else if (size == 3)
out.offset = (dataPtr[1] << 24) + (dataPtr[2] << 16) + (dataPtr[3] << 8) + dataPtr[4];
offset += size + 2;
return true;
}
out.fieldSize = dataPtr[0] & 0x1F;
if (out.fieldSize <= 28) {
if (out.fieldType == DataType::Unknown) {
out.fieldType = static_cast<DataType>(dataPtr[1] + 7);
if ((out.fieldType <= DataType::Map) || (out.fieldType > DataType::Float) || (availSize < 3))
return false;
offset += 2;
}
else {
offset += 1;
}
}
else if (out.fieldSize == 29) {
if (availSize < 2) return false;
out.fieldSize = dataPtr[1] + 29;
offset += 2; offset += 2;
} }
else if (out.fieldSize == 30) { else {
if (availSize < 3) return false; offset += 1;
out.fieldSize = (dataPtr[1] << 8) + dataPtr[2] + 285;
offset += 3;
} }
else if (out.fieldSize == 31) {
if (availSize < 4) return false;
out.fieldSize = (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 65821;
offset += 4;
}
return true;
} }
else if (out.fieldSize == 29) {
void Loader::fromBigEndian(uchar *buf, quint32 len) if (availSize < 2) return false;
{ out.fieldSize = dataPtr[1] + 29;
if (__IS_LITTLE_ENDIAN__) offset += 2;
std::reverse(buf, buf + len); }
else if (out.fieldSize == 30) {
if (availSize < 3) return false;
out.fieldSize = (dataPtr[1] << 8) + dataPtr[2] + 285;
offset += 3;
}
else if (out.fieldSize == 31) {
if (availSize < 4) return false;
out.fieldSize = (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 65821;
offset += 4;
} }
QVariant Loader::readMapValue(quint32 &offset, quint32 count) return true;
{ }
QVariantHash map;
void GeoIPDatabase::fromBigEndian(uchar *buf, quint32 len) const
{
if (__IS_LITTLE_ENDIAN__)
std::reverse(buf, buf + len);
}
for (quint32 i = 0; i < count; ++i) { QVariant GeoIPDatabase::readMapValue(quint32 &offset, quint32 count) const
QVariant field = readDataField(offset); {
if (field.userType() != QMetaType::QString) QVariantHash map;
return QVariant();
QString key = field.toString(); for (quint32 i = 0; i < count; ++i) {
field = readDataField(offset); QVariant field = readDataField(offset);
if (field.userType() == QVariant::Invalid) if (field.userType() != QMetaType::QString)
return QVariant(); return QVariant();
map[key] = field; QString key = field.toString();
} field = readDataField(offset);
if (field.userType() == QVariant::Invalid)
return QVariant();
return map; map[key] = field;
} }
QVariant Loader::readArrayValue(quint32 &offset, quint32 count) return map;
{ }
QVariantList array;
for (quint32 i = 0; i < count; ++i) { QVariant GeoIPDatabase::readArrayValue(quint32 &offset, quint32 count) const
QVariant field = readDataField(offset); {
if (field.userType() == QVariant::Invalid) QVariantList array;
return QVariant();
array.append(field); for (quint32 i = 0; i < count; ++i) {
} QVariant field = readDataField(offset);
if (field.userType() == QVariant::Invalid)
return QVariant();
return array; array.append(field);
} }
return array;
}
namespace
{
#if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0)) #if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
Q_IPV6ADDR createMappedAddress(quint32 ip4) Q_IPV6ADDR createMappedAddress(quint32 ip4)
{ {

48
src/core/net/private/geoipdatabase.h
Unescape View File

@ -30,16 +30,19 @@
#define GEOIPDATABASE_H #define GEOIPDATABASE_H
#include <QtGlobal> #include <QtGlobal>
#include <QCoreApplication>
class QHostAddress; class QHostAddress;
class QString; class QString;
class QByteArray; class QByteArray;
class QDateTime; class QDateTime;
struct GeoIPData; struct DataFieldDescriptor;
class GeoIPDatabase class GeoIPDatabase
{ {
Q_DECLARE_TR_FUNCTIONS(GeoIPDatabase)
public: public:
static GeoIPDatabase *load(const QString &filename, QString &error); static GeoIPDatabase *load(const QString &filename, QString &error);
static GeoIPDatabase *load(const QByteArray &data, QString &error); static GeoIPDatabase *load(const QByteArray &data, QString &error);
@ -52,9 +55,48 @@ public:
QString lookup(const QHostAddress &hostAddr) const; QString lookup(const QHostAddress &hostAddr) const;
private: private:
GeoIPDatabase(GeoIPData *geoIPData); GeoIPDatabase(quint32 size);
bool parseMetadata(const QVariantHash &metadata, QString &error);
bool loadDB(QString &error) const;
QVariantHash readMetadata() const;
QVariant readDataField(quint32 &offset) const;
bool readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const;
void fromBigEndian(uchar *buf, quint32 len) const;
QVariant readMapValue(quint32 &offset, quint32 count) const;
QVariant readArrayValue(quint32 &offset, quint32 count) const;
template<typename T>
QVariant readPlainValue(quint32 &offset, quint8 len) const
{
T value = 0;
const uchar *const data = m_data + offset;
const quint32 availSize = m_size - offset;
if ((len > 0) && (len <= sizeof(T) && (availSize >= len))) {
// copy input data to last 'len' bytes of 'value'
uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len);
memcpy(dst, data, len);
fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T));
offset += len;
}
return QVariant::fromValue(value);
}
GeoIPData *m_geoIPData; // Metadata
quint16 m_ipVersion;
quint16 m_recordSize;
quint32 m_nodeCount;
int m_nodeSize;
int m_indexSize;
int m_recordBytes;
QDateTime m_buildEpoch;
// Search data
mutable QHash<quint32, QString> m_countries;
quint32 m_size;
const uchar *m_data;
}; };
#endif // GEOIPDATABASE_H #endif // GEOIPDATABASE_H

Write Preview
Loading…
Cancel
Save