d4708
/
qBittorrent
mirror of https://github.com/d47081/qBittorrent.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Don't save the GeoIP nodes beforehand. 

It saves a lot of memory and isn't much faster.
adaptive-webui-19844
sledgehammer999 10 years ago
parent
787b824d90
commit
ef5b3b90c3
2 changed files with 377 additions and 447 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 776
      src/core/net/private/geoipdatabase.cpp
  2. 48
      src/core/net/private/geoipdatabase.h

776
src/core/net/private/geoipdatabase.cpp
Unescape View File

@ -27,12 +27,8 @@ @@ -27,12 +27,8 @@
*/
#include <QDebug>
#include <QCoreApplication>
#include <QVariant>
#include <QHash>
#include <QList>
#include <QScopedArrayPointer>
#include <QScopedPointer>
#include <QHostAddress>
#include <QDateTime>
#include <QFile>
@ -40,28 +36,15 @@ @@ -40,28 +36,15 @@
#include "core/types.h"
#include "geoipdatabase.h"
struct Node
{
quint32 left;
quint32 right;
};
struct GeoIPData
{
// Metadata
quint16 ipVersion;
quint16 recordSize;
quint32 nodeCount;
QDateTime buildEpoch;
// Search data
QList<Node> index;
QHash<quint32, QString> countries;
};
namespace
{
const quint32 __ENDIAN_TEST__ = 0x00000001;
const bool __IS_LITTLE_ENDIAN__ = (reinterpret_cast<const uchar *>(&__ENDIAN_TEST__)[0] == 0x01);
const int MAX_FILE_SIZE = 10485760; // 10MB
const char DB_TYPE[] = "GeoLite2-Country";
const quint32 MAX_METADATA_SIZE = 131072; // 128KB
const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
const char DATA_SECTION_SEPARATOR[16] = { 0 };
enum class DataType
{
@ -83,89 +66,60 @@ namespace @@ -83,89 +66,60 @@ namespace
Float = 15
};
struct DataFieldDescriptor
{
DataType fieldType;
union
{
quint32 fieldSize;
quint32 offset; // Pointer
};
};
const int MAX_FILE_SIZE = 10485760; // 10MB
const char DB_TYPE[] = "GeoLite2-Country";
const quint32 MAX_METADATA_SIZE = 131072; // 128KB
const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
const char DATA_SECTION_SEPARATOR[16] = { 0 };
#if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
Q_IPV6ADDR createMappedAddress(quint32 ip4);
#endif
}
class Loader
struct DataFieldDescriptor
{
DataType fieldType;
union
{
Q_DECLARE_TR_FUNCTIONS(GeoIPDatabase)
public:
GeoIPData *load(const QString &filename);
GeoIPData *load(const QByteArray &data);
QString error() const;
private:
bool parseMetadata(const QVariantHash &metadata);
bool loadDB();
QVariantHash readMetadata();
QVariant readDataField(quint32 &offset);
bool readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out);
void fromBigEndian(uchar *buf, quint32 len);
QVariant readMapValue(quint32 &offset, quint32 count);
QVariant readArrayValue(quint32 &offset, quint32 count);
template<typename T>
QVariant readPlainValue(quint32 &offset, quint8 len)
{
T value = 0;
const uchar *const data = m_data + offset;
const quint32 availSize = m_size - offset;
if ((len > 0) && (len <= sizeof(T) && (availSize >= len))) {
// copy input data to last 'len' bytes of 'value'
uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len);
memcpy(dst, data, len);
fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T));
offset += len;
}
return QVariant::fromValue(value);
}
private:
const uchar *m_data;
quint32 m_size;
QString m_error;
GeoIPData *m_geoIPData;
quint32 fieldSize;
quint32 offset; // Pointer
};
}
// GeoIPDatabase
};
GeoIPDatabase::GeoIPDatabase(GeoIPData *geoIPData)
: m_geoIPData(geoIPData)
GeoIPDatabase::GeoIPDatabase(quint32 size)
: m_ipVersion(0)
, m_recordSize(0)
, m_nodeCount(0)
, m_nodeSize(0)
, m_indexSize(0)
, m_recordBytes(0)
, m_size(size)
, m_data(new uchar[size])
{
}
GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
{
GeoIPDatabase *db = 0;
QFile file(filename);
if (file.size() > MAX_FILE_SIZE) {
error = tr("Unsupported database file size.");
return 0;
}
Loader loader;
GeoIPData *geoIPData = loader.load(filename);
if (!geoIPData)
error = loader.error();
else
db = new GeoIPDatabase(geoIPData);
if (!file.open(QFile::ReadOnly)) {
error = file.errorString();
return 0;
}
db = new GeoIPDatabase(file.size());
if (file.read((char *)db->m_data, db->m_size) != db->m_size) {
error = file.errorString();
delete db;
return 0;
}
if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error)) {
delete db;
return 0;
}
return db;
}
@ -173,20 +127,26 @@ GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error) @@ -173,20 +127,26 @@ GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error)
{
GeoIPDatabase *db = 0;
if (data.size() > MAX_FILE_SIZE) {
error = tr("Unsupported database file size.");
return 0;
}
db = new GeoIPDatabase(data.size());
Loader loader;
GeoIPData *geoIPData = loader.load(data);
if (!geoIPData)
error = loader.error();
else
db = new GeoIPDatabase(geoIPData);
memcpy((char *)db->m_data, data.constData(), db->m_size);
if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error)) {
delete db;
return 0;
}
return db;
}
GeoIPDatabase::~GeoIPDatabase()
{
delete m_geoIPData;
delete [] m_data;
}
QString GeoIPDatabase::type() const
@ -196,12 +156,12 @@ QString GeoIPDatabase::type() const @@ -196,12 +156,12 @@ QString GeoIPDatabase::type() const
quint16 GeoIPDatabase::ipVersion() const
{
return m_geoIPData->ipVersion;
return m_ipVersion;
}
QDateTime GeoIPDatabase::buildEpoch() const
{
return m_geoIPData->buildEpoch;
return m_buildEpoch;
}
QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
@ -213,402 +173,330 @@ QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const @@ -213,402 +173,330 @@ QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
#else
Q_IPV6ADDR addr = hostAddr.toIPv6Address();
#endif
const quint32 nodeCount = static_cast<quint32>(m_geoIPData->index.size());
Node node = m_geoIPData->index[0];
const uchar *ptr = m_data;
for (int i = 0; i < 16; ++i) {
for (int j = 0; j < 8; ++j) {
bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1);
quint32 id = (right ? node.right : node.left);
if (id == nodeCount)
// Interpret the left/right record as number
if (right)
ptr += m_recordBytes;
quint32 id = 0;
uchar *idPtr = reinterpret_cast<uchar *>(&id);
memcpy(&idPtr[4 - m_recordBytes], ptr, m_recordBytes);
fromBigEndian(idPtr, 4);
if (id == m_nodeCount) {
return QString();
else if (id > nodeCount)
return m_geoIPData->countries[id];
else
node = m_geoIPData->index[id];
}
else if (id > m_nodeCount) {
QString country = m_countries.value(id);
if (country.isEmpty()) {
const quint32 offset = id - m_nodeCount - sizeof(DATA_SECTION_SEPARATOR);
quint32 tmp = offset + m_indexSize + sizeof(DATA_SECTION_SEPARATOR);
QVariant val = readDataField(tmp);
if (val.userType() == QMetaType::QVariantHash) {
country = val.toHash()["country"].toHash()["iso_code"].toString();
m_countries[id] = country;
}
}
return country;
}
else {
ptr = m_data + (id * m_nodeSize);
}
}
}
return QString();
}
namespace
{
// Loader
GeoIPData *Loader::load(const QString &filename)
{
QFile file(filename);
if (file.size() > MAX_FILE_SIZE) {
m_error = tr("Unsupported database file size.");
return 0;
}
if (!file.open(QFile::ReadOnly)) {
m_error = file.errorString();
return 0;
}
m_size = file.size();
QScopedArrayPointer<uchar> data(new uchar[m_size]);
m_data = data.data();
if (file.read((char *)m_data, m_size) != m_size) {
m_error = file.errorString();
return 0;
}
QScopedPointer<GeoIPData> geoIPData(new GeoIPData);
m_geoIPData = geoIPData.data();
if (!parseMetadata(readMetadata()) || !loadDB())
return 0;
return geoIPData.take();
}
GeoIPData *Loader::load(const QByteArray &data)
{
if (data.size() > MAX_FILE_SIZE) {
m_error = tr("Unsupported database file size.");
return 0;
}
m_size = data.size();
m_data = reinterpret_cast<const uchar *>(data.constData());
#define CHECK_METADATA_REQ(key, type) \
if (!metadata.contains(#key)) { \
error = errMsgNotFound.arg(#key); \
return false; \
} \
else if (metadata.value(#key).userType() != QMetaType::type) { \
error = errMsgInvalid.arg(#key); \
return false; \
}
QScopedPointer<GeoIPData> geoIPData(new GeoIPData);
m_geoIPData = geoIPData.data();
if (!parseMetadata(readMetadata()) || !loadDB())
return 0;
#define CHECK_METADATA_OPT(key, type) \
if (metadata.contains(#key)) { \
if (metadata.value(#key).userType() != QMetaType::type) { \
error = errMsgInvalid.arg(#key); \
return false; \
} \
}
return geoIPData.take();
bool GeoIPDatabase::parseMetadata(const QVariantHash &metadata, QString &error)
{
const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
qDebug() << "Parsing MaxMindDB metadata...";
CHECK_METADATA_REQ(binary_format_major_version, UShort);
CHECK_METADATA_REQ(binary_format_minor_version, UShort);
uint versionMajor = metadata.value("binary_format_major_version").toUInt();
uint versionMinor = metadata.value("binary_format_minor_version").toUInt();
if (versionMajor != 2) {
error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
return false;
}
QString Loader::error() const
{
return m_error;
CHECK_METADATA_REQ(ip_version, UShort);
m_ipVersion = metadata.value("ip_version").value<quint16>();
if (m_ipVersion != 6) {
error = tr("Unsupported IP version: %1").arg(m_ipVersion);
return false;
}
#define CHECK_METADATA_REQ(key, type) \
if (!metadata.contains(#key)) { \
m_error = errMsgNotFound.arg(#key); \
return false; \
} \
else if (metadata.value(#key).userType() != QMetaType::type) { \
m_error = errMsgInvalid.arg(#key); \
return false; \
CHECK_METADATA_REQ(record_size, UShort);
m_recordSize = metadata.value("record_size").value<quint16>();
if (m_recordSize != 24) {
error = tr("Unsupported record size: %1").arg(m_recordSize);
return false;
}
#define CHECK_METADATA_OPT(key, type) \
if (metadata.contains(#key)) { \
if (metadata.value(#key).userType() != QMetaType::type) { \
m_error = errMsgInvalid.arg(#key); \
return false; \
} \
m_nodeSize = m_recordSize / 4;
m_recordBytes = m_nodeSize / 2;
CHECK_METADATA_REQ(node_count, UInt);
m_nodeCount = metadata.value("node_count").value<quint32>();
m_indexSize = m_nodeCount * m_nodeSize;
CHECK_METADATA_REQ(database_type, QString);
QString dbType = metadata.value("database_type").toString();
if (dbType != DB_TYPE) {
error = tr("Invalid database type: %1").arg(dbType);
return false;
}
bool Loader::parseMetadata(const QVariantHash &metadata)
{
const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
qDebug() << "Parsing MaxMindDB metadata...";
CHECK_METADATA_REQ(binary_format_major_version, UShort);
CHECK_METADATA_REQ(binary_format_minor_version, UShort);
uint versionMajor = metadata.value("binary_format_major_version").toUInt();
uint versionMinor = metadata.value("binary_format_minor_version").toUInt();
if (versionMajor != 2) {
m_error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
return false;
}
CHECK_METADATA_REQ(ip_version, UShort);
m_geoIPData->ipVersion = metadata.value("ip_version").value<quint16>();
if (m_geoIPData->ipVersion != 6) {
m_error = tr("Unsupported IP version: %1").arg(m_geoIPData->ipVersion);
return false;
}
CHECK_METADATA_REQ(build_epoch, ULongLong);
m_buildEpoch = QDateTime::fromTime_t(metadata.value("build_epoch").toULongLong());
CHECK_METADATA_REQ(record_size, UShort);
m_geoIPData->recordSize = metadata.value("record_size").value<quint16>();
if (m_geoIPData->recordSize != 24) {
m_error = tr("Unsupported record size: %1").arg(m_geoIPData->recordSize);
return false;
}
CHECK_METADATA_OPT(languages, QVariantList);
CHECK_METADATA_OPT(description, QVariantHash);
CHECK_METADATA_REQ(node_count, UInt);
m_geoIPData->nodeCount = metadata.value("node_count").value<quint32>();
CHECK_METADATA_REQ(database_type, QString);
QString dbType = metadata.value("database_type").toString();
if (dbType != DB_TYPE) {
m_error = tr("Invalid database type: %1").arg(dbType);
return false;
}
CHECK_METADATA_REQ(build_epoch, ULongLong);
m_geoIPData->buildEpoch = QDateTime::fromTime_t(metadata.value("build_epoch").toULongLong());
CHECK_METADATA_OPT(languages, QVariantList);
CHECK_METADATA_OPT(description, QVariantHash);
return true;
}
return true;
bool GeoIPDatabase::loadDB(QString &error) const
{
qDebug() << "Parsing MaxMindDB index tree...";
const int nodeSize = m_recordSize / 4; // in bytes
const int indexSize = m_nodeCount * nodeSize;
if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
|| (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0)) {
error = tr("Database corrupted: no data section found.");
return false;
}
bool Loader::loadDB()
{
qDebug() << "Parsing MaxMindDB index tree...";
const int nodeSize = m_geoIPData->recordSize / 4; // in bytes
const int indexSize = m_geoIPData->nodeCount * nodeSize;
if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
|| (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0)) {
m_error = tr("Database corrupted: no data section found.");
return false;
}
m_geoIPData->index.reserve(m_geoIPData->nodeCount);
const int recordBytes = nodeSize / 2;
const uchar *ptr = m_data;
bool left = true;
Node node;
for (quint32 i = 0; i < (2 * m_geoIPData->nodeCount); ++i) {
quint32 id = 0;
uchar *idPtr = reinterpret_cast<uchar *>(&id);
memcpy(&idPtr[4 - recordBytes], ptr, recordBytes);
fromBigEndian(idPtr, 4);
if ((id > m_geoIPData->nodeCount) && !m_geoIPData->countries.contains(id)) {
const quint32 offset = id - m_geoIPData->nodeCount - sizeof(DATA_SECTION_SEPARATOR);
quint32 tmp = offset + indexSize + sizeof(DATA_SECTION_SEPARATOR);
QVariant val = readDataField(tmp);
if (val.userType() == QMetaType::QVariantHash) {
m_geoIPData->countries[id] = val.toHash()["country"].toHash()["iso_code"].toString();
}
else if (val.userType() == QVariant::Invalid) {
m_error = tr("Database corrupted: invalid data type at DATA@%1").arg(offset, 8, 16, QLatin1Char('0'));
return false;
}
else {
m_error = tr("Invalid database: unsupported data type at DATA@%1").arg(offset, 8, 16, QLatin1Char('0'));
return false;
}
}
if (left) {
node.left = id;
}
else {
node.right = id;
m_geoIPData->index << node;
}
left = !left;
ptr += recordBytes;
}
return true;
}
return true;
QVariantHash GeoIPDatabase::readMetadata() const
{
const char *ptr = reinterpret_cast<const char *>(m_data);
quint32 size = m_size;
if (m_size > MAX_METADATA_SIZE) {
ptr += m_size - MAX_METADATA_SIZE;
size = MAX_METADATA_SIZE;
}
QVariantHash Loader::readMetadata()
{
const char *ptr = reinterpret_cast<const char *>(m_data);
quint32 size = m_size;
if (m_size > MAX_METADATA_SIZE) {
ptr += m_size - MAX_METADATA_SIZE;
size = MAX_METADATA_SIZE;
}
const QByteArray data = QByteArray::fromRawData(ptr, size);
int index = data.lastIndexOf(METADATA_BEGIN_MARK);
if (index >= 0) {
if (m_size > MAX_METADATA_SIZE)
index += (m_size - MAX_METADATA_SIZE); // from begin of all data
quint32 offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
QVariant metadata = readDataField(offset);
m_size = index; // truncate m_size to not contain metadata section
if (metadata.userType() == QMetaType::QVariantHash)
return metadata.toHash();
}
return QVariantHash();
const QByteArray data = QByteArray::fromRawData(ptr, size);
int index = data.lastIndexOf(METADATA_BEGIN_MARK);
if (index >= 0) {
if (m_size > MAX_METADATA_SIZE)
index += (m_size - MAX_METADATA_SIZE); // from begin of all data
quint32 offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
QVariant metadata = readDataField(offset);
if (metadata.userType() == QMetaType::QVariantHash)
return metadata.toHash();
}
QVariant Loader::readDataField(quint32 &offset)
{
DataFieldDescriptor descr;
if (!readDataFieldDescriptor(offset, descr))
return QVariantHash();
}
QVariant GeoIPDatabase::readDataField(quint32 &offset) const
{
DataFieldDescriptor descr;
if (!readDataFieldDescriptor(offset, descr))
return QVariant();
quint32 locOffset = offset;
bool usePointer = false;
if (descr.fieldType == DataType::Pointer) {
usePointer = true;
// convert offset from data section to global
locOffset = descr.offset + (m_nodeCount * m_recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
if (!readDataFieldDescriptor(locOffset, descr))
return QVariant();
}
quint32 locOffset = offset;
bool usePointer = false;
if (descr.fieldType == DataType::Pointer) {
usePointer = true;
// convert offset from data section to global
locOffset = descr.offset + (m_geoIPData->nodeCount * m_geoIPData->recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
if (!readDataFieldDescriptor(locOffset, descr))
return QVariant();
}
QVariant fieldValue;
switch (descr.fieldType) {
case DataType::Pointer:
qDebug() << "* Illegal Pointer using";
break;
case DataType::String:
fieldValue = QString::fromUtf8(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
locOffset += descr.fieldSize;
break;
case DataType::Double:
if (descr.fieldSize == 8)
fieldValue = readPlainValue<double>(locOffset, descr.fieldSize);
else
qDebug() << "* Invalid field size for type: Double";
break;
case DataType::Bytes:
fieldValue = QByteArray(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
locOffset += descr.fieldSize;
break;
case DataType::Integer16:
fieldValue = readPlainValue<quint16>(locOffset, descr.fieldSize);
break;
case DataType::Integer32:
fieldValue = readPlainValue<quint32>(locOffset, descr.fieldSize);
break;
case DataType::Map:
fieldValue = readMapValue(locOffset, descr.fieldSize);
break;
case DataType::SignedInteger32:
fieldValue = readPlainValue<qint32>(locOffset, descr.fieldSize);
break;
case DataType::Integer64:
fieldValue = readPlainValue<quint64>(locOffset, descr.fieldSize);
break;
case DataType::Integer128:
qDebug() << "* Unsupported data type: Integer128";
break;
case DataType::Array:
fieldValue = readArrayValue(locOffset, descr.fieldSize);
break;
case DataType::DataCacheContainer:
qDebug() << "* Unsupported data type: DataCacheContainer";
break;
case DataType::EndMarker:
qDebug() << "* Unsupported data type: EndMarker";
break;
case DataType::Boolean:
fieldValue = QVariant::fromValue(static_cast<bool>(descr.fieldSize));
break;
case DataType::Float:
if (descr.fieldSize == 4)
fieldValue = readPlainValue<float>(locOffset, descr.fieldSize);
else
qDebug() << "* Invalid field size for type: Float";
break;
}
QVariant fieldValue;
switch (descr.fieldType) {
case DataType::Pointer:
qDebug() << "* Illegal Pointer using";
break;
case DataType::String:
fieldValue = QString::fromUtf8(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
locOffset += descr.fieldSize;
break;
case DataType::Double:
if (descr.fieldSize == 8)
fieldValue = readPlainValue<double>(locOffset, descr.fieldSize);
else
qDebug() << "* Invalid field size for type: Double";
break;
case DataType::Bytes:
fieldValue = QByteArray(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
locOffset += descr.fieldSize;
break;
case DataType::Integer16:
fieldValue = readPlainValue<quint16>(locOffset, descr.fieldSize);
break;
case DataType::Integer32:
fieldValue = readPlainValue<quint32>(locOffset, descr.fieldSize);
break;
case DataType::Map:
fieldValue = readMapValue(locOffset, descr.fieldSize);
break;
case DataType::SignedInteger32:
fieldValue = readPlainValue<qint32>(locOffset, descr.fieldSize);
break;
case DataType::Integer64:
fieldValue = readPlainValue<quint64>(locOffset, descr.fieldSize);
break;
case DataType::Integer128:
qDebug() << "* Unsupported data type: Integer128";
break;
case DataType::Array:
fieldValue = readArrayValue(locOffset, descr.fieldSize);
break;
case DataType::DataCacheContainer:
qDebug() << "* Unsupported data type: DataCacheContainer";
break;
case DataType::EndMarker:
qDebug() << "* Unsupported data type: EndMarker";
break;
case DataType::Boolean:
fieldValue = QVariant::fromValue(static_cast<bool>(descr.fieldSize));
break;
case DataType::Float:
if (descr.fieldSize == 4)
fieldValue = readPlainValue<float>(locOffset, descr.fieldSize);
else
qDebug() << "* Invalid field size for type: Float";
break;
}
if (!usePointer)
offset = locOffset;
return fieldValue;
}
if (!usePointer)
offset = locOffset;
return fieldValue;
bool GeoIPDatabase::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const
{
const uchar *dataPtr = m_data + offset;
int availSize = m_size - offset;
if (availSize < 1) return false;
out.fieldType = static_cast<DataType>((dataPtr[0] & 0xE0) >> 5);
if (out.fieldType == DataType::Pointer) {
int size = ((dataPtr[0] & 0x18) >> 3);
if (availSize < (size + 2)) return false;
if (size == 0)
out.offset = ((dataPtr[0] & 0x07) << 8) + dataPtr[1];
else if (size == 1)
out.offset = ((dataPtr[0] & 0x07) << 16) + (dataPtr[1] << 8) + dataPtr[2] + 2048;
else if (size == 2)
out.offset = ((dataPtr[0] & 0x07) << 24) + (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 526336;
else if (size == 3)
out.offset = (dataPtr[1] << 24) + (dataPtr[2] << 16) + (dataPtr[3] << 8) + dataPtr[4];
offset += size + 2;
return true;
}
bool Loader::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out)
{
const uchar *dataPtr = m_data + offset;
int availSize = m_size - offset;
if (availSize < 1) return false;
out.fieldType = static_cast<DataType>((dataPtr[0] & 0xE0) >> 5);
if (out.fieldType == DataType::Pointer) {
int size = ((dataPtr[0] & 0x18) >> 3);
if (availSize < (size + 2)) return false;
if (size == 0)
out.offset = ((dataPtr[0] & 0x07) << 8) + dataPtr[1];
else if (size == 1)
out.offset = ((dataPtr[0] & 0x07) << 16) + (dataPtr[1] << 8) + dataPtr[2] + 2048;
else if (size == 2)
out.offset = ((dataPtr[0] & 0x07) << 24) + (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 526336;
else if (size == 3)
out.offset = (dataPtr[1] << 24) + (dataPtr[2] << 16) + (dataPtr[3] << 8) + dataPtr[4];
offset += size + 2;
return true;
}
out.fieldSize = dataPtr[0] & 0x1F;
if (out.fieldSize <= 28) {
if (out.fieldType == DataType::Unknown) {
out.fieldType = static_cast<DataType>(dataPtr[1] + 7);
if ((out.fieldType <= DataType::Map) || (out.fieldType > DataType::Float) || (availSize < 3))
return false;
offset += 2;
}
else {
offset += 1;
}
}
else if (out.fieldSize == 29) {
if (availSize < 2) return false;
out.fieldSize = dataPtr[1] + 29;
out.fieldSize = dataPtr[0] & 0x1F;
if (out.fieldSize <= 28) {
if (out.fieldType == DataType::Unknown) {
out.fieldType = static_cast<DataType>(dataPtr[1] + 7);
if ((out.fieldType <= DataType::Map) || (out.fieldType > DataType::Float) || (availSize < 3))
return false;
offset += 2;
}
else if (out.fieldSize == 30) {
if (availSize < 3) return false;
out.fieldSize = (dataPtr[1] << 8) + dataPtr[2] + 285;
offset += 3;
else {
offset += 1;
}
else if (out.fieldSize == 31) {
if (availSize < 4) return false;
out.fieldSize = (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 65821;
offset += 4;
}
return true;
}
void Loader::fromBigEndian(uchar *buf, quint32 len)
{
if (__IS_LITTLE_ENDIAN__)
std::reverse(buf, buf + len);
else if (out.fieldSize == 29) {
if (availSize < 2) return false;
out.fieldSize = dataPtr[1] + 29;
offset += 2;
}
else if (out.fieldSize == 30) {
if (availSize < 3) return false;
out.fieldSize = (dataPtr[1] << 8) + dataPtr[2] + 285;
offset += 3;
}
else if (out.fieldSize == 31) {
if (availSize < 4) return false;
out.fieldSize = (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 65821;
offset += 4;
}
QVariant Loader::readMapValue(quint32 &offset, quint32 count)
{
QVariantHash map;
return true;
}
void GeoIPDatabase::fromBigEndian(uchar *buf, quint32 len) const
{
if (__IS_LITTLE_ENDIAN__)
std::reverse(buf, buf + len);
}
for (quint32 i = 0; i < count; ++i) {
QVariant field = readDataField(offset);
if (field.userType() != QMetaType::QString)
return QVariant();
QVariant GeoIPDatabase::readMapValue(quint32 &offset, quint32 count) const
{
QVariantHash map;
QString key = field.toString();
field = readDataField(offset);
if (field.userType() == QVariant::Invalid)
return QVariant();
for (quint32 i = 0; i < count; ++i) {
QVariant field = readDataField(offset);
if (field.userType() != QMetaType::QString)
return QVariant();
map[key] = field;
}
QString key = field.toString();
field = readDataField(offset);
if (field.userType() == QVariant::Invalid)
return QVariant();
return map;
map[key] = field;
}
QVariant Loader::readArrayValue(quint32 &offset, quint32 count)
{
QVariantList array;
return map;
}
for (quint32 i = 0; i < count; ++i) {
QVariant field = readDataField(offset);
if (field.userType() == QVariant::Invalid)
return QVariant();
QVariant GeoIPDatabase::readArrayValue(quint32 &offset, quint32 count) const
{
QVariantList array;
array.append(field);
}
for (quint32 i = 0; i < count; ++i) {
QVariant field = readDataField(offset);
if (field.userType() == QVariant::Invalid)
return QVariant();
return array;
array.append(field);
}
return array;
}
namespace
{
#if (QT_VERSION < QT_VERSION_CHECK(5, 0, 0))
Q_IPV6ADDR createMappedAddress(quint32 ip4)
{

48
src/core/net/private/geoipdatabase.h
Unescape View File

@ -30,16 +30,19 @@ @@ -30,16 +30,19 @@
#define GEOIPDATABASE_H
#include <QtGlobal>
#include <QCoreApplication>
class QHostAddress;
class QString;
class QByteArray;
class QDateTime;
struct GeoIPData;
struct DataFieldDescriptor;
class GeoIPDatabase
{
Q_DECLARE_TR_FUNCTIONS(GeoIPDatabase)
public:
static GeoIPDatabase *load(const QString &filename, QString &error);
static GeoIPDatabase *load(const QByteArray &data, QString &error);
@ -52,9 +55,48 @@ public: @@ -52,9 +55,48 @@ public:
QString lookup(const QHostAddress &hostAddr) const;
private:
GeoIPDatabase(GeoIPData *geoIPData);
GeoIPDatabase(quint32 size);
bool parseMetadata(const QVariantHash &metadata, QString &error);
bool loadDB(QString &error) const;
QVariantHash readMetadata() const;
QVariant readDataField(quint32 &offset) const;
bool readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const;
void fromBigEndian(uchar *buf, quint32 len) const;
QVariant readMapValue(quint32 &offset, quint32 count) const;
QVariant readArrayValue(quint32 &offset, quint32 count) const;
template<typename T>
QVariant readPlainValue(quint32 &offset, quint8 len) const
{
T value = 0;
const uchar *const data = m_data + offset;
const quint32 availSize = m_size - offset;
if ((len > 0) && (len <= sizeof(T) && (availSize >= len))) {
// copy input data to last 'len' bytes of 'value'
uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len);
memcpy(dst, data, len);
fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T));
offset += len;
}
return QVariant::fromValue(value);
}
GeoIPData *m_geoIPData;
// Metadata
quint16 m_ipVersion;
quint16 m_recordSize;
quint32 m_nodeCount;
int m_nodeSize;
int m_indexSize;
int m_recordBytes;
QDateTime m_buildEpoch;
// Search data
mutable QHash<quint32, QString> m_countries;
quint32 m_size;
const uchar *m_data;
};
#endif // GEOIPDATABASE_H

Write Preview
Loading…
Cancel
Save