Pieter Wuille
8 years ago
31 changed files with 1414 additions and 1151 deletions
@ -1,89 +0,0 @@
@@ -1,89 +0,0 @@
|
||||
body { |
||||
margin-left: 0.5in; |
||||
margin-right: 0.5in; |
||||
background: white; |
||||
color: black; |
||||
} |
||||
|
||||
h1 { |
||||
margin-left: -0.2in; |
||||
font-size: 14pt; |
||||
} |
||||
h2 { |
||||
margin-left: -0in; |
||||
font-size: 12pt; |
||||
} |
||||
h3 { |
||||
margin-left: -0in; |
||||
} |
||||
h4 { |
||||
margin-left: -0in; |
||||
} |
||||
hr { |
||||
margin-left: -0in; |
||||
} |
||||
|
||||
/* Definition lists: definition term bold */ |
||||
dt { |
||||
font-weight: bold; |
||||
} |
||||
|
||||
address { |
||||
text-align: center; |
||||
} |
||||
code,samp,var { |
||||
color: blue; |
||||
} |
||||
kbd { |
||||
color: #600000; |
||||
} |
||||
div.note p { |
||||
float: right; |
||||
width: 3in; |
||||
margin-right: 0%; |
||||
padding: 1px; |
||||
border: 2px solid #6060a0; |
||||
background-color: #fffff0; |
||||
} |
||||
|
||||
ul { |
||||
margin-top: -0em; |
||||
margin-bottom: -0em; |
||||
} |
||||
|
||||
ol { |
||||
margin-top: -0em; |
||||
margin-bottom: -0em; |
||||
} |
||||
|
||||
UL.nobullets { |
||||
list-style-type: none; |
||||
list-style-image: none; |
||||
margin-left: -1em; |
||||
} |
||||
|
||||
p { |
||||
margin: 1em 0 1em 0; |
||||
padding: 0 0 0 0; |
||||
} |
||||
|
||||
pre { |
||||
line-height: 1.3em; |
||||
padding: 0.4em 0 0.8em 0; |
||||
margin: 0 0 0 0; |
||||
border: 0 0 0 0; |
||||
color: blue; |
||||
} |
||||
|
||||
.datatable { |
||||
margin-left: auto; |
||||
margin-right: auto; |
||||
margin-top: 2em; |
||||
margin-bottom: 2em; |
||||
border: 1px solid; |
||||
} |
||||
|
||||
.datatable td,th { |
||||
padding: 0 0.5em 0 0.5em; |
||||
text-align: right; |
||||
} |
||||
@ -1,213 +0,0 @@
@@ -1,213 +0,0 @@
|
||||
<!DOCTYPE html> |
||||
<html> |
||||
<head> |
||||
<link rel="stylesheet" type="text/css" href="doc.css" /> |
||||
<title>Leveldb file layout and compactions</title> |
||||
</head> |
||||
|
||||
<body> |
||||
|
||||
<h1>Files</h1> |
||||
|
||||
The implementation of leveldb is similar in spirit to the |
||||
representation of a single |
||||
<a href="http://research.google.com/archive/bigtable.html"> |
||||
Bigtable tablet (section 5.3)</a>. |
||||
However the organization of the files that make up the representation |
||||
is somewhat different and is explained below. |
||||
|
||||
<p> |
||||
Each database is represented by a set of files stored in a directory. |
||||
There are several different types of files as documented below: |
||||
<p> |
||||
<h2>Log files</h2> |
||||
<p> |
||||
A log file (*.log) stores a sequence of recent updates. Each update |
||||
is appended to the current log file. When the log file reaches a |
||||
pre-determined size (approximately 4MB by default), it is converted |
||||
to a sorted table (see below) and a new log file is created for future |
||||
updates. |
||||
<p> |
||||
A copy of the current log file is kept in an in-memory structure (the |
||||
<code>memtable</code>). This copy is consulted on every read so that read |
||||
operations reflect all logged updates. |
||||
<p> |
||||
<h2>Sorted tables</h2> |
||||
<p> |
||||
A sorted table (*.sst) stores a sequence of entries sorted by key. |
||||
Each entry is either a value for the key, or a deletion marker for the |
||||
key. (Deletion markers are kept around to hide obsolete values |
||||
present in older sorted tables). |
||||
<p> |
||||
The set of sorted tables are organized into a sequence of levels. The |
||||
sorted table generated from a log file is placed in a special <code>young</code> |
||||
level (also called level-0). When the number of young files exceeds a |
||||
certain threshold (currently four), all of the young files are merged |
||||
together with all of the overlapping level-1 files to produce a |
||||
sequence of new level-1 files (we create a new level-1 file for every |
||||
2MB of data.) |
||||
<p> |
||||
Files in the young level may contain overlapping keys. However files |
||||
in other levels have distinct non-overlapping key ranges. Consider |
||||
level number L where L >= 1. When the combined size of files in |
||||
level-L exceeds (10^L) MB (i.e., 10MB for level-1, 100MB for level-2, |
||||
...), one file in level-L, and all of the overlapping files in |
||||
level-(L+1) are merged to form a set of new files for level-(L+1). |
||||
These merges have the effect of gradually migrating new updates from |
||||
the young level to the largest level using only bulk reads and writes |
||||
(i.e., minimizing expensive seeks). |
||||
|
||||
<h2>Manifest</h2> |
||||
<p> |
||||
A MANIFEST file lists the set of sorted tables that make up each |
||||
level, the corresponding key ranges, and other important metadata. |
||||
A new MANIFEST file (with a new number embedded in the file name) |
||||
is created whenever the database is reopened. The MANIFEST file is |
||||
formatted as a log, and changes made to the serving state (as files |
||||
are added or removed) are appended to this log. |
||||
<p> |
||||
<h2>Current</h2> |
||||
<p> |
||||
CURRENT is a simple text file that contains the name of the latest |
||||
MANIFEST file. |
||||
<p> |
||||
<h2>Info logs</h2> |
||||
<p> |
||||
Informational messages are printed to files named LOG and LOG.old. |
||||
<p> |
||||
<h2>Others</h2> |
||||
<p> |
||||
Other files used for miscellaneous purposes may also be present |
||||
(LOCK, *.dbtmp). |
||||
|
||||
<h1>Level 0</h1> |
||||
When the log file grows above a certain size (1MB by default): |
||||
<ul> |
||||
<li>Create a brand new memtable and log file and direct future updates here |
||||
<li>In the background: |
||||
<ul> |
||||
<li>Write the contents of the previous memtable to an sstable |
||||
<li>Discard the memtable |
||||
<li>Delete the old log file and the old memtable |
||||
<li>Add the new sstable to the young (level-0) level. |
||||
</ul> |
||||
</ul> |
||||
|
||||
<h1>Compactions</h1> |
||||
|
||||
<p> |
||||
When the size of level L exceeds its limit, we compact it in a |
||||
background thread. The compaction picks a file from level L and all |
||||
overlapping files from the next level L+1. Note that if a level-L |
||||
file overlaps only part of a level-(L+1) file, the entire file at |
||||
level-(L+1) is used as an input to the compaction and will be |
||||
discarded after the compaction. Aside: because level-0 is special |
||||
(files in it may overlap each other), we treat compactions from |
||||
level-0 to level-1 specially: a level-0 compaction may pick more than |
||||
one level-0 file in case some of these files overlap each other. |
||||
|
||||
<p> |
||||
A compaction merges the contents of the picked files to produce a |
||||
sequence of level-(L+1) files. We switch to producing a new |
||||
level-(L+1) file after the current output file has reached the target |
||||
file size (2MB). We also switch to a new output file when the key |
||||
range of the current output file has grown enough to overlap more than |
||||
ten level-(L+2) files. This last rule ensures that a later compaction |
||||
of a level-(L+1) file will not pick up too much data from level-(L+2). |
||||
|
||||
<p> |
||||
The old files are discarded and the new files are added to the serving |
||||
state. |
||||
|
||||
<p> |
||||
Compactions for a particular level rotate through the key space. In |
||||
more detail, for each level L, we remember the ending key of the last |
||||
compaction at level L. The next compaction for level L will pick the |
||||
first file that starts after this key (wrapping around to the |
||||
beginning of the key space if there is no such file). |
||||
|
||||
<p> |
||||
Compactions drop overwritten values. They also drop deletion markers |
||||
if there are no higher numbered levels that contain a file whose range |
||||
overlaps the current key. |
||||
|
||||
<h2>Timing</h2> |
||||
|
||||
Level-0 compactions will read up to four 1MB files from level-0, and |
||||
at worst all the level-1 files (10MB). I.e., we will read 14MB and |
||||
write 14MB. |
||||
|
||||
<p> |
||||
Other than the special level-0 compactions, we will pick one 2MB file |
||||
from level L. In the worst case, this will overlap ~ 12 files from |
||||
level L+1 (10 because level-(L+1) is ten times the size of level-L, |
||||
and another two at the boundaries since the file ranges at level-L |
||||
will usually not be aligned with the file ranges at level-L+1). The |
||||
compaction will therefore read 26MB and write 26MB. Assuming a disk |
||||
IO rate of 100MB/s (ballpark range for modern drives), the worst |
||||
compaction cost will be approximately 0.5 second. |
||||
|
||||
<p> |
||||
If we throttle the background writing to something small, say 10% of |
||||
the full 100MB/s speed, a compaction may take up to 5 seconds. If the |
||||
user is writing at 10MB/s, we might build up lots of level-0 files |
||||
(~50 to hold the 5*10MB). This may significantly increase the cost of |
||||
reads due to the overhead of merging more files together on every |
||||
read. |
||||
|
||||
<p> |
||||
Solution 1: To reduce this problem, we might want to increase the log |
||||
switching threshold when the number of level-0 files is large. Though |
||||
the downside is that the larger this threshold, the more memory we will |
||||
need to hold the corresponding memtable. |
||||
|
||||
<p> |
||||
Solution 2: We might want to decrease write rate artificially when the |
||||
number of level-0 files goes up. |
||||
|
||||
<p> |
||||
Solution 3: We work on reducing the cost of very wide merges. |
||||
Perhaps most of the level-0 files will have their blocks sitting |
||||
uncompressed in the cache and we will only need to worry about the |
||||
O(N) complexity in the merging iterator. |
||||
|
||||
<h2>Number of files</h2> |
||||
|
||||
Instead of always making 2MB files, we could make larger files for |
||||
larger levels to reduce the total file count, though at the expense of |
||||
more bursty compactions. Alternatively, we could shard the set of |
||||
files into multiple directories. |
||||
|
||||
<p> |
||||
An experiment on an <code>ext3</code> filesystem on Feb 04, 2011 shows |
||||
the following timings to do 100K file opens in directories with |
||||
varying number of files: |
||||
<table class="datatable"> |
||||
<tr><th>Files in directory</th><th>Microseconds to open a file</th></tr> |
||||
<tr><td>1000</td><td>9</td> |
||||
<tr><td>10000</td><td>10</td> |
||||
<tr><td>100000</td><td>16</td> |
||||
</table> |
||||
So maybe even the sharding is not necessary on modern filesystems? |
||||
|
||||
<h1>Recovery</h1> |
||||
|
||||
<ul> |
||||
<li> Read CURRENT to find name of the latest committed MANIFEST |
||||
<li> Read the named MANIFEST file |
||||
<li> Clean up stale files |
||||
<li> We could open all sstables here, but it is probably better to be lazy... |
||||
<li> Convert log chunk to a new level-0 sstable |
||||
<li> Start directing new writes to a new log file with recovered sequence# |
||||
</ul> |
||||
|
||||
<h1>Garbage collection of files</h1> |
||||
|
||||
<code>DeleteObsoleteFiles()</code> is called at the end of every |
||||
compaction and at the end of recovery. It finds the names of all |
||||
files in the database. It deletes all log files that are not the |
||||
current log file. It deletes all table files that are not referenced |
||||
from some level and are not the output of an active compaction. |
||||
|
||||
</body> |
||||
</html> |
||||
@ -0,0 +1,170 @@
@@ -0,0 +1,170 @@
|
||||
## Files |
||||
|
||||
The implementation of leveldb is similar in spirit to the representation of a |
||||
single [Bigtable tablet (section 5.3)](http://research.google.com/archive/bigtable.html). |
||||
However the organization of the files that make up the representation is |
||||
somewhat different and is explained below. |
||||
|
||||
Each database is represented by a set of files stored in a directory. There are |
||||
several different types of files as documented below: |
||||
|
||||
### Log files |
||||
|
||||
A log file (*.log) stores a sequence of recent updates. Each update is appended |
||||
to the current log file. When the log file reaches a pre-determined size |
||||
(approximately 4MB by default), it is converted to a sorted table (see below) |
||||
and a new log file is created for future updates. |
||||
|
||||
A copy of the current log file is kept in an in-memory structure (the |
||||
`memtable`). This copy is consulted on every read so that read operations |
||||
reflect all logged updates. |
||||
|
||||
## Sorted tables |
||||
|
||||
A sorted table (*.ldb) stores a sequence of entries sorted by key. Each entry is |
||||
either a value for the key, or a deletion marker for the key. (Deletion markers |
||||
are kept around to hide obsolete values present in older sorted tables). |
||||
|
||||
The set of sorted tables are organized into a sequence of levels. The sorted |
||||
table generated from a log file is placed in a special **young** level (also |
||||
called level-0). When the number of young files exceeds a certain threshold |
||||
(currently four), all of the young files are merged together with all of the |
||||
overlapping level-1 files to produce a sequence of new level-1 files (we create |
||||
a new level-1 file for every 2MB of data.) |
||||
|
||||
Files in the young level may contain overlapping keys. However files in other |
||||
levels have distinct non-overlapping key ranges. Consider level number L where |
||||
L >= 1. When the combined size of files in level-L exceeds (10^L) MB (i.e., 10MB |
||||
for level-1, 100MB for level-2, ...), one file in level-L, and all of the |
||||
overlapping files in level-(L+1) are merged to form a set of new files for |
||||
level-(L+1). These merges have the effect of gradually migrating new updates |
||||
from the young level to the largest level using only bulk reads and writes |
||||
(i.e., minimizing expensive seeks). |
||||
|
||||
### Manifest |
||||
|
||||
A MANIFEST file lists the set of sorted tables that make up each level, the |
||||
corresponding key ranges, and other important metadata. A new MANIFEST file |
||||
(with a new number embedded in the file name) is created whenever the database |
||||
is reopened. The MANIFEST file is formatted as a log, and changes made to the |
||||
serving state (as files are added or removed) are appended to this log. |
||||
|
||||
### Current |
||||
|
||||
CURRENT is a simple text file that contains the name of the latest MANIFEST |
||||
file. |
||||
|
||||
### Info logs |
||||
|
||||
Informational messages are printed to files named LOG and LOG.old. |
||||
|
||||
### Others |
||||
|
||||
Other files used for miscellaneous purposes may also be present (LOCK, *.dbtmp). |
||||
|
||||
## Level 0 |
||||
|
||||
When the log file grows above a certain size (1MB by default): |
||||
Create a brand new memtable and log file and direct future updates here |
||||
In the background: |
||||
Write the contents of the previous memtable to an sstable |
||||
Discard the memtable |
||||
Delete the old log file and the old memtable |
||||
Add the new sstable to the young (level-0) level. |
||||
|
||||
## Compactions |
||||
|
||||
When the size of level L exceeds its limit, we compact it in a background |
||||
thread. The compaction picks a file from level L and all overlapping files from |
||||
the next level L+1. Note that if a level-L file overlaps only part of a |
||||
level-(L+1) file, the entire file at level-(L+1) is used as an input to the |
||||
compaction and will be discarded after the compaction. Aside: because level-0 |
||||
is special (files in it may overlap each other), we treat compactions from |
||||
level-0 to level-1 specially: a level-0 compaction may pick more than one |
||||
level-0 file in case some of these files overlap each other. |
||||
|
||||
A compaction merges the contents of the picked files to produce a sequence of |
||||
level-(L+1) files. We switch to producing a new level-(L+1) file after the |
||||
current output file has reached the target file size (2MB). We also switch to a |
||||
new output file when the key range of the current output file has grown enough |
||||
to overlap more than ten level-(L+2) files. This last rule ensures that a later |
||||
compaction of a level-(L+1) file will not pick up too much data from |
||||
level-(L+2). |
||||
|
||||
The old files are discarded and the new files are added to the serving state. |
||||
|
||||
Compactions for a particular level rotate through the key space. In more detail, |
||||
for each level L, we remember the ending key of the last compaction at level L. |
||||
The next compaction for level L will pick the first file that starts after this |
||||
key (wrapping around to the beginning of the key space if there is no such |
||||
file). |
||||
|
||||
Compactions drop overwritten values. They also drop deletion markers if there |
||||
are no higher numbered levels that contain a file whose range overlaps the |
||||
current key. |
||||
|
||||
### Timing |
||||
|
||||
Level-0 compactions will read up to four 1MB files from level-0, and at worst |
||||
all the level-1 files (10MB). I.e., we will read 14MB and write 14MB. |
||||
|
||||
Other than the special level-0 compactions, we will pick one 2MB file from level |
||||
L. In the worst case, this will overlap ~ 12 files from level L+1 (10 because |
||||
level-(L+1) is ten times the size of level-L, and another two at the boundaries |
||||
since the file ranges at level-L will usually not be aligned with the file |
||||
ranges at level-L+1). The compaction will therefore read 26MB and write 26MB. |
||||
Assuming a disk IO rate of 100MB/s (ballpark range for modern drives), the worst |
||||
compaction cost will be approximately 0.5 second. |
||||
|
||||
If we throttle the background writing to something small, say 10% of the full |
||||
100MB/s speed, a compaction may take up to 5 seconds. If the user is writing at |
||||
10MB/s, we might build up lots of level-0 files (~50 to hold the 5*10MB). This |
||||
may significantly increase the cost of reads due to the overhead of merging more |
||||
files together on every read. |
||||
|
||||
Solution 1: To reduce this problem, we might want to increase the log switching |
||||
threshold when the number of level-0 files is large. Though the downside is that |
||||
the larger this threshold, the more memory we will need to hold the |
||||
corresponding memtable. |
||||
|
||||
Solution 2: We might want to decrease write rate artificially when the number of |
||||
level-0 files goes up. |
||||
|
||||
Solution 3: We work on reducing the cost of very wide merges. Perhaps most of |
||||
the level-0 files will have their blocks sitting uncompressed in the cache and |
||||
we will only need to worry about the O(N) complexity in the merging iterator. |
||||
|
||||
### Number of files |
||||
|
||||
Instead of always making 2MB files, we could make larger files for larger levels |
||||
to reduce the total file count, though at the expense of more bursty |
||||
compactions. Alternatively, we could shard the set of files into multiple |
||||
directories. |
||||
|
||||
An experiment on an ext3 filesystem on Feb 04, 2011 shows the following timings |
||||
to do 100K file opens in directories with varying number of files: |
||||
|
||||
|
||||
| Files in directory | Microseconds to open a file | |
||||
|-------------------:|----------------------------:| |
||||
| 1000 | 9 | |
||||
| 10000 | 10 | |
||||
| 100000 | 16 | |
||||
|
||||
So maybe even the sharding is not necessary on modern filesystems? |
||||
|
||||
## Recovery |
||||
|
||||
* Read CURRENT to find name of the latest committed MANIFEST |
||||
* Read the named MANIFEST file |
||||
* Clean up stale files |
||||
* We could open all sstables here, but it is probably better to be lazy... |
||||
* Convert log chunk to a new level-0 sstable |
||||
* Start directing new writes to a new log file with recovered sequence# |
||||
|
||||
## Garbage collection of files |
||||
|
||||
`DeleteObsoleteFiles()` is called at the end of every compaction and at the end |
||||
of recovery. It finds the names of all files in the database. It deletes all log |
||||
files that are not the current log file. It deletes all table files that are not |
||||
referenced from some level and are not the output of an active compaction. |
||||
@ -1,549 +0,0 @@
@@ -1,549 +0,0 @@
|
||||
<!DOCTYPE html> |
||||
<html> |
||||
<head> |
||||
<link rel="stylesheet" type="text/css" href="doc.css" /> |
||||
<title>Leveldb</title> |
||||
</head> |
||||
|
||||
<body> |
||||
<h1>Leveldb</h1> |
||||
<address>Jeff Dean, Sanjay Ghemawat</address> |
||||
<p> |
||||
The <code>leveldb</code> library provides a persistent key value store. Keys and |
||||
values are arbitrary byte arrays. The keys are ordered within the key |
||||
value store according to a user-specified comparator function. |
||||
|
||||
<p> |
||||
<h1>Opening A Database</h1> |
||||
<p> |
||||
A <code>leveldb</code> database has a name which corresponds to a file system |
||||
directory. All of the contents of database are stored in this |
||||
directory. The following example shows how to open a database, |
||||
creating it if necessary: |
||||
<p> |
||||
<pre> |
||||
#include <cassert> |
||||
#include "leveldb/db.h" |
||||
|
||||
leveldb::DB* db; |
||||
leveldb::Options options; |
||||
options.create_if_missing = true; |
||||
leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db); |
||||
assert(status.ok()); |
||||
... |
||||
</pre> |
||||
If you want to raise an error if the database already exists, add |
||||
the following line before the <code>leveldb::DB::Open</code> call: |
||||
<pre> |
||||
options.error_if_exists = true; |
||||
</pre> |
||||
<h1>Status</h1> |
||||
<p> |
||||
You may have noticed the <code>leveldb::Status</code> type above. Values of this |
||||
type are returned by most functions in <code>leveldb</code> that may encounter an |
||||
error. You can check if such a result is ok, and also print an |
||||
associated error message: |
||||
<p> |
||||
<pre> |
||||
leveldb::Status s = ...; |
||||
if (!s.ok()) cerr << s.ToString() << endl; |
||||
</pre> |
||||
<h1>Closing A Database</h1> |
||||
<p> |
||||
When you are done with a database, just delete the database object. |
||||
Example: |
||||
<p> |
||||
<pre> |
||||
... open the db as described above ... |
||||
... do something with db ... |
||||
delete db; |
||||
</pre> |
||||
<h1>Reads And Writes</h1> |
||||
<p> |
||||
The database provides <code>Put</code>, <code>Delete</code>, and <code>Get</code> methods to |
||||
modify/query the database. For example, the following code |
||||
moves the value stored under key1 to key2. |
||||
<pre> |
||||
std::string value; |
||||
leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value); |
||||
if (s.ok()) s = db->Put(leveldb::WriteOptions(), key2, value); |
||||
if (s.ok()) s = db->Delete(leveldb::WriteOptions(), key1); |
||||
</pre> |
||||
|
||||
<h1>Atomic Updates</h1> |
||||
<p> |
||||
Note that if the process dies after the Put of key2 but before the |
||||
delete of key1, the same value may be left stored under multiple keys. |
||||
Such problems can be avoided by using the <code>WriteBatch</code> class to |
||||
atomically apply a set of updates: |
||||
<p> |
||||
<pre> |
||||
#include "leveldb/write_batch.h" |
||||
... |
||||
std::string value; |
||||
leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value); |
||||
if (s.ok()) { |
||||
leveldb::WriteBatch batch; |
||||
batch.Delete(key1); |
||||
batch.Put(key2, value); |
||||
s = db->Write(leveldb::WriteOptions(), &batch); |
||||
} |
||||
</pre> |
||||
The <code>WriteBatch</code> holds a sequence of edits to be made to the database, |
||||
and these edits within the batch are applied in order. Note that we |
||||
called <code>Delete</code> before <code>Put</code> so that if <code>key1</code> is identical to <code>key2</code>, |
||||
we do not end up erroneously dropping the value entirely. |
||||
<p> |
||||
Apart from its atomicity benefits, <code>WriteBatch</code> may also be used to |
||||
speed up bulk updates by placing lots of individual mutations into the |
||||
same batch. |
||||
|
||||
<h1>Synchronous Writes</h1> |
||||
By default, each write to <code>leveldb</code> is asynchronous: it |
||||
returns after pushing the write from the process into the operating |
||||
system. The transfer from operating system memory to the underlying |
||||
persistent storage happens asynchronously. The <code>sync</code> flag |
||||
can be turned on for a particular write to make the write operation |
||||
not return until the data being written has been pushed all the way to |
||||
persistent storage. (On Posix systems, this is implemented by calling |
||||
either <code>fsync(...)</code> or <code>fdatasync(...)</code> or |
||||
<code>msync(..., MS_SYNC)</code> before the write operation returns.) |
||||
<pre> |
||||
leveldb::WriteOptions write_options; |
||||
write_options.sync = true; |
||||
db->Put(write_options, ...); |
||||
</pre> |
||||
Asynchronous writes are often more than a thousand times as fast as |
||||
synchronous writes. The downside of asynchronous writes is that a |
||||
crash of the machine may cause the last few updates to be lost. Note |
||||
that a crash of just the writing process (i.e., not a reboot) will not |
||||
cause any loss since even when <code>sync</code> is false, an update |
||||
is pushed from the process memory into the operating system before it |
||||
is considered done. |
||||
|
||||
<p> |
||||
Asynchronous writes can often be used safely. For example, when |
||||
loading a large amount of data into the database you can handle lost |
||||
updates by restarting the bulk load after a crash. A hybrid scheme is |
||||
also possible where every Nth write is synchronous, and in the event |
||||
of a crash, the bulk load is restarted just after the last synchronous |
||||
write finished by the previous run. (The synchronous write can update |
||||
a marker that describes where to restart on a crash.) |
||||
|
||||
<p> |
||||
<code>WriteBatch</code> provides an alternative to asynchronous writes. |
||||
Multiple updates may be placed in the same <code>WriteBatch</code> and |
||||
applied together using a synchronous write (i.e., |
||||
<code>write_options.sync</code> is set to true). The extra cost of |
||||
the synchronous write will be amortized across all of the writes in |
||||
the batch. |
||||
|
||||
<p> |
||||
<h1>Concurrency</h1> |
||||
<p> |
||||
A database may only be opened by one process at a time. |
||||
The <code>leveldb</code> implementation acquires a lock from the |
||||
operating system to prevent misuse. Within a single process, the |
||||
same <code>leveldb::DB</code> object may be safely shared by multiple |
||||
concurrent threads. I.e., different threads may write into or fetch |
||||
iterators or call <code>Get</code> on the same database without any |
||||
external synchronization (the leveldb implementation will |
||||
automatically do the required synchronization). However other objects |
||||
(like Iterator and WriteBatch) may require external synchronization. |
||||
If two threads share such an object, they must protect access to it |
||||
using their own locking protocol. More details are available in |
||||
the public header files. |
||||
<p> |
||||
<h1>Iteration</h1> |
||||
<p> |
||||
The following example demonstrates how to print all key,value pairs |
||||
in a database. |
||||
<p> |
||||
<pre> |
||||
leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions()); |
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) { |
||||
cout << it->key().ToString() << ": " << it->value().ToString() << endl; |
||||
} |
||||
assert(it->status().ok()); // Check for any errors found during the scan |
||||
delete it; |
||||
</pre> |
||||
The following variation shows how to process just the keys in the |
||||
range <code>[start,limit)</code>: |
||||
<p> |
||||
<pre> |
||||
for (it->Seek(start); |
||||
it->Valid() && it->key().ToString() < limit; |
||||
it->Next()) { |
||||
... |
||||
} |
||||
</pre> |
||||
You can also process entries in reverse order. (Caveat: reverse |
||||
iteration may be somewhat slower than forward iteration.) |
||||
<p> |
||||
<pre> |
||||
for (it->SeekToLast(); it->Valid(); it->Prev()) { |
||||
... |
||||
} |
||||
</pre> |
||||
<h1>Snapshots</h1> |
||||
<p> |
||||
Snapshots provide consistent read-only views over the entire state of |
||||
the key-value store. <code>ReadOptions::snapshot</code> may be non-NULL to indicate |
||||
that a read should operate on a particular version of the DB state. |
||||
If <code>ReadOptions::snapshot</code> is NULL, the read will operate on an |
||||
implicit snapshot of the current state. |
||||
<p> |
||||
Snapshots are created by the DB::GetSnapshot() method: |
||||
<p> |
||||
<pre> |
||||
leveldb::ReadOptions options; |
||||
options.snapshot = db->GetSnapshot(); |
||||
... apply some updates to db ... |
||||
leveldb::Iterator* iter = db->NewIterator(options); |
||||
... read using iter to view the state when the snapshot was created ... |
||||
delete iter; |
||||
db->ReleaseSnapshot(options.snapshot); |
||||
</pre> |
||||
Note that when a snapshot is no longer needed, it should be released |
||||
using the DB::ReleaseSnapshot interface. This allows the |
||||
implementation to get rid of state that was being maintained just to |
||||
support reading as of that snapshot. |
||||
<h1>Slice</h1> |
||||
<p> |
||||
The return value of the <code>it->key()</code> and <code>it->value()</code> calls above |
||||
are instances of the <code>leveldb::Slice</code> type. <code>Slice</code> is a simple |
||||
structure that contains a length and a pointer to an external byte |
||||
array. Returning a <code>Slice</code> is a cheaper alternative to returning a |
||||
<code>std::string</code> since we do not need to copy potentially large keys and |
||||
values. In addition, <code>leveldb</code> methods do not return null-terminated |
||||
C-style strings since <code>leveldb</code> keys and values are allowed to |
||||
contain '\0' bytes. |
||||
<p> |
||||
C++ strings and null-terminated C-style strings can be easily converted |
||||
to a Slice: |
||||
<p> |
||||
<pre> |
||||
leveldb::Slice s1 = "hello"; |
||||
|
||||
std::string str("world"); |
||||
leveldb::Slice s2 = str; |
||||
</pre> |
||||
A Slice can be easily converted back to a C++ string: |
||||
<pre> |
||||
std::string str = s1.ToString(); |
||||
assert(str == std::string("hello")); |
||||
</pre> |
||||
Be careful when using Slices since it is up to the caller to ensure that |
||||
the external byte array into which the Slice points remains live while |
||||
the Slice is in use. For example, the following is buggy: |
||||
<p> |
||||
<pre> |
||||
leveldb::Slice slice; |
||||
if (...) { |
||||
std::string str = ...; |
||||
slice = str; |
||||
} |
||||
Use(slice); |
||||
</pre> |
||||
When the <code>if</code> statement goes out of scope, <code>str</code> will be destroyed and the |
||||
backing storage for <code>slice</code> will disappear. |
||||
<p> |
||||
<h1>Comparators</h1> |
||||
<p> |
||||
The preceding examples used the default ordering function for key, |
||||
which orders bytes lexicographically. You can however supply a custom |
||||
comparator when opening a database. For example, suppose each |
||||
database key consists of two numbers and we should sort by the first |
||||
number, breaking ties by the second number. First, define a proper |
||||
subclass of <code>leveldb::Comparator</code> that expresses these rules: |
||||
<p> |
||||
<pre> |
||||
class TwoPartComparator : public leveldb::Comparator { |
||||
public: |
||||
// Three-way comparison function: |
||||
// if a < b: negative result |
||||
// if a > b: positive result |
||||
// else: zero result |
||||
int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const { |
||||
int a1, a2, b1, b2; |
||||
ParseKey(a, &a1, &a2); |
||||
ParseKey(b, &b1, &b2); |
||||
if (a1 < b1) return -1; |
||||
if (a1 > b1) return +1; |
||||
if (a2 < b2) return -1; |
||||
if (a2 > b2) return +1; |
||||
return 0; |
||||
} |
||||
|
||||
// Ignore the following methods for now: |
||||
const char* Name() const { return "TwoPartComparator"; } |
||||
void FindShortestSeparator(std::string*, const leveldb::Slice&) const { } |
||||
void FindShortSuccessor(std::string*) const { } |
||||
}; |
||||
</pre> |
||||
Now create a database using this custom comparator: |
||||
<p> |
||||
<pre> |
||||
TwoPartComparator cmp; |
||||
leveldb::DB* db; |
||||
leveldb::Options options; |
||||
options.create_if_missing = true; |
||||
options.comparator = &cmp; |
||||
leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db); |
||||
... |
||||
</pre> |
||||
<h2>Backwards compatibility</h2> |
||||
<p> |
||||
The result of the comparator's <code>Name</code> method is attached to the |
||||
database when it is created, and is checked on every subsequent |
||||
database open. If the name changes, the <code>leveldb::DB::Open</code> call will |
||||
fail. Therefore, change the name if and only if the new key format |
||||
and comparison function are incompatible with existing databases, and |
||||
it is ok to discard the contents of all existing databases. |
||||
<p> |
||||
You can however still gradually evolve your key format over time with |
||||
a little bit of pre-planning. For example, you could store a version |
||||
number at the end of each key (one byte should suffice for most uses). |
||||
When you wish to switch to a new key format (e.g., adding an optional |
||||
third part to the keys processed by <code>TwoPartComparator</code>), |
||||
(a) keep the same comparator name (b) increment the version number |
||||
for new keys (c) change the comparator function so it uses the |
||||
version numbers found in the keys to decide how to interpret them. |
||||
<p> |
||||
<h1>Performance</h1> |
||||
<p> |
||||
Performance can be tuned by changing the default values of the |
||||
types defined in <code>include/leveldb/options.h</code>. |
||||
|
||||
<p> |
||||
<h2>Block size</h2> |
||||
<p> |
||||
<code>leveldb</code> groups adjacent keys together into the same block and such a |
||||
block is the unit of transfer to and from persistent storage. The |
||||
default block size is approximately 4096 uncompressed bytes. |
||||
Applications that mostly do bulk scans over the contents of the |
||||
database may wish to increase this size. Applications that do a lot |
||||
of point reads of small values may wish to switch to a smaller block |
||||
size if performance measurements indicate an improvement. There isn't |
||||
much benefit in using blocks smaller than one kilobyte, or larger than |
||||
a few megabytes. Also note that compression will be more effective |
||||
with larger block sizes. |
||||
<p> |
||||
<h2>Compression</h2> |
||||
<p> |
||||
Each block is individually compressed before being written to |
||||
persistent storage. Compression is on by default since the default |
||||
compression method is very fast, and is automatically disabled for |
||||
uncompressible data. In rare cases, applications may want to disable |
||||
compression entirely, but should only do so if benchmarks show a |
||||
performance improvement: |
||||
<p> |
||||
<pre> |
||||
leveldb::Options options; |
||||
options.compression = leveldb::kNoCompression; |
||||
... leveldb::DB::Open(options, name, ...) .... |
||||
</pre> |
||||
<h2>Cache</h2> |
||||
<p> |
||||
The contents of the database are stored in a set of files in the |
||||
filesystem and each file stores a sequence of compressed blocks. If |
||||
<code>options.cache</code> is non-NULL, it is used to cache frequently used |
||||
uncompressed block contents. |
||||
<p> |
||||
<pre> |
||||
#include "leveldb/cache.h" |
||||
|
||||
leveldb::Options options; |
||||
options.cache = leveldb::NewLRUCache(100 * 1048576); // 100MB cache |
||||
leveldb::DB* db; |
||||
leveldb::DB::Open(options, name, &db); |
||||
... use the db ... |
||||
delete db |
||||
delete options.cache; |
||||
</pre> |
||||
Note that the cache holds uncompressed data, and therefore it should |
||||
be sized according to application level data sizes, without any |
||||
reduction from compression. (Caching of compressed blocks is left to |
||||
the operating system buffer cache, or any custom <code>Env</code> |
||||
implementation provided by the client.) |
||||
<p> |
||||
When performing a bulk read, the application may wish to disable |
||||
caching so that the data processed by the bulk read does not end up |
||||
displacing most of the cached contents. A per-iterator option can be |
||||
used to achieve this: |
||||
<p> |
||||
<pre> |
||||
leveldb::ReadOptions options; |
||||
options.fill_cache = false; |
||||
leveldb::Iterator* it = db->NewIterator(options); |
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) { |
||||
... |
||||
} |
||||
</pre> |
||||
<h2>Key Layout</h2> |
||||
<p> |
||||
Note that the unit of disk transfer and caching is a block. Adjacent |
||||
keys (according to the database sort order) will usually be placed in |
||||
the same block. Therefore the application can improve its performance |
||||
by placing keys that are accessed together near each other and placing |
||||
infrequently used keys in a separate region of the key space. |
||||
<p> |
||||
For example, suppose we are implementing a simple file system on top |
||||
of <code>leveldb</code>. The types of entries we might wish to store are: |
||||
<p> |
||||
<pre> |
||||
filename -> permission-bits, length, list of file_block_ids |
||||
file_block_id -> data |
||||
</pre> |
||||
We might want to prefix <code>filename</code> keys with one letter (say '/') and the |
||||
<code>file_block_id</code> keys with a different letter (say '0') so that scans |
||||
over just the metadata do not force us to fetch and cache bulky file |
||||
contents. |
||||
<p> |
||||
<h2>Filters</h2> |
||||
<p> |
||||
Because of the way <code>leveldb</code> data is organized on disk, |
||||
a single <code>Get()</code> call may involve multiple reads from disk. |
||||
The optional <code>FilterPolicy</code> mechanism can be used to reduce |
||||
the number of disk reads substantially. |
||||
<pre> |
||||
leveldb::Options options; |
||||
options.filter_policy = NewBloomFilterPolicy(10); |
||||
leveldb::DB* db; |
||||
leveldb::DB::Open(options, "/tmp/testdb", &db); |
||||
... use the database ... |
||||
delete db; |
||||
delete options.filter_policy; |
||||
</pre> |
||||
The preceding code associates a |
||||
<a href="http://en.wikipedia.org/wiki/Bloom_filter">Bloom filter</a> |
||||
based filtering policy with the database. Bloom filter based |
||||
filtering relies on keeping some number of bits of data in memory per |
||||
key (in this case 10 bits per key since that is the argument we passed |
||||
to NewBloomFilterPolicy). This filter will reduce the number of unnecessary |
||||
disk reads needed for <code>Get()</code> calls by a factor of |
||||
approximately a 100. Increasing the bits per key will lead to a |
||||
larger reduction at the cost of more memory usage. We recommend that |
||||
applications whose working set does not fit in memory and that do a |
||||
lot of random reads set a filter policy. |
||||
<p> |
||||
If you are using a custom comparator, you should ensure that the filter |
||||
policy you are using is compatible with your comparator. For example, |
||||
consider a comparator that ignores trailing spaces when comparing keys. |
||||
<code>NewBloomFilterPolicy</code> must not be used with such a comparator. |
||||
Instead, the application should provide a custom filter policy that |
||||
also ignores trailing spaces. For example: |
||||
<pre> |
||||
class CustomFilterPolicy : public leveldb::FilterPolicy { |
||||
private: |
||||
FilterPolicy* builtin_policy_; |
||||
public: |
||||
CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) { } |
||||
~CustomFilterPolicy() { delete builtin_policy_; } |
||||
|
||||
const char* Name() const { return "IgnoreTrailingSpacesFilter"; } |
||||
|
||||
void CreateFilter(const Slice* keys, int n, std::string* dst) const { |
||||
// Use builtin bloom filter code after removing trailing spaces |
||||
std::vector<Slice> trimmed(n); |
||||
for (int i = 0; i < n; i++) { |
||||
trimmed[i] = RemoveTrailingSpaces(keys[i]); |
||||
} |
||||
return builtin_policy_->CreateFilter(&trimmed[i], n, dst); |
||||
} |
||||
|
||||
bool KeyMayMatch(const Slice& key, const Slice& filter) const { |
||||
// Use builtin bloom filter code after removing trailing spaces |
||||
return builtin_policy_->KeyMayMatch(RemoveTrailingSpaces(key), filter); |
||||
} |
||||
}; |
||||
</pre> |
||||
<p> |
||||
Advanced applications may provide a filter policy that does not use |
||||
a bloom filter but uses some other mechanism for summarizing a set |
||||
of keys. See <code>leveldb/filter_policy.h</code> for detail. |
||||
<p> |
||||
<h1>Checksums</h1> |
||||
<p> |
||||
<code>leveldb</code> associates checksums with all data it stores in the file system. |
||||
There are two separate controls provided over how aggressively these |
||||
checksums are verified: |
||||
<p> |
||||
<ul> |
||||
<li> <code>ReadOptions::verify_checksums</code> may be set to true to force |
||||
checksum verification of all data that is read from the file system on |
||||
behalf of a particular read. By default, no such verification is |
||||
done. |
||||
<p> |
||||
<li> <code>Options::paranoid_checks</code> may be set to true before opening a |
||||
database to make the database implementation raise an error as soon as |
||||
it detects an internal corruption. Depending on which portion of the |
||||
database has been corrupted, the error may be raised when the database |
||||
is opened, or later by another database operation. By default, |
||||
paranoid checking is off so that the database can be used even if |
||||
parts of its persistent storage have been corrupted. |
||||
<p> |
||||
If a database is corrupted (perhaps it cannot be opened when |
||||
paranoid checking is turned on), the <code>leveldb::RepairDB</code> function |
||||
may be used to recover as much of the data as possible |
||||
<p> |
||||
</ul> |
||||
<h1>Approximate Sizes</h1> |
||||
<p> |
||||
The <code>GetApproximateSizes</code> method can used to get the approximate |
||||
number of bytes of file system space used by one or more key ranges. |
||||
<p> |
||||
<pre> |
||||
leveldb::Range ranges[2]; |
||||
ranges[0] = leveldb::Range("a", "c"); |
||||
ranges[1] = leveldb::Range("x", "z"); |
||||
uint64_t sizes[2]; |
||||
leveldb::Status s = db->GetApproximateSizes(ranges, 2, sizes); |
||||
</pre> |
||||
The preceding call will set <code>sizes[0]</code> to the approximate number of |
||||
bytes of file system space used by the key range <code>[a..c)</code> and |
||||
<code>sizes[1]</code> to the approximate number of bytes used by the key range |
||||
<code>[x..z)</code>. |
||||
<p> |
||||
<h1>Environment</h1> |
||||
<p> |
||||
All file operations (and other operating system calls) issued by the |
||||
<code>leveldb</code> implementation are routed through a <code>leveldb::Env</code> object. |
||||
Sophisticated clients may wish to provide their own <code>Env</code> |
||||
implementation to get better control. For example, an application may |
||||
introduce artificial delays in the file IO paths to limit the impact |
||||
of <code>leveldb</code> on other activities in the system. |
||||
<p> |
||||
<pre> |
||||
class SlowEnv : public leveldb::Env { |
||||
.. implementation of the Env interface ... |
||||
}; |
||||
|
||||
SlowEnv env; |
||||
leveldb::Options options; |
||||
options.env = &env; |
||||
Status s = leveldb::DB::Open(options, ...); |
||||
</pre> |
||||
<h1>Porting</h1> |
||||
<p> |
||||
<code>leveldb</code> may be ported to a new platform by providing platform |
||||
specific implementations of the types/methods/functions exported by |
||||
<code>leveldb/port/port.h</code>. See <code>leveldb/port/port_example.h</code> for more |
||||
details. |
||||
<p> |
||||
In addition, the new platform may need a new default <code>leveldb::Env</code> |
||||
implementation. See <code>leveldb/util/env_posix.h</code> for an example. |
||||
|
||||
<h1>Other Information</h1> |
||||
|
||||
<p> |
||||
Details about the <code>leveldb</code> implementation may be found in |
||||
the following documents: |
||||
<ul> |
||||
<li> <a href="impl.html">Implementation notes</a> |
||||
<li> <a href="table_format.txt">Format of an immutable Table file</a> |
||||
<li> <a href="log_format.txt">Format of a log file</a> |
||||
</ul> |
||||
|
||||
</body> |
||||
</html> |
||||
@ -0,0 +1,523 @@
@@ -0,0 +1,523 @@
|
||||
leveldb |
||||
======= |
||||
|
||||
_Jeff Dean, Sanjay Ghemawat_ |
||||
|
||||
The leveldb library provides a persistent key value store. Keys and values are |
||||
arbitrary byte arrays. The keys are ordered within the key value store |
||||
according to a user-specified comparator function. |
||||
|
||||
## Opening A Database |
||||
|
||||
A leveldb database has a name which corresponds to a file system directory. All |
||||
of the contents of database are stored in this directory. The following example |
||||
shows how to open a database, creating it if necessary: |
||||
|
||||
```c++ |
||||
#include <cassert> |
||||
#include "leveldb/db.h" |
||||
|
||||
leveldb::DB* db; |
||||
leveldb::Options options; |
||||
options.create_if_missing = true; |
||||
leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db); |
||||
assert(status.ok()); |
||||
... |
||||
``` |
||||
|
||||
If you want to raise an error if the database already exists, add the following |
||||
line before the `leveldb::DB::Open` call: |
||||
|
||||
```c++ |
||||
options.error_if_exists = true; |
||||
``` |
||||
|
||||
## Status |
||||
|
||||
You may have noticed the `leveldb::Status` type above. Values of this type are |
||||
returned by most functions in leveldb that may encounter an error. You can check |
||||
if such a result is ok, and also print an associated error message: |
||||
|
||||
```c++ |
||||
leveldb::Status s = ...; |
||||
if (!s.ok()) cerr << s.ToString() << endl; |
||||
``` |
||||
|
||||
## Closing A Database |
||||
|
||||
When you are done with a database, just delete the database object. Example: |
||||
|
||||
```c++ |
||||
... open the db as described above ... |
||||
... do something with db ... |
||||
delete db; |
||||
``` |
||||
|
||||
## Reads And Writes |
||||
|
||||
The database provides Put, Delete, and Get methods to modify/query the database. |
||||
For example, the following code moves the value stored under key1 to key2. |
||||
|
||||
```c++ |
||||
std::string value; |
||||
leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value); |
||||
if (s.ok()) s = db->Put(leveldb::WriteOptions(), key2, value); |
||||
if (s.ok()) s = db->Delete(leveldb::WriteOptions(), key1); |
||||
``` |
||||
|
||||
## Atomic Updates |
||||
|
||||
Note that if the process dies after the Put of key2 but before the delete of |
||||
key1, the same value may be left stored under multiple keys. Such problems can |
||||
be avoided by using the `WriteBatch` class to atomically apply a set of updates: |
||||
|
||||
```c++ |
||||
#include "leveldb/write_batch.h" |
||||
... |
||||
std::string value; |
||||
leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value); |
||||
if (s.ok()) { |
||||
leveldb::WriteBatch batch; |
||||
batch.Delete(key1); |
||||
batch.Put(key2, value); |
||||
s = db->Write(leveldb::WriteOptions(), &batch); |
||||
} |
||||
``` |
||||
|
||||
The `WriteBatch` holds a sequence of edits to be made to the database, and these |
||||
edits within the batch are applied in order. Note that we called Delete before |
||||
Put so that if key1 is identical to key2, we do not end up erroneously dropping |
||||
the value entirely. |
||||
|
||||
Apart from its atomicity benefits, `WriteBatch` may also be used to speed up |
||||
bulk updates by placing lots of individual mutations into the same batch. |
||||
|
||||
## Synchronous Writes |
||||
|
||||
By default, each write to leveldb is asynchronous: it returns after pushing the |
||||
write from the process into the operating system. The transfer from operating |
||||
system memory to the underlying persistent storage happens asynchronously. The |
||||
sync flag can be turned on for a particular write to make the write operation |
||||
not return until the data being written has been pushed all the way to |
||||
persistent storage. (On Posix systems, this is implemented by calling either |
||||
`fsync(...)` or `fdatasync(...)` or `msync(..., MS_SYNC)` before the write |
||||
operation returns.) |
||||
|
||||
```c++ |
||||
leveldb::WriteOptions write_options; |
||||
write_options.sync = true; |
||||
db->Put(write_options, ...); |
||||
``` |
||||
|
||||
Asynchronous writes are often more than a thousand times as fast as synchronous |
||||
writes. The downside of asynchronous writes is that a crash of the machine may |
||||
cause the last few updates to be lost. Note that a crash of just the writing |
||||
process (i.e., not a reboot) will not cause any loss since even when sync is |
||||
false, an update is pushed from the process memory into the operating system |
||||
before it is considered done. |
||||
|
||||
Asynchronous writes can often be used safely. For example, when loading a large |
||||
amount of data into the database you can handle lost updates by restarting the |
||||
bulk load after a crash. A hybrid scheme is also possible where every Nth write |
||||
is synchronous, and in the event of a crash, the bulk load is restarted just |
||||
after the last synchronous write finished by the previous run. (The synchronous |
||||
write can update a marker that describes where to restart on a crash.) |
||||
|
||||
`WriteBatch` provides an alternative to asynchronous writes. Multiple updates |
||||
may be placed in the same WriteBatch and applied together using a synchronous |
||||
write (i.e., `write_options.sync` is set to true). The extra cost of the |
||||
synchronous write will be amortized across all of the writes in the batch. |
||||
|
||||
## Concurrency |
||||
|
||||
A database may only be opened by one process at a time. The leveldb |
||||
implementation acquires a lock from the operating system to prevent misuse. |
||||
Within a single process, the same `leveldb::DB` object may be safely shared by |
||||
multiple concurrent threads. I.e., different threads may write into or fetch |
||||
iterators or call Get on the same database without any external synchronization |
||||
(the leveldb implementation will automatically do the required synchronization). |
||||
However other objects (like Iterator and `WriteBatch`) may require external |
||||
synchronization. If two threads share such an object, they must protect access |
||||
to it using their own locking protocol. More details are available in the public |
||||
header files. |
||||
|
||||
## Iteration |
||||
|
||||
The following example demonstrates how to print all key,value pairs in a |
||||
database. |
||||
|
||||
```c++ |
||||
leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions()); |
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) { |
||||
cout << it->key().ToString() << ": " << it->value().ToString() << endl; |
||||
} |
||||
assert(it->status().ok()); // Check for any errors found during the scan |
||||
delete it; |
||||
``` |
||||
|
||||
The following variation shows how to process just the keys in the range |
||||
[start,limit): |
||||
|
||||
```c++ |
||||
for (it->Seek(start); |
||||
it->Valid() && it->key().ToString() < limit; |
||||
it->Next()) { |
||||
... |
||||
} |
||||
``` |
||||
|
||||
You can also process entries in reverse order. (Caveat: reverse iteration may be |
||||
somewhat slower than forward iteration.) |
||||
|
||||
```c++ |
||||
for (it->SeekToLast(); it->Valid(); it->Prev()) { |
||||
... |
||||
} |
||||
``` |
||||
|
||||
## Snapshots |
||||
|
||||
Snapshots provide consistent read-only views over the entire state of the |
||||
key-value store. `ReadOptions::snapshot` may be non-NULL to indicate that a |
||||
read should operate on a particular version of the DB state. If |
||||
`ReadOptions::snapshot` is NULL, the read will operate on an implicit snapshot |
||||
of the current state. |
||||
|
||||
Snapshots are created by the `DB::GetSnapshot()` method: |
||||
|
||||
```c++ |
||||
leveldb::ReadOptions options; |
||||
options.snapshot = db->GetSnapshot(); |
||||
... apply some updates to db ... |
||||
leveldb::Iterator* iter = db->NewIterator(options); |
||||
... read using iter to view the state when the snapshot was created ... |
||||
delete iter; |
||||
db->ReleaseSnapshot(options.snapshot); |
||||
``` |
||||
|
||||
Note that when a snapshot is no longer needed, it should be released using the |
||||
`DB::ReleaseSnapshot` interface. This allows the implementation to get rid of |
||||
state that was being maintained just to support reading as of that snapshot. |
||||
|
||||
## Slice |
||||
|
||||
The return value of the `it->key()` and `it->value()` calls above are instances |
||||
of the `leveldb::Slice` type. Slice is a simple structure that contains a length |
||||
and a pointer to an external byte array. Returning a Slice is a cheaper |
||||
alternative to returning a `std::string` since we do not need to copy |
||||
potentially large keys and values. In addition, leveldb methods do not return |
||||
null-terminated C-style strings since leveldb keys and values are allowed to |
||||
contain `'\0'` bytes. |
||||
|
||||
C++ strings and null-terminated C-style strings can be easily converted to a |
||||
Slice: |
||||
|
||||
```c++ |
||||
leveldb::Slice s1 = "hello"; |
||||
|
||||
std::string str("world"); |
||||
leveldb::Slice s2 = str; |
||||
``` |
||||
|
||||
A Slice can be easily converted back to a C++ string: |
||||
|
||||
```c++ |
||||
std::string str = s1.ToString(); |
||||
assert(str == std::string("hello")); |
||||
``` |
||||
|
||||
Be careful when using Slices since it is up to the caller to ensure that the |
||||
external byte array into which the Slice points remains live while the Slice is |
||||
in use. For example, the following is buggy: |
||||
|
||||
```c++ |
||||
leveldb::Slice slice; |
||||
if (...) { |
||||
std::string str = ...; |
||||
slice = str; |
||||
} |
||||
Use(slice); |
||||
``` |
||||
|
||||
When the if statement goes out of scope, str will be destroyed and the backing |
||||
storage for slice will disappear. |
||||
|
||||
## Comparators |
||||
|
||||
The preceding examples used the default ordering function for key, which orders |
||||
bytes lexicographically. You can however supply a custom comparator when opening |
||||
a database. For example, suppose each database key consists of two numbers and |
||||
we should sort by the first number, breaking ties by the second number. First, |
||||
define a proper subclass of `leveldb::Comparator` that expresses these rules: |
||||
|
||||
```c++ |
||||
class TwoPartComparator : public leveldb::Comparator { |
||||
public: |
||||
// Three-way comparison function: |
||||
// if a < b: negative result |
||||
// if a > b: positive result |
||||
// else: zero result |
||||
int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const { |
||||
int a1, a2, b1, b2; |
||||
ParseKey(a, &a1, &a2); |
||||
ParseKey(b, &b1, &b2); |
||||
if (a1 < b1) return -1; |
||||
if (a1 > b1) return +1; |
||||
if (a2 < b2) return -1; |
||||
if (a2 > b2) return +1; |
||||
return 0; |
||||
} |
||||
|
||||
// Ignore the following methods for now: |
||||
const char* Name() const { return "TwoPartComparator"; } |
||||
void FindShortestSeparator(std::string*, const leveldb::Slice&) const {} |
||||
void FindShortSuccessor(std::string*) const {} |
||||
}; |
||||
``` |
||||
|
||||
Now create a database using this custom comparator: |
||||
|
||||
```c++ |
||||
TwoPartComparator cmp; |
||||
leveldb::DB* db; |
||||
leveldb::Options options; |
||||
options.create_if_missing = true; |
||||
options.comparator = &cmp; |
||||
leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db); |
||||
... |
||||
``` |
||||
|
||||
### Backwards compatibility |
||||
|
||||
The result of the comparator's Name method is attached to the database when it |
||||
is created, and is checked on every subsequent database open. If the name |
||||
changes, the `leveldb::DB::Open` call will fail. Therefore, change the name if |
||||
and only if the new key format and comparison function are incompatible with |
||||
existing databases, and it is ok to discard the contents of all existing |
||||
databases. |
||||
|
||||
You can however still gradually evolve your key format over time with a little |
||||
bit of pre-planning. For example, you could store a version number at the end of |
||||
each key (one byte should suffice for most uses). When you wish to switch to a |
||||
new key format (e.g., adding an optional third part to the keys processed by |
||||
`TwoPartComparator`), (a) keep the same comparator name (b) increment the |
||||
version number for new keys (c) change the comparator function so it uses the |
||||
version numbers found in the keys to decide how to interpret them. |
||||
|
||||
## Performance |
||||
|
||||
Performance can be tuned by changing the default values of the types defined in |
||||
`include/leveldb/options.h`. |
||||
|
||||
### Block size |
||||
|
||||
leveldb groups adjacent keys together into the same block and such a block is |
||||
the unit of transfer to and from persistent storage. The default block size is |
||||
approximately 4096 uncompressed bytes. Applications that mostly do bulk scans |
||||
over the contents of the database may wish to increase this size. Applications |
||||
that do a lot of point reads of small values may wish to switch to a smaller |
||||
block size if performance measurements indicate an improvement. There isn't much |
||||
benefit in using blocks smaller than one kilobyte, or larger than a few |
||||
megabytes. Also note that compression will be more effective with larger block |
||||
sizes. |
||||
|
||||
### Compression |
||||
|
||||
Each block is individually compressed before being written to persistent |
||||
storage. Compression is on by default since the default compression method is |
||||
very fast, and is automatically disabled for uncompressible data. In rare cases, |
||||
applications may want to disable compression entirely, but should only do so if |
||||
benchmarks show a performance improvement: |
||||
|
||||
```c++ |
||||
leveldb::Options options; |
||||
options.compression = leveldb::kNoCompression; |
||||
... leveldb::DB::Open(options, name, ...) .... |
||||
``` |
||||
|
||||
### Cache |
||||
|
||||
The contents of the database are stored in a set of files in the filesystem and |
||||
each file stores a sequence of compressed blocks. If options.cache is non-NULL, |
||||
it is used to cache frequently used uncompressed block contents. |
||||
|
||||
```c++ |
||||
#include "leveldb/cache.h" |
||||
|
||||
leveldb::Options options; |
||||
options.cache = leveldb::NewLRUCache(100 * 1048576); // 100MB cache |
||||
leveldb::DB* db; |
||||
leveldb::DB::Open(options, name, &db); |
||||
... use the db ... |
||||
delete db |
||||
delete options.cache; |
||||
``` |
||||
|
||||
Note that the cache holds uncompressed data, and therefore it should be sized |
||||
according to application level data sizes, without any reduction from |
||||
compression. (Caching of compressed blocks is left to the operating system |
||||
buffer cache, or any custom Env implementation provided by the client.) |
||||
|
||||
When performing a bulk read, the application may wish to disable caching so that |
||||
the data processed by the bulk read does not end up displacing most of the |
||||
cached contents. A per-iterator option can be used to achieve this: |
||||
|
||||
```c++ |
||||
leveldb::ReadOptions options; |
||||
options.fill_cache = false; |
||||
leveldb::Iterator* it = db->NewIterator(options); |
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) { |
||||
... |
||||
} |
||||
``` |
||||
|
||||
### Key Layout |
||||
|
||||
Note that the unit of disk transfer and caching is a block. Adjacent keys |
||||
(according to the database sort order) will usually be placed in the same block. |
||||
Therefore the application can improve its performance by placing keys that are |
||||
accessed together near each other and placing infrequently used keys in a |
||||
separate region of the key space. |
||||
|
||||
For example, suppose we are implementing a simple file system on top of leveldb. |
||||
The types of entries we might wish to store are: |
||||
|
||||
filename -> permission-bits, length, list of file_block_ids |
||||
file_block_id -> data |
||||
|
||||
We might want to prefix filename keys with one letter (say '/') and the |
||||
`file_block_id` keys with a different letter (say '0') so that scans over just |
||||
the metadata do not force us to fetch and cache bulky file contents. |
||||
|
||||
### Filters |
||||
|
||||
Because of the way leveldb data is organized on disk, a single `Get()` call may |
||||
involve multiple reads from disk. The optional FilterPolicy mechanism can be |
||||
used to reduce the number of disk reads substantially. |
||||
|
||||
```c++ |
||||
leveldb::Options options; |
||||
options.filter_policy = NewBloomFilterPolicy(10); |
||||
leveldb::DB* db; |
||||
leveldb::DB::Open(options, "/tmp/testdb", &db); |
||||
... use the database ... |
||||
delete db; |
||||
delete options.filter_policy; |
||||
``` |
||||
|
||||
The preceding code associates a Bloom filter based filtering policy with the |
||||
database. Bloom filter based filtering relies on keeping some number of bits of |
||||
data in memory per key (in this case 10 bits per key since that is the argument |
||||
we passed to `NewBloomFilterPolicy`). This filter will reduce the number of |
||||
unnecessary disk reads needed for Get() calls by a factor of approximately |
||||
a 100. Increasing the bits per key will lead to a larger reduction at the cost |
||||
of more memory usage. We recommend that applications whose working set does not |
||||
fit in memory and that do a lot of random reads set a filter policy. |
||||
|
||||
If you are using a custom comparator, you should ensure that the filter policy |
||||
you are using is compatible with your comparator. For example, consider a |
||||
comparator that ignores trailing spaces when comparing keys. |
||||
`NewBloomFilterPolicy` must not be used with such a comparator. Instead, the |
||||
application should provide a custom filter policy that also ignores trailing |
||||
spaces. For example: |
||||
|
||||
```c++ |
||||
class CustomFilterPolicy : public leveldb::FilterPolicy { |
||||
private: |
||||
FilterPolicy* builtin_policy_; |
||||
|
||||
public: |
||||
CustomFilterPolicy() : builtin_policy_(NewBloomFilterPolicy(10)) {} |
||||
~CustomFilterPolicy() { delete builtin_policy_; } |
||||
|
||||
const char* Name() const { return "IgnoreTrailingSpacesFilter"; } |
||||
|
||||
void CreateFilter(const Slice* keys, int n, std::string* dst) const { |
||||
// Use builtin bloom filter code after removing trailing spaces |
||||
std::vector<Slice> trimmed(n); |
||||
for (int i = 0; i < n; i++) { |
||||
trimmed[i] = RemoveTrailingSpaces(keys[i]); |
||||
} |
||||
return builtin_policy_->CreateFilter(&trimmed[i], n, dst); |
||||
} |
||||
}; |
||||
``` |
||||
|
||||
Advanced applications may provide a filter policy that does not use a bloom |
||||
filter but uses some other mechanism for summarizing a set of keys. See |
||||
`leveldb/filter_policy.h` for detail. |
||||
|
||||
## Checksums |
||||
|
||||
leveldb associates checksums with all data it stores in the file system. There |
||||
are two separate controls provided over how aggressively these checksums are |
||||
verified: |
||||
|
||||
`ReadOptions::verify_checksums` may be set to true to force checksum |
||||
verification of all data that is read from the file system on behalf of a |
||||
particular read. By default, no such verification is done. |
||||
|
||||
`Options::paranoid_checks` may be set to true before opening a database to make |
||||
the database implementation raise an error as soon as it detects an internal |
||||
corruption. Depending on which portion of the database has been corrupted, the |
||||
error may be raised when the database is opened, or later by another database |
||||
operation. By default, paranoid checking is off so that the database can be used |
||||
even if parts of its persistent storage have been corrupted. |
||||
|
||||
If a database is corrupted (perhaps it cannot be opened when paranoid checking |
||||
is turned on), the `leveldb::RepairDB` function may be used to recover as much |
||||
of the data as possible |
||||
|
||||
## Approximate Sizes |
||||
|
||||
The `GetApproximateSizes` method can used to get the approximate number of bytes |
||||
of file system space used by one or more key ranges. |
||||
|
||||
```c++ |
||||
leveldb::Range ranges[2]; |
||||
ranges[0] = leveldb::Range("a", "c"); |
||||
ranges[1] = leveldb::Range("x", "z"); |
||||
uint64_t sizes[2]; |
||||
leveldb::Status s = db->GetApproximateSizes(ranges, 2, sizes); |
||||
``` |
||||
|
||||
The preceding call will set `sizes[0]` to the approximate number of bytes of |
||||
file system space used by the key range `[a..c)` and `sizes[1]` to the |
||||
approximate number of bytes used by the key range `[x..z)`. |
||||
|
||||
## Environment |
||||
|
||||
All file operations (and other operating system calls) issued by the leveldb |
||||
implementation are routed through a `leveldb::Env` object. Sophisticated clients |
||||
may wish to provide their own Env implementation to get better control. |
||||
For example, an application may introduce artificial delays in the file IO |
||||
paths to limit the impact of leveldb on other activities in the system. |
||||
|
||||
```c++ |
||||
class SlowEnv : public leveldb::Env { |
||||
... implementation of the Env interface ... |
||||
}; |
||||
|
||||
SlowEnv env; |
||||
leveldb::Options options; |
||||
options.env = &env; |
||||
Status s = leveldb::DB::Open(options, ...); |
||||
``` |
||||
|
||||
## Porting |
||||
|
||||
leveldb may be ported to a new platform by providing platform specific |
||||
implementations of the types/methods/functions exported by |
||||
`leveldb/port/port.h`. See `leveldb/port/port_example.h` for more details. |
||||
|
||||
In addition, the new platform may need a new default `leveldb::Env` |
||||
implementation. See `leveldb/util/env_posix.h` for an example. |
||||
|
||||
## Other Information |
||||
|
||||
Details about the leveldb implementation may be found in the following |
||||
documents: |
||||
|
||||
1. [Implementation notes](impl.md) |
||||
2. [Format of an immutable Table file](table_format.md) |
||||
3. [Format of a log file](log_format.md) |
||||
@ -0,0 +1,75 @@
@@ -0,0 +1,75 @@
|
||||
leveldb Log format |
||||
================== |
||||
The log file contents are a sequence of 32KB blocks. The only exception is that |
||||
the tail of the file may contain a partial block. |
||||
|
||||
Each block consists of a sequence of records: |
||||
|
||||
block := record* trailer? |
||||
record := |
||||
checksum: uint32 // crc32c of type and data[] ; little-endian |
||||
length: uint16 // little-endian |
||||
type: uint8 // One of FULL, FIRST, MIDDLE, LAST |
||||
data: uint8[length] |
||||
|
||||
A record never starts within the last six bytes of a block (since it won't fit). |
||||
Any leftover bytes here form the trailer, which must consist entirely of zero |
||||
bytes and must be skipped by readers. |
||||
|
||||
Aside: if exactly seven bytes are left in the current block, and a new non-zero |
||||
length record is added, the writer must emit a FIRST record (which contains zero |
||||
bytes of user data) to fill up the trailing seven bytes of the block and then |
||||
emit all of the user data in subsequent blocks. |
||||
|
||||
More types may be added in the future. Some Readers may skip record types they |
||||
do not understand, others may report that some data was skipped. |
||||
|
||||
FULL == 1 |
||||
FIRST == 2 |
||||
MIDDLE == 3 |
||||
LAST == 4 |
||||
|
||||
The FULL record contains the contents of an entire user record. |
||||
|
||||
FIRST, MIDDLE, LAST are types used for user records that have been split into |
||||
multiple fragments (typically because of block boundaries). FIRST is the type |
||||
of the first fragment of a user record, LAST is the type of the last fragment of |
||||
a user record, and MIDDLE is the type of all interior fragments of a user |
||||
record. |
||||
|
||||
Example: consider a sequence of user records: |
||||
|
||||
A: length 1000 |
||||
B: length 97270 |
||||
C: length 8000 |
||||
|
||||
**A** will be stored as a FULL record in the first block. |
||||
|
||||
**B** will be split into three fragments: first fragment occupies the rest of |
||||
the first block, second fragment occupies the entirety of the second block, and |
||||
the third fragment occupies a prefix of the third block. This will leave six |
||||
bytes free in the third block, which will be left empty as the trailer. |
||||
|
||||
**C** will be stored as a FULL record in the fourth block. |
||||
|
||||
---- |
||||
|
||||
## Some benefits over the recordio format: |
||||
|
||||
1. We do not need any heuristics for resyncing - just go to next block boundary |
||||
and scan. If there is a corruption, skip to the next block. As a |
||||
side-benefit, we do not get confused when part of the contents of one log |
||||
file are embedded as a record inside another log file. |
||||
|
||||
2. Splitting at approximate boundaries (e.g., for mapreduce) is simple: find the |
||||
next block boundary and skip records until we hit a FULL or FIRST record. |
||||
|
||||
3. We do not need extra buffering for large records. |
||||
|
||||
## Some downsides compared to recordio format: |
||||
|
||||
1. No packing of tiny records. This could be fixed by adding a new record type, |
||||
so it is a shortcoming of the current implementation, not necessarily the |
||||
format. |
||||
|
||||
2. No compression. Again, this could be fixed by adding new record types. |
||||
@ -1,75 +0,0 @@
@@ -1,75 +0,0 @@
|
||||
The log file contents are a sequence of 32KB blocks. The only |
||||
exception is that the tail of the file may contain a partial block. |
||||
|
||||
Each block consists of a sequence of records: |
||||
block := record* trailer? |
||||
record := |
||||
checksum: uint32 // crc32c of type and data[] ; little-endian |
||||
length: uint16 // little-endian |
||||
type: uint8 // One of FULL, FIRST, MIDDLE, LAST |
||||
data: uint8[length] |
||||
|
||||
A record never starts within the last six bytes of a block (since it |
||||
won't fit). Any leftover bytes here form the trailer, which must |
||||
consist entirely of zero bytes and must be skipped by readers. |
||||
|
||||
Aside: if exactly seven bytes are left in the current block, and a new |
||||
non-zero length record is added, the writer must emit a FIRST record |
||||
(which contains zero bytes of user data) to fill up the trailing seven |
||||
bytes of the block and then emit all of the user data in subsequent |
||||
blocks. |
||||
|
||||
More types may be added in the future. Some Readers may skip record |
||||
types they do not understand, others may report that some data was |
||||
skipped. |
||||
|
||||
FULL == 1 |
||||
FIRST == 2 |
||||
MIDDLE == 3 |
||||
LAST == 4 |
||||
|
||||
The FULL record contains the contents of an entire user record. |
||||
|
||||
FIRST, MIDDLE, LAST are types used for user records that have been |
||||
split into multiple fragments (typically because of block boundaries). |
||||
FIRST is the type of the first fragment of a user record, LAST is the |
||||
type of the last fragment of a user record, and MIDDLE is the type of |
||||
all interior fragments of a user record. |
||||
|
||||
Example: consider a sequence of user records: |
||||
A: length 1000 |
||||
B: length 97270 |
||||
C: length 8000 |
||||
A will be stored as a FULL record in the first block. |
||||
|
||||
B will be split into three fragments: first fragment occupies the rest |
||||
of the first block, second fragment occupies the entirety of the |
||||
second block, and the third fragment occupies a prefix of the third |
||||
block. This will leave six bytes free in the third block, which will |
||||
be left empty as the trailer. |
||||
|
||||
C will be stored as a FULL record in the fourth block. |
||||
|
||||
=================== |
||||
|
||||
Some benefits over the recordio format: |
||||
|
||||
(1) We do not need any heuristics for resyncing - just go to next |
||||
block boundary and scan. If there is a corruption, skip to the next |
||||
block. As a side-benefit, we do not get confused when part of the |
||||
contents of one log file are embedded as a record inside another log |
||||
file. |
||||
|
||||
(2) Splitting at approximate boundaries (e.g., for mapreduce) is |
||||
simple: find the next block boundary and skip records until we |
||||
hit a FULL or FIRST record. |
||||
|
||||
(3) We do not need extra buffering for large records. |
||||
|
||||
Some downsides compared to recordio format: |
||||
|
||||
(1) No packing of tiny records. This could be fixed by adding a new |
||||
record type, so it is a shortcoming of the current implementation, |
||||
not necessarily the format. |
||||
|
||||
(2) No compression. Again, this could be fixed by adding new record types. |
||||
@ -0,0 +1,107 @@
@@ -0,0 +1,107 @@
|
||||
leveldb File format |
||||
=================== |
||||
|
||||
<beginning_of_file> |
||||
[data block 1] |
||||
[data block 2] |
||||
... |
||||
[data block N] |
||||
[meta block 1] |
||||
... |
||||
[meta block K] |
||||
[metaindex block] |
||||
[index block] |
||||
[Footer] (fixed size; starts at file_size - sizeof(Footer)) |
||||
<end_of_file> |
||||
|
||||
The file contains internal pointers. Each such pointer is called |
||||
a BlockHandle and contains the following information: |
||||
|
||||
offset: varint64 |
||||
size: varint64 |
||||
|
||||
See [varints](https://developers.google.com/protocol-buffers/docs/encoding#varints) |
||||
for an explanation of varint64 format. |
||||
|
||||
1. The sequence of key/value pairs in the file are stored in sorted |
||||
order and partitioned into a sequence of data blocks. These blocks |
||||
come one after another at the beginning of the file. Each data block |
||||
is formatted according to the code in `block_builder.cc`, and then |
||||
optionally compressed. |
||||
|
||||
2. After the data blocks we store a bunch of meta blocks. The |
||||
supported meta block types are described below. More meta block types |
||||
may be added in the future. Each meta block is again formatted using |
||||
`block_builder.cc` and then optionally compressed. |
||||
|
||||
3. A "metaindex" block. It contains one entry for every other meta |
||||
block where the key is the name of the meta block and the value is a |
||||
BlockHandle pointing to that meta block. |
||||
|
||||
4. An "index" block. This block contains one entry per data block, |
||||
where the key is a string >= last key in that data block and before |
||||
the first key in the successive data block. The value is the |
||||
BlockHandle for the data block. |
||||
|
||||
5. At the very end of the file is a fixed length footer that contains |
||||
the BlockHandle of the metaindex and index blocks as well as a magic number. |
||||
|
||||
metaindex_handle: char[p]; // Block handle for metaindex |
||||
index_handle: char[q]; // Block handle for index |
||||
padding: char[40-p-q];// zeroed bytes to make fixed length |
||||
// (40==2*BlockHandle::kMaxEncodedLength) |
||||
magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) |
||||
|
||||
## "filter" Meta Block |
||||
|
||||
If a `FilterPolicy` was specified when the database was opened, a |
||||
filter block is stored in each table. The "metaindex" block contains |
||||
an entry that maps from `filter.<N>` to the BlockHandle for the filter |
||||
block where `<N>` is the string returned by the filter policy's |
||||
`Name()` method. |
||||
|
||||
The filter block stores a sequence of filters, where filter i contains |
||||
the output of `FilterPolicy::CreateFilter()` on all keys that are stored |
||||
in a block whose file offset falls within the range |
||||
|
||||
[ i*base ... (i+1)*base-1 ] |
||||
|
||||
Currently, "base" is 2KB. So for example, if blocks X and Y start in |
||||
the range `[ 0KB .. 2KB-1 ]`, all of the keys in X and Y will be |
||||
converted to a filter by calling `FilterPolicy::CreateFilter()`, and the |
||||
resulting filter will be stored as the first filter in the filter |
||||
block. |
||||
|
||||
The filter block is formatted as follows: |
||||
|
||||
[filter 0] |
||||
[filter 1] |
||||
[filter 2] |
||||
... |
||||
[filter N-1] |
||||
|
||||
[offset of filter 0] : 4 bytes |
||||
[offset of filter 1] : 4 bytes |
||||
[offset of filter 2] : 4 bytes |
||||
... |
||||
[offset of filter N-1] : 4 bytes |
||||
|
||||
[offset of beginning of offset array] : 4 bytes |
||||
lg(base) : 1 byte |
||||
|
||||
The offset array at the end of the filter block allows efficient |
||||
mapping from a data block offset to the corresponding filter. |
||||
|
||||
## "stats" Meta Block |
||||
|
||||
This meta block contains a bunch of stats. The key is the name |
||||
of the statistic. The value contains the statistic. |
||||
|
||||
TODO(postrelease): record following stats. |
||||
|
||||
data size |
||||
index size |
||||
key size (uncompressed) |
||||
value size (uncompressed) |
||||
number of entries |
||||
number of data blocks |
||||
@ -1,104 +0,0 @@
@@ -1,104 +0,0 @@
|
||||
File format |
||||
=========== |
||||
|
||||
<beginning_of_file> |
||||
[data block 1] |
||||
[data block 2] |
||||
... |
||||
[data block N] |
||||
[meta block 1] |
||||
... |
||||
[meta block K] |
||||
[metaindex block] |
||||
[index block] |
||||
[Footer] (fixed size; starts at file_size - sizeof(Footer)) |
||||
<end_of_file> |
||||
|
||||
The file contains internal pointers. Each such pointer is called |
||||
a BlockHandle and contains the following information: |
||||
offset: varint64 |
||||
size: varint64 |
||||
See https://developers.google.com/protocol-buffers/docs/encoding#varints |
||||
for an explanation of varint64 format. |
||||
|
||||
(1) The sequence of key/value pairs in the file are stored in sorted |
||||
order and partitioned into a sequence of data blocks. These blocks |
||||
come one after another at the beginning of the file. Each data block |
||||
is formatted according to the code in block_builder.cc, and then |
||||
optionally compressed. |
||||
|
||||
(2) After the data blocks we store a bunch of meta blocks. The |
||||
supported meta block types are described below. More meta block types |
||||
may be added in the future. Each meta block is again formatted using |
||||
block_builder.cc and then optionally compressed. |
||||
|
||||
(3) A "metaindex" block. It contains one entry for every other meta |
||||
block where the key is the name of the meta block and the value is a |
||||
BlockHandle pointing to that meta block. |
||||
|
||||
(4) An "index" block. This block contains one entry per data block, |
||||
where the key is a string >= last key in that data block and before |
||||
the first key in the successive data block. The value is the |
||||
BlockHandle for the data block. |
||||
|
||||
(6) At the very end of the file is a fixed length footer that contains |
||||
the BlockHandle of the metaindex and index blocks as well as a magic number. |
||||
metaindex_handle: char[p]; // Block handle for metaindex |
||||
index_handle: char[q]; // Block handle for index |
||||
padding: char[40-p-q]; // zeroed bytes to make fixed length |
||||
// (40==2*BlockHandle::kMaxEncodedLength) |
||||
magic: fixed64; // == 0xdb4775248b80fb57 (little-endian) |
||||
|
||||
"filter" Meta Block |
||||
------------------- |
||||
|
||||
If a "FilterPolicy" was specified when the database was opened, a |
||||
filter block is stored in each table. The "metaindex" block contains |
||||
an entry that maps from "filter.<N>" to the BlockHandle for the filter |
||||
block where "<N>" is the string returned by the filter policy's |
||||
"Name()" method. |
||||
|
||||
The filter block stores a sequence of filters, where filter i contains |
||||
the output of FilterPolicy::CreateFilter() on all keys that are stored |
||||
in a block whose file offset falls within the range |
||||
|
||||
[ i*base ... (i+1)*base-1 ] |
||||
|
||||
Currently, "base" is 2KB. So for example, if blocks X and Y start in |
||||
the range [ 0KB .. 2KB-1 ], all of the keys in X and Y will be |
||||
converted to a filter by calling FilterPolicy::CreateFilter(), and the |
||||
resulting filter will be stored as the first filter in the filter |
||||
block. |
||||
|
||||
The filter block is formatted as follows: |
||||
|
||||
[filter 0] |
||||
[filter 1] |
||||
[filter 2] |
||||
... |
||||
[filter N-1] |
||||
|
||||
[offset of filter 0] : 4 bytes |
||||
[offset of filter 1] : 4 bytes |
||||
[offset of filter 2] : 4 bytes |
||||
... |
||||
[offset of filter N-1] : 4 bytes |
||||
|
||||
[offset of beginning of offset array] : 4 bytes |
||||
lg(base) : 1 byte |
||||
|
||||
The offset array at the end of the filter block allows efficient |
||||
mapping from a data block offset to the corresponding filter. |
||||
|
||||
"stats" Meta Block |
||||
------------------ |
||||
|
||||
This meta block contains a bunch of stats. The key is the name |
||||
of the statistic. The value contains the statistic. |
||||
TODO(postrelease): record following stats. |
||||
data size |
||||
index size |
||||
key size (uncompressed) |
||||
value size (uncompressed) |
||||
number of entries |
||||
number of data blocks |
||||
@ -0,0 +1,129 @@
@@ -0,0 +1,129 @@
|
||||
// Copyright 2016 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// A portable implementation of crc32c, optimized to handle
|
||||
// four bytes at a time.
|
||||
//
|
||||
// In a separate source file to allow this accelerated CRC32C function to be
|
||||
// compiled with the appropriate compiler flags to enable x86 SSE 4.2
|
||||
// instructions.
|
||||
|
||||
#include <stdint.h> |
||||
#include <string.h> |
||||
#include "port/port.h" |
||||
|
||||
#if defined(LEVELDB_PLATFORM_POSIX_SSE) |
||||
|
||||
#if defined(_MSC_VER) |
||||
#include <intrin.h> |
||||
#elif defined(__GNUC__) && defined(__SSE4_2__) |
||||
#include <nmmintrin.h> |
||||
#include <cpuid.h> |
||||
#endif |
||||
|
||||
#endif // defined(LEVELDB_PLATFORM_POSIX_SSE)
|
||||
|
||||
namespace leveldb { |
||||
namespace port { |
||||
|
||||
#if defined(LEVELDB_PLATFORM_POSIX_SSE) |
||||
|
||||
// Used to fetch a naturally-aligned 32-bit word in little endian byte-order
|
||||
static inline uint32_t LE_LOAD32(const uint8_t *p) { |
||||
// SSE is x86 only, so ensured that |p| is always little-endian.
|
||||
uint32_t word; |
||||
memcpy(&word, p, sizeof(word)); |
||||
return word; |
||||
} |
||||
|
||||
#if defined(_M_X64) || defined(__x86_64__) // LE_LOAD64 is only used on x64.
|
||||
|
||||
// Used to fetch a naturally-aligned 64-bit word in little endian byte-order
|
||||
static inline uint64_t LE_LOAD64(const uint8_t *p) { |
||||
uint64_t dword; |
||||
memcpy(&dword, p, sizeof(dword)); |
||||
return dword; |
||||
} |
||||
|
||||
#endif // defined(_M_X64) || defined(__x86_64__)
|
||||
|
||||
static inline bool HaveSSE42() { |
||||
#if defined(_MSC_VER) |
||||
int cpu_info[4]; |
||||
__cpuid(cpu_info, 1); |
||||
return (cpu_info[2] & (1 << 20)) != 0; |
||||
#elif defined(__GNUC__) |
||||
unsigned int eax, ebx, ecx, edx; |
||||
__get_cpuid(1, &eax, &ebx, &ecx, &edx); |
||||
return (ecx & (1 << 20)) != 0; |
||||
#else |
||||
return false; |
||||
#endif |
||||
} |
||||
|
||||
#endif // defined(LEVELDB_PLATFORM_POSIX_SSE)
|
||||
|
||||
// For further improvements see Intel publication at:
|
||||
// http://download.intel.com/design/intarch/papers/323405.pdf
|
||||
uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size) { |
||||
#if !defined(LEVELDB_PLATFORM_POSIX_SSE) |
||||
return 0; |
||||
#else |
||||
static bool have = HaveSSE42(); |
||||
if (!have) { |
||||
return 0; |
||||
} |
||||
|
||||
const uint8_t *p = reinterpret_cast<const uint8_t *>(buf); |
||||
const uint8_t *e = p + size; |
||||
uint32_t l = crc ^ 0xffffffffu; |
||||
|
||||
#define STEP1 do { \ |
||||
l = _mm_crc32_u8(l, *p++); \ |
||||
} while (0) |
||||
#define STEP4 do { \ |
||||
l = _mm_crc32_u32(l, LE_LOAD32(p)); \ |
||||
p += 4; \ |
||||
} while (0) |
||||
#define STEP8 do { \ |
||||
l = _mm_crc32_u64(l, LE_LOAD64(p)); \ |
||||
p += 8; \ |
||||
} while (0) |
||||
|
||||
if (size > 16) { |
||||
// Process unaligned bytes
|
||||
for (unsigned int i = reinterpret_cast<uintptr_t>(p) % 8; i; --i) { |
||||
STEP1; |
||||
} |
||||
|
||||
// _mm_crc32_u64 is only available on x64.
|
||||
#if defined(_M_X64) || defined(__x86_64__) |
||||
// Process 8 bytes at a time
|
||||
while ((e-p) >= 8) { |
||||
STEP8; |
||||
} |
||||
// Process 4 bytes at a time
|
||||
if ((e-p) >= 4) { |
||||
STEP4; |
||||
} |
||||
#else // !(defined(_M_X64) || defined(__x86_64__))
|
||||
// Process 4 bytes at a time
|
||||
while ((e-p) >= 4) { |
||||
STEP4; |
||||
} |
||||
#endif // defined(_M_X64) || defined(__x86_64__)
|
||||
} |
||||
// Process the last few bytes
|
||||
while (p != e) { |
||||
STEP1; |
||||
} |
||||
#undef STEP8 |
||||
#undef STEP4 |
||||
#undef STEP1 |
||||
return l ^ 0xffffffffu; |
||||
#endif // defined(LEVELDB_PLATFORM_POSIX_SSE)
|
||||
} |
||||
|
||||
} // namespace port
|
||||
} // namespace leveldb
|
||||
@ -0,0 +1,66 @@
@@ -0,0 +1,66 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "leveldb/env.h" |
||||
|
||||
#include "port/port.h" |
||||
#include "util/testharness.h" |
||||
#include "util/env_posix_test_helper.h" |
||||
|
||||
namespace leveldb { |
||||
|
||||
static const int kDelayMicros = 100000; |
||||
static const int kReadOnlyFileLimit = 4; |
||||
static const int kMMapLimit = 4; |
||||
|
||||
class EnvPosixTest { |
||||
public: |
||||
Env* env_; |
||||
EnvPosixTest() : env_(Env::Default()) { } |
||||
|
||||
static void SetFileLimits(int read_only_file_limit, int mmap_limit) { |
||||
EnvPosixTestHelper::SetReadOnlyFDLimit(read_only_file_limit); |
||||
EnvPosixTestHelper::SetReadOnlyMMapLimit(mmap_limit); |
||||
} |
||||
}; |
||||
|
||||
TEST(EnvPosixTest, TestOpenOnRead) { |
||||
// Write some test data to a single file that will be opened |n| times.
|
||||
std::string test_dir; |
||||
ASSERT_OK(env_->GetTestDirectory(&test_dir)); |
||||
std::string test_file = test_dir + "/open_on_read.txt"; |
||||
|
||||
FILE* f = fopen(test_file.c_str(), "w"); |
||||
ASSERT_TRUE(f != NULL); |
||||
const char kFileData[] = "abcdefghijklmnopqrstuvwxyz"; |
||||
fputs(kFileData, f); |
||||
fclose(f); |
||||
|
||||
// Open test file some number above the sum of the two limits to force
|
||||
// open-on-read behavior of POSIX Env leveldb::RandomAccessFile.
|
||||
const int kNumFiles = kReadOnlyFileLimit + kMMapLimit + 5; |
||||
leveldb::RandomAccessFile* files[kNumFiles] = {0}; |
||||
for (int i = 0; i < kNumFiles; i++) { |
||||
ASSERT_OK(env_->NewRandomAccessFile(test_file, &files[i])); |
||||
} |
||||
char scratch; |
||||
Slice read_result; |
||||
for (int i = 0; i < kNumFiles; i++) { |
||||
ASSERT_OK(files[i]->Read(i, 1, &read_result, &scratch)); |
||||
ASSERT_EQ(kFileData[i], read_result[0]); |
||||
} |
||||
for (int i = 0; i < kNumFiles; i++) { |
||||
delete files[i]; |
||||
} |
||||
ASSERT_OK(env_->DeleteFile(test_file)); |
||||
} |
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
// All tests currently run with the same read-only file limits.
|
||||
leveldb::EnvPosixTest::SetFileLimits(leveldb::kReadOnlyFileLimit, |
||||
leveldb::kMMapLimit); |
||||
return leveldb::test::RunAllTests(); |
||||
} |
||||
@ -0,0 +1,28 @@
@@ -0,0 +1,28 @@
|
||||
// Copyright 2017 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ |
||||
#define STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_ |
||||
|
||||
namespace leveldb { |
||||
|
||||
class EnvPosixTest; |
||||
|
||||
// A helper for the POSIX Env to facilitate testing.
|
||||
class EnvPosixTestHelper { |
||||
private: |
||||
friend class EnvPosixTest; |
||||
|
||||
// Set the maximum number of read-only files that will be opened.
|
||||
// Must be called before creating an Env.
|
||||
static void SetReadOnlyFDLimit(int limit); |
||||
|
||||
// Set the maximum number of read-only files that will be mapped via mmap.
|
||||
// Must be called before creating an Env.
|
||||
static void SetReadOnlyMMapLimit(int limit); |
||||
}; |
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_UTIL_ENV_POSIX_TEST_HELPER_H_
|
||||
Loading…
Reference in new issue