Browse Source

Squashed 'src/leveldb/' changes from 7924331..7d41e6f

7d41e6f Merge upstream LevelDB 1.18
803d692 Release 1.18

git-subtree-dir: src/leveldb
git-subtree-split: 7d41e6f89ff04ce9e6a742932924796f69c6e23d
0.15
Pieter Wuille 10 years ago
parent
commit
4b0e2d75d7
  1. 36
      CONTRIBUTING.md
  2. 28
      Makefile
  3. 138
      README.md
  4. 15
      build_detect_platform
  5. 3
      db/db_bench.cc
  6. 4
      db/db_impl.cc
  7. 2
      db/db_test.cc
  8. 6
      db/dbformat.h
  9. 225
      db/dumpfile.cc
  10. 204
      db/leveldb_main.cc
  11. 4
      db/log_format.h
  12. 6
      db/log_reader.cc
  13. 4
      db/log_reader.h
  14. 2
      db/log_test.cc
  15. 2
      db/repair.cc
  16. 5
      db/skiplist.h
  17. 4
      db/write_batch_internal.h
  18. 2
      doc/bench/db_bench_tree_db.cc
  19. 4
      doc/impl.html
  20. 4
      doc/log_format.txt
  21. 9
      helpers/memenv/memenv.cc
  22. 2
      include/leveldb/cache.h
  23. 2
      include/leveldb/db.h
  24. 25
      include/leveldb/dumpfile.h
  25. 2
      include/leveldb/env.h
  26. 2
      include/leveldb/iterator.h
  27. 2
      include/leveldb/options.h
  28. 21
      port/atomic_pointer.h
  29. 9
      port/port_posix.h
  30. 5
      port/thread_annotations.h
  31. 2
      table/block.cc
  32. 2
      table/block_builder.h
  33. 2
      table/format.cc
  34. 14
      table/table.cc
  35. 2
      util/bloom.cc
  36. 10
      util/env_posix.cc
  37. 6
      util/hash.cc
  38. 54
      util/hash_test.cc
  39. 9
      util/logging.cc
  40. 4
      util/logging.h

36
CONTRIBUTING.md

@ -0,0 +1,36 @@
# Contributing
We'd love to accept your code patches! However, before we can take them, we
have to jump a couple of legal hurdles.
## Contributor License Agreements
Please fill out either the individual or corporate Contributor License
Agreement as appropriate.
* If you are an individual writing original source code and you're sure you
own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual).
* If you work for a company that wants to allow you to contribute your work,
then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate).
Follow either of the two links above to access the appropriate CLA and
instructions for how to sign and return it.
## Submitting a Patch
1. Sign the contributors license agreement above.
2. Decide which code you want to submit. A submission should be a set of changes
that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues).
Please don't mix more than one logical change per submission, because it makes
the history hard to follow. If you want to make a change
(e.g. add a sample or feature) that doesn't have a corresponding issue in the
issue tracker, please create one.
3. **Submitting**: When you are ready to submit, send us a Pull Request. Be
sure to include the issue number you fixed and the name you used to sign
the CLA.
## Writing Code ##
If your contribution contains code, please make sure that it follows
[the style guide](http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml).
Otherwise we will have to ask you to make changes, and that's no fun for anyone.

28
Makefile

@ -6,9 +6,12 @@
# Uncomment exactly one of the lines labelled (A), (B), and (C) below # Uncomment exactly one of the lines labelled (A), (B), and (C) below
# to switch between compilation modes. # to switch between compilation modes.
OPT ?= -O2 -DNDEBUG # (A) Production use (optimized mode) # (A) Production use (optimized mode)
# OPT ?= -g2 # (B) Debug mode, w/ full line-level debugging symbols OPT ?= -O2 -DNDEBUG
# OPT ?= -O2 -g2 -DNDEBUG # (C) Profiling mode: opt, but w/debugging symbols # (B) Debug mode, w/ full line-level debugging symbols
# OPT ?= -g2
# (C) Profiling mode: opt, but w/debugging symbols
# OPT ?= -O2 -g2 -DNDEBUG
#----------------------------------------------- #-----------------------------------------------
# detect what platform we're building on # detect what platform we're building on
@ -29,6 +32,11 @@ MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
TESTUTIL = ./util/testutil.o TESTUTIL = ./util/testutil.o
TESTHARNESS = ./util/testharness.o $(TESTUTIL) TESTHARNESS = ./util/testharness.o $(TESTUTIL)
# Note: iOS should probably be using libtool, not ar.
ifeq ($(PLATFORM), IOS)
AR=xcrun ar
endif
TESTS = \ TESTS = \
arena_test \ arena_test \
autocompact_test \ autocompact_test \
@ -43,6 +51,7 @@ TESTS = \
env_test \ env_test \
filename_test \ filename_test \
filter_block_test \ filter_block_test \
hash_test \
issue178_test \ issue178_test \
issue200_test \ issue200_test \
log_test \ log_test \
@ -72,7 +81,7 @@ SHARED = $(SHARED1)
else else
# Update db.h if you change these. # Update db.h if you change these.
SHARED_MAJOR = 1 SHARED_MAJOR = 1
SHARED_MINOR = 17 SHARED_MINOR = 18
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT) SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED2 = $(SHARED1).$(SHARED_MAJOR)
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
@ -152,6 +161,9 @@ filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS)
filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS) $(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
@ -194,17 +206,17 @@ IOSARCH=-arch armv6 -arch armv7 -arch armv7s -arch arm64
.cc.o: .cc.o:
mkdir -p ios-x86/$(dir $@) mkdir -p ios-x86/$(dir $@)
$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
mkdir -p ios-arm/$(dir $@) mkdir -p ios-arm/$(dir $@)
xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@
lipo ios-x86/$@ ios-arm/$@ -create -output $@ xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@
.c.o: .c.o:
mkdir -p ios-x86/$(dir $@) mkdir -p ios-x86/$(dir $@)
$(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@ xcrun -sdk iphonesimulator $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
mkdir -p ios-arm/$(dir $@) mkdir -p ios-arm/$(dir $@)
xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@ xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@
lipo ios-x86/$@ ios-arm/$@ -create -output $@ xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@
else else
.cc.o: .cc.o:

138
README.md

@ -0,0 +1,138 @@
**LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.**
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
# Features
* Keys and values are arbitrary byte arrays.
* Data is stored sorted by key.
* Callers can provide a custom comparison function to override the sort order.
* The basic operations are `Put(key,value)`, `Get(key)`, `Delete(key)`.
* Multiple changes can be made in one atomic batch.
* Users can create a transient snapshot to get a consistent view of data.
* Forward and backward iteration is supported over the data.
* Data is automatically compressed using the [Snappy compression library](http://code.google.com/p/snappy).
* External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions.
* [Detailed documentation](http://htmlpreview.github.io/?https://github.com/google/leveldb/blob/master/doc/index.html) about how to use the library is included with the source code.
# Limitations
* This is not a SQL database. It does not have a relational data model, it does not support SQL queries, and it has no support for indexes.
* Only a single process (possibly multi-threaded) can access a particular database at a time.
* There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library.
# Performance
Here is a performance report (with explanations) from the run of the
included db_bench program. The results are somewhat noisy, but should
be enough to get a ballpark performance estimate.
## Setup
We use a database with a million entries. Each entry has a 16 byte
key, and a 100 byte value. Values used by the benchmark compress to
about half their original size.
LevelDB: version 1.1
Date: Sun May 1 12:11:26 2011
CPU: 4 x Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz
CPUCache: 4096 KB
Keys: 16 bytes each
Values: 100 bytes each (50 bytes after compression)
Entries: 1000000
Raw Size: 110.6 MB (estimated)
File Size: 62.9 MB (estimated)
## Write performance
The "fill" benchmarks create a brand new database, in either
sequential, or random order. The "fillsync" benchmark flushes data
from the operating system to the disk after every operation; the other
write operations leave the data sitting in the operating system buffer
cache for a while. The "overwrite" benchmark does random writes that
update existing keys in the database.
fillseq : 1.765 micros/op; 62.7 MB/s
fillsync : 268.409 micros/op; 0.4 MB/s (10000 ops)
fillrandom : 2.460 micros/op; 45.0 MB/s
overwrite : 2.380 micros/op; 46.5 MB/s
Each "op" above corresponds to a write of a single key/value pair.
I.e., a random write benchmark goes at approximately 400,000 writes per second.
Each "fillsync" operation costs much less (0.3 millisecond)
than a disk seek (typically 10 milliseconds). We suspect that this is
because the hard disk itself is buffering the update in its memory and
responding before the data has been written to the platter. This may
or may not be safe based on whether or not the hard disk has enough
power to save its memory in the event of a power failure.
## Read performance
We list the performance of reading sequentially in both the forward
and reverse direction, and also the performance of a random lookup.
Note that the database created by the benchmark is quite small.
Therefore the report characterizes the performance of leveldb when the
working set fits in memory. The cost of reading a piece of data that
is not present in the operating system buffer cache will be dominated
by the one or two disk seeks needed to fetch the data from disk.
Write performance will be mostly unaffected by whether or not the
working set fits in memory.
readrandom : 16.677 micros/op; (approximately 60,000 reads per second)
readseq : 0.476 micros/op; 232.3 MB/s
readreverse : 0.724 micros/op; 152.9 MB/s
LevelDB compacts its underlying storage data in the background to
improve read performance. The results listed above were done
immediately after a lot of random writes. The results after
compactions (which are usually triggered automatically) are better.
readrandom : 11.602 micros/op; (approximately 85,000 reads per second)
readseq : 0.423 micros/op; 261.8 MB/s
readreverse : 0.663 micros/op; 166.9 MB/s
Some of the high cost of reads comes from repeated decompression of blocks
read from disk. If we supply enough cache to the leveldb so it can hold the
uncompressed blocks in memory, the read performance improves again:
readrandom : 9.775 micros/op; (approximately 100,000 reads per second before compaction)
readrandom : 5.215 micros/op; (approximately 190,000 reads per second after compaction)
## Repository contents
See doc/index.html for more explanation. See doc/impl.html for a brief overview of the implementation.
The public interface is in include/*.h. Callers should not include or
rely on the details of any other header files in this package. Those
internal APIs may be changed without warning.
Guide to header files:
* **include/db.h**: Main interface to the DB: Start here
* **include/options.h**: Control over the behavior of an entire database,
and also control over the behavior of individual reads and writes.
* **include/comparator.h**: Abstraction for user-specified comparison function.
If you want just bytewise comparison of keys, you can use the default
comparator, but clients can write their own comparator implementations if they
want custom ordering (e.g. to handle different character encodings, etc.)
* **include/iterator.h**: Interface for iterating over data. You can get
an iterator from a DB object.
* **include/write_batch.h**: Interface for atomically applying multiple
updates to a database.
* **include/slice.h**: A simple module for maintaining a pointer and a
length into some other byte array.
* **include/status.h**: Status is returned from many of the public interfaces
and is used to report success and various kinds of errors.
* **include/env.h**:
Abstraction of the OS environment. A posix implementation of this interface is
in util/env_posix.cc
* **include/table.h, include/table_builder.h**: Lower-level modules that most
clients probably won't use directly

15
build_detect_platform

@ -20,7 +20,7 @@
# #
# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following: # The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
# #
# -DLEVELDB_CSTDATOMIC_PRESENT if <cstdatomic> is present # -DLEVELDB_ATOMIC_PRESENT if <atomic> is present
# -DLEVELDB_PLATFORM_POSIX for Posix-based platforms # -DLEVELDB_PLATFORM_POSIX for Posix-based platforms
# -DSNAPPY if the Snappy library is present # -DSNAPPY if the Snappy library is present
# #
@ -72,6 +72,12 @@ if [ "$CXX" = "g++" ]; then
fi fi
case "$TARGET_OS" in case "$TARGET_OS" in
CYGWIN_*)
PLATFORM=OS_LINUX
COMMON_FLAGS="$MEMCMP_FLAG -lpthread -DOS_LINUX -DCYGWIN"
PLATFORM_LDFLAGS="-lpthread"
PORT_FILE=port/port_posix.cc
;;
Darwin) Darwin)
PLATFORM=OS_MACOSX PLATFORM=OS_MACOSX
COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX" COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
@ -185,13 +191,14 @@ if [ "$CROSS_COMPILE" = "true" ]; then
else else
CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$" CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$"
# If -std=c++0x works, use <cstdatomic>. Otherwise use port_posix.h. # If -std=c++0x works, use <atomic> as fallback for when memory barriers
# are not available.
$CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF $CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
#include <cstdatomic> #include <atomic>
int main() {} int main() {}
EOF EOF
if [ "$?" = 0 ]; then if [ "$?" = 0 ]; then
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_CSTDATOMIC_PRESENT" COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_ATOMIC_PRESENT"
PLATFORM_CXXFLAGS="-std=c++0x" PLATFORM_CXXFLAGS="-std=c++0x"
else else
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX" COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"

3
db/db_bench.cc

@ -431,7 +431,7 @@ class Benchmark {
benchmarks = sep + 1; benchmarks = sep + 1;
} }
// Reset parameters that may be overriddden bwlow // Reset parameters that may be overridden below
num_ = FLAGS_num; num_ = FLAGS_num;
reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads); reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
value_size_ = FLAGS_value_size; value_size_ = FLAGS_value_size;
@ -811,7 +811,6 @@ class Benchmark {
void SeekRandom(ThreadState* thread) { void SeekRandom(ThreadState* thread) {
ReadOptions options; ReadOptions options;
std::string value;
int found = 0; int found = 0;
for (int i = 0; i < reads_; i++) { for (int i = 0; i < reads_; i++) {
Iterator* iter = db_->NewIterator(options); Iterator* iter = db_->NewIterator(options);

4
db/db_impl.cc

@ -392,7 +392,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number,
reporter.info_log = options_.info_log; reporter.info_log = options_.info_log;
reporter.fname = fname.c_str(); reporter.fname = fname.c_str();
reporter.status = (options_.paranoid_checks ? &status : NULL); reporter.status = (options_.paranoid_checks ? &status : NULL);
// We intentially make log::Reader do checksumming even if // We intentionally make log::Reader do checksumming even if
// paranoid_checks==false so that corruptions cause entire commits // paranoid_checks==false so that corruptions cause entire commits
// to be skipped instead of propagating bad information (like overly // to be skipped instead of propagating bad information (like overly
// large sequence numbers). // large sequence numbers).
@ -1267,7 +1267,7 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) {
break; break;
} }
// Append to *reuslt // Append to *result
if (result == first->batch) { if (result == first->batch) {
// Switch to temporary batch instead of disturbing caller's batch // Switch to temporary batch instead of disturbing caller's batch
result = tmp_batch_; result = tmp_batch_;

2
db/db_test.cc

@ -626,7 +626,7 @@ TEST(DBTest, GetEncountersEmptyLevel) {
// * sstable B in level 2 // * sstable B in level 2
// Then do enough Get() calls to arrange for an automatic compaction // Then do enough Get() calls to arrange for an automatic compaction
// of sstable A. A bug would cause the compaction to be marked as // of sstable A. A bug would cause the compaction to be marked as
// occuring at level 1 (instead of the correct level 0). // occurring at level 1 (instead of the correct level 0).
// Step 1: First place sstables in levels 0 and 2 // Step 1: First place sstables in levels 0 and 2
int compaction_count = 0; int compaction_count = 0;

6
db/dbformat.h

@ -2,8 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_LEVELDB_DB_FORMAT_H_ #ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_
#define STORAGE_LEVELDB_DB_FORMAT_H_ #define STORAGE_LEVELDB_DB_DBFORMAT_H_
#include <stdio.h> #include <stdio.h>
#include "leveldb/comparator.h" #include "leveldb/comparator.h"
@ -227,4 +227,4 @@ inline LookupKey::~LookupKey() {
} // namespace leveldb } // namespace leveldb
#endif // STORAGE_LEVELDB_DB_FORMAT_H_ #endif // STORAGE_LEVELDB_DB_DBFORMAT_H_

225
db/dumpfile.cc

@ -0,0 +1,225 @@
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <stdio.h>
#include "db/dbformat.h"
#include "db/filename.h"
#include "db/log_reader.h"
#include "db/version_edit.h"
#include "db/write_batch_internal.h"
#include "leveldb/env.h"
#include "leveldb/iterator.h"
#include "leveldb/options.h"
#include "leveldb/status.h"
#include "leveldb/table.h"
#include "leveldb/write_batch.h"
#include "util/logging.h"
namespace leveldb {
namespace {
bool GuessType(const std::string& fname, FileType* type) {
size_t pos = fname.rfind('/');
std::string basename;
if (pos == std::string::npos) {
basename = fname;
} else {
basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1);
}
uint64_t ignored;
return ParseFileName(basename, &ignored, type);
}
// Notified when log reader encounters corruption.
class CorruptionReporter : public log::Reader::Reporter {
public:
WritableFile* dst_;
virtual void Corruption(size_t bytes, const Status& status) {
std::string r = "corruption: ";
AppendNumberTo(&r, bytes);
r += " bytes; ";
r += status.ToString();
r.push_back('\n');
dst_->Append(r);
}
};
// Print contents of a log file. (*func)() is called on every record.
Status PrintLogContents(Env* env, const std::string& fname,
void (*func)(uint64_t, Slice, WritableFile*),
WritableFile* dst) {
SequentialFile* file;
Status s = env->NewSequentialFile(fname, &file);
if (!s.ok()) {
return s;
}
CorruptionReporter reporter;
reporter.dst_ = dst;
log::Reader reader(file, &reporter, true, 0);
Slice record;
std::string scratch;
while (reader.ReadRecord(&record, &scratch)) {
(*func)(reader.LastRecordOffset(), record, dst);
}
delete file;
return Status::OK();
}
// Called on every item found in a WriteBatch.
class WriteBatchItemPrinter : public WriteBatch::Handler {
public:
WritableFile* dst_;
virtual void Put(const Slice& key, const Slice& value) {
std::string r = " put '";
AppendEscapedStringTo(&r, key);
r += "' '";
AppendEscapedStringTo(&r, value);
r += "'\n";
dst_->Append(r);
}
virtual void Delete(const Slice& key) {
std::string r = " del '";
AppendEscapedStringTo(&r, key);
r += "'\n";
dst_->Append(r);
}
};
// Called on every log record (each one of which is a WriteBatch)
// found in a kLogFile.
static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) {
std::string r = "--- offset ";
AppendNumberTo(&r, pos);
r += "; ";
if (record.size() < 12) {
r += "log record length ";
AppendNumberTo(&r, record.size());
r += " is too small\n";
dst->Append(r);
return;
}
WriteBatch batch;
WriteBatchInternal::SetContents(&batch, record);
r += "sequence ";
AppendNumberTo(&r, WriteBatchInternal::Sequence(&batch));
r.push_back('\n');
dst->Append(r);
WriteBatchItemPrinter batch_item_printer;
batch_item_printer.dst_ = dst;
Status s = batch.Iterate(&batch_item_printer);
if (!s.ok()) {
dst->Append(" error: " + s.ToString() + "\n");
}
}
Status DumpLog(Env* env, const std::string& fname, WritableFile* dst) {
return PrintLogContents(env, fname, WriteBatchPrinter, dst);
}
// Called on every log record (each one of which is a WriteBatch)
// found in a kDescriptorFile.
static void VersionEditPrinter(uint64_t pos, Slice record, WritableFile* dst) {
std::string r = "--- offset ";
AppendNumberTo(&r, pos);
r += "; ";
VersionEdit edit;
Status s = edit.DecodeFrom(record);
if (!s.ok()) {
r += s.ToString();
r.push_back('\n');
} else {
r += edit.DebugString();
}
dst->Append(r);
}
Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) {
return PrintLogContents(env, fname, VersionEditPrinter, dst);
}
Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) {
uint64_t file_size;
RandomAccessFile* file = NULL;
Table* table = NULL;
Status s = env->GetFileSize(fname, &file_size);
if (s.ok()) {
s = env->NewRandomAccessFile(fname, &file);
}
if (s.ok()) {
// We use the default comparator, which may or may not match the
// comparator used in this database. However this should not cause
// problems since we only use Table operations that do not require
// any comparisons. In particular, we do not call Seek or Prev.
s = Table::Open(Options(), file, file_size, &table);
}
if (!s.ok()) {
delete table;
delete file;
return s;
}
ReadOptions ro;
ro.fill_cache = false;
Iterator* iter = table->NewIterator(ro);
std::string r;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
r.clear();
ParsedInternalKey key;
if (!ParseInternalKey(iter->key(), &key)) {
r = "badkey '";
AppendEscapedStringTo(&r, iter->key());
r += "' => '";
AppendEscapedStringTo(&r, iter->value());
r += "'\n";
dst->Append(r);
} else {
r = "'";
AppendEscapedStringTo(&r, key.user_key);
r += "' @ ";
AppendNumberTo(&r, key.sequence);
r += " : ";
if (key.type == kTypeDeletion) {
r += "del";
} else if (key.type == kTypeValue) {
r += "val";
} else {
AppendNumberTo(&r, key.type);
}
r += " => '";
AppendEscapedStringTo(&r, iter->value());
r += "'\n";
dst->Append(r);
}
}
s = iter->status();
if (!s.ok()) {
dst->Append("iterator error: " + s.ToString() + "\n");
}
delete iter;
delete table;
delete file;
return Status::OK();
}
} // namespace
Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) {
FileType ftype;
if (!GuessType(fname, &ftype)) {
return Status::InvalidArgument(fname + ": unknown file type");
}
switch (ftype) {
case kLogFile: return DumpLog(env, fname, dst);
case kDescriptorFile: return DumpDescriptor(env, fname, dst);
case kTableFile: return DumpTable(env, fname, dst);
default:
break;
}
return Status::InvalidArgument(fname + ": not a dump-able file type");
}
} // namespace leveldb

204
db/leveldb_main.cc

@ -3,212 +3,38 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <stdio.h> #include <stdio.h>
#include "db/dbformat.h" #include "leveldb/dumpfile.h"
#include "db/filename.h"
#include "db/log_reader.h"
#include "db/version_edit.h"
#include "db/write_batch_internal.h"
#include "leveldb/env.h" #include "leveldb/env.h"
#include "leveldb/iterator.h"
#include "leveldb/options.h"
#include "leveldb/status.h" #include "leveldb/status.h"
#include "leveldb/table.h"
#include "leveldb/write_batch.h"
#include "util/logging.h"
namespace leveldb { namespace leveldb {
namespace { namespace {
bool GuessType(const std::string& fname, FileType* type) { class StdoutPrinter : public WritableFile {
size_t pos = fname.rfind('/');
std::string basename;
if (pos == std::string::npos) {
basename = fname;
} else {
basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1);
}
uint64_t ignored;
return ParseFileName(basename, &ignored, type);
}
// Notified when log reader encounters corruption.
class CorruptionReporter : public log::Reader::Reporter {
public:
virtual void Corruption(size_t bytes, const Status& status) {
printf("corruption: %d bytes; %s\n",
static_cast<int>(bytes),
status.ToString().c_str());
}
};
// Print contents of a log file. (*func)() is called on every record.
bool PrintLogContents(Env* env, const std::string& fname,
void (*func)(Slice)) {
SequentialFile* file;
Status s = env->NewSequentialFile(fname, &file);
if (!s.ok()) {
fprintf(stderr, "%s\n", s.ToString().c_str());
return false;
}
CorruptionReporter reporter;
log::Reader reader(file, &reporter, true, 0);
Slice record;
std::string scratch;
while (reader.ReadRecord(&record, &scratch)) {
printf("--- offset %llu; ",
static_cast<unsigned long long>(reader.LastRecordOffset()));
(*func)(record);
}
delete file;
return true;
}
// Called on every item found in a WriteBatch.
class WriteBatchItemPrinter : public WriteBatch::Handler {
public: public:
uint64_t offset_; virtual Status Append(const Slice& data) {
uint64_t sequence_; fwrite(data.data(), 1, data.size(), stdout);
return Status::OK();
virtual void Put(const Slice& key, const Slice& value) {
printf(" put '%s' '%s'\n",
EscapeString(key).c_str(),
EscapeString(value).c_str());
}
virtual void Delete(const Slice& key) {
printf(" del '%s'\n",
EscapeString(key).c_str());
} }
virtual Status Close() { return Status::OK(); }
virtual Status Flush() { return Status::OK(); }
virtual Status Sync() { return Status::OK(); }
}; };
// Called on every log record (each one of which is a WriteBatch)
// found in a kLogFile.
static void WriteBatchPrinter(Slice record) {
if (record.size() < 12) {
printf("log record length %d is too small\n",
static_cast<int>(record.size()));
return;
}
WriteBatch batch;
WriteBatchInternal::SetContents(&batch, record);
printf("sequence %llu\n",
static_cast<unsigned long long>(WriteBatchInternal::Sequence(&batch)));
WriteBatchItemPrinter batch_item_printer;
Status s = batch.Iterate(&batch_item_printer);
if (!s.ok()) {
printf(" error: %s\n", s.ToString().c_str());
}
}
bool DumpLog(Env* env, const std::string& fname) {
return PrintLogContents(env, fname, WriteBatchPrinter);
}
// Called on every log record (each one of which is a WriteBatch)
// found in a kDescriptorFile.
static void VersionEditPrinter(Slice record) {
VersionEdit edit;
Status s = edit.DecodeFrom(record);
if (!s.ok()) {
printf("%s\n", s.ToString().c_str());
return;
}
printf("%s", edit.DebugString().c_str());
}
bool DumpDescriptor(Env* env, const std::string& fname) {
return PrintLogContents(env, fname, VersionEditPrinter);
}
bool DumpTable(Env* env, const std::string& fname) {
uint64_t file_size;
RandomAccessFile* file = NULL;
Table* table = NULL;
Status s = env->GetFileSize(fname, &file_size);
if (s.ok()) {
s = env->NewRandomAccessFile(fname, &file);
}
if (s.ok()) {
// We use the default comparator, which may or may not match the
// comparator used in this database. However this should not cause
// problems since we only use Table operations that do not require
// any comparisons. In particular, we do not call Seek or Prev.
s = Table::Open(Options(), file, file_size, &table);
}
if (!s.ok()) {
fprintf(stderr, "%s\n", s.ToString().c_str());
delete table;
delete file;
return false;
}
ReadOptions ro;
ro.fill_cache = false;
Iterator* iter = table->NewIterator(ro);
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ParsedInternalKey key;
if (!ParseInternalKey(iter->key(), &key)) {
printf("badkey '%s' => '%s'\n",
EscapeString(iter->key()).c_str(),
EscapeString(iter->value()).c_str());
} else {
char kbuf[20];
const char* type;
if (key.type == kTypeDeletion) {
type = "del";
} else if (key.type == kTypeValue) {
type = "val";
} else {
snprintf(kbuf, sizeof(kbuf), "%d", static_cast<int>(key.type));
type = kbuf;
}
printf("'%s' @ %8llu : %s => '%s'\n",
EscapeString(key.user_key).c_str(),
static_cast<unsigned long long>(key.sequence),
type,
EscapeString(iter->value()).c_str());
}
}
s = iter->status();
if (!s.ok()) {
printf("iterator error: %s\n", s.ToString().c_str());
}
delete iter;
delete table;
delete file;
return true;
}
bool DumpFile(Env* env, const std::string& fname) {
FileType ftype;
if (!GuessType(fname, &ftype)) {
fprintf(stderr, "%s: unknown file type\n", fname.c_str());
return false;
}
switch (ftype) {
case kLogFile: return DumpLog(env, fname);
case kDescriptorFile: return DumpDescriptor(env, fname);
case kTableFile: return DumpTable(env, fname);
default: {
fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str());
break;
}
}
return false;
}
bool HandleDumpCommand(Env* env, char** files, int num) { bool HandleDumpCommand(Env* env, char** files, int num) {
StdoutPrinter printer;
bool ok = true; bool ok = true;
for (int i = 0; i < num; i++) { for (int i = 0; i < num; i++) {
ok &= DumpFile(env, files[i]); Status s = DumpFile(env, files[i], &printer);
if (!s.ok()) {
fprintf(stderr, "%s\n", s.ToString().c_str());
ok = false;
}
} }
return ok; return ok;
} }
} } // namespace
} // namespace leveldb } // namespace leveldb
static void Usage() { static void Usage() {

4
db/log_format.h

@ -26,8 +26,8 @@ static const int kMaxRecordType = kLastType;
static const int kBlockSize = 32768; static const int kBlockSize = 32768;
// Header is checksum (4 bytes), type (1 byte), length (2 bytes). // Header is checksum (4 bytes), length (2 bytes), type (1 byte).
static const int kHeaderSize = 4 + 1 + 2; static const int kHeaderSize = 4 + 2 + 1;
} // namespace log } // namespace log
} // namespace leveldb } // namespace leveldb

6
db/log_reader.cc

@ -167,14 +167,14 @@ uint64_t Reader::LastRecordOffset() {
return last_record_offset_; return last_record_offset_;
} }
void Reader::ReportCorruption(size_t bytes, const char* reason) { void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
ReportDrop(bytes, Status::Corruption(reason)); ReportDrop(bytes, Status::Corruption(reason));
} }
void Reader::ReportDrop(size_t bytes, const Status& reason) { void Reader::ReportDrop(uint64_t bytes, const Status& reason) {
if (reporter_ != NULL && if (reporter_ != NULL &&
end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) { end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
reporter_->Corruption(bytes, reason); reporter_->Corruption(static_cast<size_t>(bytes), reason);
} }
} }

4
db/log_reader.h

@ -94,8 +94,8 @@ class Reader {
// Reports dropped bytes to the reporter. // Reports dropped bytes to the reporter.
// buffer_ must be updated to remove the dropped bytes prior to invocation. // buffer_ must be updated to remove the dropped bytes prior to invocation.
void ReportCorruption(size_t bytes, const char* reason); void ReportCorruption(uint64_t bytes, const char* reason);
void ReportDrop(size_t bytes, const Status& reason); void ReportDrop(uint64_t bytes, const Status& reason);
// No copying allowed // No copying allowed
Reader(const Reader&); Reader(const Reader&);

2
db/log_test.cc

@ -463,7 +463,7 @@ TEST(LogTest, ErrorJoinsRecords) {
ASSERT_EQ("correct", Read()); ASSERT_EQ("correct", Read());
ASSERT_EQ("EOF", Read()); ASSERT_EQ("EOF", Read());
const int dropped = DroppedBytes(); const size_t dropped = DroppedBytes();
ASSERT_LE(dropped, 2*kBlockSize + 100); ASSERT_LE(dropped, 2*kBlockSize + 100);
ASSERT_GE(dropped, 2*kBlockSize); ASSERT_GE(dropped, 2*kBlockSize);
} }

2
db/repair.cc

@ -186,7 +186,7 @@ class Repairer {
reporter.env = env_; reporter.env = env_;
reporter.info_log = options_.info_log; reporter.info_log = options_.info_log;
reporter.lognum = log; reporter.lognum = log;
// We intentially make log::Reader do checksumming so that // We intentionally make log::Reader do checksumming so that
// corruptions cause entire commits to be skipped instead of // corruptions cause entire commits to be skipped instead of
// propagating bad information (like overly large sequence // propagating bad information (like overly large sequence
// numbers). // numbers).

5
db/skiplist.h

@ -1,3 +1,6 @@
#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_
#define STORAGE_LEVELDB_DB_SKIPLIST_H_
// Copyright (c) 2011 The LevelDB Authors. All rights reserved. // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
@ -377,3 +380,5 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const {
} }
} // namespace leveldb } // namespace leveldb
#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_

4
db/write_batch_internal.h

@ -21,10 +21,10 @@ class WriteBatchInternal {
// Set the count for the number of entries in the batch. // Set the count for the number of entries in the batch.
static void SetCount(WriteBatch* batch, int n); static void SetCount(WriteBatch* batch, int n);
// Return the seqeunce number for the start of this batch. // Return the sequence number for the start of this batch.
static SequenceNumber Sequence(const WriteBatch* batch); static SequenceNumber Sequence(const WriteBatch* batch);
// Store the specified number as the seqeunce number for the start of // Store the specified number as the sequence number for the start of
// this batch. // this batch.
static void SetSequence(WriteBatch* batch, SequenceNumber seq); static void SetSequence(WriteBatch* batch, SequenceNumber seq);

2
doc/bench/db_bench_tree_db.cc

@ -338,7 +338,7 @@ class Benchmark {
bool write_sync = false; bool write_sync = false;
if (name == Slice("fillseq")) { if (name == Slice("fillseq")) {
Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1); Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1);
DBSynchronize(db_);
} else if (name == Slice("fillrandom")) { } else if (name == Slice("fillrandom")) {
Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1); Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1);
DBSynchronize(db_); DBSynchronize(db_);

4
doc/impl.html

@ -111,7 +111,7 @@ A compaction merges the contents of the picked files to produce a
sequence of level-(L+1) files. We switch to producing a new sequence of level-(L+1) files. We switch to producing a new
level-(L+1) file after the current output file has reached the target level-(L+1) file after the current output file has reached the target
file size (2MB). We also switch to a new output file when the key file size (2MB). We also switch to a new output file when the key
range of the current output file has grown enough to overlap more then range of the current output file has grown enough to overlap more than
ten level-(L+2) files. This last rule ensures that a later compaction ten level-(L+2) files. This last rule ensures that a later compaction
of a level-(L+1) file will not pick up too much data from level-(L+2). of a level-(L+1) file will not pick up too much data from level-(L+2).
@ -151,7 +151,7 @@ compaction cost will be approximately 0.5 second.
If we throttle the background writing to something small, say 10% of If we throttle the background writing to something small, say 10% of
the full 100MB/s speed, a compaction may take up to 5 seconds. If the the full 100MB/s speed, a compaction may take up to 5 seconds. If the
user is writing at 10MB/s, we might build up lots of level-0 files user is writing at 10MB/s, we might build up lots of level-0 files
(~50 to hold the 5*10MB). This may signficantly increase the cost of (~50 to hold the 5*10MB). This may significantly increase the cost of
reads due to the overhead of merging more files together on every reads due to the overhead of merging more files together on every
read. read.

4
doc/log_format.txt

@ -33,8 +33,8 @@ The FULL record contains the contents of an entire user record.
FIRST, MIDDLE, LAST are types used for user records that have been FIRST, MIDDLE, LAST are types used for user records that have been
split into multiple fragments (typically because of block boundaries). split into multiple fragments (typically because of block boundaries).
FIRST is the type of the first fragment of a user record, LAST is the FIRST is the type of the first fragment of a user record, LAST is the
type of the last fragment of a user record, and MID is the type of all type of the last fragment of a user record, and MIDDLE is the type of
interior fragments of a user record. all interior fragments of a user record.
Example: consider a sequence of user records: Example: consider a sequence of user records:
A: length 1000 A: length 1000

9
helpers/memenv/memenv.cc vendored

@ -55,14 +55,15 @@ class FileState {
} }
const uint64_t available = size_ - offset; const uint64_t available = size_ - offset;
if (n > available) { if (n > available) {
n = available; n = static_cast<size_t>(available);
} }
if (n == 0) { if (n == 0) {
*result = Slice(); *result = Slice();
return Status::OK(); return Status::OK();
} }
size_t block = offset / kBlockSize; assert(offset / kBlockSize <= SIZE_MAX);
size_t block = static_cast<size_t>(offset / kBlockSize);
size_t block_offset = offset % kBlockSize; size_t block_offset = offset % kBlockSize;
if (n <= kBlockSize - block_offset) { if (n <= kBlockSize - block_offset) {
@ -167,7 +168,7 @@ class SequentialFileImpl : public SequentialFile {
if (pos_ > file_->Size()) { if (pos_ > file_->Size()) {
return Status::IOError("pos_ > file_->Size()"); return Status::IOError("pos_ > file_->Size()");
} }
const size_t available = file_->Size() - pos_; const uint64_t available = file_->Size() - pos_;
if (n > available) { if (n > available) {
n = available; n = available;
} }
@ -177,7 +178,7 @@ class SequentialFileImpl : public SequentialFile {
private: private:
FileState* file_; FileState* file_;
size_t pos_; uint64_t pos_;
}; };
class RandomAccessFileImpl : public RandomAccessFile { class RandomAccessFileImpl : public RandomAccessFile {

2
include/leveldb/cache.h

@ -96,4 +96,4 @@ class Cache {
} // namespace leveldb } // namespace leveldb
#endif // STORAGE_LEVELDB_UTIL_CACHE_H_ #endif // STORAGE_LEVELDB_INCLUDE_CACHE_H_

2
include/leveldb/db.h

@ -14,7 +14,7 @@ namespace leveldb {
// Update Makefile if you change these // Update Makefile if you change these
static const int kMajorVersion = 1; static const int kMajorVersion = 1;
static const int kMinorVersion = 17; static const int kMinorVersion = 18;
struct Options; struct Options;
struct ReadOptions; struct ReadOptions;

25
include/leveldb/dumpfile.h

@ -0,0 +1,25 @@
// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
#define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
#include <string>
#include "leveldb/env.h"
#include "leveldb/status.h"
namespace leveldb {
// Dump the contents of the file named by fname in text format to
// *dst. Makes a sequence of dst->Append() calls; each call is passed
// the newline-terminated text corresponding to a single item found
// in the file.
//
// Returns a non-OK result if fname does not name a leveldb storage
// file, or if the file cannot be read.
Status DumpFile(Env* env, const std::string& fname, WritableFile* dst);
} // namespace leveldb
#endif // STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_

2
include/leveldb/env.h

@ -142,7 +142,7 @@ class Env {
// useful for computing deltas of time. // useful for computing deltas of time.
virtual uint64_t NowMicros() = 0; virtual uint64_t NowMicros() = 0;
// Sleep/delay the thread for the perscribed number of micro-seconds. // Sleep/delay the thread for the prescribed number of micro-seconds.
virtual void SleepForMicroseconds(int micros) = 0; virtual void SleepForMicroseconds(int micros) = 0;
private: private:

2
include/leveldb/iterator.h

@ -61,7 +61,7 @@ class Iterator {
// Return the value for the current entry. The underlying storage for // Return the value for the current entry. The underlying storage for
// the returned slice is valid only until the next modification of // the returned slice is valid only until the next modification of
// the iterator. // the iterator.
// REQUIRES: !AtEnd() && !AtStart() // REQUIRES: Valid()
virtual Slice value() const = 0; virtual Slice value() const = 0;
// If an error has occurred, return it. Else return an ok status. // If an error has occurred, return it. Else return an ok status.

2
include/leveldb/options.h

@ -153,7 +153,7 @@ struct ReadOptions {
// If "snapshot" is non-NULL, read as of the supplied snapshot // If "snapshot" is non-NULL, read as of the supplied snapshot
// (which must belong to the DB that is being read and which must // (which must belong to the DB that is being read and which must
// not have been released). If "snapshot" is NULL, use an impliicit // not have been released). If "snapshot" is NULL, use an implicit
// snapshot of the state at the beginning of this read operation. // snapshot of the state at the beginning of this read operation.
// Default: NULL // Default: NULL
const Snapshot* snapshot; const Snapshot* snapshot;

21
port/atomic_pointer.h

@ -5,14 +5,13 @@
// AtomicPointer provides storage for a lock-free pointer. // AtomicPointer provides storage for a lock-free pointer.
// Platform-dependent implementation of AtomicPointer: // Platform-dependent implementation of AtomicPointer:
// - If the platform provides a cheap barrier, we use it with raw pointers // - If the platform provides a cheap barrier, we use it with raw pointers
// - If cstdatomic is present (on newer versions of gcc, it is), we use // - If <atomic> is present (on newer versions of gcc, it is), we use
// a cstdatomic-based AtomicPointer. However we prefer the memory // a <atomic>-based AtomicPointer. However we prefer the memory
// barrier based version, because at least on a gcc 4.4 32-bit build // barrier based version, because at least on a gcc 4.4 32-bit build
// on linux, we have encountered a buggy <cstdatomic> // on linux, we have encountered a buggy <atomic> implementation.
// implementation. Also, some <cstdatomic> implementations are much // Also, some <atomic> implementations are much slower than a memory-barrier
// slower than a memory-barrier based implementation (~16ns for // based implementation (~16ns for <atomic> based acquire-load vs. ~1ns for
// <cstdatomic> based acquire-load vs. ~1ns for a barrier based // a barrier based acquire-load).
// acquire-load).
// This code is based on atomicops-internals-* in Google's perftools: // This code is based on atomicops-internals-* in Google's perftools:
// http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase // http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase
@ -20,8 +19,8 @@
#define PORT_ATOMIC_POINTER_H_ #define PORT_ATOMIC_POINTER_H_
#include <stdint.h> #include <stdint.h>
#ifdef LEVELDB_CSTDATOMIC_PRESENT #ifdef LEVELDB_ATOMIC_PRESENT
#include <cstdatomic> #include <atomic>
#endif #endif
#ifdef OS_WIN #ifdef OS_WIN
#include <windows.h> #include <windows.h>
@ -126,7 +125,7 @@ class AtomicPointer {
}; };
// AtomicPointer based on <cstdatomic> // AtomicPointer based on <cstdatomic>
#elif defined(LEVELDB_CSTDATOMIC_PRESENT) #elif defined(LEVELDB_ATOMIC_PRESENT)
class AtomicPointer { class AtomicPointer {
private: private:
std::atomic<void*> rep_; std::atomic<void*> rep_;
@ -207,7 +206,7 @@ class AtomicPointer {
inline void NoBarrier_Store(void* v) { rep_ = v; } inline void NoBarrier_Store(void* v) { rep_ = v; }
}; };
// We have neither MemoryBarrier(), nor <cstdatomic> // We have neither MemoryBarrier(), nor <atomic>
#else #else
#error Please implement AtomicPointer for this platform. #error Please implement AtomicPointer for this platform.

9
port/port_posix.h

@ -21,14 +21,11 @@
#else #else
#define PLATFORM_IS_LITTLE_ENDIAN false #define PLATFORM_IS_LITTLE_ENDIAN false
#endif #endif
#elif defined(OS_FREEBSD) #elif defined(OS_FREEBSD) || defined(OS_OPENBSD) ||\
defined(OS_NETBSD) || defined(OS_DRAGONFLYBSD)
#include <sys/types.h> #include <sys/types.h>
#include <sys/endian.h> #include <sys/endian.h>
#define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN) #define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN)
#elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\
defined(OS_DRAGONFLYBSD)
#include <sys/types.h>
#include <sys/endian.h>
#elif defined(OS_HPUX) #elif defined(OS_HPUX)
#define PLATFORM_IS_LITTLE_ENDIAN false #define PLATFORM_IS_LITTLE_ENDIAN false
#elif defined(OS_ANDROID) #elif defined(OS_ANDROID)
@ -55,7 +52,7 @@
#if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\ #if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\
defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\ defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\
defined(OS_ANDROID) || defined(OS_HPUX) defined(OS_ANDROID) || defined(OS_HPUX) || defined(CYGWIN)
// Use fread/fwrite/fflush on platforms without _unlocked variants // Use fread/fwrite/fflush on platforms without _unlocked variants
#define fread_unlocked fread #define fread_unlocked fread
#define fwrite_unlocked fwrite #define fwrite_unlocked fwrite

5
port/thread_annotations.h

@ -2,7 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be // Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H #ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_
#define STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_
// Some environments provide custom macros to aid in static thread-safety // Some environments provide custom macros to aid in static thread-safety
// analysis. Provide empty definitions of such macros unless they are already // analysis. Provide empty definitions of such macros unless they are already
@ -56,4 +57,4 @@
#define NO_THREAD_SAFETY_ANALYSIS #define NO_THREAD_SAFETY_ANALYSIS
#endif #endif
#endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H #endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_

2
table/block.cc

@ -46,7 +46,7 @@ Block::~Block() {
// Helper routine: decode the next block entry starting at "p", // Helper routine: decode the next block entry starting at "p",
// storing the number of shared key bytes, non_shared key bytes, // storing the number of shared key bytes, non_shared key bytes,
// and the length of the value in "*shared", "*non_shared", and // and the length of the value in "*shared", "*non_shared", and
// "*value_length", respectively. Will not derefence past "limit". // "*value_length", respectively. Will not dereference past "limit".
// //
// If any errors are detected, returns NULL. Otherwise, returns a // If any errors are detected, returns NULL. Otherwise, returns a
// pointer to the key delta (just past the three decoded values). // pointer to the key delta (just past the three decoded values).

2
table/block_builder.h

@ -21,7 +21,7 @@ class BlockBuilder {
// Reset the contents as if the BlockBuilder was just constructed. // Reset the contents as if the BlockBuilder was just constructed.
void Reset(); void Reset();
// REQUIRES: Finish() has not been callled since the last call to Reset(). // REQUIRES: Finish() has not been called since the last call to Reset().
// REQUIRES: key is larger than any previously added key // REQUIRES: key is larger than any previously added key
void Add(const Slice& key, const Slice& value); void Add(const Slice& key, const Slice& value);

2
table/format.cc

@ -48,7 +48,7 @@ Status Footer::DecodeFrom(Slice* input) {
const uint64_t magic = ((static_cast<uint64_t>(magic_hi) << 32) | const uint64_t magic = ((static_cast<uint64_t>(magic_hi) << 32) |
(static_cast<uint64_t>(magic_lo))); (static_cast<uint64_t>(magic_lo)));
if (magic != kTableMagicNumber) { if (magic != kTableMagicNumber) {
return Status::InvalidArgument("not an sstable (bad magic number)"); return Status::Corruption("not an sstable (bad magic number)");
} }
Status result = metaindex_handle_.DecodeFrom(input); Status result = metaindex_handle_.DecodeFrom(input);

14
table/table.cc

@ -41,7 +41,7 @@ Status Table::Open(const Options& options,
Table** table) { Table** table) {
*table = NULL; *table = NULL;
if (size < Footer::kEncodedLength) { if (size < Footer::kEncodedLength) {
return Status::InvalidArgument("file is too short to be an sstable"); return Status::Corruption("file is too short to be an sstable");
} }
char footer_space[Footer::kEncodedLength]; char footer_space[Footer::kEncodedLength];
@ -58,7 +58,11 @@ Status Table::Open(const Options& options,
BlockContents contents; BlockContents contents;
Block* index_block = NULL; Block* index_block = NULL;
if (s.ok()) { if (s.ok()) {
s = ReadBlock(file, ReadOptions(), footer.index_handle(), &contents); ReadOptions opt;
if (options.paranoid_checks) {
opt.verify_checksums = true;
}
s = ReadBlock(file, opt, footer.index_handle(), &contents);
if (s.ok()) { if (s.ok()) {
index_block = new Block(contents); index_block = new Block(contents);
} }
@ -92,6 +96,9 @@ void Table::ReadMeta(const Footer& footer) {
// TODO(sanjay): Skip this if footer.metaindex_handle() size indicates // TODO(sanjay): Skip this if footer.metaindex_handle() size indicates
// it is an empty block. // it is an empty block.
ReadOptions opt; ReadOptions opt;
if (rep_->options.paranoid_checks) {
opt.verify_checksums = true;
}
BlockContents contents; BlockContents contents;
if (!ReadBlock(rep_->file, opt, footer.metaindex_handle(), &contents).ok()) { if (!ReadBlock(rep_->file, opt, footer.metaindex_handle(), &contents).ok()) {
// Do not propagate errors since meta info is not needed for operation // Do not propagate errors since meta info is not needed for operation
@ -120,6 +127,9 @@ void Table::ReadFilter(const Slice& filter_handle_value) {
// We might want to unify with ReadBlock() if we start // We might want to unify with ReadBlock() if we start
// requiring checksum verification in Table::Open. // requiring checksum verification in Table::Open.
ReadOptions opt; ReadOptions opt;
if (rep_->options.paranoid_checks) {
opt.verify_checksums = true;
}
BlockContents block; BlockContents block;
if (!ReadBlock(rep_->file, opt, filter_handle, &block).ok()) { if (!ReadBlock(rep_->file, opt, filter_handle, &block).ok()) {
return; return;

2
util/bloom.cc

@ -29,7 +29,7 @@ class BloomFilterPolicy : public FilterPolicy {
} }
virtual const char* Name() const { virtual const char* Name() const {
return "leveldb.BuiltinBloomFilter"; return "leveldb.BuiltinBloomFilter2";
} }
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const { virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {

10
util/env_posix.cc

@ -3,8 +3,6 @@
// found in the LICENSE file. See the AUTHORS file for names of contributors. // found in the LICENSE file. See the AUTHORS file for names of contributors.
#if !defined(LEVELDB_PLATFORM_WINDOWS) #if !defined(LEVELDB_PLATFORM_WINDOWS)
#include <deque>
#include <set>
#include <dirent.h> #include <dirent.h>
#include <errno.h> #include <errno.h>
#include <fcntl.h> #include <fcntl.h>
@ -18,9 +16,8 @@
#include <sys/types.h> #include <sys/types.h>
#include <time.h> #include <time.h>
#include <unistd.h> #include <unistd.h>
#if defined(LEVELDB_PLATFORM_ANDROID) #include <deque>
#include <sys/stat.h> #include <set>
#endif
#include "leveldb/env.h" #include "leveldb/env.h"
#include "leveldb/slice.h" #include "leveldb/slice.h"
#include "port/port.h" #include "port/port.h"
@ -296,7 +293,8 @@ class PosixEnv : public Env {
public: public:
PosixEnv(); PosixEnv();
virtual ~PosixEnv() { virtual ~PosixEnv() {
fprintf(stderr, "Destroying Env::Default()\n"); char msg[] = "Destroying Env::Default()\n";
fwrite(msg, 1, sizeof(msg), stderr);
abort(); abort();
} }

6
util/hash.cc

@ -34,13 +34,13 @@ uint32_t Hash(const char* data, size_t n, uint32_t seed) {
// Pick up remaining bytes // Pick up remaining bytes
switch (limit - data) { switch (limit - data) {
case 3: case 3:
h += data[2] << 16; h += static_cast<unsigned char>(data[2]) << 16;
FALLTHROUGH_INTENDED; FALLTHROUGH_INTENDED;
case 2: case 2:
h += data[1] << 8; h += static_cast<unsigned char>(data[1]) << 8;
FALLTHROUGH_INTENDED; FALLTHROUGH_INTENDED;
case 1: case 1:
h += data[0]; h += static_cast<unsigned char>(data[0]);
h *= m; h *= m;
h ^= (h >> r); h ^= (h >> r);
break; break;

54
util/hash_test.cc

@ -0,0 +1,54 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "util/hash.h"
#include "util/testharness.h"
namespace leveldb {
class HASH { };
TEST(HASH, SignedUnsignedIssue) {
const unsigned char data1[1] = {0x62};
const unsigned char data2[2] = {0xc3, 0x97};
const unsigned char data3[3] = {0xe2, 0x99, 0xa5};
const unsigned char data4[4] = {0xe1, 0x80, 0xb9, 0x32};
const unsigned char data5[48] = {
0x01, 0xc0, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x14, 0x00, 0x00, 0x00,
0x00, 0x00, 0x04, 0x00,
0x00, 0x00, 0x00, 0x14,
0x00, 0x00, 0x00, 0x18,
0x28, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x02, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
};
ASSERT_EQ(Hash(0, 0, 0xbc9f1d34), 0xbc9f1d34);
ASSERT_EQ(
Hash(reinterpret_cast<const char*>(data1), sizeof(data1), 0xbc9f1d34),
0xef1345c4);
ASSERT_EQ(
Hash(reinterpret_cast<const char*>(data2), sizeof(data2), 0xbc9f1d34),
0x5b663814);
ASSERT_EQ(
Hash(reinterpret_cast<const char*>(data3), sizeof(data3), 0xbc9f1d34),
0x323c078f);
ASSERT_EQ(
Hash(reinterpret_cast<const char*>(data4), sizeof(data4), 0xbc9f1d34),
0xed21633a);
ASSERT_EQ(
Hash(reinterpret_cast<const char*>(data5), sizeof(data5), 0x12345678),
0xf333dabb);
}
} // namespace leveldb
int main(int argc, char** argv) {
return leveldb::test::RunAllTests();
}

9
util/logging.cc

@ -45,15 +45,6 @@ std::string EscapeString(const Slice& value) {
return r; return r;
} }
bool ConsumeChar(Slice* in, char c) {
if (!in->empty() && (*in)[0] == c) {
in->remove_prefix(1);
return true;
} else {
return false;
}
}
bool ConsumeDecimalNumber(Slice* in, uint64_t* val) { bool ConsumeDecimalNumber(Slice* in, uint64_t* val) {
uint64_t v = 0; uint64_t v = 0;
int digits = 0; int digits = 0;

4
util/logging.h

@ -32,10 +32,6 @@ extern std::string NumberToString(uint64_t num);
// Escapes any non-printable characters found in "value". // Escapes any non-printable characters found in "value".
extern std::string EscapeString(const Slice& value); extern std::string EscapeString(const Slice& value);
// If *in starts with "c", advances *in past the first character and
// returns true. Otherwise, returns false.
extern bool ConsumeChar(Slice* in, char c);
// Parse a human-readable number from "*in" into *value. On success, // Parse a human-readable number from "*in" into *value. On success,
// advances "*in" past the consumed number and sets "*val" to the // advances "*in" past the consumed number and sets "*val" to the
// numeric value. Otherwise, returns false and leaves *in in an // numeric value. Otherwise, returns false and leaves *in in an

Loading…
Cancel
Save