aboutsummaryrefslogtreecommitdiffstats
path: root/log_db
diff options
context:
space:
mode:
Diffstat (limited to 'log_db')
-rw-r--r--log_db/benches/benchmark.rs82
-rw-r--r--log_db/benches/utils.rs75
-rw-r--r--log_db/src/common.rs2
-rw-r--r--log_db/src/lib.rs6
4 files changed, 91 insertions, 74 deletions
diff --git a/log_db/benches/benchmark.rs b/log_db/benches/benchmark.rs
index 6d21c2a..38f1eb3 100644
--- a/log_db/benches/benchmark.rs
+++ b/log_db/benches/benchmark.rs
@@ -5,30 +5,17 @@ use log_db::*;
use tempfile;
use utils::*;
-#[derive(Eq, PartialEq, Clone, Debug)]
-enum Field {
- Id,
- Name,
- Data,
-}
-
pub fn upsert_various_initial_sizes(c: &mut Criterion) {
let mut group = c.benchmark_group("upsert_various_initial_sizes");
- for size in [100, 1000, 10000, 100_000, 1_000_000, 10_000_000] {
+ for size in [0, 1000, 10_000, 100_000, 1_000_000, 10_000_000] {
let data_dir_obj = tempfile::tempdir().expect("Failed to get tmpdir");
let data_dir = &data_dir_obj
.path()
.to_str()
.expect("Failed to convert tmpdir path to str");
- let mut db = DB::configure()
+ let mut db = DB::<Inst>::configure()
.data_dir(&data_dir)
- .fields(&[
- (Field::Id, ValueType::int()),
- (Field::Name, ValueType::string()),
- (Field::Data, ValueType::bytes()),
- ])
- .primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB");
@@ -36,8 +23,8 @@ pub fn upsert_various_initial_sizes(c: &mut Criterion) {
group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, &_size| {
b.iter(|| {
- let record = random_record(0, size as i64 + 1);
- let _ = db.upsert(black_box(&record));
+ let inst = random_inst(0, size as i64 + 1);
+ let _ = db.upsert(black_box(inst));
});
});
}
@@ -46,25 +33,21 @@ pub fn upsert_various_initial_sizes(c: &mut Criterion) {
pub fn upsert_various_initial_sizes_compacted(c: &mut Criterion) {
let mut group = c.benchmark_group("upsert_various_initial_sizes_compacted");
- for size in [100, 1000, 10000, 100_000, 1_000_000, 10_000_000] {
+ for size in [0, 1000, 10_000, 100_000, 1_000_000, 10_000_000] {
let data_dir_obj = tempfile::tempdir().expect("Failed to get tmpdir");
let data_dir = &data_dir_obj
.path()
.to_str()
.expect("Failed to convert tmpdir path to str");
- let sample_record = random_record(0, 1);
- let record_length = sample_record.serialize().len();
+ let record_length = 1 + // tombstone tag
+ 1 + 8 + // int tag + int value
+ 1 + 8 + 5 + // string tag + string length + string value
+ 1 + 8 + 10; // bytes tag + bytes length + bytes value
- let mut db = DB::configure()
+ let mut db = DB::<Inst>::configure()
.data_dir(&data_dir)
- .fields(&[
- (Field::Id, ValueType::int()),
- (Field::Name, ValueType::string()),
- (Field::Data, ValueType::bytes()),
- ])
.segment_size(1000 * record_length)
- .primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB");
@@ -73,11 +56,16 @@ pub fn upsert_various_initial_sizes_compacted(c: &mut Criterion) {
.expect("Failed to do maintenance tasks");
group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, &_size| {
+ let mut i = 0;
b.iter(|| {
- let record = random_record(0, size as i64 + 1);
- let _ = db.upsert(black_box(&record));
- db.do_maintenance_tasks()
- .expect("Failed to do maintenance tasks");
+ let inst = random_inst(0, size as i64 + 1);
+ let _ = db.upsert(black_box(inst));
+
+ if i % 100 == 0 {
+ db.do_maintenance_tasks()
+ .expect("Failed to do maintenance tasks");
+ }
+ i += 1;
});
});
}
@@ -93,21 +81,15 @@ pub fn upsert_write_durability(c: &mut Criterion) {
.path()
.to_str()
.expect("Failed to convert tmpdir path to str");
- let mut db = DB::configure()
+ let mut db = DB::<Inst>::configure()
.data_dir(&data_dir)
- .fields(&[
- (Field::Id, ValueType::int()),
- (Field::Name, ValueType::string()),
- (Field::Data, ValueType::bytes()),
- ])
.write_durability(mode.clone())
- .primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB");
b.iter(|| {
- let record = random_record(0, 1000);
- let _ = db.upsert(black_box(&record));
+ let inst = random_inst(0, 1000);
+ let _ = db.upsert(black_box(inst));
});
});
}
@@ -116,20 +98,14 @@ pub fn upsert_write_durability(c: &mut Criterion) {
pub fn get_from_disk_various_initial_sizes(c: &mut Criterion) {
let mut group = c.benchmark_group("get_from_disk_various_initial_sizes");
- for size in [0, 10, 100, 1000, 3300, 6700, 10000, 50000, 100_000] {
+ for size in [0, 1000, 10_000, 100_000, 1_000_000, 10_000_000] {
let data_dir_obj = tempfile::tempdir().expect("Failed to get tmpdir");
let data_dir = &data_dir_obj
.path()
.to_str()
.expect("Failed to convert tmpdir path to str");
- let mut db = DB::configure()
+ let mut db = DB::<Inst>::configure()
.data_dir(&data_dir)
- .fields(&[
- (Field::Id, ValueType::int()),
- (Field::Name, ValueType::string()),
- (Field::Data, ValueType::bytes()),
- ])
- .primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB");
prefill_db(&mut db, size, false).expect("Failed to prefill DB");
@@ -146,20 +122,14 @@ pub fn get_from_disk_various_initial_sizes(c: &mut Criterion) {
pub fn get_from_disk_various_initial_sizes_compacted(c: &mut Criterion) {
let mut group = c.benchmark_group("get_from_disk_various_initial_sizes_compacted");
- for size in [0, 10, 100, 1000, 3300, 6700, 10000, 50000, 100_000] {
+ for size in [0, 1000, 10_000, 100_000, 1_000_000, 10_000_000] {
let data_dir_obj = tempfile::tempdir().expect("Failed to get tmpdir");
let data_dir = &data_dir_obj
.path()
.to_str()
.expect("Failed to convert tmpdir path to str");
- let mut db = DB::configure()
+ let mut db = DB::<Inst>::configure()
.data_dir(&data_dir)
- .fields(&[
- (Field::Id, ValueType::int()),
- (Field::Name, ValueType::string()),
- (Field::Data, ValueType::bytes()),
- ])
- .primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB");
diff --git a/log_db/benches/utils.rs b/log_db/benches/utils.rs
index 9aed90e..05ea30f 100644
--- a/log_db/benches/utils.rs
+++ b/log_db/benches/utils.rs
@@ -3,6 +3,57 @@ use rand::distributions::Alphanumeric;
use rand::Rng;
use std::fmt::Debug;
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub enum Field {
+ Id,
+ Name,
+ Data,
+}
+
+#[derive(PartialEq, Eq, Debug, Clone)]
+pub struct Inst {
+ id: i64,
+ name: String,
+ data: Vec<u8>,
+}
+impl Recordable for Inst {
+ type Field = Field;
+ fn schema() -> Vec<(Field, ValueType)> {
+ vec![
+ (Field::Id, ValueType::int()),
+ (Field::Name, ValueType::string()),
+ (Field::Data, ValueType::bytes()),
+ ]
+ }
+ fn primary_key() -> Self::Field {
+ Field::Id
+ }
+ fn into_record(self) -> Vec<Value> {
+ vec![
+ Value::Int(self.id),
+ Value::String(self.name),
+ Value::Bytes(self.data),
+ ]
+ }
+ fn from_record(record: Vec<Value>) -> Self {
+ let mut it = record.into_iter();
+ Inst {
+ id: match it.next().unwrap() {
+ Value::Int(id) => id,
+ _ => panic!("Expected Int"),
+ },
+ name: match it.next().unwrap() {
+ Value::String(name) => name,
+ _ => panic!("Expected String"),
+ },
+ data: match it.next().unwrap() {
+ Value::Bytes(data) => data,
+ _ => panic!("Expected Bytes"),
+ },
+ }
+ }
+}
+
// Function to generate a random integer
pub fn random_int(from: i64, to: i64) -> i64 {
let mut rng = rand::thread_rng();
@@ -21,23 +72,19 @@ pub fn random_bytes(len: usize) -> Vec<u8> {
(0..len).map(|_| rng.gen()).collect()
}
-// Function to generate a random record
-pub fn random_record(from_id: i64, to_id: i64) -> Record {
- Record::from(&[
- Value::Int(random_int(from_id, to_id)), // Random int value between 0..1000
- Value::String(random_string(5)), // Random string of length 5
- Value::Bytes(random_bytes(10)), // Random bytes of length 10
- ])
+// Function to generate a random Inst
+pub fn random_inst(from_id: i64, to_id: i64) -> Inst {
+ Inst {
+ id: random_int(from_id, to_id), // Random int value between 0..1000
+ name: random_string(5), // Random string of length 5
+ data: random_bytes(10), // Random bytes of length 10
+ }
}
-pub fn prefill_db<T: Eq + Clone + Debug>(
- db: &mut DB<T>,
- n_records: usize,
- compact: bool,
-) -> Result<(), DBError> {
+pub fn prefill_db(db: &mut DB<Inst>, n_records: usize, compact: bool) -> Result<(), DBError> {
for _ in 0..n_records {
- let record = random_record(0, n_records as i64);
- db.upsert(&record)?;
+ let inst = random_inst(0, n_records as i64);
+ db.upsert(inst)?;
if compact {
db.do_maintenance_tasks()?;
}
diff --git a/log_db/src/common.rs b/log_db/src/common.rs
index 4f5751d..8bdcd9a 100644
--- a/log_db/src/common.rs
+++ b/log_db/src/common.rs
@@ -216,7 +216,7 @@ impl MetadataHeader {
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum ReadConsistency {
/// Reads by client A are guaranteed to see writes by themselves and any writes by other clients B
- /// that were done before last index refresh.
+ /// that were done before last index refresh. You must call `refresh_indexes()` manually to refresh indexes.
Eventual,
/// Reads by client A are guaranteed to see all writes. This is slower: all reads must first
/// refresh indexes.
diff --git a/log_db/src/lib.rs b/log_db/src/lib.rs
index e9a25c1..4d9b82f 100644
--- a/log_db/src/lib.rs
+++ b/log_db/src/lib.rs
@@ -247,7 +247,9 @@ impl<R: Recordable> DB<R> {
Ok(db)
}
- fn refresh_indexes(&mut self) -> Result<(), DBError> {
+ /// Refresh the in-memory indexes from the log files.
+ /// This needs to only be called if the read consistency is set to `ReadConsistency::Eventual`.
+ pub fn refresh_indexes(&mut self) -> Result<(), DBError> {
let active_symlink_path = self.data_dir.join(ACTIVE_SYMLINK_FILENAME);
let active_target = fs::read_link(active_symlink_path)?;
let active_metadata_path = self.data_dir.join(active_target);
@@ -796,8 +798,6 @@ impl<R: Recordable> DB<R> {
self.active_metadata_file.unlock()?;
- self.refresh_indexes()?;
-
Ok(())
}

Want to contribute? Host your changes somewhere and send me the link: jan (at) sydanviita.fi. Also, check out my website.


forge.jan.systems — powered by cgit