aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--README.md108
-rw-r--r--log_db/benches/benchmark.rs12
-rw-r--r--log_db/src/lib.rs15
-rw-r--r--log_db/tests/integration.rs30
-rw-r--r--py_bindings/.gitignore72
-rw-r--r--py_bindings/pyproject.toml15
-rw-r--r--py_bindings/src/lib.rs234
7 files changed, 448 insertions, 38 deletions
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..9ec2def
--- /dev/null
+++ b/README.md
@@ -0,0 +1,108 @@
+# LogDB
+
+An educational endeavor in implementing a log-structured database with a focus on simplicity, understandability and performance.
+
+LogDB has the following features:
+
+- Log-structured single-table storage, based on a durable append-only log
+- In-memory indexes for fast lookups (primary and secondary)
+- Log rotation and compaction for efficient storage even with larger databases
+- Multiple concurrent readers and a single writer, using filesystem locks for synchronization
+- Simple data types: `Int`, `Float`, `String`, `Bytes` (arbitrary bytestring), and `Null`
+- A Rust API for interacting with the database, as well as Python bindings for the Rust API
+
+LogDB does not support:
+
+- Authentication or authorization in any capacity
+- Multiple tables
+- Schema evolution, other than adding new nullable fields
+
+Possible future features:
+- Transactions
+
+## Inspiration
+
+The most significant sources of inspiration for LogDB are:
+- [SQLite](https://www.sqlite.org/index.html) for its filesystem storage and locking mechanisms.
+- [Designing Data-Intensive Applications (book)](https://www.oreilly.com/library/view/designing-data-intensive-applications/9781491903063/)
+ for its excellent overview of database internals and in-depth analysis of log-structured storage.
+ LogDB is heavily based on the design outlined in chapter 3.
+
+## Usage in Rust
+
+Add LogDB as a dependency in your `Cargo.toml`.
+
+```toml
+[dependencies]
+log_db = { git = "https://github.com/jantuomi/log_db.git" }
+```
+
+```rust
+use log_db::*;
+
+// Configure and initialize the database
+let mut db = DB::configure()
+ .fields(vec![
+ (Field::Id, RecordField::int()),
+ (Field::Data, RecordField::bytes()),
+ ])
+ .primary_key(Field::Id)
+ .initialize()?;
+
+// Define a record matching the `fields` schema
+let record = Record {
+ values: vec![
+ RecordValue::Int(1),
+ RecordValue::Bytes(vec![1, 2, 3, 4]),
+ ],
+};
+
+// Insert or update the record based on the primary key (ID, first value)
+db.upsert(&record)?;
+
+// Get the record by primary key
+let found = db.get(RecordValue::Int(1))?;
+```
+
+## Tests
+
+Run the tests with:
+
+```sh
+cargo test
+```
+
+Generate the benchmark reports with:
+
+```sh
+cargo bench
+```
+
+## Python bindings
+
+To build the Python bindings, run:
+
+```sh
+cd py_bindings
+python -m venv venv
+. venv/bin/activate
+maturin develop # for the development version, or
+maturin build --release # for the release version
+```
+
+Then you can use the Python bindings like so:
+
+```python
+from log_db_py import DB, RecordValue, RecordField, Record
+
+config = DB.configure();
+config.primary_key = "id"
+config.fields = [("id", RecordField.int().nullable())]
+
+db = config.initialize()
+db.upsert(Record(RecordValue.int(10)))
+```
+
+## Copyright and license
+
+LogDB is licensed under the Apache License, Version 2.0. © 2024 Jan Tuomi.
diff --git a/log_db/benches/benchmark.rs b/log_db/benches/benchmark.rs
index 384c9b7..5174bdb 100644
--- a/log_db/benches/benchmark.rs
+++ b/log_db/benches/benchmark.rs
@@ -25,7 +25,7 @@ pub fn upsert_various_initial_sizes(c: &mut Criterion) {
.expect("Failed to convert tmpdir path to str");
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -60,7 +60,7 @@ pub fn upsert_write_durability(c: &mut Criterion) {
.expect("Failed to convert tmpdir path to str");
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -90,7 +90,7 @@ pub fn get_from_disk_various_initial_sizes(c: &mut Criterion) {
let mut db = DB::configure()
.data_dir(&data_dir)
.memtable_capacity(0)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -122,7 +122,7 @@ pub fn get_various_memtable_capacities(c: &mut Criterion) {
// Create a db instance for prefilling
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -140,7 +140,7 @@ pub fn get_various_memtable_capacities(c: &mut Criterion) {
group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, &_size| {
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -175,7 +175,7 @@ fn reverse_read_file_with_various_buffer_sizes(c: &mut Criterion) {
// Create a db instance for prefilling
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
diff --git a/log_db/src/lib.rs b/log_db/src/lib.rs
index 6787cf1..9c25dd8 100644
--- a/log_db/src/lib.rs
+++ b/log_db/src/lib.rs
@@ -21,19 +21,19 @@ use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use std::thread;
-pub struct ConfigBuilder<'a, Field: Eq + Clone + Debug> {
+pub struct ConfigBuilder<Field: Eq + Clone + Debug> {
data_dir: Option<String>,
segment_size: Option<usize>,
memtable_capacity: Option<usize>,
- fields: Option<&'a Vec<(Field, RecordField)>>,
+ fields: Option<Vec<(Field, RecordField)>>,
primary_key: Option<Field>,
secondary_keys: Option<Vec<Field>>,
memtable_evict_policy: Option<MemtableEvictPolicy>,
write_durability: Option<WriteDurability>,
}
-impl<'a, Field: Eq + Clone + Debug> ConfigBuilder<'a, Field> {
- pub fn new() -> ConfigBuilder<'a, Field> {
+impl<'a, Field: Eq + Clone + Debug> ConfigBuilder<Field> {
+ pub fn new() -> ConfigBuilder<Field> {
ConfigBuilder::<Field> {
data_dir: None,
segment_size: None,
@@ -68,8 +68,8 @@ impl<'a, Field: Eq + Clone + Debug> ConfigBuilder<'a, Field> {
}
/// The field schema of the database.
- pub fn fields(&mut self, fields: &'a Vec<(Field, RecordField)>) -> &mut Self {
- self.fields = Some(fields);
+ pub fn fields(&mut self, fields: Vec<(Field, RecordField)>) -> &mut Self {
+ self.fields = Some(fields.clone());
self
}
@@ -114,6 +114,7 @@ impl<'a, Field: Eq + Clone + Debug> ConfigBuilder<'a, Field> {
memtable_capacity: self.memtable_capacity.unwrap_or(1_000_000),
fields: self
.fields
+ .as_ref()
.ok_or(io::Error::new(
io::ErrorKind::InvalidInput,
"Required config value \"fields\" is not set",
@@ -161,7 +162,7 @@ pub struct DB<Field: Eq + Clone + Debug> {
impl<Field: Eq + Clone + Debug> DB<Field> {
/// Create a new database configuration builder.
- pub fn configure() -> ConfigBuilder<'static, Field> {
+ pub fn configure() -> ConfigBuilder<Field> {
ConfigBuilder::new()
}
diff --git a/log_db/tests/integration.rs b/log_db/tests/integration.rs
index a2453b2..76ed853 100644
--- a/log_db/tests/integration.rs
+++ b/log_db/tests/integration.rs
@@ -38,7 +38,7 @@ fn test_initialize() {
let data_dir = tmp_dir();
let _db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -53,7 +53,7 @@ fn test_upsert_and_get_with_primary_memtable() {
let data_dir = tmp_dir();
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -86,7 +86,7 @@ fn test_upsert_and_get_without_memtable() {
let mut db = DB::configure()
.data_dir(&data_dir)
.memtable_capacity(0)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string().nullable()),
(Field::Data, RecordField::bytes()),
@@ -164,7 +164,7 @@ fn test_upsert_fails_on_null_in_non_nullable_field() {
let data_dir = tmp_dir();
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![(Field::Id, RecordField::int())])
+ .fields(vec![(Field::Id, RecordField::int())])
.primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB instance");
@@ -181,7 +181,7 @@ fn test_upsert_fails_on_invalid_number_of_values() {
let data_dir = tmp_dir();
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -205,7 +205,7 @@ fn test_upsert_fails_on_invalid_value_type() {
let data_dir = tmp_dir();
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -229,7 +229,7 @@ fn test_upsert_and_get_from_secondary_memtable() {
let data_dir = tmp_dir();
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -291,7 +291,7 @@ fn test_initialize_and_read_from_primary_memtable_fixture_db2() {
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -326,7 +326,7 @@ fn test_initialize_without_memtables_fixture_db3() {
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Name, RecordField::string()),
(Field::Data, RecordField::bytes()),
@@ -357,7 +357,7 @@ fn test_multiple_writing_threads() {
threads.push(thread::spawn(move || {
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![(Field::Id, RecordField::int())])
+ .fields(vec![(Field::Id, RecordField::int())])
.primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB instance");
@@ -376,7 +376,7 @@ fn test_multiple_writing_threads() {
// Read the records
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![(Field::Id, RecordField::int())])
+ .fields(vec![(Field::Id, RecordField::int())])
.primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB instance");
@@ -406,7 +406,7 @@ fn test_one_writer_and_multiple_reading_threads() {
threads.push(thread::spawn(move || {
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![(Field::Id, RecordField::int())])
+ .fields(vec![(Field::Id, RecordField::int())])
.primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB instance");
@@ -437,7 +437,7 @@ fn test_one_writer_and_multiple_reading_threads() {
threads.push(thread::spawn(move || {
let mut db = DB::configure()
.data_dir(&data_dir)
- .fields(&vec![(Field::Id, RecordField::int())])
+ .fields(vec![(Field::Id, RecordField::int())])
.primary_key(Field::Id)
.initialize()
.expect("Failed to initialize DB instance");
@@ -462,7 +462,7 @@ fn test_literal_escape_is_escaped() {
let mut db = DB::configure()
.data_dir(&data_dir)
.memtable_capacity(0) // disable memtables
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Data, RecordField::bytes()),
])
@@ -505,7 +505,7 @@ fn test_log_is_rotated_when_capacity_reached() {
.data_dir(&data_dir)
.memtable_capacity(0) // disable memtables
.segment_size(10 * record_len) // small log segment size
- .fields(&vec![
+ .fields(vec![
(Field::Id, RecordField::int()),
(Field::Data, RecordField::bytes()),
])
diff --git a/py_bindings/.gitignore b/py_bindings/.gitignore
new file mode 100644
index 0000000..c8f0442
--- /dev/null
+++ b/py_bindings/.gitignore
@@ -0,0 +1,72 @@
+/target
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version
diff --git a/py_bindings/pyproject.toml b/py_bindings/pyproject.toml
new file mode 100644
index 0000000..01bdbce
--- /dev/null
+++ b/py_bindings/pyproject.toml
@@ -0,0 +1,15 @@
+[build-system]
+requires = ["maturin>=1.7,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "log_db_py"
+requires-python = ">=3.8"
+classifiers = [
+ "Programming Language :: Rust",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Programming Language :: Python :: Implementation :: PyPy",
+]
+dynamic = ["version"]
+[tool.maturin]
+features = ["pyo3/extension-module"]
diff --git a/py_bindings/src/lib.rs b/py_bindings/src/lib.rs
index abee9f2..fcf9052 100644
--- a/py_bindings/src/lib.rs
+++ b/py_bindings/src/lib.rs
@@ -1,16 +1,230 @@
-use log_db::*;
+use log_db;
+use pyo3::exceptions::PyException;
+use pyo3::prelude::*;
+use pyo3::types::PyTuple;
-pub fn add(left: u64, right: u64) -> u64 {
- left + right
+type Field = String;
+
+#[pyclass]
+#[derive(Clone)]
+struct RecordField {
+ record_field: log_db::RecordField,
+}
+
+#[pymethods]
+impl RecordField {
+ #[staticmethod]
+ fn int() -> Self {
+ RecordField {
+ record_field: log_db::RecordField::int(),
+ }
+ }
+
+ #[staticmethod]
+ fn float() -> Self {
+ RecordField {
+ record_field: log_db::RecordField::float(),
+ }
+ }
+
+ #[staticmethod]
+ fn string() -> Self {
+ RecordField {
+ record_field: log_db::RecordField::string(),
+ }
+ }
+
+ #[staticmethod]
+ fn bytes() -> Self {
+ RecordField {
+ record_field: log_db::RecordField::bytes(),
+ }
+ }
+
+ fn nullable(&self) -> Self {
+ RecordField {
+ record_field: self.record_field.clone().nullable(),
+ }
+ }
}
-#[cfg(test)]
-mod tests {
- use super::*;
+#[pyclass]
+#[derive(Clone)]
+struct MemtableEvictPolicy {
+ memtable_evict_policy: log_db::MemtableEvictPolicy,
+}
+
+#[pyclass]
+#[derive(Clone)]
+struct WriteDurability {
+ write_durability: log_db::WriteDurability,
+}
+
+#[pyclass]
+struct Config {
+ #[pyo3(get, set)]
+ data_dir: Option<String>,
+ #[pyo3(get, set)]
+ segment_size: Option<usize>,
+ #[pyo3(get, set)]
+ memtable_capacity: Option<usize>,
+ #[pyo3(get, set)]
+ fields: Option<Vec<(Field, RecordField)>>,
+ #[pyo3(get, set)]
+ primary_key: Option<Field>,
+ #[pyo3(get, set)]
+ secondary_keys: Option<Vec<Field>>,
+ #[pyo3(get, set)]
+ memtable_evict_policy: Option<MemtableEvictPolicy>,
+ #[pyo3(get, set)]
+ write_durability: Option<WriteDurability>,
+}
+
+#[pymethods]
+impl Config {
+ pub fn initialize(&self) -> PyResult<DB> {
+ let mut config = log_db::DB::configure();
+ if self.data_dir.is_some() {
+ config.data_dir(&self.data_dir.as_ref().unwrap().to_string());
+ }
+ if self.segment_size.is_some() {
+ config.segment_size(self.segment_size.unwrap());
+ }
+ if self.memtable_capacity.is_some() {
+ config.memtable_capacity(self.memtable_capacity.unwrap());
+ }
+ if self.fields.is_some() {
+ let mut fields = Vec::new();
+ for (field, record_field) in self.fields.as_ref().unwrap() {
+ fields.push((field.to_string(), record_field.record_field.clone()));
+ }
+ config.fields(fields);
+ }
+ if self.primary_key.is_some() {
+ config.primary_key(self.primary_key.as_ref().unwrap().to_string());
+ }
+ if self.secondary_keys.is_some() {
+ let tmp = self.secondary_keys.as_ref().unwrap();
+ config.secondary_keys(tmp.clone());
+ }
+ if self.memtable_evict_policy.is_some() {
+ let tmp = self.memtable_evict_policy.as_ref().unwrap();
+ config.memtable_evict_policy(tmp.memtable_evict_policy.clone());
+ }
+ if self.write_durability.is_some() {
+ let tmp = self.write_durability.as_ref().unwrap();
+ config.write_durability(tmp.write_durability.clone());
+ }
- #[test]
- fn it_works() {
- let result = add(2, 2);
- assert_eq!(result, 4);
+ let db = config.initialize().map_err(|e| PyException::new_err(e))?;
+ Ok(DB { db })
}
}
+
+#[pyclass]
+#[derive(Clone)]
+struct RecordValue {
+ record_value: log_db::RecordValue,
+}
+
+#[pymethods]
+impl RecordValue {
+ #[staticmethod]
+ fn int(value: i64) -> Self {
+ RecordValue {
+ record_value: log_db::RecordValue::Int(value),
+ }
+ }
+
+ #[staticmethod]
+ fn float(value: f64) -> Self {
+ RecordValue {
+ record_value: log_db::RecordValue::Float(value),
+ }
+ }
+
+ #[staticmethod]
+ fn string(value: &str) -> Self {
+ RecordValue {
+ record_value: log_db::RecordValue::String(value.to_string()),
+ }
+ }
+
+ #[staticmethod]
+ fn bytes(value: &[u8]) -> Self {
+ RecordValue {
+ record_value: log_db::RecordValue::Bytes(value.to_vec()),
+ }
+ }
+
+ #[staticmethod]
+ fn null() -> Self {
+ RecordValue {
+ record_value: log_db::RecordValue::Null,
+ }
+ }
+}
+
+#[pyclass]
+struct Record {
+ values: Vec<RecordValue>,
+}
+
+#[pymethods]
+impl Record {
+ #[new]
+ #[pyo3(signature = (*py_args))]
+ fn new(py_args: Vec<RecordValue>) -> Self {
+ Record { values: py_args }
+ }
+}
+
+#[pyclass]
+struct DB {
+ db: log_db::DB<Field>,
+}
+
+#[pymethods]
+impl DB {
+ fn upsert(&mut self, record: &Record) -> PyResult<()> {
+ let values = record
+ .values
+ .iter()
+ .map(|v| v.record_value.clone())
+ .collect();
+
+ self.db
+ .upsert(&log_db::Record { values })
+ .map_err(|e| PyException::new_err(e))?;
+ Ok(())
+ }
+
+ #[staticmethod]
+ pub fn configure() -> Config {
+ Config {
+ data_dir: None,
+ segment_size: None,
+ memtable_capacity: None,
+ fields: None,
+ primary_key: None,
+ secondary_keys: None,
+ memtable_evict_policy: None,
+ write_durability: None,
+ }
+ }
+}
+
+// #[pyfunction]
+// fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
+// Ok((a + b).to_string())
+// }
+
+#[pymodule]
+fn log_db_py(m: &Bound<'_, PyModule>) -> PyResult<()> {
+ //m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
+ m.add_class::<DB>()?;
+ m.add_class::<RecordField>()?;
+ m.add_class::<RecordValue>()?;
+ m.add_class::<Record>()?;
+ Ok(())
+}

Want to contribute? Host your changes somewhere and send me the link: jan (at) sydanviita.fi. Also, check out my website.


forge.jan.systems — powered by cgit