From e17048eddfe2df86abd2d498da2a33b9c3dd8a72 Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Fri, 2 May 2025 23:38:23 +0300 Subject: Add sort_asc param, impl demo chat app --- demo/app.py | 231 +++++++------------------------- demo/requirements.txt | 1 + demo/static/styles.css | 76 +---------- demo/templates/base.html.j2 | 17 ++- demo/templates/frag_aside.html.j2 | 27 ---- demo/templates/frag_aside_link.html.j2 | 9 -- demo/templates/frag_error.html.j2 | 16 --- demo/templates/frag_form_delete.html.j2 | 7 - demo/templates/frag_form_find.html.j2 | 7 - demo/templates/frag_form_range.html.j2 | 21 --- demo/templates/frag_form_upsert.html.j2 | 5 - demo/templates/frag_messages.html.j2 | 12 ++ demo/templates/frag_results.html.j2 | 18 --- demo/templates/page_error.html.j2 | 9 +- demo/templates/page_index.html.j2 | 27 ++++ demo/templates/page_main.html.j2 | 11 -- log_db/src/common.rs | 2 + log_db/src/engine.rs | 23 +++- log_db/tests/integration.rs | 3 + py_bindings/log_db.pyi | 6 +- py_bindings/src/lib.rs | 36 ++++- 21 files changed, 175 insertions(+), 389 deletions(-) delete mode 100644 demo/templates/frag_aside.html.j2 delete mode 100644 demo/templates/frag_aside_link.html.j2 delete mode 100644 demo/templates/frag_error.html.j2 delete mode 100644 demo/templates/frag_form_delete.html.j2 delete mode 100644 demo/templates/frag_form_find.html.j2 delete mode 100644 demo/templates/frag_form_range.html.j2 delete mode 100644 demo/templates/frag_form_upsert.html.j2 create mode 100644 demo/templates/frag_messages.html.j2 delete mode 100644 demo/templates/frag_results.html.j2 create mode 100644 demo/templates/page_index.html.j2 delete mode 100644 demo/templates/page_main.html.j2 diff --git a/demo/app.py b/demo/app.py index be79432..bab20f0 100644 --- a/demo/app.py +++ b/demo/app.py @@ -1,19 +1,23 @@ from dotenv import load_dotenv +from werkzeug.utils import redirect load_dotenv() import os import sys import traceback import tempfile -from typing import cast from flask import Flask, render_template, request from flask_compress import Compress # type: ignore +from flask_apscheduler import APScheduler #type: ignore +import datetime +from dataclasses import dataclass -import log_db +import uuid from log_db import DB, Bound, Value app = Flask(__name__) Compress(app) +scheduler = APScheduler() BASE_URL = os.environ["BASE_URL"] DB_DIR = os.environ["DB_DIR"] if "DB_DIR" in os.environ else tempfile.TemporaryDirectory().name @@ -22,26 +26,27 @@ print("""\n""" f"""BASE_URL: {BASE_URL}\n""" f"""DB_DIR: {DB_DIR}\n""") -db_fields = ["id", "name"] -db_types = ["int", "string"] +db_fields = ["id", "ts", "username", "message"] +db_types = ["string", "int", "string", "string"] + +@dataclass +class Message: + id: str + ts: str + username: str + msg: str db = DB \ .configure() \ .data_dir(DB_DIR) \ .fields(db_fields) \ .primary_key("id") \ - .secondary_keys(["name"]) \ + .secondary_keys(["ts"]) \ .initialize() def error(e: str, code: int): error_text = f"HTTP {code}: {e}" - #return render_template("page_error.html.j2", error = error_text) - - htmp_target = request.form.get("htmp") or request.args.get("htmp") - if htmp_target: - return render_template("frag_error.html.j2", error = error_text, container = htmp_target) - else: - return render_template("page_error.html.j2", error = error_text) + return render_template("page_error.html.j2", error = error_text) @app.errorhandler(Exception) def error_handler(e: Exception): @@ -52,191 +57,53 @@ def error_handler(e: Exception): return error("Internal Server Error", 500) @app.get("/") -def index_default(): - return index("find") - -@app.get("/") -def index(op: str): - if op not in ["find", "range", "upsert", "delete"]: - return error("Invalid operation", 400) - - rows = db.range_by("id", Bound.unbounded(), Bound.unbounded(), limit=100) - rows = [[value_to_str(v) for v in row] for row in rows] +def index(): + messages = get_messages() htmp_target = request.form.get("htmp") or request.args.get("htmp") if htmp_target: - return render_template(f"frag_form_{op}.html.j2") + return render_template("frag_messages.html.j2", messages = messages) else: - return render_template('page_main.html.j2', - selected_form = f"frag_form_{op}.html.j2", - field_names = ["id", "name"], - rows = rows, - ) + return render_template("page_index.html.j2", messages = messages) -@app.post("/find") +@app.post("/") def query_find(): - field = request.form.get("field") - if not field: raise ValueError("Field is required") + username = request.form.get("username") + if not username: raise ValueError("username is required") - values = request.form.get("values") - if not values: raise ValueError("Values are required") + message = request.form.get("message") + if not message: raise ValueError("message is required") - field_index = db_fields.index(field) - if field_index == -1: raise ValueError(f"Field '{field}' not found") + id = uuid.uuid4().hex + ts = int(datetime.datetime.now().timestamp()) - try: - values = [cast_to_value(field, v) for v in values.split("\n")] - values = [v for v in values if v is not None] - except ValueError as e: - return error(str(e), 400) - - tagged_rows = db.batch_find_by(field, values, limit=100) - rows = [[value_to_str(v) for v in row] for (_, row) in tagged_rows] + db.upsert([Value.string(id), Value.int(ts), Value.string(username), Value.string(message)]) htmp_target = request.form.get("htmp") or request.args.get("htmp") if htmp_target: - return render_template("frag_results.html.j2", - field_names = ["id", "name"], - rows = rows, - ) + messages = get_messages() + return render_template("frag_messages.html.j2", messages = messages) else: - return render_template('page_main.html.j2', - selected_form = "frag_form_find.html.j2", - field_names = ["id", "name"], - rows = rows, - ) + return redirect("/") -@app.post("/range") -def query_range(): - field = request.form.get("field") - if not field: raise ValueError("Field is required") - - from_type = request.form.get("from_type") - if not from_type: raise ValueError("From type is required") - - to_type = request.form.get("to_type") - if not to_type: raise ValueError("To type is required") - - field_index = db_fields.index(field) - if field_index == -1: raise ValueError(f"Field '{field}' not found") - - match from_type: - case "unbounded": - bound_lower = Bound.unbounded() - case "included": - from_value = request.form.get("from_value") - try: - if not from_value: raise ValueError("From value is required") - from_value = cast_to_value(field, from_value) - except ValueError as e: - return error(str(e), 400) - bound_lower = Bound.included(cast(Value, from_value)) - case "excluded": - from_value = request.form.get("from_value") - try: - if not from_value: raise ValueError("From value is required") - from_value = cast_to_value(field, from_value) - except ValueError as e: - return error(str(e), 400) - bound_lower = Bound.excluded(cast(Value, from_value)) - case _: - return error("Invalid from type", 400) - - match to_type: - case "unbounded": - bound_upper = Bound.unbounded() - case "included": - to_value = request.form.get("to_value") - try: - if not to_value: raise ValueError("To value is required") - to_value = cast_to_value(field, to_value) - except ValueError as e: - return error(str(e), 400) - bound_upper = Bound.included(cast(Value, to_value)) - case "excluded": - to_value = request.form.get("to_value") - try: - if not to_value: raise ValueError("To value is required") - to_value = cast_to_value(field, to_value) - except ValueError as e: - return error(str(e), 400) - bound_upper = Bound.excluded(cast(Value, to_value)) - case _: - return error("Invalid to type", 400) - - tagged_rows = db.range_by(field, bound_lower, bound_upper, limit=100) - rows = [[value_to_str(v) for v in row] for row in tagged_rows] +def get_messages(): + messages = db.range_by("ts", Bound.unbounded(), Bound.unbounded(), limit=100, sort_asc=False) - htmp_target = request.form.get("htmp") or request.args.get("htmp") - if htmp_target: - return render_template("frag_results.html.j2", - field_names = ["id", "name"], - rows = rows, - ) - else: - return render_template('page_main.html.j2', - selected_form = "frag_form_range.html.j2", - field_names = ["id", "name"], - rows = rows, + return [ + Message( + id=id.as_string(), + ts=datetime.datetime.fromtimestamp(ts.as_int()).isoformat(), + username=username.as_string(), + msg=msg.as_string() ) + for [id, ts, username, msg] in messages + ] -@app.post("/delete") -def query_delete(): - field = request.form.get("field") - if not field: raise ValueError("Field is required") - - value = request.form.get("value") - if not value: raise ValueError("Value is required") - - field_index = db_fields.index(field) - if field_index == -1: raise ValueError(f"Field '{field}' not found") - - try: - value = cast_to_value(field, value) - except ValueError as e: - return error(str(e), 400) - - rows = db.delete_by(field, cast(Value, value)) - rows = [[value_to_str(v) for v in row] for row in rows] - - htmp_target = request.form.get("htmp") or request.args.get("htmp") - if htmp_target: - return render_template("frag_results.html.j2", - field_names = ["id", "name"], - rows = rows, - ) - else: - return render_template('page_main.html.j2', - selected_form = "frag_form_delete.html.j2", - field_names = ["id", "name"], - rows = rows, - ) - - -# Utils - -def cast_to_value(field: str, str_value: str) -> Value | None: - str_value = str_value.strip() - if str_value == "": return None - if str_value[0] == "\"": - if str_value[-1] != "\"": raise ValueError(f"Invalid string: {str_value}") - str_value = str_value[1:-1] - - field_index = db_fields.index(field) - if field_index == -1: raise ValueError(f"Field '{field}' not found") - - type = db_types[field_index] - - match type: - case "int": return Value.int(int(str_value)) - case "string": return Value.string(str_value) - case _: raise ValueError(f"Unsupported type: {type}") +@scheduler.task('interval', id='my_job', minutes=1) +def run_maintenance(): + db.do_maintenance_tasks() -def value_to_str(value: Value) -> str: - match value.kind(): - case log_db.VALUE_INT: return f"{str(value.as_int())} (int)" - case log_db.VALUE_STRING: return f"\"{value.as_string()}\" (string)" - case log_db.VALUE_DECIMAL: return f"{value.as_decimal()} (decimal)" - case log_db.VALUE_BYTES: return f"{value.as_bytes()} (bytes)" - case log_db.VALUE_NULL: return "null" - case _: raise ValueError(f"Unsupported value kind: {value.kind()}") +if __name__ == '__main__': + scheduler.init_app(app) + scheduler.start() + app.run() diff --git a/demo/requirements.txt b/demo/requirements.txt index b6b9bff..9d4ef16 100644 --- a/demo/requirements.txt +++ b/demo/requirements.txt @@ -9,3 +9,4 @@ MarkupSafe==3.0.2 python-dotenv==1.0.1 Werkzeug==3.1.3 zstandard==0.23.0 +Flask-APScheduler==1.13.1 diff --git a/demo/static/styles.css b/demo/static/styles.css index 82bed74..bcf99dc 100644 --- a/demo/static/styles.css +++ b/demo/static/styles.css @@ -31,16 +31,13 @@ body { line-height: 24px; } -#container { +#messages { + max-height: 60vh; + overflow-y: scroll; + border: 1px solid black; + padding: 8px; display: flex; - flex-flow: row wrap; - justify-content: space-between; - align-items: flex-start; -} - -a, -.contrast { - color: var(--blue); + flex-direction: column-reverse; } h1 { @@ -77,64 +74,3 @@ input[type="submit"] { margin-top: 20px; } - -button.link, -input[type="submit"].link { - background: none; - color: var(--blue); - padding: 0; - margin: 0; - text-decoration: underline; - font-family: "Inclusive Sans"; -} - -button.red, -input[type="submit"].red { - background-color: var(--red); -} - -button.blue, -input[type="submit"].blue { - background-color: var(--blue); -} - -button.yellow, -input[type="submit"].yellow { - background-color: var(--yellow); -} - -input[type="text"], -input[type*="date"] { - font-size: 16px; - color: var(--black); -} - -input[type="text"]:disabled, -input[type*="date"]:disabled { - color: var(--gray); -} - -aside { - width: 250px; -} - -main { - flex: 1; - overflow-y: scroll; - max-height: 100%; -} - -table { - width: 100%; -} - -th, -td { - min-width: 100px; - text-align: left; - padding: 7px 5px; -} - -tr:nth-of-type(odd) td { - background-color: var(--light-gray-2); -} diff --git a/demo/templates/base.html.j2 b/demo/templates/base.html.j2 index 47b3f6c..06ab5fc 100644 --- a/demo/templates/base.html.j2 +++ b/demo/templates/base.html.j2 @@ -6,14 +6,25 @@ {% block head_meta %} - Database browser - + Demo chat app + {% endblock %} -

Database browser

+

Demo chat app

{% block content %}{% endblock %} + + + diff --git a/demo/templates/frag_aside.html.j2 b/demo/templates/frag_aside.html.j2 deleted file mode 100644 index 6c5658f..0000000 --- a/demo/templates/frag_aside.html.j2 +++ /dev/null @@ -1,27 +0,0 @@ -{# -Params - selected_form Form fragment to include -#} - - diff --git a/demo/templates/frag_aside_link.html.j2 b/demo/templates/frag_aside_link.html.j2 deleted file mode 100644 index 9cd6041..0000000 --- a/demo/templates/frag_aside_link.html.j2 +++ /dev/null @@ -1,9 +0,0 @@ -{# -Params - text Text to render in the link - href URL to link to -#} - -
- -
diff --git a/demo/templates/frag_error.html.j2 b/demo/templates/frag_error.html.j2 deleted file mode 100644 index c5034cd..0000000 --- a/demo/templates/frag_error.html.j2 +++ /dev/null @@ -1,16 +0,0 @@ -{# -Params - error Error message to display - container (optional) id for a div to wrap the error message in -#} - -{% if container %} -
-{% endif %} - -

Error

-{{ error }} - -{% if container %} -
-{% endif %} diff --git a/demo/templates/frag_form_delete.html.j2 b/demo/templates/frag_form_delete.html.j2 deleted file mode 100644 index 22306c1..0000000 --- a/demo/templates/frag_form_delete.html.j2 +++ /dev/null @@ -1,7 +0,0 @@ -
- - - - - -
diff --git a/demo/templates/frag_form_find.html.j2 b/demo/templates/frag_form_find.html.j2 deleted file mode 100644 index cadd44b..0000000 --- a/demo/templates/frag_form_find.html.j2 +++ /dev/null @@ -1,7 +0,0 @@ -
- - - - - -
diff --git a/demo/templates/frag_form_range.html.j2 b/demo/templates/frag_form_range.html.j2 deleted file mode 100644 index 84b8024..0000000 --- a/demo/templates/frag_form_range.html.j2 +++ /dev/null @@ -1,21 +0,0 @@ -
- - - - - - - - - - - -
diff --git a/demo/templates/frag_form_upsert.html.j2 b/demo/templates/frag_form_upsert.html.j2 deleted file mode 100644 index 68820f5..0000000 --- a/demo/templates/frag_form_upsert.html.j2 +++ /dev/null @@ -1,5 +0,0 @@ -
- - - -
diff --git a/demo/templates/frag_messages.html.j2 b/demo/templates/frag_messages.html.j2 new file mode 100644 index 0000000..0473d16 --- /dev/null +++ b/demo/templates/frag_messages.html.j2 @@ -0,0 +1,12 @@ +{# +Params + messages List of message objects to display +#} + +
+ {% for m in messages %} +
+ {{ m.ts }} {{ m.username }}: {{ m.msg }} +
+ {% endfor %} +
diff --git a/demo/templates/frag_results.html.j2 b/demo/templates/frag_results.html.j2 deleted file mode 100644 index bfb54e9..0000000 --- a/demo/templates/frag_results.html.j2 +++ /dev/null @@ -1,18 +0,0 @@ - - - - {% for field_name in field_names %} - - {% endfor %} - - - - {% for row in rows %} - - {% for field in row %} - - {% endfor %} - - {% endfor %} - -
{{ field_name }}
{{ field }}
diff --git a/demo/templates/page_error.html.j2 b/demo/templates/page_error.html.j2 index 3233283..a3f260e 100644 --- a/demo/templates/page_error.html.j2 +++ b/demo/templates/page_error.html.j2 @@ -1,5 +1,12 @@ {% extends "base.html.j2" %} {% block content %} -{% include "frag_error.html.j2" %} +
+

Error

+

An error occurred while processing your request.

+ +
+{{ error }}
+
+
{% endblock %} diff --git a/demo/templates/page_index.html.j2 b/demo/templates/page_index.html.j2 new file mode 100644 index 0000000..2da14ab --- /dev/null +++ b/demo/templates/page_index.html.j2 @@ -0,0 +1,27 @@ +{% extends "base.html.j2" %} + +{% block content %} +
+

Messages

+ {% include "frag_messages.html.j2" %} + +
+ + + +
+ + +
+{% endblock %} diff --git a/demo/templates/page_main.html.j2 b/demo/templates/page_main.html.j2 deleted file mode 100644 index 907275f..0000000 --- a/demo/templates/page_main.html.j2 +++ /dev/null @@ -1,11 +0,0 @@ -{% extends "base.html.j2" %} - -{% block content %} -
- {% include "frag_aside.html.j2" %} - -
- {% include "frag_results.html.j2" %} -
-
-{% endblock %} diff --git a/log_db/src/common.rs b/log_db/src/common.rs index ca9f451..0f6a058 100644 --- a/log_db/src/common.rs +++ b/log_db/src/common.rs @@ -564,9 +564,11 @@ impl RangeBounds for OwnedBounds { pub struct QueryParams { pub offset: usize, pub limit: usize, + pub sort_asc: bool, } pub static DEFAULT_QUERY_PARAMS: QueryParams = QueryParams { offset: 0, limit: usize::MAX, + sort_asc: true, }; diff --git a/log_db/src/engine.rs b/log_db/src/engine.rs index 0701191..6fb9edf 100644 --- a/log_db/src/engine.rs +++ b/log_db/src/engine.rs @@ -306,13 +306,19 @@ impl Engine { tagged.extend(mapped); } + if !params.sort_asc { + tagged.reverse(); + } let bound_low = params.offset; let bound_high = (params.offset + params.limit).min(tagged.len()); let sliced = &tagged[bound_low..bound_high]; - let tagged_records = self.read_tagged_log_keys(sliced.into_iter())?; + let mut tagged_records = self.read_tagged_log_keys(sliced.into_iter())?; debug!("Read {} records", tagged_records.len()); + if !params.sort_asc { + tagged_records.reverse(); + } Ok(tagged_records) } @@ -420,15 +426,26 @@ impl Engine { self.secondary_memtables[index].range(indexable_bounds) }; - let log_key_batches: Vec<(usize, &LogKey)> = + let mut log_key_batches: Vec<(usize, &LogKey)> = log_keys.into_iter().map(|log_key| (0, log_key)).collect(); + if !params.sort_asc { + log_key_batches.reverse(); + } + let bound_low = params.offset; let bound_high = (params.offset + params.limit).min(log_key_batches.len()); let sliced = &log_key_batches[bound_low..bound_high]; let tagged_records = self.read_tagged_log_keys(sliced.into_iter()); - Ok(tagged_records?.into_iter().map(|(_, rec)| rec).collect()) + let mut result_records: Vec = + tagged_records?.into_iter().map(|(_, rec)| rec).collect(); + + if !params.sort_asc { + result_records.reverse(); + } + + Ok(result_records) } /// Ensures that the `self.metadata_file` and `self.data_file` handles are still pointing to the correct files. diff --git a/log_db/tests/integration.rs b/log_db/tests/integration.rs index e7c6d73..2012faf 100644 --- a/log_db/tests/integration.rs +++ b/log_db/tests/integration.rs @@ -909,6 +909,7 @@ fn test_find_by_with_offset_and_limit() { &QueryParams { offset: 2, limit: 3, + sort_asc: true, }, ) .unwrap() @@ -953,6 +954,7 @@ fn test_batch_find_by_with_offset_and_limit() { &QueryParams { offset: 1, limit: 2, + sort_asc: true, }, ) .unwrap() @@ -995,6 +997,7 @@ fn test_range_by_with_offset_and_limit() { &QueryParams { offset: 1, limit: 3, + sort_asc: true, }, ) .unwrap() diff --git a/py_bindings/log_db.pyi b/py_bindings/log_db.pyi index aeb6421..9a6db62 100644 --- a/py_bindings/log_db.pyi +++ b/py_bindings/log_db.pyi @@ -26,11 +26,11 @@ class DB: def configure() -> Config: ... def upsert(self, record: Record) -> None: ... def get(self, key: str) -> Record: ... - def find_by(self, key: str, value: "Value", offset: int = ..., limit: int = ...) -> list[Record]: ... - def batch_find_by(self, key: str, values: list["Value"], offset: int = ..., limit: int = ...) -> list[tuple[int, Record]]: ... + def find_by(self, key: str, value: "Value", offset: int = ..., limit: int = ..., sort_asc: bool = ...) -> list[Record]: ... + def batch_find_by(self, key: str, values: list["Value"], offset: int = ..., limit: int = ..., sort_asc: bool = ...) -> list[tuple[int, Record]]: ... def delete(self, key: str) -> list[Record]: ... def delete_by(self, key: str, value: "Value") -> list[Record]: ... - def range_by(self, key: str, start: "Bound", end: "Bound", offset: int = ..., limit: int = ...) -> list[Record]: ... + def range_by(self, key: str, start: "Bound", end: "Bound", offset: int = ..., limit: int = ..., sort_asc: bool = ...) -> list[Record]: ... def tx_begin(self) -> None: ... def tx_commit(self) -> None: ... def tx_rollback(self) -> None: ... diff --git a/py_bindings/src/lib.rs b/py_bindings/src/lib.rs index a4abfde..4e125ec 100644 --- a/py_bindings/src/lib.rs +++ b/py_bindings/src/lib.rs @@ -307,15 +307,23 @@ impl DB { Ok(recs.map(|rec| py_from_record(rec))) } - #[pyo3(signature = (field, key, offset = DEFAULT_QUERY_PARAMS.offset, limit = DEFAULT_QUERY_PARAMS.limit))] + #[pyo3(signature = (field, key, + offset = DEFAULT_QUERY_PARAMS.offset, + limit = DEFAULT_QUERY_PARAMS.limit, + sort_asc = DEFAULT_QUERY_PARAMS.sort_asc))] pub fn find_by( &mut self, field: PyField, key: &Value, offset: usize, limit: usize, + sort_asc: bool, ) -> PyResult> { - let params = QueryParams { offset, limit }; + let params = QueryParams { + offset, + limit, + sort_asc, + }; let recs = self .db .find_by_with_params(&field, &key.record_value, ¶ms) @@ -324,15 +332,23 @@ impl DB { Ok(recs.into_iter().map(|rec| py_from_record(rec)).collect()) } - #[pyo3(signature = (field, keys, offset = DEFAULT_QUERY_PARAMS.offset, limit = DEFAULT_QUERY_PARAMS.limit))] + #[pyo3(signature = (field, keys, + offset = DEFAULT_QUERY_PARAMS.offset, + limit = DEFAULT_QUERY_PARAMS.limit, + sort_asc = DEFAULT_QUERY_PARAMS.sort_asc))] pub fn batch_find_by( &mut self, field: PyField, keys: Vec, offset: usize, limit: usize, + sort_asc: bool, ) -> PyResult> { - let params = QueryParams { offset, limit }; + let params = QueryParams { + offset, + limit, + sort_asc, + }; let keys: Vec = keys.into_iter().map(|key| key.record_value).collect(); let recs = self .db @@ -345,7 +361,10 @@ impl DB { .collect()) } - #[pyo3(signature = (field, start, end, offset = DEFAULT_QUERY_PARAMS.offset, limit = DEFAULT_QUERY_PARAMS.limit))] + #[pyo3(signature = (field, start, end, + offset = DEFAULT_QUERY_PARAMS.offset, + limit = DEFAULT_QUERY_PARAMS.limit, + sort_asc = DEFAULT_QUERY_PARAMS.sort_asc))] pub fn range_by( &mut self, field: PyField, @@ -353,8 +372,13 @@ impl DB { end: &PyRangeBound, offset: usize, limit: usize, + sort_asc: bool, ) -> PyResult> { - let params = QueryParams { offset, limit }; + let params = QueryParams { + offset, + limit, + sort_asc, + }; let range = OwnedBounds::new( match start { PyRangeBound::Unbounded() => StdBound::Unbounded, -- cgit v1.3