aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJan Tuomi <jan@jantuomi.fi>2025-05-15 12:34:57 +0300
committerJan Tuomi <jan@jantuomi.fi>2025-05-15 12:34:57 +0300
commitae36f44de7b9adbfa56525dd0046311ea7181288 (patch)
tree2cf714e28240204dd95e5b35bcde608d6f135d65
parent8d4fa3b038493df8f2b6108cef669d5434b1736e (diff)
Run formatter, add ruff config
-rw-r--r--.zed/settings.json13
-rw-r--r--benchmark_just_do_work.py10
-rw-r--r--benchmark_mini.py6
-rw-r--r--benchmark_throughput.py10
-rw-r--r--femtoqueue.py30
-rw-r--r--pyproject.toml5
-rw-r--r--test.py25
-rw-r--r--uv.lock43
8 files changed, 123 insertions, 19 deletions
diff --git a/.zed/settings.json b/.zed/settings.json
new file mode 100644
index 0000000..cdd051e
--- /dev/null
+++ b/.zed/settings.json
@@ -0,0 +1,13 @@
+{
+ "format_on_save": "on",
+ "languages": {
+ "Python": {
+ "formatter": {
+ "external": {
+ "command": ".venv/bin/ruff",
+ "arguments": ["format", "-"]
+ }
+ }
+ }
+ }
+}
diff --git a/benchmark_just_do_work.py b/benchmark_just_do_work.py
index b9a7696..a29c26a 100644
--- a/benchmark_just_do_work.py
+++ b/benchmark_just_do_work.py
@@ -9,6 +9,7 @@
import argparse
from femtoqueue import FemtoQueue
+
def benchmark_femtoqueue(data_dir: str, num_tasks: int = 1000):
queue = FemtoQueue(data_dir=data_dir, node_id="node1")
@@ -23,9 +24,14 @@ def benchmark_femtoqueue(data_dir: str, num_tasks: int = 1000):
break
queue.done(task)
+
if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="FemtoQueue: Just run some tasks for external benchmarking")
- parser.add_argument("data_dir", type=str, help="Data directory. Won't be cleaned up!")
+ parser = argparse.ArgumentParser(
+ description="FemtoQueue: Just run some tasks for external benchmarking"
+ )
+ parser.add_argument(
+ "data_dir", type=str, help="Data directory. Won't be cleaned up!"
+ )
args = parser.parse_args()
benchmark_femtoqueue(args.data_dir)
diff --git a/benchmark_mini.py b/benchmark_mini.py
index 273b88b..c7ed63a 100644
--- a/benchmark_mini.py
+++ b/benchmark_mini.py
@@ -3,6 +3,7 @@ import tempfile
import shutil
from femtoqueue import FemtoQueue
+
def benchmark_femtoqueue(num_tasks: int = 1000):
tmpdir = tempfile.mkdtemp()
queue = FemtoQueue(data_dir=tmpdir, node_id="node1")
@@ -26,9 +27,12 @@ def benchmark_femtoqueue(num_tasks: int = 1000):
queue.done(task)
processed += 1
process_duration = time.time() - start
- print(f"Processed in {process_duration:.4f}s ({processed / process_duration:.2f} tasks/sec)")
+ print(
+ f"Processed in {process_duration:.4f}s ({processed / process_duration:.2f} tasks/sec)"
+ )
shutil.rmtree(tmpdir)
+
if __name__ == "__main__":
benchmark_femtoqueue(1000)
diff --git a/benchmark_throughput.py b/benchmark_throughput.py
index b21c28a..c3cae27 100644
--- a/benchmark_throughput.py
+++ b/benchmark_throughput.py
@@ -5,6 +5,7 @@ import argparse
from random import randint
from femtoqueue import FemtoQueue
+
def run_benchmark(duration_seconds: int, payload_size: int = 100):
print(f"Running throughput benchmark for {duration_seconds} sec. See -h for help.")
tmpdir = tempfile.mkdtemp()
@@ -47,9 +48,16 @@ def run_benchmark(duration_seconds: int, payload_size: int = 100):
print(f"Total tasks processed: {total_count}")
print(f"Overall throughput: {total_count / elapsed:.2f} tasks/sec")
+
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="FemtoQueue throughput benchmark")
- parser.add_argument("--duration", type=int, help="Duration to run the benchmark (in seconds)", default=10, required=False)
+ parser.add_argument(
+ "--duration",
+ type=int,
+ help="Duration to run the benchmark (in seconds)",
+ default=10,
+ required=False,
+ )
args = parser.parse_args()
run_benchmark(args.duration)
diff --git a/femtoqueue.py b/femtoqueue.py
index bdb9298..faa6a5b 100644
--- a/femtoqueue.py
+++ b/femtoqueue.py
@@ -3,11 +3,13 @@ from dataclasses import dataclass
import time
from typing import Generator
+
@dataclass
class FemtoTask:
id: str
data: bytes
+
class FemtoQueue:
RESERVED_NAMES = [
"creating",
@@ -101,8 +103,12 @@ class FemtoQueue:
# Only run this every `timeout_stale_ms` milliseconds because iterating
# through all tasks is slow
timeout_sec = self.timeout_stale_ms / 1000.0
- if self.latest_stale_check_ts is not None and now - self.latest_stale_check_ts < timeout_sec:
+ if (
+ self.latest_stale_check_ts is not None
+ and now - self.latest_stale_check_ts < timeout_sec
+ ):
return
+
self.latest_stale_check_ts = now
for dir_name in listdir(self.data_dir):
@@ -138,14 +144,19 @@ class FemtoQueue:
pass
# If cache empty, then check assigned tasks in progress (aborted)
- self.todo_cache = (path.join(self.dir_in_progress, x) for x in sorted(listdir(self.dir_in_progress)))
+ self.todo_cache = (
+ path.join(self.dir_in_progress, x)
+ for x in sorted(listdir(self.dir_in_progress))
+ )
try:
return next(self.todo_cache)
except StopIteration:
pass
# Then check pending tasks
- self.todo_cache = (path.join(self.dir_pending, x) for x in sorted(listdir(self.dir_pending)))
+ self.todo_cache = (
+ path.join(self.dir_pending, x) for x in sorted(listdir(self.dir_pending))
+ )
try:
return next(self.todo_cache)
except StopIteration:
@@ -166,7 +177,8 @@ class FemtoQueue:
while True:
task = self._pop_task_path()
- if task is None: return None
+ if task is None:
+ return None
id = path.basename(task)
in_progress_path = path.join(self.dir_in_progress, id)
@@ -179,7 +191,7 @@ class FemtoQueue:
with open(in_progress_path, "rb") as f:
content = f.read()
- return FemtoTask(id = id, data = content)
+ return FemtoTask(id=id, data=content)
def done(self, task: FemtoTask):
"""
@@ -196,7 +208,9 @@ class FemtoQueue:
try:
rename(in_progress_path, done_path)
except FileNotFoundError as e:
- raise Exception(f"Tried to complete a task that is not in progress, id={task.id}") from e
+ raise Exception(
+ f"Tried to complete a task that is not in progress, id={task.id}"
+ ) from e
def fail(self, task: FemtoTask):
"""
@@ -213,4 +227,6 @@ class FemtoQueue:
try:
rename(in_progress_path, failed_path)
except FileNotFoundError as e:
- raise Exception(f"Tried to fail a task that is not in progress, id={task.id}") from e
+ raise Exception(
+ f"Tried to fail a task that is not in progress, id={task.id}"
+ ) from e
diff --git a/pyproject.toml b/pyproject.toml
index df2ebc9..dd2ac96 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,3 +26,8 @@ build-backend = "setuptools.build_meta"
[tool.setuptools]
py-modules = ["femtoqueue"]
+
+[dependency-groups]
+dev = [
+ "ruff>=0.11.9",
+]
diff --git a/test.py b/test.py
index 30d9a95..2ae9c48 100644
--- a/test.py
+++ b/test.py
@@ -7,22 +7,26 @@ from femtoqueue import FemtoQueue, FemtoTask
from tempfile import mkdtemp
original_time_fn = time.time
+
+
def set_time_mock(ts: float):
def mock_time_fn():
return ts
time.time = mock_time_fn
+
def reset_time_mock():
time.time = original_time_fn
+
class TestFemtoQueue(unittest.TestCase):
def test_basic(self):
dir = mkdtemp()
- q = FemtoQueue(data_dir = dir, node_id = "node1")
+ q = FemtoQueue(data_dir=dir, node_id="node1")
# Add a JSON task
- some_data = json.dumps({ "foo": "bar" })
+ some_data = json.dumps({"foo": "bar"})
q.push(some_data.encode("utf-8"))
# The task should be in the queue with the correct payload
@@ -41,10 +45,10 @@ class TestFemtoQueue(unittest.TestCase):
def test_aborted(self):
dir = mkdtemp()
- q = FemtoQueue(data_dir = dir, node_id = "node1")
+ q = FemtoQueue(data_dir=dir, node_id="node1")
# Add a JSON task
- some_data = json.dumps({ "foo": "bar" })
+ some_data = json.dumps({"foo": "bar"})
q.push(some_data.encode("utf-8"))
# The task should be in the queue with the correct payload
@@ -55,7 +59,7 @@ class TestFemtoQueue(unittest.TestCase):
self.assertEqual(parsed_data["foo"], "bar")
# Simulate a fault and start over
- q = FemtoQueue(data_dir = dir, node_id = "node1")
+ q = FemtoQueue(data_dir=dir, node_id="node1")
# The task should still be assigned to this node
task = q.pop()
@@ -77,14 +81,18 @@ class TestFemtoQueue(unittest.TestCase):
set_time_mock(0)
# Node1 creates and claims the task
- q1 = FemtoQueue(data_dir=dir, node_id="node1", timeout_stale_ms=timeout_stale_ms)
+ q1 = FemtoQueue(
+ data_dir=dir, node_id="node1", timeout_stale_ms=timeout_stale_ms
+ )
q1.push(b"stuck")
task = q1.pop()
self.assertIsNotNone(task)
task = cast(FemtoTask, task)
# Assert that node2 can not see the task
- q2 = FemtoQueue(data_dir=dir, node_id="node2", timeout_stale_ms=timeout_stale_ms)
+ q2 = FemtoQueue(
+ data_dir=dir, node_id="node2", timeout_stale_ms=timeout_stale_ms
+ )
non_existant_task = q2.pop()
self.assertIsNone(non_existant_task)
@@ -162,5 +170,6 @@ class TestFemtoQueue(unittest.TestCase):
# Queue should now be empty
self.assertIsNone(q.pop())
-if __name__ == '__main__':
+
+if __name__ == "__main__":
unittest.main()
diff --git a/uv.lock b/uv.lock
new file mode 100644
index 0000000..4f22386
--- /dev/null
+++ b/uv.lock
@@ -0,0 +1,43 @@
+version = 1
+revision = 2
+requires-python = ">=3.10"
+
+[[package]]
+name = "femtoqueue"
+version = "0.1.5"
+source = { editable = "." }
+
+[package.dev-dependencies]
+dev = [
+ { name = "ruff" },
+]
+
+[package.metadata]
+
+[package.metadata.requires-dev]
+dev = [{ name = "ruff", specifier = ">=0.11.9" }]
+
+[[package]]
+name = "ruff"
+version = "0.11.9"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f5/e7/e55dda1c92cdcf34b677ebef17486669800de01e887b7831a1b8fdf5cb08/ruff-0.11.9.tar.gz", hash = "sha256:ebd58d4f67a00afb3a30bf7d383e52d0e036e6195143c6db7019604a05335517", size = 4132134, upload-time = "2025-05-09T16:19:41.511Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/fb/71/75dfb7194fe6502708e547941d41162574d1f579c4676a8eb645bf1a6842/ruff-0.11.9-py3-none-linux_armv6l.whl", hash = "sha256:a31a1d143a5e6f499d1fb480f8e1e780b4dfdd580f86e05e87b835d22c5c6f8c", size = 10335453, upload-time = "2025-05-09T16:18:58.2Z" },
+ { url = "https://files.pythonhosted.org/packages/74/fc/ad80c869b1732f53c4232bbf341f33c5075b2c0fb3e488983eb55964076a/ruff-0.11.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:66bc18ca783b97186a1f3100e91e492615767ae0a3be584e1266aa9051990722", size = 11072566, upload-time = "2025-05-09T16:19:01.432Z" },
+ { url = "https://files.pythonhosted.org/packages/87/0d/0ccececef8a0671dae155cbf7a1f90ea2dd1dba61405da60228bbe731d35/ruff-0.11.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:bd576cd06962825de8aece49f28707662ada6a1ff2db848d1348e12c580acbf1", size = 10435020, upload-time = "2025-05-09T16:19:03.897Z" },
+ { url = "https://files.pythonhosted.org/packages/52/01/e249e1da6ad722278094e183cbf22379a9bbe5f21a3e46cef24ccab76e22/ruff-0.11.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b1d18b4be8182cc6fddf859ce432cc9631556e9f371ada52f3eaefc10d878de", size = 10593935, upload-time = "2025-05-09T16:19:06.455Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/9a/40cf91f61e3003fe7bd43f1761882740e954506c5a0f9097b1cff861f04c/ruff-0.11.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0f3f46f759ac623e94824b1e5a687a0df5cd7f5b00718ff9c24f0a894a683be7", size = 10172971, upload-time = "2025-05-09T16:19:10.261Z" },
+ { url = "https://files.pythonhosted.org/packages/61/12/d395203de1e8717d7a2071b5a340422726d4736f44daf2290aad1085075f/ruff-0.11.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f34847eea11932d97b521450cf3e1d17863cfa5a94f21a056b93fb86f3f3dba2", size = 11748631, upload-time = "2025-05-09T16:19:12.307Z" },
+ { url = "https://files.pythonhosted.org/packages/66/d6/ef4d5eba77677eab511644c37c55a3bb8dcac1cdeb331123fe342c9a16c9/ruff-0.11.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f33b15e00435773df97cddcd263578aa83af996b913721d86f47f4e0ee0ff271", size = 12409236, upload-time = "2025-05-09T16:19:15.006Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/8f/5a2c5fc6124dd925a5faf90e1089ee9036462118b619068e5b65f8ea03df/ruff-0.11.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7b27613a683b086f2aca8996f63cb3dd7bc49e6eccf590563221f7b43ded3f65", size = 11881436, upload-time = "2025-05-09T16:19:17.063Z" },
+ { url = "https://files.pythonhosted.org/packages/39/d1/9683f469ae0b99b95ef99a56cfe8c8373c14eba26bd5c622150959ce9f64/ruff-0.11.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9e0d88756e63e8302e630cee3ce2ffb77859797cc84a830a24473939e6da3ca6", size = 13982759, upload-time = "2025-05-09T16:19:19.693Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/0b/c53a664f06e0faab596397867c6320c3816df479e888fe3af63bc3f89699/ruff-0.11.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:537c82c9829d7811e3aa680205f94c81a2958a122ac391c0eb60336ace741a70", size = 11541985, upload-time = "2025-05-09T16:19:21.831Z" },
+ { url = "https://files.pythonhosted.org/packages/23/a0/156c4d7e685f6526a636a60986ee4a3c09c8c4e2a49b9a08c9913f46c139/ruff-0.11.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:440ac6a7029f3dee7d46ab7de6f54b19e34c2b090bb4f2480d0a2d635228f381", size = 10465775, upload-time = "2025-05-09T16:19:24.401Z" },
+ { url = "https://files.pythonhosted.org/packages/43/d5/88b9a6534d9d4952c355e38eabc343df812f168a2c811dbce7d681aeb404/ruff-0.11.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:71c539bac63d0788a30227ed4d43b81353c89437d355fdc52e0cda4ce5651787", size = 10170957, upload-time = "2025-05-09T16:19:27.08Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/b8/2bd533bdaf469dc84b45815ab806784d561fab104d993a54e1852596d581/ruff-0.11.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c67117bc82457e4501473c5f5217d49d9222a360794bfb63968e09e70f340abd", size = 11143307, upload-time = "2025-05-09T16:19:29.462Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/d9/43cfba291788459b9bfd4e09a0479aa94d05ab5021d381a502d61a807ec1/ruff-0.11.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e4b78454f97aa454586e8a5557facb40d683e74246c97372af3c2d76901d697b", size = 11603026, upload-time = "2025-05-09T16:19:31.569Z" },
+ { url = "https://files.pythonhosted.org/packages/22/e6/7ed70048e89b01d728ccc950557a17ecf8df4127b08a56944b9d0bae61bc/ruff-0.11.9-py3-none-win32.whl", hash = "sha256:7fe1bc950e7d7b42caaee2a8a3bc27410547cc032c9558ee2e0f6d3b209e845a", size = 10548627, upload-time = "2025-05-09T16:19:33.657Z" },
+ { url = "https://files.pythonhosted.org/packages/90/36/1da5d566271682ed10f436f732e5f75f926c17255c9c75cefb77d4bf8f10/ruff-0.11.9-py3-none-win_amd64.whl", hash = "sha256:52edaa4a6d70f8180343a5b7f030c7edd36ad180c9f4d224959c2d689962d964", size = 11634340, upload-time = "2025-05-09T16:19:35.815Z" },
+ { url = "https://files.pythonhosted.org/packages/40/f7/70aad26e5877c8f7ee5b161c4c9fa0100e63fc4c944dc6d97b9c7e871417/ruff-0.11.9-py3-none-win_arm64.whl", hash = "sha256:bcf42689c22f2e240f496d0c183ef2c6f7b35e809f12c1db58f75d9aa8d630ca", size = 10741080, upload-time = "2025-05-09T16:19:39.605Z" },
+]