From ab39e156749dd0136d6c0c5a032a38b7674bcbab Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 28 Oct 2026 23:11:30 +0000 Subject: [PATCH 1/2] fix(peewee): retain accepted long events in overlap queries Git-Session-Id: d8a22e8f-4386-4789-bab9-5ddd743d4b08 --- aw_datastore/storages/peewee.py | 8 +- tests/test_peewee_long_events.py | 130 +++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 tests/test_peewee_long_events.py diff --git a/aw_datastore/storages/peewee.py b/aw_datastore/storages/peewee.py index d1a348d7..dfedec1c 100644 --- a/aw_datastore/storages/peewee.py +++ b/aw_datastore/storages/peewee.py @@ -1,7 +1,7 @@ import json import logging import os -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from typing import ( Any, Dict, @@ -512,10 +512,8 @@ def _where_range( endtime = endtime.astimezone(timezone.utc) if starttime: - # Faster WHERE to speed up slow query below, leads to ~2-3x speedup - # We'll assume events aren't >24h - q = q.where(starttime - timedelta(hours=24) <= EventModel.timestamp) - + # Accepted events can be longer than 24h, so a lower bound on their + # start timestamp would discard valid overlaps before this check. # This can be slow on large databases... # Tried creating various indexes and using SQLite's unlikely() function, but it had no effect q = q.where( diff --git a/tests/test_peewee_long_events.py b/tests/test_peewee_long_events.py new file mode 100644 index 00000000..96b722a0 --- /dev/null +++ b/tests/test_peewee_long_events.py @@ -0,0 +1,130 @@ +"""Accepted event durations must not be capped implicitly by range queries.""" + +from datetime import datetime, timedelta, timezone + +import pytest +from aw_core.models import Event +from aw_datastore import Datastore +from aw_datastore.storages import PeeweeStorage +from aw_datastore.storages.peewee import EventModel + +UTC = timezone.utc +BASE = datetime(2026, 10, 1, tzinfo=UTC) + + +@pytest.fixture() +def bucket(tmp_path, monkeypatch): + monkeypatch.setattr( + "aw_datastore.storages.peewee.get_data_dir", lambda _: str(tmp_path) + ) + ds = Datastore(PeeweeStorage, testing=True, filepath=str(tmp_path / "events.db")) + try: + yield ds.create_bucket( + "long-events", type="test", client="test", hostname="test" + ) + finally: + ds.storage_strategy.db.close() + + +@pytest.mark.parametrize("hours", [24, 48, 240]) +@pytest.mark.parametrize( + "start_hour,end_hour,expected", + [ + (None, None, True), + (None, -1, False), + (None, 0, True), + (0, None, True), + (12, None, True), + (None, 13, True), + (12, 13, True), + (36, 37, None), # Inside day two for >24h events; outside for 24h. + (None, 48, True), + (48, 49, None), + (240, 241, None), + (241, None, False), + ], +) +def test_long_event_overlap(bucket, hours, start_hour, end_hour, expected): + original = Event( + timestamp=BASE, duration=timedelta(hours=hours), data={"label": "long"} + ) + bucket.insert(original) + start = BASE + timedelta(hours=start_hour) if start_hour is not None else None + end = BASE + timedelta(hours=end_hour) if end_hour is not None else None + if expected is None: + expected = start_hour <= hours + + events = bucket.get(starttime=start, endtime=end) + assert bucket.get_eventcount(starttime=start, endtime=end) == int(expected) + assert len(events) == int(expected) + if expected: + event = events[0] + assert event.id == original.id + assert event.data == original.data + # Peewee already clips returned copies, including zero-length boundary hits. + clipped_start = max(BASE, start) if start else BASE + clipped_end = ( + min(BASE + original.duration, end) if end else BASE + original.duration + ) + assert event.timestamp == clipped_start + assert event.duration == clipped_end - clipped_start + # Reads must not rewrite the stored event or mutate the caller's object. + stored = bucket.get()[0] + assert stored.timestamp == original.timestamp == BASE + assert stored.duration == original.duration == timedelta(hours=hours) + + +def test_long_and_short_events_keep_order_limits_and_offsets(bucket): + bucket.insert( + [ + Event(timestamp=BASE, duration=172800, data={"label": "long"}), + Event( + timestamp=BASE + timedelta(hours=36), + duration=60, + data={"label": "short"}, + ), + Event( + timestamp=BASE + timedelta(hours=36), + duration=120, + data={"label": "tie"}, + ), + Event( + timestamp=BASE + timedelta(hours=35), + duration=60, + data={"label": "before"}, + ), + Event( + timestamp=BASE + timedelta(hours=38), + duration=60, + data={"label": "after"}, + ), + ] + ) + start, end = BASE + timedelta(hours=36), BASE + timedelta(hours=37) + offset = timezone(timedelta(hours=5, minutes=30)) + for bounds in [(start, end), (start.astimezone(offset), end.astimezone(offset))]: + result = bucket.get(starttime=bounds[0], endtime=bounds[1]) + assert [e.data["label"] for e in result] == ["short", "tie", "long"] + assert bucket.get_eventcount(*bounds) == 3 + assert bucket.get(limit=0, starttime=bounds[0], endtime=bounds[1]) == [] + assert [e.id for e in bucket.get(2, *bounds)] == [e.id for e in result[:2]] + assert bucket.get(starttime=BASE + timedelta(days=20)) == [] + assert bucket.get_eventcount(starttime=BASE + timedelta(days=20)) == 0 + + +def test_empty_long_event_range(bucket): + assert bucket.get(starttime=BASE, endtime=BASE + timedelta(hours=1)) == [] + assert bucket.get_eventcount(BASE, BASE + timedelta(hours=1)) == 0 + + +def test_actual_overlap_query_keeps_bucket_timestamp_index(bucket): + storage = bucket.ds.storage_strategy + query = EventModel.select().where( + EventModel.bucket == storage.bucket_keys[bucket.bucket_id] + ) + query = storage._where_range( + query, BASE + timedelta(hours=36), BASE + timedelta(hours=37) + ) + sql, params = query.sql() + plan = storage.db.execute_sql("EXPLAIN QUERY PLAN " + sql, params).fetchall() + assert "eventmodel_bucket_id_timestamp" in str(plan) From 74946601dbc793ec3cf19ca6749511684725c235 Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 28 Oct 2026 23:16:10 +0000 Subject: [PATCH 2/2] test(peewee): guard ordered overlap query plan Git-Session-Id: d8a22e8f-4386-4789-bab9-5ddd743d4b08 --- tests/test_peewee_long_events.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_peewee_long_events.py b/tests/test_peewee_long_events.py index 96b722a0..54fdb66c 100644 --- a/tests/test_peewee_long_events.py +++ b/tests/test_peewee_long_events.py @@ -119,8 +119,13 @@ def test_empty_long_event_range(bucket): def test_actual_overlap_query_keeps_bucket_timestamp_index(bucket): storage = bucket.ds.storage_strategy - query = EventModel.select().where( - EventModel.bucket == storage.bucket_keys[bucket.bucket_id] + query = ( + EventModel.select() + .where(EventModel.bucket == storage.bucket_keys[bucket.bucket_id]) + .order_by( + EventModel.timestamp.desc(), EventModel.duration.asc(), EventModel.id.asc() + ) + .limit(100) ) query = storage._where_range( query, BASE + timedelta(hours=36), BASE + timedelta(hours=37) @@ -128,3 +133,5 @@ def test_actual_overlap_query_keeps_bucket_timestamp_index(bucket): sql, params = query.sql() plan = storage.db.execute_sql("EXPLAIN QUERY PLAN " + sql, params).fetchall() assert "eventmodel_bucket_id_timestamp" in str(plan) + # Sorting equal-timestamp runs is fine; sorting the entire result is not. + assert "TEMP B-TREE FOR ORDER BY" not in str(plan)