perf(state): merge FTS5 segments on VACUUM + add 'hermes sessions optimize'
The FTS5 indexes (messages_fts, messages_fts_trigram) grow as a series of incremental b-tree segments — one per trigger-driven insert batch. SQLite's automerge caps at ~16 segments, so a long-lived store keeps scanning many segments per MATCH and never collapses them unless the special 'optimize' command runs. Nothing in the codebase ever ran it: vacuum() only fired after a prune that deleted rows, and even then never merged FTS segments. Changes: - SessionDB.optimize_fts(): merges each FTS5 index to a single segment, probing for the (optional/lazy) trigram table first so it is safe to call unconditionally. Layout-only — search results and snippet() are unchanged. - vacuum() now calls optimize_fts() before VACUUM so freed index pages are returned to the OS in the same pass. - 'hermes sessions optimize' CLI subcommand for on-demand reclamation + segment compaction (previously there was no way to compact the store without a prune deleting rows), with before/after size reporting. Benchmark (8000 msgs, fragmented to 8 segments/index): - segments 8 -> 1 on both indexes - porter MATCH 5.5x faster (0.449 -> 0.081 ms/q) - trigram MATCH 3.0x faster (0.632 -> 0.207 ms/q) - 8000 matches before == 8000 after, identical row ids (no functional change) Orthogonal to the structural FTS-size PRs (#20239 external-content, #27770 optional trigram) — segment merge helps regardless of those. Tests: TestOptimizeFts covers index count, search+snippet preservation, missing-trigram path, and idempotency. Full test_hermes_state.py green (227).
This commit is contained in:
@@ -2676,6 +2676,64 @@ class TestVacuum:
|
||||
db.vacuum()
|
||||
|
||||
|
||||
class TestOptimizeFts:
|
||||
def test_optimize_returns_index_count(self, db):
|
||||
"""A fresh DB has both FTS indexes; optimize merges both."""
|
||||
db.create_session(session_id="s1", source="cli")
|
||||
db.append_message(session_id="s1", role="user", content="hello world")
|
||||
assert db.optimize_fts() == 2
|
||||
|
||||
def test_optimize_preserves_search_and_snippet(self, db):
|
||||
"""Optimize is layout-only: MATCH results + snippets are unchanged."""
|
||||
db.create_session(session_id="s1", source="cli")
|
||||
for i in range(50):
|
||||
db.append_message(
|
||||
session_id="s1",
|
||||
role="user",
|
||||
content=f"needle alpha bravo charlie message {i}",
|
||||
)
|
||||
before = db.search_messages("needle")
|
||||
n = db.optimize_fts()
|
||||
assert n == 2
|
||||
after = db.search_messages("needle")
|
||||
assert len(after) == len(before)
|
||||
assert len(after) > 0
|
||||
# Snippet must still be populated (would be empty/None if the FTS
|
||||
# content shadow were lost during optimize).
|
||||
assert all(row.get("snippet") for row in after)
|
||||
# IDs and snippets are identical before/after — pure layout change.
|
||||
assert [r["id"] for r in after] == [r["id"] for r in before]
|
||||
assert [r["snippet"] for r in after] == [r["snippet"] for r in before]
|
||||
|
||||
def test_optimize_skips_missing_trigram_table(self, db):
|
||||
"""When the trigram index is absent, optimize handles only the porter
|
||||
index and does not raise."""
|
||||
db.create_session(session_id="s1", source="cli")
|
||||
db.append_message(session_id="s1", role="user", content="hello")
|
||||
# Drop the trigram table + triggers to simulate a disabled/absent index.
|
||||
with db._lock:
|
||||
for trig in (
|
||||
"messages_fts_trigram_insert",
|
||||
"messages_fts_trigram_delete",
|
||||
"messages_fts_trigram_update",
|
||||
):
|
||||
db._conn.execute(f"DROP TRIGGER IF EXISTS {trig}")
|
||||
db._conn.execute("DROP TABLE IF EXISTS messages_fts_trigram")
|
||||
assert db._fts_table_exists("messages_fts_trigram") is False
|
||||
assert db._fts_table_exists("messages_fts") is True
|
||||
# Only the porter index remains -> 1 optimized, no error.
|
||||
assert db.optimize_fts() == 1
|
||||
|
||||
def test_optimize_idempotent(self, db):
|
||||
"""Running optimize twice is safe (second pass is a no-op merge)."""
|
||||
db.create_session(session_id="s1", source="cli")
|
||||
db.append_message(session_id="s1", role="user", content="repeat me")
|
||||
assert db.optimize_fts() == 2
|
||||
assert db.optimize_fts() == 2
|
||||
# Search still works after repeated optimization.
|
||||
assert len(db.search_messages("repeat")) == 1
|
||||
|
||||
|
||||
class TestAutoMaintenance:
|
||||
def _make_old_ended(self, db, sid: str, days_old: int = 100):
|
||||
"""Create a session that is ended and was started `days_old` days ago."""
|
||||
|
||||
Reference in New Issue
Block a user