perf(state): merge FTS5 segments on VACUUM + add 'hermes sessions optimize'

The FTS5 indexes (messages_fts, messages_fts_trigram) grow as a series of
incremental b-tree segments — one per trigger-driven insert batch. SQLite's
automerge caps at ~16 segments, so a long-lived store keeps scanning many
segments per MATCH and never collapses them unless the special 'optimize'
command runs. Nothing in the codebase ever ran it: vacuum() only fired after
a prune that deleted rows, and even then never merged FTS segments.

Changes:
- SessionDB.optimize_fts(): merges each FTS5 index to a single segment,
  probing for the (optional/lazy) trigram table first so it is safe to call
  unconditionally. Layout-only — search results and snippet() are unchanged.
- vacuum() now calls optimize_fts() before VACUUM so freed index pages are
  returned to the OS in the same pass.
- 'hermes sessions optimize' CLI subcommand for on-demand reclamation +
  segment compaction (previously there was no way to compact the store
  without a prune deleting rows), with before/after size reporting.

Benchmark (8000 msgs, fragmented to 8 segments/index):
- segments 8 -> 1 on both indexes
- porter MATCH 5.5x faster (0.449 -> 0.081 ms/q)
- trigram MATCH 3.0x faster (0.632 -> 0.207 ms/q)
- 8000 matches before == 8000 after, identical row ids (no functional change)

Orthogonal to the structural FTS-size PRs (#20239 external-content,
#27770 optional trigram) — segment merge helps regardless of those.

Tests: TestOptimizeFts covers index count, search+snippet preservation,
missing-trigram path, and idempotency. Full test_hermes_state.py green (227).
This commit is contained in:
kshitijk4poor
2026-05-29 05:09:56 -07:00
committed by Teknium
parent 2159d2a729
commit 38695254f8
3 changed files with 158 additions and 0 deletions
+58
View File
@@ -2676,6 +2676,64 @@ class TestVacuum:
db.vacuum()
class TestOptimizeFts:
def test_optimize_returns_index_count(self, db):
"""A fresh DB has both FTS indexes; optimize merges both."""
db.create_session(session_id="s1", source="cli")
db.append_message(session_id="s1", role="user", content="hello world")
assert db.optimize_fts() == 2
def test_optimize_preserves_search_and_snippet(self, db):
"""Optimize is layout-only: MATCH results + snippets are unchanged."""
db.create_session(session_id="s1", source="cli")
for i in range(50):
db.append_message(
session_id="s1",
role="user",
content=f"needle alpha bravo charlie message {i}",
)
before = db.search_messages("needle")
n = db.optimize_fts()
assert n == 2
after = db.search_messages("needle")
assert len(after) == len(before)
assert len(after) > 0
# Snippet must still be populated (would be empty/None if the FTS
# content shadow were lost during optimize).
assert all(row.get("snippet") for row in after)
# IDs and snippets are identical before/after — pure layout change.
assert [r["id"] for r in after] == [r["id"] for r in before]
assert [r["snippet"] for r in after] == [r["snippet"] for r in before]
def test_optimize_skips_missing_trigram_table(self, db):
"""When the trigram index is absent, optimize handles only the porter
index and does not raise."""
db.create_session(session_id="s1", source="cli")
db.append_message(session_id="s1", role="user", content="hello")
# Drop the trigram table + triggers to simulate a disabled/absent index.
with db._lock:
for trig in (
"messages_fts_trigram_insert",
"messages_fts_trigram_delete",
"messages_fts_trigram_update",
):
db._conn.execute(f"DROP TRIGGER IF EXISTS {trig}")
db._conn.execute("DROP TABLE IF EXISTS messages_fts_trigram")
assert db._fts_table_exists("messages_fts_trigram") is False
assert db._fts_table_exists("messages_fts") is True
# Only the porter index remains -> 1 optimized, no error.
assert db.optimize_fts() == 1
def test_optimize_idempotent(self, db):
"""Running optimize twice is safe (second pass is a no-op merge)."""
db.create_session(session_id="s1", source="cli")
db.append_message(session_id="s1", role="user", content="repeat me")
assert db.optimize_fts() == 2
assert db.optimize_fts() == 2
# Search still works after repeated optimization.
assert len(db.search_messages("repeat")) == 1
class TestAutoMaintenance:
def _make_old_ended(self, db, sid: str, days_old: int = 100):
"""Create a session that is ended and was started `days_old` days ago."""