fix(cron): make sequential jobs non-blocking too + sweep MCP after jobs finish

Follow-up on the parallel-dispatch decoupling: the sequential pass for
workdir/profile jobs still ran inline in the ticker thread, so a long
workdir/profile job reintroduced the exact starvation #37312 describes,
just for env-mutating jobs. And the MCP orphan sweep ran immediately
after dispatch in sync=False mode — before jobs finished — defeating its
own 'runs after every job' contract and racing jobs still spawning MCP
children.

- Sequential jobs now queue to a persistent single-thread cron-seq pool
  (preserves one-at-a-time ordering across ticks, never blocks the tick).
- Same in-flight dedup guard now covers sequential jobs.
- MCP orphan sweep runs via a done-callback after the LAST dispatched job
  completes in async mode; inline after as_completed in sync mode.

Verified E2E: tick(sync=False) returns in ~1ms with a 1.5s sequential job
in flight; sweep fires only after that job ends.
This commit is contained in:
Teknium
2026-06-04 05:40:13 -07:00
parent eb9cde7346
commit 9fbfeb31b9
4 changed files with 254 additions and 60 deletions
+103
View File
@@ -169,3 +169,106 @@ class TestSyncMode:
barrier.wait()
time.sleep(0.1)
sched._shutdown_parallel_pool()
class TestSequentialPool:
"""Sequential (workdir/profile) jobs use the persistent cron-seq pool.
Verifies the follow-up fix: env/context-mutating jobs no longer run inline
in the ticker thread, so a long workdir/profile job can't starve the
schedule the same way the parallel path used to.
"""
def test_sequential_job_does_not_block_ticker(self, tmp_path, monkeypatch):
"""sync=False returns immediately even when a workdir job is slow."""
import cron.scheduler as sched
sched._parallel_pool = None
sched._parallel_pool_max_workers = None
sched._sequential_pool = None
sched._running_job_ids.clear()
job = {
"id": "slow-workdir",
"name": "slow-workdir",
"prompt": "test",
"schedule": "every 5m",
"enabled": True,
"next_run_at": "2020-01-01T00:00:00",
"deliver": "local",
"workdir": str(tmp_path), # makes it sequential
}
barrier = threading.Barrier(2, timeout=5)
def slow_run(j):
barrier.wait()
return True, "out", "resp", None
monkeypatch.setattr(sched, "get_due_jobs", lambda: [job])
monkeypatch.setattr(sched, "advance_next_run", lambda *_a, **_kw: None)
monkeypatch.setattr(sched, "run_job", slow_run)
monkeypatch.setattr(sched, "save_job_output", lambda *_a, **_kw: "/tmp/out")
monkeypatch.setattr(sched, "mark_job_run", lambda *_a, **_kw: None)
monkeypatch.setattr(sched, "_deliver_result", lambda *_a, **_kw: None)
start = time.monotonic()
n = sched.tick(verbose=False, sync=False)
elapsed = time.monotonic() - start
assert n == 1 # optimistic count
assert elapsed < 1.0 # did NOT block on the slow workdir job
barrier.wait()
time.sleep(0.1)
sched._shutdown_parallel_pool()
def test_sequential_running_guard_prevents_double_dispatch(self, tmp_path, monkeypatch):
"""A workdir job already in _running_job_ids is skipped on next tick."""
import cron.scheduler as sched
sched._parallel_pool = None
sched._parallel_pool_max_workers = None
sched._sequential_pool = None
sched._running_job_ids.clear()
job = {
"id": "guard-seq",
"name": "guard-seq",
"prompt": "test",
"schedule": "every 5m",
"enabled": True,
"next_run_at": "2020-01-01T00:00:00",
"deliver": "local",
"workdir": str(tmp_path),
}
# Simulate the job already running.
sched._running_job_ids.add("guard-seq")
dispatched = []
monkeypatch.setattr(sched, "get_due_jobs", lambda: [job])
monkeypatch.setattr(sched, "advance_next_run", lambda *_a, **_kw: None)
monkeypatch.setattr(sched, "run_job", lambda j: dispatched.append(j["id"]) or (True, "out", "resp", None))
monkeypatch.setattr(sched, "save_job_output", lambda *_a, **_kw: None)
monkeypatch.setattr(sched, "mark_job_run", lambda *_a, **_kw: None)
monkeypatch.setattr(sched, "_deliver_result", lambda *_a, **_kw: None)
n = sched.tick(verbose=False)
assert n == 0 # skipped, not dispatched
assert dispatched == []
sched._running_job_ids.discard("guard-seq")
sched._shutdown_parallel_pool()
def test_get_sequential_pool_is_persistent(self):
"""_get_sequential_pool returns the same single-thread pool."""
import cron.scheduler as sched
sched._sequential_pool = None
pool1 = sched._get_sequential_pool()
pool2 = sched._get_sequential_pool()
assert pool1 is pool2
sched._shutdown_parallel_pool()
assert sched._sequential_pool is None