fix(cron): make sequential jobs non-blocking too + sweep MCP after jobs finish

Follow-up on the parallel-dispatch decoupling: the sequential pass for
workdir/profile jobs still ran inline in the ticker thread, so a long
workdir/profile job reintroduced the exact starvation #37312 describes,
just for env-mutating jobs. And the MCP orphan sweep ran immediately
after dispatch in sync=False mode — before jobs finished — defeating its
own 'runs after every job' contract and racing jobs still spawning MCP
children.

- Sequential jobs now queue to a persistent single-thread cron-seq pool
  (preserves one-at-a-time ordering across ticks, never blocks the tick).
- Same in-flight dedup guard now covers sequential jobs.
- MCP orphan sweep runs via a done-callback after the LAST dispatched job
  completes in async mode; inline after as_completed in sync mode.

Verified E2E: tick(sync=False) returns in ~1ms with a 1.5s sequential job
in flight; sweep fires only after that job ends.
This commit is contained in:
Teknium
2026-06-04 05:40:13 -07:00
parent eb9cde7346
commit 9fbfeb31b9
4 changed files with 254 additions and 60 deletions
+20 -11
View File
@@ -207,20 +207,23 @@ class TestTickWorkdirPartition:
def test_workdir_jobs_run_sequentially(self, tmp_path, monkeypatch):
import cron.scheduler as sched
# Two "jobs" — one with workdir, one without. get_due_jobs returns both.
workdir_job = {"id": "a", "name": "A", "workdir": str(tmp_path)}
parallel_job = {"id": "b", "name": "B", "workdir": None}
# Two workdir jobs (both sequential) + one parallel job.
workdir_a = {"id": "a", "name": "A", "workdir": str(tmp_path)}
workdir_b = {"id": "b", "name": "B", "workdir": str(tmp_path)}
parallel_job = {"id": "c", "name": "C", "workdir": None}
monkeypatch.setattr(sched, "get_due_jobs", lambda: [workdir_job, parallel_job])
monkeypatch.setattr(sched, "get_due_jobs", lambda: [workdir_a, workdir_b, parallel_job])
monkeypatch.setattr(sched, "advance_next_run", lambda *_a, **_kw: None)
# Record call order / thread context.
import threading
calls: list[tuple[str, bool]] = []
calls: list[tuple[str, str]] = []
order_lock = threading.Lock()
def fake_run_job(job):
# Return a minimal tuple matching run_job's signature.
calls.append((job["id"], threading.current_thread().name))
with order_lock:
calls.append((job["id"], threading.current_thread().name))
return True, "output", "response", None
monkeypatch.setattr(sched, "run_job", fake_run_job)
@@ -231,16 +234,22 @@ class TestTickWorkdirPartition:
)
n = sched.tick(verbose=False)
assert n == 2
assert n == 3
ids = [c[0] for c in calls]
# Workdir jobs always come before parallel jobs.
# Sequential workdir jobs preserve submission order relative to each
# other (single-thread pool).
assert ids.index("a") < ids.index("b")
# The workdir job must run on the main thread (sequential pass).
# Workdir jobs run on the persistent single-thread cron-seq pool —
# NOT the main thread — so a long workdir job never blocks the ticker.
main_thread_name = threading.current_thread().name
workdir_thread_name = next(t for jid, t in calls if jid == "a")
assert workdir_thread_name == main_thread_name
for jid in ("a", "b"):
workdir_thread_name = next(t for j, t in calls if j == jid)
assert workdir_thread_name != main_thread_name
assert workdir_thread_name.startswith("cron-seq"), workdir_thread_name
par_thread_name = next(t for j, t in calls if j == "c")
assert par_thread_name.startswith("cron-parallel"), par_thread_name
# ---------------------------------------------------------------------------