From 8baeb5151d822276cf9660b83da092b8b56bc6bf Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Wed, 26 Feb 2025 15:00:38 +0530 Subject: [PATCH] fix: check for running jobs before migrating (#31438) --- frappe/commands/scheduler.py | 11 ++++++++++- frappe/utils/doctor.py | 6 ++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/frappe/commands/scheduler.py b/frappe/commands/scheduler.py index 14772e8740..b02ed4acbf 100755 --- a/frappe/commands/scheduler.py +++ b/frappe/commands/scheduler.py @@ -233,6 +233,8 @@ def start_worker_pool(queue, quiet=False, num_workers=2, burst=False): @click.option("--site", help="site name") @pass_context def ready_for_migration(context: CliCtxObj, site=None): + import time + from frappe.utils.doctor import any_job_pending if not site: @@ -240,7 +242,14 @@ def ready_for_migration(context: CliCtxObj, site=None): try: frappe.init(site) - pending_jobs = any_job_pending(site=site) + pending_jobs = False + + # HACK: Check at least 3 times, 1 second apart. + # Rare edge case: Scheduler hasn't seen 'maintenance_mode=1` yet + # and takes more than 3 second to schedule. + for _ in range(3): + pending_jobs |= any_job_pending(site=site) + time.sleep(1) if pending_jobs: print(f"NOT READY for migration: site {site} has pending background jobs") diff --git a/frappe/utils/doctor.py b/frappe/utils/doctor.py index cef992537e..68fd0c08b6 100644 --- a/frappe/utils/doctor.py +++ b/frappe/utils/doctor.py @@ -80,9 +80,15 @@ def get_pending_jobs(site=None): def any_job_pending(site: str) -> bool: for queue in get_queue_list(): q = get_queue(queue) + # pending jobs for job_id in q.get_job_ids(): if job_id.startswith(site): return True + + # already running jobs + for job_id in q.started_job_registry.get_job_ids(): + if job_id.startswith(site): + return True return False