From 783165c01e4bd0433814bd145a4960b92130100b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Mon, 30 Aug 2021 12:00:31 +0530 Subject: [PATCH] fix: Retry get_redis_conn until "sure" If ConnectionError or BusyLoadingError occurs, try every second for up-to 10 times. Why: `bench start` exits just as i run it at times. This happens when the worker's processes each try to fetch a redis conn but redis isnt up yet. The 3 workeer processes exit with 1 and our procman gives up too. --- frappe/utils/background_jobs.py | 9 ++++++++- requirements.txt | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 567268f760..b8f3372540 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -3,12 +3,14 @@ import socket import time from uuid import uuid4 from collections import defaultdict +from typing import List import redis -from typing import List +from redis.exceptions import BusyLoadingError, ConnectionError from rq import Connection, Queue, Worker from rq.logutils import setup_loghandlers +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed import frappe from frappe import _ @@ -233,6 +235,11 @@ def validate_queue(queue, default_queue_list=None): if queue not in default_queue_list: frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list))) +@retry( + retry=retry_if_exception_type(BusyLoadingError) | retry_if_exception_type(ConnectionError), + stop=stop_after_attempt(10), + wait=wait_fixed(1) +) def get_redis_conn(username=None, password=None): if not hasattr(frappe.local, 'conf'): raise Exception('You need to call frappe.init') diff --git a/requirements.txt b/requirements.txt index 51327953d5..be96520a02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,3 +77,4 @@ Whoosh~=2.7.4 wrapt~=1.12.1 xlrd~=2.0.1 zxcvbn-python~=4.4.24 +tenacity~=8.0.1