diff --git a/frappe/utils/background_jobs.py b/frappe/utils/background_jobs.py index 567268f760..b8f3372540 100755 --- a/frappe/utils/background_jobs.py +++ b/frappe/utils/background_jobs.py @@ -3,12 +3,14 @@ import socket import time from uuid import uuid4 from collections import defaultdict +from typing import List import redis -from typing import List +from redis.exceptions import BusyLoadingError, ConnectionError from rq import Connection, Queue, Worker from rq.logutils import setup_loghandlers +from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed import frappe from frappe import _ @@ -233,6 +235,11 @@ def validate_queue(queue, default_queue_list=None): if queue not in default_queue_list: frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list))) +@retry( + retry=retry_if_exception_type(BusyLoadingError) | retry_if_exception_type(ConnectionError), + stop=stop_after_attempt(10), + wait=wait_fixed(1) +) def get_redis_conn(username=None, password=None): if not hasattr(frappe.local, 'conf'): raise Exception('You need to call frappe.init') diff --git a/requirements.txt b/requirements.txt index 51327953d5..be96520a02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,3 +77,4 @@ Whoosh~=2.7.4 wrapt~=1.12.1 xlrd~=2.0.1 zxcvbn-python~=4.4.24 +tenacity~=8.0.1