fix: Retry get_redis_conn until "sure"

If ConnectionError or BusyLoadingError occurs, try every second for
up-to 10 times.

Why: `bench start` exits just as i run it at times. This happens when
the worker's processes each try to fetch a redis conn but redis isnt up
yet. The 3 workeer processes exit with 1 and our procman gives up too.

frappe/utils/background_jobs.py

@@ -3,12 +3,14 @@ import socket
 import time
 from uuid import uuid4
 from collections import defaultdict
+from typing import List
 
 
 import redis
-from typing import List
+from redis.exceptions import BusyLoadingError, ConnectionError
 from rq import Connection, Queue, Worker
 from rq.logutils import setup_loghandlers
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 
 import frappe
 from frappe import _
@@ -233,6 +235,11 @@ def validate_queue(queue, default_queue_list=None):
 	if queue not in default_queue_list:
 		frappe.throw(_("Queue should be one of {0}").format(', '.join(default_queue_list)))
 
+@retry(
+	retry=retry_if_exception_type(BusyLoadingError) | retry_if_exception_type(ConnectionError),
+	stop=stop_after_attempt(10),
+	wait=wait_fixed(1)
+)
 def get_redis_conn(username=None, password=None):
 	if not hasattr(frappe.local, 'conf'):
 		raise Exception('You need to call frappe.init')

requirements.txt

@@ -77,3 +77,4 @@ Whoosh~=2.7.4
 wrapt~=1.12.1
 xlrd~=2.0.1
 zxcvbn-python~=4.4.24
+tenacity~=8.0.1