fix: add env overrides for service orchestration (#21577)

This avoids having to manipulate config files in brittle bash
entrypoints that need to react to dynamic service discovery.

This significantly improves the operability of various bench sites.

frappe/__init__.py

@@ -243,7 +243,7 @@ def init(site: str, sites_path: str = ".", new_site: bool = False, force=False)
 	local.preload_assets = {"style": [], "script": []}
 	local.session = _dict()
 	local.dev_server = _dev_server
-	local.qb = get_query_builder(local.conf.db_type or "mariadb")
+	local.qb = get_query_builder(local.conf.db_type)
 	local.qb.get_query = get_query
 	setup_redis_cache_connection()
 	setup_module_map()
@@ -320,6 +320,27 @@ def get_site_config(sites_path: str | None = None, site_path: str | None = None)
 		elif local.site and not local.flags.new_site:
 			raise IncorrectSitePath(f"{local.site} does not exist")
 
+	# Generalized env variable overrides and defaults
+	def db_default_ports(db_type):
+		from frappe.database.mariadb.database import MariaDBDatabase
+
+		return {
+			"mariadb": MariaDBDatabase.default_port,  # 3306
+			"postgres": 5432,
+		}[db_type]
+
+	config["redis_queue"] = (
+		os.environ.get("FRAPPE_REDIS_QUEUE") or config.get("redis_queue") or "redis://127.0.0.1:11311"
+	)
+	config["redis_cache"] = (
+		os.environ.get("FRAPPE_REDIS_CACHE") or config.get("redis_cache") or "redis://127.0.0.1:13311"
+	)
+	config["db_type"] = os.environ.get("FRAPPE_DB_TYPE") or config.get("db_type") or "mariadb"
+	config["db_host"] = os.environ.get("FRAPPE_DB_HOST") or config.get("db_host") or "127.0.0.1"
+	config["db_port"] = (
+		os.environ.get("FRAPPE_DB_PORT") or config.get("db_port") or db_default_ports(config["db_type"])
+	)
+
 	return _dict(config)
 
 
@@ -360,7 +381,7 @@ def setup_redis_cache_connection():
 	if not cache:
 		from frappe.utils.redis_wrapper import RedisWrapper
 
-		cache = RedisWrapper.from_url(conf.get("redis_cache") or "redis://localhost:11311")
+		cache = RedisWrapper.from_url(conf.get("redis_cache"))
 
 
 def get_traceback(with_context: bool = False) -> str:

frappe/commands/utils.py

@@ -469,7 +469,7 @@ def database(context, extra_args):
 	if not site:
 		raise SiteNotSpecifiedError
 	frappe.init(site=site)
-	if not frappe.conf.db_type or frappe.conf.db_type == "mariadb":
+	if frappe.conf.db_type == "mariadb":
 		_mariadb(extra_args=extra_args)
 	elif frappe.conf.db_type == "postgres":
 		_psql(extra_args=extra_args)
@@ -505,19 +505,17 @@ def postgres(context, extra_args):
 
 
 def _mariadb(extra_args=None):
-	from frappe.database.mariadb.database import MariaDBDatabase
-
 	mysql = which("mysql")
 	command = [
 		mysql,
 		"--port",
-		str(frappe.conf.db_port or MariaDBDatabase.default_port),
+		str(frappe.conf.db_port),
 		"-u",
 		frappe.conf.db_name,
 		f"-p{frappe.conf.db_password}",
 		frappe.conf.db_name,
 		"-h",
-		frappe.conf.db_host or "localhost",
+		frappe.conf.db_host,
 		"--pager=less -SFX",
 		"--safe-updates",
 		"-A",
@@ -530,8 +528,8 @@ def _mariadb(extra_args=None):
 def _psql(extra_args=None):
 	psql = which("psql")
 
-	host = frappe.conf.db_host or "127.0.0.1"
+	host = frappe.conf.db_host
-	port = frappe.conf.db_port or "5432"
+	port = frappe.conf.db_port
 	env = os.environ.copy()
 	env["PGPASSWORD"] = frappe.conf.db_password
 	conn_string = f"postgresql://{frappe.conf.db_name}@{host}:{port}/{frappe.conf.db_name}"
@@ -666,7 +664,7 @@ def transform_database(context, table, engine, row_format, failfast):
 	skipped = 0
 	frappe.init(site=site)
 
-	if frappe.conf.db_type and frappe.conf.db_type != "mariadb":
+	if frappe.conf.db_type != "mariadb":
 		click.secho("This command only has support for MariaDB databases at this point", fg="yellow")
 		sys.exit(1)
 

frappe/database/database.py

@@ -87,8 +87,8 @@ class Database:
 		port=None,
 	):
 		self.setup_type_map()
-		self.host = host or frappe.conf.db_host or "127.0.0.1"
+		self.host = host or frappe.conf.db_host
-		self.port = port or frappe.conf.db_port or ""
+		self.port = port or frappe.conf.db_port
 		self.user = user or frappe.conf.db_name
 		self.db_name = frappe.conf.db_name
 		self._conn = None

frappe/database/db_manager.py

@@ -68,11 +68,7 @@ class DbManager:
 		if pipe:
 			print("Restoring Database file...")
 
-		command = (
+		command = "{pipe} mysql -u {user} -p{password} -h{host} -P{port} {target} {source}"
-			"{pipe} mysql -u {user} -p{password} -h{host} "
-			+ ("-P{port}" if frappe.db.port else "")
-			+ " {target} {source}"
-		)
 		command = command.format(
 			pipe=pipe,
 			user=esc(user),

frappe/database/postgres/setup_db.py

@@ -54,7 +54,7 @@ def import_db_from_sql(source_sql=None, verbose=False):
 
 	_command = (
 		f"psql {frappe.conf.db_name} "
-		f"-h {frappe.conf.db_host or 'localhost'} -p {str(frappe.conf.db_port or '5432')} "
+		f"-h {frappe.conf.db_host} -p {str(frappe.conf.db_port)} "
 		f"-U {frappe.conf.db_name}"
 	)
 

frappe/installer.py

@@ -133,7 +133,7 @@ def install_db(
 	from frappe.database import setup_database
 
 	if not db_type:
-		db_type = frappe.conf.db_type or "mariadb"
+		db_type = frappe.conf.db_type
 
 	if not root_login and db_type == "mariadb":
 		root_login = "root"
@@ -772,7 +772,7 @@ def is_downgrade(sql_file_path, verbose=False):
 
 	# This function is only tested with mariadb
 	# TODO: Add postgres support
-	if frappe.conf.db_type not in (None, "mariadb"):
+	if frappe.conf.db_type != "mariadb":
 		return False
 
 	from semantic_version import Version
@@ -824,7 +824,7 @@ def is_partial(sql_file_path):
 def partial_restore(sql_file_path, verbose=False):
 	sql_file = extract_sql_from_archive(sql_file_path)
 
-	if frappe.conf.db_type in (None, "mariadb"):
+	if frappe.conf.db_type == "mariadb":
 		from frappe.database.mariadb.setup_db import import_db_from_sql
 	elif frappe.conf.db_type == "postgres":
 		import warnings

frappe/query_builder/utils.py

@@ -31,7 +31,7 @@ class ImportMapper:
 		self.func_map = func_map
 
 	def __call__(self, *args: Any, **kwds: Any) -> Callable:
-		db = db_type_is(frappe.conf.db_type or "mariadb")
+		db = db_type_is(frappe.conf.db_type)
 		return self.func_map[db](*args, **kwds)
 
 

frappe/tests/test_commands.py

@@ -442,7 +442,7 @@ class TestCommands(BaseTestCommands):
 			f"bench new-site {site} --force --verbose "
 			f"--admin-password {frappe.conf.admin_password} "
 			f"--mariadb-root-password {frappe.conf.root_password} "
-			f"--db-type {frappe.conf.db_type or 'mariadb'} "
+			f"--db-type {frappe.conf.db_type} "
 		)
 		self.assertEqual(self.returncode, 0)
 
@@ -467,7 +467,7 @@ class TestCommands(BaseTestCommands):
 				f"bench new-site {TEST_SITE} --verbose "
 				f"--admin-password {frappe.conf.admin_password} "
 				f"--mariadb-root-password {frappe.conf.root_password} "
-				f"--db-type {frappe.conf.db_type or 'mariadb'} "
+				f"--db-type {frappe.conf.db_type} "
 			)
 
 		app_name = "frappe"

frappe/utils/logger.py

@@ -12,6 +12,7 @@ import frappe
 from frappe.utils import get_sites
 
 default_log_level = logging.WARNING if frappe._dev_server else logging.ERROR
+stream_logging = os.environ.get("FRAPPE_STREAM_LOGGING")
 
 
 def get_logger(
@@ -21,7 +22,7 @@ def get_logger(
 	filter=None,
 	max_size=100_000,
 	file_count=20,
-	stream_only=False,
+	stream_only=stream_logging,
 ) -> "logging.Logger":
 	"""Application Logger for your given module
 

node_utils.js

@@ -6,7 +6,6 @@ const bench_path = path.resolve(__dirname, "..", "..");
 function get_conf() {
 	// defaults
 	var conf = {
-		redis_async_broker_port: "redis://localhost:12311",
 		socketio_port: 3000,
 	};
 
@@ -27,16 +26,25 @@ function get_conf() {
 	read_config("config.json");
 	read_config("sites/common_site_config.json");
 
-	// set default site
+	// set overrides from environment
 	if (process.env.FRAPPE_SITE) {
 		conf.default_site = process.env.FRAPPE_SITE;
 	}
+	if (process.env.FRAPPE_REDIS_CACHE) {
+		conf.redis_cache = process.env.FRAPPE_REDIS_CACHE;
+	}
+	if (process.env.FRAPPE_REDIS_QUEUE) {
+		conf.redis_queue = process.env.FRAPPE_REDIS_QUEUE;
+	}
+	if (process.env.FRAPPE_SOCKETIO_PORT) {
+		conf.socketio_port = process.env.FRAPPE_SOCKETIO_PORT;
+	}
 	return conf;
 }
 
 function get_redis_subscriber(kind = "redis_queue", options = {}) {
 	const conf = get_conf();
-	const host = conf[kind] || conf.redis_async_broker_port;
+	const host = conf[kind];
 	return redis.createClient({ url: host, ...options });
 }