[security] encrypt passwords that need to be retrievable, except User password which should be hashed

frappe/auth.py

@@ -14,6 +14,7 @@ from frappe import conf
 from frappe.sessions import Session, clear_sessions, delete_session
 from frappe.modules.patch_handler import check_session_stopped
 from frappe.translate import get_lang_code
+from frappe.utils.password import check_password
 
 from urllib import quote
 
@@ -188,12 +189,11 @@ class LoginManager:
 
 	def check_password(self, user, pwd):
 		"""check password"""
-		user = frappe.db.sql("""select `user` from __Auth where `user`=%s
-			and `password`=password(%s)""", (user, pwd))
-		if not user:
+		try:
+			# returns user in correct case
+			return check_password(user, pwd)
+		except frappe.AuthenticationError:
 			self.fail('Incorrect password')
-		else:
-			return user[0][0] # in correct case
 
 	def fail(self, message):
 		frappe.local.response['message'] = message
@@ -290,12 +290,6 @@ class CookieManager:
 		for key in set(self.to_delete):
 			response.set_cookie(key, "", expires=expires)
 
-def _update_password(user, password):
-	frappe.db.sql("""insert into __Auth (user, `password`)
-		values (%s, password(%s))
-		on duplicate key update `password`=password(%s)""", (user,
-		password, password))
-
 @frappe.whitelist()
 def get_logged_user():
 	return frappe.session.user

frappe/commands/site.py

@@ -316,6 +316,7 @@ def move(dest_dir, site):
 def set_admin_password(context, admin_password):
 	"Set Administrator password for a site"
 	import getpass
+	from frappe.utils.password import update_password
 
 	for site in context.sites:
 		try:
@@ -325,8 +326,7 @@ def set_admin_password(context, admin_password):
 				admin_password = getpass.getpass("Administrator's password for {0}: ".format(site))
 
 			frappe.connect()
-			frappe.db.sql("""update __Auth set `password`=password(%s)
-				where user='Administrator'""", (admin_password,))
+			update_password('Administrator', admin_password)
 			frappe.db.commit()
 			admin_password = None
 		finally:

frappe/core/doctype/user/user.py

@@ -5,7 +5,7 @@ from __future__ import unicode_literals
 import frappe
 from frappe.utils import cint, has_gravatar, format_datetime, now_datetime, get_formatted_email
 from frappe import throw, msgprint, _
-from frappe.auth import _update_password
+from frappe.utils.password import update_password as _update_password
 from frappe.desk.notifications import clear_notifications
 from frappe.utils.user import get_system_managers
 import frappe.permissions
@@ -18,6 +18,11 @@ from frappe.model.document import Document
 
 class User(Document):
 	__new_password = None
+
+	def __setup__(self):
+		# because it is handled separately
+		self.flags.ignore_save_passwords = True
+
 	def autoname(self):
 		"""set name as email id"""
 		if self.get("is_admin") or self.get("is_guest"):
@@ -158,12 +163,17 @@ class User(Document):
 	def validate_reset_password(self):
 		pass
 
-	def reset_password(self):
+	def reset_password(self, send_email=False):
 		from frappe.utils import random_string, get_url
 
 		key = random_string(32)
 		self.db_set("reset_password_key", key)
-		self.password_reset_mail(get_url("/update-password?key=" + key))
+		link = get_url("/update-password?key=" + key)
+
+		if send_email:
+			self.password_reset_mail(link)
+
+		return link
 
 	def get_other_system_managers(self):
 		return frappe.db.sql("""select distinct user.name from tabUserRole user_role, tabUser user
@@ -187,10 +197,7 @@ class User(Document):
 	def send_welcome_mail_to_user(self):
 		from frappe.utils import random_string, get_url
 
-		key = random_string(32)
-		self.db_set("reset_password_key", key)
-		link = get_url("/update-password?key=" + key)
-
+		link = self.reset_password()
 		self.send_login_mail(_("Verify Your Account"), "templates/emails/new_user.html",
 			{"link": link, "site_url": get_url()})
 
@@ -237,9 +244,6 @@ class User(Document):
 		if getattr(frappe.local, "login_manager", None):
 			frappe.local.login_manager.logout(user=self.name)
 
-		# delete their password
-		frappe.db.sql("""delete from __Auth where user=%s""", (self.name,))
-
 		# delete todos
 		frappe.db.sql("""delete from `tabToDo` where owner=%s""", (self.name,))
 		frappe.db.sql("""update tabToDo set assigned_by=null where assigned_by=%s""",
@@ -289,10 +293,6 @@ class User(Document):
 			update `tabUser` set email=%s
 			where name=%s""", (newdn, newdn))
 
-		# update __Auth table
-		if not merge:
-			frappe.db.sql("""update __Auth set user=%s where user=%s""", (newdn, olddn))
-
 	def append_roles(self, *roles):
 		"""Add roles to user"""
 		current_roles = [d.role for d in self.get("user_roles")]
@@ -411,6 +411,7 @@ def update_password(new_password, key=None, old_password=None):
 		user = frappe.db.get_value("User", {"reset_password_key":key})
 		if not user:
 			return _("Cannot Update: Incorrect / Expired Link.")
+
 	elif old_password:
 		# verify old password
 		frappe.local.login_manager.check_password(frappe.session.user, old_password)
@@ -474,7 +475,7 @@ def reset_password(user):
 	try:
 		user = frappe.get_doc("User", user)
 		user.validate_reset_password()
-		user.reset_password()
+		user.reset_password(send_email=True)
 
 		return _("Password reset instructions have been sent to your email")
 

frappe/data/Framework.sql

@@ -199,9 +199,13 @@ CREATE TABLE `tabSingles` (
 
 DROP TABLE IF EXISTS `__Auth`;
 CREATE TABLE `__Auth` (
-  `user` VARCHAR(255) NOT NULL PRIMARY KEY,
-  `password` VARCHAR(255) NOT NULL,
-  KEY `user` (`user`)
+	`doctype` VARCHAR(140) NOT NULL,
+	`name` VARCHAR(255) NOT NULL,
+	`fieldname` VARCHAR(140) NOT NULL,
+	`password` VARCHAR(255) NOT NULL,
+	`salt` VARCHAR(140),
+	`encrypted` INT(1) NOT NULL DEFAULT 0,
+	PRIMARY KEY (`doctype`, `name`, `fieldname`)
 ) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --

frappe/desk/page/setup_wizard/setup_wizard.py

@@ -9,6 +9,7 @@ from frappe.translate import (set_default_language, get_dict,
 	get_lang_dict, send_translations, get_language_from_code)
 from frappe.geo.country_info import get_country_info
 from frappe.utils.file_manager import save_file
+from frappe.utils.password import update_password
 
 @frappe.whitelist()
 def setup_complete(args):
@@ -79,8 +80,7 @@ def update_user_name(args):
 		doc.flags.no_welcome_mail = True
 		doc.insert()
 		frappe.flags.mute_emails = _mute_emails
-		from frappe.auth import _update_password
-		_update_password(args.get("email"), args.get("password"))
+		update_password(args.get("email"), args.get("password"))
 
 	else:
 		args['name'] = frappe.session.user

frappe/email/doctype/email_account/email_account.py

@@ -96,7 +96,7 @@ class EmailAccount(Document):
 
 			server = SMTPServer(login = getattr(self, "login_id", None) \
 					or self.email_id,
-				password = self.password,
+				password = self.get_password(),
 				server = self.smtp_server,
 				port = cint(self.smtp_port),
 				use_ssl = cint(self.use_tls)
@@ -109,7 +109,7 @@ class EmailAccount(Document):
 			"host": self.email_server,
 			"use_ssl": self.use_ssl,
 			"username": getattr(self, "login_id", None) or self.email_id,
-			"password": self.password,
+			"password": self.get_password(),
 			"use_imap": self.use_imap
 		}
 

frappe/email/smtp.py

@@ -78,6 +78,8 @@ def get_default_outgoing_email_account(raise_exception_not_set=True):
 		}
 	'''
 	email_account = _get_email_account({"enable_outgoing": 1, "default_outgoing": 1})
+	if email_account:
+		email_account.password = email_account.get_password()
 
 	if not email_account and frappe.conf.get("mail_server"):
 		# from site_config.json

frappe/installer.py

@@ -16,6 +16,7 @@ from frappe.model.sync import sync_for
 from frappe.utils.fixtures import sync_fixtures
 from frappe.website import render, statics
 from frappe.desk.doctype.desktop_icon.desktop_icon import sync_from_app
+from frappe.utils.password import create_auth_table
 
 def install_db(root_login="root", root_password=None, db_name=None, source_sql=None,
 	admin_password=None, verbose=True, force=0, site_config=None, reinstall=False):
@@ -68,12 +69,6 @@ def create_database_and_user(force, verbose):
 	# close root connection
 	frappe.db.close()
 
-def create_auth_table():
-	frappe.db.sql_ddl("""create table if not exists __Auth (
-		`user` VARCHAR(180) NOT NULL PRIMARY KEY,
-		`password` VARCHAR(180) NOT NULL
-		) ENGINE=InnoDB DEFAULT CHARSET=utf8""")
-
 def create_list_settings_table():
 	frappe.db.sql_ddl("""create table if not exists __ListSettings (
 		`user` VARCHAR(180) NOT NULL,

frappe/model/base_document.py

@@ -12,6 +12,7 @@ from frappe.model.utils.link_count import notify_link_count
 from frappe.modules import load_doctype_module
 from frappe.model import display_fieldtypes
 from frappe.model.db_schema import type_map, varchar_len
+from frappe.utils.password import get_decrypted_password, set_encrypted_password
 
 _classes = {}
 
@@ -570,6 +571,29 @@ class BaseDocument(object):
 
 			self.set(fieldname, sanitized_value)
 
+	def _save_passwords(self):
+		'''Save password field values in __Auth table'''
+		if self.flags.ignore_save_passwords:
+			return
+
+		for df in self.meta.get('fields', {'fieldtype': 'Password'}):
+			new_password = self.get(df.fieldname)
+			if new_password and not self.is_dummy_password(new_password):
+				# is not a dummy password like '*****'
+				set_encrypted_password(self.doctype, self.name, new_password, df.fieldname)
+
+				# set dummy password like '*****'
+				self.set(df.fieldname, '*'*len(new_password))
+
+	def get_password(self, fieldname='password', raise_exception=True):
+		if not self.is_dummy_password(self.get(fieldname)):
+			return self.get(fieldname)
+
+		return get_decrypted_password(self.doctype, self.name, fieldname, raise_exception=raise_exception)
+
+	def is_dummy_password(self, pwd):
+		return ''.join(set(pwd))=='*'
+
 	def precision(self, fieldname, parentfield=None):
 		"""Returns float precision for a particular field (or get global default).
 

frappe/model/delete_doc.py

@@ -8,6 +8,7 @@ import frappe.model.meta
 from frappe.model.dynamic_links import get_dynamic_link_map
 import frappe.defaults
 from frappe.utils.file_manager import remove_all
+from frappe.utils.password import delete_all_passwords_for
 from frappe import _
 from frappe.model.naming import revert_series_if_last
 
@@ -36,6 +37,9 @@ def delete_doc(doctype=None, name=None, force=0, ignore_doctypes=None, for_reloa
 		# delete attachments
 		remove_all(doctype, name)
 
+		# delete passwords
+		delete_all_passwords_for(doctype, name)
+
 		doc = None
 		if doctype=="DocType":
 			if for_reload:

frappe/model/document.py

@@ -366,6 +366,7 @@ class Document(BaseDocument):
 		self._validate_constants()
 		self._validate_length()
 		self._sanitize_content()
+		self._save_passwords()
 
 		children = self.get_all_children()
 		for d in children:
@@ -373,6 +374,7 @@ class Document(BaseDocument):
 			d._validate_constants()
 			d._validate_length()
 			d._sanitize_content()
+			d._save_passwords()
 
 		if self.is_new():
 			# don't set fields like _assign, _comments for new doc

frappe/model/rename_doc.py

@@ -7,6 +7,7 @@ from frappe import _
 from frappe.utils import cint
 from frappe.model.naming import validate_name
 from frappe.model.dynamic_links import get_dynamic_link_map
+from frappe.utils.password import rename_password
 
 @frappe.whitelist()
 def rename_doc(doctype, old, new, force=False, merge=False, ignore_permissions=False):
@@ -57,6 +58,9 @@ def rename_doc(doctype, old, new, force=False, merge=False, ignore_permissions=F
 
 	rename_versions(doctype, old, new)
 
+	if not merge:
+		rename_password(doctype, old, new)
+
 	# update user_permissions
 	frappe.db.sql("""update tabDefaultValue set defvalue=%s where parenttype='User Permission'
 		and defkey=%s and defvalue=%s""", (new, doctype, old))

frappe/patches.txt

@@ -127,3 +127,4 @@ frappe.patches.v7_0.desktop_icons_hidden_by_admin_as_blocked
 frappe.patches.v7_0.add_communication_in_doc
 frappe.patches.v7_0.update_send_after_in_bulk_email
 frappe.patches.v7_0.setup_list_settings
+frappe.patches.v7_0.update_auth

frappe/patches/v7_0/update_auth.py

@@ -0,0 +1,34 @@
+from __future__ import unicode_literals
+import frappe
+from frappe.utils.password import create_auth_table, set_encrypted_password
+
+def execute():
+	if '__OldAuth' not in frappe.db.get_tables():
+		frappe.db.sql_ddl('''alter table `__Auth` rename `__OldAuth`''')
+
+	create_auth_table()
+
+	# user passwords
+	frappe.db.sql('''insert ignore into `__Auth` (doctype, name, fieldname, `password`)
+		(select 'User', user, 'password', `password` from `__OldAuth`)''')
+
+	frappe.db.commit()
+
+	# other password fields
+	for doctype in frappe.db.sql_list('''select distinct parent from `tabDocField`
+		where fieldtype="Password" and parent != "User"'''):
+
+		frappe.reload_doctype(doctype)
+		meta = frappe.get_meta(doctype)
+
+		for df in meta.get('fields', {'fieldtype': 'Password'}):
+			for d in frappe.db.sql('''select name, `{fieldname}` from `tab{doctype}`
+				where `{fieldname}` is not null'''.format(fieldname=df.fieldname, doctype=doctype), as_dict=True):
+
+				set_encrypted_password(doctype, d.name, d.get(df.fieldname), fieldname=df.fieldname)
+
+			frappe.db.sql('''update `tab{doctype}` set `{fieldname}`=repeat("*", char_length(`{fieldname}`))'''
+				.format(doctype=doctype, fieldname=df.fieldname))
+			frappe.db.commit()
+
+	frappe.db.sql_ddl('''drop table `__OldAuth`''')

frappe/public/css/website.css

@@ -367,6 +367,11 @@ a.no-decoration:active {
 .indicator-right.light-blue::after {
   background: #7CD6FD;
 }
+.modal-header .indicator {
+  float: left;
+  margin-top: 7.5px;
+  margin-right: 3px;
+}
 html,
 body {
   font-family: "Open Sans", "Helvetica Neue", Serif;

frappe/tests/test_password.py

@@ -0,0 +1,98 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+from __future__ import unicode_literals
+import frappe
+import unittest
+from frappe.utils.password import update_password, check_password
+
+class TestPassword(unittest.TestCase):
+	def setUp(self):
+		frappe.delete_doc('Email Account', 'Test Email Account Password')
+		frappe.delete_doc('Email Account', 'Test Email Account Password-new')
+
+	def test_encrypted_password(self):
+		doc = self.make_email_account()
+
+		new_password = 'test-password'
+		doc.password = new_password
+		doc.save()
+
+		self.assertEquals(doc.password, '*'*len(new_password))
+
+		auth_password = frappe.db.sql('''select `password` from `__Auth`
+			where doctype=%(doctype)s and name=%(name)s and fieldname="password"''', doc.as_dict())[0][0]
+
+		# encrypted
+		self.assertTrue(auth_password != new_password)
+
+		# decrypted
+		self.assertEquals(doc.get_password(), new_password)
+
+		return doc, new_password
+
+	def make_email_account(self, name='Test Email Account Password'):
+		if not frappe.db.exists('Email Account', name):
+			return frappe.get_doc({
+				'doctype': 'Email Account',
+				'email_account_name': name,
+				'append_to': 'Communication',
+				'smtp_server': 'test.example.com',
+				'pop3_server': 'pop.test.example.com',
+				'email_id': 'test@example.com',
+				'password': 'password',
+			}).insert()
+
+		else:
+			return frappe.get_doc('Email Account', name)
+
+	def test_hashed_password(self, user='test@example.com'):
+		old_password = 'testpassword'
+		new_password = 'testpassword-new'
+
+		update_password(user, new_password)
+
+		auth = frappe.db.sql('''select `password`, `salt` from `__Auth`
+			where doctype='User' and name=%s and fieldname="password"''', user, as_dict=True)[0]
+
+		self.assertTrue(auth.password != new_password)
+		self.assertTrue(auth.salt)
+
+		# stored password = password(plain_text_password + salt)
+		self.assertEquals(frappe.db.sql('select password(concat(%s, %s))', (new_password, auth.salt))[0][0], auth.password)
+
+		self.assertTrue(check_password(user, new_password))
+
+		# revert back to old
+		update_password(user, old_password)
+		self.assertTrue(check_password(user, old_password))
+
+		# shouldn't work with old password
+		self.assertRaises(frappe.AuthenticationError, check_password, user, new_password)
+
+	def test_password_on_rename_user(self):
+		password = 'test-rename-password'
+
+		doc = self.make_email_account()
+		doc.password = password
+		doc.save()
+
+		old_name = doc.name
+		new_name = old_name + '-new'
+		frappe.rename_doc(doc.doctype, old_name, new_name)
+
+		new_doc = frappe.get_doc(doc.doctype, new_name)
+		self.assertEquals(new_doc.get_password(), password)
+		self.assertTrue(not frappe.db.sql('''select `password` from `__Auth`
+			where doctype=%s and name=%s and fieldname="password"''', (doc.doctype, doc.name)))
+
+		frappe.rename_doc(doc.doctype, new_name, old_name)
+		self.assertTrue(frappe.db.sql('''select `password` from `__Auth`
+			where doctype=%s and name=%s and fieldname="password"''', (doc.doctype, doc.name)))
+
+	def test_password_on_delete(self):
+		doc = self.make_email_account()
+		doc.delete()
+
+		self.assertTrue(not frappe.db.sql('''select `password` from `__Auth`
+			where doctype=%s and name=%s and fieldname="password"''', (doc.doctype, doc.name)))
+

frappe/utils/install.py

@@ -5,6 +5,7 @@ from __future__ import unicode_literals
 
 import frappe
 import getpass
+from frappe.utils.password import update_password
 
 def before_install():
 	frappe.reload_doc("core", "doctype", "docfield")
@@ -30,8 +31,7 @@ def after_install():
 	frappe.get_doc("User", "Administrator").add_roles(*frappe.db.sql_list("""select name from tabRole"""))
 
 	# update admin password
-	from frappe.auth import _update_password
-	_update_password("Administrator", get_admin_password())
+	update_password("Administrator", get_admin_password())
 
 	# setup wizard now in frappe
 	frappe.db.set_default('desktop:home_page', 'setup-wizard');

frappe/utils/password.py

@@ -0,0 +1,103 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe import _
+from frappe.utils import cstr, encode
+from cryptography.fernet import Fernet
+
+def get_decrypted_password(doctype, name, fieldname='password', raise_exception=True):
+	auth = frappe.db.sql('''select `password` from `__Auth`
+		where doctype=%(doctype)s and name=%(name)s and fieldname=%(fieldname)s and encrypted=1''',
+		{ 'doctype': doctype, 'name': name, 'fieldname': fieldname })
+
+	if auth and auth[0][0]:
+		return decrypt(auth[0][0])
+
+	elif raise_exception:
+		frappe.throw(_('Password not found'), frappe.AuthenticationError)
+
+def set_encrypted_password(doctype, name, pwd, fieldname='password'):
+	frappe.db.sql("""insert into __Auth (doctype, name, fieldname, `password`, encrypted)
+		values (%(doctype)s, %(name)s, %(fieldname)s, %(pwd)s, 1)
+		on duplicate key update `password`=%(pwd)s, encrypted=1""",
+		{ 'doctype': doctype, 'name': name, 'fieldname': fieldname, 'pwd': encrypt(pwd) })
+
+def check_password(user, pwd, doctype='User', fieldname='password'):
+	'''Checks if user and password are correct, else raises frappe.AuthenticationError'''
+
+	auth = frappe.db.sql("""select name, `password`, salt from `__Auth`
+		where doctype=%(doctype)s and name=%(name)s and fieldname=%(fieldname)s and encrypted=0
+		and (
+			(salt is null and `password`=password(%(pwd)s))
+			or `password`=password(concat(%(pwd)s, salt))
+		)""",{ 'doctype': doctype, 'name': user, 'fieldname': fieldname, 'pwd': pwd }, as_dict=True)
+
+	if not auth:
+		raise frappe.AuthenticationError('Incorrect User or Password')
+
+	salt = auth[0].salt
+	if not salt:
+		# sets salt and updates password
+		update_password(user, pwd, doctype, fieldname)
+
+	# lettercase agnostic
+	user = auth[0].name
+
+	return user
+
+def update_password(user, pwd, doctype='User', fieldname='password'):
+	salt = frappe.generate_hash()
+
+	frappe.db.sql("""insert into __Auth (doctype, name, fieldname, `password`, salt, encrypted)
+		values (%(doctype)s, %(name)s, %(fieldname)s, password(concat(%(pwd)s, %(salt)s)), %(salt)s, 0)
+		on duplicate key update
+			`password`=password(concat(%(pwd)s, %(salt)s)), salt=%(salt)s, encrypted=0""",
+		{ 'doctype': doctype, 'name': user, 'fieldname': fieldname, 'pwd': pwd, 'salt': salt })
+
+def delete_all_passwords_for(doctype, name):
+	frappe.db.sql("""delete from __Auth where doctype=%(doctype)s and name=%(name)s""",
+		{ 'doctype': doctype, 'name': name })
+
+def rename_password(doctype, old_name, new_name):
+	# NOTE: fieldname is not considered, since the document is renamed
+	frappe.db.sql("""update __Auth set name=%(new_name)s
+		where doctype=%(doctype)s and name=%(old_name)s""",
+		{ 'doctype': doctype, 'new_name': new_name, 'old_name': old_name })
+
+def create_auth_table():
+	# same as Framework.sql
+	frappe.db.sql_ddl("""create table if not exists __Auth (
+			`doctype` VARCHAR(140) NOT NULL,
+			`name` VARCHAR(255) NOT NULL,
+			`fieldname` VARCHAR(140) NOT NULL,
+			`password` VARCHAR(255) NOT NULL,
+			`salt` VARCHAR(140),
+			`encrypted` INT(1) NOT NULL DEFAULT 0,
+			PRIMARY KEY (`doctype`, `name`, `fieldname`)
+		) ENGINE=InnoDB ROW_FORMAT=COMPRESSED CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci""")
+
+def encrypt(pwd):
+	if len(pwd) > 100:
+		# encrypting > 100 chars will lead to truncation
+		frappe.throw(_('Password cannot be more than 100 characters long'))
+
+	cipher_suite = Fernet(encode(get_encryption_key()))
+	cipher_text = cstr(cipher_suite.encrypt(encode(pwd)))
+	return cipher_text
+
+def decrypt(pwd):
+	cipher_suite = Fernet(encode(get_encryption_key()))
+	plain_text = cstr(cipher_suite.decrypt(encode(pwd)))
+	return plain_text
+
+def get_encryption_key():
+	from frappe.installer import update_site_config
+
+	if 'encryption_key' not in frappe.local.conf:
+		encryption_key = Fernet.generate_key()
+		update_site_config('encryption_key', encryption_key)
+		frappe.local.conf.encryption_key = encryption_key
+
+	return frappe.local.conf.encryption_key

requirements.txt

@@ -34,3 +34,4 @@ Pillow
 beautifulsoup4
 rq
 schedule
+cryptography