Merge pull request #1784 from vjFaLk/frappe-limits

[WIP] Move Frappe subscription features to Frappe App as Limits

frappe/commands/site.py

@@ -4,6 +4,9 @@ import hashlib, os
 import frappe
 from frappe.commands import pass_context, get_site
 from frappe.commands.scheduler import _is_scheduler_enabled
+from frappe.commands import get_site
+from frappe.limits import set_limits, get_limits
+from frappe.installer import update_site_config
 
 @click.command('new-site')
 @click.argument('site')
@@ -329,6 +332,54 @@ def set_admin_password(context, admin_password):
 		finally:
 			frappe.destroy()
 
+
+@click.command('set-limit')
+@click.option('--site', help='site name')
+@click.argument('limit', type=click.Choice(['email', 'space', 'user', 'expiry']))
+@click.argument('value')
+@pass_context
+def set_limit(context, site, limit, value):
+	"""Sets user / space / email limit for a site"""
+	import datetime
+	if not site:
+		site = get_site(context)
+
+	with frappe.init_site(site):
+		if limit == 'expiry':
+			try:
+				datetime.datetime.strptime(value, '%Y-%m-%d')
+			except ValueError:
+				raise ValueError("Incorrect data format, should be YYYY-MM-DD")
+		else:
+			limit += '_limit'
+			# Space can be float, while other should be integers
+			val = float(value) if limit == 'space_limit' else int(value)
+
+		set_limits({limit : value})
+
+
+@click.command('clear-limit')
+@click.option('--site', help='site name')
+@click.argument('limit', type=click.Choice(['email', 'space', 'user', 'expiry']))
+@pass_context
+def clear_limit(context, site, limit):
+	"""Clears given limit from the site config, and removes limit from site config if its empty"""
+	from frappe.limits import clear_limit as _clear_limit
+	if not site:
+		site = get_site(context)
+
+	with frappe.init_site(site):
+		if not limit == 'expiry':
+			limit += '_limit'
+
+		_clear_limit(limit)
+
+		# Remove limits from the site_config, if it's empty
+		cur_limits = get_limits()
+		if not cur_limits:
+			update_site_config('limits', 'None', validate=False)
+
+
 commands = [
 	add_system_manager,
 	backup,
@@ -344,5 +395,7 @@ commands = [
 	run_patch,
 	set_admin_password,
 	uninstall,
+	set_limit,
+	clear_limit,
 	_use,
 ]

frappe/commands/utils.py

@@ -371,13 +371,17 @@ def make_app(destination, app_name):
 @click.command('set-config')
 @click.argument('key')
 @click.argument('value')
+@click.option('--as-dict', is_flag=True, default=False)
 @pass_context
-def set_config(context, key, value):
+def set_config(context, key, value, as_dict=False):
 	"Insert/Update a value in site_config.json"
 	from frappe.installer import update_site_config
+	import ast
+	if as_dict:
+		value = ast.literal_eval(value)
 	for site in context.sites:
 		frappe.init(site=site)
-		update_site_config(key, value)
+		update_site_config(key, value, validate=False)
 		frappe.destroy()
 
 @click.command('version')

frappe/core/doctype/file/file.py

@@ -8,21 +8,21 @@ record of files
 naming for same name files: file.gif, file-1.gif, file-2.gif etc
 """
 
-import frappe, frappe.utils
-from frappe.utils.file_manager import delete_file_data_content, get_content_hash, get_random_filename
-from frappe import _
-
-from frappe.utils.nestedset import NestedSet
-from frappe.utils import strip
+import frappe
 import json
 import urllib
-from PIL import Image, ImageOps
-import os
+import os, subprocess
 import requests
 import requests.exceptions
 import StringIO
 import mimetypes, imghdr
-from frappe.utils import get_files_path
+
+from frappe.utils.file_manager import delete_file_data_content, get_content_hash, get_random_filename
+from frappe import _
+from frappe.utils.nestedset import NestedSet
+from frappe.limits import get_limits, set_limits
+from frappe.utils import strip, get_url, get_files_path, flt
+from PIL import Image, ImageOps
 
 class FolderNotEmpty(frappe.ValidationError): pass
 
@@ -351,3 +351,67 @@ def check_file_permission(file_url):
 			return True
 
 	raise frappe.PermissionError
+
+def validate_space_limit(file_size):
+	"""Stop from writing file if max space limit is reached"""
+	from frappe.installer import update_site_config
+	from frappe.utils import cint
+	from frappe.utils.file_manager import MaxFileSizeReachedError
+
+	frappe_limits = get_limits()
+
+	if not frappe_limits:
+		return
+
+	if not frappe_limits.has_key('space_limit'):
+		return
+
+	# In Gigabytes
+	space_limit = flt(flt(frappe_limits['space_limit']) * 1024, 2)
+
+	# in Kilobytes
+	used_space = flt(frappe_limits['files_size']) + flt(frappe_limits['backup_size']) + flt(frappe_limits['database_size'])
+	file_size = file_size / (1024.0**2)
+
+	# Stop from attaching file
+	if flt(used_space + file_size, 2) > space_limit:
+		frappe.throw(_("You have exceeded the max space of {0} for your plan. {1} or {2}.").format(
+			"<b>{0}MB</b>".format(cint(space_limit)) if (space_limit < 1024) else "<b>{0}GB</b>".format(frappe_limits['space_limit']),
+			'<a href="#usage-info">{0}</a>'.format(_("Click here to check your usage")),
+			'<a href="#upgrade">{0}</a>'.format(_("upgrade to a higher plan")),
+		), MaxFileSizeReachedError)
+
+	# update files size in frappe subscription
+	new_files_size = flt(frappe_limits['files_size']) + file_size
+	set_limits({'files_size': file_size})
+
+def update_sizes():
+	from frappe.installer import update_site_config
+	# public files
+	files_path = frappe.get_site_path("public", "files")
+	files_size = flt(subprocess.check_output(['du', '-ms', files_path]).split()[0])
+
+	# private files
+	files_path = frappe.get_site_path("private", "files")
+	if os.path.exists(files_path):
+		files_size += flt(subprocess.check_output(['du', '-ms', files_path]).split()[0])
+
+	# backups
+	backup_path = frappe.get_site_path("private", "backups")
+	backup_size = subprocess.check_output(['du', '-ms', backup_path]).split()[0]
+
+	database_size = get_database_size()
+
+	set_limits({'files_size': files_size,
+				'backup_size': backup_size,
+				'database_size': database_size})
+
+def get_database_size():
+	db_name = frappe.conf.db_name
+
+	# This query will get the database size in MB
+	db_size = frappe.db.sql('''
+		SELECT table_schema "database_name", sum( data_length + index_length ) / 1024 / 1024 "database_size"
+		FROM information_schema.TABLES WHERE table_schema = %s GROUP BY table_schema''', db_name, as_dict=True)
+
+	return db_size[0].get('database_size')

frappe/core/doctype/file/test_file.py

@@ -95,3 +95,20 @@ class TestFile(unittest.TestCase):
 
 		folder = frappe.get_doc("File", "Home/Test Folder 1/Test Folder 3")
 		self.assertRaises(frappe.ValidationError, folder.delete)
+
+	def test_file_upload_limit(self):
+		from frappe.utils.file_manager import MaxFileSizeReachedError
+		from frappe.limits import set_limits, clear_limit
+		from frappe import _dict
+
+		set_limits({'space_limit': 1, 'files_size': (1024 * 1024), 'database_size': 0, 'backup_size': 0})
+
+		# Rebuild the frappe.local.conf to take up the changes from site_config
+		frappe.local.conf = _dict(frappe.get_site_config())
+
+		self.assertRaises(MaxFileSizeReachedError, save_file, '_test_max_space.txt',
+			'This files test for max space usage', "", "", self.get_folder("Test Folder 2", "Home").name)
+
+		# Scrub the site_config and rebuild frappe.local.conf
+		clear_limit("space_limit")
+		frappe.local.conf = _dict(frappe.get_site_config())

frappe/core/doctype/user/test_user.py

@@ -3,8 +3,14 @@
 from __future__ import unicode_literals
 
 import frappe, unittest
+import requests
 
 from frappe.model.delete_doc import delete_doc
+from frappe.utils.data import today, add_to_date
+from frappe import _dict
+from frappe.limits import SiteExpiredError, set_limits, clear_limit
+from frappe.utils import get_url
+from frappe.installer import update_site_config
 
 test_records = frappe.get_test_records('User')
 
@@ -72,3 +78,40 @@ class TestUser(unittest.TestCase):
 		me.add_roles("System Manager")
 
 		self.assertTrue("System Manager" in [d.role for d in me.get("user_roles")])
+
+	def test_user_limit_for_site(self):
+		from frappe.core.doctype.user.user import get_total_users
+
+		set_limits({'user_limit': get_total_users()})
+
+		# reload site config
+		from frappe import _dict
+		frappe.local.conf = _dict(frappe.get_site_config())
+
+		# Create a new user
+		user = frappe.new_doc('User')
+		user.email = 'test_max_users@example.com'
+		user.first_name = 'Test_max_user'
+
+		self.assertRaises(frappe.utils.user.MaxUsersReachedError, user.add_roles, 'System Manager')
+
+		if frappe.db.exists('User', 'test_max_users@example.com'):
+			frappe.delete_doc('User', 'test_max_users@example.com')
+
+		# Clear the user limit
+		clear_limit('user_limit')
+
+	def test_site_expiry(self):
+		set_limits({'expiry': add_to_date(today(), days=-1)})
+		frappe.local.conf = _dict(frappe.get_site_config())
+
+		frappe.db.commit()
+
+		res = requests.post(get_url(), params={'cmd': 'login', 'usr': 'test@example.com', 'pwd': 'testpassword',
+			'device': 'desktop'})
+
+		# While site is expired status code returned is 417 Failed Expectation
+		self.assertEqual(res.status_code, 417)
+
+		clear_limit("expiry")
+		frappe.local.conf = _dict(frappe.get_site_config())

frappe/core/page/usage_info/usage_info.css

@@ -0,0 +1,3 @@
+#page-usage-info .indicator-right::after {
+	margin-left: 8px;
+}

frappe/core/page/usage_info/usage_info.html

@@ -0,0 +1,71 @@
+<div class="padding" style="max-width: 500px;">
+    <h3>Users</h3>
+    {% var users_percent = ((users / user_limit) * 100); %}
+    <div class="progress">
+        <div class="progress-bar progress-bar-{%= (users_percent < 75 ? "success" : "warning") %}" style="width: {{ users_percent }}%">
+        </div>
+    </div>
+
+    <table class="table table-bordered">
+        <thead>
+            <tr>
+                <th style="width: 33%">Current Users</th>
+                <th style="width: 33%">Max Users</th>
+                <th style="width: 33%">Remaining</th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr>
+                <td>{%= users %}</td>
+                <td>{%= user_limit %}</td>
+                <td class="{%= ((user_limit - users) > 1) ? "text-success" : "text-warning" %}">{%= user_limit - users %}</td>
+            </tr>
+        </tbody>
+    </table>
+    <br>
+    <h3>Disk Space</h3>
+
+    {% var database_percent = ((database_size / max) * 100); %}
+    {% var files_percent = ((files_size / max) * 100); %}
+    {% var backup_percent = ((backup_size / max) * 100); %}
+    <div class="progress">
+        <div class="progress-bar progress-bar-success" style="width: {%= database_percent %}%">
+        </div>
+        <div class="progress-bar progress-bar-info" style="width: {%= files_percent %}%">
+        </div>
+        <div class="progress-bar progress-bar-warning" style="width: {%= backup_percent %}%">
+        </div>
+    </div>
+
+    <table class="table table-bordered">
+        <thead>
+            <tr>
+                <th style="width: 50%">Type</th>
+                <th style="width: 50%">Size (MB)</th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr>
+                <td><span class="indicator-right green">Database Size</span></td>
+                <td>{%= database_size %} MB</td>
+            </tr>
+            <tr>
+                <td><span class="indicator-right purple">Files Size</span></td>
+                <td>{%= files_size %} MB</td>
+            </tr>
+            <tr>
+                <td><span class="indicator-right orange">Backup Size</span></td>
+                <td>{%= backup_size %} MB</td>
+            </tr>
+            <tr>
+                <td><b>Total</b></td>
+                <td><b>{%= total %} MB</b></td>
+            </tr>
+            <tr>
+                <td><b>Available</b></td>
+                <td class="{%= ((max - total) > 50) ? "" : "text-warning" %}">
+                    <b>{%= flt(max - total, 2) %} MB</b></td>
+            </tr>
+        </tbody>
+    </table>
+</div>

frappe/core/page/usage_info/usage_info.js

@@ -0,0 +1,24 @@
+frappe.pages['usage-info'].on_page_load = function(wrapper) {
+	var page = frappe.ui.make_app_page({
+		parent: wrapper,
+		title: 'Usage Info',
+		single_column: true
+	});
+
+	frappe.call({
+			method: "frappe.limits.get_limits",
+			callback: function(doc) {
+				doc = doc.message;
+				if(!doc.database_size) doc.database_size = 26;
+				if(!doc.files_size) doc.files_size = 1;
+				if(!doc.backup_size) doc.backup_size = 1;
+
+				doc.max = flt(doc.space_limit * 1024);
+				doc.total = (doc.database_size + doc.files_size + doc.backup_size);
+				doc.users = keys(frappe.boot.user_info).length - 2;
+
+				$(frappe.render_template("usage_info", doc)).appendTo(page.main);
+		}
+	});
+
+}

frappe/core/page/usage_info/usage_info.json

@@ -0,0 +1,22 @@
+{
+ "content": null, 
+ "creation": "2016-06-02 18:14:53.475842", 
+ "docstatus": 0, 
+ "doctype": "Page", 
+ "idx": 0, 
+ "modified": "2016-06-02 18:14:53.475842", 
+ "modified_by": "Administrator", 
+ "module": "Core", 
+ "name": "usage-info", 
+ "owner": "Administrator", 
+ "page_name": "usage-info", 
+ "roles": [
+  {
+   "role": "System Manager"
+  }
+ ], 
+ "script": null, 
+ "standard": "Yes", 
+ "style": null, 
+ "title": "Usage Info"
+}

frappe/exceptions.py

@@ -61,4 +61,3 @@ class UniqueValidationError(ValidationError): pass
 class AppNotInstalledError(ValidationError): pass
 class IncorrectSitePath(NotFound): pass
 class ImplicitCommitError(ValidationError): pass
-

frappe/hooks.py

@@ -63,7 +63,9 @@ calendars = ["Event"]
 
 on_session_creation = [
 	"frappe.core.doctype.communication.feed.login_feed",
-	"frappe.core.doctype.user.user.notifify_admin_access_to_system_manager"
+	"frappe.core.doctype.user.user.notifify_admin_access_to_system_manager",
+	"frappe.limits.check_if_expired", #Unsure of where to move
+	"frappe.utils.scheduler.reset_enabled_scheduler_events",
 ]
 
 # permissions
@@ -88,6 +90,9 @@ standard_queries = {
 }
 
 doc_events = {
+	"User": {
+		"validate": "frappe.utils.user.validate_user_limit"
+	},
 	"*": {
 		"after_insert": "frappe.email.doctype.email_alert.email_alert.trigger_email_alerts",
 		"validate": "frappe.email.doctype.email_alert.email_alert.trigger_email_alerts",
@@ -125,6 +130,10 @@ scheduler_events = {
 		"frappe.sessions.clear_expired_sessions",
 		"frappe.email.doctype.email_alert.email_alert.trigger_daily_alerts",
 		"frappe.async.remove_old_task_logs",
+		"frappe.utils.scheduler.disable_scheduler_on_expiry",
+		"frappe.utils.scheduler.restrict_scheduler_events_if_dormant",
+		"frappe.core.doctype.file.file.update_sizes"
+
 	],
 	"daily_long": [
 		"frappe.integrations.doctype.dropbox_backup.dropbox_backup.take_backups_daily"
@@ -161,3 +170,4 @@ bot_parsers = [
 	'frappe.utils.bot.CountBot'
 ]
 
+before_write_file = "frappe.core.doctype.file.file.validate_space_limit"

frappe/installer.py

@@ -256,16 +256,17 @@ def make_site_config(db_name=None, db_password=None, site_config=None):
 		with open(site_file, "w") as f:
 			f.write(json.dumps(site_config, indent=1, sort_keys=True))
 
-def update_site_config(key, value):
+def update_site_config(key, value, validate=True):
 	"""Update a value in site_config"""
 	with open(get_site_config_path(), "r") as f:
 		site_config = json.loads(f.read())
 
-	# int
-	try:
-		value = int(value)
-	except ValueError:
-		pass
+	# In case of non-int value
+	if validate:
+		try:
+			value = int(value)
+		except ValueError:
+			pass
 
 	# boolean
 	if value in ("False", "True"):

frappe/limits.py

@@ -0,0 +1,62 @@
+from __future__ import unicode_literals
+import frappe
+import subprocess, os
+from frappe.model.document import Document
+from frappe.core.doctype.user.user import get_total_users
+from frappe.utils import flt, cint, now_datetime, getdate, get_site_path
+from frappe.installer import update_site_config
+from frappe.utils.file_manager import MaxFileSizeReachedError
+from frappe.utils.data import formatdate
+from frappe import _
+
+class SiteExpiredError(frappe.ValidationError):
+	pass
+
+
+def has_expired():
+	if frappe.session.user=="Administrator":
+		return False
+
+	if not get_limits():
+		return False
+
+	expires_on = get_limits().get("expiry")
+	if not expires_on:
+		return False
+
+	if now_datetime().date() <= getdate(expires_on):
+		return False
+
+	return True
+
+def check_if_expired():
+	"""check if account is expired. If expired, do not allow login"""
+	if not has_expired():
+		return
+	# if expired, stop user from logging in
+	expires_on = formatdate(get_limits().get("expiry"))
+	
+	frappe.throw("""Your subscription expired on <b>{}</b>.
+		To extend please drop a mail at <b>support@erpnext.com</b>""".format(expires_on),
+		SiteExpiredError)
+
+@frappe.whitelist()
+def get_limits():
+	return frappe.get_conf().get("limits")
+
+
+def set_limits(limits):
+		# Add/Update current config options in site_config
+	frappe_limits = get_limits() or {}
+	for key in limits.keys():
+		frappe_limits[key] = limits[key]
+
+	update_site_config("limits", frappe_limits, validate=False)
+
+
+def clear_limit(limit):
+	frappe_limits = get_limits() or {}
+	if limit in frappe_limits:
+		del frappe_limits[limit]
+
+	update_site_config("limits", frappe_limits, validate=False)

frappe/utils/data.py

@@ -556,7 +556,7 @@ def filter_strip_join(some_list, sep):
 
 def get_url(uri=None, full_address=False):
 	"""get app url from request"""
-	host_name = frappe.local.conf.host_name
+	host_name = frappe.local.conf.host_name or frappe.local.conf.hostname
 
 	if uri and (uri.startswith("http://") or uri.startswith("https://")):
 		return uri

frappe/utils/scheduler.py

@@ -14,10 +14,15 @@ import frappe
 import json
 import schedule
 import time
+import os
 import frappe.utils
-from frappe.utils import get_sites
+from frappe.utils import get_sites, get_site_path, touch_file
 from datetime import datetime
 from background_jobs import enqueue, get_jobs, queue_timeout
+from frappe.limits import has_expired
+from frappe.utils.data import get_datetime, now_datetime
+from frappe.core.doctype.user.user import STANDARD_USERS
+from frappe.installer import update_site_config
 
 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S'
 
@@ -218,7 +223,6 @@ def get_error_report(from_date=None, to_date=None, limit=10):
 	else:
 		return 0, "<p>Scheduler didn't encounter any problems.</p>"
 
-
 def scheduler_task(site, event, handler, now=False):
 	'''This is a wrapper function that runs a hooks.scheduler_events method'''
 	frappe.logger(__name__).info('running {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
@@ -239,3 +243,47 @@ def scheduler_task(site, event, handler, now=False):
 		frappe.db.commit()
 
 	frappe.logger(__name__).info('ran {handler} for {site} for event: {event}'.format(handler=handler, site=site, event=event))
+
+
+def reset_enabled_scheduler_events(login_manager):
+	if login_manager.info.user_type == "System User":
+		try:
+			frappe.db.set_global('enabled_scheduler_events', None)
+		except MySQLdb.OperationalError, e:
+			if e.args[0]==1205:
+				frappe.get_logger().error("Error in reset_enabled_scheduler_events")
+			else:
+				raise
+		else:
+			is_dormant = frappe.conf.get('dormant')
+			if is_dormant:
+				update_site_config('dormant', 'None')
+
+def disable_scheduler_on_expiry():
+	if has_expired():
+		disable_scheduler()
+
+def restrict_scheduler_events_if_dormant():
+	if is_dormant():
+		restrict_scheduler_events()
+		update_site_config('dormant', True)
+
+def restrict_scheduler_events(*args, **kwargs):
+	val = json.dumps(["daily", "daily_long", "weekly", "weekly_long", "monthly", "monthly_long"])
+	frappe.db.set_global('enabled_scheduler_events', val)
+
+
+def is_dormant(since = 345600):
+	last_active = get_datetime(get_last_active())
+	# Get now without tz info
+	now = now_datetime().replace(tzinfo=None)
+	time_since_last_active = now - last_active
+	if time_since_last_active.total_seconds() > since:  # 4 days
+		return True
+	return False
+
+def get_last_active():
+	return frappe.db.sql("""select max(ifnull(last_active, "2000-01-01 00:00:00")) from `tabUser`
+		where user_type = 'System User' and name not in ({standard_users})"""\
+		.format(standard_users=", ".join(["%s"]*len(STANDARD_USERS))),
+		STANDARD_USERS)[0][0]

frappe/utils/user.py

@@ -6,6 +6,9 @@ from __future__ import unicode_literals
 import frappe, json
 from frappe import _dict
 import frappe.share
+from frappe import _
+
+class MaxUsersReachedError(frappe.ValidationError): pass
 
 class UserPermissions:
 	"""
@@ -203,11 +206,11 @@ class UserPermissions:
 
 		d.all_reports = self.get_all_reports()
 		return d
-		
+
 	def get_all_reports(self):
-		reports =  frappe.db.sql("""select name, report_type, ref_doctype from tabReport 
+		reports =  frappe.db.sql("""select name, report_type, ref_doctype from tabReport
 		    where ref_doctype in ('{0}')""".format("', '".join(self.can_get_report)), as_dict=1)
-			
+
 		return frappe._dict((d.name, d) for d in reports)
 
 def get_user_fullname(user):
@@ -291,6 +294,7 @@ def is_website_user():
 def is_system_user(username):
 	return frappe.db.get_value("User", {"name": username, "enabled": 1, "user_type": "System User"})
 
+
 def get_users():
 	from frappe.core.doctype.user.user import get_system_users
 	users = []
@@ -303,3 +307,36 @@ def get_users():
 		})
 
 	return users
+
+
+def validate_user_limit(doc, method):
+	"""
+		This is called using validate hook, because welcome email is sent in on_update.
+		We don't want welcome email sent if max users are exceeded.
+	"""
+	from frappe.limits import get_limits
+	from frappe.core.doctype.user.user import get_total_users
+	frappe_limits = get_limits()
+
+	if doc.user_type == "Website User":
+		return
+
+	if not doc.enabled:
+		# don't validate max users when saving a disabled user
+		return
+
+	user_limit = frappe_limits.get("user_limit") if frappe_limits else None
+
+	if not user_limit:
+		return
+
+	total_users = get_total_users()
+
+	if doc.is_new():
+		# get_total_users gets existing users in database
+		# a new record isn't inserted yet, so adding 1
+		total_users += 1
+
+	if total_users > user_limit:
+		frappe.throw(_("Sorry. You have reached the maximum user limit for your subscription. You can either disable an existing user or buy a higher subscription plan."),
+			MaxUsersReachedError)

frappe/website/utils.py

@@ -207,4 +207,3 @@ def get_full_index(route=None, doctype="Web Page", extn = False):
 		return children
 
 	return get_children(route or "")
-