From 0fdb4b29ab0136bf718f29cf46072e03d36bbb24 Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Tue, 25 Sep 2012 14:22:22 +0530 Subject: [PATCH] __session_cache removed and introducted memcache for caching --- core/doctype/control_panel/control_panel.py | 2 +- webnotes/__init__.py | 29 ++++++-- webnotes/auth.py | 11 ++- webnotes/boot.py | 27 ++++---- webnotes/db.py | 35 ++-------- webnotes/handler.py | 6 +- webnotes/install_lib/install.py | 9 --- webnotes/memc.py | 67 ++++++++++++++++++ webnotes/model/code.py | 2 +- webnotes/model/utils.py | 1 - webnotes/{session_cache.py => sessions.py} | 67 ++++-------------- webnotes/widgets/auto_master.py | 76 --------------------- wnf.py | 21 +++--- 13 files changed, 139 insertions(+), 214 deletions(-) create mode 100644 webnotes/memc.py rename webnotes/{session_cache.py => sessions.py} (54%) delete mode 100644 webnotes/widgets/auto_master.py diff --git a/core/doctype/control_panel/control_panel.py b/core/doctype/control_panel/control_panel.py index 532cb0029a..2a9405b384 100644 --- a/core/doctype/control_panel/control_panel.py +++ b/core/doctype/control_panel/control_panel.py @@ -27,7 +27,7 @@ import webnotes from webnotes.utils import cint, flt from webnotes.model.doc import Document from webnotes.model.code import get_obj -from webnotes import session, form, is_testing, msgprint, errprint +from webnotes import session, form, msgprint, errprint get_value = webnotes.conn.get_value diff --git a/webnotes/__init__.py b/webnotes/__init__.py index 1131833667..dd55f061a6 100644 --- a/webnotes/__init__.py +++ b/webnotes/__init__.py @@ -36,24 +36,39 @@ code_fields_dict = { 'Control Panel':[('startup_code', 'js'), ('startup_css', 'css')] } -version = 'v170' +class DictObj(dict): + """dict like object that exposes keys as attributes""" + def __getattr__(self, key): + return self.get(key) + def __setattr__(self, key, value): + self[key] = value + def __getstate__(self): + return self + def __setstate__(self, d): + self.update(d) + form_dict = {} -auth_obj = None conn = None +_memc = None form = None session = None user = None -is_testing = None incoming_cookies = {} add_cookies = {} # append these to outgoing request cookies = {} -auto_masters = {} -tenant_id = None -response = {'message':'', 'exc':''} +response = DictObj({'message':'', 'exc':''}) debug_log = [] message_log = [] +# memcache +def cache(): + global _memc + if not _memc: + from webnotes.memc import MClient + _memc = MClient(['localhost:11211']) + return _memc + class ValidationError(Exception): pass @@ -220,7 +235,7 @@ def whitelist(allow_guest=False, allow_roles=[]): def clear_cache(user=None): """clear boot cache""" - from webnotes.session_cache import clear + from webnotes.sessions import clear clear(user) def get_roles(user=None, with_standard=True): diff --git a/webnotes/auth.py b/webnotes/auth.py index 753186cb30..8fffc95acc 100644 --- a/webnotes/auth.py +++ b/webnotes/auth.py @@ -110,7 +110,7 @@ class LoginManager: def __init__(self): if webnotes.form_dict.get('cmd')=='login': # clear cache - from webnotes.session_cache import clear_cache + from webnotes.sessions import clear_cache clear_cache(webnotes.form_dict.get('usr')) self.authenticate() @@ -272,7 +272,7 @@ class Session: def __init__(self, user=None): self.user = user self.sid = webnotes.form_dict.get('sid') or webnotes.incoming_cookies.get('sid', 'Guest') - self.data = {'user':user,'data':{}} + self.data = webnotes.DictObj({'user':user,'data':{}}) if webnotes.form_dict.get('cmd')=='login': self.start() @@ -299,9 +299,9 @@ class Session: r = self.get_session_record() if r: - self.data = {'data': (r[1] and eval(r[1]) or {}), - 'user':r[0], 'sid': self.sid} - else: + self.data = webnotes.DictObj({'data': (r[1] and eval(r[1]) or {}), + 'user':r[0], 'sid': self.sid}) + else: self.start_as_guest() def start_as_guest(self): @@ -324,7 +324,6 @@ class Session: self.data['user'] = webnotes.login_manager.user self.data['sid'] = sid self.data['data']['session_ip'] = os.environ.get('REMOTE_ADDR'); - self.data['data']['tenant_id'] = webnotes.form_dict.get('tenant_id', 0) # get ipinfo if webnotes.conn.get_global('get_ip_info'): diff --git a/webnotes/boot.py b/webnotes/boot.py index bb9bd7a40a..c2acd9f744 100644 --- a/webnotes/boot.py +++ b/webnotes/boot.py @@ -25,19 +25,21 @@ from __future__ import unicode_literals bootstrap client session """ +import webnotes +import webnotes.model.doc +import webnotes.widgets.page +import webnotes.cms + def get_bootinfo(): """build and return boot info""" - import webnotes - bootinfo = {} + bootinfo = webnotes.DictObj() doclist = [] - webnotes.conn.begin() # profile get_profile(bootinfo) # control panel - import webnotes.model.doc cp = webnotes.model.doc.getsingle('Control Panel') from webnotes.utils import cint @@ -48,7 +50,6 @@ def get_bootinfo(): bootinfo['sysdefaults'] = webnotes.utils.get_defaults() if webnotes.session['user'] != 'Guest': - import webnotes.widgets.menus bootinfo['user_info'] = get_fullnames() bootinfo['sid'] = webnotes.session['sid']; @@ -64,20 +65,20 @@ def get_bootinfo(): # plugins try: - import startup.event_handlers - if getattr(startup.event_handlers, 'boot_session', None): - startup.event_handlers.boot_session(bootinfo) + from startup import event_handlers + if getattr(event_handlers, 'boot_session', None): + event_handlers.boot_session(bootinfo) except ImportError: pass - - webnotes.conn.commit() + + from webnotes.model.utils import compress + bootinfo['docs'] = compress(bootinfo['docs']) return bootinfo def get_fullnames(): """map of user fullnames""" - import webnotes ret = webnotes.conn.sql("""select name, concat(ifnull(first_name, ''), if(ifnull(last_name, '')!='', ' ', ''), ifnull(last_name, '')), @@ -96,15 +97,11 @@ def get_fullnames(): def get_profile(bootinfo): """get profile info""" - import webnotes bootinfo['profile'] = webnotes.user.load_profile() webnotes.session['data']['profile'] = bootinfo['profile'] def add_home_page(bootinfo, doclist): """load home page""" - import webnotes - import webnotes.widgets.page - import webnotes.cms home_page = webnotes.cms.get_home_page(webnotes.session['user']) or 'Login Page' diff --git a/webnotes/db.py b/webnotes/db.py index 9389c74a93..556be15dc1 100644 --- a/webnotes/db.py +++ b/webnotes/db.py @@ -44,10 +44,8 @@ class Database: if use_default: self.user = conf.db_name - self.is_testing = 0 self.in_transaction = 0 self.transaction_writes = 0 - self.testing_tables = [] self.auto_commit_on_many_writes = 0 self.password = password or webnotes.get_db_password(self.user) @@ -63,9 +61,8 @@ class Database: """ Connect to a database """ - self._conn = MySQLdb.connect(user=self.user, host=self.host, passwd=self.password, use_unicode=True) + self._conn = MySQLdb.connect(user=self.user, host=self.host, passwd=self.password, use_unicode=True, charset='utf8') self._conn.converter[246]=float - self._conn.set_character_set('utf8') self._cursor = self._conn.cursor() def use(self, db_name): @@ -106,13 +103,13 @@ class Database: result = self._cursor.fetchall() ret = [] for r in result: - dict = {} + row_dict = webnotes.DictObj({}) for i in range(len(r)): val = self.convert_to_simple_type(r[i], formatted) if as_utf8 and type(val) is unicode: val = val.encode('utf-8') - dict[self._cursor.description[i][0]] = val - ret.append(dict) + row_dict[self._cursor.description[i][0]] = val + ret.append(row_dict) return ret def validate_query(self, q): @@ -241,30 +238,6 @@ class Database: nres.append(nr) return nres - # ====================================================================================== - - def replace_tab_by_test(self, query): - """ - Relace all ``tab`` + doctype to ``test`` + doctype - """ - if self.is_testing: - tl = self.get_testing_tables() - for t in tl: - query = query.replace(t, 'test' + t[3:]) - return query - - def get_testing_tables(self): - """ - Get list of all tables for which `tab` is to be replaced by `test` before a query is executed - """ - if not self.testing_tables: - testing_tables = ['tab'+r[0] for r in self.sql('SELECT name from tabDocType where docstatus<2 and (issingle=0 or issingle is null)', allow_testing = 0)] - testing_tables+=['tabSeries','tabSingles'] # tabSessions is not included here - return self.testing_tables - - # ====================================================================================== - # get a single value from a record - def get_value(self, doctype, docname, fieldname, ignore=None): """ Get a single / multiple value from a record. diff --git a/webnotes/handler.py b/webnotes/handler.py index e069ecdcc7..55d6f9833a 100755 --- a/webnotes/handler.py +++ b/webnotes/handler.py @@ -24,6 +24,7 @@ from __future__ import unicode_literals import sys, os import webnotes import webnotes.utils +import webnotes.sessions form = webnotes.form form_dict = webnotes.form_dict @@ -52,10 +53,7 @@ def get_cgi_fields(): @webnotes.whitelist(allow_guest=True) def startup(): - import webnotes - import webnotes.session_cache - - webnotes.response.update(webnotes.session_cache.get()) + webnotes.response.update(webnotes.sessions.get()) def cleanup_docs(): import webnotes.model.utils diff --git a/webnotes/install_lib/install.py b/webnotes/install_lib/install.py index ff8c5cde26..09b7fad7e4 100755 --- a/webnotes/install_lib/install.py +++ b/webnotes/install_lib/install.py @@ -101,7 +101,6 @@ class Installer: import webnotes self.create_sessions_table() self.create_scheduler_log() - self.create_session_cache() self.create_cache_item() self.create_auth_table() @@ -133,14 +132,6 @@ class Installer: method varchar(200), error text ) engine=MyISAM""") - - def create_session_cache(self): - import webnotes - self.dbman.drop_table('__SessionCache') - webnotes.conn.sql("""create table `__SessionCache` ( - user VARCHAR(120), - country VARCHAR(120), - cache LONGTEXT) ENGINE=InnoDB""") def create_cache_item(self): import webnotes diff --git a/webnotes/memc.py b/webnotes/memc.py new file mode 100644 index 0000000000..a4b422e88e --- /dev/null +++ b/webnotes/memc.py @@ -0,0 +1,67 @@ +# Copyright (c) 2012 Web Notes Technologies Pvt Ltd (http://erpnext.com) +# +# MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +from __future__ import unicode_literals + +import memcache, conf +from webnotes.utils import cstr + +class MClient(memcache.Client): + """memcache client that will automatically prefix conf.db_name + and maintain a key list""" + def n(self, key): + return (conf.db_name + ":" + key.replace(" ", "_")).encode('utf-8') + + def set_value(self, key, val): + self.set(self.n(key), val) + self.add_to_key_list(key) + + def get_value(self, key): + return self.get(self.n(key)) + + def delete_value(self, key): + self.delete(self.n(key)) + + def add_to_key_list(self, key): + key_list = self.get_value('key_list') or [] + if key not in key_list: + key_list.append(key) + self.set(self.n("key_list"), key_list) + + def flush_keys(self, startswith): + """flush keys from known key_list""" + if not startswith: + for key in self.get_value('key_list'): + self.delete_value(key) + + self.delete_value('key_list') + else: + deleted = [] + keys = self.get_value('key_list') or [] + for key in keys: + if key.startswith(startswith): + self.delete(self.n(key)) + deleted.append(key) + + for d in deleted: + keys.remove(d) + + self.set_value("key_list", keys) diff --git a/webnotes/model/code.py b/webnotes/model/code.py index 22a22cf526..4526c8d44f 100644 --- a/webnotes/model/code.py +++ b/webnotes/model/code.py @@ -43,7 +43,7 @@ from webnotes.model import db_exists from webnotes.model.doc import Document, addchild, getchildren, make_autoname from webnotes.model.utils import getlist from webnotes.model.code import get_obj, get_server_obj, run_server_obj, updatedb, check_syntax -from webnotes import session, form, is_testing, msgprint, errprint +from webnotes import session, form, msgprint, errprint set = webnotes.conn.set sql = webnotes.conn.sql diff --git a/webnotes/model/utils.py b/webnotes/model/utils.py index d25b7b76bf..e702cbc461 100644 --- a/webnotes/model/utils.py +++ b/webnotes/model/utils.py @@ -81,7 +81,6 @@ def compress(doclist): tmp.append(v) vl.append(tmp) - #errprint(str({'_vl':vl,'_kl':kl})) return {'_vl':vl,'_kl':kl} diff --git a/webnotes/session_cache.py b/webnotes/sessions.py similarity index 54% rename from webnotes/session_cache.py rename to webnotes/sessions.py index 0a97c41059..a2bdbbcbd7 100644 --- a/webnotes/session_cache.py +++ b/webnotes/sessions.py @@ -28,81 +28,40 @@ Session bootstraps info needed by common client side activities including permission, homepage, control panel variables, system defaults etc """ import webnotes +import conf +import json @webnotes.whitelist() def clear(user=None): """clear all cache""" - import webnotes clear_cache(user) webnotes.response['message'] = "Cache Cleared" def clear_cache(user=''): """clear cache""" - import webnotes - if user: - webnotes.conn.sql("delete from __SessionCache where user=%s", user) - webnotes.conn.sql("update tabSessions set sessiondata=NULL where user=%s", user) - else: - webnotes.conn.sql("delete from __SessionCache") - webnotes.conn.sql("update tabSessions set sessiondata=NULL") + webnotes.cache().flush_keys("bootinfo:") + webnotes.cache().flush_keys("doctype:") # doctype cache - import webnotes.utils.cache - webnotes.utils.cache.clear() - + from webnotes.utils.cache import clear + clear() + # rebuild a cache for guest if webnotes.session: webnotes.session['data'] = {} - + def get(): """get session boot info""" - import webnotes - import conf - # get country - country = webnotes.session['data'].get('ipinfo', {}).get('countryName', 'Unknown Country') - # check if cache exists if not getattr(conf,'auto_cache_clear',None): - cache = load(country) + cache = webnotes.cache().get_value('bootinfo:' + webnotes.session.user) if cache: return cache # if not create it - import webnotes.boot - bootinfo = webnotes.boot.get_bootinfo() - add_to_cache(bootinfo, country) + from webnotes.boot import get_bootinfo + bootinfo = get_bootinfo() + webnotes.cache().set_value('bootinfo:' + webnotes.session.user, bootinfo) - return bootinfo - -def load(country): - """load from cache""" - import json - try: - sd = webnotes.conn.sql("select cache from __SessionCache where user='%s' %s" % (webnotes.session['user'], (country and (" and country='%s'" % country) or ''))) - if sd: - return json.loads(sd[0][0]) - else: - return None - except Exception, e: - if e.args[0]==1146: - make_cache_table() - else: - raise e - -def add_to_cache(bootinfo, country): - """add to cache""" - import json - import webnotes.model.utils - - if bootinfo.get('docs'): - bootinfo['docs'] = webnotes.model.utils.compress(bootinfo['docs']) - - # delete earlier (?) - webnotes.conn.sql("""delete from __SessionCache where user=%s - and country=%s""", (webnotes.session['user'], country)) - - # make new - webnotes.conn.sql("""insert into `__SessionCache` - (user, country, cache) VALUES (%s, %s, %s)""", \ - (webnotes.session['user'], country, json.dumps(bootinfo))) + return bootinfo \ No newline at end of file diff --git a/webnotes/widgets/auto_master.py b/webnotes/widgets/auto_master.py deleted file mode 100644 index 4b829403c6..0000000000 --- a/webnotes/widgets/auto_master.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) 2012 Web Notes Technologies Pvt Ltd (http://erpnext.com) -# -# MIT License (MIT) -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE -# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -# auto masters -# _ + fieldname is the table -# 'value' is the column name, pkey - -from __future__ import unicode_literals -import webnotes - -# create masters for a doctype -def create_auto_masters(dt): - fl = webnotes.conn.sql("select fieldname from tabDocField where fieldtype='Data' and options='Suggest' and parent=%s", dt) - for f in fl: - make_auto_master(f[0]) - -# create master table -def make_auto_master(fieldname): - try: - webnotes.conn.sql("select `value` from `__%s` limit 1" % fieldname) - except Exception, e: - if e.args[0]==1146: - webnotes.conn.commit() - webnotes.conn.sql("create table `__%s` (`value` varchar(220), primary key (`value`))" % fieldname) - webnotes.conn.begin() - -# get auto master fields -def get_master_fields(dt): - if not webnotes.session['data'].get('auto_masters'): - webnotes.session['data']['auto_masters'] = {} - - if webnotes.session['data']['auto_masters'].get(dt, None)==None: - fl = webnotes.conn.sql("select fieldname from tabDocField where fieldtype='Data' \ - and options='Suggest' and parent=%s", dt) - webnotes.session['data']['auto_masters'][dt] = fl - return webnotes.session['data']['auto_masters'][dt] - - -# update value -def update_auto_masters(doc): - if not doc.doctype: - return - - fl = get_master_fields(doc.doctype) - - # save masters in session cache - for f in fl: - if doc.fields.get(f[0]): - add_to_master(f[0], doc.fields.get(f[0])) - -# add to master -def add_to_master(fieldname, value): - try: - webnotes.conn.sql("insert into `__%s` (`value`) values (%s)" % (fieldname,'%s'), value) - except Exception, e: - # primary key - pass diff --git a/wnf.py b/wnf.py index 877e5cc512..7fe0bd92ba 100755 --- a/wnf.py +++ b/wnf.py @@ -25,7 +25,7 @@ from __future__ import unicode_literals import os, sys -def replace_code(start, txt1, txt2, extn, search=None): +def replace_code(start, txt1, txt2, extn, search=None, force=False): """replace all txt1 by txt2 in files with extension (extn)""" import webnotes.utils import os, re @@ -40,13 +40,13 @@ def replace_code(start, txt1, txt2, extn, search=None): content = f.read() if re.search(search, content): - res = search_replace_with_prompt(fpath, txt1, txt2) + res = search_replace_with_prompt(fpath, txt1, txt2, force) if res == 'skip': return 'skip' -def search_replace_with_prompt(fpath, txt1, txt2): +def search_replace_with_prompt(fpath, txt1, txt2, force=False): """ Search and replace all txt1 by txt2 in the file with confirmation""" from termcolor import colored @@ -59,12 +59,15 @@ def search_replace_with_prompt(fpath, txt1, txt2): print fpath print colored(txt1, 'red').join(c[:-1].split(txt1)) a = '' - while a.lower() not in ['y', 'n', 'skip']: - a = raw_input('Do you want to Change [y/n/skip]?') - if a.lower() == 'y': + if force: c = c.replace(txt1, txt2) - elif a.lower() == 'skip': - return 'skip' + else: + while a.lower() not in ['y', 'n', 'skip']: + a = raw_input('Do you want to Change [y/n/skip]?') + if a.lower() == 'y': + c = c.replace(txt1, txt2) + elif a.lower() == 'skip': + return 'skip' tmp.append(c) with open(fpath, 'w') as f: @@ -262,7 +265,7 @@ def run(): # code replace elif options.replace: print options.replace - replace_code('.', options.replace[0], options.replace[1], options.replace[2]) + replace_code('.', options.replace[0], options.replace[1], options.replace[2], force=options.force) # git elif options.status: