vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

__session_cache removed and introducted memcache for caching

Browse source
This commit is contained in:
Rushabh Mehta 2012-09-25 14:22:22 +05:30
parent f486cad30c
commit 0fdb4b29ab
13 changed files with 139 additions and 214 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
core/doctype/control_panel/control_panel.py
View file

@ -27,7 +27,7 @@ import webnotes
from webnotes.utils import cint, flt
from webnotes.model.doc import Document
from webnotes.model.code import get_obj
from webnotes import session, form, is_testing, msgprint, errprint
from webnotes import session, form, msgprint, errprint
get_value = webnotes.conn.get_value

29
webnotes/__init__.py
View file

@ -36,24 +36,39 @@ code_fields_dict = {
'Control Panel':[('startup_code', 'js'), ('startup_css', 'css')]
}
version = 'v170'
class DictObj(dict):
"""dict like object that exposes keys as attributes"""
def __getattr__(self, key):
return self.get(key)
def __setattr__(self, key, value):
self[key] = value
def __getstate__(self):
return self
def __setstate__(self, d):
self.update(d)
form_dict = {}
auth_obj = None
conn = None
_memc = None
form = None
session = None
user = None
is_testing = None
incoming_cookies = {}
add_cookies = {} # append these to outgoing request
cookies = {}
auto_masters = {}
tenant_id = None
response = {'message':'', 'exc':''}
response = DictObj({'message':'', 'exc':''})
debug_log = []
message_log = []
# memcache
def cache():
global _memc
if not _memc:
from webnotes.memc import MClient
_memc = MClient(['localhost:11211'])
return _memc
class ValidationError(Exception):
pass
@ -220,7 +235,7 @@ def whitelist(allow_guest=False, allow_roles=[]):
def clear_cache(user=None):
"""clear boot cache"""
from webnotes.session_cache import clear
from webnotes.sessions import clear
clear(user)
def get_roles(user=None, with_standard=True):

11
webnotes/auth.py
View file

@ -110,7 +110,7 @@ class LoginManager:
def __init__(self):
if webnotes.form_dict.get('cmd')=='login':
# clear cache
from webnotes.session_cache import clear_cache
from webnotes.sessions import clear_cache
clear_cache(webnotes.form_dict.get('usr'))
self.authenticate()
@ -272,7 +272,7 @@ class Session:
def __init__(self, user=None):
self.user = user
self.sid = webnotes.form_dict.get('sid') or webnotes.incoming_cookies.get('sid', 'Guest')
self.data = {'user':user,'data':{}}
self.data = webnotes.DictObj({'user':user,'data':{}})
if webnotes.form_dict.get('cmd')=='login':
self.start()
@ -299,9 +299,9 @@ class Session:
r = self.get_session_record()
if r:
self.data = {'data': (r[1] and eval(r[1]) or {}),
'user':r[0], 'sid': self.sid}
else:
self.data = webnotes.DictObj({'data': (r[1] and eval(r[1]) or {}),
'user':r[0], 'sid': self.sid})
else:
self.start_as_guest()
def start_as_guest(self):
@ -324,7 +324,6 @@ class Session:
self.data['user'] = webnotes.login_manager.user
self.data['sid'] = sid
self.data['data']['session_ip'] = os.environ.get('REMOTE_ADDR');
self.data['data']['tenant_id'] = webnotes.form_dict.get('tenant_id', 0)
# get ipinfo
if webnotes.conn.get_global('get_ip_info'):

27
webnotes/boot.py
View file

@ -25,19 +25,21 @@ from __future__ import unicode_literals
bootstrap client session
"""
import webnotes
import webnotes.model.doc
import webnotes.widgets.page
import webnotes.cms
def get_bootinfo():
"""build and return boot info"""
import webnotes
bootinfo = {}
bootinfo = webnotes.DictObj()
doclist = []
webnotes.conn.begin()
# profile
get_profile(bootinfo)
# control panel
import webnotes.model.doc
cp = webnotes.model.doc.getsingle('Control Panel')
from webnotes.utils import cint
@ -48,7 +50,6 @@ def get_bootinfo():
bootinfo['sysdefaults'] = webnotes.utils.get_defaults()
if webnotes.session['user'] != 'Guest':
import webnotes.widgets.menus
bootinfo['user_info'] = get_fullnames()
bootinfo['sid'] = webnotes.session['sid'];
@ -64,20 +65,20 @@ def get_bootinfo():
# plugins
try:
import startup.event_handlers
if getattr(startup.event_handlers, 'boot_session', None):
startup.event_handlers.boot_session(bootinfo)
from startup import event_handlers
if getattr(event_handlers, 'boot_session', None):
event_handlers.boot_session(bootinfo)
except ImportError:
pass
webnotes.conn.commit()
from webnotes.model.utils import compress
bootinfo['docs'] = compress(bootinfo['docs'])
return bootinfo
def get_fullnames():
"""map of user fullnames"""
import webnotes
ret = webnotes.conn.sql("""select name,
concat(ifnull(first_name, ''),
if(ifnull(last_name, '')!='', ' ', ''), ifnull(last_name, '')),
@ -96,15 +97,11 @@ def get_fullnames():
def get_profile(bootinfo):
"""get profile info"""
import webnotes
bootinfo['profile'] = webnotes.user.load_profile()
webnotes.session['data']['profile'] = bootinfo['profile']
def add_home_page(bootinfo, doclist):
"""load home page"""
import webnotes
import webnotes.widgets.page
import webnotes.cms
home_page = webnotes.cms.get_home_page(webnotes.session['user']) or 'Login Page'

35
webnotes/db.py
View file

@ -44,10 +44,8 @@ class Database:
if use_default:
self.user = conf.db_name
self.is_testing = 0
self.in_transaction = 0
self.transaction_writes = 0
self.testing_tables = []
self.auto_commit_on_many_writes = 0
self.password = password or webnotes.get_db_password(self.user)
@ -63,9 +61,8 @@ class Database:
"""
Connect to a database
"""
self._conn = MySQLdb.connect(user=self.user, host=self.host, passwd=self.password, use_unicode=True)
self._conn = MySQLdb.connect(user=self.user, host=self.host, passwd=self.password, use_unicode=True, charset='utf8')
self._conn.converter[246]=float
self._conn.set_character_set('utf8')
self._cursor = self._conn.cursor()
def use(self, db_name):
@ -106,13 +103,13 @@ class Database:
result = self._cursor.fetchall()
ret = []
for r in result:
dict = {}
row_dict = webnotes.DictObj({})
for i in range(len(r)):
val = self.convert_to_simple_type(r[i], formatted)
if as_utf8 and type(val) is unicode:
val = val.encode('utf-8')
dict[self._cursor.description[i][0]] = val
ret.append(dict)
row_dict[self._cursor.description[i][0]] = val
ret.append(row_dict)
return ret
def validate_query(self, q):
@ -241,30 +238,6 @@ class Database:
nres.append(nr)
return nres
# ======================================================================================
def replace_tab_by_test(self, query):
"""
Relace all ``tab`` + doctype to ``test`` + doctype
"""
if self.is_testing:
tl = self.get_testing_tables()
for t in tl:
query = query.replace(t, 'test' + t[3:])
return query
def get_testing_tables(self):
"""
Get list of all tables for which `tab` is to be replaced by `test` before a query is executed
"""
if not self.testing_tables:
testing_tables = ['tab'+r[0] for r in self.sql('SELECT name from tabDocType where docstatus<2 and (issingle=0 or issingle is null)', allow_testing = 0)]
testing_tables+=['tabSeries','tabSingles'] # tabSessions is not included here
return self.testing_tables
# ======================================================================================
# get a single value from a record
def get_value(self, doctype, docname, fieldname, ignore=None):
"""
Get a single / multiple value from a record.

6
webnotes/handler.py
View file

@ -24,6 +24,7 @@ from __future__ import unicode_literals
import sys, os
import webnotes
import webnotes.utils
import webnotes.sessions
form = webnotes.form
form_dict = webnotes.form_dict
@ -52,10 +53,7 @@ def get_cgi_fields():
@webnotes.whitelist(allow_guest=True)
def startup():
import webnotes
import webnotes.session_cache
webnotes.response.update(webnotes.session_cache.get())
webnotes.response.update(webnotes.sessions.get())
def cleanup_docs():
import webnotes.model.utils

9
webnotes/install_lib/install.py
View file

@ -101,7 +101,6 @@ class Installer:
import webnotes
self.create_sessions_table()
self.create_scheduler_log()
self.create_session_cache()
self.create_cache_item()
self.create_auth_table()
@ -133,14 +132,6 @@ class Installer:
method varchar(200),
error text
) engine=MyISAM""")
def create_session_cache(self):
import webnotes
self.dbman.drop_table('__SessionCache')
webnotes.conn.sql("""create table `__SessionCache` (
user VARCHAR(120),
country VARCHAR(120),
cache LONGTEXT) ENGINE=InnoDB""")
def create_cache_item(self):
import webnotes

67
webnotes/memc.py Normal file
View file

@ -0,0 +1,67 @@
# Copyright (c) 2012 Web Notes Technologies Pvt Ltd (http://erpnext.com)
#
# MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
from __future__ import unicode_literals
import memcache, conf
from webnotes.utils import cstr
class MClient(memcache.Client):
"""memcache client that will automatically prefix conf.db_name
and maintain a key list"""
def n(self, key):
return (conf.db_name + ":" + key.replace(" ", "_")).encode('utf-8')
def set_value(self, key, val):
self.set(self.n(key), val)
self.add_to_key_list(key)
def get_value(self, key):
return self.get(self.n(key))
def delete_value(self, key):
self.delete(self.n(key))
def add_to_key_list(self, key):
key_list = self.get_value('key_list') or []
if key not in key_list:
key_list.append(key)
self.set(self.n("key_list"), key_list)
def flush_keys(self, startswith):
"""flush keys from known key_list"""
if not startswith:
for key in self.get_value('key_list'):
self.delete_value(key)
self.delete_value('key_list')
else:
deleted = []
keys = self.get_value('key_list') or []
for key in keys:
if key.startswith(startswith):
self.delete(self.n(key))
deleted.append(key)
for d in deleted:
keys.remove(d)
self.set_value("key_list", keys)

2
webnotes/model/code.py
View file

@ -43,7 +43,7 @@ from webnotes.model import db_exists
from webnotes.model.doc import Document, addchild, getchildren, make_autoname
from webnotes.model.utils import getlist
from webnotes.model.code import get_obj, get_server_obj, run_server_obj, updatedb, check_syntax
from webnotes import session, form, is_testing, msgprint, errprint
from webnotes import session, form, msgprint, errprint
set = webnotes.conn.set
sql = webnotes.conn.sql

1
webnotes/model/utils.py
View file

@ -81,7 +81,6 @@ def compress(doclist):
tmp.append(v)
vl.append(tmp)
#errprint(str({'_vl':vl,'_kl':kl}))
return {'_vl':vl,'_kl':kl}

67
webnotes/session_cache.py → webnotes/sessions.py
View file

@ -28,81 +28,40 @@ Session bootstraps info needed by common client side activities including
permission, homepage, control panel variables, system defaults etc
"""
import webnotes
import conf
import json
@webnotes.whitelist()
def clear(user=None):
"""clear all cache"""
import webnotes
clear_cache(user)
webnotes.response['message'] = "Cache Cleared"
def clear_cache(user=''):
"""clear cache"""
import webnotes
if user:
webnotes.conn.sql("delete from __SessionCache where user=%s", user)
webnotes.conn.sql("update tabSessions set sessiondata=NULL where user=%s", user)
else:
webnotes.conn.sql("delete from __SessionCache")
webnotes.conn.sql("update tabSessions set sessiondata=NULL")
webnotes.cache().flush_keys("bootinfo:")
webnotes.cache().flush_keys("doctype:")
# doctype cache
import webnotes.utils.cache
webnotes.utils.cache.clear()
from webnotes.utils.cache import clear
clear()
# rebuild a cache for guest
if webnotes.session:
webnotes.session['data'] = {}
def get():
"""get session boot info"""
import webnotes
import conf
# get country
country = webnotes.session['data'].get('ipinfo', {}).get('countryName', 'Unknown Country')
# check if cache exists
if not getattr(conf,'auto_cache_clear',None):
cache = load(country)
cache = webnotes.cache().get_value('bootinfo:' + webnotes.session.user)
if cache:
return cache
# if not create it
import webnotes.boot
bootinfo = webnotes.boot.get_bootinfo()
add_to_cache(bootinfo, country)
from webnotes.boot import get_bootinfo
bootinfo = get_bootinfo()
webnotes.cache().set_value('bootinfo:' + webnotes.session.user, bootinfo)
return bootinfo
def load(country):
"""load from cache"""
import json
try:
sd = webnotes.conn.sql("select cache from __SessionCache where user='%s' %s" % (webnotes.session['user'], (country and (" and country='%s'" % country) or '')))
if sd:
return json.loads(sd[0][0])
else:
return None
except Exception, e:
if e.args[0]==1146:
make_cache_table()
else:
raise e
def add_to_cache(bootinfo, country):
"""add to cache"""
import json
import webnotes.model.utils
if bootinfo.get('docs'):
bootinfo['docs'] = webnotes.model.utils.compress(bootinfo['docs'])
# delete earlier (?)
webnotes.conn.sql("""delete from __SessionCache where user=%s
and country=%s""", (webnotes.session['user'], country))
# make new
webnotes.conn.sql("""insert into `__SessionCache`
(user, country, cache) VALUES (%s, %s, %s)""", \
(webnotes.session['user'], country, json.dumps(bootinfo)))
return bootinfo

76
webnotes/widgets/auto_master.py
View file

@ -1,76 +0,0 @@
# Copyright (c) 2012 Web Notes Technologies Pvt Ltd (http://erpnext.com)
#
# MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
# INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
# PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
# OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# auto masters
# _ + fieldname is the table
# 'value' is the column name, pkey
from __future__ import unicode_literals
import webnotes
# create masters for a doctype
def create_auto_masters(dt):
fl = webnotes.conn.sql("select fieldname from tabDocField where fieldtype='Data' and options='Suggest' and parent=%s", dt)
for f in fl:
make_auto_master(f[0])
# create master table
def make_auto_master(fieldname):
try:
webnotes.conn.sql("select `value` from `__%s` limit 1" % fieldname)
except Exception, e:
if e.args[0]==1146:
webnotes.conn.commit()
webnotes.conn.sql("create table `__%s` (`value` varchar(220), primary key (`value`))" % fieldname)
webnotes.conn.begin()
# get auto master fields
def get_master_fields(dt):
if not webnotes.session['data'].get('auto_masters'):
webnotes.session['data']['auto_masters'] = {}
if webnotes.session['data']['auto_masters'].get(dt, None)==None:
fl = webnotes.conn.sql("select fieldname from tabDocField where fieldtype='Data' \
and options='Suggest' and parent=%s", dt)
webnotes.session['data']['auto_masters'][dt] = fl
return webnotes.session['data']['auto_masters'][dt]
# update value
def update_auto_masters(doc):
if not doc.doctype:
return
fl = get_master_fields(doc.doctype)
# save masters in session cache
for f in fl:
if doc.fields.get(f[0]):
add_to_master(f[0], doc.fields.get(f[0]))
# add to master
def add_to_master(fieldname, value):
try:
webnotes.conn.sql("insert into `__%s` (`value`) values (%s)" % (fieldname,'%s'), value)
except Exception, e:
# primary key
pass

21
wnf.py
View file

@ -25,7 +25,7 @@
from __future__ import unicode_literals
import os, sys
def replace_code(start, txt1, txt2, extn, search=None):
def replace_code(start, txt1, txt2, extn, search=None, force=False):
"""replace all txt1 by txt2 in files with extension (extn)"""
import webnotes.utils
import os, re
@ -40,13 +40,13 @@ def replace_code(start, txt1, txt2, extn, search=None):
content = f.read()
if re.search(search, content):
res = search_replace_with_prompt(fpath, txt1, txt2)
res = search_replace_with_prompt(fpath, txt1, txt2, force)
if res == 'skip':
return 'skip'
def search_replace_with_prompt(fpath, txt1, txt2):
def search_replace_with_prompt(fpath, txt1, txt2, force=False):
""" Search and replace all txt1 by txt2 in the file with confirmation"""
from termcolor import colored
@ -59,12 +59,15 @@ def search_replace_with_prompt(fpath, txt1, txt2):
print fpath
print colored(txt1, 'red').join(c[:-1].split(txt1))
a = ''
while a.lower() not in ['y', 'n', 'skip']:
a = raw_input('Do you want to Change [y/n/skip]?')
if a.lower() == 'y':
if force:
c = c.replace(txt1, txt2)
elif a.lower() == 'skip':
return 'skip'
else:
while a.lower() not in ['y', 'n', 'skip']:
a = raw_input('Do you want to Change [y/n/skip]?')
if a.lower() == 'y':
c = c.replace(txt1, txt2)
elif a.lower() == 'skip':
return 'skip'
tmp.append(c)
with open(fpath, 'w') as f:
@ -262,7 +265,7 @@ def run():
# code replace
elif options.replace:
print options.replace
replace_code('.', options.replace[0], options.replace[1], options.replace[2])
replace_code('.', options.replace[0], options.replace[1], options.replace[2], force=options.force)
# git
elif options.status: