Merge branch 'master' of github.com:webnotes/wnframework into cleanup

This commit is contained in:
Rushabh Mehta 2013-11-08 16:27:47 +05:30
commit a6da76b2fb
16 changed files with 230 additions and 111 deletions

View file

@ -17,29 +17,11 @@ class DocType:
def on_trash(self):
webnotes.clear_cache(doctype=self.doc.dt)
def get_custom_server_script(doctype, plugin=None):
import os, MySQLdb
custom_script = webnotes.cache().get_value("_server_script:" + doctype)
if not custom_script:
try:
script_path = get_custom_server_script_path(doctype, plugin)
if os.path.exists(script_path):
with open(script_path, 'r') as f:
custom_script = f.read()
else:
custom_script = "Does Not Exist"
webnotes.cache().set_value("_server_script:" + doctype, custom_script)
except (webnotes.DoesNotExistError, MySQLdb.OperationalError):
# this happens when syncing
return None
return None if custom_script == "Does Not Exist" else custom_script
def make_custom_server_script_file(doctype, script=None):
import os
from webnotes.plugins import get_path
file_path = get_custom_server_script_path(doctype)
file_path = get_path(None, "DocType", doctype)
if os.path.exists(file_path):
raise IOError(file_path + " already exists")
@ -58,24 +40,4 @@ class CustomDocType(DocType):
{script}""".format(script=script or "\tpass")
with open(file_path, "w") as f:
f.write(custom_script)
def get_custom_server_script_path(doctype, plugin=None):
from webnotes.modules import scrub, get_plugin_path
from webnotes.utils import get_site_base_path
import os
# check if doctype exists
opts = webnotes.conn.get_value("DocType", doctype, ["name", "module", "plugin"])
if not opts:
raise webnotes.DoesNotExistError("""DocType "{doctype}" does not exist""".format(doctype=doctype))
name, module, doctype_plugin = opts
if not plugin:
plugin = doctype_plugin or os.path.basename(get_site_base_path())
# site_abs_path/plugin_name/module_name/doctype/doctype_name/doctype_name.py
path = os.path.join(get_plugin_path(scrub(plugin)), scrub(module),
"doctype", scrub(doctype), scrub(doctype) + ".py")
return path
f.write(custom_script.encode("utf-8"))

View file

@ -29,8 +29,12 @@ class DocType:
self.export_doc()
def export_doc(self):
# export
from webnotes.modules.export_file import export_to_files
if self.doc.is_standard == 'Yes' and (conf.get('developer_mode') or 0) == 1:
from webnotes.modules.export_file import export_to_files
export_to_files(record_list=[['Report', self.doc.name]],
record_module=webnotes.conn.get_value("DocType", self.doc.ref_doctype, "module"))
record_module=webnotes.conn.get_value("DocType", self.doc.ref_doctype, "module"))
elif self.doc.is_standard == 'No' and self.doc.report_type == "Script Report":
from webnotes.plugins import get_plugin_name
export_to_files(record_list=[['Report', self.doc.name]],
record_module=webnotes.conn.get_value("DocType", self.doc.ref_doctype, "module"),
plugin=get_plugin_name("Report", self.doc.name), create_init=False)

View file

@ -32,7 +32,7 @@ function cint(v, def) {
if(v===false)
return 0;
v=v+'';
v=lstrip(v, ['0']);
if(v!=="0")v=lstrip(v, ['0']);
v=parseInt(v);
if(isNaN(v))v=def===undefined?0:def;
return v;

View file

@ -14,8 +14,10 @@ wn.tools.downloadify = function(data, roles, me) {
var flash_disabled = (navigator.mimeTypes["application/x-shockwave-flash"] == undefined);
var download_from_server = function() {
open_url_post("/?cmd=webnotes.utils.datautils.send_csv_to_client",
{args: {data: data, filename: me.title}}, true);
open_url_post("/", {
args: { data: data, filename: me.title },
cmd: "webnotes.utils.datautils.send_csv_to_client"
}, true);
}
// save file > abt 200 kb using server call

View file

@ -14,4 +14,5 @@ requests
six
termcolor
werkzeug
gunicorn
gunicorn
slugify

View file

@ -171,7 +171,7 @@ class LoginManager:
ip_list = [i.strip() for i in ip_list]
for ip in ip_list:
if webnotes.get_request_header('REMOTE_ADDR', '').startswith(ip):
if webnotes.get_request_header('REMOTE_ADDR', '').startswith(ip) or webnotes.get_request_header('X-Forwarded-For', '').startswith(ip):
return
webnotes.msgprint('Not allowed from this IP Address')

View file

@ -18,7 +18,7 @@ class Database:
the `conn` global variable. the `sql` method is also global to run queries
"""
def __init__(self, host=None, user=None, password=None, ac_name=None, use_default = 0):
self.host = host or 'localhost'
self.host = host or conf.db_host or 'localhost'
self.user = user or conf.db_name
if ac_name:

View file

@ -19,7 +19,7 @@ def get_server_obj(doc, doclist = [], basedoctype = ''):
# for test
import webnotes
from webnotes.modules import scrub, get_doctype_module
from core.doctype.custom_script.custom_script import get_custom_server_script
from webnotes.plugins import get_code_and_execute
# get doctype details
module = get_doctype_module(doc.doctype) or "core"
@ -33,13 +33,10 @@ def get_server_obj(doc, doclist = [], basedoctype = ''):
return DocType(doc, doclist)
# custom?
custom_script = get_custom_server_script(doc.doctype)
if custom_script:
opts = {"DocType": DocType}
exec custom_script in opts
return opts["CustomDocType"](doc, doclist)
namespace = {"DocType": DocType}
get_code_and_execute(module, "DocType", doc.doctype, namespace=namespace)
if namespace.get("CustomDocType"):
return namespace["CustomDocType"](doc, doclist)
else:
return DocType(doc, doclist)

View file

@ -228,11 +228,11 @@ def cache_name(doctype, processed):
return "doctype:" + doctype + suffix
def clear_cache(doctype=None):
import webnotes.plugins
def clear_single(dt):
webnotes.cache().delete_value(cache_name(dt, False))
webnotes.cache().delete_value(cache_name(dt, True))
webnotes.cache().delete_value("_server_script:" + dt)
webnotes.plugins.clear_cache("DocType", dt)
if doctype_cache and doctype in doctype_cache:
del doctype_cache[dt]

View file

@ -32,10 +32,6 @@ def get_module_path(module):
return os.path.join(app_path, 'lib', m)
else:
return os.path.join(app_path, 'app', m)
def get_plugin_path(plugin):
from webnotes.utils import get_site_path
return get_site_path(webnotes.conf.get("plugins_path"), scrub(plugin))
def get_doc_path(module, doctype, name):
dt, dn = scrub_dt_dn(doctype, name)

View file

@ -5,12 +5,13 @@ from __future__ import unicode_literals
import webnotes, os
import webnotes.model.doc
from webnotes.modules import scrub, get_module_path, lower_case_files_for, scrub_dt_dn, get_plugin_path
from webnotes.modules import scrub, get_module_path, lower_case_files_for, scrub_dt_dn
from webnotes.plugins import get_plugin_path
def export_doc(doc):
export_to_files([[doc.doctype, doc.name]])
def export_to_files(record_list=None, record_module=None, verbose=0, plugin=None):
def export_to_files(record_list=None, record_module=None, verbose=0, plugin=None, create_init=None):
"""
Export record_list to files. record_list is a list of lists ([doctype],[docname] ) ,
"""
@ -21,21 +22,22 @@ def export_to_files(record_list=None, record_module=None, verbose=0, plugin=None
if record_list:
for record in record_list:
write_document_file(webnotes.model.doc.get(record[0], record[1]),
record_module, plugin=plugin)
record_module, plugin=plugin, create_init=create_init)
def write_document_file(doclist, record_module=None, plugin=None):
def write_document_file(doclist, record_module=None, plugin=None, create_init=None):
from webnotes.modules.utils import pprint_doclist
doclist = [filter_fields(d.fields) for d in doclist]
module = record_module or get_module_name(doclist)
code_type = doclist[0]['doctype'] in lower_case_files_for
if create_init is None:
create_init = doclist[0]['doctype'] in lower_case_files_for
# create folder
folder = create_folder(module, doclist[0]['doctype'], doclist[0]['name'], code_type, plugin=plugin)
folder = create_folder(module, doclist[0]['doctype'], doclist[0]['name'], create_init, plugin=plugin)
# write the data file
fname = (code_type and scrub(doclist[0]['name'])) or doclist[0]['name']
fname = (doclist[0]['doctype'] in lower_case_files_for and scrub(doclist[0]['name'])) or doclist[0]['name']
with open(os.path.join(folder, fname +'.txt'),'w+') as txtfile:
txtfile.write(pprint_doclist(doclist))
@ -71,7 +73,7 @@ def get_module_name(doclist):
return module
def create_folder(module, dt, dn, code_type, plugin=None):
def create_folder(module, dt, dn, create_init, plugin=None):
if plugin:
module_path = os.path.join(get_plugin_path(plugin), scrub(module))
else:
@ -85,7 +87,7 @@ def create_folder(module, dt, dn, code_type, plugin=None):
webnotes.create_folder(folder)
# create init_py_files
if code_type:
if create_init:
create_init_py(module_path, dt, dn)
return folder

110
webnotes/plugins.py Normal file
View file

@ -0,0 +1,110 @@
# Copyright (c) 2013, Web Notes Technologies Pvt. Ltd.
# MIT License. See license.txt
from __future__ import unicode_literals
import webnotes
def get_code_and_execute(module, doctype, docname, plugin=None, namespace=None):
code = get_code(module, doctype, docname, plugin)
return exec_code(code, namespace)
def exec_code(code, namespace=None):
if namespace is None: namespace = {}
if code:
exec code in namespace
return namespace
def get_code(module, doctype, docname, plugin=None):
import MySQLdb
try:
code = read_file(module, doctype, docname, plugin, cache=True)
except MySQLdb.OperationalError:
# this happens when syncing
return None
return code
def get_cache_key(doctype, docname, extn="py"):
return "Plugin File:{doctype}:{docname}:{extn}".format(doctype=doctype, docname=docname, extn=extn)
def get_plugin_name(doctype, docname):
import os
from webnotes.utils import get_site_base_path
plugin = None
meta = webnotes.get_doctype(doctype)
if meta.get_field("plugin"):
plugin = webnotes.conn.get_value(doctype, docname, "plugin")
if not plugin:
plugin = os.path.basename(get_site_base_path())
return plugin
def read_file(module, doctype, docname, plugin=None, extn="py", cache=False):
import os
content = None
if cache:
content = webnotes.cache().get_value(get_cache_key(doctype, docname, extn))
if not content:
path = get_path(module, doctype, docname, plugin, extn)
if os.path.exists(path):
with open(path, 'r') as f:
content = f.read() or "Does Not Exist"
if cache:
webnotes.cache().set_value(get_cache_key(doctype, docname, extn), content)
return None if (content == "Does Not Exist") else content
def get_path(module, doctype, docname, plugin=None, extn="py"):
from webnotes.modules import scrub
import os
if not module: module = webnotes.conn.get_value(doctype, docname, "module")
if not plugin: plugin = get_plugin_name(doctype, docname)
# site_abs_path/plugins/module/doctype/docname/docname.py
return os.path.join(get_plugin_path(scrub(plugin)), scrub(module),
scrub(doctype), scrub(docname), scrub(docname) + "." + extn)
def get_plugin_path(plugin=None):
from webnotes.modules import scrub
from webnotes.utils import get_site_path
return get_site_path(webnotes.conf.get("plugins_path"), scrub(plugin))
def remove_init_files():
import os
from webnotes.utils import get_site_path, cstr
for path, folders, files in os.walk(get_site_path(webnotes.conf.get("plugins_path"))):
for f in files:
# cstr(f) is required when filenames are non-ascii
if cstr(f) in ("__init__.py", "__init__.pyc"):
os.remove(os.path.join(path, f))
def clear_cache(doctype=None, docname=None):
import os
from webnotes.utils import get_site_path
def titlecase(txt):
return txt.replace("_", " ").title()
def clear_single(dt, dn):
dt = titlecase(dt)
dn = titlecase(dn)
webnotes.cache().delete_value(get_cache_key(dt, dn, "py"))
webnotes.cache().delete_value(get_cache_key(dt, dn, "js"))
if not (doctype and docname):
for path, folders, files in os.walk(get_site_path(webnotes.conf.get("plugins_path"))):
if files:
dt = os.path.basename(os.path.dirname(path))
dn = os.path.basename(path)
clear_single(dt, dn)
else:
clear_single(doctype, docname)

View file

@ -14,6 +14,7 @@ import json
from webnotes.utils import cint
import webnotes.model.doctype
import webnotes.defaults
import webnotes.plugins
@webnotes.whitelist()
def clear(user=None):
@ -26,6 +27,9 @@ def clear_cache(user=None):
# clear doctype cache
webnotes.model.doctype.clear_cache()
# clear plugins code cache
webnotes.plugins.clear_cache()
if user:
cache.delete_value("bootinfo:" + user)

View file

@ -23,10 +23,11 @@ class BackupGenerator:
"""
This class contains methods to perform On Demand Backup
To initialize, specify (db_name, user, password, db_file_name=None)
To initialize, specify (db_name, user, password, db_file_name=None, db_host="localhost")
If specifying db_file_name, also append ".sql.gz"
"""
def __init__(self, db_name, user, password, backup_path_db=None, backup_path_files=None):
def __init__(self, db_name, user, password, backup_path_db=None, backup_path_files=None, db_host="localhost"):
self.db_host = db_host
self.db_name = db_name
self.user = user
self.password = password
@ -43,10 +44,14 @@ class BackupGenerator:
last_db, last_file = self.get_recent_backup(older_than)
if not (self.backup_path_files and self.backup_path_db):
self.set_backup_file_name()
if not last_db and not last_file:
if not (last_db and last_file):
self.take_dump()
if not ignore_files:
self.zip_files()
else:
self.backup_path_files = last_file
self.backup_path_db = last_db
def set_backup_file_name(self):
import random
@ -87,7 +92,7 @@ class BackupGenerator:
# escape reserved characters
args = dict([item[0], webnotes.utils.esc(item[1], '$ ')]
for item in self.__dict__.copy().items())
cmd_string = """mysqldump -u %(user)s -p%(password)s %(db_name)s | gzip -c > %(backup_path_db)s""" % args
cmd_string = """mysqldump -u %(user)s -p%(password)s %(db_name)s -h %(db_host)s | gzip -c > %(backup_path_db)s""" % args
err, out = webnotes.utils.execute_in_shell(cmd_string)
def send_email(self):
@ -131,7 +136,7 @@ def get_backup():
#if verbose: print webnotes.conn.cur_db_name + " " + conf.db_password
delete_temp_backups()
odb = BackupGenerator(webnotes.conn.cur_db_name, webnotes.conn.cur_db_name,\
webnotes.get_db_password(webnotes.conn.cur_db_name))
webnotes.get_db_password(webnotes.conn.cur_db_name), db_host = webnotes.conn.host)
odb.get_backup()
recipient_list = odb.send_email()
webnotes.msgprint("""A download link to your backup will be emailed \
@ -143,16 +148,13 @@ def scheduled_backup(older_than=6, ignore_files=False, backup_path_db=None, back
deletes backups older than 7 days
takes backup"""
odb = new_backup(older_than, ignore_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files)
from webnotes.utils import now
print "backup taken -", odb.backup_path_db, "- on", now()
return odb
def new_backup(older_than=6, ignore_files=False, backup_path_db=None, backup_path_files=None):
delete_temp_backups(older_than=168)
odb = BackupGenerator(webnotes.conn.cur_db_name, webnotes.conn.cur_db_name,\
webnotes.get_db_password(webnotes.conn.cur_db_name),
backup_path_db=backup_path_db, backup_path_files=backup_path_files)
backup_path_db=backup_path_db, backup_path_files=backup_path_files, db_host = webnotes.conn.host)
odb.get_backup(older_than, ignore_files)
return odb
@ -198,25 +200,25 @@ def get_backup_path():
if __name__ == "__main__":
"""
is_file_old db_name user password
get_backup db_name user password
is_file_old db_name user password db_host
get_backup db_name user password db_host
"""
import sys
cmd = sys.argv[1]
if cmd == "is_file_old":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4])
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost")
is_file_old(odb.db_file_name)
if cmd == "get_backup":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4])
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost")
odb.get_backup()
if cmd == "take_dump":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4])
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost")
odb.take_dump()
if cmd == "send_email":
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4])
odb = BackupGenerator(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5] or "localhost")
odb.send_email("abc.sql.gz")
if cmd == "delete_temp_backups":

View file

@ -11,33 +11,50 @@ from webnotes import _
from webnotes.modules import scrub, get_module_path
from webnotes.utils import flt, cint
import webnotes.widgets.reportview
import webnotes.plugins
@webnotes.whitelist()
def get_script(report_name):
report = webnotes.doc("Report", report_name)
doctype = webnotes.conn.get_value("DocType", report.ref_doctype, "module")
script_path = os.path.join(get_module_path(doctype),
"report", scrub(report.name), scrub(report.name) + ".js")
module = webnotes.conn.get_value("DocType", report.ref_doctype, "module")
module_path = get_module_path(module)
report_folder = os.path.join(module_path, "report", scrub(report.name))
script_path = os.path.join(report_folder, scrub(report.name) + ".js")
script = None
if os.path.exists(script_path):
with open(script_path, "r") as script:
script = script.read()
if not script and report.is_standard == "No":
script = webnotes.plugins.read_file(module, "Report", report.name, extn="js", cache=True)
if not script and report.javascript:
script = report.javascript
if not script:
script = "wn.query_reports['%s']={}" % report_name
# load translations
if webnotes.lang != "en":
from webnotes.translate import get_lang_data
locale_path = os.path.join(get_module_path(webnotes.conn.get_value(doctype)),"report", scrub(report.name))
messages = get_lang_data(locale_path,
webnotes.lang, 'js')
webnotes.response["__messages"] = messages
if os.path.exists(report_folder):
messages = get_lang_data(report_folder, webnotes.lang, 'js')
webnotes.response["__messages"] = messages
else:
# TODO check if language files get exported here
plugins_report_folder = webnotes.plugins.get_path(module, "Report", report.name)
if os.path.exists(plugins_report_folder):
messages = get_lang_data(plugins_report_folder, webnotes.lang, 'js')
webnotes.response["__messages"] = messages
if os.path.exists(script_path):
with open(script_path, "r") as script:
return script.read()
elif report.javascript:
return report.javascript
else:
return "wn.query_reports['%s']={}" % report_name
return script
@webnotes.whitelist()
def run(report_name, filters=None):
from webnotes.plugins import get_code_and_execute
report = webnotes.doc("Report", report_name)
if filters and isinstance(filters, basestring):
@ -58,9 +75,13 @@ def run(report_name, filters=None):
result = [list(t) for t in webnotes.conn.sql(report.query, filters)]
columns = [c[0] for c in webnotes.conn.get_description()]
else:
method_name = scrub(webnotes.conn.get_value("DocType", report.ref_doctype, "module")) \
+ ".report." + scrub(report.name) + "." + scrub(report.name) + ".execute"
columns, result = webnotes.get_method(method_name)(filters or {})
module = webnotes.conn.get_value("DocType", report.ref_doctype, "module")
if report.is_standard=="Yes":
method_name = scrub(module) + ".report." + scrub(report.name) + "." + scrub(report.name) + ".execute"
columns, result = webnotes.get_method(method_name)(filters or {})
else:
namespace = get_code_and_execute(module, "Report", report.name)
columns, result = namespace["execute"](filters or {})
result = get_filtered_data(report.ref_doctype, columns, result)
@ -91,7 +112,7 @@ def get_filtered_data(ref_doctype, columns, data):
linked_doctypes = get_linked_doctypes(columns)
match_filters = get_user_match_filters(linked_doctypes, ref_doctype)
if match_filters:
matched_columns = get_matched_columns(linked_doctypes, match_filters)
for row in data:
@ -123,7 +144,6 @@ def get_user_match_filters(doctypes, ref_doctype):
match_filters = {}
doctypes_meta = {}
tables = []
doctypes[ref_doctype] = None
for dt in doctypes:
tables.append("`tab" + dt + "`")
@ -139,6 +159,9 @@ def get_user_match_filters(doctypes, ref_doctype):
return match_filters
def get_matched_columns(linked_doctypes, match_filters):
if "owner" in match_filters:
match_filters["profile"] = match_filters["owner"]
col_idx_map = {}
for dt, idx in linked_doctypes.items():
link_field = dt.lower().replace(" ", "_")

24
wnf.py
View file

@ -96,6 +96,7 @@ def setup_utilities(parser):
# update
parser.add_argument("-u", "--update", nargs="*", metavar=("REMOTE", "BRANCH"),
help="Perform git pull, run patches, sync schema and rebuild files/translations")
parser.add_argument("--reload_gunicorn", default=False, action="store_true", help="reload gunicorn on update")
parser.add_argument("--patch", nargs=1, metavar="PATCH-MODULE",
help="Run a particular patch [-f]")
parser.add_argument("-l", "--latest", default=False, action="store_true",
@ -253,18 +254,24 @@ def make_demo_fresh(site=None):
# utilities
@cmd
def update(remote=None, branch=None, site=None):
def update(remote=None, branch=None, site=None, reload_gunicorn=False):
pull(remote=remote, branch=branch, site=site)
# maybe there are new framework changes, any consequences?
reload(webnotes)
if not site: build()
latest(site=site)
if reload_gunicorn:
import subprocess
subprocess.check_output("killall -HUP gunicorn".split())
@cmd
def latest(site=None, verbose=True):
import webnotes.modules.patch_handler
import webnotes.model.sync
import webnotes.plugins
webnotes.connect(site=site)
@ -277,6 +284,10 @@ def latest(site=None, verbose=True):
# sync
webnotes.model.sync.sync_all()
# remove __init__.py from plugins
webnotes.plugins.remove_init_files()
except webnotes.modules.patch_handler.PatchError, e:
print "\n".join(webnotes.local.patch_log_list)
raise e
@ -302,6 +313,10 @@ def patch(patch_module, site=None, force=False):
@cmd
def update_all_sites(remote=None, branch=None, verbose=True):
pull(remote, branch)
# maybe there are new framework changes, any consequences?
reload(webnotes)
build()
for site in get_sites():
latest(site=site, verbose=verbose)
@ -326,11 +341,12 @@ def watch():
def backup(site=None, with_files=False, verbose=True, backup_path_db=None, backup_path_files=None):
from webnotes.utils.backups import scheduled_backup
webnotes.connect(site=site)
print backup_path_db
odb = scheduled_backup(ignore_files=not with_files, backup_path_db=backup_path_db, backup_path_files=backup_path_files)
if verbose:
from webnotes.utils import now
print "backup taken -", odb.backup_path_db, "- on", now()
print "database backup taken -", odb.backup_path_db, "- on", now()
if with_files:
print "files backup taken -", odb.backup_path_files, "- on", now()
return odb
@cmd
@ -552,7 +568,7 @@ def mysql(site=None):
import commands, os
msq = commands.getoutput('which mysql')
webnotes.init(site=site)
os.execv(msq, [msq, '-u', webnotes.conf.db_name, '-p'+webnotes.conf.db_password, webnotes.conf.db_name])
os.execv(msq, [msq, '-u', webnotes.conf.db_name, '-p'+webnotes.conf.db_password, webnotes.conf.db_name, webnotes.conf.db_host or "localhost"])
webnotes.destroy()
@cmd