From 9f07d404c758983fbbbcbca0aab034906938e7ff Mon Sep 17 00:00:00 2001 From: Rushabh Mehta Date: Wed, 20 Jul 2011 17:58:37 +0530 Subject: [PATCH] started cleanup of modules --- cgi-bin/webnotes/model/doc.py | 5 +- cgi-bin/webnotes/model/doctype.py | 3 +- cgi-bin/webnotes/model/sql_sync.py | 58 ++++++ cgi-bin/webnotes/modules/__init__.py | 116 +++++++++--- cgi-bin/webnotes/modules/compress.py | 3 +- cgi-bin/webnotes/modules/import_module.py | 206 ---------------------- cgi-bin/webnotes/utils/__init__.py | 15 +- cgi-bin/webnotes/utils/jstimestamp.py | 4 +- 8 files changed, 172 insertions(+), 238 deletions(-) create mode 100644 cgi-bin/webnotes/model/sql_sync.py diff --git a/cgi-bin/webnotes/model/doc.py b/cgi-bin/webnotes/model/doc.py index b9c9f8a9d1..117337f421 100755 --- a/cgi-bin/webnotes/model/doc.py +++ b/cgi-bin/webnotes/model/doc.py @@ -389,7 +389,10 @@ class Document: def _get_user_defaults(self): if not self._user_defaults: - self._user_defaults = webnotes.user.get_defaults() + if webnotes.user: + self._user_defaults = webnotes.user.get_defaults() + else: + self.defaults = {} def check_perm(self, verbose=0): import webnotes diff --git a/cgi-bin/webnotes/model/doctype.py b/cgi-bin/webnotes/model/doctype.py index b6badd098a..599226972d 100644 --- a/cgi-bin/webnotes/model/doctype.py +++ b/cgi-bin/webnotes/model/doctype.py @@ -156,7 +156,8 @@ class _DocType: update_fields = ('description', 'depends_on') - from webnotes.modules import get_file_timestamp, get_item_file + from webnotes.modules import get_item_file + from webnotes.utils import get_file_timestamp doc = doclist[0] # main doc file_name = get_item_file(doc.module, 'DocType', doc.name) diff --git a/cgi-bin/webnotes/model/sql_sync.py b/cgi-bin/webnotes/model/sql_sync.py new file mode 100644 index 0000000000..6167d13857 --- /dev/null +++ b/cgi-bin/webnotes/model/sql_sync.py @@ -0,0 +1,58 @@ +""" +Sync sql files (that contain triggers, stored procs etc) into the database +calling sync will walk through all .sql files in the modules file structure +and execute them if they are not executed or their timestamps have changed + +All sql timestamps will be saved in a table '__sql_timestamps' +""" + +# modules path +import webnotes +import webnotes.defs + +def get_sql_files(): + """ + Returns list of .sql files from + """ + import os + ret = [] + for walk_tuple in os.walk(webnotes.defs.modules_path): + if os.path.split(walk_tuple[0])[-1]=='doctype': + for sql_file in filter(lambda x: x.endswith('.sql'), walk_tuple[2]): + ret.append[os.path.join(walk_tuple[0], sql_file)] + return ret + +def run_sql_file(fn): + """ + Checks if timestamp matches, if not runs it + """ + from webnotes.modules import ModuleFile + mf = ModuleFile(fn) + if mf.is_new(): + webnotes.conn.sql(mf.read()) + mf.update() + +def get_sql_timestamp(fn): + """ + Returns the last updated timestamp of the sql file + from the __sql_timestamps table. If the table does not + exist, it will create it + """ + try: + ts = webnotes.conn.sql("select tstamp from __sql_timestamp where file_name=%s", fn) + if ts: + return ts[0][0] + except Exception, e: + if e.args[0]==1147: + # create the table + webnotes.conn.commit() + webnotes.conn.sql(""" + create table __sql_timestamp ( + file_name varchar(320) primary key, + tstamp varchar(40))""") + webnotes.conn.begin() + else: + raise e + +def update_sql_timestamp(fn, ts): + pass \ No newline at end of file diff --git a/cgi-bin/webnotes/modules/__init__.py b/cgi-bin/webnotes/modules/__init__.py index cbe182e97e..bc38a2f96a 100644 --- a/cgi-bin/webnotes/modules/__init__.py +++ b/cgi-bin/webnotes/modules/__init__.py @@ -30,39 +30,15 @@ def get_item_timestamp(module, dt, dn): """ Return ths timestamp of the given item (if exists) """ + from webnotes.utils import get_file_timestamp return get_file_timestamp(get_item_file(module, dt, dn)) -def get_file_timestamp(fn): - """ - Returns timestamp of the given file - """ - import os - from webnotes.utils import cint - - try: - return str(cint(os.stat(fn).st_mtime)) - except OSError, e: - if e.args[0]!=2: - raise e - else: - return None def get_module_path(module): """ Returns path of the given module (imports it and reads it from __file__) """ - import webnotes.defs, os, os.path - - try: - exec ('import ' + scrub(module)) in locals() - modules_path = eval(scrub(module) + '.__file__') - - modules_path = os.path.sep.join(modules_path.split(os.path.sep)[:-1]) - except ImportError, e: - # get module path by importing the module - modules_path = os.path.join(webnotes.defs.modules_path, scrub(module)) - - return modules_path + return Module(module).get_path() def switch_module(dt, dn, to, frm=None, export=None): """ @@ -85,3 +61,91 @@ def switch_module(dt, dn, to, frm=None, export=None): for ext in ('py','js','html','css'): os.system('cp %s %s') + + + + + +class Module: + """ + Represents a module in the framework + """ + def __init__(self, name): + self.name = name + + def get_path(self): + """ + Returns path of the module (imports it and reads it from __file__) + """ + import webnotes.defs, os + + try: + exec ('import ' + scrub(self.name)) in locals() + modules_path = eval(scrub(self.name) + '.__file__') + + modules_path = os.path.sep.join(modules_path.split(os.path.sep)[:-1]) + except ImportError, e: + modules_path = os.path.join(webnotes.defs.modules_path, scrub(self.name)) + + return modules_path + + def get_file(self, *path): + """ + Returns ModuleFile object, in path specifiy the package name and file name + For example:: + Module('accounts').get_file('doctype','account.txt') + """ + return ModuleFile(self, os.path.join(self.get_path(), os.path.join(*path))) + +class ModuleFile: + """ + Module file class + """ + + def __init__(self, path): + self.path = os.path.join(*path) + + def is_new(self): + """ + Returns true if file does not match with last updated timestamp + """ + import webnotes.utils + self.timestamp = webnotes.utils.get_file_timestamp(self.path) + + if self.timestamp != self.get_db_timestamp(): + return True + + def get_db_timestamp(self): + """ + Returns the timestamp of the file + """ + try: + ts = webnotes.conn.sql("select tstamp from __file_timestamp where file_name=%s", fn) + if ts: + return ts[0][0] + except Exception, e: + if e.args[0]==1147: + # create the table + webnotes.conn.commit() + webnotes.conn.sql(""" + create table __file_timestamp ( + file_name varchar(180) primary key, + tstamp varchar(40))""") + webnotes.conn.begin() + else: + raise e + + def update(self): + """ + Update the timestamp into the database + (must be called after is_new) + """ + webnotes.conn.sql(""" + insert into __file_timestamp(file_name, tstamp) + values (%s, %s) on duplicate key update""", (self.path, self.timestamp)) + + def read(self): + """ + Return the file content + """ + return open(self.path,'r').read() \ No newline at end of file diff --git a/cgi-bin/webnotes/modules/compress.py b/cgi-bin/webnotes/modules/compress.py index eb9871abee..da2b8bba71 100644 --- a/cgi-bin/webnotes/modules/compress.py +++ b/cgi-bin/webnotes/modules/compress.py @@ -13,7 +13,8 @@ def get_js_code(fn, extn='js'): Get js code from a file (uncompressed) """ import webnotes - from webnotes.modules import scrub, get_file_timestamp + from webnotes.modules import scrub + from webnotes.utils import get_file_timestamp src_file_name = fn + '.' + extn diff --git a/cgi-bin/webnotes/modules/import_module.py b/cgi-bin/webnotes/modules/import_module.py index 141299c4af..0708f95009 100644 --- a/cgi-bin/webnotes/modules/import_module.py +++ b/cgi-bin/webnotes/modules/import_module.py @@ -4,9 +4,6 @@ Imports Documents from modules (.txt) files in the filesystem import webnotes -# -# imports / updates all files in a module into the database -# def import_module(module, verbose=0): "imports the all the records and files from the given module" from webnotes.modules import get_module_path @@ -31,9 +28,6 @@ def import_module(module, verbose=0): import_attachments(module) -# -# get doclist from file -# def get_doclist(path, doctype, docname): "returns a doclist (list of dictionaries) of multiple records for the given parameters" import os @@ -50,9 +44,6 @@ def get_doclist(path, doctype, docname): else: return None -# -# import a file into the database -# def import_file(module, doctype, docname, path=None): "imports a given file into the database" @@ -66,9 +57,6 @@ def import_file(module, doctype, docname, path=None): from webnotes.utils.transfer import set_doc set_doc(doclist, 1, 1, 1) -# -# list folders in a dir -# def listfolders(path, only_name=0): """returns the list of folders (with paths) in the given path, if only_name is set, it returns only the folder names""" @@ -84,197 +72,3 @@ def listfolders(path, only_name=0): return out - - - - - -# ============================================================================== -# Import from files -# ============================================================================= -def import_from_files(modules = [], record_list = [], sync_cp = 0, target_db=None, target_ac=None): - - if target_db or target_ac: - init_db_login(target_ac, target_db) - - from webnotes.utils import transfer - # Get paths of folder which will be imported - folder_list = get_folder_paths(modules, record_list) - ret = [] - - if folder_list: - # get all doclist - all_doclist = get_all_doclist(folder_list) - - # import doclist - ret += accept_module(all_doclist) - - # import attachments - for m in modules: - import_attachments(m) - - # sync control panel - if sync_cp: - ret.append(sync_control_panel()) - else: - ret.append("Module/Record not found") - - return ret - - -# ============================================================================== -# Get list of folder path -# ============================================================================= -# record_list in format [[module,dt,dn], ..] -def get_folder_paths(modules, record_list): - import os - import webnotes - import fnmatch - import webnotes.defs - from webnotes.modules import transfer_types, get_module_path, scrub - - folder_list=[] - - # get the folder list - if record_list: - for record in record_list: - if scrub(record[1]) in ('doctype', 'page', 'search_criteria'): - record[1], record[2] = scrub(record[1]), scrub(record[2]) - - folder_list.append(os.path.join(get_module_path(scrub(record[0])), \ - record[1], record[2].replace('/','_'))) - - if modules: - # system modules will be transferred in a predefined order and before all other modules - sys_mod_ordered_list = ['roles', 'core','application_internal', 'mapper', 'settings'] - all_mod_ordered_list = [t for t in sys_mod_ordered_list if t in modules] + list(set(modules).difference(sys_mod_ordered_list)) - - for module in all_mod_ordered_list: - mod_path = get_module_path(module) - types_list = listfolders(mod_path, 1) - - # list of types - types_list = list(set(types_list).difference(['control_panel'])) - all_transfer_types =[t for t in transfer_types if t in types_list] + list(set(types_list).difference(transfer_types)) - - # build the folders - for d in all_transfer_types: - if d not in ('files', 'startup', 'patches'): - # get all folders inside type - folder_list+=listfolders(os.path.join(mod_path, d)) - - return folder_list - - -# ============================================================================== -# Get doclist for all folder -# ============================================================================= - - -def get_all_doclist(folder_list): - import fnmatch - import os - - doclist = [] - all_doclist = [] - - # build into doclist - for folder in folder_list: - # get the doclist - file_list = os.listdir(folder) - for each in file_list: - - if fnmatch.fnmatch(each,'*.txt'): - doclist = eval(open(os.path.join(folder,each),'r').read()) - # add code - all_doclist.append(doclist) - - return all_doclist - - -# ============================================================================== -# accept a module coming from a remote server -# ============================================================================== -def accept_module(super_doclist): - import webnotes - import webnotes.utils - from webnotes.utils import transfer - msg, i = [], 0 - - for dl in super_doclist: - if dl[0]['doctype']!='Control Panel': - msg.append(transfer.set_doc(dl, 1, 1, 1)) - - if dl[0]['doctype']=='Module Def': - update_module_timestamp(dl[0]['name']) - - if not webnotes.conn.in_transaction: - webnotes.conn.sql("START TRANSACTION") - - # clear cache - webnotes.conn.sql("DELETE from __DocTypeCache") - webnotes.conn.sql("COMMIT") - - return msg - -# ============================================================================= -# Update timestamp in Module Def table -# ============================================================================= -def update_module_timestamp(mod): - import webnotes, webnotes.defs, os - - try: - file = open(os.path.join(webnotes.defs.modules_path, mod, 'module.info'), 'r') - except Exception, e: - if e.args[0]==2: - return # module.info - else: - raise e - - module_info = eval(file.read()) - file.close() - -# ============================================================================= - -def update_module_timestamp_query(mod, timestamp): - import webnotes - webnotes.conn.sql("start transaction") - webnotes.conn.sql("update `tabModule Def` set last_updated_date=%s where name=%s", (timestamp, mod)) - webnotes.conn.sql("commit") - - -# ============================================================================= -# Import Attachments -# ============================================================================= - -def import_attachments(m): - import os, webnotes.defs - import webnotes.utils.file_manager - from webnotes.modules import get_module_path - - out = [] - - # get list - try: - folder = os.path.join(get_module_path(m), 'files') - fl = os.listdir(folder) - except OSError, e: - if e.args[0]==2: - return - else: - raise e - - # import files - for f in fl: - if not os.path.isdir(os.path.join(folder, f)): - # delete - webnotes.utils.file_manager.delete_file(f) - - # import - file = open(os.path.join(folder, f),'r') - webnotes.utils.file_manager.save_file(f, file.read(), m) - file.close() - - out.append(f) - - return out diff --git a/cgi-bin/webnotes/utils/__init__.py b/cgi-bin/webnotes/utils/__init__.py index 22c854faf1..e770963a7c 100644 --- a/cgi-bin/webnotes/utils/__init__.py +++ b/cgi-bin/webnotes/utils/__init__.py @@ -620,7 +620,20 @@ def get_diff_dict(d1, d2): return ret - +def get_file_timestamp(fn): + """ + Returns timestamp of the given file + """ + import os + from webnotes.utils import cint + + try: + return str(cint(os.stat(fn).st_mtime)) + except OSError, e: + if e.args[0]!=2: + raise e + else: + return None diff --git a/cgi-bin/webnotes/utils/jstimestamp.py b/cgi-bin/webnotes/utils/jstimestamp.py index db918a8a69..10725d54c4 100644 --- a/cgi-bin/webnotes/utils/jstimestamp.py +++ b/cgi-bin/webnotes/utils/jstimestamp.py @@ -33,11 +33,11 @@ class generateTimestamp: def get_timestamp_dict(jsdir,filelist): tsdict={} import os - import webnotes.modules as webmod + from webnotes.utils import get_file_timestamp oldcwd = os.getcwd() os.chdir(jsdir) for filename in generateTimestamp.list_js_files('.'): - ts = webmod.get_file_timestamp(filename) + ts = get_file_timestamp(filename) filename = filename.lstrip('./') filename = filename.rstrip('.js') filename = filename.replace('/','.')