vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

started cleanup of modules

Browse source
This commit is contained in:
Rushabh Mehta 2011-07-20 17:58:37 +05:30
parent fe2042d078
commit 9f07d404c7
8 changed files with 172 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cgi-bin/webnotes/model/doc.py
View file

@ -389,7 +389,10 @@ class Document:
def _get_user_defaults(self):
if not self._user_defaults:
self._user_defaults = webnotes.user.get_defaults()
if webnotes.user:
self._user_defaults = webnotes.user.get_defaults()
else:
self.defaults = {}
def check_perm(self, verbose=0):
import webnotes

3
cgi-bin/webnotes/model/doctype.py
View file

@ -156,7 +156,8 @@ class _DocType:
update_fields = ('description', 'depends_on')
from webnotes.modules import get_file_timestamp, get_item_file
from webnotes.modules import get_item_file
from webnotes.utils import get_file_timestamp
doc = doclist[0] # main doc
file_name = get_item_file(doc.module, 'DocType', doc.name)

58
cgi-bin/webnotes/model/sql_sync.py Normal file
View file

@ -0,0 +1,58 @@
"""
Sync sql files (that contain triggers, stored procs etc) into the database
calling sync will walk through all .sql files in the modules file structure
and execute them if they are not executed or their timestamps have changed
All sql timestamps will be saved in a table '__sql_timestamps'
"""
# modules path
import webnotes
import webnotes.defs
def get_sql_files():
"""
Returns list of .sql files from
"""
import os
ret = []
for walk_tuple in os.walk(webnotes.defs.modules_path):
if os.path.split(walk_tuple[0])[-1]=='doctype':
for sql_file in filter(lambda x: x.endswith('.sql'), walk_tuple[2]):
ret.append[os.path.join(walk_tuple[0], sql_file)]
return ret
def run_sql_file(fn):
"""
Checks if timestamp matches, if not runs it
"""
from webnotes.modules import ModuleFile
mf = ModuleFile(fn)
if mf.is_new():
webnotes.conn.sql(mf.read())
mf.update()
def get_sql_timestamp(fn):
"""
Returns the last updated timestamp of the sql file
from the __sql_timestamps table. If the table does not
exist, it will create it
"""
try:
ts = webnotes.conn.sql("select tstamp from __sql_timestamp where file_name=%s", fn)
if ts:
return ts[0][0]
except Exception, e:
if e.args[0]==1147:
# create the table
webnotes.conn.commit()
webnotes.conn.sql("""
create table __sql_timestamp (
file_name varchar(320) primary key,
tstamp varchar(40))""")
webnotes.conn.begin()
else:
raise e
def update_sql_timestamp(fn, ts):
pass

116
cgi-bin/webnotes/modules/__init__.py
View file

@ -30,39 +30,15 @@ def get_item_timestamp(module, dt, dn):
"""
Return ths timestamp of the given item (if exists)
"""
from webnotes.utils import get_file_timestamp
return get_file_timestamp(get_item_file(module, dt, dn))
def get_file_timestamp(fn):
"""
Returns timestamp of the given file
"""
import os
from webnotes.utils import cint
try:
return str(cint(os.stat(fn).st_mtime))
except OSError, e:
if e.args[0]!=2:
raise e
else:
return None
def get_module_path(module):
"""
Returns path of the given module (imports it and reads it from __file__)
"""
import webnotes.defs, os, os.path
try:
exec ('import ' + scrub(module)) in locals()
modules_path = eval(scrub(module) + '.__file__')
modules_path = os.path.sep.join(modules_path.split(os.path.sep)[:-1])
except ImportError, e:
# get module path by importing the module
modules_path = os.path.join(webnotes.defs.modules_path, scrub(module))
return modules_path
return Module(module).get_path()
def switch_module(dt, dn, to, frm=None, export=None):
"""
@ -85,3 +61,91 @@ def switch_module(dt, dn, to, frm=None, export=None):
for ext in ('py','js','html','css'):
os.system('cp %s %s')
class Module:
"""
Represents a module in the framework
"""
def __init__(self, name):
self.name = name
def get_path(self):
"""
Returns path of the module (imports it and reads it from __file__)
"""
import webnotes.defs, os
try:
exec ('import ' + scrub(self.name)) in locals()
modules_path = eval(scrub(self.name) + '.__file__')
modules_path = os.path.sep.join(modules_path.split(os.path.sep)[:-1])
except ImportError, e:
modules_path = os.path.join(webnotes.defs.modules_path, scrub(self.name))
return modules_path
def get_file(self, *path):
"""
Returns ModuleFile object, in path specifiy the package name and file name
For example::
Module('accounts').get_file('doctype','account.txt')
"""
return ModuleFile(self, os.path.join(self.get_path(), os.path.join(*path)))
class ModuleFile:
"""
Module file class
"""
def __init__(self, path):
self.path = os.path.join(*path)
def is_new(self):
"""
Returns true if file does not match with last updated timestamp
"""
import webnotes.utils
self.timestamp = webnotes.utils.get_file_timestamp(self.path)
if self.timestamp != self.get_db_timestamp():
return True
def get_db_timestamp(self):
"""
Returns the timestamp of the file
"""
try:
ts = webnotes.conn.sql("select tstamp from __file_timestamp where file_name=%s", fn)
if ts:
return ts[0][0]
except Exception, e:
if e.args[0]==1147:
# create the table
webnotes.conn.commit()
webnotes.conn.sql("""
create table __file_timestamp (
file_name varchar(180) primary key,
tstamp varchar(40))""")
webnotes.conn.begin()
else:
raise e
def update(self):
"""
Update the timestamp into the database
(must be called after is_new)
"""
webnotes.conn.sql("""
insert into __file_timestamp(file_name, tstamp)
values (%s, %s) on duplicate key update""", (self.path, self.timestamp))
def read(self):
"""
Return the file content
"""
return open(self.path,'r').read()

3
cgi-bin/webnotes/modules/compress.py
View file

@ -13,7 +13,8 @@ def get_js_code(fn, extn='js'):
Get js code from a file (uncompressed)
"""
import webnotes
from webnotes.modules import scrub, get_file_timestamp
from webnotes.modules import scrub
from webnotes.utils import get_file_timestamp
src_file_name = fn + '.' + extn

206
cgi-bin/webnotes/modules/import_module.py
View file

@ -4,9 +4,6 @@ Imports Documents from modules (.txt) files in the filesystem
import webnotes
#
# imports / updates all files in a module into the database
#
def import_module(module, verbose=0):
"imports the all the records and files from the given module"
from webnotes.modules import get_module_path
@ -31,9 +28,6 @@ def import_module(module, verbose=0):
import_attachments(module)
#
# get doclist from file
#
def get_doclist(path, doctype, docname):
"returns a doclist (list of dictionaries) of multiple records for the given parameters"
import os
@ -50,9 +44,6 @@ def get_doclist(path, doctype, docname):
else:
return None
#
# import a file into the database
#
def import_file(module, doctype, docname, path=None):
"imports a given file into the database"
@ -66,9 +57,6 @@ def import_file(module, doctype, docname, path=None):
from webnotes.utils.transfer import set_doc
set_doc(doclist, 1, 1, 1)
#
# list folders in a dir
#
def listfolders(path, only_name=0):
"""returns the list of folders (with paths) in the given path,
if only_name is set, it returns only the folder names"""
@ -84,197 +72,3 @@ def listfolders(path, only_name=0):
return out
# ==============================================================================
# Import from files
# =============================================================================
def import_from_files(modules = [], record_list = [], sync_cp = 0, target_db=None, target_ac=None):
if target_db or target_ac:
init_db_login(target_ac, target_db)
from webnotes.utils import transfer
# Get paths of folder which will be imported
folder_list = get_folder_paths(modules, record_list)
ret = []
if folder_list:
# get all doclist
all_doclist = get_all_doclist(folder_list)
# import doclist
ret += accept_module(all_doclist)
# import attachments
for m in modules:
import_attachments(m)
# sync control panel
if sync_cp:
ret.append(sync_control_panel())
else:
ret.append("Module/Record not found")
return ret
# ==============================================================================
# Get list of folder path
# =============================================================================
# record_list in format [[module,dt,dn], ..]
def get_folder_paths(modules, record_list):
import os
import webnotes
import fnmatch
import webnotes.defs
from webnotes.modules import transfer_types, get_module_path, scrub
folder_list=[]
# get the folder list
if record_list:
for record in record_list:
if scrub(record[1]) in ('doctype', 'page', 'search_criteria'):
record[1], record[2] = scrub(record[1]), scrub(record[2])
folder_list.append(os.path.join(get_module_path(scrub(record[0])), \
record[1], record[2].replace('/','_')))
if modules:
# system modules will be transferred in a predefined order and before all other modules
sys_mod_ordered_list = ['roles', 'core','application_internal', 'mapper', 'settings']
all_mod_ordered_list = [t for t in sys_mod_ordered_list if t in modules] + list(set(modules).difference(sys_mod_ordered_list))
for module in all_mod_ordered_list:
mod_path = get_module_path(module)
types_list = listfolders(mod_path, 1)
# list of types
types_list = list(set(types_list).difference(['control_panel']))
all_transfer_types =[t for t in transfer_types if t in types_list] + list(set(types_list).difference(transfer_types))
# build the folders
for d in all_transfer_types:
if d not in ('files', 'startup', 'patches'):
# get all folders inside type
folder_list+=listfolders(os.path.join(mod_path, d))
return folder_list
# ==============================================================================
# Get doclist for all folder
# =============================================================================
def get_all_doclist(folder_list):
import fnmatch
import os
doclist = []
all_doclist = []
# build into doclist
for folder in folder_list:
# get the doclist
file_list = os.listdir(folder)
for each in file_list:
if fnmatch.fnmatch(each,'*.txt'):
doclist = eval(open(os.path.join(folder,each),'r').read())
# add code
all_doclist.append(doclist)
return all_doclist
# ==============================================================================
# accept a module coming from a remote server
# ==============================================================================
def accept_module(super_doclist):
import webnotes
import webnotes.utils
from webnotes.utils import transfer
msg, i = [], 0
for dl in super_doclist:
if dl[0]['doctype']!='Control Panel':
msg.append(transfer.set_doc(dl, 1, 1, 1))
if dl[0]['doctype']=='Module Def':
update_module_timestamp(dl[0]['name'])
if not webnotes.conn.in_transaction:
webnotes.conn.sql("START TRANSACTION")
# clear cache
webnotes.conn.sql("DELETE from __DocTypeCache")
webnotes.conn.sql("COMMIT")
return msg
# =============================================================================
# Update timestamp in Module Def table
# =============================================================================
def update_module_timestamp(mod):
import webnotes, webnotes.defs, os
try:
file = open(os.path.join(webnotes.defs.modules_path, mod, 'module.info'), 'r')
except Exception, e:
if e.args[0]==2:
return # module.info
else:
raise e
module_info = eval(file.read())
file.close()
# =============================================================================
def update_module_timestamp_query(mod, timestamp):
import webnotes
webnotes.conn.sql("start transaction")
webnotes.conn.sql("update `tabModule Def` set last_updated_date=%s where name=%s", (timestamp, mod))
webnotes.conn.sql("commit")
# =============================================================================
# Import Attachments
# =============================================================================
def import_attachments(m):
import os, webnotes.defs
import webnotes.utils.file_manager
from webnotes.modules import get_module_path
out = []
# get list
try:
folder = os.path.join(get_module_path(m), 'files')
fl = os.listdir(folder)
except OSError, e:
if e.args[0]==2:
return
else:
raise e
# import files
for f in fl:
if not os.path.isdir(os.path.join(folder, f)):
# delete
webnotes.utils.file_manager.delete_file(f)
# import
file = open(os.path.join(folder, f),'r')
webnotes.utils.file_manager.save_file(f, file.read(), m)
file.close()
out.append(f)
return out

15
cgi-bin/webnotes/utils/__init__.py
View file

@ -620,7 +620,20 @@ def get_diff_dict(d1, d2):
return ret
def get_file_timestamp(fn):
"""
Returns timestamp of the given file
"""
import os
from webnotes.utils import cint
try:
return str(cint(os.stat(fn).st_mtime))
except OSError, e:
if e.args[0]!=2:
raise e
else:
return None

4
cgi-bin/webnotes/utils/jstimestamp.py
View file

@ -33,11 +33,11 @@ class generateTimestamp:
def get_timestamp_dict(jsdir,filelist):
tsdict={}
import os
import webnotes.modules as webmod
from webnotes.utils import get_file_timestamp
oldcwd = os.getcwd()
os.chdir(jsdir)
for filename in generateTimestamp.list_js_files('.'):
ts = webmod.get_file_timestamp(filename)
ts = get_file_timestamp(filename)
filename = filename.lstrip('./')
filename = filename.rstrip('.js')
filename = filename.replace('/','.')