"""
Contains the Document class representing an object / record
"""

import webnotes
import webnotes.model.meta

from webnotes.utils import *

# actually should be "BaseDocType" - deprecated. Only for v160
class SuperDocType:
	def __init__(self):
		pass
		
	def __getattr__(self, name):
		if self.__dict__.has_key(name):
			return self.__dict__[name]
		elif self.super and hasattr(self.super, name):
			return getattr(self.super, name)
		else:
			raise AttributeError, 'BaseDocType Attribute Error'

class Document:
	"""
	   The wn(meta-data)framework equivalent of a Database Record.
	   Stores,Retrieves,Updates the record in the corresponding table.
	   Runs the triggers required.

	   The `Document` class represents the basic Object-Relational Mapper (ORM). The object type is defined by
	   `DocType` and the object ID is represented by `name`:: 
	   
	      Please note the anamoly in the Web Notes Framework that `ID` is always called as `name`

	   If both `doctype` and `name` are specified in the constructor, then the object is loaded from the database.
	   If only `doctype` is given, then the object is not loaded
	   If `fielddata` is specfied, then the object is created from the given dictionary.
	       
	      **Note 1:**
	      
		 The getter and setter of the object are overloaded to map to the fields of the object that
		 are loaded when it is instantiated.
	       
		 For example: doc.name will be the `name` field and doc.owner will be the `owner` field

	      **Note 2 - Standard Fields:**
	      
		 * `name`: ID / primary key
		 * `owner`: creator of the record
		 * `creation`: datetime of creation
		 * `modified`: datetime of last modification
		 * `modified_by` : last updating user
		 * `docstatus` : Status 0 - Saved, 1 - Submitted, 2- Cancelled
		 * `parent` : if child (table) record, this represents the parent record
		 * `parenttype` : type of parent record (if any)
		 * `parentfield` : table fieldname of parent record (if any)
		 * `idx` : Index (sequence) of the child record	
	"""
	
	def __init__(self, doctype = '', name = '', fielddata = {}, prefix='tab'):
		self._roles = []
		self._perms = []
		self._user_defaults = {}
		self._prefix = prefix
		
		if fielddata: 
			self.fields = fielddata
		else: 
			self.fields = {}
		
		if not self.fields.has_key('name'):
			self.fields['name']='' # required on save
		if not self.fields.has_key('doctype'):
			self.fields['doctype']='' # required on save			
		if not self.fields.has_key('owner'):
			self.fields['owner']='' # required on save			

		if doctype:
			self.fields['doctype'] = doctype
		if name:
			self.fields['name'] = name
		self.__initialized = 1

		if (doctype and name):
			self._loadfromdb(doctype, name)

	def __nonzero__(self):
		return True

	def __str__(self):
		return str(self.fields)

	# Load Document
	# ---------------------------------------------------------------------------

	def _loadfromdb(self, doctype = None, name = None):
		if name: self.name = name
		if doctype: self.doctype = doctype
				
		if webnotes.model.meta.is_single(self.doctype):
			self._loadsingle()
		else:
			dataset = webnotes.conn.sql('select * from `%s%s` where name="%s"' % (self._prefix, self.doctype, self.name.replace('"', '\"')))
			if not dataset:
				webnotes.msgprint('%s %s does not exist' % (self.doctype, self.name), 1)
				raise Exception, '[WNF] %s %s does not exist' % (self.doctype, self.name)
			self._load_values(dataset[0], webnotes.conn.get_description())

	# Load Fields from dataset

	def _load_values(self, data, description):
		for i in range(len(description)):
			v = data[i]
			self.fields[description[i][0]] = webnotes.conn.convert_to_simple_type(v)

	def _merge_values(self, data, description):
			v = data[i]
			if v: # only if value, over-write
				self.fields[description[i][0]] = webnotes.conn.convert_to_simple_type(v)
			
	# Load Single Type

	def _loadsingle(self):
		self.name = self.doctype
		dataset = webnotes.conn.sql("select field, value from tabSingles where doctype='%s'" % self.doctype)
		for d in dataset: self.fields[d[0]] = d[1]

	# Setter
			
	def __setattr__(self, name, value):
		# normal attribute
		if not self.__dict__.has_key('_Document__initialized'): 
			self.__dict__[name] = value
		elif self.__dict__.has_key(name):
		else:
			# field attribute
			f = self.__dict__['fields']
			f[name] = value

	# Getter

		elif self.fields.has_key(name):
			return self.fields[name]	
		else:
			return ''

	# Get Amendement number
	
	def _get_amended_name(self):
		am_id = 1
		am_prefix = self.amended_from
		if webnotes.conn.sql('select amended_from from `tab%s` where name = "%s"' % (self.doctype, self.amended_from))[0][0] or '':
			am_id = cint(self.amended_from.split('-')[-1]) + 1
			am_prefix = '-'.join(self.amended_from.split('-')[:-1]) # except the last hyphen
			
		self.name = am_prefix + '-' + str(am_id)

	# Set Name

	def _set_name(self, autoname, istable):
		self.localname = self.name

		# get my object
		import webnotes.model.code
		so = webnotes.model.code.get_server_obj(self, [])

		# amendments
		if self.amended_from: 
			self._get_amended_name()

		# by method
		elif so and hasattr(so, 'autoname'):
			r = webnotes.model.code.run_server_obj(so, 'autoname')
			if r: return r
			
		# based on a field
		elif autoname and autoname.startswith('field:'):
			n = self.fields[autoname[6:]]
			if not n:
				raise Exception, 'Name is required'
			self.name = n.strip()
		
		# based on expression
		elif autoname and autoname.startswith('eval:'):
			doc = self # for setting
			self.name = eval(autoname[5:])
		
		# call the method!
		elif autoname and autoname!='Prompt': 
			self.name = make_autoname(autoname, self.doctype)
				
		# given
		elif self.fields.get('__newname',''): 
			self.name = self.fields['__newname']

		# default name for table
		elif istable: 
			self.name = make_autoname('#########', self.doctype)
					
	# Validate Name
	
	def _validate_name(self, case):

		if webnotes.conn.sql('select name from `tab%s` where name=%s' % (self.doctype,'%s'), self.name):
			raise NameError, 'Name %s already exists' % self.name
		
		# no name
		if not self.name: return 'No Name Specified for %s' % self.doctype
		
		# new..
		if self.name.startswith('New '+self.doctype):
			return 'There were some errors setting the name, please contact the administrator'
		
		if case=='Title Case': self.name = self.name.title()
		if case=='UPPER CASE': self.name = self.name.upper()
		
		self.name = self.name.strip() # no leading and trailing blanks

		forbidden = ['%', "'", '"', '#', '*', '?', '`']
		for f in forbidden:
			if f in self.name:
				webnotes.msgprint('%s not allowed in ID (name)' % f, raise_exception =1)

	# Insert
	
	def _makenew(self, autoname, istable, case='', make_autoname=1):
		# set owner
		if not self.owner: self.owner = webnotes.session['user']
		
		# set name
		if make_autoname:
			self._set_name(autoname, istable)
		
		# validate name
		self._validate_name(case)
				
		# insert!
		webnotes.conn.sql("""insert into `tab%s` (name, owner, creation, modified, modified_by) values ('%s', '%s', '%s', '%s', '%s')""" % (self.doctype, self.name, webnotes.session['user'], now(), now(), webnotes.session['user']))

	# Update Values
	
	def _update_single(self, link_list):
		update_str = ["(%s, 'modified', %s)",]
		values = [self.doctype, now()]
		
		webnotes.conn.sql("delete from tabSingles where doctype='%s'" % self.doctype)
		for f in self.fields.keys():
			if not (f in ('modified', 'doctype', 'name', 'perm', 'localname', 'creation'))\
				and (not f.startswith('__')): # fields not saved

				# validate links
				if link_list and link_list.get(f):
					self.fields[f] = self._validate_link(link_list[f][0], self.fields[f])

				if self.fields[f]==None:
					update_str.append("(%s,%s,NULL)")
					values.append(self.doctype)
					values.append(f)
				else:
					update_str.append("(%s,%s,%s)")
					values.append(self.fields[f])
		webnotes.conn.sql("insert into tabSingles(doctype, field, value) values %s" % (', '.join(update_str)), values)

	# Validate Links

	def validate_links(self, link_list):
		err_list = []
			# validate links
			old_val = self.fields[f]
			if link_list and link_list.get(f):
				self.fields[f] = self._validate_link(link_list[f][0], self.fields[f])

			if old_val and not self.fields[f]:
				s = link_list[f][1] + ': ' + old_val
				err_list.append(s)
				
		return err_list

	def make_link_list(self):
		res = webnotes.model.meta.get_link_fields(self.doctype)

		link_list = {}
		for i in res: link_list[i[0]] = (i[1], i[2]) # options, label
		return link_list
	
	def _validate_link(self, dt, dn):
		if not dt: return dn
		if not dn: return None
		if dt.lower().startswith('link:'):
			dt = dt[5:]
		if '\n' in dt:
			dt = dt.split('\n')[0]
		tmp = webnotes.conn.sql("""SELECT name FROM `tab%s` WHERE name = %s""" % (dt, '%s'), dn)
		return tmp and tmp[0][0] or ''# match case

	# Update query
		
	def _update_values(self, issingle, link_list, ignore_fields=0):
		if issingle:
			self._update_single(link_list)
		else:
			update_str, values = [], []
			# set modified timestamp
			self.modified = now()
			self.modified_by = webnotes.session['user']
			for f in self.fields.keys():
				if (not (f in ('doctype', 'name', 'perm', 'localname', 'creation','_user_tags'))) \
					and (not f.startswith('__')): # fields not saved
					
					# validate links
					if link_list and link_list.get(f):
						self.fields[f] = self._validate_link(link_list[f][0], self.fields[f])

					if self.fields[f]==None or self.fields[f]=='':
						update_str.append("`%s`=NULL" % f)
						if ignore_fields:
							try: r = webnotes.conn.sql("update `tab%s` set `%s`=NULL where name=%s" % (self.doctype, f, '%s'), self.name)
							except: pass
					else:
						values.append(self.fields[f])
						update_str.append("`%s`=%s" % (f, '%s'))
							try: r = webnotes.conn.sql("update `tab%s` set `%s`=%s where name=%s" % (self.doctype, f, '%s', '%s'), (self.fields[f], self.name))
			if values:
				if not ignore_fields:
					# update all in one query
					r = webnotes.conn.sql("update `tab%s` set %s where name='%s'" % (self.doctype, ', '.join(update_str), self.name), values)

	# Save values
	
	def save(self, new=0, check_links=1, ignore_fields=0, make_autoname = 1):
		"""
	      Saves the current record in the database. If new = 1, creates a new instance of the record.
	      Also clears temperory fields starting with `__`
	      
	      * if check_links is set, it validates all `Link` fields
	      * if ignore_fields is sets, it does not throw an exception for any field that does not exist in the 
		database table
		"""	
		res = webnotes.model.meta.get_dt_values(self.doctype, 'autoname, issingle, istable, name_case', as_dict=1)
		res = res and res[0] or {}

		# if required, make new
		if new or (not new and self.fields.get('__islocal')) and (not res.get('issingle')):
			r = self._makenew(res.get('autoname'), res.get('istable'), res.get('name_case'), make_autoname)
			if r: 
				return r
				
		# save the values
		self._update_values(res.get('issingle'), check_links and self.make_link_list() or {}, ignore_fields)
		self._clear_temp_fields()

	# check permissions

	def _get_perms(self):
		if not self._perms:
			self._perms = webnotes.conn.sql("select role, `match` from tabDocPerm where parent=%s and ifnull(`read`,0) = 1 and ifnull(permlevel,0)=0", self.doctype)

	def _get_roles(self):
		# check if roles match/
		if not self._roles:
			if webnotes.user:
				self._roles = webnotes.user.get_roles()
			else:
				self._roles = ['Guest']

	def _get_user_defaults(self):
		if not self._user_defaults:
			self._user_defaults = webnotes.user.get_defaults()

	def check_perm(self, verbose=0):
		import webnotes
		
		# find roles with read access for this record at 0
		self._get_perms()
		self._get_roles()
		self._get_user_defaults()
	
		has_perm, match = 0, []
		
		# loop through everything to find if there is a match
		for r in self._perms:
			if r[0] in self._roles:
				has_perm = 1
				if r[1] and match != -1:
					match.append(r[1]) # add to match check
				else:
					match = -1 # has permission and no match, so match not required!
		
		if has_perm and match and match != -1:
			for m in match:
				if self.fields.get(m, 'no value') in self._user_defaults.get(m, 'no default'):
					has_perm = 1
					break # permission found! break
				else:
					has_perm = 0
					if verbose:
						webnotes.msgprint("Value not allowed: '%s' for '%s'" % (self.fields.get(m, 'no value'), m))
					
		# check for access key
		if webnotes.form and webnotes.form.has_key('akey'):
			import webnotes.utils.encrypt
			if webnotes.utils.encrypt.decrypt(webnotes.form.getvalue('akey')) == self.name:
				webnotes.response['print_access'] = 1
				
		return has_perm

	# Cleanup
		
	def _clear_temp_fields(self):
		# clear temp stuff
		keys = self.fields.keys()
		for f in keys:
			if f.startswith('__'): 
				del self.fields[f]

	# Table methods

	def clear_table(self, doclist, tablefield, save=0):
		"""
		Clears the child records from the given `doclist` for a particular `tablefield`
		"""
		import webnotes.model.doclist
		
		for d in webnotes.model.doclist.getlist(doclist, tablefield):
			d.fields['__oldparent'] = d.parent
			d.parent = 'old_parent:' + d.parent # for client to send it back while saving
			d.docstatus = 2
			if save and not d.fields.get('__islocal'):
				d.save()
		self.fields['__unsaved'] = 1

	def addchild(self, fieldname, childtype = '', local=0, doclist=None):
		"""
	      Returns a child record of the give `childtype`.
	      
	      * if local is set, it does not save the record
	      * if doclist is passed, it append the record to the doclist
		"""
		if not childtype:
			childtype = db_getchildtype(self.doctype, fieldname)
	
		d = Document()
		d.parent = self.name
		d.parenttype = self.doctype
		d.parentfield = fieldname
		d.doctype = childtype
		d.docstatus = 0;
		d.name = ''
		d.owner = webnotes.session['user']
		
		if local:
			d.fields['__islocal'] = '1' # for Client to identify unsaved doc
		else: 
			d.save(new=1)
	
		if doclist != None:
			doclist.append(d)
	
		return d

def addchild(parent, fieldname, childtype = '', local=0, doclist=None):
	"""
	
   Create a child record to the parent doc.
   
   Example::
   
     c = Document('Contact','ABC')
     d = addchild(c, 'contact_updates', 'Contact Update', local = 1)
     d.last_updated = 'Phone call'
     d.save(1)
	"""
	return parent.addchild(fieldname, childtype, local, doclist)

# Remove Child
# ------------

def removechild(d, is_local = 0):
	"""
	   Sets the docstatus of the object d to 2 (deleted) and appends an 'old_parent:' to the parent name
	"""
	if not is_local:
		set(d, 'docstatus', 2)
		set(d, 'parent', 'old_parent:' + d.parent)
	else:
		d.parent = 'old_parent:' + d.parent
		d.docstatus = 2
			
# Naming
# ------
def make_autoname(key, doctype=''):
	"""
   Creates an autoname from the given key:
   
   **Autoname rules:**
      
         * The key is separated by '.'
         * '####' represents a series. The string before this part becomes the prefix:
            Example: ABC.#### creates a series ABC0001, ABC0002 etc
         * 'MM' represents the current month
         * 'YY' and 'YYYY' represent the current year

   *Example:*
   
         * DE/./.YY./.MM./.##### will create a series like
           DE/09/01/0001 where 09 is the year, 01 is the month and 0001 is the series
	"""
	n = ''
	l = key.split('.')
	for e in l:
		en = ''
		if e.startswith('#'):
			digits = len(e)
			en = getseries(n, digits, doctype)
		elif e=='YY': 
			import time
			en = time.strftime('%y')
		elif e=='MM': 
			en = time.strftime('%m')		
		elif e=='YYYY': 
			en = time.strftime('%Y')		
		else: en = e
		n+=en
	return n

# Get Series for Autoname
# -----------------------
def getseries(key, digits, doctype=''):
	# series created ?
	if webnotes.conn.sql("select name from tabSeries where name='%s'" % key):

		# yes, update it
		webnotes.conn.sql("update tabSeries set current = current+1 where name='%s'" % key)

		# find the series counter
		r = webnotes.conn.sql("select current from tabSeries where name='%s'" % key)
		n = r[0][0]
	else:
	
		# no, create it
		webnotes.conn.sql("insert into tabSeries (name, current) values ('%s', 1)" % key)
		n = 1
	return ('%0'+str(digits)+'d') % n

# Get Children

def getchildren(name, childtype, field='', parenttype='', from_doctype=0, prefix='tab'):
	import webnotes
	
	tmp = ''
	
	if field: 
		tmp = ' and parentfield="%s" ' % field
	if parenttype: 
		tmp = ' and parenttype="%s" ' % parenttype

	try:
		dataset = webnotes.conn.sql("select * from `%s%s` where parent='%s' %s order by idx" % (prefix, childtype, name, tmp))
		desc = webnotes.conn.get_description()
	except Exception, e:
		if prefix=='arc' and e.args[0]==1146:
			return []
		else:
			raise e

	l = []
	
	for i in dataset:
		d._load_values(i, desc)
		l.append(d)
	
	return l

# Check if "Guest" is allowed to view this page
# ---------------------------------------------

def check_page_perm(doc):
	if doc.name=='Login Page':
		return
	if doc.publish:
		return

	if not webnotes.conn.sql("select name from `tabPage Role` where parent=%s and role='Guest'", doc.name):
		webnotes.response['exc_type'] = 'PermissionError'
		raise Exception, '[WNF] No read permission for %s %s' % ('Page', doc.name)

def get_report_builder_code(doc):
	if doc.doctype=='Search Criteria':
		from webnotes.model.code import get_code
		
		if doc.standard != 'No':
			doc.report_script = get_code(doc.module, 'Search Criteria', doc.name, 'js')
			doc.custom_query = get_code(doc.module, 'Search Criteria', doc.name, 'sql')

# called from everywhere
# load a record and its child records and bundle it in a list - doclist
# ---------------------------------------------------------------------

def get(dt, dn='', with_children = 1, from_get_obj = 0, prefix = 'tab'):
	"""
	Returns a doclist containing the main record and all child records
	"""	
	import webnotes 
	import webnotes.model
	import webnotes.defs

	dn = dn or dt

	# load the main doc
	doc = Document(dt, dn, prefix=prefix)

	# check permission - for doctypes, pages
	if (dt in ('DocType', 'Page', 'Control Panel', 'Search Criteria')) or (from_get_obj and webnotes.session.get('user') != 'Guest'):
		if dt=='Page' and webnotes.session['user'] == 'Guest':
			check_page_perm(doc)
	else:
		if not doc.check_perm():
			webnotes.response['exc_type'] = 'PermissionError'
			raise webnotes.ValidationError, '[WNF] No read permission for %s %s' % (dt, dn)

	# import report_builder code
	get_report_builder_code(doc)

	if not with_children:
		# done
		return [doc,]
	
	# get all children types
	tablefields = webnotes.model.meta.get_table_fields(dt)

	# load chilren
	doclist = [doc,]
	for t in tablefields:
		doclist += getchildren(doc.name, t[0], t[1], dt, prefix=prefix)

	return doclist

# setup all tables for multi-tenant
# ---------------------------------
def setup_tables():
	import webnotes.multi_tenant
	
	tl = webnotes.conn.sql("show tables")
	for t in tl:
		add_tenant_id(t[0])
		change_primary_key(t[0])

def add_tenant_id(tname):
	webnotes.conn.sql("alter table `%s` add column _tenant_id int(10) default 0 not null")
		
def change_primary_key(tname):
	webnotes.conn.sql("alter table `%s` drop primary key name")
	webnotes.conn.sql("alter table `%s` add primary key (name, _tenant_id)")
		
# Tree (Hierarchical) Nested Set Model (nsm)
# 
# To use the nested set model,
# use the following pattern
# 1. name your parent field as "parent_node" if not have a property nsm_parent_field as your field name in the document class
# 2. have a field called "old_parent" in your fields list - this identifies whether the parent has been changed
# 3. call update_nsm(doc_obj) in the on_upate method

# ------------------------------------------

# called in the on_update method
def update_nsm(doc_obj):
	# get fields, data from the DocType
	d = doc_obj.doc
	pf, opf = 'parent_node', 'old_parent'
	if hasattr(doc_obj,'nsm_parent_field'):
		pf = doc_obj.nsm_parent_field
	if hasattr(doc_obj,'nsm_oldparent_field'):
		opf = doc_obj.nsm_oldparent_field
	p, op = d.fields[pf], d.fields.get(opf, '')

	# has parent changed (?) or parent is None (root)
	if not doc_obj.doc.lft and not doc_obj.doc.rgt:
		update_add_node(doc_obj.doc.doctype, doc_obj.doc.name, p or '', pf)
	elif op != p:
		update_remove_node(doc_obj.doc.doctype, doc_obj.doc.name)
	# set old parent
	webnotes.conn.set(d, opf, p or '')
	
def rebuild_tree(doctype, parent_field):
	# get all roots
	right = 1
	result = webnotes.conn.sql("SELECT name FROM `tab%s` WHERE `%s`='' or `%s` IS NULL" % (doctype, parent_field, parent_field))
	for r in result:
		right = rebuild_node(doctype, r[0], right, parent_field)
		
def rebuild_node(doctype, parent, left, parent_field):
	# the right value of this node is the left value + 1
	right = left+1

	# get all children of this node
	result = webnotes.conn.sql("SELECT name FROM `tab%s` WHERE `%s`='%s'" % (doctype, parent_field, parent))

	# we've got the left value, and now that we've processed
	# the children of this node we also know the right value
	webnotes.conn.sql('UPDATE `tab%s` SET lft=%s, rgt=%s WHERE name="%s"' % (doctype,left,right,parent))

	#return the right value of this node + 1
	return right+1
	
def update_add_node(doctype, name, parent, parent_field):
	# get the last sibling of the parent
	if parent:
		right = webnotes.conn.sql("select rgt from `tab%s` where name='%s'" % (doctype, parent))[0][0]
	else: # root
		right = webnotes.conn.sql("select ifnull(max(rgt),0)+1 from `tab%s` where ifnull(`%s`,'') =''" % (doctype, parent_field))[0][0]
	right = right or 1
	
	# update all on the right
	webnotes.conn.sql("update `tab%s` set rgt = rgt+2 where rgt >= %s" %(doctype,right))
	webnotes.conn.sql("update `tab%s` set lft = lft+2 where lft >= %s" %(doctype,right))
	
	#$ update index of new node
	webnotes.conn.sql("update `tab%s` set lft=%s, rgt=%s where name='%s'" % (doctype,right,right+1,name))
	return right

def update_remove_node(doctype, name):
	left = webnotes.conn.sql("select lft from `tab%s` where name='%s'" % (doctype,name))
	if left[0][0]:
		# reset this node
		webnotes.conn.sql("update `tab%s` set lft=0, rgt=0 where name='%s'" % (doctype,name))

		# update all on the right
		webnotes.conn.sql("update `tab%s` set rgt = rgt-2 where rgt > %s" %(doctype,left[0][0]))
		webnotes.conn.sql("update `tab%s` set lft = lft-2 where lft > %s" %(doctype,left[0][0]))

import webnotes.model.doc
import webnotes.model.code

conn = webnotes.conn

class Page:
	"""
	   A page class helps in loading a Page in the system. On loading
	   
	   * Page will import Client Script from other pages where specified by `$import(page_name)`
	   * Execute dynamic HTML if the `content` starts with `#python`
	"""	
	def __init__(self, name):
		self.name = name

	def load(self):	
		"""
		Returns :term:`doclist` of the `Page`
		"""
		from webnotes.modules import compress
		
		doclist = webnotes.model.doc.get('Page', self.name)
		doc = doclist[0]

		doc.fields['__script'] = compress.get_page_js(doc)
		doc.script = None

		template = '%(content)s'
		# load code from template
		if doc.template:
			template = get_code(webnotes.conn.get_value('Page Template', doc.template, 'module'), 'Page Template', doc.template, 'html', fieldname='template')

		doc.content = get_code(doc.module, 'page', doc.name, 'html') or doc.content
				
		# execute content
		if doc.content and doc.content.startswith('#!python'):
			doc.__content = template % {'content': webnotes.model.code.execute(doc.content).get('content')}
		else:
			doc.__content = template % {'content': doc.content or ''}

		# local stylesheet
		css = get_code(doc.module, 'page', doc.name, 'css')
		if css: doc.style = css
			
		# add stylesheet
		if doc.stylesheet:
			doclist += self.load_stylesheet(doc.stylesheet)
		
		return doclist

	def load_stylesheet(self, stylesheet):
		# load stylesheet
		loaded = eval(webnotes.form_dict.get('stylesheets') or '[]')
		if not stylesheet in loaded:
			import webnotes.model.doc
			from webnotes.model.code import get_code
			
			# doclist
			sslist = webnotes.model.doc.get('Stylesheet', stylesheet)
			
			# stylesheet from file
			css = get_code(sslist[0].module, 'Stylesheet', stylesheet, 'css')
			
			if css: sslist[0].stylesheet = css
			
			return sslist
		else:
			return []

def get(name):
	"""
	   Return the :term:`doclist` of the `Page` specified by `name`
	"""
	return Page(name).load()

def getpage():
	"""
	   Load the page from `webnotes.form` and send it via `webnotes.response`
	"""
	doclist = get(webnotes.form.getvalue('name'))
		
	# send
	webnotes.response['docs'] = doclist
	
# util __init__.py

user_time_zone = None
month_name = ['','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
month_name_full = ['','January','February','March','April','May','June','July','August','September','October','November','December']
no_value_fields = ['Section Break', 'Column Break', 'HTML', 'Table', 'FlexTable', 'Button', 'Image', 'Graph']
default_fields = ['doctype','name','owner','creation','modified','modified_by','parent','parentfield','parenttype','idx','docstatus']

def getCSVelement(v):
	"""
		 Returns the CSV value of `v`, For example: 
		 
		 * apple becomes "apple"
		 * hi"there becomes "hi""there"
	"""
	v = cstr(v)
	if not v: return ''
	if (',' in v) or ('\n' in v) or ('"' in v):
		if '"' in v: v = v.replace('"', '""')
		return '"'+v+'"'
	else: return v or ''

def validate_email_add(email_str):
	"""
	Validates the email string
	"""
	s = email_str
	if '<' in s:
		s = s.split('<')[1].split('>')[0]
	if s: s = s.strip().lower()
	import re
	#return re.match("^[a-zA-Z0-9._%-]+@[a-zA-Z0-9._%-]+.[a-zA-Z]{2,6}$", email_str)
	return re.match("[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", s)

def sendmail(recipients, sender='', msg='', subject='[No Subject]', parts=[], cc=[], attach=[]):
	"""
	Send an email. For more details see :func:`email_lib.sendmail`
	"""
	import webnotes.utils.email_lib
	return email_lib.sendmail(recipients, sender, msg, subject, parts, cc, attach)
	
def generate_hash():
	"""
		 Generates reandom hash for session id
	"""
	import sha, time
	return sha.new(str(time.time())).hexdigest()

def db_exists(dt, dn):
	return webnotes.conn.sql('select name from `tab%s` where name="%s"' % (dt, dn))

def load_json(arg):
	
	# already a dictionary?
	if type(arg)!=str:
		return arg
	
	try: import json
	except: import simplejson as json

	#return json.loads(unicode(arg, 'iso-8859-15'))
	return json.loads(arg)
	
# Get Traceback
# ==============================================================================

def getTraceback():
	"""
		 Returns the traceback of the Exception
	"""
	import sys, traceback, string
	type, value, tb = sys.exc_info()
	
	body = "Traceback (innermost last):\n"
	list = traceback.format_tb(tb, None) + traceback.format_exception_only(type, value)
	body = body + "%-20s %s" % (string.join(list[:-1], ""), list[-1])
	
	if webnotes.logger:
		webnotes.logger.error('Db:'+(webnotes.conn and webnotes.conn.cur_db_name or '') + ' - ' + body)
	
	return body

# Log

def log(event, details):
	webnotes.logger.info(details)

# Date and Time

def getdate(string_date):
	"""
		 Coverts string date (yyyy-mm-dd) to datetime.date object
	"""
	import datetime

	if type(string_date)==unicode:
		string_date = str(string_date)
	
	if type(string_date) in (datetime.datetime, datetime.date): 
		return string_date
	
	if ' ' in string_date:
		string_date = string_date.split(' ')[0]
	t = string_date.split('-')
	if len(t)==3:
		return datetime.date(cint(t[0]), cint(t[1]), cint(t[2]))
	else:
		return ''

def add_days(date, days):
	"""
		 Adds `days` to the given `string_date`
	"""
	if not date:
		date = now_datetime()

	if type(date) not in (datetime.datetime, datetime.date): 
		date = getdate(date)

	return (date + datetime.timedelta(days)).strftime('%Y-%m-%d')

def add_months(string_date, months):
	return webnotes.conn.sql("select DATE_ADD('%s',INTERVAL '%s' MONTH)" % (getdate(string_date),months))[0][0]

def add_years(string_date, years):
	return webnotes.conn.sql("select DATE_ADD('%s',INTERVAL '%s' YEAR)" % (getdate(string_date),years))[0][0]

def date_diff(string_ed_date, string_st_date=None):
	return webnotes.conn.sql("SELECT DATEDIFF('%s','%s')" %(getdate(string_ed_date), getdate(string_st_date)))[0][0]

def now_datetime():
	global user_time_zone
	from datetime import datetime
	from pytz import timezone
	
	# get localtime
	if not user_time_zone:
		user_time_zone = webnotes.conn.get_value('Control Panel', None, 'time_zone') or 'Asia/Calcutta'

	# convert to UTC
	utcnow = timezone('UTC').localize(datetime.utcnow())

	# convert to user time zone
	return utcnow.astimezone(timezone(user_time_zone))

def now():
	"""
		 Returns `time.strftime('%Y-%m-%d %H:%M:%S')`
	"""
	return now_datetime().strftime('%Y-%m-%d %H:%M:%S')
	
def nowdate():
	"""
		 Returns time.strftime('%Y-%m-%d')
	"""
	return now_datetime().strftime('%Y-%m-%d')

def get_first_day(dt, d_years=0, d_months=0):
	"""
	 Returns the first day of the month for the date specified by date object
	 Also adds `d_years` and `d_months` if specified
	"""
	# d_years, d_months are "deltas" to apply to dt
	y, m = dt.year + d_years, dt.month + d_months
	a, m = divmod(m-1, 12)
	return datetime.date(y+a, m+1, 1)

def get_last_day(dt):
	"""
	 Returns last day of the month using:
	 `get_first_day(dt, 0, 1) + datetime.timedelta(-1)`
	"""
	return get_first_day(dt, 0, 1) + datetime.timedelta(-1)

user_format = None
"""
	 User format specified in :term:`Control Panel`
	 
	 Examples:
	 
	 * dd-mm-yyyy
	 * mm-dd-yyyy
	 * dd/mm/yyyy
"""

def formatdate(string_date):
	"""
	 	Convers the given string date to :data:`user_format`
	"""
	global user_format
	if not user_format:
		user_format = webnotes.conn.get_value('Control Panel', None, 'date_format')
	d = string_date.split('-');
	out = user_format
	return out.replace('dd', ('%.2i' % cint(d[2]))).replace('mm', ('%.2i' % cint(d[1]))).replace('yyyy', d[0])
	
def dict_to_str(args, sep='&'):
	"""
	Converts a dictionary to URL
	"""
	import urllib
	t = []
	for k in args.keys():
		t.append(str(k)+'='+urllib.quote(str(args[k] or '')))
	return sep.join(t)

def timestamps_equal(t1, t2):
	"""Returns true if same the two string timestamps are same"""
	scrub = lambda x: x.replace(':', ' ').replace('-',' ').split()

	t1, t2 = scrub(t1), scrub(t2)
	
	if len(t1) != len(t2):
		return
	
	for i in range(len(t1)):
		if t1[i]!=t2[i]:
			return
	return 1

def global_date_format(date):

	if type(date) in (str, unicode):
	
	return date.strftime('%d') + ' ' + month_name_full[int(date.strftime('%m'))] + ' ' + date.strftime('%Y')
	
# Datatype

def isNull(v):
	"""
	Returns true if v='' or v is `None`
	"""
	return (v=='' or v==None)
	
def has_common(l1, l2):
	"""
	Returns true if there are common elements in lists l1 and l2
	"""
	for l in l1:
		if l in l2: 
			return 1
	return 0
	
def flt(s):
	"""
	Convert to float (ignore commas)
	"""
	if type(s)==str: # if string
		s = s.replace(',','')
	try: tmp = float(s)
	except: tmp = 0
	return tmp

def cint(s):
	"""
	Convert to integer
	"""
	try: tmp = int(float(s))
	return tmp

def cstr(s):
	"""	
	Convert to string
	"""
	if s==None: 
		return ''
	else:
		if hasattr(s, 'encode'):
			try:
				s = s.encode('utf-8', 'ignore')
			except:
				pass
		return str(s)

def str_esc_quote(s):
	"""
	Escape quotes
	"""
	if s==None:return ''
	return s.replace("'","\'")

def replace_newlines(s):
	"""
	Replace newlines by '<br>'
	"""
	return s.replace("\n","<br>")

def parse_val(v):
	"""
	Converts to simple datatypes from SQL query results
	"""
	
	try: import decimal # for decimal Python 2.5 (?)
	except: pass

	if type(v)==datetime.date:
		v = str(v)
	elif type(v)==datetime.timedelta:
		v = ':'.join(str(v).split(':')[:2])
	elif type(v)==datetime.datetime:
		v = str(v)
	elif type(v)==long: v=int(v)

	try:
		if type(v)==decimal.Decimal: v=float(v)

	return v
	
def fmt_money(amount, fmt = '%.2f'):
	"""
	Convert to string with commas for thousands, millions etc
	"""
	curr = webnotes.conn.get_value('Control Panel', None, 'currency_format') or 'Millions'

	val = 2
	if curr == 'Millions': val = 3

	if cstr(amount).find('.') == -1:	temp = '00'
	else: temp = cstr(amount).split('.')[1]

	l = []
	minus = ''
	if flt(amount) < 0: minus = '-'

	amount = ''.join(cstr(amount).split(','))
	amount = cstr(abs(flt(amount))).split('.')[0]
	
	# main logic	
	if len(cstr(amount)) > 3:
		nn = amount[len(amount)-3:]
		l.append(nn)
		amount = amount[0:len(amount)-3]
		while len(cstr(amount)) > val:
			nn = amount[len(amount)-val:]
			l.insert(0,nn)
			amount = amount[0:len(amount)-val]
	
	if len(amount) > 0:	l.insert(0,amount)

	amount = ','.join(l)+'.'+temp
	amount = minus + amount
	return amount

#
# convet currency to words
#
def money_in_words(number, main_currency = None, fraction_currency=None):
	"""
	Returns string in words with currency and fraction currency. 
	"""
	
	d = get_defaults()
	if not main_currency:
		main_currency = d.get('currency', 'INR')
	if not fraction_currency:
		fraction_currency = d.get('fraction_currency', 'paise')

	n = str(flt(number))
	main, fraction = n.split('.')
	if len(fraction)==1: fraction += '0'
	
	out = main_currency + ' ' + in_words(main).title()
	if cint(fraction):
		out = out + ' and ' + in_words(fraction).title() + ' ' + fraction_currency

	return out + ' only.'

#
# convert number to words
#
def in_words(integer):
	"""
	Returns string in words for the given integer.
	"""

	in_million = webnotes.conn.get_value('Control Panel',None,'currency_format')=='Millions' and 1 or 0
	
	n=int(integer)
	known = {0: 'zero', 1: 'one', 2: 'two', 3: 'three', 4: 'four', 5: 'five', 6: 'six', 7: 'seven', 8: 'eight', 9: 'nine', 10: 'ten',
		11: 'eleven', 12: 'twelve', 13: 'thirteen', 14: 'fourteen', 15: 'fifteen', 16: 'sixteen', 17: 'seventeen', 18: 'eighteen',
		19: 'nineteen', 20: 'twenty', 30: 'thirty', 40: 'forty', 50: 'fifty', 60: 'sixty', 70: 'seventy', 80: 'eighty', 90: 'ninety'}
	
	def psn(n, known, xpsn):
		import sys; 
		if n in known: return known[n]
		bestguess, remainder = str(n), 0

		if n<=20:
			print >>sys.stderr, n, "How did this happen?"
			assert 0
		elif n < 100:
			bestguess= xpsn((n//10)*10, known, xpsn) + '-' + xpsn(n%10, known, xpsn)
			return bestguess
		elif n < 1000:
			bestguess= xpsn(n//100, known, xpsn) + ' ' + 'hundred'
			remainder = n%100
		else:
			if in_million:
				if n < 1000000:
					bestguess= xpsn(n//1000, known, xpsn) + ' ' + 'thousand'
					remainder = n%1000
				elif n < 1000000000:
					bestguess= xpsn(n//1000000, known, xpsn) + ' ' + 'million'
					remainder = n%1000000
				else:
					bestguess= xpsn(n//1000000000, known, xpsn) + ' ' + 'billion'
					remainder = n%1000000000				
			else:
				if n < 100000:
				elif n < 10000000:
					bestguess= xpsn(n//100000, known, xpsn) + ' ' + 'lakh'
					remainder = n%100000
				else:
					bestguess= xpsn(n//10000000, known, xpsn) + ' ' + 'crore'
					remainder = n%10000000
		if remainder:
			if remainder >= 100:
				comma = ','
			else:
				comma = ''
			return bestguess + comma + ' ' + xpsn(remainder, known, xpsn)
		else:

	return psn(n, known, psn)
	
# Get Defaults

def get_defaults(key=None):
	"""
	Get dictionary of default values from the :term:`Control Panel`, or a value if key is passed
	"""
	if key:
		res = webnotes.conn.sql('select defvalue from `tabDefaultValue` where parent = "Control Panel" where defkey=%s', key)
		return res and res[0][0] or None
	else:
		res = webnotes.conn.sql('select defkey, defvalue from `tabDefaultValue` where parent = "Control Panel"')
		d = {}
		for rec in res: 
			d[rec[0]] = rec[1] or ''
		return d

def set_default(key, val):
	"""
	Set / add a default value to :term:`Control Panel`
	"""
	res = webnotes.conn.sql('select defkey from `tabDefaultValue` where defkey="%s" and parent = "Control Panel"' % key)
	if res:
		webnotes.conn.sql('update `tabDefaultValue` set defvalue="%s" where parent = "Control Panel" and defkey="%s"' % (val, key))
	else:
		from webnotes.model.doc import Document
		d = Document('DefaultValue')
		d.parent = 'Control Panel'
		d.parenttype = 'Control Panel'
		d.parentfield = 'system_defaults'
		d.defkey = key
		d.defvalue = val
		d.save(1)

#
# Clear recycle bin
#
def clear_recycle_bin():
	sql = webnotes.conn.sql
	
	tl = sql('show tables')
	total_deleted = 0
	for t in tl:
		fl = [i[0] for i in sql('desc `%s`' % t[0])]
		
		if 'name' in fl:
			total_deleted += sql("select count(*) from `%s` where name like '__overwritten:%%'" % t[0])[0][0]
			sql("delete from `%s` where name like '__overwritten:%%'" % t[0])

		if 'parent' in fl:	
			total_deleted += sql("select count(*) from `%s` where parent like '__oldparent:%%'" % t[0])[0][0]
			sql("delete from `%s` where parent like '__oldparent:%%'" % t[0])
	
			total_deleted += sql("select count(*) from `%s` where parent like 'oldparent:%%'" % t[0])[0][0]
			sql("delete from `%s` where parent like 'oldparent:%%'" % t[0])

			total_deleted += sql("select count(*) from `%s` where parent like 'old_parent:%%'" % t[0])[0][0]
			sql("delete from `%s` where parent like 'old_parent:%%'" % t[0])

	return "%s records deleted" % str(int(total_deleted))

# Send Error Report

def send_error_report():
	m = ''
	acc_id = webnotes.conn.get_value('Control Panel',None,'account_id') or ''
	if acc_id: m = 'Account Id : '+acc_id
	form = webnotes.form
	err_msg = '''
		%s <br>
		Comment: %s
		Err Msg : %s
	''' % (m, form.getvalue('msg') or '', form.getvalue('err_msg'))
	sendmail([webnotes.conn.get_value('Control Panel',None,'support_email_id') or 'support@iwebnotes.com'], sender=webnotes.session['user'], msg=err_msg, subject='Error Report '+m)

# pretty print a dict

def pprint_dict(d, level=1):
	indent = ''
	for i in range(0,level):
		indent += '\t'
	lines = []
	kl = d.keys()
	kl.sort()
	for key in kl:
		tmp = {key: d[key]}
		lines.append(indent + str(tmp)[1:-1] )
	return indent + '{\n' \
			+ indent + ',\n\t'.join(lines) \
			+ '\n' + indent + '}'

# Database Module
# --------------------

import MySQLdb
from webnotes import defs

class Database:
	"""
	   Open a database connection with the given parmeters, if use_default is True, use the
	   login details from `defs.py`. This is called by the request handler and is accessible using
	   the `conn` global variable. the `sql` method is also global to run queries
	"""
	def __init__(self, host='', user='', password='', ac_name = '', use_default = 0):
		self.host = host or 'localhost'
		self.user = user or getattr(defs, 'default_db_name', '')
		self.password = password or getattr(defs, 'db_password', '')

		if ac_name:
			self.user = self.get_db_login(ac_name) or defs.default_db_name
		
		if use_default:
			self.user = defs.default_db_name

		self.is_testing = 0
		self.in_transaction = 0
		self.transaction_writes = 0
		self.testing_tables = []
		
		self.connect()
		if self.user != 'root':
			self.use(self.user)
		
		if webnotes.logger:
			webnotes.logger.debug('Database object initialized for:%s',self.user)

	def get_db_login(self, ac_name):
		return getattr(defs,'db_name_map').get(ac_name, getattr(defs,'default_db_name'))

	def connect(self):
		"""
		      Connect to a database
		"""
		self._conn = MySQLdb.connect(user=self.user, host=self.host, passwd=self.password)
		try:
			self._conn.set_character_set('utf8')
		except:
			pass

		self._cursor = self._conn.cursor()
		
		return self._cursor
	
	def use(self, db_name):
		"""
		      `USE` db_name
		"""
		self._conn.select_db(db_name)
		self.cur_db_name = db_name
	
	def check_transaction_status(self, query):
		"""
		      Update *in_transaction* and check if "START TRANSACTION" is not called twice
		"""
		if self.in_transaction and query and query.strip().split()[0].lower() in ['start', 'alter', 'drop', 'create']:
			raise Exception, 'This statement can cause implicit commit'

		if query and query.strip().lower()=='start transaction':
			self.in_transaction = 1
			self.transaction_writes = 0
			
		if query and query.strip().split()[0].lower() in ['commit', 'rollback']:
			self.in_transaction = 0

		if self.in_transaction and query[:6].lower() in ['update', 'insert']:
			self.transaction_writes += 1
			if self.transaction_writes > 5000:
				webnotes.msgprint('A very long query was encountered. If you are trying to import data, please do so using smaller files')
				raise Exception, 'Bad Query!!! Too many writes'
	
	def fetch_as_dict(self, formatted=0):
		"""
		      Internal - get results as dictionary
		"""
		result = self._cursor.fetchall()
		ret = []
		for r in result:
			dict = {}
			for i in range(len(r)):
				dict[self._cursor.description[i][0]] = self.convert_to_simple_type(r[i], formatted)
			ret.append(dict)
		return ret
	
	def validate_query(self, q):
		cmd = q.strip().lower().split()[0]
		if cmd in ['alter', 'drop', 'truncate'] and webnotes.user.name != 'Administrator':
			webnotes.msgprint('Not allowed to execute query')
			raise Execption

	# ======================================================================================
	
	def sql(self, query, values=(), as_dict = 0, as_list = 0, formatted = 0, ignore_no_table = 1, debug=0):
		"""
		      * Execute a `query`, with given `values`
		      * returns as a dictionary if as_dict = 1
		      * returns as a list of lists (with cleaned up dates and decimals) if as_list = 1
		"""			
		# in transaction validations
		self.check_transaction_status(query)
		
		if getattr(defs,'multi_tenant',None):
			query = self.add_multi_tenant_condition(query)
			
		# execute
		if values!=():
			self._cursor.execute(query, values)
			if debug: webnotes.msgprint(query % values)
				
		else:
			self._cursor.execute(query)	
			if debug: webnotes.msgprint(query)

		# scrub output if required
		if as_dict:
			return self.fetch_as_dict(formatted)
		elif as_list:
			return self.convert_to_lists(self._cursor.fetchall(), formatted)
		else:
			return self._cursor.fetchall()

	# add condition for tenant id
	def add_multi_tenant_condition(query):
		import webnotes.multi_tenant
		return webnotes.multi_tenant.query_parser.add_condition(query)
		
	def get_description(self):
		"""
		      Get metadata of the last query
		"""
		return self._cursor.description

	def convert_to_simple_type(self, v, formatted=0):
		try: import decimal # for decimal Python 2.5 onwards
		except: pass
		import datetime
		from webnotes.utils import formatdate, fmt_money

		# date
		if type(v)==datetime.date:
			v = str(v)
			if formatted:
				v = formatdate(v)
		
		# time	
		elif type(v)==datetime.timedelta:
			h = int(v.seconds/60/60)
			v = str(h) + ':' + str(v.seconds/60 - h*60)
			if v[1]==':': 
				v='0'+v
		
		# datetime
		elif type(v)==datetime.datetime:
			v = str(v)
		
		# long
		elif type(v)==long: 
			v=int(v)

		# decimal
		try:
			if type(v)==decimal.Decimal: 
				v=float(v)
		
		# convert to strings... (if formatted)
		if formatted:
			if type(v)==float:
				v=fmt_money(v)
			if type(v)==int:
				v=str(v)
		
		return v

	def convert_to_lists(self, res, formatted=0):
		"""
		      Convert the given result set to a list of lists (with cleaned up dates and decimals)
		"""
		nres = []
		for r in res:
			nr = []
			for c in r:
				nr.append(self.convert_to_simple_type(c, formatted))
			nres.append(nr)
		return nres

	def replace_tab_by_test(self, query):
		"""
		      Relace all ``tab`` + doctype to ``test`` + doctype
		"""
		if self.is_testing:
			tl = self.get_testing_tables()
			for t in tl:
				query = query.replace(t, 'test' + t[3:])
		return query
		
	def get_testing_tables(self):
		"""
		      Get list of all tables for which `tab` is to be replaced by `test` before a query is executed
		"""
		if not self.testing_tables:
			testing_tables = ['tab'+r[0] for r in self.sql('SELECT name from tabDocType where docstatus<2 and (issingle=0 or issingle is null)', allow_testing = 0)]
			testing_tables+=['tabSeries','tabSingles'] # tabSessions is not included here
		return self.testing_tables

	# get a single value from a record

	def get_value(self, doctype, docname, fieldname):
		"""
		      Get a single / multiple value from a record.

		      For Single DocType, let docname be = None
		"""
			
		fl = fieldname
		if docname and (docname!=doctype or docname=='DocType'):
			if type(fieldname) in (list, tuple):
				fl = '`, `'.join(fieldname)

			r = self.sql("select `%s` from `tab%s` where name='%s'" % (fl, doctype, docname))
			return r and (len(r[0]) > 1 and r[0] or r[0][0]) or None
		else:
				fl = "', '".join(fieldname)

			r = self.sql("select value from tabSingles where field in ('%s') and doctype='%s'" % (fieldname, doctype))
			return r and (len(r) > 1 and (i[0] for i in r) or r[0][0]) or None

	def set_value(self, dt, dn, field, val):
		from webnotes.utils import now
		if dn and dt!=dn:
			self.sql("update `tab"+dt+"` set `"+field+"`=%s, modified=%s where name=%s", (val, now(), dn))
		else:
			if self.sql("select value from tabSingles where field=%s and doctype=%s", (field, dt)):
				self.sql("update tabSingles set value=%s where field=%s and doctype=%s", (val, field, dt))
			else:
				self.sql("insert into tabSingles(doctype, field, value) values (%s, %s, %s)", (dt, field, val))
				
	def set(self, doc, field, val):
		self.set_value(doc.doctype, doc.name, field, val)
		doc.fields[field] = val

	def set_global(self, key, val, user='__global'):
		res = self.sql('select defkey from `tabDefaultValue` where defkey=%s and parent=%s', (key, user))
		if res:
			self.sql('update `tabDefaultValue` set defvalue=%s where parent=%s and defkey=%s', (str(val), user, key))
		else:
			self.sql('insert into `tabDefaultValue` (name, defkey, defvalue, parent) values (%s,%s,%s,%s)', (user+'_'+key, key, str(val), user))

	def get_global(self, key, user='__global'):
		g = self.sql("select defvalue from tabDefaultValue where defkey=%s and parent=%s", (key, user))
		return g and g[0][0] or None

	def begin(self):
		if not self.in_transaction:
			self.sql("start transaction")
	
	def commit(self):
		self.sql("commit")

	def rollback(self):
		self.sql("ROLLBACK")

	def field_exists(self, dt, fn):
		"""
		      Returns True if `fn` exists in `DocType` `dt`
		"""	
		return self.sql("select name from tabDocField where fieldname=%s and parent=%s", (dt, fn))

	def exists(self, dt, dn):
		"""
		      Returns true if the record exists
		"""	
		try:
			return self.sql('select name from `tab%s` where name=%s' % (dt, '%s'), dn)
		except:
			return None

	def close(self):
		"""
		      Close my connection
		"""
		if self._conn:
			self._conn.close()

# to generate sitemaps

frame_xml = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">%s
</urlset>"""

link_xml = """\n<url><loc>%s</loc><lastmod>%s</lastmod></url>"""

# generate the sitemap XML
def generate_xml(conn, site_prefix):
	global frame_xml, link_xml

	# settings
	max_doctypes = 10
	max_items = 1000
	
	site_map = ''
	
	if site_prefix:
		# list of all Guest pages (static content)
		for r in conn.sql("SELECT name, modified FROM tabPage WHERE ifnull(publish,0)=1 ORDER BY modified DESC"):
			page_url = site_prefix + '#!' + urllib.quote(r[0])
			site_map += link_xml % (page_url, r[1].strftime('%Y-%m-%d'))
		
		# list of all Records that are viewable by guests (Blogs, Articles etc)
		try:
			from event_handlers import get_sitemap_items
			for i in get_sitemap_items(site_prefix):
				site_map += link_xml % (i[0], i[1])
		except ImportError, e:
			pass

	return frame_xml % site_map

"""
Add, manage, fire triggers (events / observers)
Standard events called by the framework are "save, submit, cancel"
"""

def insert_trigger(doctype, docname, event_name, method):
	"inserts a new trigger record"
	
	from webnotes.model.doc import Document
	d = Document('DocTrigger')
	d.doc_type = doctype
	d.doc_name = docname
	d.event_name = event_name
	d.method = method
	d.save(1)
	
def add_trigger(doctype, docname, event_name, method):
	"""Add a trigger to an event on a doctype, docname. The specified method will be called.
	Wild card '*' is allowed in doctype, docname and/or event_name"""

	try:
		if not trigger_exists(doctype, docname, event_name, method):
			insert_trigger(doctype, docname, event_name, method)
		if e.args[0]==1146: 
			setup()
		else: raise e

def trigger_exists(doctype, docname, event_name, method):
	"returns true if trigger exists"
	return webnotes.conn.sql("select name from tabDocTrigger where doc_name=%s and doc_type=%s and event_name=%s and method=%s", \
		(doctype, docname, event_name, method))

def remove_trigger(doctype, docname, event_name, method):
	"Remove a trigger on an event"
	webnotes.conn.sql("delete from tabDocTrigger where doc_name=%s and doc_type=%s and event_name=%s and method=%s", \
	
def fire_event(doc, event_name):
	"Fire all triggers on an event and passes the doc to the trigger"
	try:
		tl = webnotes.conn.sql("""select method from tabDocTrigger 
			where (doc_type=%s or doc_type='*')
			and (doc_name=%s or doc_name='*')
			and (event_name=%s or event_name='*')""", (doc.doctype, doc.name, event_name))
		if e.args[0]==1146:
			setup()
			tl = []
		
	for t in tl:
		module, method = '.'.join(t[0].split('.')[:-1]), t[0].split('.')[-1]
		exec 'from %s import %s' % (module, method) in locals()
		locals()[method](doc)
#
# setup
#
def setup():
	"creates the DocTrigger table from core"
	webnotes.conn.commit()
	from webnotes.modules.module_manager import reload_doc
	reload_doc('core','doctype','doctrigger')
	webnotes.conn.begin()
	
"""
	This module contains classes for managing incoming emails
"""

class IncomingMail:
	"""
		Single incoming email object. Extracts, text / html and attachments from the email
	"""
	def __init__(self, content):
		"""
			Parse the incoming mail content
		"""
		import email
		
		self.mail = email.message_from_string(content)
		self.text_content = ''
		self.html_content = ''
		self.attachments = []
		self.parse()

	def get_text_content(self):
		"""
			Returns the text parts of the email. If None, then HTML parts
		"""
		return self.text_content or self.html_content

	def get_charset(self, part):
		"""
			Guesses character set
		"""
		charset = part.get_content_charset()
		if not charset:
			import chardet
			charset = chardet.detect(str(part))['encoding']

		return charset
			
	def get_payload(self, part, charset):
		"""
			get utf-8 encoded part content
		"""
		return unicode(part.get_payload(decode=True),str(charset),"ignore").encode('utf8','replace')
		
	def get_attachment(self, part, charset):
		"""
			Extracts an attachment
		"""
		self.attachments.append({
			'content-type': part.get_content_type(),
			'filename': part.get_filename(),
			'content': self.get_payload(part, charset)
		})
				
	def parse(self):
		"""
			Extracts text, html and attachments from the mail
		"""
		for part in self.mail.walk():
			self.process_part(part)

	def get_thread_id(self):
		"""
			Extracts thread id of the message between first [] 
			from the subject
		"""
		subject = self.mail.get('Subject', '')
		if '[' in subject and ']' in subject:
			return subject.split('[')[1].split(']')[0]

	def process_part(self, part):
		"""
			Process a single part of an email
		"""
		charset = self.get_charset(part)
		content_type = part.get_content_type()

		if content_type == 'text/plain':
			self.text_content += self.get_payload(part, charset)

		if content_type == 'text/html':
			self.html_content += self.get_payload(part, charset)

		if part.get_filename():
			self.get_attachment(part, charset)

class POP3Mailbox:
	"""
		A simple pop3 mailbox, abstracts connection and mail extraction
		To use, subclass it and override method process_message(from, subject, text, thread_id)
	"""
	
	def __init__(self, settings_doc):
		"""
			settings_doc must contain
			is_ssl, host, username, password
		"""
		self.settings = Document(settings_doc, settings_doc)

		"""
			Connects to the mailbox
		"""
		import poplib
		
		if self.settings.use_ssl:
			self.pop = poplib.POP3_SSL(self.settings.host)
		else:
			self.pop = poplib.POP3(self.settings.host)
		self.pop.user(self.settings.username)
		self.pop.pass_(self.settings.password)
		
	def get_messages(self):
		"""
			Loads messages from the mailbox and calls
			process_message for each message
		"""
		
		if not self.check_mails():
			return # nothing to do
		
		num = len(self.pop.list()[1])
		for m in range(num):
			msg = self.pop.retr(m+1)
			self.process_message(IncomingMail('\n'.join(msg[1])))
			self.pop.dele(m+1)
		self.pop.quit()
		
	def check_mails(self):
		"""
			To be overridden
			If mailbox is to be scanned, returns true
		"""
		return true
	
	def process_message(self, mail):
		"""
			To be overriden
		"""
		pass

"""
Server side methods called from DocBrowser
"""

from webnotes.utils import cint, cstr

sql = webnotes.conn.sql

def get_menu_items():
	"""
	   Returns a list of items to show in `Options` of the Web Notes Toolbar
	   List contains Pages and Single DocTypes
	"""
	import webnotes.utils

	rl = webnotes.user.get_roles() + [webnotes.session['user']]
	role_options = ["role = '"+r+"'" for r in rl]
	
	menuitems = []
	
	# pages
	pages = sql("select distinct parent from `tabPage Role` where docstatus!=2 and (%s)" % (' OR '.join(role_options)))

	for p in pages:
		tmp = sql("select icon, parent_node, menu_index, show_in_menu from tabPage where name = '%s'" % p[0])
		if tmp and tmp[0][3]:
			menuitems.append(['Page', p[0] or '', tmp[0][1] or '', tmp[0][0] or '', webnotes.utils.cint(tmp[0][2])])
			
	# singles
	tmp = sql("select smallicon, parent_node, menu_index, name from tabDocType where (show_in_menu = 1 and show_in_menu is not null)")
	singles = {}
	for t in tmp: singles[t[3]] = t
		
	for p in webnotes.user.can_read:
		tmp = singles.get(p, None)
		if tmp: menuitems.append([p, p, tmp[1] or '', tmp[0] or '', int(tmp[2] or 0)])
		
	return menuitems
	
# --------------------------------------------------------------
def has_result():
	return sql("select name from `tab%s` limit 1" % webnotes.form_dict.get('dt')) and 'Yes' or 'No'

def is_submittable(dt):
	return sql("select name from tabDocPerm where parent=%s and ifnull(submit,0)=1 and docstatus<1 limit 1", dt)

def can_cancel(dt):
	return sql('select name from tabDocPerm where parent="%s" and ifnull(cancel,0)=1 and docstatus<1 and role in ("%s") limit 1' % (dt, '", "'.join(webnotes.user.get_roles())))

def get_dt_trend(dt):
	ret = {}
	for r in sql("select datediff(now(),modified), count(*) from `tab%s` where datediff(now(),modified) between 0 and 30 group by date(modified)" % dt):
		ret[cint(r[0])] = cint(r[1])
	return ret

def get_columns(out, sf, fl, dt, tag_fields):
	if not fl:
		fl = sf

	# subject
	subject = webnotes.conn.get_value('DocType', dt, 'subject')
	if subject:
		out['subject'] = subject
		
		# get fields from subject
		import re
		fl = re.findall('\%\( (?P<name> [^)]*) \)s', subject, re.VERBOSE)
		
		if tag_fields:
			fl += [t.strip() for t in tag_fields.split(',')]

	res = []
	for f in tuple(set(fl)):
		if f:
			res += [[c or '' for c in r] for r in sql("select fieldname, label, fieldtype, options from tabDocField where parent='%s' and fieldname='%s'" % (dt, f))]
			
	return res

# NOTE: THIS SHOULD BE CACHED IN DOCTYPE CACHE

def get_dt_details():
	"""
		Returns details called by DocBrowser this includes:
		the filters, columns, subject and tag_fields
		also if the doctype is of type "submittable"
	"""
	fl = eval(webnotes.form_dict.get('fl'))
	dt = webnotes.form_dict.get('dt')
	tag_fields, description = webnotes.conn.get_value('DocType', dt, ['tag_fields', 'description'])

	submittable = is_submittable(dt) and 1 or 0
 
	out = {
		'submittable':(is_submittable(dt) and 1 or 0), 
		'can_cancel':(can_cancel(dt) and 1 or 0)
	}

	# filters
	# -------

	sf = sql("select search_fields from tabDocType where name=%s", dt)[0][0] or ''

	# get fields from in_filter (if not in search_fields)
	if not sf.strip():
		res = sql("select fieldname, label, fieldtype, options from tabDocField where parent=%s and `in_filter` = 1 and ifnull(fieldname,'') != ''", dt)
		sf = [s[0] for s in res]
	else:
		sf = [s.strip() for s in sf.split(',')]
		res = sql("select fieldname, label, fieldtype, options from tabDocField where parent='%s' and fieldname in (%s)" % (dt, '"'+'","'.join(sf)+'"'))

	# select "link" options
	res = [[c or '' for c in r] for r in res]
	for r in res:
		if r[2]=='Select' and r[3] and r[3].startswith('link:'):
			tdt = r[3][5:]
			ol = sql("select name from `tab%s` where docstatus!=2 order by name asc" % tdt)
			r[3] = "\n".join([''] + [o[0] for o in ol])

	if not res:
		out['filters'] = [['name', 'ID', 'Data', '']]
	else:
		out['filters'] = [['name', 'ID', 'Data', '']] + res
	
	# columns
	# -------
	res = get_columns(out, sf, fl, dt, tag_fields)
	
	from webnotes.widgets.tags import check_user_tags
	check_user_tags(dt)
	
	out['columns'] = [['name', 'ID', 'Link', dt], ['modified', 'Modified', 'Data', ''], ['_user_tags', 'Tags', 'Data', '']] + res
	out['tag_fields'] = tag_fields
	out['description'] = description
	
	return out

def get_trend():
	return {'trend': get_dt_trend(webnotes.form_dict.get('dt'))}

#
# delete and archive in docbrowser
#
def delete_items():
	il = eval(webnotes.form_dict.get('items'))
	from webnotes.model import delete_doc
	from webnotes.model.code import get_obj
	
	for d in il:
		dt_obj = get_obj(d[0], d[1])
		if hasattr(dt_obj, 'on_trash'):
			dt_obj.on_trash()
		delete_doc(d[0], d[1])

def archive_items():
	
	from webnotes.utils.archive import archive_doc
	for d in il:
		archive_doc(d[0], d[1], webnotes.form_dict.get('action')=='Restore' and 1 or 0)

import webnotes.model

def xzip(a,b):
	d = {}
	for i in range(len(a)):
		d[a[i]] = b[i]
	return d
	
def expand(docs):
	"""
   Expand a doclist sent from the client side. (Internally used by the request handler)
	"""
	from webnotes.utils import load_json

	docs = load_json(docs)
	clist = []
	for d in docs['_vl']:
		doc = xzip(docs['_kl'][d[0]], d);
		clist.append(doc)
	return clist

def compress(doclist):
	"""
	   Compress a doclist before sending it to the client side. (Internally used by the request handler)

	"""	
	if doclist and hasattr(doclist[0],'fields'):
		docs = [d.fields for d in doclist]
	else:
		docs = doclist
		
	kl, vl = {}, []
	for d in docs:
		dt = d['doctype']
		if not (dt in kl.keys()):
			fl = d.keys()
			forbidden = ['server_code_compiled']
			nl = ['doctype','localname','__oldparent','__unsaved']
			
			# add client script for doctype, doctype due to ambiguity
			if dt=='DocType': nl.append('__client_script')
			
			for f in fl:
				if not (f in nl) and not (f in forbidden):
					nl.append(f)
			kl[dt] = nl

		## values
		fl = kl[dt]
		nl = []
		for f in fl:
			v = d.get(f)

			if type(v)==long:
				v=int(v)
			nl.append(v)
		vl.append(nl)
	#errprint(str({'_vl':vl,'_kl':kl}))
	return {'_vl':vl,'_kl':kl}

# Get Children List (for scripts utility)
# ---------------------------------------

def getlist(doclist, field):
	"""
   Filter a list of records for a specific field from the full doclist
   
   Example::
   
     # find all phone call details     
     dl = getlist(self.doclist, 'contact_updates')
     pl = []
     for d in dl:
       if d.type=='Phone':
         pl.append(d)
	"""
	
	l = []
	for d in doclist:
		if d.parent and (not d.parent.lower().startswith('old_parent:')) and d.parentfield == field:
			l.append(d)
	return l

# Copy doclist

def copy_doclist(doclist, no_copy = []):
	"""
      Save & return a copy of the given doclist
      Pass fields that are not to be copied in `no_copy`
	"""
	
	cl = []
	
	# main doc
	c = Document(fielddata = doclist[0].fields.copy())
	
	# clear no_copy fields
	for f in no_copy: 
		if c.fields.has_key(f):
			c.fields[f] = None
	
	c.name = None
	c.save(1)
	cl.append(c)
	
	# new parent name
	parent = c.name
	
	# children
	for d in doclist[1:]:
		c = Document(fielddata = d.fields.copy())
		c.name = None
		
		# clear no_copy fields
		for f in no_copy: 
			if c.fields.has_key(f):
				c.fields[f] = None

		c.parent = parent
		c.save(1)
		cl.append(c)

	return cl

# Validate Multiple Links

def validate_links_doclist(doclist):
	"""
	Validate link fields and return link fields that are not correct.
	Calls the `validate_links` function on the Document object
	"""
	ref, err_list = {}, []
		if not ref.get(d.doctype):
			ref[d.doctype] = d.make_link_list()
			
		err_list += d.validate_links(ref[d.doctype])
	return ', '.join(err_list)
	
# Get list of field values
# ------------------------

def getvaluelist(doclist, fieldname):
	"""
   Returns a list of values of a particualr fieldname from all Document object in a doclist
	"""
	l = []
		l.append(d.fields[fieldname])
	return l

def _make_html(doc, link_list):

	from webnotes.utils import cstr
	out = '<table class="simpletable">'
	for k in doc.fields.keys():
		if k!='server_code_compiled':		
			v = cstr(doc.fields[k])
			
			# link field
			if v and (k in link_list.keys()):
				dt = link_list[k]
				if type(dt)==str and dt.startswith('link:'):
					dt = dt[5:]
				v = '<a href="index.cgi?page=Form/%s/%s">%s</a>' % (dt, v, v) 
				
			out += '\t<tr><td>%s</td><td>%s</td></tr>\n' % (cstr(k), v)
		
	out += '</table>'
	return out

def to_html(doclist):
	"""
Return a simple HTML format of the doclist
	"""
	out = ''
	link_lists = {}
	
		if not link_lists.get(d.doctype):
			link_lists[d.doctype] = d.make_link_list()

		out += _make_html(d, link_lists[d.doctype])
		
	return out
	
import webnotes.db
import webnotes.utils
import webnotes.profile
import webnotes.defs

# =================================================================================
# HTTPRequest

class HTTPRequest:

		# Get Environment variables
		self.domain = webnotes.get_env_vars('HTTP_HOST')
		if self.domain and self.domain.startswith('www.'):
			self.domain = self.domain[4:]

		webnotes.remote_ip = webnotes.get_env_vars('REMOTE_ADDR')					

		# load cookies
		webnotes.cookie_manager = CookieManager()

		# set db
		self.set_db()

		# check status
		if webnotes.conn.get_global("__session_status")=='stop':
			webnotes.msgprint(webnotes.conn.get_global("__session_status_message"))
			raise Exception

		# -----------------------------
		# start transaction
		webnotes.conn.begin()

		# login
		webnotes.login_manager = LoginManager()

		# start session
		webnotes.session_obj = Session()
		webnotes.session = webnotes.session_obj.data
		webnotes.tenant_id = webnotes.session.get('tenant_id', 0)

		# write out cookies if sid is supplied (this is a pre-logged in redirect)
		if webnotes.form_dict.get('sid'):
			webnotes.cookie_manager.set_cookies()

		# run login triggers
		if webnotes.form_dict.get('cmd')=='login':
			webnotes.login_manager.run_trigger('on_login_post_session')
			
		# load profile
		self.setup_profile()

		webnotes.conn.commit()
		# end transaction

	# setup profile
	# -------------

	def setup_profile(self):
		webnotes.user = webnotes.profile.Profile()
		# load the profile data
		if webnotes.session['data'].get('profile'):
			webnotes.user.load_from_session(webnotes.session['data']['profile'])
		else:
			webnotes.user.load_profile()	

	# get account name
	# ------------------

	def get_ac_name(self):
		# login
		if webnotes.form_dict.get('acx'):
			return webnotes.form_dict.get('acx')
		
		# in form
		elif webnotes.form_dict.get('ac_name'):
			return webnotes.form_dict.get('ac_name')
			
		# in cookie
		elif webnotes.incoming_cookies.get('ac_name'):
			return webnotes.incoming_cookies.get('ac_name')
			
	# set database login

	def set_db(self, ac_name = None):
		
		# select based on subdomain
		if getattr(webnotes.defs,'domain_name_map', {}).get(self.domain):
			db_name = webnotes.defs.domain_name_map[self.domain]

		# select based on ac_name
		else:
			ac_name = self.get_ac_name()
			if ac_name:
				db_name = getattr(webnotes.defs,'db_name_map',{}).\
					get(ac_name, ac_name)
			else:
				db_name = getattr(webnotes.defs,'default_db_name','')
	
		webnotes.conn = webnotes.db.Database(user = db_name,password = getattr(webnotes.defs,'db_password',''))
		webnotes.ac_name = ac_name

# Login Manager

class LoginManager:
		self.cp = None
			# clear cache
			from webnotes.session_cache import clear_cache
			clear_cache(webnotes.form_dict.get('usr'))				

			self.authenticate()
			self.post_login()
			webnotes.response['message'] = 'Logged In'

	# run triggers, write cookies
	# ---------------------------
	
	def post_login(self):
		self.validate_ip_address()
		self.run_trigger()
	
	# check password
	# --------------
	
	def authenticate(self, user=None, pwd=None):
		if not (user and pwd):	
			user, pwd = webnotes.form_dict.get('usr'), webnotes.form_dict.get('pwd')

		if not (user and pwd):
			webnotes.msgprint('Incomplete Login Details', raise_exception=1)
		
		# custom authentication (for single-sign on)
		self.load_control_panel()
		if hasattr(self.cp, 'authenticate'):
			self.user = self.cp.authenticate()
		
		# check the password
		if user=='Administrator':
			p = webnotes.conn.sql("select name from tabProfile where name=%s and (`password`=%s OR `password`=PASSWORD(%s))", (user, pwd, pwd))
		else:
			p = webnotes.conn.sql("select name from tabProfile where name=%s and (`password`=%s  OR `password`=PASSWORD(%s)) and IFNULL(enabled,0)=1", (user, pwd, pwd))
			
		if not p:
			webnotes.msgprint('Authentication Failed', raise_exception=1)
			
		self.user = p[0][0]
	
	# triggers
	# --------
	
	def load_control_panel(self):
		try:
			if not self.cp:
				self.cp = webnotes.model.code.get_obj('Control Panel')
		except Exception, e:
			webnotes.response['Control Panel Exception'] = webnotes.utils.getTraceback()
	
	# --------
	def run_trigger(self, method='on_login'):
		try:
			import event_handlers
			if hasattr(event_handlers, method):
				getattr(event_handlers, method)(self)
			return
			pass
	
		# deprecated
		if self.cp and hasattr(self.cp, method):
			getattr(self.cp, method)(self)

	# ip validation
	
	def validate_ip_address(self):
		try:
			ip = webnotes.conn.sql("select ip_address from tabProfile where name = '%s'" % self.user)[0][0] or ''
		except: return
			
		ip = ip.replace(",", "\n").split('\n')
		ip = [i.strip() for i in ip]
			
		if ret and ip:
			if not (webnotes.remote_ip.startswith(ip[0]) or (webnotes.remote_ip in ip)):
				raise Exception, 'Not allowed from this IP Address'	

	# login as guest
	
	def login_as_guest(self):
		res = webnotes.conn.sql("select name from tabProfile where name='Guest' and ifnull(enabled,0)=1")
		if not res:
			raise Exception, "No Guest Access"
		self.user = 'Guest'
		self.post_login()

	# Logout
	# ------
	
	def call_on_logout_event(self):
		cp = webnotes.model.code.get_obj('Control Panel', 'Control Panel')
		if hasattr(cp, 'on_logout'):
			cp.on_logout(self)

	def logout(self, arg=''):
		self.run_trigger('on_logout')
		webnotes.conn.sql('update tabSessions set status="Logged Out" where sid="%s"' % webnotes.session['sid'])
		
# Cookie Manager

class CookieManager:
		import Cookie
		webnotes.cookies = Cookie.SimpleCookie()
		self.get_incoming_cookies()

	# get incoming cookies
	# --------------------
	def get_incoming_cookies(self):
		import os
		cookies = {}
		if 'HTTP_COOKIE' in os.environ:
			c = os.environ['HTTP_COOKIE']
			webnotes.cookies.load(c)
			for c in webnotes.cookies.values():
				cookies[c.key] = c.value
					
		webnotes.incoming_cookies = cookies
		
	# Set cookies
	# -----------
	
	def set_cookies(self):
		if webnotes.conn.cur_db_name:
			webnotes.cookies['account_id'] = webnotes.conn.cur_db_name
		
		# ac_name	
		webnotes.cookies['ac_name'] = webnotes.ac_name or ''
		
		if webnotes.session.get('sid'):
			webnotes.cookies['sid'] = webnotes.session['sid']

			# sid expires in 3 days
			import datetime
			expires = datetime.datetime.now() + datetime.timedelta(days=3)

			webnotes.cookies['sid']['expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S')		

	# Set Remember Me
	# ---------------

	def set_remember_me(self):
		if webnotes.utils.cint(webnotes.form_dict.get('remember_me')):
			remember_days = webnotes.conn.get_value('Control Panel',None,'remember_for_days') or 7
			webnotes.cookies['remember_me'] = 1

			expires = datetime.datetime.now() + datetime.timedelta(days=remember_days)

			for k in webnotes.cookies.keys():
				webnotes.cookies[k]['expires'] = expires.strftime('%a, %d %b %Y %H:%M:%S')	

# Session 

class Session:
	def __init__(self, user=None):
		self.user = user
		self.sid = webnotes.form_dict.get('sid') or webnotes.incoming_cookies.get('sid')
		self.data = {'user':user,'data':{}}

			self.start()
			return
			
		self.load()
	
	# start a session
	def load(self):
		
		r=None
		try:
			r = webnotes.conn.sql("select user, sessiondata, status from tabSessions where sid='%s'" % self.sid)
			if e.args[0]==1054:
				self.add_status_column()
			else:
				raise e
	
		if r:
			r=r[0]
			
			# ExipredSession
			if r[2]=='Expired' and (webnotes.form_dict.get('cmd')!='resume_session'):
				if r[0]=='Guest' or (not webnotes.form_dict.get('cmd')) or webnotes.form_dict.get('cmd')=='logout':
					webnotes.login_manager.login_as_guest()
					self.start()
				else:
					webnotes.response['session_status'] = 'Session Expired'
					raise Exception, 'Session Expired'
			elif r[2]=='Logged Out':
				webnotes.login_manager.login_as_guest()
				self.start()
				# allow refresh or logout
				if webnotes.form_dict.get('cmd') and webnotes.form_dict.get('cmd')!='logout':
					webnotes.response['session_status'] = 'Logged Out'
					raise Exception, 'Logged Out'
			else:
				self.data = {'data':eval(r[1]), 'user':r[0], 'sid': self.sid}
		else:				
			webnotes.login_manager.login_as_guest()
			
	def start(self):
		import os
		import webnotes.utils
		
		# generate sid
		self.data['user'] = webnotes.login_manager.user
		self.data['sid'] = webnotes.utils.generate_hash()
		self.data['data']['session_ip'] = os.environ.get('REMOTE_ADDR');
		self.data['data']['tenant_id'] = webnotes.form_dict.get('tenant_id', 0)

		# get ipinfo
		if webnotes.conn.get_global('get_ip_info'):
			self.get_ipinfo()
		
		# insert session
		try:
			self.insert_session_record()
				self.insert_session_record()
			else:
				raise e

		# update profile
		webnotes.conn.sql("UPDATE tabProfile SET last_login = '%s', last_ip = '%s' where name='%s'" % (webnotes.utils.now(), webnotes.remote_ip, self.data['user']))

		# set cookies to write
		webnotes.session = self.data
		webnotes.cookie_manager.set_cookies()

	# resume session
	def resume(self):
		pwd = webnotes.form_dict.get('pwd')
		webnotes.login_manager.authenticate(self.data['user'], pwd)
		webnotes.conn.sql("update tabSessions set status='Active' where sid=%s", self.data['sid'])
		return 'Logged In'
	
	# update session
	def update(self):
		# update session
		webnotes.conn.sql("update tabSessions set sessiondata=%s, user=%s, lastupdate=NOW() where sid=%s" , (str(self.data['data']), self.data['user'], self.data['sid']))	

		self.check_expired()

	# check expired
	def check_expired(self):
		# in control panel?
		exp_sec = webnotes.conn.get_value('Control Panel', None, 'session_expiry') or '6:00:00'
		
		# set sessions as expired
		try:
			webnotes.conn.sql("update from tabSessions where TIMEDIFF(NOW(), lastupdate) > %s SET `status`='Expired'", exp_sec)
		
		# clear out old sessions
		webnotes.conn.sql("delete from tabSessions where TIMEDIFF(NOW(), lastupdate) > '72:00:00'")

	# -----------------------------
	def add_status_column(self):
		webnotes.conn.sql("alter table tabSessions add column `status` varchar(20)")

	# Get IP Info from ipinfodb.com
	def get_ipinfo(self):
		import os
		
		try:
			import pygeoip
		except:
			return
		
		gi = pygeoip.GeoIP('data/GeoIP.dat')
		self.data['data']['ipinfo'] = {'countryName': gi.country_name_by_addr(os.environ.get('REMOTE_ADDR'))}
			
	def insert_session_record(self):
		webnotes.conn.sql("insert into tabSessions (sessiondata, user, lastupdate, sid, status) values (%s , %s, NOW(), %s, 'Active')", (str(self.data['data']), self.data['user'], self.data['sid']))
		
"""
Syncs a database table to the `DocType` (metadata)

.. note:: This module is only used internally

"""
import os

type_map = {
	'currency':		('decimal', '14,2')
	,'int':			('int', '11')
	,'float':		('decimal', '14,6')
	,'check':		('int', '1')
	,'small text':	('text', '')
	,'long text':	('text', '')
	,'code':		('text', '')
	,'text editor':	('text', '')
	,'date':		('date', '')
	,'time':		('time', '')
	,'text':		('text', '')
	,'data':		('varchar', '180')
	,'link':		('varchar', '180')
	,'password':	('varchar', '180')
	,'select':		('varchar', '180')
	,'read only':	('varchar', '180')
	,'blob':		('longblob', '')
}

default_columns = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'parent',\
	 'parentfield', 'parenttype', 'idx']

default_shortcuts = ['_Login', '__user', '_Full Name', 'Today', '__today']

from webnotes.utils import cint

# -------------------------------------------------
# Class database table

class DbTable:
	def __init__(self, doctype, prefix = 'tab'):
		self.doctype = doctype
		self.name = prefix + doctype
		self.columns = {}
		self.current_columns = {}
		
		# lists for change
		self.add_column = []
		self.change_type = []
		self.add_index = []
		self.drop_index = []
		self.set_default = []
		
		# load
		self.get_columns_from_docfields()

	def create(self):
		add_text = ''
		
		# columns
		t = self.get_column_definitions()
		if t: add_text += ',\n'.join(self.get_column_definitions()) + ',\n'
		
		# index
		t = self.get_index_definitions()
		if t: add_text += ',\n'.join(self.get_index_definitions()) + ',\n'
	
		# create table
		webnotes.conn.sql("""create table `%s` (
			name varchar(120) not null primary key, 
			creation datetime,
			modified datetime,
			modified_by varchar(40), 
			owner varchar(40),
			docstatus int(1) default '0', 
			parent varchar(120),
			parentfield varchar(120), 
			parenttype varchar(120), 
			idx int(8),
			%sindex parent(parent)) ENGINE=InnoDB""" % (self.name, add_text))

	def get_columns_from_docfields(self):
		fl = webnotes.conn.sql("SELECT * FROM tabDocField WHERE parent = '%s'" % self.doctype, as_dict = 1)

		for f in fl:
			if not f.get('no_column'):
				self.columns[f['fieldname']] = DbColumn(self, f['fieldname'], f['fieldtype'], f.get('length'), f['default'], f['search_index'], f['options'])
	
	def get_columns_from_db(self):
		self.show_columns = webnotes.conn.sql("desc `%s`" % self.name)
		for c in self.show_columns:
			self.current_columns[c[0]] = {'name': c[0], 'type':c[1], 'index':c[3], 'default':c[4]}
	
	def get_column_definitions(self):
		column_list = [] + default_columns
		ret = []
		for k in self.columns.keys():
			if k not in column_list:
				d = self.columns[k].get_definition()
				if d:
					ret.append('`'+ k+ '` ' + d)
					column_list.append(k)
		return ret
	
	def get_index_definitions(self):
		ret = []
			if type_map.get(self.columns[k].fieldtype) and type_map.get(self.columns[k].fieldtype.lower())[0] not in ('text', 'blob'):
				ret.append('index `' + k + '`(`' + k + '`)')
		return ret

	# GET foreign keys
	def get_foreign_keys(self):
		fk_list = []
		txt = webnotes.conn.sql("show create table `%s`" % self.name)[0][1]
		for line in txt.split('\n'):
			if line.strip().startswith('CONSTRAINT') and line.find('FOREIGN')!=-1:
				try:
					fk_list.append((line.split('`')[3], line.split('`')[1]))
				except IndexError, e:
					pass

		return fk_list

	# Drop foreign keys
	def drop_foreign_keys(self):
		if not self.drop_foreign_key:
			return

		fk_list = self.get_foreign_keys()
		
		# make dictionary of constraint names
		fk_dict = {}
		for f in fk_list:
			fk_dict[f[0]] = f[1]
			
		# drop
		for col in self.drop_foreign_key:
			webnotes.conn.sql("set foreign_key_checks=0")
			webnotes.conn.sql("alter table `%s` drop foreign key `%s`" % (self.name, fk_dict[col.fieldname]))
			webnotes.conn.sql("set foreign_key_checks=1")
		
	def sync(self):
		if not self.name in DbManager(webnotes.conn).get_tables_list(webnotes.conn.cur_db_name):
			self.create()
		else:
			self.alter()
	
	def alter(self):
		self.get_columns_from_db()
		for col in self.columns.values():
			col.check(self.current_columns.get(col.fieldname, None))

		for col in self.add_column:
			webnotes.conn.sql("alter table `%s` add column `%s` %s" % (self.name, col.fieldname, col.get_definition()))

		for col in self.change_type:
			webnotes.conn.sql("alter table `%s` change `%s` `%s` %s" % (self.name, col.fieldname, col.fieldname, col.get_definition()))

		for col in self.add_index:
			webnotes.conn.sql("alter table `%s` add index `%s`(`%s`)" % (self.name, col.fieldname, col.fieldname))

		for col in self.drop_index:
			if col.fieldname != 'name': # primary key
				webnotes.conn.sql("alter table `%s` drop index `%s`" % (self.name, col.fieldname))

		for col in self.set_default:
			webnotes.conn.sql("alter table `%s` alter column `%s` set default %s" % (self.name, col.fieldname, '%s'), col.default)

# Class database column

class DbColumn:
	def __init__(self, table, fieldname, fieldtype, length, default, set_index, options):
		self.table = table
		self.fieldname = fieldname
		self.fieldtype = fieldtype
		self.length = length
		self.set_index = set_index
		self.default = default
		self.options = options

	def get_definition(self, with_default=1):
		d = type_map.get(self.fieldtype.lower())

		if not d:
			return
			
		ret = d[0]
		if d[1]:
			ret += '(' + d[1] + ')'
		if with_default and self.default and (self.default not in default_shortcuts):
			ret += ' default "' + self.default.replace('"', '\"') + '"'
		return ret
		
	def check(self, current_def):
		column_def = self.get_definition(0)

		# no columns
		if not column_def:
			return
		
		# to add?
		if not current_def:
			self.fieldname = validate_column_name(self.fieldname)
			self.table.add_column.append(self)
			return

		# type
		if current_def['type'] != column_def:
			self.table.change_type.append(self)
		
		# index
		else:
			if (current_def['index'] and not self.set_index):
				self.table.drop_index.append(self)
			
			if (not current_def['index'] and self.set_index and not (column_def in ['text','blob'])):
				self.table.add_index.append(self)
		
		# default
		if (self.default and (current_def['default'] != self.default) and (self.default not in default_shortcuts) and not (column_def in ['text','blob'])):
			self.table.set_default.append(self)

class DbManager:
	"""
	Basically, a wrapper for oft-used mysql commands. like show tables,databases, variables etc... 

	#TODO:
		0.  Simplify / create settings for the restore database source folder 
		0a. Merge restore database and extract_sql(from webnotes_server_tools).
		1. Setter and getter for different mysql variables.
		2. Setter and getter for mysql variables at global level??
	"""	
	def __init__(self,conn):
		"""
		Pass root_conn here for access to all databases.
		"""
 		if conn:
 			self.conn = conn
		
	def get_variables(self,regex):
		"""
		Get variables that match the passed pattern regex
		"""
		return list(self.conn.sql("SHOW VARIABLES LIKE '%s'"%regex))

	def drop_all_databases(self):
		"""
		Danger: will delete all databases except test,mysql.
		"""
		db_list = self.get_database_list()
		for db in db_list:
			self.drop_database(db)
			
	def get_table_schema(self,table):
		"""
		Just returns the output of Desc tables.
		"""
		return list(self.conn.sql("DESC %s"%table))
		
	def get_tables_list(self,target):	
		"""
		
		"""
		try:
			self.conn.use(target)
			res = self.conn.sql("SHOW TABLES;")
			table_list = []
			for table in res:
				table_list.append(table[0])
			return table_list

		except Exception,e:
			raise e

	def create_user(self,user,password):
		#Create user if it doesn't exist.
		try:
			if password:
				self.conn.sql("CREATE USER '%s'@'localhost' IDENTIFIED BY '%s';" % (user[:16], password))
			else:
				self.conn.sql("CREATE USER '%s'@'localhost';"%user[:16])
			raise e

	def delete_user(self,target):
	# delete user if exists
		try:
			self.conn.sql("DROP USER '%s'@'localhost';" % target)
			if e.args[0]==1396:
				pass
			else:
				raise e

	def create_database(self,target):
		
		try:
			self.conn.sql("CREATE DATABASE IF NOT EXISTS `%s` ;" % target)
			raise e

	def drop_database(self,target):
		try:
			self.conn.sql("DROP DATABASE IF EXISTS `%s`;"%target)
			raise e

	def grant_all_privileges(self,target,user):
		try:
			self.conn.sql("GRANT ALL PRIVILEGES ON `%s` . * TO '%s'@'localhost';" % (target, user))
			raise e

	def grant_select_privilges(self,db,table,user):
		try:
			if table:
				self.conn.sql("GRANT SELECT ON %s.%s to '%s'@'localhost';" % (db,table,user))
			else:
				self.conn.sql("GRANT SELECT ON %s.* to '%s'@'localhost';" % (db,user))
			raise e

	def flush_privileges(self):
		try:
			self.conn.sql("FLUSH PRIVILEGES")
			raise e

	def get_database_list(self):
		try:
			db_list = []
			ret_db_list = self.conn.sql("SHOW DATABASES")
			for db in ret_db_list:
				if db[0] not in ['information_schema', 'mysql', 'test', 'accounts']:
					db_list.append(db[0])
			return db_list
			raise e

	def restore_database(self,target,source,root_password):
		import webnotes.defs
		mysql_path = getattr(webnotes.defs, 'mysql_path', None)
		mysql = mysql_path and os.path.join(mysql_path, 'mysql') or 'mysql'
		
		try:
			ret = os.system("%s -u root -p%s %s < %s"%(mysql, root_password, target, source))
			raise e

	def drop_table(self,table_name):
		try:
			self.conn.sql("DROP TABLE IF EXISTS %s "%(table_name))
			raise e	

	def set_transaction_isolation_level(self,scope='SESSION',level='READ COMMITTED'):
		#Sets the transaction isolation level. scope = global/session
		try:
			self.conn.sql("SET %s TRANSACTION ISOLATION LEVEL %s"%(scope,level))
			raise e

# validate column name to be code-friendly

def validate_column_name(n):
	n = n.replace(' ','_').strip().lower()
	import re
	if not re.match("[a-zA-Z_][a-zA-Z0-9_]*$", n):
		webnotes.msgprint('err:%s is not a valid fieldname.<br>A valid name must contain letters / numbers / spaces.<br><b>Tip: </b>You can change the Label after the fieldname has been set' % n)
		raise Exception
	return n

# sync table - called from form.py

def updatedb(dt, archive=0):
	"""
	Syncs a `DocType` to the table
	   * creates if required
	   * updates columns
	   * updates indices
	"""
	res = webnotes.conn.sql("select ifnull(issingle, 0) from tabDocType where name=%s", dt)
	if not res:
		raise Exception, 'Wrong doctype "%s" in updatedb' % dt
	
	if not res[0][0]:
		tab = DbTable(dt, archive and 'arc' or 'tab')
		tab.sync()

# patch to remove foreign keys
# ----------------------------

def remove_all_foreign_keys():
	webnotes.conn.sql("set foreign_key_checks = 0")
	for t in webnotes.conn.sql("select name from tabDocType where ifnull(issingle,0)=0"):
		dbtab = webnotes.model.db_schema.DbTable(t[0])
		try:
			fklist = dbtab.get_foreign_keys()
			if e.args[0]==1146:
				fklist = []
			else:
				raise e
				
		for f in fklist:
			webnotes.conn.sql("alter table `tab%s` drop foreign key `%s`" % (t[0], f[1]))

import os, os.path, shutil

# This code is original from jsmin by Douglas Crockford, it was translated to
# Python by Baruch Even. The original code had the following copyright and
# license.
#
# /* jsmin.c
#    2007-05-22
#
# Copyright (c) 2002 Douglas Crockford  (www.crockford.com)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# The Software shall be used for Good, not Evil.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# */

from StringIO import StringIO

def jsmin(js):
    ins = StringIO(js)
    outs = StringIO()
    JavascriptMinify().minify(ins, outs)
    str = outs.getvalue()
    if len(str) > 0 and str[0] == '\n':
        str = str[1:]
    return str

def isAlphanum(c):
    """return true if the character is a letter, digit, underscore,
           dollar sign, or non-ASCII character.
    """
    return ((c >= 'a' and c <= 'z') or (c >= '0' and c <= '9') or
            (c >= 'A' and c <= 'Z') or c == '_' or c == '$' or c == '\\' or (c is not None and ord(c) > 126));

class UnterminatedComment(Exception):
    pass

class UnterminatedStringLiteral(Exception):
    pass

class UnterminatedRegularExpression(Exception):
    pass

class JavascriptMinify(object):

    def _outA(self):
        self.outstream.write(self.theA)
    def _outB(self):
        self.outstream.write(self.theB)

    def _get(self):
        """return the next character from stdin. Watch out for lookahead. If
           the character is a control character, translate it to a space or
           linefeed.
        """
        c = self.theLookahead
        self.theLookahead = None
        if c == None:
            c = self.instream.read(1)
        if c >= ' ' or c == '\n':
            return c
        if c == '': # EOF
            return '\000'
        if c == '\r':
            return '\n'
        return ' '

    def _peek(self):
        self.theLookahead = self._get()
        return self.theLookahead

    def _next(self):
        """get the next character, excluding comments. peek() is used to see
           if an unescaped '/' is followed by a '/' or '*'.
        """
        c = self._get()
        if c == '/' and self.theA != '\\':
            p = self._peek()
            if p == '/':
                c = self._get()
                while c > '\n':
                    c = self._get()
                return c
            if p == '*':
                while 1:
                    if c == '*':
                        if self._peek() == '/':
                            self._get()
                            return ' '
                    if c == '\000':
                        raise UnterminatedComment()

        return c

    def _action(self, action):
        """do something! What you do is determined by the argument:
           1   Output A. Copy B to A. Get the next B.
           2   Copy B to A. Get the next B. (Delete A).
           3   Get the next B. (Delete B).
           action treats a string as a single character. Wow!
           action recognizes a regular expression if it is preceded by ( or , or =.
        """
        if action <= 1:
            self._outA()

        if action <= 2:
            self.theA = self.theB
            if self.theA == "'" or self.theA == '"':
                while 1:
                    self._outA()
                    self.theA = self._get()
                    if self.theA == self.theB:
                        break
                    if self.theA <= '\n':
                        raise UnterminatedStringLiteral()
                    if self.theA == '\\':
                        self._outA()
                        self.theA = self._get()

        if action <= 3:
            self.theB = self._next()
            if self.theB == '/' and (self.theA == '(' or self.theA == ',' or
                                     self.theA == '=' or self.theA == ':' or
                                     self.theA == '[' or self.theA == '?' or
                                     self.theA == '!' or self.theA == '&' or
                                     self.theA == '|' or self.theA == ';' or
                                     self.theA == '{' or self.theA == '}' or
                                     self.theA == '\n'):
                self._outA()
                self._outB()
                while 1:
                    if self.theA == '/':
                        break
                    elif self.theA == '\\':
                    elif self.theA <= '\n':
                        raise UnterminatedRegularExpression()
                self.theB = self._next()

    def _jsmin(self):
        """Copy the input to the output, deleting the characters which are
           insignificant to JavaScript. Comments will be removed. Tabs will be
           replaced with spaces. Carriage returns will be replaced with linefeeds.
           Most spaces and linefeeds will be removed.
        """
        self.theA = '\n'
        self._action(3)

        while self.theA != '\000':
            if self.theA == ' ':
                if isAlphanum(self.theB):
                    self._action(1)
                else:
                    self._action(2)
            elif self.theA == '\n':
                if self.theB in ['{', '[', '(', '+', '-']:
                elif self.theB == ' ':
                    self._action(3)
                else:
                    if isAlphanum(self.theB):
                        self._action(1)
                    else:
                        self._action(2)
            else:
                if self.theB == ' ':
                    if isAlphanum(self.theA):
                    else:
                        self._action(3)
                elif self.theB == '\n':
                    if self.theA in ['}', ']', ')', '+', '-', '"', '\'']:
                    else:
                        if isAlphanum(self.theA):
                            self._action(1)
                        else:
                            self._action(3)
                else:

    def minify(self, instream, outstream):
        self.instream = instream
        self.outstream = outstream
        self.theB = None

        self._jsmin()
        self.instream.close()

"""
	Utilities for using modules
"""

transfer_types = ['Role', 'Print Format','DocType','Page','DocType Mapper','GL Mapper','Search Criteria', 'Patch']

def scrub(txt):
	return txt.replace(' ','_').replace('-', '_').replace('/', '_').lower()

def scrub_dt_and_dn(dt, dn):
	"""
		Returns in lowercase and code friendly names of doctype and name for certain types
	"""
	ndt, ndn = dt, dn
	if dt.lower() in ('doctype', 'search criteria', 'page'):
		ndt, ndn = scrub(dt), scrub(dn)

	return ndt, ndn

def get_item_file(module, dt, dn):
	"""
		Returns the path of the item file
	"""
	import os
	ndt, ndn = scrub_dt_and_dn(dt, dn)

	return os.path.join(get_module_path(module), ndt, ndn, ndn + '.txt')
	
def get_item_timestamp(module, dt, dn):
	"""
		Return ths timestamp of the given item (if exists)
	"""
	return get_file_timestamp(get_item_file(module, dt, dn))

def get_file_timestamp(fn):
	"""
		Returns timestamp of the given file
	"""
	import os
	from webnotes.utils import cint
	
	try:
		return str(cint(os.stat(fn).st_mtime))
	except OSError, e:
		if e.args[0]!=2:
			raise e
		else:
			
def get_module_path(module):
	"""
		Returns path of the given module (imports it and reads it from __file__)
	"""
	import webnotes.defs, os, os.path
	
	try:
		exec ('import ' + scrub(module)) in locals()
		modules_path = eval(scrub(module) + '.__file__')
		
		modules_path = os.path.sep.join(modules_path.split(os.path.sep)[:-1])
	except ImportError, e:
		# get module path by importing the module
		modules_path = os.path.join(webnotes.defs.modules_path, scrub(module))
		
	return modules_path
	
def switch_module(dt, dn, to, frm=None, export=None):
	"""
		Change the module of the given doctype, if export is true, then also export txt and copy
		code files from src
	"""
	import os
	webnotes.conn.sql("update `tab"+dt+"` set module=%s where name=%s", (to, dn))

	if export:
		export_doc(dt, dn)

		# copy code files
		if dt in ('DocType', 'Page', 'Search Criteria'):
			from_path = os.path.join(get_module_path(frm), scrub(dt), scrub(dn), scrub(dn))
			to_path = os.path.join(get_module_path(to), scrub(dt), scrub(dn), scrub(dn))

			# make dire if exists
			os.system('mkdir -p %s' % os.path.join(get_module_path(to), scrub(dt), scrub(dn)))

			for ext in ('py','js','html','css'):
				os.system('cp %s %s')

"""
This is where all the plug-in code is executed. The standard method for DocTypes is declaration of a 
standardized `DocType` class that has the methods of any DocType. When an object is instantiated using the
`get_obj` method, it creates an instance of the `DocType` class of that particular DocType and sets the 
`doc` and `doclist` attributes that represent the fields (properties) of that record.

methods in following modules are imported for backward compatibility

	* webnotes.*
	* webnotes.utils.*
	* webnotes.model.doc.*
	* webnotes.model.doclist.*
"""
custom_class = '''
# Please edit this list and import only required elements

from webnotes.utils import add_days, add_months, add_years, cint, cstr, date_diff, default_fields, flt, fmt_money, formatdate, generate_hash, getTraceback, get_defaults, get_first_day, get_last_day, getdate, has_common, month_name, now, nowdate, replace_newlines, sendmail, set_default, str_esc_quote, user_format, validate_email_add
from webnotes.model import db_exists
from webnotes.model.doc import Document, addchild, removechild, getchildren, make_autoname, SuperDocType
from webnotes.model.doclist import getlist, copy_doclist
from webnotes.model.code import get_obj, get_server_obj, run_server_obj, updatedb, check_syntax
from webnotes import session, form, is_testing, msgprint, errprint

set = webnotes.conn.set
get_value = webnotes.conn.get_value
in_transaction = webnotes.conn.in_transaction
convert_to_lists = webnotes.conn.convert_to_lists

# -----------------------------------------------------------------------------------------

class CustomDocType(DocType):
  def __init__(self, doc, doclist):
    DocType.__init__(self, doc, doclist)
'''

#=================================================================================
# execute a script with a lot of globals - deprecated

def execute(code, doc=None, doclist=[]):
	"""
	Execute the code, if doc is given, then return the instance of the `DocType` class created
	"""
	# functions used in server script of DocTypes
	# --------------------------------------------------	
	from webnotes.utils import add_days, add_months, add_years, cint, cstr, date_diff, default_fields, flt, fmt_money, formatdate, generate_hash, getTraceback, get_defaults, get_first_day, get_last_day, getdate, has_common, month_name, now, nowdate, replace_newlines, sendmail, set_default, str_esc_quote, user_format, validate_email_add
	from webnotes.model import db_exists
	from webnotes.model.doc import Document, addchild, removechild, getchildren, make_autoname, SuperDocType
	from webnotes.model.doclist import getlist, copy_doclist
	from webnotes import session, form, is_testing, msgprint, errprint

	set = webnotes.conn.set
	get_value = webnotes.conn.get_value
	in_transaction = webnotes.conn.in_transaction
	convert_to_lists = webnotes.conn.convert_to_lists
	if webnotes.user:
		get_roles = webnotes.user.get_roles
	locals().update({'get_obj':get_obj, 'get_server_obj':get_server_obj, 'run_server_obj':run_server_obj, 'updatedb':updatedb, 'check_syntax':check_syntax})
	version = 'v170'
	NEWLINE = '\n'
	BACKSLASH = '\\'

	# execute it
	# -----------------
	exec code in locals()
	
	# if doc
	if doc:
		d = DocType(doc, doclist)
		return d
		
	if locals().get('page_html'):
		return page_html

	if locals().get('out'):
		return out

# load the DocType class from module & return an instance

def get_custom_script(doctype, script_type):
	"""
		Returns custom script if set in doctype `Custom Script`
	"""
	try:
		custom_script = webnotes.conn.sql("select script from `tabCustom Script` where dt=%s and script_type=%s", (doctype, script_type))
			
	if custom_script and custom_script[0][0]:
		return custom_script[0][0]
		
def get_server_obj(doc, doclist = [], basedoctype = ''):
	"""
	Returns the instantiated `DocType` object. Will also manage caching & compiling
	"""
	# for test
	
	# get doctype details
	module = webnotes.conn.get_value('DocType', doc.doctype, 'module')
	
	# no module specified (must be really old), can't get code so quit
	if not module:
		return
		
	module = module.replace(' ','_').lower()
	dt = doc.doctype.replace(' ','_').lower()

	# import
	try:
		exec 'from %s.doctype.%s.%s import DocType' % (module, dt, dt)
		# declare it here
		class DocType:
			def __init__(self, d, dl):
				self.doc, self.doclist = d, dl

	# custom?
	custom_script = get_custom_script(doc.doctype, 'Server')
	if custom_script:
		global custom_class		
		
		exec custom_class + custom_script.replace('\t','  ') in locals()
			
		return CustomDocType(doc, doclist)
	else:
		return DocType(doc, doclist)
	
# get object (from dt and/or dn or doclist)

def get_obj(dt = None, dn = None, doc=None, doclist=[], with_children = 0):
	"""
	   Returns the instantiated `DocType` object. Here you can pass the DocType and name (ID) to get the object.
	   If with_children is true, then all child records will be laoded and added in the doclist.
	"""	
	if dt:
		import webnotes.model.doc
		
		if not dn:
			dn = dt
		if with_children:
			doclist = webnotes.model.doc.get(dt, dn, from_get_obj=1)
		else:
			doclist = webnotes.model.doc.get(dt, dn, with_children = 0, from_get_obj=1)
		return get_server_obj(doclist[0], doclist)
	else:
		return get_server_obj(doc, doclist)
		
# get object and run method

def run_server_obj(server_obj, method_name, arg=None):
	"""
	   Executes a method (`method_name`) from the given object (`server_obj`)
	"""
	if server_obj and hasattr(server_obj, method_name):
		if arg:
			return getattr(server_obj, method_name)(arg)
		else:
			return getattr(server_obj, method_name)()
	else:
		raise Exception, 'No method %s' % method_name

# deprecated methods to keep v160 apps happy

def updatedb(doctype, userfields = [], args = {}):
	pass

def check_syntax(code):
	return ''

#===================================================================================
def get_code(module, dt, dn, extn, is_static=None, fieldname=None):
	from webnotes.modules import scrub, get_module_path
	import os, webnotes
	
	# get module (if required)
		module = webnotes.conn.sql("select module from `tab%s` where name=%s" % (dt,'%s'),dn)[0][0]

	# no module, quit
		return ''
	
	# file names
	if scrub(dt) in ('page','doctype','search_criteria'):
		dt, dn = scrub(dt), scrub(dn)

	# get file name
	fname = dn + '.' + extn
	if is_static:
		fname = dn + '_static.' + extn

	# code
	code = ''
	try:
		file = open(os.path.join(get_module_path(scrub(module)), dt, dn, fname), 'r')
		code = file.read()
		file.close()	
	except IOError, e:
		# no file, try from db
		if fieldname:
			code = webnotes.conn.get_value(dt, dn, fieldname)

	return code

def upload():

	# get record details
	dt = form.getvalue('doctype')
	dn = form.getvalue('docname')
	at_id = form.getvalue('at_id')

	# save
	fid, fname = save_uploaded()
	
	if fid:
		# refesh the form!
		webnotes.response['result'] = """
<script type='text/javascript'>
window.parent.wn.widgets.form.file_upload_done('%s', '%s', '%s', '%s', '%s');
window.parent.frms['%s'].show_doc('%s');
</script>
			""" % (dt, dn, fid, fname.replace("'", "\\'"), at_id, dt, dn)

# -------------------------------------------------------

def make_thumbnail(blob, size):
	from PIL import Image
	import cStringIO
				
	fobj = cStringIO.StringIO(blob)
	image = Image.open(fobj)
	image.thumbnail((tn,tn*2), Image.ANTIALIAS)
	outfile = cStringIO.StringIO()
	image.save(outfile, 'JPEG')
	outfile.seek(0)
	fcontent = outfile.read()
	
	return fcontent

def save_uploaded(js_okay='window.parent.msgprint("File Upload Successful")', js_fail=''):
	
	webnotes.response['type'] = 'iframe'

	form, fid, fname = webnotes.form, None, None

	try:
		# has attachment?
		if 'filedata' in form:
			i = form['filedata']
	
			fname, content = i.filename, i.file.read()
	
			# thumbnail
			if webnotes.form_dict.get('thumbnail'):
				try:
					content = make_thumbnail(content, int(form.get('thumbnail')))
					# change extension to jpg
					fname = '.'.join(fname.split('.')[:-1])+'.jpg'
				except Exception, e:
					pass
		
			# get the file id
			fid = save_file(fname, content)
			
			# okay
			webnotes.response['result'] = """<script type='text/javascript'>%s</script>""" % js_okay
		else:
			webnotes.response['result'] = """<script type='text/javascript'>window.parent.msgprint("No file"); %s</script>""" % js_fail
			
		webnotes.response['result'] = """<script type='text/javascript'>window.parent.msgprint("%s"); window.parent.errprint("%s"); %s</script>""" % (str(e), webnotes.utils.getTraceback().replace('\n','<br>').replace('"', '\\"'), js_fail)
	
	return fid, fname

def save_file(fname, content, module=None):

	# some browsers return the full path
	if '\\' in fname:
		fname = fname.split('\\')[-1]
	if '/' in fname:
		fname = fname.split('/')[-1]

	# generate the ID (?)
	f = Document('File Data')
	f.file_name = fname
	if module:
		f.module = module
	f.save(1)
	
	write_file(f.name, content)

	return f.name

def write_file(fid, content):
	import webnotes, os, webnotes.defs

	# test size
	max_file_size = 1000000
	if hasattr(webnotes.defs, 'max_file_size'):
		max_file_size = webnotes.defs.max_file_size

	if len(content) > max_file_size:
		raise Exception, 'Maximum File Limit (%s MB) Crossed' % (int(max_file_size / 1000000))

	# no slashes
	fid = fid.replace('/','-')

	# save to a folder (not accessible to public)
	folder = webnotes.get_files_path()
		
	# create account folder (if not exists)
	webnotes.create_folder(folder)

	# write the file
	file = open(os.path.join(folder, fid),'w+')
	file.write(content)
	file.close()
		
def get_file_system_name(fname):
	# get system name from File Data table
	return webnotes.conn.sql("select name, file_name from `tabFile Data` where name=%s or file_name=%s", (fname, fname))

def delete_file(fname, verbose=0):
	import webnotes, os
		
	for f in get_file_system_name(fname):
		webnotes.conn.sql("delete from `tabFile Data` where name=%s", f[0])
	
		# delete file
		file_id = f[0].replace('/','-')
		try:
			os.remove(os.path.join(webnotes.get_files_path(), file_id))
		except OSError, e:
			if e.args[0]!=2:
				raise e
		
		if verbose: webnotes.msgprint('File Deleted')

# Get File

def get_file(fname):
	in_fname = fname
	
	# from the "File" table
	if webnotes.conn.exists('File',fname):
		fname = webnotes.conn.sql("select file_list from tabFile where name=%s", fname)
		fname = fname and fname[0][0]
		fname = fname.split('\n')[0].split(',')[1]

	if get_file_system_name(fname):
		f = get_file_system_name(fname)[0]
	else:
		f = None

	# read the file
	import os
		
	file_id = f[0].replace('/','-')
	file = open(os.path.join(webnotes.get_files_path(), file_id), 'r')
	content = file.read()
	return [f[1], content]

# Conversion Patch

def convert_to_files(verbose=0):
	
	# nfiles
	fl = webnotes.conn.sql("select name from `tabFile Data`")
	for f in fl:
		# get the blob
		blob = webnotes.conn.sql("select blob_content from `tabFile Data` where name=%s", f[0])[0][0]
		
		if blob:

			if hasattr(blob, 'tostring'):
				blob = blob.tostring()

			# write the file
			write_file(f[0], blob)
						
			if verbose:
				webnotes.msgprint('%s updated' % f[0])

"""
Simple Scheduler

This scheduler is used to fire events across multiple databases. A database
master_scheduler is maintained with one event and one log table

Events are added by different databases in the master scheduler using the
`set_event` method and they are executed by the cron.

__main__ will call run

To install:
-----------

python install_lib.py [root] [password] master_scheduler

In cron:
--------

python [path]webnotes/utils/scheduler.py

"""

class Scheduler:
		if hasattr(self,'conn'): return
		
		import webnotes.defs, webnotes.db
		try:
			self.conn = webnotes.db.Database(user='master_scheduler', password=webnotes.defs.db_password)
			self.setup()
			self.connect()
		
	def set(self, event, interval, recurring, db_name=None):
		"""
			Add an event to the Event table in the master scheduler
		"""

		if not db_name:
			import webnotes
			db_name = webnotes.conn.cur_db_name

		self.clear(db_name, event)
		self.conn.sql("""insert into 
			Event (`db_name`, `event`, `interval`, next_execution, recurring) 
			values  (%s, %s, %s, ADDTIME(NOW(), SEC_TO_TIME(%s)), %s)
			""", (webnotes.conn.cur_db_name, event, interval, interval, recurring))

	def get_events(self, db_name=None):
		"""
			Returns list of upcoming events
		"""
			
		return self.conn.sql("select * from Event where db_name=%s", db_name, as_dict=1)

	def get_log(self, db_name=None):
		"""
			Returns log of events
		"""

		return self.conn.sql("select * from EventLog where db_name=%s limit 50", db_name, as_dict=1)

	def clear(self, db_name, event):
		"""
			Clears the event
		"""
		self.conn.sql("delete from Event where `event`=%s and db_name=%s", (event, db_name))

	def execute(self, db_name, event):
		"""
			Executes event in the specifed db
		"""
		import webnotes, webnotes.defs, webnotes.db

		try:
			webnotes.conn = webnotes.db.Database(user=db_name, password=webnotes.defs.db_password)
			webnotes.session = {'user':'Administrator'}

			module = '.'.join(event.split('.')[:-1])
			method = event.split('.')[-1]
		
			exec 'from %s import %s' % (module, method)
		
			webnotes.conn.begin()
			ret = locals()[method]()
			webnotes.conn.commit()
			webnotes.conn.close()
			
			self.log(db_name, event, ret or 'Ok')
			
			self.log(db_name, event, webnotes.getTraceback())

	def log(self, db_name, event, traceback):
		"""
			Log an event error
		"""
		self.conn.sql("insert into `EventLog`(db_name, event, log, executed_on) values (%s, %s, %s, now())", \
			(db_name, event, traceback))
			
		# delete old logs
		self.conn.sql("delete from EventLog where executed_on < subdate(curdate(), interval 30 day)")

	def run(self):
		"""
			Run scheduled (due) events and reset time for recurring events
		"""
		el = self.conn.sql("""select `db_name`, `event`, `recurring`, `interval`
			from `Event` 
			where next_execution < NOW()""", as_dict=1)
		
		for e in el:
			# execute the event
			self.execute(e['db_name'], e['event'])
			
			# if recurring, update next_execution
			if e['recurring']:
				self.conn.sql("update Event set next_execution = addtime(now(), sec_to_time(%s))", e['interval'])
			
			# else clear
			else:
				self.clear(e['db_name'], e['event'])

def set_event(event, interval=60*60*24, recurring=1):
	"""
		Adds an event to the master scheduler
	"""
	return Scheduler().set(event, interval, recurring)

def cancel_event(event):
	"""
		Cancels an event
	"""
	return Scheduler().clear(webnotes.conn.cur_db_name, event)

# to be called from cron
if __name__=='__main__':
	import os,sys

	cgi_bin_path = os.path.sep.join(__file__.split(os.path.sep)[:-3])

	sys.path.append(cgi_bin_path)

	sys.path.append(getattr(webnotes.defs,'modules_path',None))

	sch = Scheduler()
	sch.connect()
	sch.run()
	
#==============================================================================
# SYNC
def reload_doc(module, dt, dn):
	"alias for webnotes.modules.import_module.import_file"
	from webnotes.modules.import_module import import_file

	import_file(module, dt, dn)

#
# get list of doctypes and their last update times
#
def get_doc_list(dt):
	"""
	returns the list of records and their last update times from the table
	if the column _last_update does not exist, it will add it to the table
	"""
	
	module = dt=='Module Def' and 'name' or 'module'
	q = "select %s, name, _last_update from `tab%s`" % (module, dt)
	try:
		return webnotes.conn.sql(q)
		if e.args[0]==1054:
			webnotes.conn.sql("alter table `tab%s` add column _last_update varchar(32)" % dt)
			return webnotes.conn.sql(q)
		elif e.args[0]==1146:
			return []
		else:
			raise e

#
# sync dt
#
def sync_one_doc(d, dt, ts):
	from webnotes.model.db_schema import updatedb
	reload_doc(d[0], dt, d[1])
		
	# update schema(s)
	if dt=='DocType':
		updatedb(d[1])
	webnotes.conn.sql("update `tab%s` set _last_update=%s where name=%s" % (dt, '%s', '%s'), (ts, d[1]))

#
# sync doctypes, mappers and 
#
def sync_meta():

	tl = ['DocType', 'DocType Mapper', 'Module Def']

	for dt in tl:
		dtl = get_doc_list(dt)
				
		for d in filter(lambda x: x[0], dtl):
			try:
				ndt, ndn = dt, d[1]
				if dt == 'DocType':
					ndt, ndn = scrub(dt), scrub(d[1])
					
				mp = get_module_path(scrub(d[0]))
				ts = cint(os.stat(os.path.join(mp, ndt, ndn, ndn + '.txt')).st_mtime)
				
				if d[2] != str(ts):
					sync_one_doc(d, dt, ts)
			except OSError, e:
				pass

def get_module_details(m):
	from export_module import get_module_items
	return {'in_files': get_module_items_from_files(m), \
		'in_system':[[i[0], i[1], get_modified(i[0], i[1])] for i in get_module_items(m)]}

def get_modified(dt, dn):
	try:
		return str(webnotes.conn.sql("select modified from `tab%s` where replace(name,' ','_')=%s" % (dt,'%s'), dn)[0][0])
	except:
		pass

def get_module_items_from_files(m):
	import os, webnotes.defs
	from import_module import listfolders

	items = []
	for item_type in listfolders(os.path.join(webnotes.defs.modules_path, m), 1):
		for item_name in listfolders(os.path.join(webnotes.defs.modules_path, m, item_type), 1):
			# read the file
			file = open(os.path.join(webnotes.defs.modules_path, m, item_type, item_name, item_name)+'.txt','r')
			doclist = eval(file.read())
			file.close()
			
			# append
			items.append([item_type, item_name, doclist[0]['modified']])
			
	return items

def get_last_update_for(mod):
	try:
		return webnotes.conn.sql("select last_updated_date from `tabModule Def` where name=%s", mod)[0][0]
	except:
		return ''

def init_db_login(ac_name, db_name):
	import webnotes.db
	import webnotes.profile
	
	if ac_name:
		webnotes.conn = webnotes.db.Database(ac_name = ac_name)
		webnotes.conn.use(webnotes.conn.user)
	elif db_name:
		webnotes.conn = webnotes.db.Database(user=db_name)
		webnotes.conn.use(db_name)
	else:
		webnotes.conn = webnotes.db.Database(use_default=1)
			
	webnotes.session = {'user':'Administrator'}
	webnotes.user = webnotes.profile.Profile()

# Return module names present in File System
def get_modules_from_filesystem():
	
	modules = listfolders(webnotes.defs.modules_path, 1)
	out = []
	modules.sort()
	modules = filter(lambda x: x!='patches', modules)
	
	for m in modules:
		file = open(os.path.join(webnotes.defs.modules_path, m, 'module.info'), 'r')
		out.append([m, eval(file.read()), get_last_update_for(m), \
			webnotes.conn.exists('Module Def',m) and 'Installed' or 'Not Installed'])
		file.close()

	return out

class Profile:
	"""
	A profile object is created at the beginning of every request with details of the use.
	The global profile object is `webnotes.user`
	"""
	def __init__(self, name=''):
		self.name = name or webnotes.session.get('user')
		self.roles = []

		self.can_create = []
		self.can_read = []
		self.can_write = []
		self.can_get_report = []
		
	def _load_roles(self):
		res = webnotes.conn.sql('select role from tabUserRole where parent = "%s"' % self.name)
		for t in res:
			if t[0]: self.roles.append(t[0])
		if webnotes.session.get('user') == 'Guest':
			self.roles.append('Guest')
		else:
			self.roles.append('All')
			
		return self.roles

	def get_roles(self):
		"""
		get list of roles
		"""
		if self.roles:
			return self.roles
		
		return self._load_roles()
	
	def get_allow_list(self, key):
		"""
	      	Internal - get list of DocType where `key` is allowed. Key is either 'read', 'write' or 'create'
		"""
		conn = webnotes.conn
		roles = self.get_roles()
		return [r[0] for r in conn.sql('SELECT DISTINCT t1.parent FROM `tabDocPerm` t1, tabDocType t2 WHERE t1.`%s`=1 AND t1.parent not like "old_parent:%%" AND t1.parent = t2.name AND IFNULL(t2.istable,0) = 0 AND t1.role in ("%s") order by t1.parent' % (key, '", "'.join(roles)))]
	
	def get_create_list(self):
		"""
		Get list of DocTypes the user can create. Will filter DocTypes tagged with 'not_in_create' and table
		"""
		cl = self.get_allow_list('create')
		no_create_list = [r[0] for r in conn.sql('select name from tabDocType where ifnull(in_create,0)=1 or ifnull(istable,0)=1 or ifnull(issingle,0)=1')]
		self.can_create = filter(lambda x: x not in no_create_list, cl)
		return self.can_create
		
	def get_read_list(self):
		"""
		Get list of DocTypes the user can read
		"""
		self.can_read = list(set(self.get_allow_list('read') + self.get_allow_list('write')))
		return self.can_read
	
	def get_report_list(self):

		# get all tables list
		res = conn.sql('SELECT parent, options from tabDocField where fieldtype="Table"')
		table_types, all_tabletypes = {}, []
		
		# make a dictionary fo all table types
		for t in res: 
			all_tabletypes.append(t[1])
			if not table_types.has_key(t[0]):
				table_types[t[0]] = []
			table_types[t[0]].append(t[1])
	
		no_search_list = [r[0] for r in conn.sql('SELECT name FROM tabDocType WHERE read_only = 1 ORDER BY name')]
		# make the lists
		for f in self.can_read:
			tl = table_types.get(f, None)
			if tl:
				for t in tl:
					if t and (not t in self.can_get_report) and (not t in no_search_list):
						self.can_get_report.append(t)
			
			if f and (not f in self.can_get_report) and (not f in no_search_list): 
				self.can_get_report.append(f)
	
		return self.can_get_report
		
	def get_write_list(self):
		"""
		Get list of DocTypes the user can write
		"""	
		self.can_write = self.get_allow_list('write')
		return self.can_write

	def get_home_page(self):
		"""
		Get the name of the user's home page from the `Control Panel`
		"""
		try:
			hpl = webnotes.conn.sql("select role, home_page from `tabDefault Home Page` where parent='Control Panel' order by idx asc")
			for h in hpl:
				if h[0] in self.get_roles():
					return h[1]
		except:
			pass
		return webnotes.conn.get_value('Control Panel',None,'home_page')

	def get_defaults(self):
		"""
		Get the user's default values based on user and role profile
		"""
		roles = self.get_roles() + [self.name]
		res = webnotes.conn.sql('select defkey, defvalue from `tabDefaultValue` where parent in ("%s")' % '", "'.join(roles))
	
		self.defaults = {'owner': [self.name,]}

			if not self.defaults.has_key(rec[0]): 
				self.defaults[rec[0]] = []
			self.defaults[rec[0]].append(rec[1] or '')

		return self.defaults
		
	def get_hide_tips(self):
		try:
			return webnotes.conn.sql("select hide_tips from tabProfile where name=%s", self.name)[0][0] or 0
		except:
			return 0
		
	def get_random_password(self):
		"""
		Generate a random password
		"""	
		import string
		from random import choice
		
		size = 9
		pwd = ''.join([choice(string.letters + string.digits) for i in range(size)])
		return pwd

	def reset_password(self):
		"""
		Reset the user's password and send an email
		"""
		pwd = self.get_random_password()
		
		# get profile
		profile = webnotes.conn.sql("SELECT name, email, first_name, last_name FROM tabProfile WHERE name=%s OR email=%s",(self.name, self.name))
		
		if not profile:
			raise Exception, "Profile %s not found" % self.name
		
		# update tab Profile
		webnotes.conn.sql("UPDATE tabProfile SET password=password(%s) WHERE name=%s", (pwd, profile[0][0]))
		
		self.send_email("Password Reset", "<p>Dear %s%s,</p><p>your password has been changed to %s</p><p>[Automatically Generated]</p>" % (profile[0][2], (profile[0][3] and (' ' + profile[0][3]) or ''), pwd), profile[0][1])
		
	def send_email(self, subj, mess, email):
		import webnotes.utils.email_lib
		
		webnotes.utils.email_lib.sendmail(email, msg=mess, subject=subj)
	
	# update recent documents
	def update_recent(self, dt, dn):
		"""
		Update the user's `Recent` list with the given `dt` and `dn`
		"""
	
		# get list of child tables, so we know what not to add in the recent list
		child_tables = [t[0] for t in conn.sql('select name from tabDocType where istable = 1')]
		if not (dt in ['Print Format', 'Start Page', 'Event', 'ToDo Item', 'Search Criteria']) and not webnotes.is_testing and not (dt in child_tables):
			r = webnotes.conn.sql("select recent_documents from tabProfile where name=%s", self.name)[0][0] or ''
			new_str = dt+'~~~'+dn + '\n'
			if new_str in r:
				r = r.replace(new_str, '')

			self.recent = new_str + r
			
			if len(self.recent.split('\n')) > 50:
				self.recent = '\n'.join(self.recent.split('\n')[:50])
			
			webnotes.conn.sql("update tabProfile set recent_documents=%s where name=%s", (self.recent, self.name))
			
	def load_profile(self):
		"""
	      	Return a dictionary of user properites to be stored in the session
		"""
		t = webnotes.conn.sql('select email, first_name, last_name, recent_documents from tabProfile where name = %s', self.name)[0]

		d = {}
		d['name'] = self.name
		d['email'] = t[0] or ''
		d['first_name'] = t[1] or ''
		d['last_name'] = t[2] or ''
		d['recent'] = t[3] or ''
		
		d['hide_tips'] = self.get_hide_tips()
		
		d['roles'] = self.get_roles()
		d['defaults'] = self.get_defaults()
		
		d['can_create'] = self.get_create_list()
		d['can_read'] = self.get_read_list()
		d['can_write'] = self.get_write_list()
		d['can_get_report'] = self.get_report_list()
		
		return d
		
	def load_from_session(self, d):
		"""
		Setup the user profile from the dictionary saved in the session (generated by `load_profile`)
		"""
		self.can_create = d['can_create']
		self.can_read = d['can_read']
		self.can_write = d['can_write']
		self.can_get_report = d['can_get_report']

		self.roles = d['roles']
		self.defaults = d['defaults']

def get_user_img():
	if not webnotes.form.getvalue('username'):
		webnotes.response['message'] = 'no_img_m'
		return

	f = webnotes.conn.sql("select file_list from tabProfile where name=%s", webnotes.form.getvalue('username',''))
	if f:
		if f[0][0]:
			lst = f[0][0].split('\n')	
			webnotes.response['message'] = lst[0].split(',')[1]
		else:
			webnotes.response['message'] = 'no_img_m'		
	else:

class FrameworkServer:
	"""
	   Connect to a remote server via HTTP (webservice).
	   
	   * `remote_host` is the the address of the remote server
	   * `path` is the path of the Framework (excluding index.cgi)
	"""
	def __init__(self, remote_host, path, user='', password='', account='', cookies=None, opts=None, https = 0):
		# validate
		if not (remote_host and path):
			raise Exception, "Server address and path necessary"

		if not ((user and password) or (cookies)):
			raise Exception, "Either cookies or user/password necessary"
	
		self.remote_host = remote_host
		self.path = path
		self.cookies = cookies or {}
		self.webservice_method='POST'
		self.account = account
		self.account_id = None
		self.https = https
		self.conn = None

		# login
		if not cookies:
			args = { 'usr': user, 'pwd': password, 'acx': account }
			
			if opts:
				args.update(opts)
			
			res = self.http_get_response('login', args)
		
			ret = res.read()
			try:
				ret = eval(ret)
			except Exception, e:
				webnotes.msgprint(ret)
				raise Exception, e
				
			if ret.get('message') and ret.get('message')!='Logged In':
				raise Exception, ret.get('message')
				
			if ret.get('exc'):
				raise Exception, ret.get('exc')
				
			self._extract_cookies(res)

			self.account_id = self.cookies.get('account_id')
			self.sid = self.cookies.get('sid')
			
			self.login_response = res
			self.login_return = ret

	# -----------------------------------------------------------------------------------------

	def http_get_response(self, method, args):
		"""
		Run a method on the remote server, with the given arguments
		"""
		# get response from remote server
	
		import urllib, urllib2, os

		args['cmd'] = method
		if self.path.startswith('/'): self.path = self.path[1:]
				
		protocol = self.https and 'https://' or 'http://'
		req = urllib2.Request(protocol + os.path.join(self.remote_host, self.path, 'index.cgi'), urllib.urlencode(args))
		for key in self.cookies:
			req.add_header('cookie', '; '.join(['%s=%s' % (key, self.cookies[key]) for key in self.cookies]))
		return urllib2.urlopen(req)

	def _extract_cookies(self, res):
		cookies = Cookie.SimpleCookie()
		cookies.load(res.headers.get('set-cookie'))
		for c in cookies.values():
			self.cookies[c.key] = c.value.rstrip(',')

	def runserverobj(self, doctype, docname, method, arg=''):
		"""
		Returns the response of a remote method called on a system object specified by `doctype` and `docname`
		"""
		res = self.http_get_response('runserverobj', args = {
			'doctype':doctype
			,'docname':docname
			,'method':method
			,'arg':arg
		})
		ret = eval(res.read())
		if ret.get('exc'):
			raise Exception, ret.get('exc')
		return ret
	
	def run_method(self, method, args):
		res = self.http_get_response(method, args)
		return ret

# session_cache.py

# clear cache
# ==================================================

def clear():
	clear_cache()

	webnotes.response['message'] = "Cache Cleared"

def clear_cache(user=''):
	try:
		if user:
			webnotes.conn.sql("delete from __SessionCache where user=%s", user)
		else:
			webnotes.conn.sql("delete from __SessionCache")
			make_cache_table()
		else:
			raise e

# load cache

def get():
	
	# get country
	country = webnotes.session['data'].get('ipinfo', {}).get('countryName', 'Unknown Country')

	# run patches
	try:
		import webnotes.modules.patch
		webnotes.modules.patch.run()
	except ImportError, e: 
		pass # no patches - do nothing

	# check if cache exists
	if not getattr(webnotes.defs,'auto_cache_clear',None):
		cache = load(country)
		if cache:
			return cache
	
	# if not create it
	sd = build()
	dump(sd, country)

	# update profile from cache
	webnotes.session['data']['profile'] = sd['profile']	
		
	return sd

def load(country):
	
	try:
		sd = webnotes.conn.sql("select cache from __SessionCache where user='%s' %s" % (webnotes.session['user'], (country and (" and country='%s'" % country) or '')))
		if sd:
			return eval(sd[0][0])
		else:
		else:
			raise e

# make the cache table
				
def make_cache_table():
	webnotes.conn.sql("create table `__SessionCache` (user VARCHAR(120), country VARCHAR(120), cache LONGTEXT)")

# dump session to cache

def dump(sd, country):
	import webnotes.model.doclist

	if sd.get('docs'):
		sd['docs'] = webnotes.model.doclist.compress(sd['docs'])

	# delete earlier (?)
	webnotes.conn.sql("delete from __SessionCache where user=%s and country=%s", (webnotes.session['user'], country))

	# make new
	webnotes.conn.sql("insert into `__SessionCache` (user, country, cache) VALUES (%s, %s, %s)", (webnotes.session['user'], country, str(sd)))

def get_letter_heads():
	try:
		lh = {}
		ret = webnotes.conn.sql("select name, content from `tabLetter Head` where ifnull(disabled,0)=0")
		for r in ret:
			lh[r[0]] = r[1]
		return lh
			return {}
		else:
			raise Exception, e

# load startup.js and startup.css from the modules/startup folder

def load_startup(cp):
	from webnotes.modules import compress
	
	try: from webnotes.defs import modules_path
	except ImportError: return
	
	import os

	try:
		cp.startup_code = compress.get_js_code(os.path.join(modules_path, 'startup', 'startup'))
		startup_css = open(os.path.join(modules_path, 'startup', 'startup.css'), 'r')
		cp.startup_css = startup_css.read()
		startup_css.close()
		if e.args[0]!=2: # no startup module!
			raise e

# build it

def build():
	sd = {}

	import webnotes.model.doc
	import webnotes.model.doctype
	import webnotes.widgets.page
	import webnotes.widgets.menus
	
	sd['profile'] = webnotes.user.load_profile()

	doclist = []
	doclist += webnotes.model.doc.get('Control Panel')
	cp = doclist[0]
	load_startup(cp)
	
	doclist += webnotes.model.doctype.get('Event')
	doclist += webnotes.model.doctype.get('Search Criteria')
	home_page = webnotes.user.get_home_page()

	if home_page:
		doclist += webnotes.widgets.page.get(home_page)

	sd['account_name'] = cp.account_id or ''
	sd['sysdefaults'] = webnotes.utils.get_defaults()
	sd['n_online'] = int(sql("SELECT COUNT(DISTINCT user) FROM tabSessions")[0][0] or 0)
	sd['docs'] = doclist
	sd['letter_heads'] = get_letter_heads()
	sd['home_page'] = home_page or ''
	sd['start_items'] = webnotes.widgets.menus.get_menu_items()
	if webnotes.session['data'].get('ipinfo'):
		sd['ipinfo'] = webnotes.session['data']['ipinfo']
		
	webnotes.session['data']['profile'] = sd['profile']
	sd['dt_labels'] = webnotes.model.get_dt_labels()
	
	return sd

# model __init__.py

def check_if_doc_is_linked(dt, dn):
	"""
		Raises excption if the given doc(dt, dn) is linked in another record.
	"""

	ll = get_link_fields(dt)
	for l in ll:
		link_dt, link_field = l
		issingle = sql("select issingle from tabDocType where name = '%s'" % link_dt)

		# no such doctype (?)
		if not issingle: continue
		
		if issingle[0][0]:
			item = sql("select doctype from `tabSingles` where field='%s' and value = '%s' and doctype = '%s' " % (link_field, dn, l[0]))
			if item:
				webnotes.msgprint("Cannot delete %s <b>%s</b> because it is linked in <b>%s</b>" % (dt, dn, item[0][0]), raise_exception=1)
			
		else:
			item = sql("select name from `tab%s` where `%s`='%s' and docstatus!=2 limit 1" % (link_dt, link_field, dn))
			if item:
				webnotes.msgprint("Cannot delete %s <b>%s</b> because it is linked in %s <b>%s</b>" % (dt, dn, link_dt, item[0][0]), raise_exception=1)

def delete_doc(doctype=None, name=None, doclist = None):
	"""
		Deletes a doc(dt, dn) and validates if it is not submitted and not linked in a live record
	"""
	import webnotes.model.meta

	# get from form
	if not doctype:
		doctype = webnotes.form_dict.get('dt')
		name = webnotes.form_dict.get('dn')
	
		webnotes.msgprint('Nothing to delete!', raise_exception =1)

	# already deleted..?
	if not webnotes.conn.exists(doctype, name):
		return

	tablefields = webnotes.model.meta.get_table_fields(doctype)
	
	# check if submitted
	d = webnotes.conn.sql("select docstatus from `tab%s` where name=%s" % (doctype, '%s'), name)
	if d and int(d[0][0]) == 1:
		webnotes.msgprint("Submitted Record '%s' '%s' cannot be deleted" % (doctype, name))
	
	# call on_trash if required
	if doclist:
		obj = get_obj(doclist=doclist)
	else:
		obj = get_obj(doctype, name)

	if hasattr(obj,'on_trash'):
		obj.on_trash()
	
	# check if links exist
	check_if_doc_is_linked(doctype, name)
	
	try:
		webnotes.conn.sql("delete from `tab%s` where name='%s' limit 1" % (doctype, name))
		for t in tablefields:
			webnotes.conn.sql("delete from `tab%s` where parent = %s" % (t[0], '%s'), name)
		if e.args[0]==1451:
			webnotes.msgprint("Cannot delete %s '%s' as it is referenced in another record. You must delete the referred record first" % (doctype, name))
		
		raise e
	
	return 'okay'

# new feature added 9-Jun-10 to allow doctypes to have labels 
def get_dt_labels():
	d = {}
	try:
		res = webnotes.conn.sql("select name, dt_label from `tabDocType Label`")
	except:
		return {}
		
		d[r[0]] = r[1]
	
	return d

def get_search_criteria(dt):
	# load search criteria for reports (all)
	dl = []
	try: # bc
		sc_list = webnotes.conn.sql("select name from `tabSearch Criteria` where doc_type = '%s' or parent_doc_type = '%s' and (disabled!=1 OR disabled IS NULL)" % (dt, dt))
		for sc in sc_list:
			dl += webnotes.model.doc.get('Search Criteria', sc[0])
		pass # no search criteria
	return dl

# Rename Doc

def get_link_fields(dt):
	"""
		Returns linked fields for dt as a tuple of (linked_doctype, linked_field)
	"""
	return webnotes.conn.sql("select parent, fieldname from tabDocField where parent not like 'old%%' and ((options = '%s' and fieldtype='Link') or (options = 'link:%s' and fieldtype='Select'))" % (dt, dt))

def rename(dt, old, new, is_doctype = 0):
	"""
		Renames a doc(dt, old) to doc(dt, new) and updates all linked fields of type "Link" or "Select" with "link:"
	"""

	# rename doc
	sql("update `tab%s` set name='%s' where name='%s'" % (dt, new, old))

	# get child docs
	ct = sql("select options from tabDocField where parent = '%s' and fieldtype='Table'" % dt)
	for c in ct:
		sql("update `tab%s` set parent='%s' where parent='%s'" % (c[0], new, old))

	# get links (link / select)
	for l in ll:
		is_single = sql("select issingle from tabDocType where name = '%s'" % l[0])
		is_single = is_single and webnotes.utils.cint(is_single[0][0]) or 0
		if is_single:
			sql("update `tabSingles` set value='%s' where field='%s' and value = '%s' and doctype = '%s' " % (new, l[1], old, l[0]))
		else:
			sql("update `tab%s` set `%s`='%s' where `%s`='%s'" % (l[0], l[1], new, l[1], old))

	# doctype
	if is_doctype:
		sql("RENAME TABLE `tab%s` TO `tab%s`" % (old, new))

		# get child docs (update parenttype)
		ct = sql("select options from tabDocField where parent = '%s' and fieldtype='Table'" % new)
		for c in ct:
			sql("update `tab%s` set parenttype='%s' where parenttype='%s'" % (c[0], new, old))

	"""
		Clears temporary records that have been deleted
	"""

	for t in tl:
		
	webnotes.msgprint("%s records deleted" % str(int(total_deleted)))
	
# Make Table Copy

def copytables(srctype, src, srcfield, tartype, tar, tarfield, srcfields, tarfields=[]):

	if not tarfields: 
		tarfields = srcfields
	l = []
	data = webnotes.model.doc.getchildren(src.name, srctype, srcfield)
	for d in data:
		newrow = webnotes.model.doc.addchild(tar, tarfield, tartype, local = 1)
		newrow.idx = d.idx
	
		for i in range(len(srcfields)):
			newrow.fields[tarfields[i]] = d.fields[srcfields[i]]
			
		l.append(newrow)
	return l

# DB Exists

	return webnotes.conn.exists(dt, dn)

def import_docs(docs = []):
	import webnotes.model.code

	doc_list = {}
	created_docs = []
	already_exists = []

	out, tmp ="", ""

		cur_doc = Document(fielddata = d)
		if not cur_doc.parent in already_exists: # parent should not exist
			try:
				cur_doc.save(1)
				out += "Created: " + cur_doc.name + "\n"
				created_docs.append(cur_doc)

				# make in groups
				if cur_doc.parent:
					if not doc_list.has_key(cur_doc.parent):
						doc_list[cur_doc.parent] = []
					doc_list[cur_doc.parent].append(cur_doc)

				out += "Creation Warning/Error: " + cur_doc.name + " :"+ str(e) + "\n"
				already_exists.append(cur_doc.name)

	# Run scripts for main docs
	for m in created_docs:
		if doc_list.has_key(m.name):
			tmp = webnotes.model.code.run_server_obj(webnotes.model.code.get_server_obj(m, doc_list.get(m.name, [])),'on_update')

			# update database (in case of DocType)
			if m.doctype=='DocType':
				import webnotes.model.doctype
				try: webnotes.model.doctype.update_doctype(doc_list.get(m.name, []))
				except: pass			
			out += 'Executed: '+ str(m.name) + ', Err:' + str(tmp) + "\n"

	return out

#======================================================================================================================================

flt = webnotes.utils.flt
cint = webnotes.utils.cint
cstr = webnotes.utils.cstr

class CSVImport:
		self.msg = []
		self.csv_data = None
		self.import_date_format = None

	def validate_doctype(self, dt_list):
		cl, tables, self.dt_list, self.prompt_autoname_flag = 0, [t[0] for t in sql("show tables")], [], 0
		self.msg.append('<p><b>Identifying Documents</b></p>')
		dtd = sql("select name, istable, autoname from `tabDocType` where name = '%s' " % dt_list[0])
		if dtd and dtd[0][0]:
			self.msg.append('<div style="color: GREEN">Identified Document: ' + dt_list[0] + '</div>')
			self.dt_list.append(dt_list[0])
			if dtd[0][2] and 'Prompt' in dtd[0][2]: self.prompt_autoname_flag = 1
			if flt(dtd[0][1]):
				res1 = sql("select parent, fieldname from tabDocField where options='%s' and fieldtype='Table' and docstatus!=2" % self.dt_list[0])
				if res1 and res1[0][0] == dt_list[1]:
					self.msg.append('<div style="color: GREEN">Identified Document: ' + dt_list[1] + '</div>')
					self.dt_list.append(dt_list[1])
				else : 
					self.msg.append('<div style="color:RED"> Error: At Row 1, Column 2 => %s is not a valid Document </div>' % dt_list[1])
					self.validate_success = 0

				if res1 and res1[0][1] == dt_list[2]:
					self.msg.append('<div style="color: GREEN" >Identified Document Fieldname: ' + dt_list[2] + '</div>')
					self.dt_list.append(dt_list[2])
				else :
					self.msg.append('<div style="color:RED"> Error: At Row 1, Column 3 => %s is not a valid Fieldname </div>' % dt_list[2])
			elif dt_list[1]:
				self.msg.append('<div style="color:RED"> Error: At Row 1, Column 1 => %s is not a Table.	</div>' % dt_list[0])
				self.validate_success = 0
		else:
			self.msg.append('<div style="color:RED"> Error: At Row 1, Column 1 => %s is not a valid Document </div>' % dt_list[0])
			self.validate_success = 0

	def validate_fields(self, lb_list):
		self.msg.append('<p><b>Checking fieldnames for %s</b></p>' % self.dt_list[0])
		if len(self.dt_list) > 1 and self.overwrite:
			self.msg.append('<div style="color:RED"> Error: Overwrite is not possible for Document %s </div>' % self.dt_list[0])
			return
		
		elif self.overwrite and 'Name' != lb_list[0]:
			self.msg.append('<div style="color:RED"> Error : At Row 4 and Column 1: To Overwrite fieldname should be Name </div>')
			return
		# labelnames
		res = self.validate_success and [d[0] for d in sql("select label from tabDocField where parent='%s' and docstatus!=2 and ifnull(hidden,'') in ('',0)" % self.dt_list[0])] or []

		if len(self.dt_list) > 1 and self.dt_list[1]:
			self.fields.append('parent')
			lb_list.pop(lb_list.index(self.dt_list[1]))

		dtd = sql("select autoname from `tabDocType` where name = '%s' " % self.dt_list[0])[0][0]
		if self.prompt_autoname_flag or self.overwrite:
			self.fields.append('name')
			res.append('Name')
			lb_list.pop(lb_list.index('Name'))

		cl = 1
		for l in lb_list:
			try:
				if l:
					if not (l in res):
						self.msg.append('<div style="color: RED">Error : At Row 4 and Column %s Field %s is not present in %s</div>' % (cl, l, self.dt_list[0]))
						self.validate_success = 0
						# this condition is for child doctype
				
					else:	self.fields.append(sql("select fieldname from tabDocField where parent ='%s' and label = '%s' and ifnull(fieldname,'') !='' " % (self.dt_list[0], l))[0][0] or '')
				self.msg.append('<div style="color: RED"> At Row 4 and Column %s : =>ERROR: %s </div>' % ( cl, e))
			cl = cl + 1
	
		if not self.overwrite:
			# get_reqd_fields
			reqd_list = [d[0] for d in sql("select label from `tabDocField` where parent = '%s' and ifnull(reqd,'') not in ('', 0)  and docstatus !=2" % self.dt_list[0])	if d[0] not in lb_list] or []
		
			# Check if Reqd field not present in self.fields
			if reqd_list:
				self.msg.append('<div style="color: RED"> Error : At Row 4 Mandatory Fields %s of Document %s are Required. </div>' %(reqd_list , self.dt_list[0]))
		if self.validate_success:
			self.msg.append('<div style="color: GREEN">Fields OK for %s</div>' % self.dt_list[0])
					
	def validate_headers(self):
		self.validate_doctype(self.doctype_data)
			self.validate_fields(self.labels)

	# Date parsing
	# --------------------------------------------------------------------
	def parse_date(self, r, c, d):
		out = ''
		try:
			if self.import_date_format=='yyyy-mm-dd':
				tmpd = d.split('-')
			
				if len(tmpd)==3:
					out = tmpd[0] + '-'+tmpd[1] + '-' + tmpd[2]
		
			elif d and self.import_date_format=='dd-mm-yyyy':
					
					out = tmpd[2]+'-'+tmpd[1]+'-'+tmpd[0]
			
			elif d and self.import_date_format=='mm/dd/yyyy':
				tmpd = d.split('/')
			
					out = tmpd[2]+'-'+tmpd[0]+'-'+tmpd[1]
				
			elif d and self.import_date_format=='mm/dd/yy':
			
					out = '20'+tmpd[2]+'-'+tmpd[0]+'-'+tmpd[1]
		
			elif d and self.import_date_format=='dd/mm/yyyy':

			elif d and self.import_date_format=='dd/mm/yy':
					out = '20'+tmpd[2]+'-'+tmpd[1]+'-'+tmpd[0]

			if len(tmpd) != 3:
				self.msg.append('<div style="color: RED"> At Row %s and Column %s : => Date Format selected as %s does not match with Date Format in File</div>' % (r, c, str(self.import_date_format)))
			else:
			
				import datetime
				dt = out.split('-')
				datetime.date(int(dt[0]),int(dt[1]), int(dt[2]))
			self.msg.append('<div style="color: RED"> At Row %s and Column %s : =>ERROR: %s </div>' % (r, c, e))
		self.msg.append(out)
		return out

	def check_select_link_data(self, r, c, f, d, s = '', l = ''):
		options = ''
		try:
			if d and f:
				dt = sql("select options, label from `tabDocField` where fieldname ='%s' and parent = '%s' " % (f, self.dt_list[0]))
				dt, lbl = (dt and dt[0][0] and dt[0][0].strip() or None), dt and dt[0][1].strip()
				if dt:
					options = l and dt and [n[0] for n in sql("select name from `tab%s` " % (('link:' in dt and dt[5:]) or dt))] or s and dt.split('\n') or ''
					if options and d not in options :
						msg = '<div style="color: RED">At Row ' + str(r) + ' and Column ' + str(c)+ ' : => Data "' + str(d) + '" in field ['+ str(lbl) +'] Not Found in '
						msg = msg.__add__( s and  str( 'Select Options [' +str(dt.replace('\n', ',')) +']' ) or str('Master ' + str('link:' in dt and dt[5:] or dt)))
						msg = msg.__add__('</div>\n')
						self.msg.append(msg)

			self.msg.append('<div style="color: RED"> ERROR: %s </div>' % (str(webnotes.utils.getTraceback())))
		return d

	def get_field_type_list(self):
		# get_date_fields
		date_list = [d[0] for d in sql("select fieldname from `tabDocField` where parent = '%s' and fieldtype='Date' and docstatus !=2" % self.dt_list[0])]

		# get_link_fields
		link_list = [d[0] for d in sql("select fieldname from `tabDocField` where parent = '%s' and ((fieldtype='Link' and ifnull(options,'') != '') or (fieldtype='Select' and ifnull(options,'') like '%%link:%%')) and docstatus !=2 " % self.dt_list[0])]

		# get_select_fileds
		select_list = [d[0] for d in sql("select fieldname from `tabDocField` where parent = '%s' and fieldtype='Select' and ifnull(options,'') not like '%%link:%%' and docstatus !=2" % self.dt_list[0])]
		
		# get_reqd_fields
		reqd_list = self.overwrite and ['name'] or [d[0] for d in sql("select fieldname from `tabDocField` where parent = '%s' and ifnull(reqd,'') not in ('', 0)  and docstatus !=2" % self.dt_list[0])]

		if len(self.dt_list)> 1 and 'parent' not in reqd_list: reqd_list.append('parent')

		if self.prompt_autoname_flag and 'name' not in reqd_list: reqd_list.append('name')
			 
		return date_list, link_list, select_list, reqd_list

	def validate_data(self):
		self.msg.append('<p><b>Checking Data for %s</b></p>' % self.dt_list[0])
		date_list, link_list, select_list, reqd_list = self.get_field_type_list()

		# load data
		row = 5
		for d in self.data:
			self.validate_success, fd, col = 1, {}, 1
			self.msg.append('<p><b>Checking Row %s </b></p>' % (row))
			for i in range(len(d)):
				if i < len(self.fields):
					f = self.fields[i]
					try:
						# Check Reqd Fields
						if (f in reqd_list) and not d[i]:
							self.msg.append('<div style="color: RED">Error: At Row %s and Column %s, Field %s is Mandatory.</div>' % (row, col, f))
							self.validate_success = 0
					
						# Check Date Fields
						if d[i] and f and f in date_list : fd[f] = self.parse_date(row, col, d[i])
	
						# Check Link Fields
						elif d[i] and f in link_list:
							fd[f] = self.check_select_link_data(row, col, f, d[i], l='Link')
						
						# Check Select Fields
						elif d[i] and f in select_list:
							fd[f] = self.check_select_link_data(row, col, f, d[i], s= 'Select')
												
						# Need To Perform Check For Other Data Type Too	
						else:	fd[f] = d[i]

					except Exception:
						self.msg.append('<div style="color: RED"> ERROR: %sData:%s and %s and %s and %s</div>' % (str(webnotes.utils.getTraceback()) + '\n', str(d), str(f), str(date_list), str(link_list)))
				elif d[i]:
					self.msg.append('<div style="color: RED">At Row %s and Column %s</div>' % (row,col))
					self.msg.append('<div style="color: ORANGE">Ignored</div>')
				col = col + 1 
			if self.validate_success:
				self.msg.append('<div style="color: GREEN">At Row %s and Column %s, Data Verification Completed </div>' % (row,col))
				self.update_data(fd,row)
			row = row + 1

	def update_data(self, fd, row):
		# load metadata
		cur_doc = Document(fielddata = fd)
		cur_doc.doctype, cur_doc.parenttype, cur_doc.parentfield = self.dt_list[0], len(self.dt_list) > 1 and self.dt_list[1] or '', len(self.dt_list) > 1 and self.dt_list[2] or ''
		obj = ''
		# save the document
		try:
			if webnotes.conn.in_transaction:
				sql("COMMIT")
			sql("START TRANSACTION")
			if cur_doc.name and webnotes.conn.exists(self.dt_list[0], cur_doc.name):
				if self.overwrite:
					cur_doc.save()
					obj = webnotes.model.code.get_obj(cur_doc.parent and cur_doc.parent_type or cur_doc.doctype, cur_doc.parent or cur_doc.name, with_children = 1)
					self.msg.append('<div style="color: ORANGE">Row %s => Over-written: %s</div>' % (row, cur_doc.name))
				else:
					self.msg.append('<div style="color: ORANGE">Row %s => Ignored: %s</div>' % (row, cur_doc.name))
			else:
				if cur_doc.parent and webnotes.conn.exists(cur_doc.parenttype, cur_doc.parent) or not cur_doc.parent:
					cur_doc.save(1)
					obj = webnotes.model.code.get_obj(cur_doc.parent and cur_doc.parenttype or cur_doc.doctype, cur_doc.parent or cur_doc.name, with_children = 1)
					self.msg.append('<div style="color: GREEN">Row %s => Created: %s</div>' % (row, cur_doc.name))
				else: 
					self.msg.append('<div style="color: RED">Row %s => Invalid %s : %s</div>' % (row, cur_doc.parenttype, cur_doc.parent))
		except Exception:
			self.msg.append('<div style="color: RED"> Validation: %s</div>' % str(webnotes.utils.getTraceback()))
		try:
			if obj:
				if hasattr(obj, 'validate') : obj.validate()
				if hasattr(obj, 'on_update') : obj.on_update()
				if hasattr(obj, 'on_submit') : obj.on_submit()
			sql("COMMIT")

			sql("ROLLBACK")
			self.msg.append('<div style="color: RED"> Validation: %s</div>' % str(webnotes.message_log[-1:]))
			
	# do import
	def import_csv(self, csv_data, import_date_format = 'yyyy-mm-dd', overwrite = 0):
		import csv
		self.validate_success, self.csv_data = 1, self.convert_csv_data_into_list(csv.reader(csv_data.splitlines()))
		self.import_date_format, self.overwrite = import_date_format, overwrite
		if len(self.csv_data) > 4:
			
			self.doctype_data, self.labels, self.data = self.csv_data[0][:4], self.csv_data[3], self.csv_data[4:]
			self.fields = []
		
			import webnotes.model.code
			from webnotes.model.doc import Document
			sql = webnotes.conn.sql
	        
			self.validate_headers()
				self.validate_data()
		else:
			self.msg.append('<p><b>No data entered in file.</b></p>')
		return '\n'.join(self.msg)

	def convert_csv_data_into_list(self,csv_data):
		st_list = []
		for s in csv_data:
			st_list.append([d.strip() for d in s])
		return st_list

# Get Template method
# -----------------------------------------------------------------

def get_template():

	from webnotes.utils import getCSVelement
	
	# get form values
	dt = form.getvalue('dt')
	overwrite = cint(form.getvalue('overwrite')) or 0

	pt, pf = '', ''
	tmp_lbl, tmp_ml = [],[]
	
	# is table?
	dtd = sql("select istable, autoname from tabDocType where name='%s'" % dt)
	if dtd and dtd[0][0]:
		res1 = sql("select parent, fieldname from tabDocField where options='%s' and fieldtype='Table' and docstatus!=2" % dt)
		if res1:
			pt, pf = res1[0][0], res1[0][1]

	# line 1
	dset = []
	if pt and pf:
		lbl, ml = [pt], ['[Mandatory]']
		line1 = '%s,%s,%s' % (getCSVelement(dt), getCSVelement(pt), getCSVelement(pf))
		line2 = ',,,,,,Please fill valid %(p)s No in %(p)s column.' % {'p':getCSVelement(pt)}
	else:
		if dtd[0][1]=='Prompt' or overwrite:
			lbl, ml= ['Name'], ['[Mandatory][Special Characters are not allowed]']
		else:
			lbl, ml= [], []
		line1 = '%s' % getCSVelement(dt)
		line2 = (overwrite and ',,,,,,Please fill valid %(d)s No in %(n)s' % {'d':dt,'n': 'Name'}) or ',,'

	# Help on Line 
	line1 = line1 + ',,,Please fill columns which are Mandatory., Please do not modify the structure'
	
	# fieldnames
	res = sql("select fieldname, fieldtype, label, reqd, hidden from tabDocField where parent='%s' and docstatus!=2" % dt)

		# restrict trash_reason field, hidden and required fields 
		if not r[1] in webnotes.model.no_value_fields and r[0] != 'trash_reason' and not r[4] and not r[3]:
			tmp_lbl.append(getCSVelement(r[2]))
			tmp_ml.append('')
		# restrict trash_reason field and hidden fields and add Mandatory indicator for required fields
		elif not r[1] in webnotes.model.no_value_fields and r[0] != 'trash_reason' and not r[4] and r[3]:
			lbl.append(getCSVelement(r[2]))
			ml.append(getCSVelement('[Mandatory]'))
	
	dset.append(line1)
	dset.append(line2)
	dset.append(','.join(ml + tmp_ml))
	dset.append(','.join(lbl + tmp_lbl))
	
	txt = '\n'.join(dset)
	webnotes.response['result'] = txt
	webnotes.response['type'] = 'csv'
	webnotes.response['doctype'] = dt

form = webnotes.form

from webnotes.utils.email_lib import get_footer
from webnotes.utils.email_lib.send import EMail

class FormEmail:
	"""
		Represents an email sent from a Form
	"""
		"""
			Get paramteres from the cgi form object
		"""
		self.__dict__.update(webnotes.form_dict)		

		self.recipients = None
		if self.sendto:
			self.recipients = self.sendto.replace(';', ',')
			self.recipients = self.recipients.split(',')

	def update_contacts(self):
		"""
			Add new email contact to database
		"""

		for r in self.recipients:
			r = r.strip()
			try:
				if not webnotes.conn.sql("select email_id from tabContact where email_id=%s", r):
					d = Document('Contact')
					d.email_id = r
					d.save(1)
				if e.args[0]==1146: pass # no table
				else: raise e
	
	def make_full_links(self):
		"""
			Adds server name the relative links, so that images etc can be seen correctly
		"""
		# only domain
		if not self.__dict__.get('full_domain'):
			return
			
		def make_full_link(match):
			import os
			link = match.group('name')
			if not link.startswith('http'):
				link = os.path.join(self.full_domain, link)
			return 'src="%s"' % link

		import re
		p = re.compile('src[ ]*=[ ]*" (?P<name> [^"]*) "', re.VERBOSE)
		self.body = p.sub(make_full_link, self.body)

		p = re.compile("src[ ]*=[ ]*' (?P<name> [^']*) '", re.VERBOSE)

	def get_form_link(self):
		"""
			Returns publicly accessible form link
		"""
		public_domain = webnotes.conn.get_value('Control Panel', None, 'public_domain')
		from webnotes.utils.encrypt import encrypt

		if not public_domain:
			return ''

		args = {
			'dt': self.dt, 
			'dn':self.dn, 
			'acx': webnotes.conn.get_value('Control Panel', None, 'account_id'),
			'server': public_domain,
			'akey': encrypt(self.dn)
		}
		return '<div>If you are unable to view the form below <a href="http://%(server)s/index.cgi?page=Form/%(dt)s/%(dn)s&acx=%(acx)s&akey=%(akey)s">click here to see it in your browser</div>' % args

	def set_attachments(self):
		"""
			Set attachments to the email from the form
		"""
		al = []
		try:
			al = webnotes.conn.sql('select file_list from `tab%s` where name="%s"' % (form.getvalue('dt'), form.getvalue('dn')))
			if al:
				al = (al[0][0] or '').split('\n')
				pass # no attachments in single types!
			else:
		return al
	
	def build_message(self):
		"""
			Builds the message object
		"""

		self.email = EMail(self.sendfrom, self.recipients, self.subject, alternative = 1)

		from webnotes.utils.email_lib.html2text import html2text

		self.make_full_links()

		# message
		if not self.__dict__.get('message'):
			self.message = 'Please find attached %s: %s\n' % (self.dt, self.dn)

		html_message = text_message = self.message.replace('\n','<br>')
		
		# separator
		html_message += '<div style="margin:17px 0px; border-bottom:1px solid #AAA"></div>'

		# form itself (only in the html message)
		html_message += self.body

		# form link
		html_message += self.get_form_link()
		text_message += self.get_form_link()

		# footer
		footer = get_footer()
		if footer: 
			html_message += footer
			text_message += footer

		# message as text
		self.email.set_text(html2text(text_message))
		self.email.set_html(html_message)
					
	def send(self):
		"""
			Send the form with html attachment
		"""

		if not self.recipients:
			webnotes.msgprint('No one to send to!')
			return
			
		self.build_message()
		
		# print format (as attachment also - for text-only clients)
		self.email.add_attachment(self.dn.replace(' ','').replace('/','-') + '.html', self.body)

		# attachments
		# self.with_attachments comes from http form variables
		# i.e. with_attachments=1
		if cint(self.with_attachments):
			for a in self.set_attachments():
				a and self.email.attach_file(a.split(',')[0])

		# cc
		if self.cc:
			self.email.cc = [self.cc]
		
		self.email.send(send_now=1)
		webnotes.msgprint('Sent')

class DocType:
	def __init__(self, d, dl):
		self.doc, self.doclist = d,dl

	# export
	def on_update(self):
		
		if hasattr(webnotes.defs, 'developer_mode') and webnotes.defs.developer_mode:
			from webnotes.modules.export_module import export_to_files	
			from webnotes.modules import get_module_path, scurb
			import os
			
			export_to_files(record_list=[['Stylesheet', self.doc.name]])

			file = open(os.path.join(get_module_path(self.doc.module), 'Stylesheet', scrub(self.doc.name), scrub(self.doc.name) + '.html'), 'w')
			file.write(self.doc.content)
			file.close()	

"""
Simple Caching:

Stores key-value pairs in database and enables simple caching

get_item(key).get() returns the cached value if not expired (else returns null)
get_item(key).set(interval = 60000) sets a value to cache, expiring after x seconds
get_item(key).clear() clears an old value
setup() sets up cache
"""

class CacheItem:
	def __init__(self, key):
		"""create a new cache"""
		self.key = key
	
	def get(self):
		"""get value"""
		try:
			return webnotes.conn.sql("select `value` from __CacheItem where `key`=%s and expires_on > NOW()", self.key)[0][0]
	
	def set(self, value, interval=6000):
		"""set a new value, with interval"""
		try:
			self.clear()
			webnotes.conn.sql("""INSERT INTO 
					__CacheItem (`key`, `value`, expires_on) 
				VALUES 
					(%s, %s, addtime(now(), sec_to_time(%s)))
				""", (self.key, str(value), interval))
			if e.args[0]==1146: 
				setup()
				self.set(value, interval)
			else: raise e
	
	def clear(self):
		"""clear the item"""
		webnotes.conn.sql("delete from __CacheItem where `key`=%s", self.key)

	webnotes.conn.sql("""create table __CacheItem(
		`key` VARCHAR(180) NOT NULL PRIMARY KEY,
		`value` TEXT,
		`expires_on` TIMESTAMP
		)""")

def get_item(key):
	"""returns get CacheItem object"""
	return CacheItem(key)
	pass

# Search

# this is called when a new doctype is setup for search - to set the filters
def getsearchfields():

	sf = webnotes.conn.sql("select search_fields from tabDocType where name=%s", webnotes.form.getvalue("doctype"))
	sf = sf and sf[0][0] or ''
	sf = [s.strip() for s in sf.split(',')]
	if sf and sf[0]:
		res =  webnotes.conn.sql("select fieldname, label, fieldtype, options from tabDocField where parent='%s' and fieldname in (%s)" % (webnotes.form.getvalue("doctype","_NA"), '"'+'","'.join(sf)+'"'))
	else:
		res = []

			dt = r[3][5:]
			ol = webnotes.conn.sql("select name from `tab%s` where docstatus!=2 order by name asc" % dt)
			r[3] = '\n'.join([''] + [o[0] for o in ol])

	webnotes.response['searchfields'] = [['name', 'ID', 'Data', '']] + res

def make_query(fields, dt, key, txt, start, length):
	return  """SELECT %(fields)s 
		FROM `tab%(dt)s` 
		WHERE `tab%(dt)s`.`%(key)s` LIKE '%(txt)s' AND `tab%(dt)s`.docstatus != 2
		ORDER BY `tab%(dt)s`.`%(key)s` 
		DESC LIMIT %(start)s, %(len)s """ % {
			'fields': fields,
			'dt': dt,
			'key': key,
			'txt': txt + '%',
			'start': start, 
			'len': length
		}

def get_std_fields_list(dt, key):
	# get additional search fields
	sflist = webnotes.conn.sql("select search_fields from tabDocType where name = '%s'" % dt)
	sflist = sflist and sflist[0][0] and sflist[0][0].split(',') or []

	sflist = ['name'] + sflist
	if not key in sflist:
		sflist = sflist + [key]

	return ['`tab%s`.`%s`' % (dt, f.strip()) for f in sflist]

def build_for_autosuggest(res):
	
	results = []
		info = ''
		if len(r) > 1:
			info = ','.join([cstr(t) for t in r[1:]])
			if len(info) > 30:
				info = info[:30] + '...'
				
		results.append({'id':r[0], 'value':r[0], 'info':info})
	return results
	
def scrub_custom_query(query, key, txt):
	if '%(key)s' in query:
		query = query.replace('%(key)s', key)
	if '%s' in query:
		query = query.replace('%s', ((txt or '') + '%'))
	return query

# this is called by the Link Field
def search_link():
	import webnotes.widgets.query_builder

	txt = webnotes.form.getvalue('txt')
	dt = webnotes.form.getvalue('dt')
	query = webnotes.form.getvalue('query')
	
	if query:
		res = webnotes.conn.sql(scrub_custom_query(query, 'name', txt))
	else:
		q = make_query(', '.join(get_std_fields_list(dt, 'name')), dt, 'name', txt, '0', '10')
		res = webnotes.widgets.query_builder.runquery(q, ret=1)

	# make output
	webnotes.response['results'] = build_for_autosuggest(res)

# this is called by the search box
def search_widget():

	dt = webnotes.form.getvalue('doctype')
	txt = webnotes.form.getvalue('txt') or ''
	key = webnotes.form.getvalue('searchfield') or 'name' # key field
	user_query = webnotes.form.getvalue('query') or ''

	if user_query:
		query = scrub_custom_query(user_query, key, txt)
	else:
		query = make_query(', '.join(get_std_fields_list(dt, key)), dt, key, txt, webnotes.form.getvalue('start') or 0, webnotes.form.getvalue('page_len') or 50)
	
	webnotes.widgets.query_builder.runquery(query)

"""
 DocType module
 ==============
 
 This module has the DocType class that represents a "DocType" as metadata.
 This is usually called by the form builder or report builder.

 Key functions:
	* manage cache - read / write
	* merge client-side scripts
	* update properties from the modules .txt files

 Cache management:
	* Cache is stored in __DocTypeCache
"""

import webnotes.model.doclist

from webnotes.utils import cstr

class _DocType:
	"""
	   The _DocType object is created internally using the module's `get` method.
	"""
	
	# is cache modified ?
	# =================================================================
	
	def is_modified(self):
		"""
			Returns true if modified
		"""		
		try:
			# doctype modified
			modified = webnotes.conn.sql("select modified from tabDocType where name=%s", self.name)[0][0]
	
			# cache modified
			cache_modified = webnotes.conn.sql("SELECT modified from `__DocTypeCache` where name='%s'" % self.name)[0][0]
			
		except IndexError, e:
			return 1

		return cache_modified != modified

	# write to cache

	def _update_cache(self, doclist):
		import zlib

		if webnotes.conn.sql("SELECT name FROM __DocTypeCache WHERE name=%s", self.name):
			webnotes.conn.sql("UPDATE `__DocTypeCache` SET `modified`=%s, `content`=%s WHERE name=%s", (doclist[0].modified, zlib.compress(str([d.fields for d in doclist]),2), self.name))
		else:
			webnotes.conn.sql("INSERT INTO `__DocTypeCache` (`name`, `modified`, `content`) VALUES (%s, %s, %s)" , (self.name, doclist[0].modified, zlib.compress(str([d.fields for d in doclist]))))

	# read from cache

	def _load_from_cache(self):
	
		doclist = eval(zlib.decompress(webnotes.conn.sql("SELECT content from `__DocTypeCache` where name=%s", self.name)[0][0]))
		return [webnotes.model.doc.Document(fielddata = d) for d in doclist]

	# load options for "link:" type 'Select' fields
			
	def _load_select_options(self, doclist):
		for d in doclist:
			if d.doctype=='DocField' and d.fieldtype=='Select' and d.options and d.options[:5].lower()=='link:':
				op = d.options.split('\n')
				if len(op)>1 and op[1][:4].lower() == 'sql:':
					ol = webnotes.conn.sql(op[1][4:].replace('__user', webnotes.session['user']))	
				else:
					t = op[0][5:].strip()
					op = op[1:]
					op = [oc.replace('__user', webnotes.session['user']) for oc in op]
					
					try:
						# select options will always come from the user db
						ol = webnotes.conn.sql("select name from `tab%s` where %s docstatus!=2 order by name asc" % (t, op and (' AND '.join(op) + ' AND ') or ''))
					except:
						ol = []
				ol = [''] + [o[0] or '' for o in ol]
				d.options = '\n'.join(ol)

	# clear un-necessary code from going to the client side

	def _clear_code(self, doclist):
		if self.name != 'DocType':
			if doclist[0].server_code: doclist[0].server_code = None
			if doclist[0].server_code_core: doclist[0].server_code_core = None
			if doclist[0].client_script: doclist[0].client_script = None
			if doclist[0].client_script_core: doclist[0].client_script_core = None
			doclist[0].server_code_compiled = None

	# build a list of all the records required for the DocType
	
	def _get_last_update(self, dt):
		"""
			Returns last update timestamp
		"""
		try:
			last_update = webnotes.conn.sql("select _last_update from tabDocType where name=%s", dt)[0][0]
			if e.args[0]==1054: 
				self._setup_last_update(dt)
				last_update = None
		
		return last_update

	def _setup_last_update(self, doctype):
		"""
			Adds _last_update column to tabDocType

		"""
		webnotes.conn.sql("alter table `tabDocType` add column _last_update varchar(32)")
		
	def _get_fields(self, file_name):
		"""
			Returns a dictionary of DocFields by fieldname or label
		"""
		try:
			doclist = open(file_name, 'r').read()
		except:
			return
		doclist = eval(doclist)
		fields = {}
			if d['doctype']=='DocField':
				if d['fieldname'] or d['label']:
					fields[d['fieldname'] or d['label']] = d
		return fields
		
	def _update_field_properties(self, doclist):
		"""
			Updates properties like description, depends on from the database based on the timestamp
			of the .txt file. Adds a column _last_updated if not exists in the database and uses
			it to update the file..
			
			This feature is built because description is changed / updated quite often and is tedious to
			write a patch every time. Can be extended to cover more updates
		"""
		
		update_fields = ('description', 'depends_on')

		from webnotes.modules import get_file_timestamp, get_item_file

		doc = doclist[0] # main doc
		file_name = get_item_file(doc.module, 'DocType', doc.name)
		time_stamp = get_file_timestamp(file_name)
		last_update = self._get_last_update(doc.name)
		
		# this is confusing because we are updating the fields of fields
		
		if last_update != time_stamp:

			# there are updates!
			fields = self._get_fields(file_name)
			if fields:
				for d in doclist:
				
					# for each field in teh outgoing doclist
					if d.doctype=='DocField':
						key = d.fieldname or d.label
					
						# if it has a fieldname or label
						if key and key in fields:
						
							# update the values
							for field_to_update in update_fields:
								new_value = fields[key][field_to_update]
							
								# in doclist
								d.fields[field_to_update] = new_value
							
								# in database
								webnotes.conn.sql("update tabDocField set `%s` = %s where parent=%s and `%s`=%s" % \
									(field_to_update, '%s', '%s', (d.fieldname and 'fieldname' or 'label'), '%s'), \
									(new_value, doc.name, key))
					
				webnotes.conn.sql("update tabDocType set _last_update=%s where name=%s", (time_stamp, doc.name))
	
	def _override_field_properties(self, doclist):
		"""
			Override field properties that are updated by "Property Setter"
			The term "property" is used to define a fieldname of a field to avoid confusing
			terminology
		"""
		# load field properties and add them to a dictionary
		property_dict = {}
		dt = doclist[0].name
		try:
			for f in webnotes.conn.sql("select doc_name, property, property_type, value from `tabProperty Setter` where doc_type=%s", dt, as_dict=1):
				if not f['doc_name'] in property_dict:
					property_dict[f['doc_name']] = []
				property_dict[f['doc_name']].append(f)
				# no override table
				pass
			else: 
				raise e

		# loop over fields and override property
			if d.doctype=='DocField' and d.name in property_dict:
				for p in property_dict[d.name]:
					if p['property_type']=='Check':
						d.fields[p['property']] = int(p['value'])
					else:
						d.fields[p['property']] = p['value']
					
		# override properties in the main doctype
		if dt in property_dict:
			for p in property_dict[self.name]:
				doclist[0].fields[p['property']] = p['value']
				
	def make_doclist(self):
		"""
		      returns the :term:`doclist` for consumption by the client
		      
		      * it cleans up the server code
		      * executes all `$import` tags in client code
		      * replaces `link:` in the `Select` fields
		      * loads all related `Search Criteria`
		      * updates the cache
		"""
		
		tablefields = webnotes.model.meta.get_table_fields(self.name)

		if self.is_modified():
			# yes
			doclist = webnotes.model.doc.get('DocType', self.name, 1)
			
			self._update_field_properties(doclist)
			self._override_field_properties(doclist)
			
			# table doctypes
			for t in tablefields: 
				table_doclist = webnotes.model.doc.get('DocType', t[0], 1)
				
				self._override_field_properties(table_doclist)
				doclist += table_doclist

			# don't save compiled server code

		else:
			doclist = self._load_from_cache()
			
		doclist[0].fields['__client_script'] = compress.get_doctype_js(self.name)
		self._load_select_options(doclist)
		self._clear_code(doclist)

def clear_cache():
	webnotes.conn.sql("delete from __DocTypeCache")
	
# Load "DocType" - called by form builder, report buider and from code.py (when there is no cache)
#=================================================================================================

def get(dt):
	"""
	Load "DocType" - called by form builder, report buider and from code.py (when there is no cache)
	"""
	doclist = _DocType(dt).make_doclist()
		
# ToDO and Reminder
# -----------------

def add_todo(user, date, priority, desc, ref_type, ref_name):
	nlist = []
	if type(user)==list:
		for i in user:
			nlist.append(add_todo_item(i, date, priority, desc, ref_type, ref_name))
		return nlist
	else:
		return add_todo_item(user, date, priority, desc, ref_type, ref_name)	
	
def add_todo_item(user, date, priority, desc, ref_type, ref_name):
		date = nowdate()

	d = Document('ToDo Item')
	d.owner = user
	d.date = date
	d.priority = priority
	d.description = desc
	d.reference_type = ref_type
	d.reference_name = ref_name
	d.save(1)
	return d.name
	
def remove_todo(name):
	if type(name)==list:
		for i in name:
			sql("delete from `tabToDo Item` where name='%s'" % i)
	else:
		sql("delete from `tabToDo Item` where name='%s'" % name)

def get_todo_list():
	c = getcursor()
	try:
		role_options = ["role = '"+r+"'" for r in roles]
		role_options = role_options and ' OR ' + ' OR '.join(role_options) or ''
		c.execute("select * from `tabToDo Item` where owner='%s' %s" % (session['user'], role_options))
	except: # deprecated
		c.execute("select * from `tabToDo Item` where owner='%s'" % session['user'])
	dataset = c.fetchall()
	l = []
	for i in range(len(dataset)):
		d = Document('ToDo Item')
		d.loadfields(dataset, i, c.description)
		
	return l

import os,sys

cgi_bin_path = os.path.sep.join(__file__.split(os.path.sep)[:-3])

sys.path.append(cgi_bin_path)

#
# make a copy of defs.py (if not exists)
#		
def copy_defs():
	global cgi_bin_path
	if not os.path.exists(os.path.join(cgi_bin_path, 'webnotes', 'defs.py')):
		ret = os.system('cp '+ os.path.join(cgi_bin_path, 'webnotes', 'defs_template.py')+\
			' '+os.path.join(cgi_bin_path, 'webnotes', 'defs.py'))
		print 'Made copy of defs.py'

#
# Main Installer Class
#
class Installer:
	def __init__(self, root_login, root_password):

		import webnotes.db
	
		self.root_password = root_password
		from webnotes.model.db_schema import DbManager
		
		self.conn = webnotes.db.Database(user=root_login, password=root_password)			
		webnotes.conn=self.conn
		webnotes.session= {'user':'Administrator'}
		self.dbman = DbManager(self.conn)
		self.mysql_path = hasattr(webnotes.defs, 'mysql_path') and webnotes.defs.mysql_path or ''

	#
	# run framework related cleanups
	#
	def framework_cleanups(self, target):

		self.dbman.drop_table('__DocTypeCache')
		webnotes.conn.sql("create table `__DocTypeCache` (name VARCHAR(120), modified DATETIME, content TEXT, server_code_compiled TEXT)")

		# set the basic passwords
		webnotes.conn.sql("update tabProfile set password = password('admin') where name='Administrator'")

	def import_core_module(self):
		"""
			Imports the "Core" module from .txt file and creates
			Creates profile Administrator
		"""
		from webnotes.modules.import_module import import_module
		from webnotes.modules.module_manager import reload_doc

		reload_doc('core','doctype','doctype')
		reload_doc('core','doctype','docfield')
		reload_doc('core','doctype','docperm')

		import_module('core')
		
	def create_users(self):
		"""
			Create Administrator / Guest
		"""
		
		p = Document('Profile')
		p.name = p.first_name = 'Administrator'
		p.email = 'admin@localhost'
		p.save(new = 1)
		
		ur = Document('UserRole')
		ur.parent = 'Administrator'
		ur.role = 'Administrator'
		ur.parenttype = 'Profile'
		ur.parentfield = 'userroles'
		p.enabled = 1
		ur.save(1)

		p.name = p.first_name = 'Guest'
		p.email = 'guest@localhost'
		
		ur.parent = 'Guest'
		ur.role = 'Guest'

	#
	# main script to create a database from
	#
	def import_from_db(self, target, source_path='', password = 'admin', verbose=0):
		"""
		a very simplified version, just for the time being..will eventually be deprecated once the framework stabilizes.
		"""
		import webnotes.defs		
		
		# delete user (if exists)
		self.dbman.delete_user(target)

		# create user and db
		self.dbman.create_user(target,getattr(webnotes.defs,'db_password',None))
		if verbose: print "Created user %s" % target
	
		# create a database
		self.dbman.create_database(target)
		if verbose: print "Created database %s" % target
		
		# grant privileges to user
		self.dbman.grant_all_privileges(target,target)
		if verbose: print "Granted privileges to user %s and database %s" % (target, target)

		# flush user privileges
		self.dbman.flush_privileges()

		self.conn.use(target)
		
		# import in target
		if verbose: print "Starting database import..."

		# get the path of the sql file to import
		source_given = True
		if not source_path:
			source_given = False
			source_path = os.path.join(os.path.sep.join(os.path.abspath(webnotes.__file__).split(os.path.sep)[:-3]), 'data', 'Framework.sql')

		self.dbman.restore_database(target, source_path, self.root_password)
		if verbose: print "Imported from database %s" % source_path

		if not source_given:
			if verbose: print "Importing core module..."
			self.import_core_module()
			self.create_users()

		# framework cleanups
		self.framework_cleanups(target)
		if verbose: print "Ran framework startups on %s" % target
		
		return target		

def make_scheduler(root_login, root_password, verbose):
	"""
		Make the database where all scheduler events will be stored from multiple datbases
		See webnotes.utils.scheduler for more information
	"""
	conn = webnotes.db.Database(user=root_login, password=root_password)			

	from webnotes.model.db_schema import DbManager

	dbman = DbManager(conn)
	
	# delete user (if exists)
	dbman.delete_user('master_scheduler')

	# create user and db
	dbman.create_user('master_scheduler', getattr(webnotes.defs,'db_password',None))
	if verbose: print "Created user master_scheduler"

	# create a database
	dbman.create_database('master_scheduler')
	if verbose: print "Created database master_scheduler" 

	# grant privileges to user
	dbman.grant_all_privileges('master_scheduler','master_scheduler')

	# flush user privileges
	dbman.flush_privileges()
	
	conn.use('master_scheduler')
	
	# create events table
	conn.sql("""create table Event(
		`db_name` varchar(60),
		`event` varchar(180),
		`interval` int(20),
		`next_execution` timestamp,
		`recurring` int(1),
		primary key (`db_name`, `event`),
		index next_execution(next_execution)
		)""")
	
	conn.sql("""create table EventLog(
		`db_name` varchar(180), 
		`executed_on` timestamp,
		`log` text,
		index executed_on(executed_on))
	""")
#
# load the options
#
def get_parser():
	from optparse import OptionParser

	parser = OptionParser(usage="usage: %prog [options] ROOT_LOGIN ROOT_PASSWORD DBNAME")
	parser.add_option("-x", "--database-password", dest="password", default="admin", help="Optional: New password for the Framework Administrator, default 'admin'")	
	parser.add_option("-s", "--source", dest="source_path", default=None, help="Optional: Path of the sql file from which you want to import the instance, default 'data/Framework.sql'")
	
	return parser

#
# execution here
#

	parser = get_parser()
	(options, args)	= parser.parse_args()
	
	try:
	
	except ImportError:
		copy_defs()

	if len(args)==3:
		
		root_login, root_password, db_name = args[0], args[1], args[2]
		
		if db_name=='master_scheduler':
			make_scheduler(root_login, root_password, 1)
		else:
			inst = Installer(root_login, root_password)

			inst.import_from_db(db_name, source_path=options.source_path, \
				password = options.password, verbose = 1)

			print "Database created, please edit defs.py to get started"		
	else:
		parser.print_help()
		
# metadata

def get_dt_values(doctype, fields, as_dict = 0):
	return webnotes.conn.sql('SELECT %s FROM tabDocType WHERE name="%s"' % (fields, doctype), as_dict = as_dict)

def set_dt_value(doctype, field, value):
	return webnotes.conn.set_value('DocType', doctype, field, value)

def is_single(doctype):
	try:
		return get_dt_values(doctype, 'issingle')[0][0]
	except IndexError, e:
		raise Exception, 'Cannot determine whether %s is single' % doctype

def get_parent_dt(dt):
	parent_dt = webnotes.conn.sql('select parent from tabDocField where fieldtype="Table" and options="%s" and (parent not like "old_parent:%%") limit 1' % dt)
	return parent_dt and parent_dt[0][0] or ''

def set_fieldname(field_id, fieldname):
	webnotes.conn.set_value('DocField', field_id, 'fieldname', fieldname)

def get_link_fields(doctype):
	return webnotes.conn.sql("SELECT fieldname, options, label FROM tabDocField WHERE parent='%s' and (fieldtype='Link' or (fieldtype='Select' and `options` like 'link:%%'))" % (doctype))

def get_table_fields(doctype):
	return webnotes.conn.sql("select options, fieldname from tabDocField where parent='%s' and fieldtype='Table'" % doctype)
	
	dl = []
	sc_list = webnotes.conn.sql("select name from `tabSearch Criteria` where doc_type = '%s' or parent_doc_type = '%s' and (disabled!=1 OR disabled IS NULL)" % (dt, dt))
	for sc in sc_list:
		if sc[0]:
	return dl

def get_print_format_html(name):
	html = webnotes.conn.sql('select html from `tabPrint Format` where name="%s"' % name)
	return html and html[0][0] or ''
	
	# export
		from webnotes.modules.export_module import export_to_files	
		from webnotes.modules import get_module_path, scrub
		import os
		
			export_to_files(record_list=[['Page Template', self.doc.name]])

			file = open(os.path.join(get_module_path(self.doc.module), 'Page Template', self.doc.name, self.doc.name + '.html'), 'w')

"""
Server side functions for tagging.

- Tags can be added to any record (doctype, name) in the system.
- Items are filtered by tags
- Top tags are shown in the sidebar (?)
- Tags are also identified by the tag_fields property of the DocType

Discussion:

Tags are shown in the docbrowser and ideally where-ever items are searched.
There should also be statistics available for tags (like top tags etc)

Design:

- free tags (user_tags) are stored in __user_tags
- doctype tags are set in tag_fields property of the doctype
- top tags merges the tags from both the lists (only refreshes once an hour (max))

"""

def check_user_tags(dt):
	"if the user does not have a tags column, then it creates one"
	try:
		webnotes.conn.sql("select `_user_tags` from `tab%s` limit 1" % dt)
		if e.args[0] == 1054:
			DocTags(dt).setup()
			
#
# Add a new tag
#
def add_tag():
	"adds a new tag to a record, and creates the Tag master"
	
	f = webnotes.form_dict
	tag, color = f.get('tag'), f.get('color')
	dt, dn = f.get('dt'), f.get('dn')
	
	DocTags(dt).add(dn, tag)
		
	return tag

#
# remove tag
#
def remove_tag():
	"removes tag from the record"
	tag, dt, dn = f.get('tag'), f.get('dt'), f.get('dn')
	
	DocTags(dt).remove(dn, tag)

from webnotes.utils import cint, cstr, load_json
		
class DocTags:
	"""Tags for a particular doctype"""
	def __init__(self, dt):
		self.dt = dt
		
	def get_tag_fields(self):
		"""returns tag_fields property"""
		return webnotes.conn.get_value('DocType', self.dt, 'tag_fields')
		
	def get_tags(self, dn):
		"""returns tag for a particular item"""
		return webnotes.conn.get_value(self.dt, dn, '_user_tags') or ''

	def create(self, tag):
		try:
			webnotes.conn.sql("insert into tabTag(name) values (%s) on duplicate key ignore", tag)
			if e.args[0]==1147:
				self.setup_tag_master()
				self.create(tag)

	def add(self, dn, tag):
		"""add a new user tag"""
		self.create(tag)
		tl = self.get_tags(dn).split(',')
		if not tag in tl:
			tl.append(tag)
			self.update(dn, tl)
			TagCounter(self.dt).update(tag, 1)

	def remove(self, dn, tag):
		"""remove a user tag"""
		self.update(dn, filter(lambda x:x!=tag, tl))
		TagCounter(self.dt).update(tag, -1)

	def update(self, dn, tl):
		"""updates the _user_tag column in the table"""

		tl = list(set(filter(lambda x: x, tl)))
					
		try:
			webnotes.conn.sql("update `tab%s` set _user_tags=%s where name=%s" % \
				(self.dt,'%s','%s'), (',' + ','.join(tl), dn))
				self.setup()
				self.update(dn, tl)

	def setup_tags(self):
		"""creates the tabTag table if not exists"""
		reload_doc('core','doctype','tag')
		
	def setup(self):
		"""adds the _user_tags column if not exists"""
		webnotes.conn.sql("alter table `tab%s` add column `_user_tags` varchar(180)" % self.dt)

class TagCounter:
	"""
		Tag Counter stores tag count per doctype in table _tag_cnt
	"""
	def __init__(self, doctype):

	# setup / update tag cnt
	# keeps tags in _tag_cnt (doctype, tag, cnt)
	# if doctype cnt does not exist
	# creates it for the first time
	def update(self, tag, diff):
		"updates tag cnt for a doctype and tag"
		cnt = webnotes.conn.sql("select cnt from `_tag_cnt` where doctype=%s and tag=%s", (self.doctype, tag))

		if not cnt:
			# first time? build a cnt and add
			self.new_tag(tag, 1)
		else:
			webnotes.conn.sql("update `_tag_cnt` set cnt = ifnull(cnt,0) + (%s) where doctype=%s and tag=%s",\
				(diff, self.doctype, tag))

	def new_tag(self, tag, cnt=0, dt=None):
		"Creates a new row for the tag and doctype"
		webnotes.conn.sql("insert into `_tag_cnt`(doctype, tag, cnt) values (%s, %s, %s)", \
			(dt or self.doctype, tag, cnt))

	def build(self, dt):
		"Builds / rebuilds the counting"		
		webnotes.conn.sql("delete from _tag_cnt where doctype=%s", dt)
		
		# count
		tags = {}
		for ut in webnotes.conn.sql("select _user_tags from `tab%s`" % dt):
			if ut[0]:
				tag_list = ut[0].split(',')
				for t in tag_list:
					if t:
						tags[t] = tags.get(t, 0) + 1

		# insert
		for t in tags:
			self.new_tag(t, tags[t], dt)
						
	def load_top(self):
		try:
			return webnotes.conn.sql("select tag, cnt from `_tag_cnt` where doctype=%s and cnt>0 order by cnt desc limit 10", self.doctype, as_list = 1)
				return self.load_top()

		"creates the tag cnt table from the DocType"
		webnotes.conn.sql("""
		create table `_tag_cnt` (
			doctype varchar(180), tag varchar(22), cnt int(10),
			primary key (doctype, tag), index cnt(cnt)) ENGINE=InnoDB
		""")
		
		# build all
		for dt in webnotes.conn.sql("select name from tabDocType where ifnull(issingle,0)=0 and docstatus<2"):
			try:
				self.build(dt[0])
				if e.args[0]==1054: pass

def get_top_field_tags(dt):
	tf = webnotes.conn.get_value('DocType', dt, 'tag_fields')
	if not tf: return []
	
	# restrict to only 2 fields
	tf = tuple(set(tf.split(',')))[:2]
	tl = []
	
	for t in tf:
		t = t.strip()
		# disastrous query but lets try it!
		tl += webnotes.conn.sql("""select `%s`, count(*), '%s' from `tab%s` 
			where docstatus!=2 
			and ifnull(`%s`, '')!=''
			group by `%s` 
			order by count(*) desc 
			limit 10""" % (t, t, dt, t, t), as_list=1)

	if tl:
		tl.sort(lambda x, y: y[1]-x[1])

	return tl[:10]

# returns the top ranked 10 tags for the
# doctype. 
# merges the top tags from fields and user tags
def get_top_tags(args=''):
	"returns the top 10 tags for the doctype from fields (7) and users (3)"
	tl = None
	dt = webnotes.form_dict['doctype']
	
	from webnotes.utils.cache import get_item
	
	# if not reload, try and load from cache
	if not cint(webnotes.form_dict.get('refresh')):
		tl = get_item('tags-' + dt).get()
	
	if tl:
		return eval(tl)
	else:
		tl = TagCounter(dt).load_top() + get_top_field_tags(dt)
		if tl:
			tl.sort(lambda x, y: y[1]-x[1])
			tl = tl[:20]
			
		# set in cache and don't reload for an hour
		get_item('tags-' + dt).set(tl, 60*60)
	
		return tl
	
#:    HTML Template of index.cgi
index_template = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
<html>
<head id="head">
<!-- Web Notes Framework : www.webnotesframework.org -->

  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
  <meta name="robots" content="index, follow" />
  <meta name="keywords" content="%(keywords)s" />
  <meta name="description" content="%(site_description)s" />
  <meta name="generator" content="Web Notes Framework Version v170 - Open Source Web Application Framework" />  
  
  <title>%(title)s</title>
  <link type="text/css" rel="stylesheet" href="css/jquery-ui.css">
  <link type="text/css" rel="stylesheet" href="css/default.css">
  <link rel="Shortcut Icon" href="/favicon.ico">
  
  <script language="JavaScript" src="js/jquery/jquery.min.js"></script>
  <script language="JavaScript" src="js/jquery/jquery-ui.min.js"></script>
  <script type="text/javascript" src="js/tiny_mce_33/jquery.tinymce.js"></script>
  <script language="JavaScript" src="js/wnf.compressed.js"></script>
  %(import_form)s
  <script language="JavaScript">var _startup_data = %(startup_data)s;</script>
  <!--[if IE]><script language="javascript" type="text/javascript" src="js/jquery/excanvas.min.js"></script><![endif]-->
  %(add_in_head)s
  
  <script type="text/javascript">
    window.dhtmlHistory.create({ debugMode: false });
  </script>
</head>
<body>

<div id="dialog_back"></div>

<div id="startup_div" style="padding: 8px; font-size: 14px;"></div>

<!-- Main Starts -->
<div id="body_div"> 

	<!--static (no script) content-->
	<div class="no_script">
		%(content)s
	</div>

</div>

%(add_in_body)s
</body>
</html>
'''

redirect_template = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
<html>
<head>
<title>%s</title>
<meta http-equiv="REFRESH" content="0; url=%s"></HEAD>
<BODY style="font-family: Arial; padding: 8px; font-size: 14px; margin: 0px;">
Redirecting...
</BODY>
</HTML>'''

page_properties = {
	'add_in_head':'',
	'add_in_body':'',
	'keywords':'',
	'site_description':'',
	'title':'',
	'content':'',
	'startup_data':'{}',
	'import_form':'<script language="JavaScript" src="js/form.compressed.js"></script>'
}

# remove 'id' attributes so they don't conflict
def replace_id(match):
	#webnotes.msgprint(match.group('name'))
	return ''
	
def scrub_ids(content):
	import re
	
	p = re.compile('id=\"(?P<name> [^\"]*)\"', re.VERBOSE)
	content = p.sub(replace_id, content)

	p = re.compile('id=\'(?P<name> [^\']*)\'', re.VERBOSE)
	
	return content

#
# load the page content and meta tags
#
def get_page_content(page):
	"""
	   Gets the HTML content from `static_content` or `content` property of a `Page`
	"""
	from webnotes.model.code import get_code
	global page_properties
	
	if not page: return
	if '/' in page: page = page.split('/')[0]
	if page=='Form': return
	
	try:
		doc = Document('Page', page)
	
		load_page_metatags(doc)
	
		# content
	
		page_properties['content'] = get_code(doc.module, 'page', doc.name, 'html', fieldname='content')
				
		# dynamic (scripted) content
		if page_properties['content'] and page_properties['content'].startswith('#!python'):
			page_properties.update(webnotes.model.code.execute(page_properties['content']))
	
		page_properties['content'] = scrub_ids(template % {'content':page_properties['content']})
	except:
		pass

#
# load metatags
#
def load_page_metatags(doc):

	try:
		import startup
	except:
		startup = ''

	# page meta-tags
	page_properties['title'] = doc.page_title or doc.name
	page_properties['keywords'] = doc.keywords or webnotes.conn.get_value('Control Panel',None,'keywords') or ''
	page_properties['site_description'] = doc.site_description or webnotes.conn.get_value('Control Panel',None,'site_description') or ''
	page_properties['add_in_head'] = getattr(startup, 'add_in_head', '')
	page_properties['add_in_body'] = getattr(startup, 'add_in_body', '')

#
# load the form as page (deprecated)
#
def get_doc_content(dt, dn):
	"""
	   Gets the HTML content of a document record by using the overridden or standard :method: `doclist.to_html`
	"""
	
	if dt in webnotes.user.get_read_list():
		# generate HTML
		do = webnotes.model.code.get_obj(dt, dn, with_children = 1)
		if hasattr(do, 'to_html'):
			return dn, do.to_html()
		else:
			import webnotes.model.doclist
			return dn, webnotes.model.doclist.to_html(do.doclist)
	else:
		return 'Forbidden - 404', '<h1>Forbidden - 404</h1>'

# find the page to load as static
# -------------------------------

def load_properties():

	page_url = webnotes.form_dict.get('_escaped_fragment_', webnotes.form_dict.get('page', ''))
	
	if page_url:
		if page_url.startswith('Page/'): 
			page_url = page_url[5:]
		page_url = ['Page', urllib.unquote(page_url)]
	else:
		page_url = ['Page', webnotes.user.get_home_page()]
		
	# load content
	# -----------------	
	get_page_content(page_url[1])

# generate the page
def load_default_properties():
	if not page_properites['keywords']:
		page_properites['keywords'] = webnotes.conn.get_value('Control Panel',None,'keywords') or ''
	if not page_properites['site_description']:
		page_properites['site_description'] = webnotes.conn.get_value('Control Panel',None,'site_description') or ''

def get():
	"""
	   returns the full rendered index.cgi
	   Gets `keywords` and `site_description` from the `Control Panel`
	"""
	no_startup = webnotes.form.getvalue('no_startup') or None

	global index_template, redirect_template
	import webnotes.session_cache
	try:
		import json
	except: # python 2.4
		import simplejson as json
	
	page = webnotes.form_dict.get('page', '')
	# sid in public display
	# ---------------------
	if webnotes.form_dict.get('sid'): 
		return redirect_template % ('Redirecting...', ('index.cgi' + (page and ('?page='+page) or '')))
	
	if '%(content)s' in index_template:
		load_properties()
		
		# load the session data
		# ---------------------
		try:
			sd = webnotes.session_cache.get()
		except:
			import webnotes.utils
			sd = {'exc':webnotes.utils.getTraceback()}
		
		# add debug messages

		sd['server_messages'] = '\n--------------\n'.join(webnotes.message_log)
		
		page_properties['startup_data'] = no_startup and '{}' or json.dumps(sd)
		
		# no form api required for guests
		if webnotes.session['user']=='Guest':
			page_properties['import_form'] = ''
		
		index_template = index_template % page_properties
		
	return index_template

"""
XTEA Block Encryption Algorithm
Author: Paul Chakravarti (paul_dot_chakravarti_at_gmail_dot_com)
License: Public Domain
""" 

def get_key():
	# Encryption key is datetime of creation of DocType, DocType"
	return webnotes.conn.sql("select creation from tabDocType where name='DocType'")[0][0].strftime('%Y%m%d%H%M%s')[:16]

def encrypt(data, encryption_key = None):
	if not encryption_key:
		encryption_key = get_key()
	return crypt(encryption_key, data).encode('hex')

def decrypt(data, encryption_key = None):
	return crypt(encryption_key, data.decode('hex'))

def crypt(key,data,iv='\00\00\00\00\00\00\00\00',n=32):
	def keygen(key,iv,n):
		while True:
			iv = xtea_encrypt(key,iv,n)
			for k in iv:
				yield ord(k)
	xor = [ chr(x^y) for (x,y) in zip(map(ord,data),keygen(key,iv,n)) ]
	return "".join(xor)

def xtea_encrypt(key,block,n=32,endian="!"):
	import struct
	v0,v1 = struct.unpack(endian+"2L",block)
	k = struct.unpack(endian+"4L",key)
	sum,delta,mask = 0L,0x9e3779b9L,0xffffffffL
	for round in range(n):
		v0 = (v0 + (((v1<<4 ^ v1>>5) + v1) ^ (sum + k[sum & 3]))) & mask
		sum = (sum + delta) & mask
		v1 = (v1 + (((v0<<4 ^ v0>>5) + v0) ^ (sum + k[sum>>11 & 3]))) & mask
	return struct.pack(endian+"2L",v0,v1)
	
def xtea_decrypt(key,block,n=32,endian="!"):

	delta,mask = 0x9e3779b9L,0xffffffffL
	sum = (delta * n) & mask
		v1 = (v1 - (((v0<<4 ^ v0>>5) + v0) ^ (sum + k[sum>>11 & 3]))) & mask
		sum = (sum - delta) & mask
		v0 = (v0 - (((v1<<4 ^ v1>>5) + v1) ^ (sum + k[sum & 3]))) & mask

#!/usr/bin/env python
"""html2text: Turn HTML into equivalent Markdown-structured text."""
__version__ = "3.02"
__author__ = "Aaron Swartz (me@aaronsw.com)"
__copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3."
__contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"]

# TODO:
#   Support decoded entities with unifiable.

try:
    True
except NameError:
    setattr(__builtins__, 'True', 1)
    setattr(__builtins__, 'False', 0)

def has_key(x, y):
    if hasattr(x, 'has_key'): return x.has_key(y)
    else: return y in x

try:
    import htmlentitydefs
    import urlparse
    import HTMLParser
except ImportError: #Python3
    import html.entities as htmlentitydefs
    import urllib.parse as urlparse
    import html.parser as HTMLParser
try: #Python3
    import urllib.request as urllib
except:
    import urllib
import optparse, re, sys, codecs, types

try: from textwrap import wrap
except: pass

# Use Unicode characters instead of their ascii psuedo-replacements
UNICODE_SNOB = 0

# Put the links after each paragraph instead of at the end.
LINKS_EACH_PARAGRAPH = 0

# Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.)
BODY_WIDTH = 78

# Don't show internal links (href="#local-anchor") -- corresponding link targets
# won't be visible in the plain text file anyway.
SKIP_INTERNAL_LINKS = False

### Entity Nonsense ###

def name2cp(k):
    if k == 'apos': return ord("'")
    if hasattr(htmlentitydefs, "name2codepoint"): # requires Python 2.3
        return htmlentitydefs.name2codepoint[k]
    else:
        k = htmlentitydefs.entitydefs[k]
        if k.startswith("&#") and k.endswith(";"): return int(k[2:-1]) # not in latin-1
        return ord(codecs.latin_1_decode(k)[0])

unifiable = {'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"', 
'copy':'(C)', 'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*',
'ndash':'-', 'oelig':'oe', 'aelig':'ae',
'agrave':'a', 'aacute':'a', 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a', 
'egrave':'e', 'eacute':'e', 'ecirc':'e', 'euml':'e', 
'igrave':'i', 'iacute':'i', 'icirc':'i', 'iuml':'i',
'ograve':'o', 'oacute':'o', 'ocirc':'o', 'otilde':'o', 'ouml':'o', 
'ugrave':'u', 'uacute':'u', 'ucirc':'u', 'uuml':'u'}

unifiable_n = {}

for k in unifiable.keys():
    unifiable_n[name2cp(k)] = unifiable[k]

def charref(name):
    if name[0] in ['x','X']:
        c = int(name[1:], 16)
    else:
        c = int(name)
    
    if not UNICODE_SNOB and c in unifiable_n.keys():
        return unifiable_n[c]
    else:
        try:
            return unichr(c)
        except NameError: #Python3
            return chr(c)

def entityref(c):
    if not UNICODE_SNOB and c in unifiable.keys():
        return unifiable[c]
    else:
        try: name2cp(c)
        except KeyError: return "&" + c + ';'
        else:
            try:
                return unichr(name2cp(c))
            except NameError: #Python3
                return chr(name2cp(c))

def replaceEntities(s):
    s = s.group(1)
    if s[0] == "#": 
        return charref(s[1:])
    else: return entityref(s)

r_unescape = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
def unescape(s):
    return r_unescape.sub(replaceEntities, s)

### End Entity Nonsense ###

def onlywhite(line):
    """Return true if the line does only consist of whitespace characters."""
    for c in line:
        if c is not ' ' and c is not '  ':
            return c is ' '
    return line

def optwrap(text):
    """Wrap all paragraphs in the provided text."""
    if not BODY_WIDTH:
        return text
    
    assert wrap, "Requires Python 2.3."
    result = ''
    newlines = 0
    for para in text.split("\n"):
        if len(para) > 0:
            if para[0] != ' ' and para[0] != '-' and para[0] != '*':
                for line in wrap(para, BODY_WIDTH):
                    result += line + "\n"
                result += "\n"
                newlines = 2
            else:
                if not onlywhite(para):
                    result += para + "\n"
                    newlines = 1
        else:
            if newlines < 2:
                newlines += 1
    return result

def hn(tag):
    if tag[0] == 'h' and len(tag) == 2:
        try:
            n = int(tag[1])
            if n in range(1, 10): return n
        except ValueError: return 0

class _html2text(HTMLParser.HTMLParser):
    def __init__(self, out=None, baseurl=''):
        HTMLParser.HTMLParser.__init__(self)
        
        if out is None: self.out = self.outtextf
        else: self.out = out
        try:
            self.outtext = unicode()
        except NameError: # Python3
            self.outtext = str()
        self.quiet = 0
        self.p_p = 0
        self.outcount = 0
        self.start = 1
        self.space = 0
        self.a = []
        self.astack = []
        self.acount = 0
        self.list = []
        self.blockquote = 0
        self.pre = 0
        self.startpre = 0
        self.lastWasNL = 0
        self.abbr_title = None # current abbreviation definition
        self.abbr_data = None # last inner HTML (for abbr being defined)
        self.abbr_list = {} # stack of abbreviations to write later
        self.baseurl = baseurl
    
    def outtextf(self, s): 
        self.outtext += s
    
    def close(self):
        HTMLParser.HTMLParser.close(self)
        
        self.pbr()
        self.o('', 0, 'end')
        
        return self.outtext
        
    def handle_charref(self, c):
        self.o(charref(c))

    def handle_entityref(self, c):
        self.o(entityref(c))
            
    def handle_starttag(self, tag, attrs):
        self.handle_tag(tag, attrs, 1)
    
    def handle_endtag(self, tag):
        self.handle_tag(tag, None, 0)
        
    def previousIndex(self, attrs):
        """ returns the index of certain set of attributes (of a link) in the
            self.a list
 
            If the set of attributes is not found, returns None
        """
        if not has_key(attrs, 'href'): return None
        
        i = -1
        for a in self.a:
            i += 1
            match = 0
            
            if has_key(a, 'href') and a['href'] == attrs['href']:
                if has_key(a, 'title') or has_key(attrs, 'title'):
                        if (has_key(a, 'title') and has_key(attrs, 'title') and
                            a['title'] == attrs['title']):
                            match = True
                else:
                    match = True

            if match: return i

    def handle_tag(self, tag, attrs, start):
        #attrs = fixattrs(attrs)
    
        if hn(tag):
            self.p()
            if start: self.o(hn(tag)*"#" + ' ')

        if tag in ['p', 'div']: self.p()
        
        if tag == "br" and start: self.o("  \n")

        if tag == "hr" and start:
            self.p()
            self.o("* * *")
            self.p()

        if tag in ["head", "style", 'script']: 
            if start: self.quiet += 1
            else: self.quiet -= 1

        if tag in ["body"]:
            self.quiet = 0 # sites like 9rules.com never close <head>
        
        if tag == "blockquote":
            if start: 
                self.p(); self.o('> ', 0, 1); self.start = 1
                self.blockquote += 1
            else:
                self.blockquote -= 1
                self.p()
        
        if tag in ['em', 'i', 'u']: self.o("_")
        if tag in ['strong', 'b']: self.o("**")
        if tag == "code" and not self.pre: self.o('`') #TODO: `` `this` ``
        if tag == "abbr":
            if start:
                attrsD = {}
                for (x, y) in attrs: attrsD[x] = y
                attrs = attrsD
                
                self.abbr_title = None
                self.abbr_data = ''
                if has_key(attrs, 'title'):
                    self.abbr_title = attrs['title']
            else:
                if self.abbr_title != None:
                    self.abbr_list[self.abbr_data] = self.abbr_title
                    self.abbr_title = None
        
        if tag == "a":
            if start:
                attrsD = {}
                if has_key(attrs, 'href') and not (SKIP_INTERNAL_LINKS and attrs['href'].startswith('#')): 
                    self.astack.append(attrs)
                    self.o("[")
                else:
                    self.astack.append(None)
            else:
                if self.astack:
                    a = self.astack.pop()
                    if a:
                        i = self.previousIndex(a)
                        if i is not None:
                            a = self.a[i]
                        else:
                            self.acount += 1
                            a['count'] = self.acount
                            a['outcount'] = self.outcount
                            self.a.append(a)
                        self.o("][" + str(a['count']) + "]")
        
        if tag == "img" and start:
            attrsD = {}
            for (x, y) in attrs: attrsD[x] = y
            attrs = attrsD
            if has_key(attrs, 'src'):
                attrs['href'] = attrs['src']
                alt = attrs.get('alt', '')
                i = self.previousIndex(attrs)
                if i is not None:
                    attrs = self.a[i]
                else:
                    self.acount += 1
                    attrs['count'] = self.acount
                    attrs['outcount'] = self.outcount
                    self.a.append(attrs)
                self.o("![")
                self.o(alt)
                self.o("]["+ str(attrs['count']) +"]")
        
        if tag == 'dl' and start: self.p()
        if tag == 'dt' and not start: self.pbr()
        if tag == 'dd' and start: self.o('    ')
        if tag == 'dd' and not start: self.pbr()
        
        if tag in ["ol", "ul"]:
            if start:
                self.list.append({'name':tag, 'num':0})
            else:
                if self.list: self.list.pop()
            
            self.p()
        
        if tag == 'li':
            if start:
                self.pbr()
                if self.list: li = self.list[-1]
                else: li = {'name':'ul', 'num':0}
                self.o("  "*len(self.list)) #TODO: line up <ol><li>s > 9 correctly.
                if li['name'] == "ul": self.o("* ")
                elif li['name'] == "ol":
                    li['num'] += 1
                    self.o(str(li['num'])+". ")
                self.start = 1
            else:
        
        if tag in ["table", "tr"] and start: self.p()
        if tag == 'td': self.pbr()
        
        if tag == "pre":
            if start:
                self.startpre = 1
                self.pre = 1
            else:
                self.pre = 0
            self.p()
            
    def pbr(self):
        if self.p_p == 0: self.p_p = 1

    def p(self): self.p_p = 2
    
    def o(self, data, puredata=0, force=0):
        if self.abbr_data is not None: self.abbr_data += data
        
        if not self.quiet: 
            if puredata and not self.pre:
                data = re.sub('\s+', ' ', data)
                if data and data[0] == ' ':
                    self.space = 1
                    data = data[1:]
            if not data and not force: return
            
            if self.startpre:
                #self.out(" :") #TODO: not output when already one there
                self.startpre = 0
            
            bq = (">" * self.blockquote)
            if not (force and data and data[0] == ">") and self.blockquote: bq += " "
            
            if self.pre:
                bq += "    "
                data = data.replace("\n", "\n"+bq)
            
            if self.start:
                self.space = 0
                self.p_p = 0
                self.start = 0

            if force == 'end':
                # It's the end.
                self.out("\n")

            if self.p_p:
                self.out(('\n'+bq)*self.p_p)
                
            if self.space:
                if not self.lastWasNL: self.out(' ')

            if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"):
                if force == "end": self.out("\n")

                newa = []
                for link in self.a:
                    if self.outcount > link['outcount']:
                        self.out("   ["+ str(link['count']) +"]: " + urlparse.urljoin(self.baseurl, link['href'])) 
                        if has_key(link, 'title'): self.out(" ("+link['title']+")")
                        self.out("\n")
                    else:
                        newa.append(link)

                if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done.

                self.a = newa
            
            if self.abbr_list and force == "end":
                for abbr, definition in self.abbr_list.items():
                    self.out("  *[" + abbr + "]: " + definition + "\n")

            self.p_p = 0
            self.out(data)
            self.lastWasNL = data and data[-1] == '\n'
            self.outcount += 1

    def handle_data(self, data):
        if r'\/script>' in data: self.quiet -= 1
        self.o(data, 1)
    
    def unknown_decl(self, data): pass

def wrapwrite(text):
    text = text.encode('utf-8')
    try: #Python3
        sys.stdout.buffer.write(text)
    except AttributeError:
        sys.stdout.write(text)

def html2text_file(html, out=wrapwrite, baseurl=''):
    h = _html2text(out, baseurl)
    h.feed(html)
    h.feed("")
    return h.close()

def html2text(html, baseurl=''):
    return optwrap(html2text_file(html, None, baseurl))

if __name__ == "__main__":
    baseurl = ''

    p = optparse.OptionParser('%prog [(filename|url) [encoding]]',
                              version='%prog ' + __version__)
    args = p.parse_args()[1]
    if len(args) > 0:
        file_ = args[0]
        encoding = None
        if len(args) == 2:
            encoding = args[1]
        if len(args) > 2:
            p.error('Too many arguments')

        if file_.startswith('http://') or file_.startswith('https://'):
            baseurl = file_
            j = urllib.urlopen(baseurl)
            text = j.read()
            if encoding is None:
                try:
                    from feedparser import _getCharacterEncoding as enc
                except ImportError:
                    enc = lambda x, y: ('utf-8', 1)
                encoding = enc(j.headers, text)[0]
                if encoding == 'us-ascii':
                    encoding = 'utf-8'
            data = text.decode(encoding)

        else:
            data = open(file_, 'rb').read()
                try:
                    from chardet import detect
                    detect = lambda x: {'encoding': 'utf-8'}
                encoding = detect(data)['encoding']
            data = data.decode(encoding)
    else:
        data = sys.stdin.read()
    wrapwrite(html2text(data, baseurl))

"""
Server side handler for "Form" events
"""

from webnotes.model.triggers import fire_event

def getdoc():
	"""
	Loads a doclist for a given document. This method is called directly from the client.
	Requries "doctype", "docname" as form variables. If "from_archive" is set, it will get from archive.
	Will also call the "onload" method on the document.
	"""

	form = webnotes.form_dict
	doctype, docname = form.get('doctype'), form.get('name')
	prefix = cint(form.get('from_archive')) and 'arc' or 'tab'

	if not (doctype and docname):
		raise Exception, 'doctype and name required!'
	
	# single
	doclist = load_single_doc(doctype, docname, (form.get('user') or webnotes.session['user']), prefix)
	
	# load doctype along with the doc
	if form.get('getdoctype'):
		import webnotes.model.doctype
		doclist += webnotes.model.doctype.get(doctype)

	# tag as archived
	if prefix == 'arc':
		doclist[0].__archived=1

#===========================================================================================

def get_comments(doctype=None, docname=None, limit=5):
	nc, cl = 0, []

		doctype, docname, limit = webnotes.form_dict.get('dt'), webnotes.form_dict.get('dn'), webnotes.form_dict.get('limit')
		
	try:
		nc = int(webnotes.conn.sql("select count(*) from `tabComment Widget Record` where comment_doctype=%s and comment_docname=%s", (doctype, docname))[0][0])
		if nc:
			cl = webnotes.conn.sql("select comment, ifnull(comment_by_fullname, comment_by) AS 'comment_by_fullname', creation from `tabComment Widget Record` where comment_doctype=%s and comment_docname=%s order by creation desc limit %s" % ('%s','%s',limit), (doctype, docname), as_dict=1)

			# no table
			make_comment_table()
		else:
			raise e

	webnotes.response['n_comments'], webnotes.response['comment_list'] = nc, cl

#
# make comment table
#
def make_comment_table():
	"Make table for comments - if missing via import module"
	from webnotes.modules import reload_doc
	reload_doc('core', 'doctype', 'comment_widget_record')
	webnotes.conn.begin()	
	
def add_comment():
	import time
	args = webnotes.form_dict

	if args.get('comment'):
		from webnotes.utils import nowdate
		
		cmt = Document('Comment Widget Record')
		for arg in ['comment', 'comment_by', 'comment_by_fullname', 'comment_doctype', 'comment_docname']:
			cmt.fields[arg] = args[arg]
		cmt.comment_date = nowdate()
		cmt.comment_time = time.strftime('%H:%M')
		cmt.save(1)
  			
def remove_comment():
	webnotes.conn.sql("delete from `tabComment Widget Record` where name=%s",args.get('id'))

	try:
		get_obj('Feed Control').upate_comment_in_feed(args['dt'], args['dn'])

def getdoctype():
	# load parent doctype too
	
	form, doclist = webnotes.form, []
	
	with_parent = form.getvalue('with_parent')

	# with parent (called from report builder)
	if with_parent:
		parent_dt = webnotes.model.meta.get_parent_dt(dt)
		if parent_dt:
			doclist = webnotes.model.doctype.get(parent_dt)
			webnotes.response['parent_dt'] = parent_dt
	
	if not doclist:
		doclist = webnotes.model.doctype.get(dt)
	
	# if single, send the record too
	if doclist[0].issingle:
		doclist += webnotes.model.doc.get(dt)

	doclist += webnotes.model.meta.get_search_criteria(dt)

def load_single_doc(dt, dn, user, prefix):

	if not dn: dn = dt
	dl = webnotes.model.doc.get(dt, dn, prefix=prefix)

	# archive, done
	if prefix=='arc':
		return dl

	try:
		so, r = webnotes.model.code.get_server_obj(dl[0], dl), None
		if hasattr(so, 'onload'):
			r = webnotes.model.code.run_server_obj(so, 'onload')
		if hasattr(so, 'custom_onload'):
			r = webnotes.model.code.run_server_obj(so, 'custom_onload')
		if r: 
			webnotes.msgprint(r)
		webnotes.errprint(webnotes.utils.getTraceback())
		webnotes.msgprint('Error in script while loading')
		raise e

	if dl and not dn.startswith('_'):
		webnotes.user.update_recent(dt, dn)

	# load search criteria ---- if doctype
		dl += webnotes.model.meta.get_search_criteria(dt)

	return dl

# Check Guest Access
def check_guest_access(doc):
	if webnotes.session['user']=='Guest' and not webnotes.conn.sql("select name from tabDocPerm where role='Guest' and parent=%s and ifnull(`read`,0)=1", doc.doctype):
		webnotes.msgprint("Guest not allowed to call this object")

# Runserverobj - run server calls from form

def runserverobj():

	method = form.getvalue('method')
	doclist, clientlist = [], []
	arg = form.getvalue('arg')
		
	if dt: # not called from a doctype (from a page)
		if not dn: dn = dt # single
		so = webnotes.model.code.get_obj(dt, dn)

	else:
		clientlist = webnotes.model.doclist.expand(form.getvalue('docs'))

		# find main doc
		for d in clientlist:
			if cint(d.get('docstatus')) != 2 and not d.get('parent'):
				main_doc = webnotes.model.doc.Document(fielddata = d)
	
		# find child docs
			doc = webnotes.model.doc.Document(fielddata = d)
			if doc.fields.get('parent'):
				doclist.append(doc)	
	
		so = webnotes.model.code.get_server_obj(main_doc, doclist)

	# check integrity
	if not check_integrity(so.doc):
		return
		
	check_guest_access(so.doc)
				
	if so:
		r = webnotes.model.code.run_server_obj(so, method, arg)
		doclist = so.doclist # reference back [in case of null]
		if r:
			try:
				if r['doclist']:
					clientlist += r['doclist']
			except:
				pass
			
			#build output as csv
			if cint(webnotes.form.getvalue('as_csv')):
				make_csv_output(r, so.doc.doctype)
			else:
				webnotes.response['message'] = r
		
		if clientlist:
			doclist.append(main_doc)
			webnotes.response['docs'] = doclist

def make_csv_output(res, dt):

	txt = []
	if type(res)==list:
			txt.append(','.join([getCSVelement(i) for i in r]))
		
		txt = '\n'.join(txt)
	
	else:
		txt = 'Output was not in list format\n' + r
					
	webnotes.response['doctype'] = dt.replace(' ','')						

# Document Save

def _get_doclist(clientlist):
	# converts doc dictionaries into Document objects

	midx = 0
	for i in range(len(clientlist)):
		if clientlist[i]['name'] == form.getvalue('docname'):
			main_doc = Document(fielddata = clientlist[i])
			midx = i
		else:
			clientlist[i] = Document(fielddata = clientlist[i])

	del clientlist[midx]
	return main_doc, clientlist

def _do_action(doc, doclist, so, method_name, docstatus=0):

	from webnotes.model.code import run_server_obj

	if so and hasattr(so, method_name):
		errmethod = method_name
		run_server_obj(so, method_name)
		if hasattr(so, 'custom_'+method_name):
			run_server_obj(so, 'custom_'+method_name)
		errmethod = ''

	# fire triggers observers (if any)
	fire_event(doc, method_name)

	# set docstatus for all children records
	if docstatus:
		for d in [doc] + doclist:
			if int(d.docstatus or 0) != 2:
				set(d, 'docstatus', docstatus)

def check_integrity(doc):
		
	if (not webnotes.model.meta.is_single(doc.doctype)) and (not doc.fields.get('__islocal')):
		tmp = webnotes.conn.sql('SELECT modified FROM `tab%s` WHERE name="%s" for update' % (doc.doctype, doc.name))
		if tmp and str(tmp[0][0]) != str(doc.modified):
			webnotes.msgprint('Document has been modified after you have opened it. To maintain the integrity of the data, you will not be able to save your changes. Please refresh this document. [%s/%s]' % (tmp[0][0], doc.modified))
			return 0
			
	return 1

def savedocs():

	from webnotes.model.code import get_server_obj
	from webnotes.widgets.auto_master import update_auto_masters

	# action
	action = form.getvalue('action')
	
	# get docs	
	doc, doclist = _get_doclist(webnotes.model.doclist.expand(form.getvalue('docs')))

	# get server object	
	server_obj = get_server_obj(doc, doclist)

	if not check_integrity(doc):
		return
	
	if not doc.check_perm(verbose=1):
		webnotes.msgprint("Not enough permission to save %s" % doc.doctype)
		return
	
	# validate links
	ret = webnotes.model.doclist.validate_links_doclist([doc] + doclist)
	if ret:
		webnotes.msgprint("[Link Validation] Could not find the following values: %s. Please correct and resave. Document Not Saved." % ret)
		return

	# saving & post-saving
	try:
		# validate befor saving and submitting
		if action in ('Save', 'Submit') and server_obj:
			if hasattr(server_obj, 'validate'):	
				t = run_server_obj(server_obj, 'validate')
			if hasattr(server_obj, 'custom_validate'):
				t = run_server_obj(server_obj, 'custom_validate')
				
		# set owner and modified times
		is_new = cint(doc.fields.get('__islocal'))
		if is_new and not doc.owner:
			doc.owner = form.getvalue('user')
		
		doc.modified, doc.modified_by = webnotes.utils.now(), webnotes.session['user']
		
		# save main doc
		try:
			t = doc.save(is_new)
			update_auto_masters(doc)
		except NameError, e:
			webnotes.msgprint('%s "%s" already exists' % (doc.doctype, doc.name))
			if webnotes.conn.sql("select docstatus from `tab%s` where name=%s" % (doc.doctype, '%s'), doc.name)[0][0]==2:
				webnotes.msgprint('[%s "%s" has been cancelled]' % (doc.doctype, doc.name))
			webnotes.errprint(webnotes.utils.getTraceback())
			raise e
		
		# save child docs
			deleted, local = d.fields.get('__deleted',0), d.fields.get('__islocal',0)
	
			if cint(local) and cint(deleted):
				pass
			elif d.fields.has_key('parent'):
				if d.parent and (not d.parent.startswith('old_parent:')):
					d.parent = doc.name # rename if reqd
					d.parenttype = doc.doctype
				d.modified, d.modified_by = webnotes.utils.now(), webnotes.session['user']
				d.save(new = cint(local))
				update_auto_masters(d)
	
		# on_update
		if action in ('Save','Submit') and server_obj:
			if hasattr(server_obj, 'on_update'):
				t = run_server_obj(server_obj, 'on_update')
				if t: webnotes.msgprint(t)
	
			if hasattr(server_obj, 'custom_on_update'):
				t = run_server_obj(server_obj, 'custom_on_update')

			fire_event(doc, 'on_update')
				
		# on_submit
		if action == 'Submit':
			_do_action(doc, doclist, server_obj, 'on_submit', 1)

		# for allow_on_submit type
		if action == 'Update':
			_do_action(doc, doclist, server_obj, 'on_update_after_submit', 0)
				
		# on_cancel
		if action == 'Cancel':
			_do_action(doc, doclist, server_obj, 'on_cancel', 2)

		# update recent documents
		webnotes.user.update_recent(doc.doctype, doc.name)

		# send updated docs
		webnotes.response['saved'] = '1'
		webnotes.response['main_doc_name'] = doc.name
		webnotes.response['docname'] = doc.name
		webnotes.response['docs'] = [doc] + doclist

		webnotes.msgprint('Did not save')
		raise e

# Print Format
def _get_print_format(match):
	name = match.group('name')
	return webnotes.model.meta.get_print_format_html(name)

def get_print_format():
	import re

	html = webnotes.model.meta.get_print_format_html(webnotes.form.getvalue('name'))

	p = re.compile('\$import\( (?P<name> [^)]*) \)', re.VERBOSE)
	out_html = ''
	if html: 
		out_html = p.sub(_get_print_format, html)

	webnotes.response['message'] = out_html
	
# remove attachment

def remove_attach():
	import webnotes.utils.file_manager
	
	fid = webnotes.form.getvalue('fid')
	webnotes.utils.file_manager.delete_file(fid, verbose=1)

# Get Fields - Counterpart to $c_get_fields
def get_fields():
	r = {}
	args = {
		'select':webnotes.form.getvalue('select')
		,'from':webnotes.form.getvalue('from')
		,'where':webnotes.form.getvalue('where')
	}
	ret = webnotes.conn.sql("select %(select)s from `%(from)s` where %(where)s limit 1" % args)
	if ret:
		fl, i = webnotes.form.getvalue('fields').split(','), 0
		for f in fl:
			r[f], i = ret[0][i], i+1
	webnotes.response['message']=r

# validate link
def validate_link():
	
	value, options, fetch = webnotes.form.getvalue('value'), webnotes.form.getvalue('options'), webnotes.form.getvalue('fetch')

	# no options, don't validate
	if not options or options=='null' or options=='undefined':
		webnotes.response['message'] = 'Ok'
		return
		
	if webnotes.conn.sql("select name from `tab%s` where name=%s" % (options, '%s'), value):
	
		# get fetch values
		if fetch:
			webnotes.response['fetch_values'] = [webnotes.utils.parse_val(c) for c in webnotes.conn.sql("select %s from `tab%s` where name=%s" % (fetch, options, '%s'), value)[0]]
	
"""
Run tests from modules. Sets up database connection, modules path and session before running test

Usage: from shell, run

python tests.py [test modules]

Options:
	test modules: list of modules separated by space
	
if no modules are specified, it will run all "tests.py" files from all modules
"""

import sys, os
import unittest

# webnotes path
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

# modules path

if webnotes.defs.__dict__.get('modules_path'):
	sys.path.append(webnotes.defs.modules_path)

def get_tests():
	"""
	Returns list of test modules identified by "tests.py"
	"""
	ret = []
	for walk_tuple in os.walk(webnotes.defs.modules_path):
		if 'tests.py' in walk_tuple[2]:
			dir_path = os.path.relpath(walk_tuple[0], webnotes.defs.modules_path)
			if dir_path=='.':
				ret.append('tests')
			else:
				ret.append(dir_path.replace('/', '.') + '.tests')
			
	return ret

	"""
	Sets up connection and session
	"""
	from webnotes.db import Database
	webnotes.conn = Database()

	setup()
		
	if len(sys.argv) > 1:
		tests_list = sys.argv[1:]
		
		# for unittest.main
		sys.argv = sys.argv[:1]
	else:
		tests_list = get_tests()
	
	for tests in tests_list:
		exec 'from %s import *' % str(tests)

	unittest.main()

"""
Sends email via outgoing server specified in "Control Panel"
Allows easy adding of Attachments of "File" objects
"""

import webnotes	
from webnotes import msgprint
import email

class EMail:
	"""
	Wrapper on the email module. Email object represents emails to be sent to the client. 
	Also provides a clean way to add binary `FileData` attachments
	Also sets all messages as multipart/alternative for cleaner reading in text-only clients
	"""
	def __init__(self, sender='', recipients=[], subject='', from_defs=0, alternative=0, reply_to=None):
		from email.mime.multipart import MIMEMultipart
		if type(recipients)==str:
			recipients = recipients.replace(';', ',')
			recipients = recipients.split(',')
			
		self.from_defs = from_defs
		self.sender = sender
		self.reply_to = reply_to or sender
		self.recipients = recipients
		self.subject = subject
		
		self.msg_root = MIMEMultipart('mixed')
		self.msg_multipart = MIMEMultipart('alternative')
		self.msg_root.attach(self.msg_multipart)
		self.cc = []
	
	def set_text(self, message):
		"""
			Attach message in the text portion of multipart/alternative
		"""
		from email.mime.text import MIMEText		
		part = MIMEText(message, 'plain')
		self.msg_multipart.attach(part)
		
	def set_html(self, message):
		"""
			Attach message in the html portion of multipart/alternative
		"""
		part = MIMEText(message, 'html')
	
	def set_message(self, message, mime_type='text/html', as_attachment=0, filename='attachment.html'):
		"""
			Append the message with MIME content to the root node (as attachment)
		"""
		from email.mime.text import MIMEText
		
		maintype, subtype = mime_type.split('/')
		part = MIMEText(message, _subtype = subtype)
		
		if as_attachment:
			part.add_header('Content-Disposition', 'attachment', filename=filename)
		
		self.msg_root.attach(part)
		
	def attach_file(self, n):
		"""
		attach a file from the `FileData` table
		"""
		from webnotes.utils.file_manager import get_file		
		res = get_file(n)
		if not res:
			return
	
		self.add_attachment(res[0], res[1])
	
	def add_attachment(self, fname, fcontent, content_type=None):
	
		from email.mime.audio import MIMEAudio
		from email.mime.base import MIMEBase
		from email.mime.image import MIMEImage
					
		import mimetypes

		if not content_type:
			content_type, encoding = mimetypes.guess_type(fname)

		if content_type is None:
			# No guess could be made, or the file is encoded (compressed), so
			# use a generic bag-of-bits type.
			content_type = 'application/octet-stream'
		
		maintype, subtype = content_type.split('/', 1)
		if maintype == 'text':
			# Note: we should handle calculating the charset
			part = MIMEText(fcontent, _subtype=subtype)
		elif maintype == 'image':
			part = MIMEImage(fcontent, _subtype=subtype)
		elif maintype == 'audio':
			part = MIMEAudio(fcontent, _subtype=subtype)
		else:
			part = MIMEBase(maintype, subtype)
			part.set_payload(fcontent)
			# Encode the payload using Base64
			from email import encoders
			encoders.encode_base64(part)
			
		# Set the filename parameter
		if fname:
			part.add_header('Content-Disposition', 'attachment', filename=fname)

	def validate(self):
		"""
		validate the email ids
		"""
		if not self.sender:
			self.sender = webnotes.conn.get_value('Control Panel',None,'auto_email_id')

		from webnotes.utils import validate_email_add
		# validate ids
		if self.sender and (not validate_email_add(self.sender)):
			webnotes.msgprint("%s is not a valid email id" % self.sender, raise_exception = 1)

		if self.reply_to and (not validate_email_add(self.reply_to)):
			webnotes.msgprint("%s is not a valid email id" % self.reply_to, raise_exception = 1)

		for e in self.recipients:
			if not validate_email_add(e):
				webnotes.msgprint("%s is not a valid email id" % e, raise_exception = 1)
	
		"""
		setup the SMTP (outgoing) server from `Control Panel` or defs.py
		"""
		if self.from_defs:
			self.server = getattr(webnotes.defs,'mail_server','')
			self.login = getattr(webnotes.defs,'mail_login','')
			self.port = getattr(webnotes.defs,'mail_port',None)
			self.password = getattr(webnotes.defs,'mail_password','')
			self.use_ssl = getattr(webnotes.defs,'use_ssl',0)

		else:	
			from webnotes.utils import cint

			# get defaults from control panel
			cp = webnotes.model.doc.Document('Control Panel','Control Panel')
			self.server = cp.outgoing_mail_server or getattr(webnotes.defs,'mail_server','')
			self.login = cp.mail_login or getattr(webnotes.defs,'mail_login','')
			self.port = cp.mail_port or getattr(webnotes.defs,'mail_port',None)
			self.password = cp.mail_password or getattr(webnotes.defs,'mail_password','')
			self.use_ssl = cint(cp.use_ssl)

	def make_msg(self):
		self.msg_root['Subject'] = self.subject
		self.msg_root['From'] = self.sender
		self.msg_root['To'] = ', '.join([r.strip() for r in self.recipients])
		if self.reply_to and self.reply_to != self.sender:
			self.msg_root['Reply-To'] = self.reply_to
			self.msg_root['CC'] = ', '.join([r.strip() for r in self.cc])
	
	def add_to_queue(self):
		# write to a file called "email_queue" or as specified in email
		q = EmailQueue()
		q.push({
			'server': self.server, 
			'port': self.port, 
			'use_ssl': self.use_ssl,
			'login': self.login,
			'password': self.password,
			'sender': self.sender,
			'recipients': self.recipients, 
			'msg': self.msg_root.as_string()
		})
		q.close()

	def send(self, send_now = 0):
		"""		
		send the message
		"""
		from webnotes.utils import cint
		
		self.setup()
		self.validate()
		self.make_msg()
		
		if (not send_now) and getattr(webnotes.defs, 'batch_emails', 0):
			self.add_to_queue()
			return

		import smtplib
		sess = smtplib.SMTP(self.server, self.port or None)
		
		if self.use_ssl: 
			sess.ehlo()
			sess.starttls()
		
		ret = sess.login(self.login, self.password)

		# check if logged correctly
		if ret[0]!=235:
			msgprint(ret[1])
		
		sess.sendmail(self.sender, self.recipients, self.msg_root.as_string())
		
		try:
			sess.quit()
		except:
			pass
			
# ===========================================
# Email Queue
# Maintains a list of emails in a file
# Flushes them when called from cron
# Defs settings:
# 	email_queue: (filename) [default: email_queue.py]
#
# From the scheduler, call: flush(qty)

class EmailQueue:
		self.server = self.login = self.sess = None
		self.filename = getattr(webnotes.defs, 'email_queue', 'email_queue.py')
	
		try:
			f = open(self.filename, 'r')
			self.queue = eval(f.read() or '[]')
			f.close()
		except IOError, e:
			if e.args[0]==2:
				self.queue = []
			else:
				raise e
		
	def push(self, email):
		self.queue.append(email)
		
		f = open(self.filename, 'w')
		f.write(str(self.queue))
		f.close()

	def get_smtp_session(self, e):
		if self.server==e['server'] and self.login==e['login'] and self.sess:
			return self.sess

		webnotes.msgprint('getting server')

		sess = smtplib.SMTP(e['server'], e['port'] or None)
		
		ret = sess.login(e['login'], e['password'])

			webnotes.msgprint(ret[1])
						
		self.sess = sess
		self.server, self.login = e['server'], e['login']
		
		return sess
		
	def flush(self, qty = 100):
		f = open(self.filename, 'r')
		
		self.queue = eval(f.read() or '[]')
		
		if len(self.queue) < 100:
			qty = len(self.queue)

		for i in range(qty):
			e = self.queue[i]
			sess = self.get_smtp_session(e)
			sess.sendmail(e['sender'], e['recipients'], e['msg'])			
		
		self.queue = self.queue[:(len(self.queue) - qty)]
		self.close()

# auto masters
# _ + fieldname is the table
# 'value' is the column name, pkey

# create masters for a doctype
def create_auto_masters(dt):
	fl = webnotes.conn.sql("select fieldname from tabDocField where fieldtype='Data' and options='Suggest' and parent=%s", dt)
	for f in fl:
		make_auto_master(f[0])

# create master table
def make_auto_master(fieldname):
	try:
		webnotes.conn.sql("select `value` from `__%s` limit 1" % fieldname)
			webnotes.conn.sql("create table `__%s` (`value` varchar(220), primary key (`value`))" % fieldname)

# get auto master fields
def get_master_fields(dt):
	if not webnotes.session['data'].get('auto_masters'):
		webnotes.session['data']['auto_masters'] = {}
	
	if webnotes.session['data']['auto_masters'].get(dt, None)==None:
		fl = webnotes.conn.sql("select fieldname from tabDocField where fieldtype='Data' and options='Suggest' and parent=%s", dt)
		webnotes.session['data']['auto_masters'][dt] = fl

	return webnotes.session['data']['auto_masters'][dt]
		
# update value
def update_auto_masters(doc):
	if not doc.doctype:
		return
		
	fl = get_master_fields(doc.doctype)

	# save masters in session cache		
	for f in fl:
		if doc.fields.get(f[0]):
			add_to_master(f[0], doc.fields.get(f[0]))

# add to master
def add_to_master(fieldname, value):
	try:
		webnotes.conn.sql("insert into `__%s` (`value`) values (%s)" % (fieldname,'%s'), value)
		# primary key
		pass

"""
Server side methods for the follower pattern (Follow button used in forms)
"""

form = webnotes.form_dict

#
# Follow
#
def follow(dt=None, dn=None, user=None, verbose=0):
	"Add as follower to a particular record. If no parameteres, then take from the http request (form)"
	
	if not dt: 
		dt, dn, user = form.get('dt'), form.get('dn'), form.get('user')
		verbose = 1

	if not user: return

	if not is_follower(dt, dn, user):
		make_follower(dt, dn, user, verbose)
	else:
		if verbose: webnotes.msgprint("%s is already a follower!" % user)

	return load_followers(dt, dn)

def make_follower(dt, dn, user, verbose):
	"Add the user as a follower"
	if has_permission(dt, user):
		d = Document('Follower')
		d.doc_type = dt
		d.doc_name = dn
		d.owner = user
		d.save(1)
	else:
		if verbose: webnotes.msgprint('%s does not have sufficient permission to follow' % user)
		
def has_permission(dt, user):
	"Check to see if the user has permission to follow"

	return webnotes.conn.sql("select name from tabDocPerm where parent=%s and ifnull(`read`,0)=1 and role in ('%s') limit 1" \
		% ('%s', ("', '".join(webnotes.user.get_roles()))), dt)
	
def is_follower(dt, dn, user):
	"returns true if given user is a follower"
	
	return webnotes.conn.sql("""
		select name from tabFollower 
		where ifnull(doc_type,'')=%s 
		and ifnull(doc_name,'')=%s 
		and owner=%s""", (dt, dn, user))
#
# Unfollow
#
def unfollow(dt=None, dn=None, user=None):
	"Unfollow a particular record. If no parameteres, then take from the http request (form)"

	if not dt:

	webnotes.conn.sql("delete from tabFollower where doc_name=%s and doc_type=%s and owner=%s", (dn, dt, user))

#
# Load followers
#
def load_followers(dt=None, dn=None):
	"returns list of followers (Full Names) for a particular object"

	if not dt: dt, dn = form.get('dt'), form.get('dn')
		
	try:
		return [t[0] for t in webnotes.conn.sql("""
			SELECT IFNULL(CONCAT(t1.first_name, if(t1.first_name IS NULL, '', ' '), t1.last_name), t1.name)
			FROM tabProfile t1, tabFollower t2 WHERE t2.doc_type=%s AND t2.doc_name=%s 
			AND t1.name = t2.owner""", (dt, dn))]
			
		if e.args[0] in (1146, 1054): 
			setup()
			return []
		else:
			raise e

#
# Email followers
#
def email_followers(dt, dn, msg_html=None, msg_text=None):
	"Send an email to all followers of this object"
	pass

#
# Update feed
#
def on_docsave(doc):
	"Add the owner and all linked Profiles as followers"
	follow(doc.doctype, doc.name, doc.owner)
	for p in get_profile_fields(doc.doctype):
		follow(doc.doctype, doc.name, doc.fields.get(p))

	update_followers(doc = doc)

#
# update the follower record timestamp and subject
#
def update_followers(dt=None, dn=None, subject=None, update_by=None, doc=None):
	"Updates the timestamp and subject in follower table (for feed generation)"
	from webnotes.utils import now
	webnotes.conn.sql("update tabFollower set modified=%s, subject=%s, modified_by=%s where doc_type=%s and doc_name=%s", \
		(now(), 
		subject or doc.fields.get('subject'), \
		update_by or webnotes.session['user'],\
		dt or doc.doctype, 
		dn or doc.name))

#
# get type of "Profile" fields
#
def get_profile_fields(dt):
	"returns a list of all profile link fields from the doctype"
	return [f[0] for f in \
		webnotes.conn.sql("select fieldname from tabDocField where parent=%s and fieldtype='Link' and options='Profile'", dt)]

#
# setup - make followers table
#
	"Make table for followers - if missing"
	reload_doc('core', 'doctype', 'follower')

# patch manager

def run(log_exception=1):
	from patches import patch
	
	next_patch = cint(webnotes.conn.get_global('next_patch'))
	
	if next_patch <= patch.last_patch:
		for i in range(next_patch, patch.last_patch+1):
			if log_exception:
				try:
					patch.execute(i)
					write_log()	
					return
			else:
				patch.execute(i)

			webnotes.conn.set_global('next_patch', str(i+1))

def write_log():
	import os
	
	patch_log = open(os.path.join(webnotes.defs.modules_path, 'patches', 'patch.log'), 'a')
	patch_log.write(('\n\nError in %s:\n' % webnotes.conn.cur_db_name) + webnotes.getTraceback())
	patch_log.close()
	
	webnotes.msgprint("There were errors in running patches, please call the Administrator")
	
# Event
# -------------

def get_cal_events(m_st, m_end):
	
	# load owned events
	res1 = sql("select name from `tabEvent` WHERE ifnull(event_date,'2000-01-01') between '%s' and '%s' and owner = '%s' and event_type != 'Public' and event_type != 'Cancel'" % (m_st, m_end, webnotes.user.name))

	# load individual events
	res2 = sql("select t1.name from `tabEvent` t1, `tabEvent User` t2 where ifnull(t1.event_date,'2000-01-01') between '%s' and '%s' and t2.person = '%s' and t1.name = t2.parent and t1.event_type != 'Cancel'" % (m_st, m_end, webnotes.user.name))

	# load role events
	roles = webnotes.user.get_roles()
	myroles = ['t2.role = "%s"' % r for r in roles]
	myroles = '(' + (' OR '.join(myroles)) + ')'
	res3 = sql("select t1.name from `tabEvent` t1, `tabEvent Role` t2  where ifnull(t1.event_date,'2000-01-01') between '%s' and '%s' and t1.name = t2.parent and t1.event_type != 'Cancel' and %s" % (m_st, m_end, myroles))
	
	# load public events
	res4 = sql("select name from `tabEvent` where ifnull(event_date,'2000-01-01') between '%s' and '%s' and event_type='Public'" % (m_st, m_end))
	
	doclist, rl = [], []
	for r in res1 + res2 + res3 + res4:
		if not r in rl:
			doclist += webnotes.model.doc.get('Event', r[0])
			rl.append(r)
	
# Load Month Events

def load_month_events():
	
	mm = form.getvalue('month')
	yy = form.getvalue('year')
	m_st = str(yy) + '-%.2i' % cint(mm) + '-01'
	m_end = str(yy) + '-%.2i' % cint(mm) + '-31'

	webnotes.response['docs'] = get_cal_events(m_st, m_end)

	def autoname(self):
		if (self.doc.name and self.doc.name.startswith('New Page')) or not self.doc.name:
			self.doc.name = self.doc.page_name.lower().replace(' ', '-')

	def onload(self):
		import os
		
		# load content
		try:
			file = open(os.path.join(get_module_path(self.doc.module), 'page', scrub(self.doc.name) + '.html'), 'r')
			self.doc.content = file.read() or ''
		except IOError, e: # no file / permission
				raise e

	# replace $image
		import re
		p = re.compile('\$image\( (?P<name> [^)]*) \)', re.VERBOSE)
		if self.doc.content:
			self.doc.content = p.sub(self.replace_by_img, self.doc.content)
	
	def replace_by_img(self, match):
		name = match.group('name')
		return '<img src="cgi-bin/getfile.cgi?ac=%s&name=%s">' % (webnotes.conn.get('Control Panel', None, 'account_id'), name)
		
	# export
		from webnotes.modules.export_module import export_to_files
		import os
		from webnotes import defs
		
		if getattr(defs,'developer_mode', 0):
			export_to_files(record_list=[['Page', self.doc.name]])
	
			if self.doc.write_content and self.doc.content:
				file = open(os.path.join(get_module_path(self.doc.module), 'page', scrub(self.doc.name), scrub(self.doc.name) + '.html'), 'w')
				file.write(self.doc.content)
				file.close()
 
form_dict = webnotes.form_dict

sql = None
session = None
errdoc = ''
errdoctype = ''
errmethod = ''
fw_folder = '/Users/rushabh/workbench/www/'

# Logs

# refresh / start page
# ------------------------------------------------------------------------------------

def startup():

	webnotes.response.update(webnotes.session_cache.get())

def cleanup_docs():
	if webnotes.response.get('docs') and type(webnotes.response['docs'])!=dict:
		webnotes.response['docs'] = webnotes.model.doclist.compress(webnotes.response['docs'])

# server calls

def runserverobj(arg=None):
	import webnotes.widgets.form
	webnotes.widgets.form.runserverobj()

def logout():
	webnotes.login_manager.logout()

# DocType Mapper

def dt_map():
	
	form_dict = webnotes.form_dict
	
	dt_list = webnotes.model.doclist.expand(form_dict.get('docs'))
	from_doctype = form_dict.get('from_doctype')
	to_doctype = form_dict.get('to_doctype')
	from_docname = form_dict.get('from_docname')
	from_to_list = form_dict.get('from_to_list')
	
	dm = get_obj('DocType Mapper', from_doctype +'-' + to_doctype)
	doclist = dm.dt_map(from_doctype, to_doctype, from_docname, Document(fielddata = dt_list[0]), [], from_to_list)
	
	m_st = str(yy) + '-' + str(mm) + '-01'
	m_end = str(yy) + '-' + str(mm) + '-31'

	import webnotes.widgets.event
	webnotes.response['docs'] = webnotes.widgets.event.get_cal_events(m_st, m_end)

# Data import

def import_csv():
	import webnotes.model.import_docs
	
	i = webnotes.model.import_docs.CSVImport()
	r = i.import_csv(form.getvalue('csv_file'), form.getvalue('dateformat'), form_dict.get('overwrite', 0) and 1)
	
	webnotes.response['type']='iframe'
	rhead = '''<style>body, html {font-family: Arial; font-size: 12px;}</style>'''
	webnotes.response['result']= rhead + r

	webnotes.model.import_docs.get_template()
	
# File Upload

def uploadfile():
	if webnotes.form_dict.get('from_form'):
		webnotes.utils.file_manager.upload()
	else:
		# save the file
		fid, fname = webnotes.utils.file_manager.save_uploaded()
		
		# do something with the uploaded file
		if fid and webnotes.form_dict.get('server_obj'):
			from webnotes.model.code import get_obj
			getattr(get_obj(webnotes.form_dict.get('server_obj')), webnotes.form_dict.get('method'))(fid, fname)
			
		# return the upload
		if fid:
			webnotes.response['result'] = '<script>window.parent.upload_callback("'+webnotes.form_dict.get('uploader_id')+'", "'+fid+'")</script>'
	
# File upload (from scripts)

def upload_many():

	# pass it on to upload_many method in Control Panel
	cp = get_obj('Control Panel')
	cp.upload_many(webnotes.form)
	
	webnotes.response['result'] = """
%s
</script>
%s
%s""" % (cp.upload_callback(webnotes.form), '\n----\n'.join(webnotes.message_log).replace("'", "\'"), '\n----\n'.join(webnotes.debug_log).replace("'", "\'").replace("\n","<br>"))

# File download
def get_file():

	res = webnotes.utils.file_manager.get_file(form.getvalue('fname'))
	if res:
		webnotes.response['type'] = 'download'
		webnotes.response['filename'] = res[0]
		
		if hasattr(res[1], 'tostring'):
			webnotes.response['filecontent'] = res[1].tostring()
		else: 
			webnotes.response['filecontent'] = res[1]
	else:
		webnotes.msgprint('[get_file] Unknown file name')
		
# Get Graph
def get_graph():

	import StringIO
	f = StringIO.StringIO()

	# call the object
	obj = server.get_obj(form_dict.get('dt'))
	plt = server.run_server_obj(obj, form_dict.get('method'), form_dict.get('arg'))
	plt.savefig(f)

	# stream out
	webnotes.response['type'] = 'download'
	webnotes.response['filename'] = webnotes.user.get_random_password() + '.png'
	webnotes.response['filecontent'] = f.getvalue()

# Reset Password

def reset_password():
	
	act = form_dict.get('account', '')
	user = form_dict.get('user', '')
	if act:
		webnotes.conn.set_db(act)

	try:
		p = webnotes.profile.Profile(user)
		p.reset_password()
		webnotes.msgprint("Password has been reset and sent to your email id.")
		webnotes.msgprint(str(e))

# Resume session

def resume_session():
	webnotes.response['message'] = webnotes.session_obj.resume()

# Create Backup

def backupdb(form_dict, session):
	db_name = server.decrypt(form_dict.get('db_name'))

	server.backup_db(db_name)

	webnotes.response['filename'] = db_name+'.tar.gz'
	webnotes.response['filecontent'] = open('../backups/' + db_name+'.tar.gz','rb').read()

def validate_cmd(cmd):
	# check if there is no direct possibility of malicious script injection
	if cmd.startswith('webnotes.model.code'):
		raise Exception, 'Cannot call any methods from webnotes.model.code directly from the handler'

	if cmd.startswith('webnotes.model.db_schema'):
		raise Exception, 'Cannot call any methods from webnotes.model.db_schema directly from the handler'

	if cmd.startswith('webnotes.conn'):
		raise Exception, 'Cannot call database connection method directly from the handler'
	
# Execution Starts Here

import webnotes.auth

# reset password

if form_dict.has_key('cmd') and (form_dict.get('cmd')=='reset_password'):
	webnotes.conn = webnotes.db.Database(use_default = 1)
	sql("START TRANSACTION")
	try:
		reset_password()
		sql("COMMIT")
		webnotes.errprint(str(e))
		sql("ROLLBACK")
	
# pre-login access - for registration etc.

elif form_dict.has_key('cmd') and (form_dict.get('cmd')=='prelogin'):

	try:
		webnotes.response['message'] = webnotes.model.code.get_obj('Profile Control').prelogin(form_dict) or ''
	except:

# main stuff

else:

	try:
		webnotes.request = webnotes.auth.HTTPRequest()
	
		if form_dict.get('cmd') != 'login' and webnotes.conn:
		
			# NOTE:
			# guest should only be allowed: 
			# getdoc (if Guest access)
			# runserverobj (if Guest access)
		
			# get command cmd
			cmd = form_dict.has_key('cmd') and form_dict.get('cmd') or ''
			read_only = form_dict.has_key('_read_only') and form_dict.get('_read_only') or None

			validate_cmd(cmd)

			module = ''
			if '.' in cmd:
				module = '.'.join(cmd.split('.')[:-1])
				cmd = cmd.split('.')[-1]

				exec 'from %s import %s' % (module, cmd) in locals()			
	
			# execute
			if locals().has_key(cmd):
				if (not webnotes.conn.in_transaction) and (not read_only):
					webnotes.conn.begin()
				
				if webnotes.form_dict.get('arg'):
					# direct method call
					ret = locals()[cmd](webnotes.form_dict.get('arg'))
				else:
					ret = locals()[cmd]()
				
				# returns with a message
				if ret:
					webnotes.response['message'] = ret
						
				# update session
				webnotes.session_obj.update()
				
				if webnotes.conn.in_transaction:
					webnotes.conn.commit()
			else:
				if cmd!='login':
					webnotes.msgprint('No Method: %s' % cmd)

	except webnotes.ValidationError:
		webnotes.conn.rollback()
	except:
		webnotes.conn and webnotes.conn.rollback()
			
#### cleanup
#-----------

if webnotes.conn:
	webnotes.conn.close()

#### go

import string
import os

acceptsGzip, out_buf, str_out = 0, None, None
try:
	if string.find(os.environ["HTTP_ACCEPT_ENCODING"], "gzip") != -1:
		acceptsGzip = 1 # problem in win ?
except:
	pass

def compressBuf(buf):
	import gzip, cStringIO
	zbuf = cStringIO.StringIO()
	zfile = gzip.GzipFile(mode = 'wb',  fileobj = zbuf, compresslevel = 5)
	zfile.write(buf)
	zfile.close()
	return zbuf.getvalue()

# CSV
# -------------------------------------------------------------------

if webnotes.response.get('type')=='csv':
	print "Content-Type: text/csv"
	print "Content-Disposition: attachment; filename="+webnotes.response['doctype'].replace(' ', '_')+".csv"
	print
	print webnotes.response['result']

# IFRAME

elif webnotes.response.get('type')=='iframe':
	print "Content-Type: text/html"
	print
	if webnotes.response.get('result'):
		print webnotes.response['result']
	if webnotes.debug_log:
		print '''<script type='text/javascript'>alert("%s");</script>''' % ('-------'.join(webnotes.debug_log).replace('"', '').replace('\n',''))

# file

elif webnotes.response.get('type')=='download':
	import mimetypes
	print "Content-Type: %s" % (mimetypes.guess_type(webnotes.response['filename'])[0] or 'application/unknown')
	print "Content-Disposition: filename="+webnotes.response['filename'].replace(' ', '_')
	print
	print webnotes.response['filecontent']

# JSON

else:
		save_log = 1
		if webnotes.debug_log[0].startswith('[Validation Error]'):
			save_log = 0

		t = '\n----------------\n'.join(webnotes.debug_log)
		if errdoctype: 
			t = t + '\nDocType: ' + errdoctype
		if errdoc: 
			t = t + '\nName: ' + errdoc
		if errmethod: 
			t = t + '\nMethod: ' + errmethod
		webnotes.response['exc'] = '<pre>'+t.replace('\n','<br>')+'</pre>'

		if save_log: # don't save validation errors
			try:  save_log(t, 'Server')
			except: pass

	if webnotes.message_log:
		webnotes.response['server_messages'] = '\n----------------\n'.join(webnotes.message_log)

	cleanup_docs()
	
	# Convert to JSON
	try:
	
	try:
		str_out = json.dumps(webnotes.response)
	except:
		str_out = str(webnotes.response).replace(', None', ', ""')
	
	if acceptsGzip and len(str_out)>512:
		out_buf = compressBuf(str_out)
		print "Content-Encoding: gzip"
		print "Content-Length: %d" % (len(out_buf))
		
	print "Content-Type: text/html; Charset: ISO-8859-1"
	
	# if there ar additional cookies defined during the request, add them here
	if webnotes.cookies or webnotes.add_cookies: 
		for c in webnotes.add_cookies.keys():
			webnotes.cookies[c] = webnotes.add_cookies[c]
			
		print webnotes.cookies
		
	print # Headers end
	
if out_buf:
	sys.stdout.write(out_buf)
elif str_out:
	print str_out

from webnotes.model.doc import Document

def set_doc(doclist, ovr=0, ignore=1, onupdate=1):
	dt = doclist[0]['doctype']
	
	if webnotes.conn.exists(doclist[0]['doctype'], doclist[0]['name']):	
		# exists, merge if possible
		
		if dt=='DocType':
			ud = UpdateDocType(doclist)
		elif dt == 'Module Def':
			ud = UpdateModuleDef(doclist)
		elif dt == 'DocType Mapper':
			ud = UpdateDocTypeMapper(doclist)
		else:
			ud = UpdateDocument(doclist)
	else:
		ud = UpdateDocument(doclist)
		
	ud.sync()
	return '\n'.join(ud.log)

#
# Class to sync incoming document
#
class UpdateDocument:
	def __init__(self, in_doclist=[]):
		self.in_doclist = in_doclist
		self.doc = Document(fielddata = in_doclist[0])
		self.modified = self.doc.modified # make a copy
		self.doclist = []
		
		self.log = []
		self.exists = 0

	# sync
		is_mod = self.is_modified()
		
		if (not self.exists) or (is_mod):
			if self.exists:
				self.delete_existing()
			self.save()
			self.update_modified()
			self.run_on_update()
	
	# check modified	
		try:
			timestamp = webnotes.conn.sql("select modified from `tab%s` where name=%s" % (self.doc.doctype, '%s'), self.doc.name)
		except Exception ,e:
			if(e.args[0]==1146):
				return
			else:
				raise e

		if timestamp:
			self.exists = 1
			if str(timestamp[0][0]) == self.doc.modified: 
				self.log.append('%s %s, No change' % (self.doc.doctype, self.doc.name))
			else: return 1
	
	# delete existing
	def delete_existing(self):
		from webnotes.model import delete_doc
		webnotes.conn.sql("set foreign_key_checks=0")
		delete_doc(self.doc.doctype, self.doc.name)
		webnotes.conn.sql("set foreign_key_checks=1")

	# update modified timestamp
	def update_modified(self):
		webnotes.conn.set(self.doc, 'modified', self.modified)
		
	def save(self):
		# parent
		self.doc.save(new = 1, ignore_fields = 1, check_links=0)
		self.doclist = [self.doc]
		self.save_children()
			
	def save_children(self):
		for df in self.in_doclist[1:]:
			self.save_one_doc(df)
			
	def save_one_doc(self, df, as_new=1):
		d = Document(fielddata = df)
		d.save(new = as_new, ignore_fields = 1, check_links=0)
		self.doclist.append(d)

	def run_on_update(self):
		from webnotes.model.code import get_server_obj
		so = get_server_obj(self.doc, self.doclist)
		if hasattr(so, 'on_update'):
			so.on_update()

#
# "Merge incoming doctype"
#
class UpdateDocumentMerge(UpdateDocument):
	def __init__(self, in_doclist):
		self.to_update_doctype = []
		UpdateDocument.__init__(self, in_doclist)

		pass
		
	def get_id(self, d):
		pass
		
	def to_update(self, d):
		return 1
	
	def child_exists(self, d):
		return self.get_id(d)
		
	def on_save(self):
		pass
	
		if self.exists:
			# save main doc
			self.keep_values(self.doc)
			self.doc.save(ignore_fields = 1, check_links=0)
			self.doclist.append(self.doc)
			self.save_children()
			self.on_save()
			self.log.append('Updated %s' % self.doc.name)
		else:
			UpdateDocument.save(self)
		
		for df in self.in_doclist[1:]: 
			d = Document(fielddata = df)
			
			# update doctype?
			if d.doctype in self.to_update_doctype:
				
				# update this record?
				if self.to_update(d):
				
					# is it new?
					if self.child_exists(d):
						self.keep_values(d)
						d.save(ignore_fields = 1, check_links=0)
						self.log.append('updated %s, %s' % (d.doctype, d.name))
					else:
						d.save(1, ignore_fields = 1, check_links=0)
						self.log.append('new %s' % d.doctype)
			self.doclist.append(d)

	def keep_values(self, d):
		if hasattr(self, 'get_orignal_values'):
			ov = self.get_orignal_values(d)
			if ov:
				d.fields.update(ov)

#
# Class to sync incoming doctype
#
class UpdateDocType(UpdateDocumentMerge):
		UpdateDocumentMerge.__init__(self, in_doclist)
		self.to_update_doctype = ['DocType', 'DocField']
		
	def to_udpate(self, d):
		if (d.fieldtype not in ['Section Break', 'Column Break', 'HTML']) and (d.fieldname or d.label):
			return 1
	
		key = d.fieldname and 'fieldname' or 'label'
		return webnotes.conn.sql("""select name, options, permlevel, reqd, print_hide, hidden
		from tabDocField where %s=%s and parent=%s""" % (key, '%s', '%s'), (d.fields[key], d.parent))
				
		self.renum()

		if d.doctype=='DocField':
			return self.get_id(d)
	
	def get_orignal_values(self, d):
			t = self.get_id(d)[0]
			return {'name': t[0], 'options': t[1], 'reqd':t[3], 'print_hide':t[4], 'hidden':t[5]}

		if d.doctype=='DocType':
			return webnotes.conn.sql("select server_code, client_script from `tabDocType` where name=%s", d.name, as_dict = 1)[0]

	# renumber the indexes	
	def renum(self):
		extra = self.get_extra_fields()
		self.clear_section_breaks()
		self.add_section_breaks_and_renum()
		self.fix_extra_fields(extra)

	# get fields not in the incoming list (to preserve order)
	def get_extra_fields(self):
		prev_field, prev_field_key, extra = '', '', []
		
		# get new fields and labels
		fieldnames = [d.get('fieldname') for d in self.in_doclist]
		labels = [d.get('label') for d in self.in_doclist]
		
		# check if all existing are present
		for f in webnotes.conn.sql("select fieldname, label, idx from tabDocField where parent=%s and fieldtype not in ('Section Break', 'Column Break', 'HTML') order by idx asc", self.doc.name):
			if f[0] and not f[0] in fieldnames:
				extra.append([f[0], f[1], prev_field, prev_field_key])
			elif f[1] and not f[1] in labels:
				
			prev_field, prev_field_key = f[0] or f[1], f[0] and 'fieldname' or 'label'
		
		return extra

	# clear section breaks
	def clear_section_breaks(self):
		webnotes.conn.sql("delete from tabDocField where fieldtype in ('Section Break', 'Column Break', 'HTML') and parent=%s and ifnull(options,'')!='Custom'", self.doc.name)

	# add section breaks
	def add_section_breaks_and_renum(self):
		for d in self.in_doclist:
			if d.get('parentfield')=='fields':
				if d.get('fieldtype') in ('Section Break', 'Column Break', 'HTML'):
					tmp = Document(fielddata = d)
					tmp.fieldname = ''
					tmp.name = None
					tmp.save(1, ignore_fields = 1, check_links=0)
				else:
					webnotes.conn.sql("update tabDocField set idx=%s where %s=%s and parent=%s" % \
						('%s', d.get('fieldname') and 'fieldname' or 'label', '%s', '%s'), (d.get('idx'), d.get('fieldname') or d.get('label'), self.doc.name))

	# adjust the extra fields
	def fix_extra_fields(self, extra):	
		# push fields down at new idx
		for e in extra:
			# get idx of the prev to extra field
			idx = 0
			if e[2]:
				idx = webnotes.conn.sql("select idx from tabDocField where %s=%s and parent=%s" % (e[3], '%s', '%s'), (e[2], self.doc.name))
				idx = idx and idx[0][0] or 0
			
			if idx:
				webnotes.conn.sql("update tabDocField set idx=idx+1 where idx>%s and parent=%s", (idx, self.doc.name))	
				webnotes.conn.sql("update tabDocField set idx=%s where %s=%s and parent=%s" % \
					('%s', e[0] and 'fieldname' or 'label', '%s', '%s'), (idx+1, e[0] or e[1], self.doc.name))

#
# update module def
#
class UpdateModuleDef(UpdateDocumentMerge):
		self.to_update_doctype = ['Module Def', 'Module Def Item']
			
	def get_id(self, d):		
		return webnotes.conn.sql("select name from `tabModule Def Item` where doc_type=%s and doc_name=%s and display_name=%s and parent=%s", (d.doc_type, d.doc_name, d.display_name, d.parent))
	
		if d.doctype=='Module Def Item': return 1
	
		if d.doctype=='Module Def Item':
			return {'name': self.get_id(d)[0][0]}
		if d.doctype=='Module Def':
			return webnotes.conn.sql("select module_seq, disabled, is_hidden from `tabModule Def` where name=%s", d.name, as_dict = 1)[0]

#
#
class UpdateDocTypeMapper(UpdateDocumentMerge):
		self.to_update_doctype = ['Field Mapper Detail', 'Table Mapper Detail']
			
		if d.doctype=='Field Mapper Detail':
			return webnotes.conn.sql("select name from `tabField Mapper Detail` where from_field=%s and to_field=%s and match_id=%s and parent=%s", (d.from_field, d.to_field, d.match_id, d.parent))
		elif d.doctype=='Table Mapper Detail':
			return webnotes.conn.sql("select name from `tabTable Mapper Detail` where from_table=%s and to_table = %s and match_id=%s and parent=%s", (d.from_table, d.to_table, d.match_id, d.parent))
		
		if d.doctype in ['Field Mapper Detail', 'Table Mapper Detail']: 

"""
	Create a database from scratch (wip)
"""

class DatabaseInstance:

	def __init__(self, conn = None,db_name = None):
		self.conn = conn
		self.db_name = db_name
		
#	def setup(self):
#		self.create_db_and_user()
#		self.create_base_tables()
#		self.import_system_module()
#		self.setup_users()
		
	def create_db_and_user(self):
		
		self.conn.sql("CREATE USER '%s'@'localhost' IDENTIFIED BY '%s'" % (self.db_name, webnotes.defs.db_password))
		self.conn.sql("CREATE DATABASE IF NOT EXISTS `%s` ;" % self.db_name)
		self.conn.sql("GRANT ALL PRIVILEGES ON `%s` . * TO '%s'@'localhost';" % (self.db_name, self.db_name))
		self.conn.sql("FLUSH PRIVILEGES")
		self.conn.sql("SET GLOBAL TRANSACTION ISOLATION LEVEL READ COMMITTED;")
		self.conn.sql("USE %s"%self.db_name)
		
	def create_base_tables(self):
		self.create_singles()
		self.create_sessions()
		self.create_doctypecache()
		self.create_role()
		self.create_docfield()
		self.create_docperm()
		self.create_docformat()
		self.create_doctype()

	def import_system_module(self):
		docs = [
			['DocType','Role']
			,['Role','Administrator']
			,['Role','Guest']
			,['Role','All']
			,['DocType','DocPerm']
			,['DocType','DocFormat']
			,['DocType','DocField']
			,['DocType','DocType']
			,['DocType','DefaultValue']
			,['DocType','Profile']
			,['DocType','UserRole']
		]
		
		# import in sequence
		for d in docs:
			import_module.import_from_files(record_list=[['System',d[0],d[1]]])
		
		# import all
		import_module.import_from_files([['System']])
			
	# singles
	# ------------------------------------------------------

	def create_singles(self):
		self.conn.sql("DROP TABLE IF EXISTS `tabSingles`")
		self.conn.sql("""CREATE TABLE `tabSingles` (
		  `doctype` varchar(40) default NULL,
		  `field` varchar(40) default NULL,
		  `value` text,
		  KEY `doctype` (`doctype`)
		) ENGINE=InnoDB DEFAULT CHARSET=latin1;""")
	
	# sessions

	def create_sessions(self):
		self.conn.sql("DROP TABLE IF EXISTS `tabSessions`;")
		self.conn.sql("""CREATE TABLE `tabSessions` (
		  `user` varchar(40) default NULL,
		  `sid` varchar(120) default NULL,
		  `sessiondata` longtext,
		  `ipaddress` varchar(16) default NULL,
		  `lastupdate` datetime default NULL
		) ENGINE=MyISAM DEFAULT CHARSET=latin1;""")

	# doc type cache

	def create_doctypecache(self):
		self.conn.sql("DROP TABLE IF EXISTS `__DocTypeCache`")
		self.conn.sql("create table `__DocTypeCache` (name VARCHAR(120), modified DATETIME, content TEXT, server_code_compiled TEXT)")
		
	# Role

	def create_role(self):
		self.conn.sql("DROP TABLE IF EXISTS `tabRole`")
		self.conn.sql("""CREATE TABLE `tabRole` (
		  `name` varchar(120) NOT NULL,
		  `creation` datetime default NULL,
		  `modified` datetime default NULL,
		  `modified_by` varchar(40) default NULL,
		  `owner` varchar(40) default NULL,
		  `docstatus` int(1) default '0',
		  `parent` varchar(120) default NULL,
		  `parentfield` varchar(120) default NULL,
		  `parenttype` varchar(120) default NULL,
		  `idx` int(8) default NULL,
		  `role_name` varchar(180) default NULL,
		  `module` varchar(180) default NULL,
		  PRIMARY KEY  (`name`),
		  KEY `parent` (`parent`)

	# DocField

	def create_docfield(self):
		self.conn.sql("DROP TABLE IF EXISTS `tabDocField`")
		self.conn.sql("""CREATE TABLE `tabDocField` (
		  `fieldname` varchar(180) default NULL,
		  `label` varchar(180) default NULL,
		  `oldfieldname` varchar(180) default NULL,
		  `fieldtype` varchar(180) default NULL,
		  `oldfieldtype` varchar(180) default NULL,
		  `options` text,
		  `search_index` int(3) default NULL,
		  `hidden` int(3) default NULL,
		  `print_hide` int(3) default NULL,
		  `report_hide` int(3) default NULL,
		  `reqd` int(3) default NULL,
		  `no_copy` int(3) default NULL,
		  `allow_on_submit` int(3) default NULL,
		  `trigger` varchar(180) default NULL,
		  `depends_on` varchar(180) default NULL,
		  `permlevel` int(3) default NULL,
		  `width` varchar(180) default NULL,
		  `default` text,
		  `description` text,
		  `colour` varchar(180) default NULL,
		  `icon` varchar(180) default NULL,
		  `in_filter` int(3) default NULL,
		  KEY `parent` (`parent`),
		  KEY `label` (`label`),
		  KEY `fieldtype` (`fieldtype`),
		  KEY `fieldname` (`fieldname`)

	# DocPerm

	def create_docperm(self):
		self.conn.sql("DROP TABLE IF EXISTS `tabDocPerm`")
		self.conn.sql("""CREATE TABLE `tabDocPerm` (
		  `permlevel` int(11) default NULL,
		  `role` varchar(180) default NULL,
		  `match` varchar(180) default NULL,
		  `read` int(3) default NULL,
		  `write` int(3) default NULL,
		  `create` int(3) default NULL,
		  `submit` int(3) default NULL,
		  `cancel` int(3) default NULL,
		  `amend` int(3) default NULL,
		  `execute` int(3) default NULL,
		
	# DocFormat

	def create_docformat(self):
		self.conn.sql("DROP TABLE IF EXISTS `tabDocFormat`")
		self.conn.sql("""CREATE TABLE `tabDocFormat` (
		  `format` varchar(180) default NULL,

	# DocType

	def create_doctype(self):
		self.conn.sql("DROP TABLE IF EXISTS `tabDocType`")
		self.conn.sql("""CREATE TABLE `tabDocType` (
		  `name` varchar(180) NOT NULL default '',
		  `owner` varchar(180) default NULL,
		  `search_fields` varchar(180) default NULL,
		  `issingle` int(1) default NULL,
		  `istable` int(1) default NULL,
		  `version` int(11) default NULL,
		  `autoname` varchar(180) default NULL,
		  `name_case` varchar(180) default NULL,
		  `read_only` int(1) default NULL,
		  `in_create` int(1) default NULL,
		  `show_in_menu` int(3) default NULL,
		  `menu_index` int(11) default NULL,
		  `parent_node` varchar(180) default NULL,
		  `smallicon` varchar(180) default NULL,
		  `allow_print` int(1) default NULL,
		  `allow_email` int(1) default NULL,
		  `allow_copy` int(1) default NULL,
		  `allow_rename` int(1) default NULL,
		  `hide_toolbar` int(1) default NULL,
		  `hide_heading` int(1) default NULL,
		  `allow_attach` int(1) default NULL,
		  `use_template` int(1) default NULL,
		  `max_attachments` int(11) default NULL,
		  `section_style` varchar(180) default NULL,
		  `client_script` text,
		  `client_script_core` text,
		  `server_code` text,
		  `server_code_core` text,
		  `server_code_compiled` text,
		  `client_string` text,
		  `server_code_error` varchar(180) default NULL,
		  `print_outline` varchar(180) default NULL,
		  `dt_template` text,
		  `is_transaction_doc` int(1) default NULL,
		  `change_log` text,
		  `read_only_onload` int(1) default NULL,
		
	def create_module_def(self):
		self.conn.sql("DROP TABLE IF EXISTS `tabModule Def`")
		self.conn.sql("CREATE TABLE `tabModule Def` (`name` varchar(120) NOT NULL, `creation` datetime default NULL, `modified` datetime 					default NULL,`modified_by` varchar(40) default NULL,`owner` varchar(40) default NULL,`docstatus` int(1) default '0',  					`parent` varchar(120) default NULL,`parentfield` varchar(120) default NULL, `parenttype` varchar(120) default NULL,				`idx` int(8) default NULL,`module_name` varchar(180) default NULL,`doctype_list` text,				 PRIMARY KEY  (`name`), KEY `parent` (`parent`)) ENGINE=InnoDB")

	def post_cleanup(self):
		self.conn.sql("use %s;" % self.db_name)
		self.conn.sql("update tabProfile set password = password('admin') where name='Administrator'")
		self.conn.sql("update tabDocType set server_code_compiled = NULL")	

		self.doc, self.doclist = d, dl
	
	def get_property_list(self, dt):
		return webnotes.conn.sql("""select fieldname, label, fieldtype 
		from tabDocField
		where parent=%s
		and fieldtype not in ('Section Break', 'Column Break', 'HTML', 'Read Only', 'Table') 
		and ifnull(fieldname, '') != ''
		order by label asc""", dt, as_dict=1)
		
	def get_setup_data(self):
		return {
			'doctypes': [d[0] for d in webnotes.conn.sql("select name from tabDocType")],
			'dt_properties': self.get_property_list('DocType'),
			'df_properties': self.get_property_list('DocField')
		}
		
	def get_field_ids(self):
		return webnotes.conn.sql("select name, fieldtype, label, fieldname from tabDocField where parent=%s", self.doc.doc_type, as_dict = 1)
	
		if self.doc.doc_type == self.doc.doc_name:
			return webnotes.conn.sql("select * from `tabDocType` where name=%s", self.doc.doc_name, as_dict = 1)[0]
		else:
			return webnotes.conn.sql("select * from `tabDocField` where name=%s", self.doc.doc_name, as_dict = 1)[0]

# main function
# -------------------------

def archive_doc(doctype, name, restore=0):
	archive_record(doctype, name, restore)
	
	tables = sql("select options from tabDocField where parent=%s and fieldtype='Table'", doctype)
	for t in tables:
		try:
			rec_list = sql("select name from `%s%s` where parent=%s" % ((restore and 'arc' or 'tab') ,t[0], '%s'), name)
			if e.args[0]==1146: # no child table
				rec_list = []
			else:
				raise e
		
		for r in rec_list:
			archive_record(t[0], r[0], restore)

# archive individual record

def archive_record(doctype, name, restore = 0):
	src_tab = (restore and 'arc' or 'tab') + doctype
	tar_tab = (restore and 'tab' or 'arc') + doctype

	# get the record
	try:
		rec = sql("select * from `%s` where name=%s for update" % (src_tab, '%s'), name, as_dict=1)[0]
			return # source table does not exist
		else:
			raise e
	
	# insert the record
	insert_record(doctype, tar_tab, name)
	
	# put it field by field (ignore missing columns)
	for field in rec.keys():
		if rec.get(field):
			update_value(src_tab, tar_tab, name, rec, field)

	# delete from main
	try:
		sql("delete from `%s` where name=%s limit 1" % (src_tab, '%s'), name)
			webnotes.msgprint("Cannot archive %s '%s' as it is referenced in another record. You must delete the referred record first" % (doctype, name))

			# delete from target, as it will create a double copy!
			sql("delete from `%s` where name=%s limit 1" % (tar_tab, '%s'), name)

# insert the record

def insert_record_name(tab, name):
	sql("insert ignore into `%s` (name) values (%s)" % (tab, '%s'), name)

# insert record

def insert_record(doctype, tar_tab, name):
	try:
		insert_record_name(tar_tab, name)
			# missing table - create it
			from webnotes.model.db_schema import updatedb
			updatedb(doctype, 1)
			
			# again
			insert_record_name(tar_tab, name)
		else:
			raise e

# update single value

def update_single_value(tab, field, value, name):
	sql("update `%s` set `%s`=%s where name=%s" % (tab, field, '%s', '%s'), (value, name))

def update_value(src_tab, tar_tab, name, rec, field):
	try:
		update_single_value(tar_tab, field, rec[field], name)
			# column missing.... add it?
			ftype = sql("show columns from `%s` like '%s'" % (src_tab, field))[0][1]
			
			webnotes.conn.commit() # causes implict commit
			sql("alter table `%s` add column `%s` %s" % (tar_tab, field, ftype))
			
			# again
			update_single_value(tar_tab, field, rec[field], name)
		else:
			raise e

"""
Imports Documents from modules (.txt) files in the filesystem
"""

#
# imports / updates all files in a module into the database
#
def import_module(module, verbose=0):
	"imports the all the records and files from the given module"
	from webnotes.modules import get_module_path
	import os
	
	not_module = ('startup', 'event_handlers', 'files', 'patches')
	if module in not_module: 
		if verbose: webnotes.msgprint('%s is not a module' % module)
		return
	
	path = get_module_path(module)
	
	doctypes = listfolders(path, 1)
	if 'doctype' in doctypes:
		doctypes.remove('doctype')
		doctypes = ['doctype'] + doctypes
	
	for doctype in doctypes:
		for docname in listfolders(os.path.join(path, doctype), 1):
			import_file(module, doctype, docname, path)
			if verbose: webnotes.msgprint('Imported %s/%s/%s' % (module, doctype, docname))
	
	import_attachments(module)

#
# get doclist from file
#
def get_doclist(path, doctype, docname):
	"returns a doclist (list of dictionaries) of multiple records for the given parameters"
	import os
	do_not_import = ('control_panel')
	
	fname = os.path.join(path,doctype,docname,docname+'.txt')
	if os.path.exists(fname) and (doctype not in do_not_import):
		f = open(fname,'r')
		dl = eval(f.read())
		f.close()
		return dl
	else:
		return None

#
# import a file into the database
#
def import_file(module, doctype, docname, path=None):
	"imports a given file into the database"
	
	if not path:
		from webnotes.modules import get_module_path
		path = get_module_path(module)
	
	doclist = get_doclist(path, doctype, docname)
	
		from webnotes.utils.transfer import set_doc
		set_doc(doclist, 1, 1, 1)

#
# list folders in a dir
#
def listfolders(path, only_name=0):
	"""returns the list of folders (with paths) in the given path, 
	if only_name is set, it returns only the folder names"""

	import os
	out = []
	for each in os.listdir(path):
		dirname = each.split(os.path.sep)[-1]
		fullpath = os.path.join(path, dirname)

		if os.path.isdir(fullpath) and not dirname.startswith('.'):
			out.append(only_name and dirname or fullname)
	return out

# Import from files
# =============================================================================
def import_from_files(modules = [], record_list = [], sync_cp = 0, target_db=None, target_ac=None):

	if target_db or target_ac:
		init_db_login(target_ac, target_db)

	from webnotes.utils import transfer
	# Get paths of folder which will be imported
	folder_list = get_folder_paths(modules, record_list)
	ret = []
		
	if folder_list:
		# get all doclist
		all_doclist = get_all_doclist(folder_list)
	
		# import doclist
		ret += accept_module(all_doclist)
	
		# import attachments
		for m in modules:
			import_attachments(m)
		
		# sync control panel
		if sync_cp:
			ret.append(sync_control_panel())
	else:
		ret.append("Module/Record not found")
		
	return ret

# Get list of folder path
# record_list in format [[module,dt,dn], ..]
def get_folder_paths(modules, record_list):
	import os
	import fnmatch
	from webnotes.modules import transfer_types, get_module_path, scrub

	folder_list=[]

	# get the folder list
	if record_list:
		for record in record_list:
			if scrub(record[1]) in ('doctype', 'page', 'search_criteria'):
				record[1], record[2] = scrub(record[1]), scrub(record[2])
			
			folder_list.append(os.path.join(get_module_path(scrub(record[0])), \
				record[1], record[2].replace('/','_')))

	if modules:
		# system modules will be transferred in a predefined order and before all other modules
		sys_mod_ordered_list = ['roles', 'core','application_internal', 'mapper', 'settings']
		all_mod_ordered_list = [t for t in sys_mod_ordered_list if t in modules] + list(set(modules).difference(sys_mod_ordered_list))
				
		for module in all_mod_ordered_list:
			mod_path = get_module_path(module)
			types_list = listfolders(mod_path, 1)
			
			# list of types
			types_list = list(set(types_list).difference(['control_panel']))
			all_transfer_types =[t for t in transfer_types if t in types_list] + list(set(types_list).difference(transfer_types))
			
			# build the folders
			for d in all_transfer_types:
				if d not in  ('files', 'startup', 'patches'):
					# get all folders inside type
					folder_list+=listfolders(os.path.join(mod_path, d))

	return folder_list

# Get doclist for all folder

def get_all_doclist(folder_list):
	import os
		
	all_doclist = []

	# build into doclist
	for folder in folder_list:
		# get the doclist
		file_list = os.listdir(folder)
		for each in file_list:

			if fnmatch.fnmatch(each,'*.txt'):
				doclist = eval(open(os.path.join(folder,each),'r').read())
				# add code
				all_doclist.append(doclist)
	
	return all_doclist
	
# accept a module coming from a remote server
def accept_module(super_doclist):
	msg, i = [], 0
	
	for dl in super_doclist:
		if dl[0]['doctype']!='Control Panel':
			msg.append(transfer.set_doc(dl, 1, 1, 1))
		
		if dl[0]['doctype']=='Module Def':
			update_module_timestamp(dl[0]['name'])

	if not webnotes.conn.in_transaction:
		webnotes.conn.sql("START TRANSACTION")
	
	# clear cache
	webnotes.conn.sql("DELETE from __DocTypeCache")
	webnotes.conn.sql("COMMIT")
	
	return msg

# Update timestamp in Module Def table
def update_module_timestamp(mod):
	import webnotes, webnotes.defs, os
	
	try:
		file = open(os.path.join(webnotes.defs.modules_path, mod, 'module.info'), 'r')
		if e.args[0]==2:
			return # module.info
		else:
			raise e
			
	module_info = eval(file.read())

def update_module_timestamp_query(mod, timestamp):
	webnotes.conn.sql("start transaction")
	webnotes.conn.sql("update `tabModule Def` set last_updated_date=%s where name=%s", (timestamp, mod))
	webnotes.conn.sql("commit")

# Import Attachments

def import_attachments(m):
	
	out = []
	
	# get list
	try:
		folder = os.path.join(get_module_path(m), 'files')
		fl = os.listdir(folder)
			return
		else:
			raise e
	
	# import files
	for f in fl:
		if not os.path.isdir(os.path.join(folder, f)):
			# delete
			webnotes.utils.file_manager.delete_file(f)
		
			# import
			file = open(os.path.join(folder, f),'r')
			webnotes.utils.file_manager.save_file(f, file.read(), m)
			
			out.append(f)
	
	return out

from webnotes.modules import scrub, get_module_path

def export_to_files(record_list=[], record_module=None, verbose=0):
	module_doclist =[]
			doclist = [d.fields for d in webnotes.model.doc.get(record[0], record[1])]
			write_document_file(doclist, record_module)

	return out

def create_init_py(modules_path, module, dt, dn):
	import os
	from webnotes.modules import scrub
	
	def create_if_not_exists(path):
		initpy = os.path.join(path, '__init__.py')
		if not os.path.exists(initpy):
			open(initpy, 'w').close()
	
	create_if_not_exists(os.path.join(modules_path, module))
	create_if_not_exists(os.path.join(modules_path, module, dt))
	create_if_not_exists(os.path.join(modules_path, module, dt, dn))
	
def create_folder(module, dt, dn):
	
	# get module path by importing the module
	modules_path = get_module_path(module)
			
	code_type = dt in ['DocType', 'Page', 'Search Criteria']
	
	# create folder
	folder = os.path.join(modules_path, code_type and scrub(dt) or dt, code_type and scrub(dn) or dn)
	
	# create init_py_files
	if code_type:
		create_init_py(modules_path, module, scrub(dt), scrub(dn))
	
	return folder

def get_module_name(doclist, record_module=None):
	# module name
	if doclist[0]['doctype'] == 'Module Def':
		module = doclist[0]['name']
	elif doclist[0]['doctype']=='Control Panel':
		module = 'Core'
	elif record_module:
		module = record_module
	else:
		module = doclist[0]['module']

	return module
	
def write_document_file(doclist, record_module=None):
	import os
	from webnotes.utils import pprint_dict

	module = get_module_name()

	# create the folder
	code_type = doclist[0]['doctype'] in ['DocType','Page','Search Criteria']
	
	folder = create_folder(module, doclist[0]['doctype'], doclist[0]['name'])
	
	# separate code files
	clear_code_fields(doclist, folder, code_type)
		
	# write the data file	
	fname = (code_type and scrub(doclist[0]['name'])) or doclist[0]['name']
	dict_list = [pprint_dict(d) for d in doclist]	
	
	txtfile = open(os.path.join(folder, fname +'.txt'),'w+')	
	txtfile.write('[\n' + ',\n'.join(dict_list) + '\n]')
	txtfile.close()

def clear_code_fields(doclist, folder, code_type):
	
	import os
	# code will be in the parent only
	code_fields = webnotes.code_fields_dict.get(doclist[0]['doctype'], [])
	
	for code_field in code_fields:
		if doclist[0].get(code_field[0]):

			doclist[0][code_field[0]] = None
		
session = webnotes.session
out = webnotes.response

def get_search_criteria_list(dt):
	sc_list = sql("select criteria_name, doc_type from `tabSearch Criteria` where doc_type = '%s' or parent_doc_type = '%s'" % (dt, dt))
	return [list(s) for s in sc_list]

def load_report_list():
	webnotes.response['rep_list'] = get_search_criteria_list(form.getvalue('dt'))

# Get, scrub metadata
# ====================================================================

def get_sql_tables(q):
	if q.find('WHERE') != -1:
		tl = q.split('FROM')[1].split('WHERE')[0].split(',')
	elif q.find('GROUP BY') != -1:
		tl = q.split('FROM')[1].split('GROUP BY')[0].split(',')
	else:
		tl = q.split('FROM')[1].split('ORDER BY')[0].split(',')
	return [t.strip().strip('`')[3:] for t in tl]

	pdt = ''
	if sql('select name from `tabDocType` where istable=1 and name="%s"' % dt):
		res = sql('select parent from `tabDocField` where fieldtype="Table" and options="%s"' % dt)
		if res: pdt = res[0][0]
	return pdt

def get_sql_meta(tl):
	std_columns = {
		'owner':('Owner', '', '', '100'), 
		'creation':('Created on', 'Date', '', '100'), 
		'modified':('Last modified on', 'Date', '', '100'), 
		'modified_by':('Modified By', '', '', '100')
	}
	
	meta = {}
	
		meta[dt] = std_columns.copy()

		# for table doctype, the ID is the parent id
		pdt = get_parent_dt(dt)
		if pdt: 
			meta[dt]['parent'] = ('ID', 'Link', pdt, '200')

		# get the field properties from DocField
		res = sql("select fieldname, label, fieldtype, options, width from tabDocField where parent='%s'" % dt)
			if r[0]:
				meta[dt][r[0]] = (r[1], r[2], r[3], r[4]);
				
		# name
		meta[dt]['name'] = ('ID', 'Link', dt, '200')
			
	return meta

# Additional conditions to fulfill match permission rules

def getmatchcondition(dt, ud, ur):
	res = sql("SELECT `role`, `match` FROM tabDocPerm WHERE parent = '%s' AND (`read`=1) AND permlevel = 0" % dt)
	cond = []
		if r[0] in ur: # role applicable to user
			if r[1]:
				defvalues = ud.get(r[1],['_NA'])
				for d in defvalues:
					cond.append('`tab%s`.`%s`="%s"' % (dt, r[1], d))
			else: # nomatch i.e. full read rights
				return ''

	return ' OR '.join(cond)
	
def add_match_conditions(q, tl, ur, ud):
	sl = []
		s = getmatchcondition(dt, ud, ur)
		if s: 
			sl.append(s)

	# insert the conditions
	if sl:
		condition_st  = q.find('WHERE')!=-1 and ' AND ' or ' WHERE '

		condition_end = q.find('ORDER BY')!=-1 and 'ORDER BY' or 'LIMIT'
		condition_end = q.find('GROUP BY')!=-1 and 'GROUP BY' or condition_end
		
		if q.find('ORDER BY')!=-1 or q.find('LIMIT')!=-1 or q.find('GROUP BY')!=-1: # if query continues beyond conditions
			q = q.split(condition_end)
			q = q[0] + condition_st + '(' + ' OR '.join(sl) + ') ' + condition_end + q[1]
		else:
			q = q + condition_st + '(' + ' OR '.join(sl) + ')'
			
	return q

# execute server-side script from Search Criteria

def exec_report(code, res, colnames=[], colwidths=[], coltypes=[], coloptions=[], filter_values={}, query='', from_export=0):
	col_idx, i, out, style, header_html, footer_html, page_template = {}, 0, None, [], '', '', ''
	for c in colnames:
		col_idx[c] = i
		i+=1
	
	# load globals (api)
	from webnotes import *
	from webnotes.utils import *
	from webnotes.model.doc import *
	from webnotes.model.doclist import getlist

	exec str(code)
	
	if out!=None:
		res = out

	return res, style, header_html, footer_html, page_template
	
def guess_type(m):
	"""
		Returns fieldtype depending on the MySQLdb Description
	"""
	import MySQLdb
	if m in MySQLdb.NUMBER:
		return 'Currency'
	elif m in MySQLdb.DATE:
		return 'Date'
	else:
		return 'Data'
		
def build_description_simple():
	colnames, coltypes, coloptions, colwidths = [], [], [], []

	for m in webnotes.conn.get_description():
		colnames.append(m[0])
		coltypes.append(guess_type[m[0]])
		coloptions.append('')
		colwidths.append('100')
	
	return colnames, coltypes, coloptions, colwidths

def build_description_standard(meta, tl):

	desc = webnotes.conn.get_description()

	# merged metadata - used if we are unable to
	# get both the table name and field name from
	# the description - in case of joins
	merged_meta = {}
	for d in meta:
		merged_meta.update(meta[d])

	for f in desc:
		fn, dt = f[0], ''
		if '.' in fn:
			dt, fn = fn.split('.')

		if (not dt) and merged_meta.get(fn):
			# no "AS" given, find type from merged description
			
			desc = merged_meta[fn]
			colnames.append(desc[0] or fn)
			coltypes.append(desc[1] or '')
			coloptions.append(desc[2] or '')
			colwidths.append(desc[3] or '100')
			
		elif meta.get(dt,{}).has_key(fn):
			# type specified for a multi-table join
			# usually from Report Builder
			
			desc = meta[dt][fn]
			
		else:
			# nothing found
			# guess
			
			colnames.append(fn)
			coltypes.append(guess_type(f[1]))
			coloptions.append('')
			colwidths.append('100')

# Entry Point - Run the query

def runquery(q='', ret=0, from_export=0):

	formatted = cint(form.getvalue('formatted'))	
	
	# CASE A: Simple Query
	if form.getvalue('simple_query') or form.getvalue('is_simple'):
		q = form.getvalue('simple_query') or form.getvalue('query')
		if q.split()[0].lower() != 'select':
			raise Exception, 'Query must be a SELECT'
		
		as_dict = cint(form.getvalue('as_dict'))
		res = sql(q, as_dict = as_dict, as_list = not as_dict, formatted=formatted)
		
		# build colnames etc from metadata
		colnames, coltypes, coloptions, colwidths = [], [], [], []
		
	# CASE B: Standard Query
	# -----------------------
	else:
		if not q: q = form.getvalue('query')

		tl = get_sql_tables(q)
		meta = get_sql_meta(tl)
			
		q = add_match_conditions(q, tl, webnotes.user.roles, webnotes.user.get_defaults())
		
		# replace special variables
		q = q.replace('__user', session['user'])
		q = q.replace('__today', webnotes.utils.nowdate())
		
		res = sql(q, as_list=1, formatted=formatted)

		colnames, coltypes, coloptions, colwidths = build_description_standard(meta, tl)
		
	# run server script
	style, header_html, footer_html, page_template = '', '', '', ''
	if form.has_key('sc_id') and form.getvalue('sc_id'):
		sc_id = form.getvalue('sc_id')
		sc_details = webnotes.conn.sql("select module, standard, server_script from `tabSearch Criteria` where name=%s", sc_id)[0]
		if sc_details[1]!='No':	
			code = get_code(sc_details[0], 'Search Criteria', sc_id, 'py')
		else:
			code = sc_details[2]
			
		if code:
			filter_values = form.has_key('filter_values') and eval(form.getvalue('filter_values','')) or {}
			res, style, header_html, footer_html, page_template = exec_report(code, res, colnames, colwidths, coltypes, coloptions, filter_values, q, from_export)
		
	out['colnames'] = colnames
	out['coltypes'] = coltypes
	out['coloptions'] = coloptions
	out['colwidths'] = colwidths
	out['header_html'] = header_html
	out['footer_html'] = footer_html
	out['page_template'] = page_template
	
	if style:
		out['style'] = style
	
	# just the data - return
	if ret==1:
		return res	

	out['values'] = res

	# return num of entries 
	qm = form.has_key('query_max') and form.getvalue('query_max') or ''
	if qm and qm.strip():
		if qm.split()[0].lower() != 'select':
			raise Exception, 'Query (Max) must be a SELECT'
		if not form.has_key('simple_query'):
			qm = add_match_conditions(qm, tl, webnotes.user.roles, webnotes.user.defaults)

		out['n_values'] = webnotes.utils.cint(sql(qm)[0][0])

# Export to CSV

def runquery_csv():

	# run query
	res = runquery(from_export = 1)
	
	q = form.getvalue('query')
	
	rep_name = form.getvalue('report_name')
	if not form.has_key('simple_query'):

		# Report Name
		if not rep_name:
			rep_name = get_sql_tables(q)[0]
	
	if not rep_name: rep_name = 'DataExport'
	
	# Headings
	heads = []
	for h in out['colnames']:
		heads.append(getCSVelement(h))
	if form.has_key('colnames'):
		for h in form.getvalue('colnames').split(','):
			heads.append(getCSVelement(h))

	# Output dataset
	dset = [rep_name, '']
	if heads:
		dset.append(','.join(heads))
	
	# Data
	for r in out['values']:
		dset.append(','.join([getCSVelement(i) for i in r]))
		
	out['result'] = txt
	out['type'] = 'csv'
	out['doctype'] = rep_name

"""
Load compressed .js page scripts

Will also replace $import(page) or $import(module.page) with the relevant js files
"""

# load "compressed" js code from modules

def get_js_code(fn, extn='js'):
	"""
		Get js code from a file (uncompressed)
	"""
	from webnotes.modules import scrub, get_file_timestamp

	src_file_name = fn + '.' + extn

	# src_timestamp = get_file_timestamp(src_file_name)
	
	# if no source, return
	#if not src_timestamp:
	#	return ''
		
	# if timestamps are not same, compress
	#if src_timestamp != get_file_timestamp(comp_file_name):
	#	compress(src_file_name, comp_file_name)
		
	# get the code
	try:
		file = open(src_file_name, 'r')
		return ''
		
	code = file.read()
	
	# return

# get doctype client

def sub_get_doctype_js(match):
	return get_doctype_js(name)

def get_doctype_js(dt):
	"""
		Returns the client-side (js) code of the DocType.
		Adds custom script
		and replaces $import(dt) with the code of that DocType
	"""
	from webnotes.model.code import get_custom_script
	
	module = scrub(webnotes.conn.get_value('DocType',dt,'module'))

	code = get_js_code(os.path.join(get_module_path(scrub(module)), 'doctype', scrub(dt), scrub(dt))) \
		+ '\n' + (get_custom_script(dt, 'Client') or '')
		
	# compile for import
	if code.strip():
		import re
		p = re.compile('\$import\( (?P<name> [^)]*) \)', re.VERBOSE)
	
		code = p.sub(sub_get_doctype_js, code)

# get page client

def sub_get_page_js(match):
	
	if '.' in name:
		name = name.split('.')
		return get_page_js(name[1], name[0])
	
	return get_page_js(Document('Page', name))

def get_page_js(page, module=None):
	"""
	Returns the js code of a page. Will replace $import (page) or $import(module.page)
	with the code from the file
	"""

	if type(page)==str:
		page_name = page
	else:
		page_name, module = page.name, page.module

	code = get_js_code(os.path.join(get_module_path(module), 'page', scrub(page_name), scrub(page_name)))
		
	if not code and type(page)!=str:
		code = page.script
	
	if code and code.strip():
		import re
	
		code = p.sub(sub_get_page_js, code)

# compress

def compress(src, comp):
	out = open(comp, 'w')
	
	jsm = JavascriptMinify()
	jsm.minify(open(src,'r'), out)
	
	out.close()

#
# license.
#
#	2007-05-22
#
#
#
#
#
# */

	ins = StringIO(js)
	outs = StringIO()
	JavascriptMinify().minify(ins, outs)
	str = outs.getvalue()
	if len(str) > 0 and str[0] == '\n':
		str = str[1:]
	return str

	"""return true if the character is a letter, digit, underscore,
		   dollar sign, or non-ASCII character.
	"""
	return ((c >= 'a' and c <= 'z') or (c >= '0' and c <= '9') or
			(c >= 'A' and c <= 'Z') or c == '_' or c == '$' or c == '\\' or (c is not None and ord(c) > 126));

	pass

	pass

	pass

	def _outA(self):
		self.outstream.write(self.theA)
	def _outB(self):
		self.outstream.write(self.theB)

	def _get(self):
		"""return the next character from stdin. Watch out for lookahead. If
		   the character is a control character, translate it to a space or
		   linefeed.
		"""
		c = self.theLookahead
		self.theLookahead = None
		if c == None:
			c = self.instream.read(1)
		if c >= ' ' or c == '\n':
			return c
		if c == '': # EOF
			return '\000'
		if c == '\r':
			return '\n'
		return ' '

	def _peek(self):
		self.theLookahead = self._get()
		return self.theLookahead

	def _next(self):
		"""get the next character, excluding comments. peek() is used to see
		   if an unescaped '/' is followed by a '/' or '*'.
		"""
		c = self._get()
		if c == '/' and self.theA != '\\':
			p = self._peek()
			if p == '/':
				c = self._get()
				while c > '\n':
					c = self._get()
				return c
			if p == '*':
				while 1:
					if c == '*':
						if self._peek() == '/':
							self._get()
							return ' '
					if c == '\000':
						raise UnterminatedComment()

		return c

	def _action(self, action):
		"""do something! What you do is determined by the argument:
		   1   Output A. Copy B to A. Get the next B.
		   2   Copy B to A. Get the next B. (Delete A).
		   3   Get the next B. (Delete B).
		   action treats a string as a single character. Wow!
		   action recognizes a regular expression if it is preceded by ( or , or =.
		"""
		if action <= 1:
			self._outA()

		if action <= 2:
			self.theA = self.theB
			if self.theA == "'" or self.theA == '"':
				while 1:
					self._outA()
					self.theA = self._get()
					if self.theA == self.theB:
						break
					if self.theA <= '\n':
						raise UnterminatedStringLiteral()
					if self.theA == '\\':
						self._outA()
						self.theA = self._get()

		if action <= 3:
			self.theB = self._next()
			if self.theB == '/' and (self.theA == '(' or self.theA == ',' or
									 self.theA == '=' or self.theA == ':' or
									 self.theA == '[' or self.theA == '?' or
									 self.theA == '!' or self.theA == '&' or
									 self.theA == '|' or self.theA == ';' or
									 self.theA == '{' or self.theA == '}' or
									 self.theA == '\n'):
				self._outA()
				self._outB()
				while 1:
					if self.theA == '/':
						break
					elif self.theA == '\\':
					elif self.theA <= '\n':
						raise UnterminatedRegularExpression()
				self.theB = self._next()

	def _jsmin(self):
		"""Copy the input to the output, deleting the characters which are
		   insignificant to JavaScript. Comments will be removed. Tabs will be
		   replaced with spaces. Carriage returns will be replaced with linefeeds.
		   Most spaces and linefeeds will be removed.
		"""
		self.theA = '\n'
		self._action(3)

		while self.theA != '\000':
			if self.theA == ' ':
				if isAlphanum(self.theB):
					self._action(1)
				else:
					self._action(2)
			elif self.theA == '\n':
				if self.theB in ['{', '[', '(', '+', '-']:
				elif self.theB == ' ':
					self._action(3)
				else:
					if isAlphanum(self.theB):
						self._action(1)
					else:
						self._action(2)
			else:
				if self.theB == ' ':
					if isAlphanum(self.theA):
					else:
						self._action(3)
				elif self.theB == '\n':
					if self.theA in ['}', ']', ')', '+', '-', '"', '\'']:
					else:
						if isAlphanum(self.theA):
							self._action(1)
						else:
							self._action(3)
				else:

	def minify(self, instream, outstream):
		self.instream = instream
		self.outstream = outstream
		self.theB = None

		self._jsmin()
		self.instream.close()

#
# import modules path
# 
import os, sys

try:
	m = getattr(webnotes.defs,'modules_path',None)
	m and sys.path.append(m)
except Exception,e:
	raise e

#
# map for identifying which field values come from files
#
code_fields_dict = {
	'Page':[('script', 'js'), ('content', 'html'), ('style', 'css'), ('static_content', 'html'), ('server_code', 'py')],
	'DocType':[('server_code_core', 'py'), ('client_script_core', 'js')],
	'Search Criteria':[('report_script', 'js'), ('server_script', 'py'), ('custom_query', 'sql')],
	'Patch':[('patch_code', 'py')],
	'Stylesheet':['stylesheet', 'css'],
	'Page Template':['template', 'html'],
	'Control Panel':[('startup_code', 'js'), ('startup_css', 'css')]
}

#
# globals
#
#: "v170" 
version = 'v170'
form_dict = {}
auth_obj = None

#: The database connection :class:`webnotes.db.Database` setup by :mod:`auth`
conn = None

#: The cgi.FieldStorage() object (Dictionary representing the formdata from the URL)
form = None

"""
   Global session dictionary.

   * session['user'] - Current user   
   * session['data'] - Returns a dictionary of the session cache
"""

user = None
is_testing = None
"""    Flag to identify if system is in :term:`Testing Mode` """

incoming_cookies = {}
add_cookies = {}
"""    Dictionary of additional cookies appended by custom code """

cookies = {}
auto_masters = {}
tenant_id = None

#
# Custom Class (no traceback)
#
class ValidationError(Exception):
	pass

#
# HTTP standard response
#
response = {'message':'', 'exc':''}
"""
   The JSON response object. Default is::
   
   {'message':'', 'exc':''}
"""

#
# the logs
#
debug_log = []
"""    List of exceptions to be shown in the :term:`Error Console` """

message_log = []
"""    List of messages to be shown to the user in a popup box at the end of the request """

	return webnotes.utils.getTraceback()

def errprint(msg):
	"""
	   Append to the :data:`debug log`
	"""
	debug_log.append(str(msg or ''))

def msgprint(msg, small=0, raise_exception=0):
	"""
	   Append to the :data:`message_log`
	"""	
	message_log.append((small and '__small:' or '')+str(msg or ''))
	if raise_exception:
		raise ValidationError

def is_apache_user():
	import os
	if os.environ.get('USER') == 'apache':
		return True 
	else:
		return (not os.environ.get('USER'))
		# os.environ does not have user, so allows a security vulnerability,fixed now. 

def get_index_path():
	import os
	return os.sep.join(os.path.dirname(os.path.abspath(__file__)).split(os.sep)[:-2])

def get_files_path():
	global conn
	import defs, os

	if not conn:
		raise Exception, 'You must login first'

	if defs.files_path:
		return os.path.join(defs.files_path, conn.cur_db_name)
	else:
		return os.path.join(get_index_path(), 'user_files', conn.cur_db_name)
	
def create_folder(path):
	"""
	Wrapper function for os.makedirs (does not throw exception if directory exists)
	"""
	import os
	
	try:
		os.makedirs(path)
		if e.args[0]==17: 
			pass
		else: 
			raise e

###############################################################################
#	BEGIN: TENTATIVE CODE FEELS LIKE A CLASS/TEMPLATE IS A BETTER IDEA FOR THESE VARIABLES.
#	Bad idea combining/using one function to set conn,user,session variables.
#	Need to split up.

def set_as_account_master():
	conn = webnotes.db.Database(use_default = 1)

def set_as_administrator():
	
	global user
	
	if is_apache_user():
		raise Exception, 'Not for web users!'

	user = webnotes.profile.Profile('Administrator')

def set_as_admin_session():
	global session
	session = {'user':'Administrator'}

#END  

def set_as_admin(db_name=None, ac_name=None):

	import os

		conn = webnotes.db.Database(ac_name = ac_name)
	else:
		set_as_account_master()
		if db_name:
			conn.use(db_name)
		
# Environment Variables
#-----------------------------------------------------------
def get_env_vars(env_var):
	return os.environ.get(env_var,'None')

remote_ip = get_env_vars('REMOTE_ADDR')		#Required for login from python shell

# Logging
# -----------------------------------------------------------

logger = None

def setup_logging():
	import logging
	import logging.handlers
	# Also please set umask for apache to 002.
	global logger

	try:
		logger = logging.getLogger('WNLogger')
		logger.setLevel(eval(defs.log_level))

		log_handler = logging.handlers.RotatingFileHandler(defs.log_file_name, maxBytes = defs.log_file_size, backupCount = defs.log_file_backup_count)	
		formatter = logging.Formatter('%(name)s - %(asctime)s - %(levelname)s\n%(message)s\n-------------------')
	
		log_handler.setFormatter(formatter)
		logger.addHandler(log_handler)
	
	except IOError,e:
		if e.args == 13:
			open(defs.log_file_name).close()

if getattr(defs, 'log_file_name', None):
	setup_logging()
	
def sendmail_html(sender, recipients, subject, html, text=None, template=None, send_now=1, reply_to=None):
	"""
		Send an html mail with alternative text and using Page Templates
	"""
	sendmail(recipients, sender, html, subject, send_now = send_now, reply_to = reply_to, template = template)

def make_html_body(content, template = None):
	"""
		Generate html content from a Page Template object
	"""
	template_html = '%(content)s'

	if template:
		template_html = get_code(webnotes.conn.get_value('Page Template', template, 'module'), 'Page Template', template, 'html', fieldname='template')

	return template_html % {'content': content}

def sendmail(recipients, sender='', msg='', subject='[No Subject]', parts=[], cc=[], attach=[], send_now=1, reply_to=None, template=None):
	"""
		send an html email as multipart with attachments and all
	"""

	from webnotes.utils.email_lib.html2text import html2text
	from webnotes.utils.email_lib.send import EMail
		
	email = EMail(sender, recipients, subject, reply_to=reply_to)
	email.cc = cc
		
	if msg:
		if template:
			msg = make_html_body(msg, template)
		else:
			# if not html, then lets put some whitespace
			if (not '<br>' in msg) or (not '<p>' in msg):
				msg = msg.replace('\n','<br>')
			
		msg = msg + (footer or '')
		email.set_text(html2text(msg))
		email.set_html(msg)
	for p in parts:
		email.set_message(p[1])
	for a in attach:
		email.attach(a)

	email.send(send_now)

def get_footer():
	"""
		Returns combination of footer from globals and Control Panel
	"""

	footer = webnotes.conn.get_value('Control Panel',None,'mail_footer') or ''
	footer += (webnotes.conn.get_global('global_mail_footer') or '')
	return footer

def send_form():
	"""
		Emails a print format (form)
		Called from form UI
	"""
	
	from webnotes.utils.email_lib.form_email import FormEmail
	FormEmail().send()

def get_contact_list():
	"""
		Returns contacts (from autosuggest)
	"""

	cond = ['`%s` like "%s%%"' % (f, webnotes.form.getvalue('txt')) for f in webnotes.form.getvalue('where').split(',')]
	cl = webnotes.conn.sql("select `%s` from `tab%s` where %s" % (
  			 webnotes.form.getvalue('select')
			,webnotes.form.getvalue('from')
			,' OR '.join(cond)
		)
	)
	webnotes.response['cl'] = filter(None, [c[0] for c in cl])
	
# Modules
# -----------

def get_module_items(mod, only_dt=0):
	dl = []
	if only_dt:
		transfer_types = ['DocType']
	else:
		transfer_types = ['Role', 'Page', 'DocType', 'DocType Mapper', 'Search Criteria']
		dl = ['Module Def,'+mod]
	
	for dt in transfer_types:
		try:
			dl2 = sql('select name from `tab%s` where module="%s"' % (dt,mod))
			dl += [(dt+','+e[0]) for e in dl2]
		except:
			pass

	if not only_dt:
		dl1 = sql('select doctype_list from `tabModule Def` where name=%s', mod)
		dl += dl1[0][0].split('\n')
	
	# build finally
	dl = [e.split(',') for e in dl]
	dl = [[e[0].strip(), e[1].strip()] for e in dl] # remove blanks
	return dl

get_defaults() (core.doctype.property_setter.property_setter.DocType method)

get_field_ids() (core.doctype.property_setter.property_setter.DocType method)

get_setup_data() (core.doctype.property_setter.property_setter.DocType method)

add_section_breaks_and_renum() (webnotes.utils.transfer.UpdateDocType method)

clear_section_breaks() (webnotes.utils.transfer.UpdateDocType method)

fix_extra_fields() (webnotes.utils.transfer.UpdateDocType method)

get_extra_fields() (webnotes.utils.transfer.UpdateDocType method)

get_orignal_values() (webnotes.utils.transfer.UpdateDocType method)

get_orignal_values() (webnotes.utils.transfer.UpdateDocTypeMapper method)

delete_existing() (webnotes.utils.transfer.UpdateDocument method)

update_modified() (webnotes.utils.transfer.UpdateDocument method)

child_exists() (webnotes.utils.transfer.UpdateDocumentMerge method)

delete_existing() (webnotes.utils.transfer.UpdateDocumentMerge method)

keep_values() (webnotes.utils.transfer.UpdateDocumentMerge method)

save_children() (webnotes.utils.transfer.UpdateDocumentMerge method)

to_update() (webnotes.utils.transfer.UpdateDocumentMerge method)

get_orignal_values() (webnotes.utils.transfer.UpdateModuleDef method)

http_get_response() (webnotes.utils.webservice.FrameworkServer method)

runserverobj() (webnotes.utils.webservice.FrameworkServer method)

load_default_properties() (in module webnotes.widgets.page_body)

add_match_conditions() (in module webnotes.widgets.query_builder)

build_description_simple() (in module webnotes.widgets.query_builder)

build_description_standard() (in module webnotes.widgets.query_builder)

get_search_criteria_list() (in module webnotes.widgets.query_builder)

write_document_file() (in module webnotes.modules.export_module)

update_module_timestamp() (in module webnotes.modules.import_module)

update_module_timestamp_query() (in module webnotes.modules.import_module)

get_last_update_for() (in module webnotes.modules.module_manager)

get_module_details() (in module webnotes.modules.module_manager)

get_module_items_from_files() (in module webnotes.modules.module_manager)

get_modules_from_filesystem() (in module webnotes.modules.module_manager)

drop_all_databases() (webnotes.model.db_schema.DbManager method)

grant_all_privileges() (webnotes.model.db_schema.DbManager method)

grant_select_privilges() (webnotes.model.db_schema.DbManager method)

set_transaction_isolation_level() (webnotes.model.db_schema.DbManager method)

get_column_definitions() (webnotes.model.db_schema.DbTable method)

get_columns_from_docfields() (webnotes.model.db_schema.DbTable method)

get_index_definitions() (webnotes.model.db_schema.DbTable method)

check_select_link_data() (webnotes.model.import_docs.CSVImport method)

convert_csv_data_into_list() (webnotes.model.import_docs.CSVImport method)

get_field_type_list() (webnotes.model.import_docs.CSVImport method)

validate_doctype() (webnotes.model.import_docs.CSVImport method)

validate_headers() (webnotes.model.import_docs.CSVImport method)

on_update() (core.doctype.page_template.page_template.DocType method)

create_base_tables() (webnotes.install_lib.db_init.DatabaseInstance method)

create_db_and_user() (webnotes.install_lib.db_init.DatabaseInstance method)

create_docfield() (webnotes.install_lib.db_init.DatabaseInstance method)

create_docformat() (webnotes.install_lib.db_init.DatabaseInstance method)

create_docperm() (webnotes.install_lib.db_init.DatabaseInstance method)

create_doctype() (webnotes.install_lib.db_init.DatabaseInstance method)

create_doctypecache() (webnotes.install_lib.db_init.DatabaseInstance method)

create_module_def() (webnotes.install_lib.db_init.DatabaseInstance method)

create_role() (webnotes.install_lib.db_init.DatabaseInstance method)

create_sessions() (webnotes.install_lib.db_init.DatabaseInstance method)

create_singles() (webnotes.install_lib.db_init.DatabaseInstance method)

import_system_module() (webnotes.install_lib.db_init.DatabaseInstance method)

post_cleanup() (webnotes.install_lib.db_init.DatabaseInstance method)

framework_cleanups() (webnotes.install_lib.install.Installer method)

import_core_module() (webnotes.install_lib.install.Installer method)

import_from_db() (webnotes.install_lib.install.Installer method)

build_message() (webnotes.utils.email_lib.form_email.FormEmail method)

get_form_link() (webnotes.utils.email_lib.form_email.FormEmail method)

make_full_links() (webnotes.utils.email_lib.form_email.FormEmail method)

set_attachments() (webnotes.utils.email_lib.form_email.FormEmail method)

update_contacts() (webnotes.utils.email_lib.form_email.FormEmail method)

replaceEntities() (in module webnotes.utils.email_lib.html2text)

get_attachment() (webnotes.utils.email_lib.receive.IncomingMail method)

get_charset() (webnotes.utils.email_lib.receive.IncomingMail method)

get_payload() (webnotes.utils.email_lib.receive.IncomingMail method)

get_text_content() (webnotes.utils.email_lib.receive.IncomingMail method)

get_thread_id() (webnotes.utils.email_lib.receive.IncomingMail method)

process_part() (webnotes.utils.email_lib.receive.IncomingMail method)

check_mails() (webnotes.utils.email_lib.receive.POP3Mailbox method)

get_messages() (webnotes.utils.email_lib.receive.POP3Mailbox method)

process_message() (webnotes.utils.email_lib.receive.POP3Mailbox method)

get_smtp_session() (webnotes.utils.email_lib.send.EmailQueue method)