Install fixes, started to rewrite translate.py webnotes/wnframework#351

setup.py

@@ -18,7 +18,7 @@ setup(
     install_requires=install_requires,
         entry_points= {
                 'console_scripts':[
-                        'webnotes = webnotes.wnf:main'
+                        'webnotes = webnotes.cli:main'
                         ]
                 }
 )

webnotes/__init__.py

@@ -98,6 +98,8 @@ def init(site, sites_path=None):
 	local.initialised = True
 	local.flags = _dict({})
 	local.rollback_observers = []
+	local.module_app = None
+	local.app_modules = None
 
 	setup_module_map()
 
@@ -415,13 +417,15 @@ def get_pymodule_path(modulename, *joins):
 def get_module_list(app_name):
 	return get_file_items(os.path.join(os.path.dirname(get_module(app_name).__file__), "modules.txt"))
 
-def get_app_list(with_webnotes=False):
+def get_all_apps(with_webnotes=False):
 	apps = get_file_items(os.path.join(local.sites_path, "apps.txt"))
 	if with_webnotes:
 		apps.insert(0, 'webnotes')
 	return apps
 
 def get_installed_apps():
+	if flags.in_install_db:
+		return []
 	def load_installed_apps():
 		return json.loads(conn.get_global("installed_apps") or "[]")
 	return cache().get_value("installed_apps", load_installed_apps)
@@ -440,19 +444,22 @@ def get_hooks():
 
 def setup_module_map():
 	_cache = cache()
-	local.app_modules = _cache.get_value("app_modules")
-	local.module_app = _cache.get_value("module_app")
+	
+	if conf.db_name:
+		local.app_modules = _cache.get_value("app_modules")
+		local.module_app = _cache.get_value("module_app")
 	
 	if not local.app_modules:
 		local.module_app, local.app_modules = {}, {}
-		for app in get_app_list(True):
+		for app in get_all_apps(True):
 			for module in get_module_list(app):
 				local.module_app[module] = app
 				local.app_modules.setdefault(app, [])
 				local.app_modules[app].append(module)
-				
-		_cache.set_value("app_modules", local.app_modules)
-		_cache.set_value("module_app", local.module_app)
+	
+		if conf.db_name:
+			_cache.set_value("app_modules", local.app_modules)
+			_cache.set_value("module_app", local.module_app)
 		
 def get_file_items(path):
 	if os.path.exists(path):
@@ -546,7 +553,7 @@ def get_jenv():
 
 		# webnotes will be loaded last, so app templates will get precedence
 		jenv = Environment(loader = ChoiceLoader([PackageLoader(app, ".") \
-			for app in get_app_list() + ["webnotes"]]))
+			for app in get_all_apps() + ["webnotes"]]))
 
 		jenv.filters["global_date_format"] = global_date_format
 		jenv.filters["markdown"] = markdown

webnotes/build.py

@@ -42,7 +42,7 @@ def make_site_public_dirs():
 			os.makedirs(dir_path)
 	
 	# symlink app/public > assets/app
-	for app_name in webnotes.get_app_list(True):
+	for app_name in webnotes.get_all_apps(True):
 		pymodule = webnotes.get_module(app_name)
 		source = os.path.join(os.path.abspath(os.path.dirname(pymodule.__file__)), 'public')
 		target = os.path.join(assets_path, app_name)
@@ -82,7 +82,7 @@ def build(no_compress=False):
 def get_build_maps():
 	"""get all build.jsons with absolute paths"""
 	# framework js and css files
-	pymodules = [webnotes.get_module(app) for app in webnotes.get_app_list(True)]
+	pymodules = [webnotes.get_module(app) for app in webnotes.get_all_apps(True)]
 	app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]
 
 	build_maps = {}

webnotes/cli.py

@@ -211,7 +211,7 @@ def setup_translation(parser):
 
 # install
 @cmd
-def install(root_login="root", root_password=None, db_name=None, source_sql=None,
+def install(db_name, root_login="root", root_password=None, source_sql=None,
 	 admin_password = 'admin', verbose=True, force=False, site_config=None):
 	from webnotes.installer import install_db, install_app
 	install_db(root_login=root_login, root_password=root_password, db_name=db_name, source_sql=source_sql,
@@ -220,10 +220,10 @@ def install(root_login="root", root_password=None, db_name=None, source_sql=None
 	webnotes.destroy()
 
 @cmd
-def install_app(app):
+def install_app(app_name, verbose=False):
 	from webnotes.installer import install_app
 	webnotes.connect()
-	install_app(app, verbose=True)
+	install_app(app_name, verbose=verbose)
 	webnotes.destroy()
 
 @cmd

webnotes/core/doctype/notification_count/notification_count.py

@@ -50,6 +50,7 @@ def get_notifications():
 	}
 
 def delete_notification_count_for(doctype):
+	if webnotes.flags.in_import: return
 	webnotes.conn.sql("""delete from `tabNotification Count` where for_doctype = %s""", doctype)
 
 def delete_event_notification_count():

webnotes/data/Framework.sql

@@ -243,3 +243,26 @@ CREATE TABLE `tabFile Data` (
   KEY `attached_to_name` (`attached_to_name`),
   KEY `attached_to_doctype` (`attached_to_doctype`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+--
+-- Table structure for table `tabDefaultValue`
+--
+
+DROP TABLE IF EXISTS `tabDefaultValue`;
+CREATE TABLE `tabDefaultValue` (
+  `name` varchar(120) NOT NULL,
+  `creation` datetime DEFAULT NULL,
+  `modified` datetime DEFAULT NULL,
+  `modified_by` varchar(40) DEFAULT NULL,
+  `owner` varchar(40) DEFAULT NULL,
+  `docstatus` int(1) DEFAULT '0',
+  `parent` varchar(120) DEFAULT NULL,
+  `parentfield` varchar(120) DEFAULT NULL,
+  `parenttype` varchar(120) DEFAULT NULL,
+  `idx` int(8) DEFAULT NULL,
+  `defvalue` text,
+  `defkey` varchar(180) DEFAULT NULL,
+  PRIMARY KEY (`name`),
+  KEY `parent` (`parent`),
+  KEY `defaultvalue_parent_defkey_index` (`parent`,`defkey`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

webnotes/data/languages.json

@@ -1,19 +0,0 @@
-{
-	"中国（简体）": "zh-cn",
-	"中國（繁體）": "zh-tw",
-	"deutsch": "de",
-	"ελληνικά": "el",
-	"english": "en",
-	"español": "es",
-	"français": "fr",
-	"हिंदी": "hi",
-	"hrvatski": "hr",
-	"italiano": "it",
-	"nederlands": "nl",
-	"português brasileiro": "pt-BR",
-	"português": "pt",
-	"српски":"sr",
-	"தமிழ்": "ta",
-	"ไทย": "th",
-	"العربية":"ar"
-}

webnotes/data/languages.txt

@@ -0,0 +1,17 @@
+ar		العربية
+de		deutsch
+el		ελληνικά
+en		english
+es		español
+fr		français
+hi		हिंदी
+hr		hrvatski
+it		italiano
+nl		nederlands
+pt-BR	português brasileiro
+pt		português
+sr		српски
+ta		தமிழ்
+th		ไทย
+zh-cn	中国（简体
+zh-tw	中國（繁體

webnotes/installer.py

@@ -16,7 +16,8 @@ from webnotes.model.sync import sync_for
 from webnotes.utils import cstr
 
 def install_db(root_login="root", root_password=None, db_name=None, source_sql=None,
-	 admin_password = 'admin', verbose=0, force=0, site_config=None):
+	admin_password = 'admin', verbose=True, force=0, site_config=None):
+	webnotes.flags.in_install_db = True
 	make_conf(db_name, site_config=site_config)
 	webnotes.local.conn = make_connection(root_login, root_password)
 	webnotes.local.session = webnotes._dict({'user':'Administrator'})
@@ -27,6 +28,7 @@ def install_db(root_login="root", root_password=None, db_name=None, source_sql=N
 	import_db_from_sql(source_sql, verbose)
 	
 	create_auth_table()
+	webnotes.flags.in_install_db = False
 
 def create_database_and_user(force, verbose):
 	db_name = webnotes.conf.db_name
@@ -73,6 +75,7 @@ def make_connection(root_login, root_password):
 	return webnotes.db.Database(user=root_login, password=root_password)
 
 def install_app(name, verbose=False):
+	webnotes.flags.in_install_app = True
 	webnotes.clear_cache()
 	manage = webnotes.get_module(name + ".manage") 
 	if hasattr(manage, "before_install"):
@@ -87,12 +90,14 @@ def install_app(name, verbose=False):
 	installed_apps = json.loads(webnotes.conn.get_global("installed_apps") or "[]") or []
 	installed_apps.append(name)
 	webnotes.conn.set_global("installed_apps", json.dumps(installed_apps))
+	webnotes.clear_cache()
 	webnotes.conn.commit()
-
+	
 	from webnotes.website.doctype.website_sitemap_config.website_sitemap_config import rebuild_website_sitemap_config
 	rebuild_website_sitemap_config()
 
 	webnotes.clear_cache()
+	webnotes.flags.in_install_app = False
 	
 
 def set_all_patches_as_completed(app):
@@ -112,6 +117,7 @@ def make_conf(db_name=None, db_password=None, site_config=None):
 	webnotes.init(site)
 
 def make_site_config(db_name=None, db_password=None, site_config=None):		
+	webnotes.create_folder(os.path.join(webnotes.local.site_path))
 	site_file = os.path.join(webnotes.local.site_path, "site_config.json")
 	
 	if not os.path.exists(site_file):

webnotes/manage.py

@@ -11,6 +11,9 @@ def get_hooks():
 	}
 
 def after_install():
+	# reset installed apps for re-install
+	webnotes.conn.set_global("installed_apps", "[]")
+	
 	# core users / roles
 	install_docs = [
 		{'doctype':'Profile', 'name':'Administrator', 'first_name':'Administrator', 

webnotes/memc.py

@@ -2,18 +2,17 @@
 # MIT License. See license.txt 
 from __future__ import unicode_literals
 
-import memcache
-from webnotes import conf
+import memcache, webnotes
 
 class MClient(memcache.Client):
 	"""memcache client that will automatically prefix conf.db_name"""
 	def n(self, key):
-		return (conf.db_name + ":" + key.replace(" ", "_")).encode('utf-8')
+		return (webnotes.conf.db_name + ":" + key.replace(" ", "_")).encode('utf-8')
 	
 	def set_value(self, key, val):
 		self.set(self.n(key), val)
 		
-	def get_value(self, key, builder=None):			
+	def get_value(self, key, builder=None):
 		val = self.get(self.n(key))
 		if not val and builder:
 			val = builder()

webnotes/translate.py

@@ -43,13 +43,8 @@ def translate(lang=None):
 	os.remove('_lang_tmp.csv')
 
 def get_all_languages():
-	try:
-		return [f[:-4] for f in os.listdir("app/translations") if f.endswith(".csv")]
-	except OSError, e:
-		if e.args[0]==2:
-			return []
-		else:
-			raise
+	languages = [a.split()[0] for a in \
+		webnotes.get_file_items(os.path.join(webnotes.local.sites_path, "languages.txt"))]
 
 def get_lang_dict():
 	languages_path = os.path.join(os.path.dirname(webnotes.__file__), "data", "languages.json")
@@ -94,12 +89,10 @@ def build_message_files():
 	if not webnotes.conn:
 		webnotes.connect()
 		
-	build_for_pages('lib/core')
-	build_for_pages('app')
-
-	build_from_doctype_code('lib/core')
-	build_from_doctype_code('app')
-
+	for app in webnotes.get_all_apps(True):
+		build_for_pages(webnotes.get_pymodule_path(app))
+		build_from_doctype_code(webnotes.get_pymodule_path(app))
+		
 	#reports
 	build_from_query_report()
 	

webnotes/website/doctype/website_sitemap_config/website_sitemap_config.py

@@ -33,7 +33,7 @@ class DocType:
 def rebuild_website_sitemap_config():
 	webnotes.conn.sql("""delete from `tabWebsite Sitemap Config`""")
 	webnotes.conn.sql("""delete from `tabWebsite Sitemap`""")
-	for app in webnotes.get_app_list(True):
+	for app in webnotes.get_installed_apps():
 		build_website_sitemap_config(app)
 
 def build_website_sitemap_config(app):		

webnotes/webutils.py

@@ -217,10 +217,14 @@ def is_signup_enabled():
 
 class WebsiteGenerator(object):
 	def setup_generator(self):
+		if webnotes.flags.in_install_app:
+			return
 		self._website_config = webnotes.conn.get_values("Website Sitemap Config", 
 			{"ref_doctype": self.doc.doctype}, "*")[0]
 		
 	def on_update(self, page_name=None):
+		if webnotes.flags.in_install_app:
+			return
 		self.setup_generator()
 		if self._website_config.condition_field:
 			if not self.doc.fields.get(self._website_config.condition_field):