Merge pull request #13145 from gavindsouza/misc-fixes-py3-1
fix: Misc fixes
This commit is contained in:
gavin
GitHub
commit
6365037b81
16 changed files with 206 additions and 216 deletions
|
|
@ -99,17 +99,7 @@ def application(request):
|
|||
frappe.monitor.stop(response)
|
||||
frappe.recorder.dump()
|
||||
|
||||
if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
|
||||
frappe.logger("frappe.web", allow_site=frappe.local.site).info({
|
||||
"site": get_site_name(request.host),
|
||||
"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
|
||||
"base_url": getattr(request, "base_url", "NOTFOUND"),
|
||||
"full_path": getattr(request, "full_path", "NOTFOUND"),
|
||||
"method": getattr(request, "method", "NOTFOUND"),
|
||||
"scheme": getattr(request, "scheme", "NOTFOUND"),
|
||||
"http_status_code": getattr(response, "status_code", "NOTFOUND")
|
||||
})
|
||||
|
||||
log_request(request, response)
|
||||
process_response(response)
|
||||
frappe.destroy()
|
||||
|
||||
|
|
@ -137,6 +127,19 @@ def init_request(request):
|
|||
if request.method != "OPTIONS":
|
||||
frappe.local.http_request = frappe.auth.HTTPRequest()
|
||||
|
||||
def log_request(request, response):
|
||||
if hasattr(frappe.local, 'conf') and frappe.local.conf.enable_frappe_logger:
|
||||
frappe.logger("frappe.web", allow_site=frappe.local.site).info({
|
||||
"site": get_site_name(request.host),
|
||||
"remote_addr": getattr(request, "remote_addr", "NOTFOUND"),
|
||||
"base_url": getattr(request, "base_url", "NOTFOUND"),
|
||||
"full_path": getattr(request, "full_path", "NOTFOUND"),
|
||||
"method": getattr(request, "method", "NOTFOUND"),
|
||||
"scheme": getattr(request, "scheme", "NOTFOUND"),
|
||||
"http_status_code": getattr(response, "status_code", "NOTFOUND")
|
||||
})
|
||||
|
||||
|
||||
def process_response(response):
|
||||
if not response:
|
||||
return
|
||||
|
|
|
|||
175
frappe/build.py
175
frappe/build.py
|
|
@ -1,14 +1,11 @@
|
|||
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
|
||||
# MIT License. See license.txt
|
||||
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import shutil
|
||||
import warnings
|
||||
import tempfile
|
||||
from tempfile import mkdtemp, mktemp
|
||||
from distutils.spawn import find_executable
|
||||
|
||||
import frappe
|
||||
|
|
@ -16,8 +13,8 @@ from frappe.utils.minify import JavascriptMinify
|
|||
|
||||
import click
|
||||
import psutil
|
||||
from six import iteritems, text_type
|
||||
from six.moves.urllib.parse import urlparse
|
||||
from urllib.parse import urlparse
|
||||
from simple_chalk import green
|
||||
|
||||
|
||||
timestamps = {}
|
||||
|
|
@ -75,8 +72,8 @@ def get_assets_link(frappe_head):
|
|||
from requests import head
|
||||
|
||||
tag = getoutput(
|
||||
"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
|
||||
" refs/tags/,,' -e 's/\^{}//'"
|
||||
r"cd ../apps/frappe && git show-ref --tags -d | grep %s | sed -e 's,.*"
|
||||
r" refs/tags/,,' -e 's/\^{}//'"
|
||||
% frappe_head
|
||||
)
|
||||
|
||||
|
|
@ -97,9 +94,7 @@ def download_frappe_assets(verbose=True):
|
|||
commit HEAD.
|
||||
Returns True if correctly setup else returns False.
|
||||
"""
|
||||
from simple_chalk import green
|
||||
from subprocess import getoutput
|
||||
from tempfile import mkdtemp
|
||||
|
||||
assets_setup = False
|
||||
frappe_head = getoutput("cd ../apps/frappe && git rev-parse HEAD")
|
||||
|
|
@ -166,7 +161,7 @@ def symlink(target, link_name, overwrite=False):
|
|||
|
||||
# Create link to target with temporary filename
|
||||
while True:
|
||||
temp_link_name = tempfile.mktemp(dir=link_dir)
|
||||
temp_link_name = mktemp(dir=link_dir)
|
||||
|
||||
# os.* functions mimic as closely as possible system functions
|
||||
# The POSIX symlink() returns EEXIST if link_name already exists
|
||||
|
|
@ -193,7 +188,8 @@ def symlink(target, link_name, overwrite=False):
|
|||
|
||||
|
||||
def setup():
|
||||
global app_paths
|
||||
global app_paths, assets_path
|
||||
|
||||
pymodules = []
|
||||
for app in frappe.get_all_apps(True):
|
||||
try:
|
||||
|
|
@ -201,6 +197,7 @@ def setup():
|
|||
except ImportError:
|
||||
pass
|
||||
app_paths = [os.path.dirname(pymodule.__file__) for pymodule in pymodules]
|
||||
assets_path = os.path.join(frappe.local.sites_path, "assets")
|
||||
|
||||
|
||||
def get_node_pacman():
|
||||
|
|
@ -210,10 +207,10 @@ def get_node_pacman():
|
|||
raise ValueError("Yarn not found")
|
||||
|
||||
|
||||
def bundle(no_compress, app=None, make_copy=False, restore=False, verbose=False, skip_frappe=False):
|
||||
def bundle(no_compress, app=None, hard_link=False, verbose=False, skip_frappe=False):
|
||||
"""concat / minify js files"""
|
||||
setup()
|
||||
make_asset_dirs(make_copy=make_copy, restore=restore)
|
||||
make_asset_dirs(hard_link=hard_link)
|
||||
|
||||
pacman = get_node_pacman()
|
||||
mode = "build" if no_compress else "production"
|
||||
|
|
@ -266,75 +263,101 @@ def get_safe_max_old_space_size():
|
|||
|
||||
return safe_max_old_space_size
|
||||
|
||||
def make_asset_dirs(make_copy=False, restore=False):
|
||||
# don't even think of making assets_path absolute - rm -rf ahead.
|
||||
assets_path = os.path.join(frappe.local.sites_path, "assets")
|
||||
def generate_assets_map():
|
||||
symlinks = {}
|
||||
|
||||
for dir_path in [os.path.join(assets_path, "js"), os.path.join(assets_path, "css")]:
|
||||
if not os.path.exists(dir_path):
|
||||
os.makedirs(dir_path)
|
||||
for app_name in frappe.get_all_apps():
|
||||
app_doc_path = None
|
||||
|
||||
for app_name in frappe.get_all_apps(True):
|
||||
pymodule = frappe.get_module(app_name)
|
||||
app_base_path = os.path.abspath(os.path.dirname(pymodule.__file__))
|
||||
|
||||
symlinks = []
|
||||
app_public_path = os.path.join(app_base_path, "public")
|
||||
# app/public > assets/app
|
||||
symlinks.append([app_public_path, os.path.join(assets_path, app_name)])
|
||||
# app/node_modules > assets/app/node_modules
|
||||
if os.path.exists(os.path.abspath(app_public_path)):
|
||||
symlinks.append(
|
||||
[
|
||||
os.path.join(app_base_path, "..", "node_modules"),
|
||||
os.path.join(assets_path, app_name, "node_modules"),
|
||||
]
|
||||
)
|
||||
app_node_modules_path = os.path.join(app_base_path, "..", "node_modules")
|
||||
app_docs_path = os.path.join(app_base_path, "docs")
|
||||
app_www_docs_path = os.path.join(app_base_path, "www", "docs")
|
||||
|
||||
app_doc_path = None
|
||||
if os.path.isdir(os.path.join(app_base_path, "docs")):
|
||||
app_assets = os.path.abspath(app_public_path)
|
||||
app_node_modules = os.path.abspath(app_node_modules_path)
|
||||
|
||||
# {app}/public > assets/{app}
|
||||
if os.path.isdir(app_assets):
|
||||
symlinks[app_assets] = os.path.join(assets_path, app_name)
|
||||
|
||||
# {app}/node_modules > assets/{app}/node_modules
|
||||
if os.path.isdir(app_node_modules):
|
||||
symlinks[app_node_modules] = os.path.join(assets_path, app_name, "node_modules")
|
||||
|
||||
# {app}/docs > assets/{app}_docs
|
||||
if os.path.isdir(app_docs_path):
|
||||
app_doc_path = os.path.join(app_base_path, "docs")
|
||||
|
||||
elif os.path.isdir(os.path.join(app_base_path, "www", "docs")):
|
||||
elif os.path.isdir(app_www_docs_path):
|
||||
app_doc_path = os.path.join(app_base_path, "www", "docs")
|
||||
|
||||
if app_doc_path:
|
||||
symlinks.append([app_doc_path, os.path.join(assets_path, app_name + "_docs")])
|
||||
app_docs = os.path.abspath(app_doc_path)
|
||||
symlinks[app_docs] = os.path.join(assets_path, app_name + "_docs")
|
||||
|
||||
for source, target in symlinks:
|
||||
source = os.path.abspath(source)
|
||||
if os.path.exists(source):
|
||||
if restore:
|
||||
if os.path.exists(target):
|
||||
if os.path.islink(target):
|
||||
os.unlink(target)
|
||||
else:
|
||||
shutil.rmtree(target)
|
||||
shutil.copytree(source, target)
|
||||
elif make_copy:
|
||||
if os.path.exists(target):
|
||||
warnings.warn("Target {target} already exists.".format(target=target))
|
||||
else:
|
||||
shutil.copytree(source, target)
|
||||
else:
|
||||
if os.path.exists(target):
|
||||
if os.path.islink(target):
|
||||
os.unlink(target)
|
||||
else:
|
||||
shutil.rmtree(target)
|
||||
try:
|
||||
symlink(source, target, overwrite=True)
|
||||
except OSError:
|
||||
print("Cannot link {} to {}".format(source, target))
|
||||
else:
|
||||
warnings.warn('Source {source} does not exist.'.format(source = source))
|
||||
pass
|
||||
return symlinks
|
||||
|
||||
|
||||
def setup_assets_dirs():
|
||||
for dir_path in (os.path.join(assets_path, x) for x in ("js", "css")):
|
||||
os.makedirs(dir_path, exist_ok=True)
|
||||
|
||||
|
||||
def clear_broken_symlinks():
|
||||
for path in os.listdir(assets_path):
|
||||
path = os.path.join(assets_path, path)
|
||||
if os.path.islink(path) and not os.path.exists(path):
|
||||
os.remove(path)
|
||||
|
||||
|
||||
|
||||
def unstrip(message):
|
||||
try:
|
||||
max_str = os.get_terminal_size().columns
|
||||
except Exception:
|
||||
max_str = 80
|
||||
_len = len(message)
|
||||
_rem = max_str - _len
|
||||
return f"{message}{' ' * _rem}"
|
||||
|
||||
|
||||
def make_asset_dirs(hard_link=False):
|
||||
setup_assets_dirs()
|
||||
clear_broken_symlinks()
|
||||
symlinks = generate_assets_map()
|
||||
|
||||
for source, target in symlinks.items():
|
||||
start_message = unstrip(f"{'Copying assets from' if hard_link else 'Linking'} {source} to {target}")
|
||||
fail_message = unstrip(f"Cannot {'copy' if hard_link else 'link'} {source} to {target}")
|
||||
|
||||
try:
|
||||
print(start_message, end="\r")
|
||||
link_assets_dir(source, target, hard_link=hard_link)
|
||||
except Exception:
|
||||
print(fail_message, end="\r")
|
||||
|
||||
print(unstrip(f"{green('✔')} Application Assets Linked") + "\n")
|
||||
|
||||
|
||||
def link_assets_dir(source, target, hard_link=False):
|
||||
if not os.path.exists(source):
|
||||
return
|
||||
|
||||
if os.path.exists(target):
|
||||
if os.path.islink(target):
|
||||
os.unlink(target)
|
||||
else:
|
||||
shutil.rmtree(target)
|
||||
|
||||
if hard_link:
|
||||
shutil.copytree(source, target, dirs_exist_ok=True)
|
||||
else:
|
||||
symlink(source, target, overwrite=True)
|
||||
|
||||
|
||||
def build(no_compress=False, verbose=False):
|
||||
assets_path = os.path.join(frappe.local.sites_path, "assets")
|
||||
|
||||
for target, sources in iteritems(get_build_maps()):
|
||||
for target, sources in get_build_maps().items():
|
||||
pack(os.path.join(assets_path, target), sources, no_compress, verbose)
|
||||
|
||||
|
||||
|
|
@ -348,7 +371,7 @@ def get_build_maps():
|
|||
if os.path.exists(path):
|
||||
with open(path) as f:
|
||||
try:
|
||||
for target, sources in iteritems(json.loads(f.read())):
|
||||
for target, sources in (json.loads(f.read() or "{}")).items():
|
||||
# update app path
|
||||
source_paths = []
|
||||
for source in sources:
|
||||
|
|
@ -381,7 +404,7 @@ def pack(target, sources, no_compress, verbose):
|
|||
timestamps[f] = os.path.getmtime(f)
|
||||
try:
|
||||
with open(f, "r") as sourcefile:
|
||||
data = text_type(sourcefile.read(), "utf-8", errors="ignore")
|
||||
data = str(sourcefile.read(), "utf-8", errors="ignore")
|
||||
|
||||
extn = f.rsplit(".", 1)[1]
|
||||
|
||||
|
|
@ -396,7 +419,7 @@ def pack(target, sources, no_compress, verbose):
|
|||
jsm.minify(tmpin, tmpout)
|
||||
minified = tmpout.getvalue()
|
||||
if minified:
|
||||
outtxt += text_type(minified or "", "utf-8").strip("\n") + ";"
|
||||
outtxt += str(minified or "", "utf-8").strip("\n") + ";"
|
||||
|
||||
if verbose:
|
||||
print("{0}: {1}k".format(f, int(len(minified) / 1024)))
|
||||
|
|
@ -426,16 +449,16 @@ def html_to_js_template(path, content):
|
|||
def scrub_html_template(content):
|
||||
"""Returns HTML content with removed whitespace and comments"""
|
||||
# remove whitespace to a single space
|
||||
content = re.sub("\s+", " ", content)
|
||||
content = re.sub(r"\s+", " ", content)
|
||||
|
||||
# strip comments
|
||||
content = re.sub("(<!--.*?-->)", "", content)
|
||||
content = re.sub(r"(<!--.*?-->)", "", content)
|
||||
|
||||
return content.replace("'", "\'")
|
||||
|
||||
|
||||
def files_dirty():
|
||||
for target, sources in iteritems(get_build_maps()):
|
||||
for target, sources in get_build_maps().items():
|
||||
for f in sources:
|
||||
if ":" in f:
|
||||
f, suffix = f.split(":")
|
||||
|
|
|
|||
|
|
@ -28,6 +28,10 @@ def pass_context(f):
|
|||
except frappe.exceptions.SiteNotSpecifiedError as e:
|
||||
click.secho(str(e), fg='yellow')
|
||||
sys.exit(1)
|
||||
except frappe.exceptions.IncorrectSitePath:
|
||||
site = ctx.obj.get("sites", "")[0]
|
||||
click.secho(f'Site {site} does not exist!', fg='yellow')
|
||||
sys.exit(1)
|
||||
|
||||
if profile:
|
||||
pr.disable()
|
||||
|
|
|
|||
|
|
@ -16,13 +16,13 @@ from frappe.utils import get_bench_path, update_progress_bar, cint
|
|||
|
||||
@click.command('build')
|
||||
@click.option('--app', help='Build assets for app')
|
||||
@click.option('--make-copy', is_flag=True, default=False, help='Copy the files instead of symlinking')
|
||||
@click.option('--restore', is_flag=True, default=False, help='Copy the files instead of symlinking with force')
|
||||
@click.option('--hard-link', is_flag=True, default=False, help='Copy the files instead of symlinking')
|
||||
@click.option('--make-copy', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking')
|
||||
@click.option('--restore', is_flag=True, default=False, help='[DEPRECATED] Copy the files instead of symlinking with force')
|
||||
@click.option('--verbose', is_flag=True, default=False, help='Verbose')
|
||||
@click.option('--force', is_flag=True, default=False, help='Force build assets instead of downloading available')
|
||||
def build(app=None, make_copy=False, restore=False, verbose=False, force=False):
|
||||
def build(app=None, hard_link=False, make_copy=False, restore=False, verbose=False, force=False):
|
||||
"Minify + concatenate JS and CSS files, build translations"
|
||||
import frappe.build
|
||||
frappe.init('')
|
||||
# don't minify in developer_mode for faster builds
|
||||
no_compress = frappe.local.conf.developer_mode or False
|
||||
|
|
@ -34,7 +34,20 @@ def build(app=None, make_copy=False, restore=False, verbose=False, force=False):
|
|||
else:
|
||||
skip_frappe = False
|
||||
|
||||
frappe.build.bundle(no_compress, app=app, make_copy=make_copy, restore=restore, verbose=verbose, skip_frappe=skip_frappe)
|
||||
if make_copy or restore:
|
||||
hard_link = make_copy or restore
|
||||
click.secho(
|
||||
"bench build: --make-copy and --restore options are deprecated in favour of --hard-link",
|
||||
fg="yellow",
|
||||
)
|
||||
|
||||
frappe.build.bundle(
|
||||
skip_frappe=skip_frappe,
|
||||
no_compress=no_compress,
|
||||
hard_link=hard_link,
|
||||
verbose=verbose,
|
||||
app=app,
|
||||
)
|
||||
|
||||
|
||||
@click.command('watch')
|
||||
|
|
@ -488,6 +501,8 @@ frappe.db.connect()
|
|||
@pass_context
|
||||
def console(context):
|
||||
"Start ipython console for a site"
|
||||
import warnings
|
||||
|
||||
site = get_site(context)
|
||||
frappe.init(site=site)
|
||||
frappe.connect()
|
||||
|
|
@ -508,6 +523,7 @@ def console(context):
|
|||
if failed_to_import:
|
||||
print("\nFailed to import:\n{}".format(", ".join(failed_to_import)))
|
||||
|
||||
warnings.simplefilter('ignore')
|
||||
IPython.embed(display_banner="", header="", colors="neutral")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -641,7 +641,7 @@ class Row:
|
|||
return
|
||||
elif df.fieldtype == "Duration":
|
||||
import re
|
||||
is_valid_duration = re.match("^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
|
||||
is_valid_duration = re.match(r"^(?:(\d+d)?((^|\s)\d+h)?((^|\s)\d+m)?((^|\s)\d+s)?)$", value)
|
||||
if not is_valid_duration:
|
||||
self.warnings.append(
|
||||
{
|
||||
|
|
@ -929,10 +929,7 @@ class Column:
|
|||
self.warnings.append(
|
||||
{
|
||||
"col": self.column_number,
|
||||
"message": _(
|
||||
"Date format could not be determined from the values in"
|
||||
" this column. Defaulting to yyyy-mm-dd."
|
||||
),
|
||||
"message": _("Date format could not be determined from the values in this column. Defaulting to yyyy-mm-dd."),
|
||||
"type": "info",
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -671,12 +671,12 @@ class DocType(Document):
|
|||
flags = {"flags": re.ASCII} if six.PY3 else {}
|
||||
|
||||
# a DocType name should not start or end with an empty space
|
||||
if re.search("^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
|
||||
if re.search(r"^[ \t\n\r]+|[ \t\n\r]+$", name, **flags):
|
||||
frappe.throw(_("DocType's name should not start or end with whitespace"), frappe.NameError)
|
||||
|
||||
# a DocType's name should not start with a number or underscore
|
||||
# and should only contain letters, numbers and underscore
|
||||
if not re.match("^(?![\W])[^\d_\s][\w ]+$", name, **flags):
|
||||
if not re.match(r"^(?![\W])[^\d_\s][\w ]+$", name, **flags):
|
||||
frappe.throw(_("DocType's name should start with a letter and it can only consist of letters, numbers, spaces and underscores"), frappe.NameError)
|
||||
|
||||
validate_route_conflict(self.doctype, self.name)
|
||||
|
|
|
|||
|
|
@ -284,7 +284,7 @@ class EmailServer:
|
|||
|
||||
flags = []
|
||||
for flag in imaplib.ParseFlags(flag_string) or []:
|
||||
pattern = re.compile("\w+")
|
||||
pattern = re.compile(r"\w+")
|
||||
match = re.search(pattern, frappe.as_unicode(flag))
|
||||
flags.append(match.group(0))
|
||||
|
||||
|
|
@ -555,7 +555,7 @@ class Email:
|
|||
|
||||
def get_thread_id(self):
|
||||
"""Extract thread ID from `[]`"""
|
||||
l = re.findall('(?<=\[)[\w/-]+', self.subject)
|
||||
l = re.findall(r'(?<=\[)[\w/-]+', self.subject)
|
||||
return l and l[0] or None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -226,7 +226,6 @@ scheduler_events = {
|
|||
"frappe.desk.doctype.event.event.send_event_digest",
|
||||
"frappe.sessions.clear_expired_sessions",
|
||||
"frappe.email.doctype.notification.notification.trigger_daily_alerts",
|
||||
"frappe.realtime.remove_old_task_logs",
|
||||
"frappe.utils.scheduler.restrict_scheduler_events_if_dormant",
|
||||
"frappe.email.doctype.auto_email_report.auto_email_report.send_daily",
|
||||
"frappe.website.doctype.personal_data_deletion_request.personal_data_deletion_request.remove_unverified_record",
|
||||
|
|
|
|||
|
|
@ -390,19 +390,16 @@ def get_conf_params(db_name=None, db_password=None):
|
|||
|
||||
|
||||
def make_site_dirs():
|
||||
site_public_path = os.path.join(frappe.local.site_path, 'public')
|
||||
site_private_path = os.path.join(frappe.local.site_path, 'private')
|
||||
for dir_path in (
|
||||
os.path.join(site_private_path, 'backups'),
|
||||
os.path.join(site_public_path, 'files'),
|
||||
os.path.join(site_private_path, 'files'),
|
||||
os.path.join(frappe.local.site_path, 'logs'),
|
||||
os.path.join(frappe.local.site_path, 'task-logs')):
|
||||
if not os.path.exists(dir_path):
|
||||
os.makedirs(dir_path)
|
||||
locks_dir = frappe.get_site_path('locks')
|
||||
if not os.path.exists(locks_dir):
|
||||
os.makedirs(locks_dir)
|
||||
for dir_path in [
|
||||
os.path.join("public", "files"),
|
||||
os.path.join("private", "backups"),
|
||||
os.path.join("private", "files"),
|
||||
"error-snapshots",
|
||||
"locks",
|
||||
"logs",
|
||||
]:
|
||||
path = frappe.get_site_path(dir_path)
|
||||
os.makedirs(path, exist_ok=True)
|
||||
|
||||
|
||||
def add_module_defs(app):
|
||||
|
|
|
|||
|
|
@ -1,56 +1,23 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
|
||||
# For license information, please see license.txt
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
|
||||
import frappe
|
||||
import os
|
||||
import time
|
||||
import redis
|
||||
from io import FileIO
|
||||
from frappe.utils import get_site_path
|
||||
from frappe import conf
|
||||
|
||||
END_LINE = '<!-- frappe: end-file -->'
|
||||
TASK_LOG_MAX_AGE = 86400 # 1 day in seconds
|
||||
redis_server = None
|
||||
|
||||
|
||||
@frappe.whitelist()
|
||||
def get_pending_tasks_for_doc(doctype, docname):
|
||||
return frappe.db.sql_list("select name from `tabAsync Task` where status in ('Queued', 'Running') and reference_doctype=%s and reference_name=%s", (doctype, docname))
|
||||
|
||||
|
||||
def set_task_status(task_id, status, response=None):
|
||||
if not response:
|
||||
response = {}
|
||||
response.update({
|
||||
"status": status,
|
||||
"task_id": task_id
|
||||
})
|
||||
emit_via_redis("task_status_change", response, room="task:" + task_id)
|
||||
|
||||
|
||||
def remove_old_task_logs():
|
||||
logs_path = get_site_path('task-logs')
|
||||
|
||||
def full_path(_file):
|
||||
return os.path.join(logs_path, _file)
|
||||
|
||||
files_to_remove = [full_path(_file) for _file in os.listdir(logs_path)]
|
||||
files_to_remove = [_file for _file in files_to_remove if is_file_old(_file) and os.path.isfile(_file)]
|
||||
for _file in files_to_remove:
|
||||
os.remove(_file)
|
||||
|
||||
|
||||
def is_file_old(file_path):
|
||||
return ((time.time() - os.stat(file_path).st_mtime) > TASK_LOG_MAX_AGE)
|
||||
|
||||
def publish_progress(percent, title=None, doctype=None, docname=None, description=None):
|
||||
publish_realtime('progress', {'percent': percent, 'title': title, 'description': description},
|
||||
user=frappe.session.user, doctype=doctype, docname=docname)
|
||||
|
||||
|
||||
def publish_realtime(event=None, message=None, room=None,
|
||||
user=None, doctype=None, docname=None, task_id=None,
|
||||
after_commit=False):
|
||||
|
|
@ -103,6 +70,7 @@ def publish_realtime(event=None, message=None, room=None,
|
|||
else:
|
||||
emit_via_redis(event, message, room)
|
||||
|
||||
|
||||
def emit_via_redis(event, message, room):
|
||||
"""Publish real-time updates via redis
|
||||
|
||||
|
|
@ -117,57 +85,17 @@ def emit_via_redis(event, message, room):
|
|||
# print(frappe.get_traceback())
|
||||
pass
|
||||
|
||||
def put_log(line_no, line, task_id=None):
|
||||
r = get_redis_server()
|
||||
if not task_id:
|
||||
task_id = frappe.local.task_id
|
||||
task_progress_room = get_task_progress_room(task_id)
|
||||
task_log_key = "task_log:" + task_id
|
||||
publish_realtime('task_progress', {
|
||||
"message": {
|
||||
"lines": {line_no: line}
|
||||
},
|
||||
"task_id": task_id
|
||||
}, room=task_progress_room)
|
||||
r.hset(task_log_key, line_no, line)
|
||||
r.expire(task_log_key, 3600)
|
||||
|
||||
|
||||
def get_redis_server():
|
||||
"""returns redis_socketio connection."""
|
||||
global redis_server
|
||||
if not redis_server:
|
||||
from redis import Redis
|
||||
redis_server = Redis.from_url(conf.get("redis_socketio")
|
||||
redis_server = Redis.from_url(frappe.conf.redis_socketio
|
||||
or "redis://localhost:12311")
|
||||
return redis_server
|
||||
|
||||
|
||||
class FileAndRedisStream(FileIO):
|
||||
def __init__(self, *args, **kwargs):
|
||||
ret = super(FileAndRedisStream, self).__init__(*args, **kwargs)
|
||||
self.count = 0
|
||||
return ret
|
||||
|
||||
def write(self, data):
|
||||
ret = super(FileAndRedisStream, self).write(data)
|
||||
if frappe.local.task_id:
|
||||
put_log(self.count, data, task_id=frappe.local.task_id)
|
||||
self.count += 1
|
||||
return ret
|
||||
|
||||
|
||||
def get_std_streams(task_id):
|
||||
stdout = FileAndRedisStream(get_task_log_file_path(task_id, 'stdout'), 'w')
|
||||
# stderr = FileAndRedisStream(get_task_log_file_path(task_id, 'stderr'), 'w')
|
||||
return stdout, stdout
|
||||
|
||||
|
||||
def get_task_log_file_path(task_id, stream_type):
|
||||
logs_dir = frappe.utils.get_site_path('task-logs')
|
||||
return os.path.join(logs_dir, task_id + '.' + stream_type)
|
||||
|
||||
|
||||
@frappe.whitelist(allow_guest=True)
|
||||
def can_subscribe_doc(doctype, docname):
|
||||
if os.environ.get('CI'):
|
||||
|
|
@ -201,9 +129,7 @@ def get_site_room():
|
|||
def get_task_progress_room(task_id):
|
||||
return "".join([frappe.local.site, ":task_progress:", task_id])
|
||||
|
||||
# frappe.chat
|
||||
def get_chat_room(room):
|
||||
room = ''.join([frappe.local.site, ":room:", room])
|
||||
|
||||
return room
|
||||
# end frappe.chat room
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
|
||||
# MIT License. See license.txt
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import frappe
|
||||
from frappe.utils import update_progress_bar
|
||||
|
||||
from whoosh.index import create_in, open_dir, EmptyIndexError
|
||||
from whoosh.fields import TEXT, ID, Schema
|
||||
|
|
@ -95,9 +95,10 @@ class FullTextSearch:
|
|||
ix = self.create_index()
|
||||
writer = ix.writer()
|
||||
|
||||
for document in self.documents:
|
||||
for i, document in enumerate(self.documents):
|
||||
if document:
|
||||
writer.add_document(**document)
|
||||
update_progress_bar("Building Index", i, len(self.documents))
|
||||
|
||||
writer.commit(optimize=True)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,16 @@
|
|||
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
|
||||
# MIT License. See license.txt
|
||||
|
||||
from __future__ import unicode_literals
|
||||
import frappe
|
||||
from bs4 import BeautifulSoup
|
||||
from whoosh.fields import TEXT, ID, Schema
|
||||
from frappe.search.full_text_search import FullTextSearch
|
||||
from frappe.website.render import render_page
|
||||
from frappe.utils import set_request
|
||||
import os
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from whoosh.fields import ID, TEXT, Schema
|
||||
|
||||
import frappe
|
||||
from frappe.search.full_text_search import FullTextSearch
|
||||
from frappe.utils import set_request, update_progress_bar
|
||||
from frappe.website.render import render_page
|
||||
|
||||
INDEX_NAME = "web_routes"
|
||||
|
||||
class WebsiteSearch(FullTextSearch):
|
||||
|
|
@ -30,11 +31,21 @@ class WebsiteSearch(FullTextSearch):
|
|||
Returns:
|
||||
self (object): FullTextSearch Instance
|
||||
"""
|
||||
routes = get_static_pages_from_all_apps()
|
||||
routes += slugs_with_web_view()
|
||||
|
||||
documents = [self.get_document_to_index(route) for route in routes]
|
||||
return documents
|
||||
if getattr(self, "_items_to_index", False):
|
||||
return self._items_to_index
|
||||
|
||||
routes = get_static_pages_from_all_apps() + slugs_with_web_view()
|
||||
|
||||
self._items_to_index = []
|
||||
|
||||
for i, route in enumerate(routes):
|
||||
update_progress_bar("Retrieving Routes", i, len(routes))
|
||||
self._items_to_index += [self.get_document_to_index(route)]
|
||||
|
||||
print()
|
||||
|
||||
return self.get_items_to_index()
|
||||
|
||||
def get_document_to_index(self, route):
|
||||
"""Render a page and parse it using BeautifulSoup
|
||||
|
|
@ -114,4 +125,4 @@ def remove_document_from_index(path):
|
|||
|
||||
def build_index_for_all_routes():
|
||||
ws = WebsiteSearch(INDEX_NAME)
|
||||
return ws.build()
|
||||
return ws.build()
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ def validate_url(txt, throw=False, valid_schemes=None):
|
|||
|
||||
Parameters:
|
||||
throw (`bool`): throws a validationError if URL is not valid
|
||||
valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this
|
||||
valid_schemes (`str` or `list`): if provided checks the given URL's scheme against this
|
||||
|
||||
Returns:
|
||||
bool: if `txt` represents a valid URL
|
||||
|
|
@ -225,14 +225,17 @@ def get_gravatar(email):
|
|||
|
||||
return gravatar_url
|
||||
|
||||
def get_traceback():
|
||||
def get_traceback() -> str:
|
||||
"""
|
||||
Returns the traceback of the Exception
|
||||
"""
|
||||
exc_type, exc_value, exc_tb = sys.exc_info()
|
||||
|
||||
if not any([exc_type, exc_value, exc_tb]):
|
||||
return ""
|
||||
|
||||
trace_list = traceback.format_exception(exc_type, exc_value, exc_tb)
|
||||
body = "".join(cstr(t) for t in trace_list)
|
||||
return body
|
||||
return "".join(cstr(t) for t in trace_list)
|
||||
|
||||
def log(event, details):
|
||||
frappe.logger().info(details)
|
||||
|
|
@ -425,7 +428,7 @@ def get_test_client():
|
|||
return Client(application)
|
||||
|
||||
def get_hook_method(hook_name, fallback=None):
|
||||
method = (frappe.get_hooks().get(hook_name))
|
||||
method = frappe.get_hooks().get(hook_name)
|
||||
if method:
|
||||
method = frappe.get_attr(method[0])
|
||||
return method
|
||||
|
|
@ -439,6 +442,16 @@ def call_hook_method(hook, *args, **kwargs):
|
|||
|
||||
return out
|
||||
|
||||
def is_cli() -> bool:
|
||||
"""Returns True if current instance is being run via a terminal
|
||||
"""
|
||||
invoked_from_terminal = False
|
||||
try:
|
||||
invoked_from_terminal = bool(os.get_terminal_size())
|
||||
except Exception:
|
||||
invoked_from_terminal = sys.stdin.isatty()
|
||||
return invoked_from_terminal
|
||||
|
||||
def update_progress_bar(txt, i, l):
|
||||
if os.environ.get("CI"):
|
||||
if i == 0:
|
||||
|
|
@ -448,7 +461,7 @@ def update_progress_bar(txt, i, l):
|
|||
sys.stdout.flush()
|
||||
return
|
||||
|
||||
if not getattr(frappe.local, 'request', None):
|
||||
if not getattr(frappe.local, 'request', None) or is_cli():
|
||||
lt = len(txt)
|
||||
try:
|
||||
col = 40 if os.get_terminal_size().columns > 80 else 20
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ def make_boilerplate(dest, app_name):
|
|||
if hook_key=="app_name" and hook_val.lower().replace(" ", "_") != hook_val:
|
||||
print("App Name must be all lowercase and without spaces")
|
||||
hook_val = ""
|
||||
elif hook_key=="app_title" and not re.match("^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE):
|
||||
elif hook_key=="app_title" and not re.match(r"^(?![\W])[^\d_\s][\w -]+$", hook_val, re.UNICODE):
|
||||
print("App Title should start with a letter and it can only consist of letters, numbers, spaces and underscores")
|
||||
hook_val = ""
|
||||
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class RedisWrapper(redis.Redis):
|
|||
return self.keys(key)
|
||||
|
||||
except redis.exceptions.ConnectionError:
|
||||
regex = re.compile(cstr(key).replace("|", "\|").replace("*", "[\w]*"))
|
||||
regex = re.compile(cstr(key).replace("|", r"\|").replace("*", r"[\w]*"))
|
||||
return [k for k in list(frappe.local.cache) if regex.match(cstr(k))]
|
||||
|
||||
def delete_keys(self, key):
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ def update_controller_context(context, controller):
|
|||
except (frappe.PermissionError, frappe.PageDoesNotExistError, frappe.Redirect):
|
||||
raise
|
||||
except:
|
||||
if not frappe.flags.in_migrate:
|
||||
if not any([frappe.flags.in_migrate, frappe.flags.in_website_search_build]):
|
||||
frappe.errprint(frappe.utils.get_traceback())
|
||||
|
||||
if hasattr(module, "get_children"):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue