vassili/seitime-frappe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 4

Merge branch 'develop' of https://github.com/frappe/frappe into chart-in-custom-script-reports-dev

Browse source
This commit is contained in:
Saqib Ansari 2021-07-31 11:59:31 +05:30
commit ea72dc2aae
1096 changed files with 7503 additions and 11814 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/helper/roulette.py vendored
View file

@ -18,7 +18,7 @@ def is_js(file):
return file.endswith("js")
def is_docs(file):
regex = re.compile('\.(md|png|jpg|jpeg)$|^.github|LICENSE')
regex = re.compile(r'\.(md|png|jpg|jpeg)$|^.github|LICENSE')
return bool(regex.search(file))

2
.github/helper/semgrep_rules/frappe_correctness.yml vendored
View file

@ -98,8 +98,6 @@ rules:
languages: [python]
severity: WARNING
paths:
exclude:
- test_*.py
include:
- "*/**/doctype/*"

4
.github/helper/semgrep_rules/security.yml vendored
View file

@ -8,10 +8,6 @@ rules:
dynamic content. Avoid it or use safe_eval().
languages: [python]
severity: ERROR
paths:
exclude:
- frappe/__init__.py
- frappe/commands/utils.py
- id: frappe-sqli-format-strings
patterns:

9
.github/helper/semgrep_rules/ux.js vendored Normal file
View file

@ -0,0 +1,9 @@
// ok: frappe-missing-translate-function-js
frappe.msgprint('{{ _("Both login and password required") }}');
// ruleid: frappe-missing-translate-function-js
frappe.msgprint('What');
// ok: frappe-missing-translate-function-js
frappe.throw(' {{ _("Both login and password required") }}. ');

18
.github/helper/semgrep_rules/ux.py vendored
View file

@ -2,30 +2,30 @@ import frappe
from frappe import msgprint, throw, _
# ruleid: frappe-missing-translate-function
# ruleid: frappe-missing-translate-function-python
throw("Error Occured")
# ruleid: frappe-missing-translate-function
# ruleid: frappe-missing-translate-function-python
frappe.throw("Error Occured")
# ruleid: frappe-missing-translate-function
# ruleid: frappe-missing-translate-function-python
frappe.msgprint("Useful message")
# ruleid: frappe-missing-translate-function
# ruleid: frappe-missing-translate-function-python
msgprint("Useful message")
# ok: frappe-missing-translate-function
# ok: frappe-missing-translate-function-python
translatedmessage = _("Hello")
# ok: frappe-missing-translate-function
# ok: frappe-missing-translate-function-python
throw(translatedmessage)
# ok: frappe-missing-translate-function
# ok: frappe-missing-translate-function-python
msgprint(translatedmessage)
# ok: frappe-missing-translate-function
# ok: frappe-missing-translate-function-python
msgprint(_("Helpful message"))
# ok: frappe-missing-translate-function
# ok: frappe-missing-translate-function-python
frappe.throw(_("Error occured"))

23
.github/helper/semgrep_rules/ux.yml vendored
View file

@ -1,15 +1,30 @@
rules:
- id: frappe-missing-translate-function
- id: frappe-missing-translate-function-python
pattern-either:
- patterns:
- pattern: frappe.msgprint("...", ...)
- pattern-not: frappe.msgprint(_("..."), ...)
- pattern-not: frappe.msgprint(__("..."), ...)
- patterns:
- pattern: frappe.throw("...", ...)
- pattern-not: frappe.throw(_("..."), ...)
- pattern-not: frappe.throw(__("..."), ...)
message: |
All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations
languages: [python, javascript, json]
languages: [python]
severity: ERROR
- id: frappe-missing-translate-function-js
pattern-either:
- patterns:
- pattern: frappe.msgprint("...", ...)
- pattern-not: frappe.msgprint(__("..."), ...)
# ignore microtemplating e.g. msgprint("{{ _("server side translation") }}")
- pattern-not: frappe.msgprint("=~/\{\{.*\_.*\}\}/i", ...)
- patterns:
- pattern: frappe.throw("...", ...)
- pattern-not: frappe.throw(__("..."), ...)
# ignore microtemplating
- pattern-not: frappe.throw("=~/\{\{.*\_.*\}\}/i", ...)
message: |
All user facing text must be wrapped in translate function. Please refer to translation documentation. https://frappeframework.com/docs/user/en/guides/basics/translations
languages: [javascript]
severity: ERROR

17
.github/semantic.yml vendored
View file

@ -11,3 +11,20 @@ allowRevertCommits: true
# For allowed PR types: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json
# Tool Reference: https://github.com/zeke/semantic-pull-requests
# By default types specified in commitizen/conventional-commit-types is used.
# See: https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json
# You can override the valid types
types:
- BREAKING CHANGE
- feat
- fix
- docs
- style
- refactor
- perf
- test
- build
- ci
- chore
- revert

83
.github/workflows/patch-mariadb-tests.yml vendored Normal file
View file

@ -0,0 +1,83 @@
name: Patch
on: [pull_request, workflow_dispatch]
jobs:
test:
runs-on: ubuntu-18.04
name: Patch Test
services:
mysql:
image: mariadb:10.3
env:
MYSQL_ALLOW_EMPTY_PASSWORD: YES
ports:
- 3306:3306
options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3
steps:
- name: Clone
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: 3.7
- name: Add to Hosts
run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
- name: Cache pip
uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Cache node modules
uses: actions/cache@v2
env:
cache-name: cache-node-modules
with:
path: ~/.npm
key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ runner.os }}-build-${{ env.cache-name }}-
${{ runner.os }}-build-
${{ runner.os }}-
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v2
id: yarn-cache
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Install Dependencies
run: bash ${GITHUB_WORKSPACE}/.github/helper/install_dependencies.sh
env:
BEFORE: ${{ env.GITHUB_EVENT_PATH.before }}
AFTER: ${{ env.GITHUB_EVENT_PATH.after }}
TYPE: server
- name: Install
run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
env:
DB: mariadb
TYPE: server
- name: Run Patch Tests
run: |
cd ~/frappe-bench/
wget https://frappeframework.com/files/v10-frappe.sql.gz
bench --site test_site --force restore ~/frappe-bench/v10-frappe.sql.gz
bench --site test_site migrate

36
.github/workflows/semgrep.yml vendored
View file

@ -1,34 +1,18 @@
name: Semgrep
on:
pull_request:
branches:
- develop
- version-13-hotfix
- version-13-pre-release
pull_request: { }
jobs:
semgrep:
name: Frappe Linter
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup python3
uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Setup semgrep
run: |
python -m pip install -q semgrep
git fetch origin $GITHUB_BASE_REF:$GITHUB_BASE_REF -q
- name: Semgrep errors
run: |
files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
[[ -d .github/helper/semgrep_rules ]] && semgrep --severity ERROR --config=.github/helper/semgrep_rules --quiet --error $files
semgrep --config="r/python.lang.correctness" --quiet --error $files
- name: Semgrep warnings
run: |
files=$(git diff --name-only --diff-filter=d $GITHUB_BASE_REF)
[[ -d .github/helper/semgrep_rules ]] && semgrep --severity WARNING --severity INFO --config=.github/helper/semgrep_rules --quiet $files
- uses: actions/checkout@v2
- uses: returntocorp/semgrep-action@v1
env:
SEMGREP_TIMEOUT: 120
with:
config: >-
r/python.lang.correctness
.github/helper/semgrep_rules

1
.github/workflows/server-mariadb-tests.yml vendored
View file

@ -91,7 +91,6 @@ jobs:
DB: mariadb
TYPE: server
- name: Run Tests
run: cd ~/frappe-bench/ && bench --site test_site run-parallel-tests --use-orchestrator --with-coverage
env:

2
.github/workflows/ui-tests.yml vendored
View file

@ -105,3 +105,5 @@ jobs:
- name: UI Tests
run: cd ~/frappe-bench/ && bench --site test_site run-ui-tests frappe --headless --parallel --ci-build-id $GITHUB_RUN_ID
env:
CYPRESS_RECORD_KEY: 4a48f41c-11b3-425b-aa88-c58048fa69eb

3
CODEOWNERS
View file

@ -4,13 +4,10 @@
# the repo. Unless a later match takes precedence,
* @frappe/frappe-review-team
website/ @prssanna
web_form/ @prssanna
templates/ @surajshetty3416
www/ @surajshetty3416
integrations/ @leela
patches/ @surajshetty3416
dashboard/ @prssanna
email/ @leela
event_streaming/ @ruchamahabal
data_import* @netchampfaris

1
cypress/integration/form.js
View file

@ -18,6 +18,7 @@ context('Form', () => {
cy.get('.primary-action').click();
cy.wait('@form_save').its('response.statusCode').should('eq', 200);
cy.visit('/app/todo');
cy.wait(300);
cy.get('.title-text').should('be.visible').and('contain', 'To Do');
cy.get('.list-row').should('contain', 'this is a test todo');
});

88
cypress/integration/form_tour.js Normal file
View file

@ -0,0 +1,88 @@
context('Form Tour', () => {
before(() => {
cy.login();
cy.visit('/app/form-tour');
return cy.window().its('frappe').then(frappe => {
return frappe.call("frappe.tests.ui_test_helpers.create_form_tour");
});
});
const open_test_form_tour = () => {
cy.visit('/app/form-tour/Test Form Tour');
cy.get('button[data-label="Show%20Tour"]').should('be.visible').and('contain', 'Show Tour').as('show_tour');
cy.get('@show_tour').click();
cy.wait(500);
cy.url().should('include', '/app/contact');
};
it('jump to a form tour', open_test_form_tour);
it('navigates a form tour', () => {
open_test_form_tour();
cy.get('#driver-popover-item').should('be.visible');
cy.get('.frappe-control[data-fieldname="first_name"]').as('first_name');
cy.get('@first_name').should('have.class', 'driver-highlighted-element');
cy.get('.driver-next-btn').as('next_btn');
// next btn shouldn't move to next step, if first name is not entered
cy.get('@next_btn').click();
cy.wait(500);
cy.get('@first_name').should('have.class', 'driver-highlighted-element');
// after filling the field, next step should be highlighted
cy.fill_field('first_name', 'Test Name', 'Data');
cy.wait(500);
cy.get('@next_btn').click();
cy.wait(500);
// assert field is highlighted
cy.get('.frappe-control[data-fieldname="last_name"]').as('last_name');
cy.get('@last_name').should('have.class', 'driver-highlighted-element');
// after filling the field, next step should be highlighted
cy.fill_field('last_name', 'Test Last Name', 'Data');
cy.wait(500);
cy.get('@next_btn').click();
cy.wait(500);
// assert field is highlighted
cy.get('.frappe-control[data-fieldname="phone_nos"]').as('phone_nos');
cy.get('@phone_nos').should('have.class', 'driver-highlighted-element');
// move to next step
cy.wait(500);
cy.get('@next_btn').click();
cy.wait(500);
// assert add row btn is highlighted
cy.get('@phone_nos').find('.grid-add-row').as('add_row');
cy.get('@add_row').should('have.class', 'driver-highlighted-element');
// add a row & move to next step
cy.wait(500);
cy.get('@add_row').click();
cy.wait(500);
// assert table field is highlighted
cy.get('.grid-row-open .frappe-control[data-fieldname="phone"]').as('phone');
cy.get('@phone').should('have.class', 'driver-highlighted-element');
// enter value in a table field
cy.fill_table_field('phone_nos', '1', 'phone', '1234567890');
// move to collapse row step
cy.wait(500);
cy.get('@next_btn').click();
cy.wait(500);
// collapse row
cy.get('.grid-row-open .grid-collapse-row').click();
cy.wait(500);
// assert save btn is highlighted
cy.get('.primary-action').should('have.class', 'driver-highlighted-element');
cy.get('@next_btn').should('contain', 'Save');
});
});

14
cypress/integration/navigation.js Normal file
View file

@ -0,0 +1,14 @@
context('Navigation', () => {
before(() => {
cy.login();
cy.visit('/app/website');
});
it('Navigate to route with hash in document name', () => {
cy.insert_doc('ToDo', {'__newname': 'ABC#123', 'description': 'Test this', 'ignore_duplicate': true});
cy.visit('/app/todo/ABC#123');
cy.title().should('eq', 'Test this - ABC#123');
cy.get_field('description', 'Text Editor').contains('Test this');
cy.go('back');
cy.title().should('eq', 'Website');
});
});

112
esbuild/esbuild.js
View file

@ -8,6 +8,7 @@ let yargs = require("yargs");
let cliui = require("cliui")();
let chalk = require("chalk");
let html_plugin = require("./frappe-html");
let rtlcss = require('rtlcss');
let postCssPlugin = require("esbuild-plugin-postcss2").default;
let ignore_assets = require("./ignore-assets");
let sass_options = require("./sass_options");
@ -96,9 +97,9 @@ async function execute() {
await clean_dist_folders(APPS);
}
let result;
let results;
try {
result = await build_assets_for_apps(APPS, FILES_TO_BUILD);
results = await build_assets_for_apps(APPS, FILES_TO_BUILD);
} catch (e) {
log_error("There were some problems during build");
log();
@ -107,13 +108,15 @@ async function execute() {
}
if (!WATCH_MODE) {
log_built_assets(result.metafile);
log_built_assets(results);
console.timeEnd(TOTAL_BUILD_TIME);
log();
} else {
log("Watching for changes...");
}
return await write_assets_json(result.metafile);
for (const result of results) {
await write_assets_json(result.metafile);
}
}
function build_assets_for_apps(apps, files) {
@ -125,6 +128,8 @@ function build_assets_for_apps(apps, files) {
let output_path = assets_path;
let file_map = {};
let style_file_map = {};
let rtl_style_file_map = {};
for (let file of files) {
let relative_app_path = path.relative(apps_path, file);
let app = relative_app_path.split(path.sep)[0];
@ -140,19 +145,32 @@ function build_assets_for_apps(apps, files) {
}
output_name = path.join(app, "dist", output_name);
if (Object.keys(file_map).includes(output_name)) {
if (Object.keys(file_map).includes(output_name) || Object.keys(style_file_map).includes(output_name)) {
log_warn(
`Duplicate output file ${output_name} generated from ${file}`
);
}
file_map[output_name] = file;
if ([".css", ".scss", ".less", ".sass", ".styl"].includes(extension)) {
style_file_map[output_name] = file;
rtl_style_file_map[output_name.replace('/css/', '/css-rtl/')] = file;
} else {
file_map[output_name] = file;
}
}
return build_files({
let build = build_files({
files: file_map,
outdir: output_path
});
let style_build = build_style_files({
files: style_file_map,
outdir: output_path
});
let rtl_style_build = build_style_files({
files: rtl_style_file_map,
outdir: output_path,
rtl_style: true
});
return Promise.all([build, style_build, rtl_style_build]);
});
}
@ -203,7 +221,33 @@ function get_files_to_build(files) {
}
function build_files({ files, outdir }) {
return esbuild.build({
let build_plugins = [
html_plugin,
vue(),
];
return esbuild.build(get_build_options(files, outdir, build_plugins));
}
function build_style_files({ files, outdir, rtl_style=false }) {
let plugins = [];
if (rtl_style) {
plugins.push(rtlcss);
}
let build_plugins = [
ignore_assets,
postCssPlugin({
plugins: plugins,
sassOptions: sass_options
})
];
plugins.push(require("autoprefixer"));
return esbuild.build(get_build_options(files, outdir, build_plugins));
}
function get_build_options(files, outdir, plugins) {
return {
entryPoints: files,
entryNames: "[dir]/[name].[hash]",
outdir,
@ -217,17 +261,9 @@ function build_files({ files, outdir }) {
PRODUCTION ? "production" : "development"
)
},
plugins: [
html_plugin,
ignore_assets,
vue(),
postCssPlugin({
plugins: [require("autoprefixer")],
sassOptions: sass_options
})
],
plugins: plugins,
watch: get_watch_config()
});
};
}
function get_watch_config() {
@ -258,16 +294,26 @@ function get_watch_config() {
async function clean_dist_folders(apps) {
for (let app of apps) {
let public_path = get_public_path(app);
await fs.promises.rmdir(path.resolve(public_path, "dist", "js"), {
recursive: true
});
await fs.promises.rmdir(path.resolve(public_path, "dist", "css"), {
recursive: true
});
let paths = [
path.resolve(public_path, "dist", "js"),
path.resolve(public_path, "dist", "css"),
path.resolve(public_path, "dist", "css-rtl")
];
for (let target of paths) {
if (fs.existsSync(target)) {
// rmdir is deprecated in node 16, this will work in both node 14 and 16
let rmdir = fs.promises.rm || fs.promises.rmdir;
await rmdir(target, { recursive: true });
}
}
}
}
function log_built_assets(metafile) {
function log_built_assets(results) {
let outputs = {};
for (const result of results) {
outputs = Object.assign(outputs, result.metafile.outputs);
}
let column_widths = [60, 20];
cliui.div(
{
@ -282,9 +328,9 @@ function log_built_assets(metafile) {
cliui.div("");
let output_by_dist_path = {};
for (let outfile in metafile.outputs) {
for (let outfile in outputs) {
if (outfile.endsWith(".map")) continue;
let data = metafile.outputs[outfile];
let data = outputs[outfile];
outfile = path.resolve(outfile);
outfile = path.relative(assets_path, outfile);
let filename = path.basename(outfile);
@ -339,7 +385,11 @@ async function write_assets_json(metafile) {
let info = metafile.outputs[output];
let asset_path = "/" + path.relative(sites_path, output);
if (info.entryPoint) {
out[path.basename(info.entryPoint)] = asset_path;
let key = path.basename(info.entryPoint);
if (key.endsWith('.css') && asset_path.includes('/css-rtl/')) {
key = `rtl_${key}`;
}
out[key] = asset_path;
}
}
@ -478,4 +528,4 @@ function log_rebuilt_assets(prev_assets, new_assets) {
log(" " + filename);
}
log();
}
}

25
frappe/__init__.py
View file

@ -21,7 +21,6 @@ if _dev_server:
from werkzeug.local import Local, release_local
import sys, importlib, inspect, json
import typing
from past.builtins import cmp
import click
# Local application imports
@ -29,6 +28,8 @@ from .exceptions import *
from .utils.jinja import (get_jenv, get_template, render_template, get_email_from_template, get_jloader)
from .utils.lazy_loader import lazy_import
from frappe.query_builder import get_query_builder
# Lazy imports
faker = lazy_import('faker')
@ -119,6 +120,7 @@ def set_user_lang(user, user_language=None):
# local-globals
db = local("db")
qb = local("qb")
conf = local("conf")
form = form_dict = local("form_dict")
request = local("request")
@ -203,6 +205,7 @@ def init(site, sites_path=None, new_site=False):
local.form_dict = _dict()
local.session = _dict()
local.dev_server = _dev_server
local.qb = get_query_builder(local.conf.db_type or "mariadb")
setup_module_map()
@ -528,16 +531,20 @@ def sendmail(recipients=[], sender="", subject="No Subject", message="No Message
if not delayed:
now = True
from frappe.email import queue
queue.send(recipients=recipients, sender=sender,
from frappe.email.doctype.email_queue.email_queue import QueueBuilder
builder = QueueBuilder(recipients=recipients, sender=sender,
subject=subject, message=message, text_content=text_content,
reference_doctype = doctype or reference_doctype, reference_name = name or reference_name, add_unsubscribe_link=add_unsubscribe_link,
unsubscribe_method=unsubscribe_method, unsubscribe_params=unsubscribe_params, unsubscribe_message=unsubscribe_message,
attachments=attachments, reply_to=reply_to, cc=cc, bcc=bcc, message_id=message_id, in_reply_to=in_reply_to,
send_after=send_after, expose_recipients=expose_recipients, send_priority=send_priority, queue_separately=queue_separately,
communication=communication, now=now, read_receipt=read_receipt, is_notification=is_notification,
communication=communication, read_receipt=read_receipt, is_notification=is_notification,
inline_images=inline_images, header=header, print_letterhead=print_letterhead, with_container=with_container)
# build email queue and send the email if send_now is True.
builder.process(send_now=now)
whitelisted = []
guest_methods = []
xss_safe_methods = []
@ -1107,9 +1114,7 @@ def setup_module_map():
if not (local.app_modules and local.module_app):
local.module_app, local.app_modules = {}, {}
for app in get_all_apps(True):
if app == "webnotes":
app = "frappe"
for app in get_all_apps(with_internal_apps=True):
local.app_modules.setdefault(app, [])
for module in get_module_list(app):
module = scrub(module)
@ -1490,7 +1495,7 @@ def get_print(doctype=None, name=None, print_format=None, style=None,
:param style: Print Format style.
:param as_pdf: Return as PDF. Default False.
:param password: Password to encrypt the pdf with. Default None"""
from frappe.website.render import build_page
from frappe.website.serve import get_response_content
from frappe.utils.pdf import get_pdf
local.form_dict.doctype = doctype
@ -1505,7 +1510,7 @@ def get_print(doctype=None, name=None, print_format=None, style=None,
options = {'password': password}
if not html:
html = build_page("printview")
html = get_response_content("printview")
if as_pdf:
return get_pdf(html, output = output, options = options)
@ -1682,7 +1687,7 @@ def get_desk_link(doctype, name):
)
def bold(text):
return '<b>{0}</b>'.format(text)
return '<strong>{0}</strong>'.format(text)
def safe_eval(code, eval_globals=None, eval_locals=None):
'''A safer `eval`'''

1
frappe/api.py
View file

@ -1,6 +1,5 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
import base64
import binascii
import json

14
frappe/app.py
View file

@ -1,10 +1,8 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import os
from six import iteritems
import logging
from werkzeug.local import LocalManager
@ -18,9 +16,9 @@ import frappe.handler
import frappe.auth
import frappe.api
import frappe.utils.response
import frappe.website.render
from frappe.utils import get_site_name, sanitize_html
from frappe.middlewares import StaticDataMiddleware
from frappe.website.serve import get_response
from frappe.utils.error import make_error_snapshot
from frappe.core.doctype.comment.comment import update_comments_in_parent_after_request
from frappe import _
@ -74,7 +72,7 @@ def application(request):
response = frappe.utils.response.download_private_file(request.path)
elif request.method in ('GET', 'HEAD', 'POST'):
response = frappe.website.render.render()
response = get_response()
else:
raise NotFound
@ -191,8 +189,9 @@ def make_form_dict(request):
frappe.throw(_("Invalid request arguments"))
try:
frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \
for k, v in iteritems(args) })
frappe.local.form_dict = frappe._dict({
k: v[0] if isinstance(v, (list, tuple)) else v for k, v in args.items()
})
except IndexError:
frappe.local.form_dict = frappe._dict(args)
@ -267,8 +266,7 @@ def handle_exception(e):
make_error_snapshot(e)
if return_as_message:
response = frappe.website.render.render("message",
http_status_code=http_status_code)
response = get_response("message", http_status_code=http_status_code)
return response

115
frappe/auth.py
View file

@ -1,35 +1,58 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See LICENSE
from urllib.parse import quote
from __future__ import unicode_literals
import datetime
from frappe import _
import frappe
import frappe.database
import frappe.utils
from frappe.utils import cint, flt, get_datetime, datetime, date_diff, today
import frappe.utils.user
from frappe import conf
from frappe.sessions import Session, clear_sessions, delete_session
from frappe.modules.patch_handler import check_session_stopped
from frappe.translate import get_lang_code
from frappe.utils.password import check_password, delete_login_failed_cache
from frappe import _, conf
from frappe.core.doctype.activity_log.activity_log import add_authentication_log
from frappe.twofactor import (should_run_2fa, authenticate_for_2factor,
confirm_otp_token, get_cached_user_pass)
from frappe.modules.patch_handler import check_session_stopped
from frappe.sessions import Session, clear_sessions, delete_session
from frappe.translate import get_language
from frappe.twofactor import authenticate_for_2factor, confirm_otp_token, get_cached_user_pass, should_run_2fa
from frappe.utils import cint, date_diff, datetime, get_datetime, today
from frappe.utils.password import check_password
from frappe.website.utils import get_home_page
from six.moves.urllib.parse import quote
class HTTPRequest:
def __init__(self):
# Get Environment variables
self.domain = frappe.request.host
if self.domain and self.domain.startswith('www.'):
self.domain = self.domain[4:]
# set frappe.local.request_ip
self.set_request_ip()
# load cookies
self.set_cookies()
# set frappe.local.db
self.connect()
# login and start/resume user session
self.set_session()
# set request language
self.set_lang()
# match csrf token from current session
self.validate_csrf_token()
# write out latest cookies
frappe.local.cookie_manager.init_cookies()
# check session status
check_session_stopped()
@property
def domain(self):
if not getattr(self, "_domain", None):
self._domain = frappe.request.host
if self._domain and self._domain.startswith('www.'):
self._domain = self._domain[4:]
return self._domain
def set_request_ip(self):
if frappe.get_request_header('X-Forwarded-For'):
frappe.local.request_ip = (frappe.get_request_header('X-Forwarded-For').split(",")[0]).strip()
@ -39,37 +62,21 @@ class HTTPRequest:
else:
frappe.local.request_ip = '127.0.0.1'
# language
self.set_lang()
# load cookies
def set_cookies(self):
frappe.local.cookie_manager = CookieManager()
# set db
self.connect()
# login
def set_session(self):
frappe.local.login_manager = LoginManager()
if frappe.form_dict._lang:
lang = get_lang_code(frappe.form_dict._lang)
if lang:
frappe.local.lang = lang
self.validate_csrf_token()
# write out latest cookies
frappe.local.cookie_manager.init_cookies()
# check status
check_session_stopped()
def validate_csrf_token(self):
if frappe.local.request and frappe.local.request.method in ("POST", "PUT", "DELETE"):
if not frappe.local.session: return
if not frappe.local.session.data.csrf_token \
or frappe.local.session.data.device=="mobile" \
or frappe.conf.get('ignore_csrf', None):
if not frappe.local.session:
return
if (
not frappe.local.session.data.csrf_token
or frappe.local.session.data.device == "mobile"
or frappe.conf.get('ignore_csrf', None)
):
# not via boot
return
@ -83,17 +90,18 @@ class HTTPRequest:
frappe.throw(_("Invalid Request"), frappe.CSRFTokenError)
def set_lang(self):
from frappe.translate import guess_language
frappe.local.lang = guess_language()
frappe.local.lang = get_language()
def get_db_name(self):
"""get database name from conf"""
return conf.db_name
def connect(self, ac_name = None):
def connect(self):
"""connect to db, from ac_name or db_name"""
frappe.local.db = frappe.database.get_db(user = self.get_db_name(), \
password = getattr(conf, 'db_password', ''))
frappe.local.db = frappe.database.get_db(
user=self.get_db_name(),
password=getattr(conf, 'db_password', '')
)
class LoginManager:
def __init__(self):
@ -146,8 +154,9 @@ class LoginManager:
self.make_session()
self.setup_boot_cache()
self.set_user_info()
self.clear_preferred_language()
def get_user_info(self, resume=False):
def get_user_info(self):
self.info = frappe.db.get_value("User", self.user,
["user_type", "first_name", "last_name", "user_image"], as_dict=1)
@ -185,11 +194,13 @@ class LoginManager:
frappe.local.response["redirect_to"] = redirect_to
frappe.cache().hdel('redirect_after_login', self.user)
frappe.local.cookie_manager.set_cookie("full_name", self.full_name)
frappe.local.cookie_manager.set_cookie("user_id", self.user)
frappe.local.cookie_manager.set_cookie("user_image", self.info.user_image or "")
def clear_preferred_language(self):
frappe.local.cookie_manager.delete_cookie("preferred_language")
def make_session(self, resume=False):
# start session
frappe.local.session_obj = Session(user=self.user, resume=resume,

2
frappe/automation/doctype/assignment_rule/assignment_rule.py
View file

@ -2,8 +2,6 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from frappe.desk.form import assign_to

2
frappe/automation/doctype/assignment_rule/test_assignment_rule.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import unittest
from frappe.utils import random_string

1
frappe/automation/doctype/assignment_rule_day/assignment_rule_day.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
# import frappe
from frappe.model.document import Document

1
frappe/automation/doctype/assignment_rule_user/assignment_rule_user.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
# import frappe
from frappe.model.document import Document

3
frappe/automation/doctype/auto_repeat/auto_repeat.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe import _
from datetime import timedelta
@ -334,7 +333,7 @@ class AutoRepeat(Document):
if self.reference_doctype and self.reference_document:
res = get_contacts_linking_to(self.reference_doctype, self.reference_document, fields=['email_id'])
res += get_contacts_linked_from(self.reference_doctype, self.reference_document, fields=['email_id'])
email_ids = list(set([d.email_id for d in res]))
email_ids = {d.email_id for d in res}
if not email_ids:
frappe.msgprint(_('No contacts linked to document'), alert=True)
else:

2
frappe/automation/doctype/auto_repeat/test_auto_repeat.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2018, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import unittest
import frappe

1
frappe/automation/doctype/auto_repeat_day/auto_repeat_day.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2020, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
# import frappe
from frappe.model.document import Document

2
frappe/automation/doctype/milestone/milestone.py
View file

@ -2,8 +2,6 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document

2
frappe/automation/doctype/milestone/test_milestone.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
#import frappe
import unittest

2
frappe/automation/doctype/milestone_tracker/milestone_tracker.py
View file

@ -2,8 +2,6 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
import frappe.cache_manager

2
frappe/automation/doctype/milestone_tracker/test_milestone_tracker.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import frappe.cache_manager
import unittest

9
frappe/boot.py
View file

@ -1,10 +1,5 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
from six import iteritems, text_type
"""
bootstrap client session
"""
@ -75,7 +70,7 @@ def get_bootinfo():
frappe.get_attr(method)(bootinfo)
if bootinfo.lang:
bootinfo.lang = text_type(bootinfo.lang)
bootinfo.lang = str(bootinfo.lang)
bootinfo.versions = {k: v['version'] for k, v in get_versions().items()}
bootinfo.error_report_email = frappe.conf.error_report_email
@ -220,7 +215,7 @@ def load_translations(bootinfo):
messages[name] = frappe._(name)
# only untranslated
messages = {k:v for k, v in iteritems(messages) if k!=v}
messages = {k: v for k, v in messages.items() if k!=v}
bootinfo["__messages"] = messages

4
frappe/build.py
View file

@ -1,11 +1,11 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
import os
import re
import json
import shutil
import subprocess
from io import StringIO
from tempfile import mkdtemp, mktemp
from distutils.spawn import find_executable
@ -402,8 +402,6 @@ def get_build_maps():
def pack(target, sources, no_compress, verbose):
from six import StringIO
outtype, outtxt = target.split(".")[-1], ""
jsm = JavascriptMinify()

22
frappe/cache_manager.py
View file

@ -1,8 +1,6 @@
# Copyright (c) 2018, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe, json
from frappe.model.document import Document
from frappe.desk.notifications import (delete_notification_count_for,
@ -55,7 +53,7 @@ def clear_domain_cache(user=None):
cache.delete_value(domain_cache_keys)
def clear_global_cache():
from frappe.website.render import clear_cache as clear_website_cache
from frappe.website.utils import clear_website_cache
clear_doctype_cache()
clear_website_cache()
@ -143,17 +141,13 @@ def build_table_count_cache():
return
_cache = frappe.cache()
data = frappe.db.multisql({
"mariadb": """
SELECT table_name AS name,
table_rows AS count
FROM information_schema.tables""",
"postgres": """
SELECT "relname" AS name,
"n_tup_ins" AS count
FROM "pg_stat_all_tables"
"""
}, as_dict=1)
table_name = frappe.qb.Field("table_name").as_("name")
table_rows = frappe.qb.Field("table_rows").as_("count")
information_schema = frappe.qb.Schema("information_schema")
query = frappe.qb.from_(information_schema.tables).select(table_name, table_rows)
data = frappe.db.sql(query, as_dict=1)
counts = {d.get('name').lstrip('tab'): d.get('count', None) for d in data}
_cache.set_value("information_schema:counts", counts)

2
frappe/chat/__init__.py
View file

@ -1,4 +1,4 @@
from __future__ import unicode_literals
import frappe
from frappe import _

2
frappe/chat/doctype/chat_message/chat_message.py
View file

@ -1,5 +1,3 @@
from __future__ import unicode_literals
# imports - standard imports
import json

2
frappe/chat/doctype/chat_profile/chat_profile.py
View file

@ -1,5 +1,3 @@
from __future__ import unicode_literals
# imports - module imports
from frappe.model.document import Document
from frappe import _

2
frappe/chat/doctype/chat_room/chat_room.py
View file

@ -1,5 +1,3 @@
from __future__ import unicode_literals
# imports - module imports
from frappe.model.document import Document
from frappe import _

2
frappe/chat/doctype/chat_room_user/chat_room_user.py
View file

@ -1,5 +1,3 @@
from __future__ import unicode_literals
# imports - module imports
from frappe.model.document import Document
import frappe

1
frappe/chat/doctype/chat_token/chat_token.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2018, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document

2
frappe/chat/util/__init__.py
View file

@ -1,5 +1,3 @@
from __future__ import unicode_literals
# imports - module imports
from frappe.chat.util.util import (
get_user_doc,

5
frappe/chat/util/test_util.py
View file

@ -1,5 +1,3 @@
from __future__ import unicode_literals
# imports - standard imports
import unittest
@ -9,7 +7,6 @@ from frappe.chat.util import (
safe_json_loads
)
import frappe
import six
class TestChatUtil(unittest.TestCase):
def test_safe_json_loads(self):
@ -20,7 +17,7 @@ class TestChatUtil(unittest.TestCase):
self.assertEqual(type(number), float)
string = safe_json_loads("foobar")
self.assertEqual(type(string), six.text_type)
self.assertEqual(type(string), str)
array = safe_json_loads('[{ "foo": "bar" }]')
self.assertEqual(type(array), list)

2
frappe/chat/util/util.py
View file

@ -1,5 +1,3 @@
from __future__ import unicode_literals
# imports - standard imports
import json
from collections.abc import MutableMapping, MutableSequence, Sequence

2
frappe/chat/website/__init__.py
View file

@ -1,4 +1,4 @@
from __future__ import unicode_literals
import frappe
from frappe.chat.util import filter_dict, safe_json_loads

19
frappe/client.py
View file

@ -1,7 +1,5 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe
from frappe import _
import frappe.model
@ -11,7 +9,6 @@ from frappe.utils import get_safe_filters
from frappe.desk.reportview import validate_args
from frappe.model.db_query import check_parent_permission
from six import iteritems, string_types, integer_types
'''
Handle RESTful requests that are mapped to the `/api/resource` route.
@ -86,7 +83,7 @@ def get_value(doctype, fieldname, filters=None, as_dict=True, debug=False, paren
frappe.throw(_("No permission for {0}").format(doctype), frappe.PermissionError)
filters = get_safe_filters(filters)
if isinstance(filters, string_types):
if isinstance(filters, str):
filters = {"name": filters}
try:
@ -135,7 +132,7 @@ def set_value(doctype, name, fieldname, value=None):
if not value:
values = fieldname
if isinstance(fieldname, string_types):
if isinstance(fieldname, str):
try:
values = json.loads(fieldname)
except ValueError:
@ -161,7 +158,7 @@ def insert(doc=None):
'''Insert a document
:param doc: JSON or dict object to be inserted'''
if isinstance(doc, string_types):
if isinstance(doc, str):
doc = json.loads(doc)
if doc.get("parent") and doc.get("parenttype"):
@ -179,7 +176,7 @@ def insert_many(docs=None):
'''Insert multiple documents
:param docs: JSON or list of dict objects to be inserted in one request'''
if isinstance(docs, string_types):
if isinstance(docs, str):
docs = json.loads(docs)
out = []
@ -205,7 +202,7 @@ def save(doc):
'''Update (save) an existing document
:param doc: JSON or dict object with the properties of the document to be updated'''
if isinstance(doc, string_types):
if isinstance(doc, str):
doc = json.loads(doc)
doc = frappe.get_doc(doc)
@ -228,7 +225,7 @@ def submit(doc):
'''Submit a document
:param doc: JSON or dict object to be submitted remotely'''
if isinstance(doc, string_types):
if isinstance(doc, str):
doc = json.loads(doc)
doc = frappe.get_doc(doc)
@ -266,7 +263,7 @@ def make_width_property_setter(doc):
'''Set width Property Setter
:param doc: Property Setter document with `width` property'''
if isinstance(doc, string_types):
if isinstance(doc, str):
doc = json.loads(doc)
if doc["doctype"]=="Property Setter" and doc["property"]=="width":
frappe.get_doc(doc).insert(ignore_permissions = True)
@ -280,7 +277,7 @@ def bulk_update(docs):
failed_docs = []
for doc in docs:
try:
ddoc = {key: val for key, val in iteritems(doc) if key not in ['doctype', 'docname']}
ddoc = {key: val for key, val in doc.items() if key not in ['doctype', 'docname']}
doctype = doc['doctype']
docname = doc['docname']
doc = frappe.get_doc(doctype, docname)

7
frappe/commands/__init__.py
View file

@ -1,7 +1,6 @@
# Copyright (c) 2015, Web Notes Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals, absolute_import, print_function
import sys
import click
import cProfile
@ -10,7 +9,7 @@ import frappe
import frappe.utils
import subprocess # nosec
from functools import wraps
from six import StringIO
from io import StringIO
from os import environ
click.disable_unicode_literals_warning = True
@ -103,7 +102,9 @@ def get_commands():
from .site import commands as site_commands
from .translate import commands as translate_commands
from .utils import commands as utils_commands
from .redis import commands as redis_commands
return list(set(scheduler_commands + site_commands + translate_commands + utils_commands))
all_commands = scheduler_commands + site_commands + translate_commands + utils_commands + redis_commands
return list(set(all_commands))
commands = get_commands()

53
frappe/commands/redis.py Normal file
View file

@ -0,0 +1,53 @@
import os
import click
import frappe
from frappe.utils.rq import RedisQueue
from frappe.installer import update_site_config
@click.command('create-rq-users')
@click.option('--set-admin-password', is_flag=True, default=False, help='Set new Redis admin(default user) password')
@click.option('--use-rq-auth', is_flag=True, default=False, help='Enable Redis authentication for sites')
def create_rq_users(set_admin_password=False, use_rq_auth=False):
"""Create Redis Queue users and add to acl and app configs.
acl config file will be used by redis server while starting the server
and app config is used by app while connecting to redis server.
"""
acl_file_path = os.path.abspath('../config/redis_queue.acl')
with frappe.init_site():
acl_list, user_credentials = RedisQueue.gen_acl_list(
set_admin_password=set_admin_password)
with open(acl_file_path, 'w') as f:
f.writelines([acl+'\n' for acl in acl_list])
sites_path = os.getcwd()
common_site_config_path = os.path.join(sites_path, 'common_site_config.json')
update_site_config("rq_username", user_credentials['bench'][0], validate=False,
site_config_path=common_site_config_path)
update_site_config("rq_password", user_credentials['bench'][1], validate=False,
site_config_path=common_site_config_path)
update_site_config("use_rq_auth", use_rq_auth, validate=False,
site_config_path=common_site_config_path)
click.secho('* ACL and site configs are updated with new user credentials. '
'Please restart Redis Queue server to enable namespaces.',
fg='green')
if set_admin_password:
env_key = 'RQ_ADMIN_PASWORD'
click.secho('* Redis admin password is successfully set up. '
'Include below line in .bashrc file for system to use',
fg='green')
click.secho(f"`export {env_key}={user_credentials['default'][1]}`")
click.secho('NOTE: Please save the admin password as you '
'can not access redis server without the password',
fg='yellow')
commands = [
create_rq_users
]

9
frappe/commands/scheduler.py
View file

@ -1,4 +1,3 @@
from __future__ import unicode_literals, absolute_import, print_function
import click
import sys
import frappe
@ -173,9 +172,13 @@ def start_scheduler():
@click.command('worker')
@click.option('--queue', type=str)
@click.option('--quiet', is_flag = True, default = False, help = 'Hide Log Outputs')
def start_worker(queue, quiet = False):
@click.option('-u', '--rq-username', default=None, help='Redis ACL user')
@click.option('-p', '--rq-password', default=None, help='Redis ACL user password')
def start_worker(queue, quiet = False, rq_username=None, rq_password=None):
"""Site is used to find redis credentals.
"""
from frappe.utils.background_jobs import start_worker
start_worker(queue, quiet = quiet)
start_worker(queue, quiet = quiet, rq_username=rq_username, rq_password=rq_password)
@click.command('ready-for-migration')
@click.option('--site', help='site name')

1
frappe/commands/translate.py
View file

@ -1,4 +1,3 @@
from __future__ import unicode_literals, absolute_import, print_function
import click
from frappe.commands import pass_context, get_site
from frappe.exceptions import SiteNotSpecifiedError

120
frappe/commands/utils.py
View file

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
import json
import os
import subprocess
@ -14,6 +12,13 @@ from frappe.exceptions import SiteNotSpecifiedError
from frappe.utils import get_bench_path, update_progress_bar, cint
DATA_IMPORT_DEPRECATION = click.style(
"[DEPRECATED] The `import-csv` command used 'Data Import Legacy' which has been deprecated.\n"
"Use `data-import` command instead to import data via 'Data Import'.",
fg="yellow"
)
@click.command('build')
@click.option('--app', help='Build assets for app')
@click.option('--apps', help='Build assets for specific apps')
@ -69,14 +74,14 @@ def watch(apps=None):
def clear_cache(context):
"Clear cache, doctype cache and defaults"
import frappe.sessions
import frappe.website.render
from frappe.website.utils import clear_website_cache
from frappe.desk.notifications import clear_notifications
for site in context.sites:
try:
frappe.connect(site)
frappe.clear_cache()
clear_notifications()
frappe.website.render.clear_cache()
clear_website_cache()
finally:
frappe.destroy()
if not context.sites:
@ -86,12 +91,12 @@ def clear_cache(context):
@pass_context
def clear_website_cache(context):
"Clear website cache"
import frappe.website.render
from frappe.website.utils import clear_website_cache
for site in context.sites:
try:
frappe.init(site=site)
frappe.connect()
frappe.website.render.clear_cache()
clear_website_cache()
finally:
frappe.destroy()
if not context.sites:
@ -222,7 +227,7 @@ def execute(context, method, args=None, kwargs=None, profile=False):
if profile:
import pstats
from six import StringIO
from io import StringIO
pr.disable()
s = StringIO()
@ -350,7 +355,8 @@ def import_doc(context, path, force=False):
if not context.sites:
raise SiteNotSpecifiedError
@click.command('import-csv')
@click.command('import-csv', help=DATA_IMPORT_DEPRECATION)
@click.argument('path')
@click.option('--only-insert', default=False, is_flag=True, help='Do not overwrite existing records')
@click.option('--submit-after-import', default=False, is_flag=True, help='Submit document after importing it')
@ -358,32 +364,8 @@ def import_doc(context, path, force=False):
@click.option('--no-email', default=True, is_flag=True, help='Send email if applicable')
@pass_context
def import_csv(context, path, only_insert=False, submit_after_import=False, ignore_encoding_errors=False, no_email=True):
"Import CSV using data import"
from frappe.core.doctype.data_import_legacy import importer
from frappe.utils.csvutils import read_csv_content
site = get_site(context)
if not os.path.exists(path):
path = os.path.join('..', path)
if not os.path.exists(path):
print('Invalid path {0}'.format(path))
sys.exit(1)
with open(path, 'r') as csvfile:
content = read_csv_content(csvfile.read())
frappe.init(site=site)
frappe.connect()
try:
importer.upload(content, submit_after_import=submit_after_import, no_email=no_email,
ignore_encoding_errors=ignore_encoding_errors, overwrite=not only_insert,
via_console=True)
frappe.db.commit()
except Exception:
print(frappe.get_traceback())
frappe.destroy()
click.secho(DATA_IMPORT_DEPRECATION)
sys.exit(1)
@click.command('data-import')
@ -569,25 +551,16 @@ def run_tests(context, app=None, module=None, doctype=None, test=(), profile=Fal
if coverage:
from coverage import Coverage
from frappe.coverage import STANDARD_INCLUSIONS, STANDARD_EXCLUSIONS, FRAPPE_EXCLUSIONS
# Generate coverage report only for app that is being tested
source_path = os.path.join(get_bench_path(), 'apps', app or 'frappe')
omit=[
'*.html',
'*.js',
'*.xml',
'*.css',
'*.less',
'*.scss',
'*.vue',
'*/doctype/*/*_dashboard.py',
'*/patches/*'
]
omit = STANDARD_EXCLUSIONS[:]
if not app or app == 'frappe':
omit.append('*/commands/*')
omit.extend(FRAPPE_EXCLUSIONS)
cov = Coverage(source=[source_path], omit=omit)
cov = Coverage(source=[source_path], omit=omit, include=STANDARD_INCLUSIONS)
cov.start()
ret = frappe.test_runner.main(app, module, doctype, context.verbose, tests=tests,
@ -654,7 +627,7 @@ def run_ui_tests(context, app, headless=False, parallel=True, ci_build_id=None):
frappe.commands.popen("yarn add cypress@^6 cypress-file-upload@^5 --no-lockfile")
# run for headless mode
run_or_open = 'run --browser firefox --record --key 4a48f41c-11b3-425b-aa88-c58048fa69eb' if headless else 'open'
run_or_open = 'run --browser firefox --record' if headless else 'open'
command = '{site_env} {password_env} {cypress} {run_or_open}'
formatted_command = command.format(site_env=site_env, password_env=password_env, cypress=cypress_path, run_or_open=run_or_open)
@ -760,22 +733,49 @@ def set_config(context, key, value, global_=False, parse=False, as_dict=False):
frappe.destroy()
@click.command('version')
def get_version():
"Show the versions of all the installed apps"
@click.command("version")
@click.option("-f", "--format", "output",
type=click.Choice(["plain", "table", "json", "legacy"]), help="Output format", default="legacy")
def get_version(output):
"""Show the versions of all the installed apps."""
from git import Repo
from frappe.utils.commands import render_table
from frappe.utils.change_log import get_app_branch
frappe.init('')
for m in sorted(frappe.get_all_apps()):
branch_name = get_app_branch(m)
module = frappe.get_module(m)
app_hooks = frappe.get_module(m + ".hooks")
frappe.init("")
data = []
if hasattr(app_hooks, '{0}_version'.format(branch_name)):
print("{0} {1}".format(m, getattr(app_hooks, '{0}_version'.format(branch_name))))
for app in sorted(frappe.get_all_apps()):
module = frappe.get_module(app)
app_hooks = frappe.get_module(app + ".hooks")
repo = Repo(frappe.get_app_path(app, ".."))
elif hasattr(module, "__version__"):
print("{0} {1}".format(m, module.__version__))
app_info = frappe._dict()
app_info.app = app
app_info.branch = get_app_branch(app)
app_info.commit = repo.head.object.hexsha[:7]
app_info.version = getattr(app_hooks, f"{app_info.branch}_version", None) or module.__version__
data.append(app_info)
{
"legacy": lambda: [
click.echo(f"{app_info.app} {app_info.version}")
for app_info in data
],
"plain": lambda: [
click.echo(f"{app_info.app} {app_info.version} {app_info.branch} ({app_info.commit})")
for app_info in data
],
"table": lambda: render_table(
[["App", "Version", "Branch", "Commit"]] +
[
[app_info.app, app_info.version, app_info.branch, app_info.commit]
for app_info in data
]
),
"json": lambda: click.echo(json.dumps(data, indent=4)),
}[output]()
@click.command('rebuild-global-search')

22
frappe/config/__init__.py
View file

@ -1,6 +1,3 @@
from __future__ import unicode_literals
import json
from six import iteritems
import frappe
from frappe import _
from frappe.desk.moduleview import (get_data, get_onboard_items, config_exists, get_module_link_items_from_list)
@ -42,18 +39,13 @@ def get_modules_from_app(app):
)
def get_all_empty_tables_by_module():
empty_tables = set(r[0] for r in frappe.db.multisql({
"mariadb": """
SELECT table_name
FROM information_schema.tables
WHERE table_rows = 0 and table_schema = "{}"
""".format(frappe.conf.db_name),
"postgres": """
SELECT "relname" as "table_name"
FROM "pg_stat_all_tables"
WHERE n_tup_ins = 0
"""
}))
table_rows = frappe.qb.Field("table_rows")
table_name = frappe.qb.Field("table_name")
information_schema = frappe.qb.Schema("information_schema")
query = frappe.qb.from_(information_schema.tables).select(table_name).where(table_rows == 0)
empty_tables = {r[0] for r in frappe.db.sql(query)}
results = frappe.get_all("DocType", fields=["name", "module"])
empty_tables_by_module = {}

3
frappe/contacts/address_and_contact.py
View file

@ -1,7 +1,6 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt
from __future__ import unicode_literals
import frappe
from frappe import _
@ -154,7 +153,7 @@ def filter_dynamic_link_doctypes(doctype, txt, searchfield, start, page_len, fil
doctypes = frappe.db.get_all("DocField", filters=filters, fields=["parent"],
distinct=True, as_list=True)
doctypes = tuple([d for d in doctypes if re.search(txt+".*", _(d[0]), re.IGNORECASE)])
doctypes = tuple(d for d in doctypes if re.search(txt+".*", _(d[0]), re.IGNORECASE))
filters.update({
"dt": ("not in", [d[0] for d in doctypes])

24
frappe/contacts/doctype/address/address.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe import throw, _
@ -10,15 +9,10 @@ from frappe.utils import cstr
from frappe.model.document import Document
from jinja2 import TemplateSyntaxError
from frappe.utils.user import is_website_user
from frappe.model.naming import make_autoname
from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links
from six import iteritems, string_types
from past.builtins import cmp
from frappe.contacts.address_and_contact import set_link_title
import functools
class Address(Document):
def __setup__(self):
@ -112,10 +106,13 @@ def get_default_address(doctype, name, sort_key='is_primary_address'):
WHERE
dl.parent = addr.name and dl.link_doctype = %s and
dl.link_name = %s and ifnull(addr.disabled, 0) = 0
""" %(sort_key, '%s', '%s'), (doctype, name))
""" %(sort_key, '%s', '%s'), (doctype, name), as_dict=True)
if out:
return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(y[1], x[1])))[0][0]
for contact in out:
if contact.get(sort_key):
return contact.name
return out[0].name
else:
return None
@ -141,7 +138,7 @@ def get_territory_from_address(address):
if not address:
return
if isinstance(address, string_types):
if isinstance(address, str):
address = frappe.get_cached_doc("Address", address)
territory = None
@ -174,14 +171,11 @@ def get_address_list(doctype, txt, filters, limit_start, limit_page_length = 20,
def has_website_permission(doc, ptype, user, verbose=False):
"""Returns true if there is a related lead or contact related to this document"""
contact_name = frappe.db.get_value("Contact", {"email_id": frappe.session.user})
if contact_name:
contact = frappe.get_doc('Contact', contact_name)
return contact.has_common_link(doc)
lead_name = frappe.db.get_value("Lead", {"email_id": frappe.session.user})
if lead_name:
return doc.has_link('Lead', lead_name)
return False
def get_address_templates(address):
@ -214,7 +208,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters):
condition = ""
meta = frappe.get_meta("Address")
for fieldname, value in iteritems(filters):
for fieldname, value in filters.items():
if meta.get_field(fieldname) or fieldname in frappe.db.DEFAULT_COLUMNS:
condition += " and {field}={value}".format(
field=fieldname,
@ -263,7 +257,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters):
def get_condensed_address(doc):
fields = ["address_title", "address_line1", "address_line2", "city", "county", "state", "country"]
return ", ".join([doc.get(d) for d in fields if doc.get(d)])
return ", ".join(doc.get(d) for d in fields if doc.get(d))
def update_preferred_address(address, field):
frappe.db.set_value('Address', address, field, 0)

2
frappe/contacts/doctype/address/test_address.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe, unittest
from frappe.contacts.doctype.address.address import get_address_display

1
frappe/contacts/doctype/address_template/address_template.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from frappe.utils import cint

4
frappe/contacts/doctype/address_template/test_address_template.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe, unittest
class TestAddressTemplate(unittest.TestCase):
@ -42,4 +40,4 @@ class TestAddressTemplate(unittest.TestCase):
"doctype": "Address Template",
"country": 'Brazil',
"template": template
}).insert()
}).insert()

16
frappe/contacts/doctype/contact/contact.py
View file

@ -1,18 +1,13 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt
from __future__ import unicode_literals
import frappe
from frappe.utils import cstr, has_gravatar, cint
from frappe.utils import cstr, has_gravatar
from frappe import _
from frappe.model.document import Document
from frappe.core.doctype.dynamic_link.dynamic_link import deduplicate_dynamic_links
from six import iteritems
from past.builtins import cmp
from frappe.model.naming import append_number_if_name_exists
from frappe.contacts.address_and_contact import set_link_title
import functools
class Contact(Document):
def autoname(self):
@ -120,7 +115,7 @@ class Contact(Document):
if len(is_primary) > 1:
frappe.throw(_("Only one {0} can be set as primary.").format(frappe.bold(frappe.unscrub(fieldname))))
primary_number_exists = False
primary_number_exists = False
for d in self.phone_nos:
if d.get(field_name) == 1:
primary_number_exists = True
@ -140,10 +135,13 @@ def get_default_contact(doctype, name):
where
dl.link_doctype=%s and
dl.link_name=%s and
dl.parenttype = "Contact"''', (doctype, name))
dl.parenttype = "Contact"''', (doctype, name), as_dict=True)
if out:
return sorted(out, key = functools.cmp_to_key(lambda x,y: cmp(cint(y[1]), cint(x[1]))))[0][0]
for contact in out:
if contact.is_primary_contact:
return contact.parent
return out[0].parent
else:
return None

2
frappe/contacts/doctype/contact/test_contact.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import unittest

1
frappe/contacts/doctype/contact_email/contact_email.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
# import frappe
from frappe.model.document import Document

1
frappe/contacts/doctype/contact_phone/contact_phone.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
# import frappe
from frappe.model.document import Document

1
frappe/contacts/doctype/gender/gender.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2017, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
from frappe.model.document import Document
class Gender(Document):

2
frappe/contacts/doctype/gender/test_gender.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import unittest
class TestGender(unittest.TestCase):

1
frappe/contacts/doctype/salutation/salutation.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2017, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
from frappe.model.document import Document
class Salutation(Document):

2
frappe/contacts/doctype/salutation/test_salutation.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import unittest
class TestSalutation(unittest.TestCase):

5
frappe/contacts/report/addresses_and_contacts/addresses_and_contacts.py
View file

@ -1,8 +1,5 @@
# Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
from six import iteritems
import frappe
from frappe import _
@ -58,7 +55,7 @@ def get_reference_addresses_and_contact(reference_doctype, reference_name):
reference_details = get_reference_details(reference_doctype, "Address", reference_list, reference_details)
reference_details = get_reference_details(reference_doctype, "Contact", reference_list, reference_details)
for reference_name, details in iteritems(reference_details):
for reference_name, details in reference_details.items():
addresses = details.get("address", [])
contacts = details.get("contact", [])
if not any([addresses, contacts]):

2
frappe/contacts/report/addresses_and_contacts/test_addresses_and_contacts.py
View file

@ -1,4 +1,4 @@
from __future__ import unicode_literals
import frappe
import frappe.defaults
import unittest

2
frappe/core/__init__.py
View file

@ -1,4 +1,2 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

1
frappe/core/doctype/__init__.py
View file

@ -1,4 +1,3 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

6
frappe/core/doctype/access_log/access_log.py
View file

@ -1,11 +1,5 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
# imports - standard imports
from __future__ import unicode_literals
# imports - module imports
import frappe
from frappe.model.document import Document

1
frappe/core/doctype/activity_log/activity_log.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2017, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
from frappe import _
from frappe.utils import get_fullname, now
from frappe.model.document import Document

14
frappe/core/doctype/activity_log/feed.py
View file

@ -1,13 +1,11 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: See license.txt
from __future__ import unicode_literals
import frappe
import frappe.permissions
from frappe.utils import get_fullname
from frappe import _
from frappe.core.doctype.activity_log.activity_log import add_authentication_log
from six import string_types
def update_feed(doc, method=None):
if frappe.flags.in_patch or frappe.flags.in_install or frappe.flags.in_import:
@ -23,7 +21,7 @@ def update_feed(doc, method=None):
feed = doc.get_feed()
if feed:
if isinstance(feed, string_types):
if isinstance(feed, str):
feed = {"subject": feed}
feed = frappe._dict(feed)
@ -31,10 +29,12 @@ def update_feed(doc, method=None):
name = feed.name or doc.name
# delete earlier feed
frappe.db.sql("""delete from `tabActivity Log`
where
reference_doctype=%s and reference_name=%s
and link_doctype=%s""", (doctype, name,feed.link_doctype))
frappe.db.delete("Activity Log", {
"reference_doctype": doctype,
"reference_name": name,
"link_doctype": feed.link_doctype
})
frappe.get_doc({
"doctype": "Activity Log",
"reference_doctype": doctype,

2
frappe/core/doctype/activity_log/test_activity_log.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import unittest
import time

1
frappe/core/doctype/block_module/block_module.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document

4
frappe/core/doctype/comment/comment.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals, absolute_import
import frappe
from frappe import _
import json
@ -11,7 +9,7 @@ from frappe.core.doctype.user.user import extract_mentions
from frappe.desk.doctype.notification_log.notification_log import enqueue_create_notification,\
get_title, get_title_html
from frappe.utils import get_fullname
from frappe.website.render import clear_cache
from frappe.website.utils import clear_cache
from frappe.database.schema import add_column
from frappe.exceptions import ImplicitCommitError

2
frappe/core/doctype/comment/test_comment.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe, json
import unittest

80
frappe/core/doctype/communication/communication.py
View file

@ -1,26 +1,26 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals, absolute_import
from collections import Counter
import frappe
from frappe import _
from frappe.model.document import Document
from frappe.utils import validate_email_address, strip_html, cstr, time_diff_in_seconds
from frappe.core.doctype.communication.email import validate_email, notify, _notify
from frappe.core.doctype.communication.email import validate_email
from frappe.core.doctype.communication.mixins import CommunicationEmailMixin
from frappe.core.utils import get_parent_doc
from frappe.utils.bot import BotReply
from frappe.utils import parse_addr
from frappe.utils import parse_addr, split_emails
from frappe.core.doctype.comment.comment import update_comment_in_doc
from email.utils import parseaddr
from six.moves.urllib.parse import unquote
from urllib.parse import unquote
from frappe.utils.user import is_system_user
from frappe.contacts.doctype.contact.contact import get_contact_name
from frappe.automation.doctype.assignment_rule.assignment_rule import apply as apply_assignment_rule
exclude_from_linked_with = True
class Communication(Document):
class Communication(Document, CommunicationEmailMixin):
"""Communication represents an external communication like Email.
"""
no_feed_on_delete = True
@ -126,6 +126,45 @@ class Communication(Document):
if self.communication_type == "Communication":
self.notify_change('delete')
@property
def sender_mailid(self):
return parse_addr(self.sender)[1] if self.sender else ""
@staticmethod
def _get_emails_list(emails=None, exclude_displayname = False):
"""Returns list of emails from given email string.
* Removes duplicate mailids
* Removes display name from email address if exclude_displayname is True
"""
emails = split_emails(emails) if isinstance(emails, str) else (emails or [])
if exclude_displayname:
return [email.lower() for email in set([parse_addr(email)[1] for email in emails]) if email]
return [email.lower() for email in set(emails) if email]
def to_list(self, exclude_displayname = True):
"""Returns to list.
"""
return self._get_emails_list(self.recipients, exclude_displayname=exclude_displayname)
def cc_list(self, exclude_displayname = True):
"""Returns cc list.
"""
return self._get_emails_list(self.cc, exclude_displayname=exclude_displayname)
def bcc_list(self, exclude_displayname = True):
"""Returns bcc list.
"""
return self._get_emails_list(self.bcc, exclude_displayname=exclude_displayname)
def get_attachments(self):
attachments = frappe.get_all(
"File",
fields=["name", "file_name", "file_url", "is_private"],
filters = {"attached_to_name": self.name, "attached_to_doctype": self.DOCTYPE}
)
return attachments
def notify_change(self, action):
frappe.publish_realtime('update_docinfo_for_{}_{}'.format(self.reference_doctype, self.reference_name), {
'doc': self.as_dict(),
@ -199,36 +238,6 @@ class Communication(Document):
if not self.sender_full_name:
self.sender_full_name = sender_email
def send(self, print_html=None, print_format=None, attachments=None,
send_me_a_copy=False, recipients=None):
"""Send communication via Email.
:param print_html: Send given value as HTML attachment.
:param print_format: Attach print format of parent document."""
self.send_me_a_copy = send_me_a_copy
self.notify(print_html, print_format, attachments, recipients)
def notify(self, print_html=None, print_format=None, attachments=None,
recipients=None, cc=None, bcc=None,fetched_from_email_account=False):
"""Calls a delayed task 'sendmail' that enqueus email in Email Queue queue
:param print_html: Send given value as HTML attachment
:param print_format: Attach print format of parent document
:param attachments: A list of filenames that should be attached when sending this email
:param recipients: Email recipients
:param cc: Send email as CC to
:param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient
"""
notify(self, print_html, print_format, attachments, recipients, cc, bcc,
fetched_from_email_account)
def _notify(self, print_html=None, print_format=None, attachments=None,
recipients=None, cc=None, bcc=None):
_notify(self, print_html, print_format, attachments, recipients, cc, bcc)
def bot_reply(self):
if self.comment_type == 'Bot' and self.communication_type == 'Chat':
reply = BotReply().get_reply(self.content)
@ -505,3 +514,4 @@ def set_avg_response_time(parent, communication):
if response_times:
avg_response_time = sum(response_times) / len(response_times)
parent.db_set("avg_response_time", avg_response_time)

349
frappe/core/doctype/communication/email.py
View file

@ -1,9 +1,6 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals, absolute_import
from six.moves import range
from six import string_types
import frappe
import json
from email.utils import formataddr
@ -16,6 +13,11 @@ import time
from frappe import _
from frappe.utils.background_jobs import enqueue
OUTGOING_EMAIL_ACCOUNT_MISSING = _("""
Unable to send mail because of a missing email account.
Please setup default Email Account from Setup > Email > Email Account
""")
@frappe.whitelist()
def make(doctype=None, name=None, content=None, subject=None, sent_or_received = "Sent",
sender=None, sender_full_name=None, recipients=None, communication_medium="Email", send_email=False,
@ -39,7 +41,6 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
:param send_me_a_copy: Send a copy to the sender (default **False**).
:param email_template: Template which is used to compose mail .
"""
is_error_report = (doctype=="User" and name==frappe.session.user and subject=="Error Report")
send_me_a_copy = cint(send_me_a_copy)
@ -77,22 +78,24 @@ def make(doctype=None, name=None, content=None, subject=None, sent_or_received =
comm.save(ignore_permissions=True)
if isinstance(attachments, string_types):
if isinstance(attachments, str):
attachments = json.loads(attachments)
# if not committed, delayed task doesn't find the communication
if attachments:
add_attachments(comm.name, attachments)
frappe.db.commit()
if cint(send_email):
frappe.flags.print_letterhead = cint(print_letterhead)
comm.send(print_html, print_format, attachments, send_me_a_copy=send_me_a_copy)
if not comm.get_outgoing_email_account():
frappe.throw(msg=OUTGOING_EMAIL_ACCOUNT_MISSING, exc=frappe.OutgoingEmailError)
comm.send_email(print_html=print_html, print_format=print_format,
send_me_a_copy=send_me_a_copy, print_letterhead=print_letterhead)
emails_not_sent_to = comm.exclude_emails_list(include_sender=send_me_a_copy)
return {
"name": comm.name,
"emails_not_sent_to": ", ".join(comm.emails_not_sent_to) if hasattr(comm, "emails_not_sent_to") else None
"emails_not_sent_to": ", ".join(emails_not_sent_to or [])
}
def validate_email(doc):
@ -113,164 +116,6 @@ def validate_email(doc):
# validate sender
def notify(doc, print_html=None, print_format=None, attachments=None,
recipients=None, cc=None, bcc=None, fetched_from_email_account=False):
"""Calls a delayed task 'sendmail' that enqueus email in Email Queue queue
:param print_html: Send given value as HTML attachment
:param print_format: Attach print format of parent document
:param attachments: A list of filenames that should be attached when sending this email
:param recipients: Email recipients
:param cc: Send email as CC to
:param bcc: Send email as BCC to
:param fetched_from_email_account: True when pulling email, the notification shouldn't go to the main recipient
"""
recipients, cc, bcc = get_recipients_cc_and_bcc(doc, recipients, cc, bcc,
fetched_from_email_account=fetched_from_email_account)
if not recipients and not cc:
return
doc.emails_not_sent_to = set(doc.all_email_addresses) - set(doc.sent_email_addresses)
if frappe.flags.in_test:
# for test cases, run synchronously
doc._notify(print_html=print_html, print_format=print_format, attachments=attachments,
recipients=recipients, cc=cc, bcc=None)
else:
enqueue(sendmail, queue="default", timeout=300, event="sendmail",
communication_name=doc.name,
print_html=print_html, print_format=print_format, attachments=attachments,
recipients=recipients, cc=cc, bcc=bcc, lang=frappe.local.lang,
session=frappe.local.session, print_letterhead=frappe.flags.print_letterhead)
def _notify(doc, print_html=None, print_format=None, attachments=None,
recipients=None, cc=None, bcc=None):
prepare_to_notify(doc, print_html, print_format, attachments)
if doc.outgoing_email_account.send_unsubscribe_message:
unsubscribe_message = _("Leave this conversation")
else:
unsubscribe_message = ""
frappe.sendmail(
recipients=(recipients or []),
cc=(cc or []),
bcc=(bcc or []),
expose_recipients="header",
sender=doc.sender,
reply_to=doc.incoming_email_account,
subject=doc.subject,
content=doc.content,
reference_doctype=doc.reference_doctype,
reference_name=doc.reference_name,
attachments=doc.attachments,
message_id=doc.message_id,
unsubscribe_message=unsubscribe_message,
delayed=True,
communication=doc.name,
read_receipt=doc.read_receipt,
is_notification=True if doc.sent_or_received =="Received" else False,
print_letterhead=frappe.flags.print_letterhead
)
def get_recipients_cc_and_bcc(doc, recipients, cc, bcc, fetched_from_email_account=False):
doc.all_email_addresses = []
doc.sent_email_addresses = []
doc.previous_email_sender = None
if not recipients:
recipients = get_recipients(doc, fetched_from_email_account=fetched_from_email_account)
if not cc:
cc = get_cc(doc, recipients, fetched_from_email_account=fetched_from_email_account)
if not bcc:
bcc = get_bcc(doc, recipients, fetched_from_email_account=fetched_from_email_account)
if fetched_from_email_account:
# email was already sent to the original recipient by the sender's email service
original_recipients, recipients = recipients, []
# send email to the sender of the previous email in the thread which this email is a reply to
#provides erratic results and can send external
#if doc.previous_email_sender:
# recipients.append(doc.previous_email_sender)
# cc that was received in the email
original_cc = split_emails(doc.cc)
# don't cc to people who already received the mail from sender's email service
cc = list(set(cc) - set(original_cc) - set(original_recipients))
remove_administrator_from_email_list(cc)
original_bcc = split_emails(doc.bcc)
bcc = list(set(bcc) - set(original_bcc) - set(original_recipients))
remove_administrator_from_email_list(bcc)
remove_administrator_from_email_list(recipients)
return recipients, cc, bcc
def remove_administrator_from_email_list(email_list):
administrator_email = list(filter(lambda emails: "Administrator" in emails, email_list))
if administrator_email:
email_list.remove(administrator_email[0])
def prepare_to_notify(doc, print_html=None, print_format=None, attachments=None):
"""Prepare to make multipart MIME Email
:param print_html: Send given value as HTML attachment.
:param print_format: Attach print format of parent document."""
view_link = frappe.utils.cint(frappe.db.get_value("System Settings", "System Settings", "attach_view_link"))
if print_format and view_link:
doc.content += get_attach_link(doc, print_format)
set_incoming_outgoing_accounts(doc)
if not doc.sender:
doc.sender = doc.outgoing_email_account.email_id
if not doc.sender_full_name:
doc.sender_full_name = doc.outgoing_email_account.name or _("Notification")
if doc.sender:
# combine for sending to get the format 'Jane <jane@example.com>'
doc.sender = get_formatted_email(doc.sender_full_name, mail=doc.sender)
doc.attachments = []
if print_html or print_format:
doc.attachments.append({"print_format_attachment":1, "doctype":doc.reference_doctype,
"name":doc.reference_name, "print_format":print_format, "html":print_html})
if attachments:
if isinstance(attachments, string_types):
attachments = json.loads(attachments)
for a in attachments:
if isinstance(a, string_types):
# is it a filename?
try:
# check for both filename and file id
file_id = frappe.db.get_list('File', or_filters={'file_name': a, 'name': a}, limit=1)
if not file_id:
frappe.throw(_("Unable to find attachment {0}").format(a))
file_id = file_id[0]['name']
_file = frappe.get_doc("File", file_id)
_file.get_content()
# these attachments will be attached on-demand
# and won't be stored in the message
doc.attachments.append({"fid": file_id})
except IOError:
frappe.throw(_("Unable to find attachment {0}").format(a))
else:
doc.attachments.append(a)
def set_incoming_outgoing_accounts(doc):
from frappe.email.doctype.email_account.email_account import EmailAccount
incoming_email_account = EmailAccount.find_incoming(
@ -283,82 +128,13 @@ def set_incoming_outgoing_accounts(doc):
if doc.sent_or_received == "Sent":
doc.db_set("email_account", doc.outgoing_email_account.name)
def get_recipients(doc, fetched_from_email_account=False):
"""Build a list of email addresses for To"""
# [EDGE CASE] doc.recipients can be None when an email is sent as BCC
recipients = split_emails(doc.recipients)
#if fetched_from_email_account and doc.in_reply_to:
# add sender of previous reply
#doc.previous_email_sender = frappe.db.get_value("Communication", doc.in_reply_to, "sender")
#recipients.append(doc.previous_email_sender)
if recipients:
recipients = filter_email_list(doc, recipients, [])
return recipients
def get_cc(doc, recipients=None, fetched_from_email_account=False):
"""Build a list of email addresses for CC"""
# get a copy of CC list
cc = split_emails(doc.cc)
if doc.reference_doctype and doc.reference_name:
if fetched_from_email_account:
# if it is a fetched email, add follows to CC
cc.append(get_owner_email(doc))
cc += get_assignees(doc)
if getattr(doc, "send_me_a_copy", False) and doc.sender not in cc:
cc.append(doc.sender)
if cc:
# exclude unfollows, recipients and unsubscribes
exclude = [] #added to remove account check
exclude += [d[0] for d in frappe.db.get_all("User", ["email"], {"thread_notify": 0}, as_list=True)]
exclude += [(parse_addr(email)[1] or "").lower() for email in recipients]
if fetched_from_email_account:
# exclude sender when pulling email
exclude += [parse_addr(doc.sender)[1]]
if doc.reference_doctype and doc.reference_name:
exclude += [d[0] for d in frappe.db.get_all("Email Unsubscribe", ["email"],
{"reference_doctype": doc.reference_doctype, "reference_name": doc.reference_name}, as_list=True)]
cc = filter_email_list(doc, cc, exclude, is_cc=True)
return cc
def get_bcc(doc, recipients=None, fetched_from_email_account=False):
"""Build a list of email addresses for BCC"""
bcc = split_emails(doc.bcc)
if bcc:
exclude = []
exclude += [d[0] for d in frappe.db.get_all("User", ["email"], {"thread_notify": 0}, as_list=True)]
exclude += [(parse_addr(email)[1] or "").lower() for email in recipients]
if fetched_from_email_account:
# exclude sender when pulling email
exclude += [parse_addr(doc.sender)[1]]
if doc.reference_doctype and doc.reference_name:
exclude += [d[0] for d in frappe.db.get_all("Email Unsubscribe", ["email"],
{"reference_doctype": doc.reference_doctype, "reference_name": doc.reference_name}, as_list=True)]
bcc = filter_email_list(doc, bcc, exclude, is_bcc=True)
return bcc
def add_attachments(name, attachments):
'''Add attachments to the given Communication'''
# loop through attachments
for a in attachments:
if isinstance(a, string_types):
if isinstance(a, str):
attach = frappe.db.get_value("File", {"name":a},
["file_name", "file_url", "is_private"], as_dict=1)
# save attachments to new doc
_file = frappe.get_doc({
"doctype": "File",
@ -370,103 +146,6 @@ def add_attachments(name, attachments):
})
_file.save(ignore_permissions=True)
def filter_email_list(doc, email_list, exclude, is_cc=False, is_bcc=False):
# temp variables
filtered = []
email_address_list = []
for email in list(set(email_list)):
email_address = (parse_addr(email)[1] or "").lower()
if not email_address:
continue
# this will be used to eventually find email addresses that aren't sent to
doc.all_email_addresses.append(email_address)
if (email in exclude) or (email_address in exclude):
continue
if is_cc:
is_user_enabled = frappe.db.get_value("User", email_address, "enabled")
if is_user_enabled==0:
# don't send to disabled users
continue
if is_bcc:
is_user_enabled = frappe.db.get_value("User", email_address, "enabled")
if is_user_enabled==0:
continue
# make sure of case-insensitive uniqueness of email address
if email_address not in email_address_list:
# append the full email i.e. "Human <human@example.com>"
filtered.append(email)
email_address_list.append(email_address)
doc.sent_email_addresses.extend(email_address_list)
return filtered
def get_owner_email(doc):
owner = get_parent_doc(doc).owner
return get_formatted_email(owner) or owner
def get_assignees(doc):
return [( get_formatted_email(d.owner) or d.owner ) for d in
frappe.db.get_all("ToDo", filters={
"reference_type": doc.reference_doctype,
"reference_name": doc.reference_name,
"status": "Open"
}, fields=["owner"])
]
def get_attach_link(doc, print_format):
"""Returns public link for the attachment via `templates/emails/print_link.html`."""
return frappe.get_template("templates/emails/print_link.html").render({
"url": get_url(),
"doctype": doc.reference_doctype,
"name": doc.reference_name,
"print_format": print_format,
"key": get_parent_doc(doc).get_signature()
})
def sendmail(communication_name, print_html=None, print_format=None, attachments=None,
recipients=None, cc=None, bcc=None, lang=None, session=None, print_letterhead=None):
try:
if lang:
frappe.local.lang = lang
if session:
# hack to enable access to private files in PDF
session['data'] = frappe._dict(session['data'])
frappe.local.session.update(session)
if print_letterhead:
frappe.flags.print_letterhead = print_letterhead
# upto 3 retries
for i in range(3):
try:
communication = frappe.get_doc("Communication", communication_name)
communication._notify(print_html=print_html, print_format=print_format, attachments=attachments,
recipients=recipients, cc=cc, bcc=bcc)
except frappe.db.InternalError as e:
# deadlock, try again
if frappe.db.is_deadlocked(e):
frappe.db.rollback()
time.sleep(1)
continue
else:
raise
else:
break
except:
traceback = frappe.log_error("frappe.core.doctype.communication.email.sendmail")
raise
@frappe.whitelist(allow_guest=True)
def mark_email_as_seen(name=None):
try:

306
frappe/core/doctype/communication/mixins.py Normal file
View file

@ -0,0 +1,306 @@
import frappe
from frappe import _
from frappe.core.utils import get_parent_doc
from frappe.utils import parse_addr, get_formatted_email, get_url
from frappe.email.doctype.email_account.email_account import EmailAccount
from frappe.desk.doctype.todo.todo import ToDo
class CommunicationEmailMixin:
"""Mixin class to handle communication mails.
"""
def is_email_communication(self):
return self.communication_type=="Communication" and self.communication_medium == "Email"
def get_owner(self):
"""Get owner of the communication docs parent.
"""
parent_doc = get_parent_doc(self)
return parent_doc.owner if parent_doc else None
def get_all_email_addresses(self, exclude_displayname=False):
"""Get all Email addresses mentioned in the doc along with display name.
"""
return self.to_list(exclude_displayname=exclude_displayname) + \
self.cc_list(exclude_displayname=exclude_displayname) + \
self.bcc_list(exclude_displayname=exclude_displayname)
def get_email_with_displayname(self, email_address):
"""Returns email address after adding displayname.
"""
display_name, email = parse_addr(email_address)
if display_name and display_name != email:
return email_address
# emailid to emailid with display name map.
email_map = {parse_addr(email)[1]: email for email in self.get_all_email_addresses()}
return email_map.get(email, email)
def mail_recipients(self, is_inbound_mail_communcation=False):
"""Build to(recipient) list to send an email.
"""
# Incase of inbound mail, recipients already received the mail, no need to send again.
if is_inbound_mail_communcation:
return []
if hasattr(self, '_final_recipients'):
return self._final_recipients
to = self.to_list()
self._final_recipients = list(filter(lambda id: id != 'Administrator', to))
return self._final_recipients
def get_mail_recipients_with_displayname(self, is_inbound_mail_communcation=False):
"""Build to(recipient) list to send an email including displayname in email.
"""
to_list = self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation)
return [self.get_email_with_displayname(email) for email in to_list]
def mail_cc(self, is_inbound_mail_communcation=False, include_sender = False):
"""Build cc list to send an email.
* if email copy is requested by sender, then add sender to CC.
* If this doc is created through inbound mail, then add doc owner to cc list
* remove all the thread_notify disabled users.
* Make sure that all users enabled in the system
* Remove admin from email list
* FixMe: Removed adding TODO owners to cc list. Check if that is needed.
"""
if hasattr(self, '_final_cc'):
return self._final_cc
cc = self.cc_list()
# Need to inform parent document owner incase communication is created through inbound mail
if include_sender:
cc.append(self.sender_mailid)
if is_inbound_mail_communcation:
cc.append(self.get_owner())
cc = set(cc) - {self.sender_mailid}
cc.update(self.get_assignees())
cc = set(cc) - set(self.filter_thread_notification_disbled_users(cc))
cc = cc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation))
cc = cc - set(self.filter_disabled_users(cc))
# # Incase of inbound mail, to and cc already received the mail, no need to send again.
if is_inbound_mail_communcation:
cc = cc - set(self.cc_list() + self.to_list())
self._final_cc = list(filter(lambda id: id != 'Administrator', cc))
return self._final_cc
def get_mail_cc_with_displayname(self, is_inbound_mail_communcation=False, include_sender = False):
cc_list = self.mail_cc(is_inbound_mail_communcation=is_inbound_mail_communcation, include_sender = include_sender)
return [self.get_email_with_displayname(email) for email in cc_list]
def mail_bcc(self, is_inbound_mail_communcation=False):
"""
* Thread_notify check
* Email unsubscribe list
* User must be enabled in the system
* remove_administrator_from_email_list
"""
if hasattr(self, '_final_bcc'):
return self._final_bcc
bcc = set(self.bcc_list())
if is_inbound_mail_communcation:
bcc = bcc - {self.sender_mailid}
bcc = bcc - set(self.filter_thread_notification_disbled_users(bcc))
bcc = bcc - set(self.mail_recipients(is_inbound_mail_communcation=is_inbound_mail_communcation))
bcc = bcc - set(self.filter_disabled_users(bcc))
# Incase of inbound mail, to and cc & bcc already received the mail, no need to send again.
if is_inbound_mail_communcation:
bcc = bcc - set(self.bcc_list() + self.to_list())
self._final_bcc = list(filter(lambda id: id != 'Administrator', bcc))
return self._final_bcc
def get_mail_bcc_with_displayname(self, is_inbound_mail_communcation=False):
bcc_list = self.mail_bcc(is_inbound_mail_communcation=is_inbound_mail_communcation)
return [self.get_email_with_displayname(email) for email in bcc_list]
def mail_sender(self):
email_account = self.get_outgoing_email_account()
if not self.sender_mailid and email_account:
return email_account.email_id
return self.sender_mailid
def mail_sender_fullname(self):
email_account = self.get_outgoing_email_account()
if not self.sender_full_name:
return (email_account and email_account.name) or _("Notification")
return self.sender_full_name
def get_mail_sender_with_displayname(self):
return get_formatted_email(self.mail_sender_fullname(), mail=self.mail_sender())
def get_content(self, print_format=None):
if print_format:
return self.content + self.get_attach_link(print_format)
return self.content
def get_attach_link(self, print_format):
"""Returns public link for the attachment via `templates/emails/print_link.html`."""
return frappe.get_template("templates/emails/print_link.html").render({
"url": get_url(),
"doctype": self.reference_doctype,
"name": self.reference_name,
"print_format": print_format,
"key": get_parent_doc(self).get_signature()
})
def get_outgoing_email_account(self):
if not hasattr(self, '_outgoing_email_account'):
if self.email_account:
self._outgoing_email_account = EmailAccount.find(self.email_account)
else:
self._outgoing_email_account = EmailAccount.find_outgoing(
match_by_email=self.sender_mailid,
match_by_doctype=self.reference_doctype
)
if self.sent_or_received == "Sent" and self._outgoing_email_account:
self.db_set("email_account", self._outgoing_email_account.name)
return self._outgoing_email_account
def get_incoming_email_account(self):
if not hasattr(self, '_incoming_email_account'):
self._incoming_email_account = EmailAccount.find_incoming(
match_by_email=self.sender_mailid,
match_by_doctype=self.reference_doctype
)
return self._incoming_email_account
def mail_attachments(self, print_format=None, print_html=None):
final_attachments = []
if print_format or print_html:
d = {'print_format': print_format, 'html': print_html, 'print_format_attachment': 1,
'doctype': self.reference_doctype, 'name': self.reference_name}
final_attachments.append(d)
for a in self.get_attachments() or []:
final_attachments.append({"fid": a['name']})
return final_attachments
def get_unsubscribe_message(self):
email_account = self.get_outgoing_email_account()
if email_account and email_account.send_unsubscribe_message:
return _("Leave this conversation")
return ''
def exclude_emails_list(self, is_inbound_mail_communcation=False, include_sender=False):
"""List of mail id's excluded while sending mail.
"""
all_ids = self.get_all_email_addresses(exclude_displayname=True)
final_ids = self.mail_recipients(is_inbound_mail_communcation = is_inbound_mail_communcation) + \
self.mail_bcc(is_inbound_mail_communcation = is_inbound_mail_communcation) + \
self.mail_cc(is_inbound_mail_communcation = is_inbound_mail_communcation, include_sender=include_sender)
return set(all_ids) - set(final_ids)
def get_assignees(self):
"""Get owners of the reference document.
"""
filters = {'status': 'Open', 'reference_name': self.reference_name,
'reference_type': self.reference_doctype}
return ToDo.get_owners(filters)
@staticmethod
def filter_thread_notification_disbled_users(emails):
"""Filter users based on notifications for email threads setting is disabled.
"""
if not emails:
return []
disabled_users = frappe.db.sql_list("""
SELECT
email
FROM
`tabUser`
where
email in %(emails)s
and
thread_notify=0
""", {'emails': tuple(emails)})
return disabled_users
@staticmethod
def filter_disabled_users(emails):
"""
"""
if not emails:
return []
disabled_users = frappe.db.sql_list("""
SELECT
email
FROM
`tabUser`
where
email in %(emails)s
and
enabled=0
""", {'emails': tuple(emails)})
return disabled_users
def sendmail_input_dict(self, print_html=None, print_format=None,
send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None):
outgoing_email_account = self.get_outgoing_email_account()
if not outgoing_email_account:
return {}
recipients = self.get_mail_recipients_with_displayname(
is_inbound_mail_communcation=is_inbound_mail_communcation
)
cc = self.get_mail_cc_with_displayname(
is_inbound_mail_communcation=is_inbound_mail_communcation,
include_sender = send_me_a_copy
)
bcc = self.get_mail_bcc_with_displayname(
is_inbound_mail_communcation=is_inbound_mail_communcation
)
if not (recipients or cc):
return {}
final_attachments = self.mail_attachments(print_format=print_format, print_html=print_html)
incoming_email_account = self.get_incoming_email_account()
return {
"recipients": recipients,
"cc": cc,
"bcc": bcc,
"expose_recipients": "header",
"sender": self.get_mail_sender_with_displayname(),
"reply_to": incoming_email_account and incoming_email_account.email_id,
"subject": self.subject,
"content": self.get_content(print_format=print_format),
"reference_doctype": self.reference_doctype,
"reference_name": self.reference_name,
"attachments": final_attachments,
"message_id": self.message_id,
"unsubscribe_message": self.get_unsubscribe_message(),
"delayed": True,
"communication": self.name,
"read_receipt": self.read_receipt,
"is_notification": (self.sent_or_received =="Received" and True) or False,
"print_letterhead": print_letterhead
}
def send_email(self, print_html=None, print_format=None,
send_me_a_copy=None, print_letterhead=None, is_inbound_mail_communcation=None):
input_dict = self.sendmail_input_dict(
print_html=print_html,
print_format=print_format,
send_me_a_copy=send_me_a_copy,
print_letterhead=print_letterhead,
is_inbound_mail_communcation=is_inbound_mail_communcation
)
if input_dict:
frappe.sendmail(**input_dict)

74
frappe/core/doctype/communication/test_communication.py
View file

@ -1,12 +1,12 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# See license.txt
from __future__ import unicode_literals
import unittest
from urllib.parse import quote
import frappe
import unittest
from six.moves.urllib.parse import quote
test_records = frappe.get_test_records('Communication')
from frappe.email.doctype.email_queue.email_queue import EmailQueue
test_records = frappe.get_test_records('Communication')
class TestCommunication(unittest.TestCase):
@ -201,6 +201,70 @@ class TestCommunication(unittest.TestCase):
self.assertIn(("Note", note.name), doc_links)
class TestCommunicationEmailMixin(unittest.TestCase):
def new_communication(self, recipients=None, cc=None, bcc=None):
recipients = ', '.join(recipients or [])
cc = ', '.join(cc or [])
bcc = ', '.join(bcc or [])
comm = frappe.get_doc({
"doctype": "Communication",
"communication_type": "Communication",
"communication_medium": "Email",
"content": "Test content",
"recipients": recipients,
"cc": cc,
"bcc": bcc
}).insert(ignore_permissions=True)
return comm
def new_user(self, email, **user_data):
user_data.setdefault('first_name', 'first_name')
user = frappe.new_doc('User')
user.email = email
user.update(user_data)
user.insert(ignore_permissions=True, ignore_if_duplicate=True)
return user
def test_recipients(self):
to_list = ['to@test.com', 'receiver <to+1@test.com>', 'to@test.com']
comm = self.new_communication(recipients = to_list)
res = comm.get_mail_recipients_with_displayname()
self.assertCountEqual(res, ['to@test.com', 'receiver <to+1@test.com>'])
comm.delete()
def test_cc(self):
to_list = ['to@test.com']
cc_list = ['cc+1@test.com', 'cc <cc+2@test.com>', 'to@test.com']
user = self.new_user(email='cc+1@test.com', thread_notify=0)
comm = self.new_communication(recipients=to_list, cc=cc_list)
res = comm.get_mail_cc_with_displayname()
self.assertCountEqual(res, ['cc <cc+2@test.com>'])
user.delete()
comm.delete()
def test_bcc(self):
bcc_list = ['bcc+1@test.com', 'cc <bcc+2@test.com>', ]
user = self.new_user(email='bcc+2@test.com', enabled=0)
comm = self.new_communication(bcc=bcc_list)
res = comm.get_mail_bcc_with_displayname()
self.assertCountEqual(res, ['bcc+1@test.com'])
user.delete()
comm.delete()
def test_sendmail(self):
to_list = ['to <to@test.com>']
cc_list = ['cc <cc+1@test.com>', 'cc <cc+2@test.com>']
comm = self.new_communication(recipients=to_list, cc=cc_list)
comm.send_email()
doc = EmailQueue.find_one_by_filters(communication=comm.name)
mail_receivers = [each.recipient for each in doc.recipients]
self.assertIsNotNone(doc)
self.assertCountEqual(to_list+cc_list, mail_receivers)
doc.delete()
comm.delete()
def create_email_account():
frappe.delete_doc_if_exists("Email Account", "_Test Comm Account 1")
@ -231,4 +295,4 @@ def create_email_account():
"enable_automatic_linking": 1
}).insert(ignore_permissions=True)
return email_account
return email_account

1
frappe/core/doctype/communication_link/communication_link.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document

1
frappe/core/doctype/custom_docperm/custom_docperm.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document

2
frappe/core/doctype/custom_docperm/test_custom_docperm.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import unittest

1
frappe/core/doctype/custom_role/custom_role.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document

2
frappe/core/doctype/custom_role/test_custom_role.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import unittest

1
frappe/core/doctype/data_export/data_export.py
View file

@ -2,7 +2,6 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
from frappe.model.document import Document
class DataExport(Document):

15
frappe/core/doctype/data_export/exporter.py
View file

@ -1,16 +1,12 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe
from frappe import _
import frappe.permissions
import re, csv, os
from frappe.utils.csvutils import UnicodeWriter
from frappe.utils import cstr, formatdate, format_datetime, parse_json, cint, format_duration
from frappe.core.doctype.data_import_legacy.importer import get_data_keys
from six import string_types
from frappe.core.doctype.access_log.access_log import make_access_log
reflags = {
@ -23,6 +19,15 @@ reflags = {
"D": re.DEBUG
}
def get_data_keys():
return frappe._dict({
"data_separator": _('Start entering data below this line'),
"main_table": _("Table") + ":",
"parent_table": _("Parent Table") + ":",
"columns": _("Column Name") + ":",
"doctype": _("DocType") + ":"
})
@frappe.whitelist()
def export_data(doctype=None, parent_doctype=None, all_doctypes=True, with_data=False,
select_columns=None, file_type='CSV', template=False, filters=None):
@ -57,7 +62,7 @@ class DataExporter:
self.docs_to_export = {}
if self.doctype:
if isinstance(self.doctype, string_types):
if isinstance(self.doctype, str):
self.doctype = [self.doctype]
if len(self.doctype) > 1:

18
frappe/core/doctype/data_import/data_import.py
View file

@ -171,9 +171,6 @@ def import_file(
i.import_data()
##############
def import_doc(path, pre_process=None):
if os.path.isdir(path):
files = [os.path.join(path, f) for f in os.listdir(path)]
@ -192,19 +189,8 @@ def import_doc(path, pre_process=None):
)
frappe.flags.mute_emails = False
frappe.db.commit()
elif f.endswith(".csv"):
validate_csv_import_file(f)
frappe.db.commit()
def validate_csv_import_file(path):
if path.endswith(".csv"):
print()
print("This method is deprecated.")
print('Import CSV files using the command "bench --site sitename data-import"')
print("Or use the method frappe.core.doctype.data_import.data_import.import_file")
print()
raise Exception("Method deprecated")
else:
raise NotImplementedError("Only .json files can be imported")
def export_json(

15
frappe/core/doctype/data_import/importer.py
View file

@ -1,7 +1,6 @@
# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import os
import io
import frappe
@ -450,7 +449,7 @@ class ImportFile:
for row in data_without_first_row:
row_values = row.get_values(parent_column_indexes)
# if the row is blank, it's a child row doc
if all([v in INVALID_VALUES for v in row_values]):
if all(v in INVALID_VALUES for v in row_values):
rows.append(row)
continue
# if we encounter a row which has values in parent columns,
@ -607,7 +606,7 @@ class Row:
if df.fieldtype == "Select":
select_options = get_select_options(df)
if select_options and value not in select_options:
options_string = ", ".join([frappe.bold(d) for d in select_options])
options_string = ", ".join(frappe.bold(d) for d in select_options)
msg = _("Value must be one of {0}").format(options_string)
self.warnings.append(
{"row": self.row_number, "field": df_as_json(df), "message": msg,}
@ -903,7 +902,7 @@ class Column:
if self.df.fieldtype == "Link":
# find all values that dont exist
values = list(set([cstr(v) for v in self.column_values[1:] if v]))
values = list({cstr(v) for v in self.column_values[1:] if v})
exists = [
d.name for d in frappe.db.get_all(self.df.options, filters={"name": ("in", values)})
]
@ -936,11 +935,11 @@ class Column:
elif self.df.fieldtype == "Select":
options = get_select_options(self.df)
if options:
values = list(set([cstr(v) for v in self.column_values[1:] if v]))
invalid = list(set(values) - set(options))
values = {cstr(v) for v in self.column_values[1:] if v}
invalid = values - set(options)
if invalid:
valid_values = ", ".join([frappe.bold(o) for o in options])
invalid_values = ", ".join([frappe.bold(i) for i in invalid])
valid_values = ", ".join(frappe.bold(o) for o in options)
invalid_values = ", ".join(frappe.bold(i) for i in invalid)
self.warnings.append(
{
"col": self.column_number,

2
frappe/core/doctype/data_import/test_data_import.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2020, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
# import frappe
import unittest

2
frappe/core/doctype/data_import/test_exporter.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import unittest
import frappe
from frappe.core.doctype.data_import.exporter import Exporter

6
frappe/core/doctype/data_import/test_importer.py
View file

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
import unittest
import frappe
from frappe.core.doctype.data_import.importer import Importer
@ -64,9 +62,9 @@ class TestImporter(unittest.TestCase):
data_import.reload()
import_log = frappe.parse_json(data_import.import_log)
self.assertEqual(import_log[0]['row_indexes'], [2,3])
expected_error = "Error: <b>Child 1 of DocType for Import</b> Row #1: Value missing for: Child Title"
expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #1: Value missing for: Child Title"
self.assertEqual(frappe.parse_json(import_log[0]['messages'][0])['message'], expected_error)
expected_error = "Error: <b>Child 1 of DocType for Import</b> Row #2: Value missing for: Child Title"
expected_error = "Error: <strong>Child 1 of DocType for Import</strong> Row #2: Value missing for: Child Title"
self.assertEqual(frappe.parse_json(import_log[0]['messages'][1])['message'], expected_error)
self.assertEqual(import_log[1]['row_indexes'], [4])

324
frappe/core/doctype/data_import_legacy/data_import_legacy.js
View file

@ -1,324 +0,0 @@
// Copyright (c) 2017, Frappe Technologies and contributors
// For license information, please see license.txt
frappe.ui.form.on('Data Import Legacy', {
onload: function(frm) {
if (frm.doc.__islocal) {
frm.set_value("action", "");
}
frappe.call({
method: "frappe.core.doctype.data_import_legacy.data_import_legacy.get_importable_doctypes",
callback: function (r) {
let importable_doctypes = r.message;
frm.set_query("reference_doctype", function () {
return {
"filters": {
"issingle": 0,
"istable": 0,
"name": ['in', importable_doctypes]
}
};
});
}
}),
// should never check public
frm.fields_dict["import_file"].df.is_private = 1;
frappe.realtime.on("data_import_progress", function(data) {
if (data.data_import === frm.doc.name) {
if (data.reload && data.reload === true) {
frm.reload_doc();
}
if (data.progress) {
let progress_bar = $(frm.dashboard.progress_area.body).find(".progress-bar");
if (progress_bar) {
$(progress_bar).removeClass("progress-bar-danger").addClass("progress-bar-success progress-bar-striped");
$(progress_bar).css("width", data.progress + "%");
}
}
}
});
},
reference_doctype: function(frm){
if (frm.doc.reference_doctype) {
frappe.model.with_doctype(frm.doc.reference_doctype);
}
},
refresh: function(frm) {
frm.disable_save();
frm.dashboard.clear_headline();
if (frm.doc.reference_doctype && !frm.doc.import_file) {
frm.page.set_indicator(__('Attach file'), 'orange');
} else {
if (frm.doc.import_status) {
const listview_settings = frappe.listview_settings['Data Import Legacy'];
const indicator = listview_settings.get_indicator(frm.doc);
frm.page.set_indicator(indicator[0], indicator[1]);
if (frm.doc.import_status === "In Progress") {
frm.dashboard.add_progress("Data Import Progress", "0");
frm.set_read_only();
frm.refresh_fields();
}
}
}
if (frm.doc.reference_doctype) {
frappe.model.with_doctype(frm.doc.reference_doctype);
}
if(frm.doc.action == "Insert new records" || frm.doc.action == "Update records") {
frm.set_df_property("action", "read_only", 1);
}
frm.add_custom_button(__("Help"), function() {
frappe.help.show_video("6wiriRKPhmg");
});
if (frm.doc.reference_doctype && frm.doc.docstatus === 0) {
frm.add_custom_button(__("Download template"), function() {
frappe.data_import.download_dialog(frm).show();
});
}
if (frm.doc.reference_doctype && frm.doc.import_file && frm.doc.total_rows &&
frm.doc.docstatus === 0 && (!frm.doc.import_status || frm.doc.import_status == "Failed")) {
frm.page.set_primary_action(__("Start Import"), function() {
frappe.call({
btn: frm.page.btn_primary,
method: "frappe.core.doctype.data_import_legacy.data_import_legacy.import_data",
args: {
data_import: frm.doc.name
}
});
}).addClass('btn btn-primary');
}
if (frm.doc.log_details) {
frm.events.create_log_table(frm);
} else {
$(frm.fields_dict.import_log.wrapper).empty();
}
},
action: function(frm) {
if(!frm.doc.action) return;
if(!frm.doc.reference_doctype) {
frappe.msgprint(__("Please select document type first."));
frm.set_value("action", "");
return;
}
if(frm.doc.action == "Insert new records") {
frm.doc.insert_new = 1;
} else if (frm.doc.action == "Update records"){
frm.doc.overwrite = 1;
}
frm.save();
},
only_update: function(frm) {
frm.save();
},
submit_after_import: function(frm) {
frm.save();
},
skip_errors: function(frm) {
frm.save();
},
ignore_encoding_errors: function(frm) {
frm.save();
},
no_email: function(frm) {
frm.save();
},
show_only_errors: function(frm) {
frm.events.create_log_table(frm);
},
create_log_table: function(frm) {
let msg = JSON.parse(frm.doc.log_details);
var $log_wrapper = $(frm.fields_dict.import_log.wrapper).empty();
$(frappe.render_template("log_details", {
data: msg.messages,
import_status: frm.doc.import_status,
show_only_errors: frm.doc.show_only_errors,
})).appendTo($log_wrapper);
}
});
frappe.provide('frappe.data_import');
frappe.data_import.download_dialog = function(frm) {
var dialog;
const filter_fields = df => frappe.model.is_value_type(df) && !df.hidden;
const get_fields = dt => frappe.meta.get_docfields(dt).filter(filter_fields);
const get_doctype_checkbox_fields = () => {
return dialog.fields.filter(df => df.fieldname.endsWith('_fields'))
.map(df => dialog.fields_dict[df.fieldname]);
};
const doctype_fields = get_fields(frm.doc.reference_doctype)
.map(df => {
let reqd = (df.reqd || df.fieldname == 'naming_series') ? 1 : 0;
return {
label: df.label,
reqd: reqd,
danger: reqd,
value: df.fieldname,
checked: 1
};
});
let fields = [
{
"label": __("Select Columns"),
"fieldname": "select_columns",
"fieldtype": "Select",
"options": "All\nMandatory\nManually",
"reqd": 1,
"onchange": function() {
const fields = get_doctype_checkbox_fields();
fields.map(f => f.toggle(true));
if(this.value == 'Mandatory' || this.value == 'Manually') {
checkbox_toggle(true);
fields.map(multicheck_field => {
multicheck_field.options.map(option => {
if(!option.reqd) return;
$(multicheck_field.$wrapper).find(`:checkbox[data-unit="${option.value}"]`)
.prop('checked', false)
.trigger('click');
});
});
} else if(this.value == 'All'){
$(dialog.body).find(`[data-fieldtype="MultiCheck"] :checkbox`)
.prop('disabled', true);
}
}
},
{
"label": __("File Type"),
"fieldname": "file_type",
"fieldtype": "Select",
"options": "Excel\nCSV",
"default": "Excel"
},
{
"label": __("Download with Data"),
"fieldname": "with_data",
"fieldtype": "Check",
"hidden": !frm.doc.overwrite,
"default": 1
},
{
"label": __("Select All"),
"fieldname": "select_all",
"fieldtype": "Button",
"depends_on": "eval:doc.select_columns=='Manually'",
click: function() {
checkbox_toggle();
}
},
{
"label": __("Unselect All"),
"fieldname": "unselect_all",
"fieldtype": "Button",
"depends_on": "eval:doc.select_columns=='Manually'",
click: function() {
checkbox_toggle(true);
}
},
{
"label": frm.doc.reference_doctype,
"fieldname": "doctype_fields",
"fieldtype": "MultiCheck",
"options": doctype_fields,
"columns": 2,
"hidden": 1
}
];
const child_table_fields = frappe.meta.get_table_fields(frm.doc.reference_doctype)
.map(df => {
return {
"label": df.options,
"fieldname": df.fieldname + '_fields',
"fieldtype": "MultiCheck",
"options": frappe.meta.get_docfields(df.options)
.filter(filter_fields)
.map(df => ({
label: df.label,
reqd: df.reqd ? 1 : 0,
value: df.fieldname,
checked: 1,
danger: df.reqd
})),
"columns": 2,
"hidden": 1
};
});
fields = fields.concat(child_table_fields);
dialog = new frappe.ui.Dialog({
title: __('Download Template'),
fields: fields,
primary_action: function(values) {
var data = values;
if (frm.doc.reference_doctype) {
var export_params = () => {
let columns = {};
if(values.select_columns) {
columns = get_doctype_checkbox_fields().reduce((columns, field) => {
const options = field.get_checked_options();
columns[field.df.label] = options;
return columns;
}, {});
}
return {
doctype: frm.doc.reference_doctype,
parent_doctype: frm.doc.reference_doctype,
select_columns: JSON.stringify(columns),
with_data: frm.doc.overwrite && data.with_data,
all_doctypes: true,
file_type: data.file_type,
template: true
};
};
let get_template_url = '/api/method/frappe.core.doctype.data_export.exporter.export_data';
open_url_post(get_template_url, export_params());
} else {
frappe.msgprint(__("Please select the Document Type."));
}
dialog.hide();
},
primary_action_label: __('Download')
});
$(dialog.body).find('div[data-fieldname="select_all"], div[data-fieldname="unselect_all"]')
.wrapAll('<div class="inline-buttons" />');
const button_container = $(dialog.body).find('.inline-buttons');
button_container.addClass('flex');
$(button_container).find('.frappe-control').map((index, button) => {
$(button).css({"margin-right": "1em"});
});
function checkbox_toggle(checked=false) {
$(dialog.body).find('[data-fieldtype="MultiCheck"]').map((index, element) => {
$(element).find(`:checkbox`).prop("checked", checked).trigger('click');
});
}
return dialog;
};

218
frappe/core/doctype/data_import_legacy/data_import_legacy.json
View file

@ -1,218 +0,0 @@
{
"actions": [],
"allow_copy": 1,
"creation": "2020-06-11 16:13:23.813709",
"doctype": "DocType",
"document_type": "Document",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"reference_doctype",
"action",
"insert_new",
"overwrite",
"only_update",
"section_break_4",
"import_file",
"column_break_4",
"error_file",
"section_break_6",
"skip_errors",
"submit_after_import",
"ignore_encoding_errors",
"no_email",
"import_detail",
"import_status",
"show_only_errors",
"import_log",
"log_details",
"amended_from",
"total_rows",
"amended_from"
],
"fields": [
{
"fieldname": "reference_doctype",
"fieldtype": "Link",
"ignore_user_permissions": 1,
"in_list_view": 1,
"label": "Document Type",
"options": "DocType",
"reqd": 1
},
{
"fieldname": "action",
"fieldtype": "Select",
"label": "Action",
"options": "Insert new records\nUpdate records",
"reqd": 1
},
{
"default": "0",
"depends_on": "eval:!doc.overwrite",
"description": "New data will be inserted.",
"fieldname": "insert_new",
"fieldtype": "Check",
"hidden": 1,
"label": "Insert new records",
"set_only_once": 1
},
{
"default": "0",
"depends_on": "eval:!doc.insert_new",
"description": "If you are updating/overwriting already created records.",
"fieldname": "overwrite",
"fieldtype": "Check",
"hidden": 1,
"label": "Update records",
"set_only_once": 1
},
{
"default": "0",
"depends_on": "overwrite",
"description": "If you don't want to create any new records while updating the older records.",
"fieldname": "only_update",
"fieldtype": "Check",
"label": "Don't create new records"
},
{
"depends_on": "eval:(!doc.__islocal)",
"fieldname": "section_break_4",
"fieldtype": "Section Break"
},
{
"fieldname": "import_file",
"fieldtype": "Attach",
"label": "Attach file for Import"
},
{
"fieldname": "column_break_4",
"fieldtype": "Column Break"
},
{
"depends_on": "eval: doc.import_status == \"Partially Successful\"",
"description": "This is the template file generated with only the rows having some error. You should use this file for correction and import.",
"fieldname": "error_file",
"fieldtype": "Attach",
"label": "Generated File"
},
{
"depends_on": "eval:(!doc.__islocal)",
"fieldname": "section_break_6",
"fieldtype": "Section Break"
},
{
"default": "0",
"description": "If this is checked, rows with valid data will be imported and invalid rows will be dumped into a new file for you to import later.",
"fieldname": "skip_errors",
"fieldtype": "Check",
"label": "Skip rows with errors"
},
{
"default": "0",
"fieldname": "submit_after_import",
"fieldtype": "Check",
"label": "Submit after importing"
},
{
"default": "0",
"fieldname": "ignore_encoding_errors",
"fieldtype": "Check",
"label": "Ignore encoding errors"
},
{
"default": "1",
"fieldname": "no_email",
"fieldtype": "Check",
"label": "Do not send Emails"
},
{
"collapsible": 1,
"collapsible_depends_on": "eval: doc.import_status == \"Failed\"",
"depends_on": "import_status",
"fieldname": "import_detail",
"fieldtype": "Section Break",
"label": "Import Log"
},
{
"fieldname": "import_status",
"fieldtype": "Select",
"label": "Import Status",
"options": "\nSuccessful\nFailed\nIn Progress\nPartially Successful",
"read_only": 1
},
{
"allow_on_submit": 1,
"default": "1",
"fieldname": "show_only_errors",
"fieldtype": "Check",
"label": "Show only errors",
"no_copy": 1,
"print_hide": 1
},
{
"allow_on_submit": 1,
"depends_on": "import_status",
"fieldname": "import_log",
"fieldtype": "HTML",
"label": "Import Log"
},
{
"allow_on_submit": 1,
"fieldname": "log_details",
"fieldtype": "Code",
"hidden": 1,
"label": "Log Details",
"read_only": 1
},
{
"fieldname": "amended_from",
"fieldtype": "Link",
"label": "Amended From",
"no_copy": 1,
"options": "Data Import",
"print_hide": 1,
"read_only": 1
},
{
"fieldname": "total_rows",
"fieldtype": "Int",
"hidden": 1,
"label": "Total Rows",
"read_only": 1
},
{
"fieldname": "amended_from",
"fieldtype": "Link",
"label": "Amended From",
"no_copy": 1,
"options": "Data Import Legacy",
"print_hide": 1,
"read_only": 1
}
],
"is_submittable": 1,
"links": [],
"max_attachments": 1,
"modified": "2020-06-11 16:13:23.813709",
"modified_by": "Administrator",
"module": "Core",
"name": "Data Import Legacy",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"read": 1,
"role": "System Manager",
"share": 1,
"submit": 1,
"write": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1,
"track_seen": 1
}

126
frappe/core/doctype/data_import_legacy/data_import_legacy.py
View file

@ -1,126 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe Technologies and contributors
# For license information, please see license.txt
import os
import frappe
import frappe.modules.import_file
from frappe import _
from frappe.core.doctype.data_import_legacy.importer import upload
from frappe.model.document import Document
from frappe.modules.import_file import import_file_by_path as _import_file_by_path
from frappe.utils.background_jobs import enqueue
from frappe.utils.data import format_datetime
class DataImportLegacy(Document):
def autoname(self):
if not self.name:
self.name = "Import on " + format_datetime(self.creation)
def validate(self):
if not self.import_file:
self.db_set("total_rows", 0)
if self.import_status == "In Progress":
frappe.throw(_("Can't save the form as data import is in progress."))
# validate the template just after the upload
# if there is total_rows in the doc, it means that the template is already validated and error free
if self.import_file and not self.total_rows:
upload(data_import_doc=self, from_data_import="Yes", validate_template=True)
@frappe.whitelist()
def get_importable_doctypes():
return frappe.cache().hget("can_import", frappe.session.user)
@frappe.whitelist()
def import_data(data_import):
frappe.db.set_value("Data Import Legacy", data_import, "import_status", "In Progress", update_modified=False)
frappe.publish_realtime("data_import_progress", {"progress": "0",
"data_import": data_import, "reload": True}, user=frappe.session.user)
from frappe.core.page.background_jobs.background_jobs import get_info
enqueued_jobs = [d.get("job_name") for d in get_info()]
if data_import not in enqueued_jobs:
enqueue(upload, queue='default', timeout=6000, event='data_import', job_name=data_import,
data_import_doc=data_import, from_data_import="Yes", user=frappe.session.user)
def import_doc(path, overwrite=False, ignore_links=False, ignore_insert=False,
insert=False, submit=False, pre_process=None):
if os.path.isdir(path):
files = [os.path.join(path, f) for f in os.listdir(path)]
else:
files = [path]
for f in files:
if f.endswith(".json"):
frappe.flags.mute_emails = True
_import_file_by_path(f, data_import=True, force=True, pre_process=pre_process, reset_permissions=True)
frappe.flags.mute_emails = False
frappe.db.commit()
elif f.endswith(".csv"):
import_file_by_path(f, ignore_links=ignore_links, overwrite=overwrite, submit=submit, pre_process=pre_process)
frappe.db.commit()
def import_file_by_path(path, ignore_links=False, overwrite=False, submit=False, pre_process=None, no_email=True):
from frappe.utils.csvutils import read_csv_content
print("Importing " + path)
with open(path, "r") as infile:
upload(rows=read_csv_content(infile.read()), ignore_links=ignore_links, no_email=no_email, overwrite=overwrite,
submit_after_import=submit, pre_process=pre_process)
def export_json(doctype, path, filters=None, or_filters=None, name=None, order_by="creation asc"):
def post_process(out):
del_keys = ('modified_by', 'creation', 'owner', 'idx')
for doc in out:
for key in del_keys:
if key in doc:
del doc[key]
for k, v in doc.items():
if isinstance(v, list):
for child in v:
for key in del_keys + ('docstatus', 'doctype', 'modified', 'name'):
if key in child:
del child[key]
out = []
if name:
out.append(frappe.get_doc(doctype, name).as_dict())
elif frappe.db.get_value("DocType", doctype, "issingle"):
out.append(frappe.get_doc(doctype).as_dict())
else:
for doc in frappe.get_all(doctype, fields=["name"], filters=filters, or_filters=or_filters, limit_page_length=0, order_by=order_by):
out.append(frappe.get_doc(doctype, doc.name).as_dict())
post_process(out)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
path = os.path.join('..', path)
with open(path, "w") as outfile:
outfile.write(frappe.as_json(out))
def export_csv(doctype, path):
from frappe.core.doctype.data_export.exporter import export_data
with open(path, "wb") as csvfile:
export_data(doctype=doctype, all_doctypes=True, template=True, with_data=True)
csvfile.write(frappe.response.result.encode("utf-8"))
@frappe.whitelist()
def export_fixture(doctype, app):
if frappe.session.user != "Administrator":
raise frappe.PermissionError
if not os.path.exists(frappe.get_app_path(app, "fixtures")):
os.mkdir(frappe.get_app_path(app, "fixtures"))
export_json(doctype, frappe.get_app_path(app, "fixtures", frappe.scrub(doctype) + ".json"), order_by="name asc")

24
frappe/core/doctype/data_import_legacy/data_import_legacy_list.js
View file

@ -1,24 +0,0 @@
frappe.listview_settings['Data Import Legacy'] = {
add_fields: ["import_status"],
has_indicator_for_draft: 1,
get_indicator: function(doc) {
let status = {
'Successful': [__("Success"), "green", "import_status,=,Successful"],
'Partially Successful': [__("Partial Success"), "blue", "import_status,=,Partially Successful"],
'In Progress': [__("In Progress"), "orange", "import_status,=,In Progress"],
'Failed': [__("Failed"), "red", "import_status,=,Failed"],
'Pending': [__("Pending"), "orange", "import_status,=,"]
}
if (doc.import_status) {
return status[doc.import_status];
}
if (doc.docstatus == 0) {
return status['Pending'];
}
return status['Pending'];
}
};

542
frappe/core/doctype/data_import_legacy/importer.py
View file

@ -1,542 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals, print_function
from six.moves import range
import requests
import frappe, json
import frappe.permissions
from frappe import _
from frappe.utils.csvutils import getlink
from frappe.utils.dateutils import parse_date
from frappe.utils import cint, cstr, flt, getdate, get_datetime, get_url, get_absolute_url, duration_to_seconds
from six import string_types
@frappe.whitelist()
def get_data_keys():
return frappe._dict({
"data_separator": _('Start entering data below this line'),
"main_table": _("Table") + ":",
"parent_table": _("Parent Table") + ":",
"columns": _("Column Name") + ":",
"doctype": _("DocType") + ":"
})
@frappe.whitelist()
def upload(rows = None, submit_after_import=None, ignore_encoding_errors=False, no_email=True, overwrite=None,
update_only = None, ignore_links=False, pre_process=None, via_console=False, from_data_import="No",
skip_errors = True, data_import_doc=None, validate_template=False, user=None):
"""upload data"""
# for translations
if user:
frappe.cache().hdel("lang", user)
frappe.set_user_lang(user)
if data_import_doc and isinstance(data_import_doc, string_types):
data_import_doc = frappe.get_doc("Data Import Legacy", data_import_doc)
if data_import_doc and from_data_import == "Yes":
no_email = data_import_doc.no_email
ignore_encoding_errors = data_import_doc.ignore_encoding_errors
update_only = data_import_doc.only_update
submit_after_import = data_import_doc.submit_after_import
overwrite = data_import_doc.overwrite
skip_errors = data_import_doc.skip_errors
else:
# extra input params
params = json.loads(frappe.form_dict.get("params") or '{}')
if params.get("submit_after_import"):
submit_after_import = True
if params.get("ignore_encoding_errors"):
ignore_encoding_errors = True
if not params.get("no_email"):
no_email = False
if params.get('update_only'):
update_only = True
if params.get('from_data_import'):
from_data_import = params.get('from_data_import')
if not params.get('skip_errors'):
skip_errors = params.get('skip_errors')
frappe.flags.in_import = True
frappe.flags.mute_emails = no_email
def get_data_keys_definition():
return get_data_keys()
def bad_template():
frappe.throw(_("Please do not change the rows above {0}").format(get_data_keys_definition().data_separator))
def check_data_length():
if not data:
frappe.throw(_("No data found in the file. Please reattach the new file with data."))
def get_start_row():
for i, row in enumerate(rows):
if row and row[0]==get_data_keys_definition().data_separator:
return i+1
bad_template()
def get_header_row(key):
return get_header_row_and_idx(key)[0]
def get_header_row_and_idx(key):
for i, row in enumerate(header):
if row and row[0]==key:
return row, i
return [], -1
def filter_empty_columns(columns):
empty_cols = list(filter(lambda x: x in ("", None), columns))
if empty_cols:
if columns[-1*len(empty_cols):] == empty_cols:
# filter empty columns if they exist at the end
columns = columns[:-1*len(empty_cols)]
else:
frappe.msgprint(_("Please make sure that there are no empty columns in the file."),
raise_exception=1)
return columns
def make_column_map():
doctype_row, row_idx = get_header_row_and_idx(get_data_keys_definition().doctype)
if row_idx == -1: # old style
return
dt = None
for i, d in enumerate(doctype_row[1:]):
if d not in ("~", "-"):
if d and doctype_row[i] in (None, '' ,'~', '-', _("DocType") + ":"):
dt, parentfield = d, None
# xls format truncates the row, so it may not have more columns
if len(doctype_row) > i+2:
parentfield = doctype_row[i+2]
doctypes.append((dt, parentfield))
column_idx_to_fieldname[(dt, parentfield)] = {}
column_idx_to_fieldtype[(dt, parentfield)] = {}
if dt:
column_idx_to_fieldname[(dt, parentfield)][i+1] = rows[row_idx + 2][i+1]
column_idx_to_fieldtype[(dt, parentfield)][i+1] = rows[row_idx + 4][i+1]
def get_doc(start_idx):
if doctypes:
doc = {}
attachments = []
last_error_row_idx = None
for idx in range(start_idx, len(rows)):
last_error_row_idx = idx # pylint: disable=W0612
if (not doc) or main_doc_empty(rows[idx]):
for dt, parentfield in doctypes:
d = {}
for column_idx in column_idx_to_fieldname[(dt, parentfield)]:
try:
fieldname = column_idx_to_fieldname[(dt, parentfield)][column_idx]
fieldtype = column_idx_to_fieldtype[(dt, parentfield)][column_idx]
if not fieldname or not rows[idx][column_idx]:
continue
d[fieldname] = rows[idx][column_idx]
if fieldtype in ("Int", "Check"):
d[fieldname] = cint(d[fieldname])
elif fieldtype in ("Float", "Currency", "Percent"):
d[fieldname] = flt(d[fieldname])
elif fieldtype == "Date":
if d[fieldname] and isinstance(d[fieldname], string_types):
d[fieldname] = getdate(parse_date(d[fieldname]))
elif fieldtype == "Datetime":
if d[fieldname]:
if " " in d[fieldname]:
_date, _time = d[fieldname].split()
else:
_date, _time = d[fieldname], '00:00:00'
_date = parse_date(d[fieldname])
d[fieldname] = get_datetime(_date + " " + _time)
else:
d[fieldname] = None
elif fieldtype == "Duration":
d[fieldname] = duration_to_seconds(cstr(d[fieldname]))
elif fieldtype in ("Image", "Attach Image", "Attach"):
# added file to attachments list
attachments.append(d[fieldname])
elif fieldtype in ("Link", "Dynamic Link", "Data") and d[fieldname]:
# as fields can be saved in the number format(long type) in data import template
d[fieldname] = cstr(d[fieldname])
except IndexError:
pass
# scrub quotes from name and modified
if d.get("name") and d["name"].startswith('"'):
d["name"] = d["name"][1:-1]
if sum([0 if not val else 1 for val in d.values()]):
d['doctype'] = dt
if dt == doctype:
doc.update(d)
else:
if not overwrite and doc.get("name"):
d['parent'] = doc["name"]
d['parenttype'] = doctype
d['parentfield'] = parentfield
doc.setdefault(d['parentfield'], []).append(d)
else:
break
return doc, attachments, last_error_row_idx
else:
doc = frappe._dict(zip(columns, rows[start_idx][1:]))
doc['doctype'] = doctype
return doc, [], None
# used in testing whether a row is empty or parent row or child row
# checked only 3 first columns since first two columns can be blank for example the case of
# importing the item variant where item code and item name will be blank.
def main_doc_empty(row):
if row:
for i in range(3,0,-1):
if len(row) > i and row[i]:
return False
return True
def validate_naming(doc):
autoname = frappe.get_meta(doctype).autoname
if autoname:
if autoname[0:5] == 'field':
autoname = autoname[6:]
elif autoname == 'naming_series:':
autoname = 'naming_series'
else:
return True
if (autoname not in doc) or (not doc[autoname]):
from frappe.model.base_document import get_controller
if not hasattr(get_controller(doctype), "autoname"):
frappe.throw(_("{0} is a mandatory field").format(autoname))
return True
users = frappe.db.sql_list("select name from tabUser")
def prepare_for_insert(doc):
# don't block data import if user is not set
# migrating from another system
if not doc.owner in users:
doc.owner = frappe.session.user
if not doc.modified_by in users:
doc.modified_by = frappe.session.user
def is_valid_url(url):
is_valid = False
if url.startswith("/files") or url.startswith("/private/files"):
url = get_url(url)
try:
r = requests.get(url)
is_valid = True if r.status_code == 200 else False
except Exception:
pass
return is_valid
def attach_file_to_doc(doctype, docname, file_url):
# check if attachment is already available
# check if the attachement link is relative or not
if not file_url:
return
if not is_valid_url(file_url):
return
files = frappe.db.sql("""Select name from `tabFile` where attached_to_doctype='{doctype}' and
attached_to_name='{docname}' and (file_url='{file_url}' or thumbnail_url='{file_url}')""".format(
doctype=doctype,
docname=docname,
file_url=file_url
))
if files:
# file is already attached
return
_file = frappe.get_doc({
"doctype": "File",
"file_url": file_url,
"attached_to_name": docname,
"attached_to_doctype": doctype,
"attached_to_field": 0,
"folder": "Home/Attachments"})
_file.save()
# header
filename, file_extension = ['','']
if not rows:
_file = frappe.get_doc("File", {"file_url": data_import_doc.import_file})
fcontent = _file.get_content()
filename, file_extension = _file.get_extension()
if file_extension == '.xlsx' and from_data_import == 'Yes':
from frappe.utils.xlsxutils import read_xlsx_file_from_attached_file
rows = read_xlsx_file_from_attached_file(file_url=data_import_doc.import_file)
elif file_extension == '.csv':
from frappe.utils.csvutils import read_csv_content
rows = read_csv_content(fcontent, ignore_encoding_errors)
else:
frappe.throw(_("Unsupported File Format"))
start_row = get_start_row()
header = rows[:start_row]
data = rows[start_row:]
try:
doctype = get_header_row(get_data_keys_definition().main_table)[1]
columns = filter_empty_columns(get_header_row(get_data_keys_definition().columns)[1:])
except:
frappe.throw(_("Cannot change header content"))
doctypes = []
column_idx_to_fieldname = {}
column_idx_to_fieldtype = {}
if skip_errors:
data_rows_with_error = header
if submit_after_import and not cint(frappe.db.get_value("DocType",
doctype, "is_submittable")):
submit_after_import = False
parenttype = get_header_row(get_data_keys_definition().parent_table)
if len(parenttype) > 1:
parenttype = parenttype[1]
# check permissions
if not frappe.permissions.can_import(parenttype or doctype):
frappe.flags.mute_emails = False
return {"messages": [_("Not allowed to Import") + ": " + _(doctype)], "error": True}
# Throw expception in case of the empty data file
check_data_length()
make_column_map()
total = len(data)
if validate_template:
if total:
data_import_doc.total_rows = total
return True
if overwrite==None:
overwrite = params.get('overwrite')
# delete child rows (if parenttype)
parentfield = None
if parenttype:
parentfield = get_parent_field(doctype, parenttype)
if overwrite:
delete_child_rows(data, doctype)
import_log = []
def log(**kwargs):
if via_console:
print((kwargs.get("title") + kwargs.get("message")).encode('utf-8'))
else:
import_log.append(kwargs)
def as_link(doctype, name):
if via_console:
return "{0}: {1}".format(doctype, name)
else:
return getlink(doctype, name)
# publish realtime task update
def publish_progress(achieved, reload=False):
if data_import_doc:
frappe.publish_realtime("data_import_progress", {"progress": str(int(100.0*achieved/total)),
"data_import": data_import_doc.name, "reload": reload}, user=frappe.session.user)
error_flag = rollback_flag = False
batch_size = frappe.conf.data_import_batch_size or 1000
for batch_start in range(0, total, batch_size):
batch = data[batch_start:batch_start + batch_size]
for i, row in enumerate(batch):
# bypass empty rows
if main_doc_empty(row):
continue
row_idx = i + start_row
doc = None
publish_progress(i)
try:
doc, attachments, last_error_row_idx = get_doc(row_idx)
validate_naming(doc)
if pre_process:
pre_process(doc)
original = None
if parentfield:
parent = frappe.get_doc(parenttype, doc["parent"])
doc = parent.append(parentfield, doc)
parent.save()
else:
if overwrite and doc.get("name") and frappe.db.exists(doctype, doc["name"]):
original = frappe.get_doc(doctype, doc["name"])
original_name = original.name
original.update(doc)
# preserve original name for case sensitivity
original.name = original_name
original.flags.ignore_links = ignore_links
original.save()
doc = original
else:
if not update_only:
doc = frappe.get_doc(doc)
prepare_for_insert(doc)
doc.flags.ignore_links = ignore_links
doc.insert()
if attachments:
# check file url and create a File document
for file_url in attachments:
attach_file_to_doc(doc.doctype, doc.name, file_url)
if submit_after_import:
doc.submit()
# log errors
if parentfield:
log(**{"row": doc.idx, "title": 'Inserted row for "%s"' % (as_link(parenttype, doc.parent)),
"link": get_absolute_url(parenttype, doc.parent), "message": 'Document successfully saved', "indicator": "green"})
elif submit_after_import:
log(**{"row": row_idx + 1, "title":'Submitted row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully submitted", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "blue"})
elif original:
log(**{"row": row_idx + 1,"title":'Updated row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully updated", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "green"})
elif not update_only:
log(**{"row": row_idx + 1, "title":'Inserted row for "%s"' % (as_link(doc.doctype, doc.name)),
"message": "Document successfully saved", "link": get_absolute_url(doc.doctype, doc.name), "indicator": "green"})
else:
log(**{"row": row_idx + 1, "title":'Ignored row for %s' % (row[1]), "link": None,
"message": "Document updation ignored", "indicator": "orange"})
except Exception as e:
error_flag = True
# build error message
if frappe.local.message_log:
err_msg = "\n".join(['<p class="border-bottom small">{}</p>'.format(json.loads(msg).get('message')) for msg in frappe.local.message_log])
else:
err_msg = '<p class="border-bottom small">{}</p>'.format(cstr(e))
error_trace = frappe.get_traceback()
if error_trace:
error_log_doc = frappe.log_error(error_trace)
error_link = get_absolute_url("Error Log", error_log_doc.name)
else:
error_link = None
log(**{
"row": row_idx + 1,
"title": 'Error for row %s' % (len(row)>1 and frappe.safe_decode(row[1]) or ""),
"message": err_msg,
"indicator": "red",
"link":error_link
})
# data with error to create a new file
# include the errored data in the last row as last_error_row_idx will not be updated for the last row
if skip_errors:
if last_error_row_idx == len(rows)-1:
last_error_row_idx = len(rows)
data_rows_with_error += rows[row_idx:last_error_row_idx]
else:
rollback_flag = True
finally:
frappe.local.message_log = []
start_row += batch_size
if rollback_flag:
frappe.db.rollback()
else:
frappe.db.commit()
frappe.flags.mute_emails = False
frappe.flags.in_import = False
log_message = {"messages": import_log, "error": error_flag}
if data_import_doc:
data_import_doc.log_details = json.dumps(log_message)
import_status = None
if error_flag and data_import_doc.skip_errors and len(data) != len(data_rows_with_error):
import_status = "Partially Successful"
# write the file with the faulty row
file_name = 'error_' + filename + file_extension
if file_extension == '.xlsx':
from frappe.utils.xlsxutils import make_xlsx
xlsx_file = make_xlsx(data_rows_with_error, "Data Import Template")
file_data = xlsx_file.getvalue()
else:
from frappe.utils.csvutils import to_csv
file_data = to_csv(data_rows_with_error)
_file = frappe.get_doc({
"doctype": "File",
"file_name": file_name,
"attached_to_doctype": "Data Import Legacy",
"attached_to_name": data_import_doc.name,
"folder": "Home/Attachments",
"content": file_data})
_file.save()
data_import_doc.error_file = _file.file_url
elif error_flag:
import_status = "Failed"
else:
import_status = "Successful"
data_import_doc.import_status = import_status
data_import_doc.save()
if data_import_doc.import_status in ["Successful", "Partially Successful"]:
data_import_doc.submit()
publish_progress(100, True)
else:
publish_progress(0, True)
frappe.db.commit()
else:
return log_message
def get_parent_field(doctype, parenttype):
parentfield = None
# get parentfield
if parenttype:
for d in frappe.get_meta(parenttype).get_table_fields():
if d.options==doctype:
parentfield = d.fieldname
break
if not parentfield:
frappe.msgprint(_("Did not find {0} for {0} ({1})").format("parentfield", parenttype, doctype))
raise Exception
return parentfield
def delete_child_rows(rows, doctype):
"""delete child rows for all parents"""
for p in list(set([r[1] for r in rows])):
if p:
frappe.db.sql("""delete from `tab{0}` where parent=%s""".format(doctype), p)

38
frappe/core/doctype/data_import_legacy/log_details.html
View file

@ -1,38 +0,0 @@
<div>
<div class="table-responsive">
<table class="table table-bordered table-hover log-details-table">
<tr>
<th style="width:10%"> {{ __("Row No") }} </th>
<th style="width:40%"> {{ __("Row Status") }} </th>
<th style="width:50%"> {{ __("Message") }} </th>
</tr>
{% for row in data %}
{% if (!show_only_errors) || (show_only_errors && row.indicator == "red") %}
<tr>
<td>
<span>{{ row.row }} </span>
</td>
<td>
<span class="indicator {{ row.indicator }}"> {{ row.title }} </span>
</td>
<td>
{% if (import_status != "Failed" || (row.indicator == "red")) { %}
<div>{{ row.message }}</div>
{% if row.link %}
<span style="width: 10%; float:right;">
<a class="btn-open no-decoration" title="Open Link" href="{{ row.link }}">
<i class="octicon octicon-arrow-right"></i>
</a>
</span>
{% endif %}
{% } else { %}
<span> {{ __("Document can't saved.") }} </span>
{% } %}
</td>
</tr>
{% endif %}
{% endfor %}
</table>
</div>
</div>

10
frappe/core/doctype/data_import_legacy/test_data_import_legacy.py
View file

@ -1,10 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2020, Frappe Technologies and Contributors
# See license.txt
from __future__ import unicode_literals
# import frappe
import unittest
class TestDataImportLegacy(unittest.TestCase):
pass

1
frappe/core/doctype/defaultvalue/__init__.py
View file

@ -1,4 +1,3 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

1
frappe/core/doctype/defaultvalue/defaultvalue.py
View file

@ -1,7 +1,6 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document

Some files were not shown because too many files have changed in this diff Show more