From acc7de7335e28b13c5590b95a42ed972198de83b Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 17 Apr 2020 18:49:04 +0530 Subject: [PATCH 01/83] feat: added pandas date_range to utils --- frappe/utils/dateutils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frappe/utils/dateutils.py b/frappe/utils/dateutils.py index d5b7a3136b..907cab3d02 100644 --- a/frappe/utils/dateutils.py +++ b/frappe/utils/dateutils.py @@ -73,3 +73,9 @@ def datetime_in_user_format(date_time): date_time = get_datetime(date_time) from frappe.utils import formatdate return formatdate(date_time.date()) + " " + date_time.strftime("%H:%M") + +def get_date_range(from_date, to_date, frequency="M", normalize=True): + from pandas import date_range + date_range = date_range(start=from_date, end=to_date, freq=frequency, normalize=normalize).to_pydatetime().tolist() + + return date_range \ No newline at end of file From da6024242769e8c0934f8bc283ae8c9d93fbc573 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 17 Apr 2020 18:49:12 +0530 Subject: [PATCH 02/83] feat: added website analytics report --- frappe/website/report/__init__.py | 0 .../report/website_analytics/__init__.py | 0 .../website_analytics/website_analytics.js | 34 ++++ .../website_analytics/website_analytics.json | 27 ++++ .../website_analytics/website_analytics.py | 145 ++++++++++++++++++ 5 files changed, 206 insertions(+) create mode 100644 frappe/website/report/__init__.py create mode 100644 frappe/website/report/website_analytics/__init__.py create mode 100644 frappe/website/report/website_analytics/website_analytics.js create mode 100644 frappe/website/report/website_analytics/website_analytics.json create mode 100644 frappe/website/report/website_analytics/website_analytics.py diff --git a/frappe/website/report/__init__.py b/frappe/website/report/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/website/report/website_analytics/__init__.py b/frappe/website/report/website_analytics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/website/report/website_analytics/website_analytics.js b/frappe/website/report/website_analytics/website_analytics.js new file mode 100644 index 0000000000..b607a16eb4 --- /dev/null +++ b/frappe/website/report/website_analytics/website_analytics.js @@ -0,0 +1,34 @@ +// Copyright (c) 2016, Frappe Technologies and contributors +// For license information, please see license.txt +/* eslint-disable */ + +frappe.query_reports["Website Analytics"] = { + "filters": [ + { + fieldname: "from_date", + label: __("From Date"), + fieldtype: "Date", + default: frappe.datetime.add_days(frappe.datetime.now_date(true), -7), + reqd: 1 + }, + { + fieldname:"to_date", + label: __("To Date"), + fieldtype: "Date", + default: frappe.datetime.now_date(true), + reqd: 1 + }, + { + fieldname: "range", + label: __("Range"), + fieldtype: "Select", + options: [ + { "value": "D", "label": __("Daily") }, + { "value": "W", "label": __("Weekly") }, + { "value": "M", "label": __("Monthly") }, + ], + default: "D", + reqd: 1 + } + ] +}; diff --git a/frappe/website/report/website_analytics/website_analytics.json b/frappe/website/report/website_analytics/website_analytics.json new file mode 100644 index 0000000000..62c5751a5c --- /dev/null +++ b/frappe/website/report/website_analytics/website_analytics.json @@ -0,0 +1,27 @@ +{ + "add_total_row": 0, + "creation": "2020-04-17 13:04:45.770148", + "disable_prepared_report": 0, + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "idx": 0, + "is_standard": "Yes", + "modified": "2020-04-17 16:10:30.168312", + "modified_by": "Administrator", + "module": "Website", + "name": "Website Analytics", + "owner": "Administrator", + "prepared_report": 0, + "ref_doctype": "Web Page View", + "report_name": "Website Analytics", + "report_type": "Script Report", + "roles": [ + { + "role": "System Manager" + }, + { + "role": "Website Manager" + } + ] +} \ No newline at end of file diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py new file mode 100644 index 0000000000..8b2d5b3806 --- /dev/null +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -0,0 +1,145 @@ +# Copyright (c) 2013, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe.utils.dateutils import get_date_range + +def execute(filters=None): + return WebsiteAnalytics(filters).run() + +class WebsiteAnalytics(object): + def __init__(self, filters=None): + self.filters = frappe._dict(filters or {}) + self.filters.to_date = frappe.utils.add_days(self.filters.to_date, 1) + self.query_filters = {'creation': ['between', [self.filters.from_date, self.filters.to_date]]} + + def run(self): + columns = self.get_columns() + data = self.get_data() + summary = self.get_report_summary() + chart = self.get_chart_data() + return columns, data, None, chart, summary + + def get_columns(self): + return [ + { + "fieldname": "path", + "label": "Page", + "fieldtype": "Data", + "width": 300 + }, + { + "fieldname": "count", + "label": "Page Views", + "fieldtype": "Int", + "width": 150 + } + ] + + def get_data(self): + data = frappe.get_all("Web Page View", fields=['path', 'count(*) as count'], filters=self.query_filters, group_by="path", order_by='count desc') + return data + + def get_chart_data(self): + def _get_field_for_chart(filters_range): + field = 'creation' + date_format = '%Y-%m-%d' + + if filters_range == "W": + field = 'ADDDATE(creation, INTERVAL 1-DAYOFWEEK(creation) DAY)' + + elif filters_range == "M": + date_format = '%Y-%m-01' + + return field, date_format + + field, date_format = _get_field_for_chart(self.filters.range) + + data = frappe.db.sql(""" + SELECT + DATE_FORMAT({0}, %s) as date, + COUNT(*) as count, + count(CASE WHEN is_unique = 1 THEN 1 END) as unique_count + FROM `tabWeb Page View` + WHERE creation BETWEEN %s AND %s + GROUP BY DATE_FORMAT({0}, %s) + ORDER BY creation + """.format(field), (date_format, self.filters.from_date, self.filters.to_date, date_format), as_dict=1, debug=1) + + return self.prepare_chart_data(data) + + def prepare_chart_data(self, data): + date_range = get_date_range(self.filters.from_date, self.filters.to_date, self.filters.range) + if self.filters.range == "M": + date_range = [frappe.utils.add_days(dd, 1) for dd in date_range] + + labels = [] + total_dataset = [] + unique_dataset = [] + + def get_data_for_date(date): + for item in data: + item_date = frappe.utils.get_datetime(item.get("date")).date() + if item_date == date.date(): + return item + return {'count': 0, 'unique_count': 0} + + + for date in date_range: + labels.append(date.strftime("%b %d %Y")) + match = get_data_for_date(date) + total_dataset.append(match.get('count', 0)) + unique_dataset.append(match.get('unique_count', 0)) + + chart = { + "data": { + 'labels': labels, + 'datasets': [ + { + 'name': "Total Views", + 'type': 'line', + 'values': total_dataset + }, + { + 'name': "Unique Visits", + 'type': 'line', + 'values': unique_dataset + } + ] + }, + "type": "axis-mixed", + 'lineOptions': { + 'regionFill': 1, + }, + 'axisOptions': { + 'xIsSeries': 1 + } + } + + return chart + + + def get_report_summary(self): + summary_data = frappe.get_all("Web Page View", fields=['is_unique', 'count(*) as count'], filters=self.query_filters, group_by="is_unique") + + total_count = 0 + unique_count = 0 + for data in summary_data: + if data.get('is_unique'): + unique_count = data.get('count') + total_count += data.get('count') + report_summary = [ + { + "value": total_count, + "label": "Total Page Views", + "datatype": "Int", + }, + { + "value": unique_count, + "label": "Unique Page Views", + "datatype": "Int", + }, + + ] + return report_summary \ No newline at end of file From 9b185802aaab09aa84dff1fce76292ea1edcdc54 Mon Sep 17 00:00:00 2001 From: prssanna Date: Mon, 27 Apr 2020 18:56:12 +0530 Subject: [PATCH 03/83] fix: fix calculation of dates and values for dashboard charts --- .../dashboard_chart/dashboard_chart.py | 108 ++++-------------- .../dashboard_chart/test_dashboard_chart.py | 58 +++++++--- 2 files changed, 64 insertions(+), 102 deletions(-) diff --git a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py index 7bed8f4504..4502748504 100644 --- a/frappe/desk/doctype/dashboard_chart/dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/dashboard_chart.py @@ -128,7 +128,6 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date): to_date = datetime.datetime.now() doctype = chart.document_type - unit_function = get_unit_function(doctype, chart.based_on, timegrain) datefield = chart.based_on aggregate_function = get_aggregate_function(chart.chart_type) value_field = chart.value_based_on or '1' @@ -141,23 +140,18 @@ def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date): data = frappe.db.get_all( doctype, fields = [ - 'extract(year from `tab{doctype}`.{datefield}) as _year'.format(doctype=doctype, datefield=datefield), - '{} as _unit'.format(unit_function), + '{} as _unit'.format(datefield), '{aggregate_function}({value_field})'.format(aggregate_function=aggregate_function, value_field=value_field), ], filters = filters, - group_by = '_year, _unit', - order_by = '_year asc, _unit asc', + group_by = '_unit', + order_by = '_unit asc', as_list = True, ignore_ifnull = True ) + result = get_result(data, timegrain, from_date, to_date) - # result given as year, unit -> convert it to end of period of that unit - result = convert_to_dates(data, timegrain) - - # add missing data points for periods where there was no result - result = add_missing_values(result, timegrain, timespan, from_date, to_date) chart_config = { "labels": [formatdate(r[0].strftime('%Y-%m-%d')) for r in result], "datasets": [{ @@ -217,75 +211,22 @@ def get_aggregate_function(chart_type): }[chart_type] -def convert_to_dates(data, timegrain): - """ Converts individual dates within data to the end of period """ - result = [] - for d in data: - if d[2] != 0: - if timegrain == 'Daily': - result.append([add_to_date('{:d}-01-01'.format(int(d[0])), days = d[1] - 1), d[2]]) - elif timegrain == 'Weekly': - result.append([add_to_date(add_to_date('{:d}-01-01'.format(int(d[0])), weeks = d[1] + 1), days = -1), d[2]]) - elif timegrain == 'Monthly': - result.append([add_to_date(add_to_date('{:d}-01-01'.format(int(d[0])), months=d[1]), days = -1), d[2]]) - elif timegrain == 'Quarterly': - result.append([add_to_date(add_to_date('{:d}-01-01'.format(int(d[0])), months=d[1] * 3), days = -1), d[2]]) - elif timegrain == 'Yearly': - result.append([add_to_date(add_to_date('{:d}-01-01'.format(int(d[0])), months=12), days = -1), d[2]]) - result[-1][0] = getdate(result[-1][0]) - - return result - -def get_unit_function(doctype, datefield, timegrain): - unit_function = '' - if timegrain=='Daily': - if frappe.db.db_type == 'mariadb': - unit_function = 'dayofyear(`tab{doctype}`.{datefield})'.format( - doctype=doctype, datefield=datefield) - else: - unit_function = 'extract(doy from `tab{doctype}`.{datefield})'.format( - doctype=doctype, datefield=datefield) - - else: - unit_function = 'extract({unit} from `tab{doctype}`.{datefield})'.format( - unit = timegrain[:-2].lower(), doctype=doctype, datefield=datefield) - - return unit_function - -def add_missing_values(data, timegrain, timespan, from_date, to_date): - # add missing intervals +def get_result(data, timegrain, from_date, to_date): + start_date = getdate(from_date) + end_date = getdate(to_date) result = [] - if timespan != 'All Time': - first_expected_date = get_period_ending(from_date, timegrain) - # fill out data before the first data point - first_data_point_date = data[0][0] if data else getdate(add_to_date(to_date, days=1)) - while first_data_point_date > first_expected_date: - result.append([first_expected_date, 0.0]) - first_expected_date = get_next_expected_date(first_expected_date, timegrain) + while start_date <= end_date: + next_date = get_next_expected_date(start_date, timegrain) + result.append([next_date, 0.0]) + start_date = next_date - # fill data points and missing points - for i, d in enumerate(data): - result.append(d) - - next_expected_date = get_next_expected_date(d[0], timegrain) - - if i < len(data)-1: - next_date = data[i+1][0] - else: - # already reached at end of data, see if we need any more dates - next_date = getdate(nowdate()) - - # if next data point is earler than the expected date - # need to fill out missing data points - while next_date > next_expected_date: - # fill missing value - result.append([next_expected_date, 0.0]) - next_expected_date = get_next_expected_date(next_expected_date, timegrain) - - # add date for the last period (if missing) - if result and get_period_ending(to_date, timegrain) > result[-1][0]: - result.append([get_period_ending(to_date, timegrain), 0.0]) + data_index = 0 + if data: + for i, d in enumerate(result): + while data_index < len(data) and getdate(data[data_index][0]) <= d[0]: + d[1] += data[data_index][1] + data_index += 1 return result @@ -314,17 +255,12 @@ def get_period_ending(date, timegrain): return getdate(date) def get_week_ending(date): - # fun fact: week ends on the day before 1st Jan of the year. - # for 2019 it is Monday + # week starts on monday + from datetime import timedelta + start = date - timedelta(days = date.weekday()) + end = start + timedelta(days=6) - week_of_the_year = int(date.strftime('%U')) - - if week_of_the_year == 52: - date = add_to_date(date, years=1) - # first day of next week - date = add_to_date('{}-01-01'.format(date.year), weeks = (week_of_the_year%52) + 1) - # last day of this week - return add_to_date(date, days=-1) + return end def get_month_ending(date): month_of_the_year = int(date.strftime('%m')) diff --git a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py index 4425c4fd45..dfc6edbf58 100644 --- a/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py +++ b/frappe/desk/doctype/dashboard_chart/test_dashboard_chart.py @@ -17,10 +17,9 @@ class TestDashboardChart(unittest.TestCase): self.assertEqual(get_period_ending('2019-04-10', 'Daily'), getdate('2019-04-10')) - # fun fact: week ends on the day before 1st Jan of the year. - # for 2019 it is Monday + # week starts on monday self.assertEqual(get_period_ending('2019-04-10', 'Weekly'), - getdate('2019-04-15')) + getdate('2019-04-14')) self.assertEqual(get_period_ending('2019-04-10', 'Monthly'), getdate('2019-04-30')) @@ -133,6 +132,34 @@ class TestDashboardChart(unittest.TestCase): frappe.db.rollback() + def test_weekly_dashboard_chart(self): + insert_test_records() + + if frappe.db.exists('Dashboard Chart', 'Test Weekly Dashboard Chart'): + frappe.delete_doc('Dashboard Chart', 'Test Weekly Dashboard Chart') + + frappe.get_doc(dict( + doctype = 'Dashboard Chart', + chart_name = 'Test Weekly Dashboard Chart', + chart_type = 'Sum', + document_type = 'Communication', + based_on = 'communication_date', + value_based_on = 'rating', + timespan = 'Select Date Range', + time_interval = 'Weekly', + from_date = datetime(2018, 12, 30), + to_date = datetime(2019, 1, 15), + filters_json = '[]', + timeseries = 1 + )).insert() + + result = get(chart_name ='Test Weekly Dashboard Chart', refresh = 1) + + self.assertEqual(result.get('datasets')[0].get('values'), [200.0, 400.0, 0.0]) + self.assertEqual(result.get('labels'), [formatdate('2019-01-06'), formatdate('2019-01-13'), formatdate('2019-01-20')]) + + frappe.db.rollback() + def test_group_by_chart_type(self): if frappe.db.exists('Dashboard Chart', 'Test Group By Dashboard Chart'): frappe.delete_doc('Dashboard Chart', 'Test Group By Dashboard Chart') @@ -155,17 +182,16 @@ class TestDashboardChart(unittest.TestCase): frappe.db.rollback() - def test_dashboard_with_single_doctype(self): - if frappe.db.exists('Dashboard Chart', 'Test Single DocType In Dashboard Chart'): - frappe.delete_doc('Dashboard Chart', 'Test Single DocType In Dashboard Chart') +def insert_test_records(): + create_new_communication(datetime(2019, 1, 10), 100) + create_new_communication(datetime(2019, 1, 6), 200) + create_new_communication(datetime(2019, 1, 8), 300) - chart_doc = frappe.get_doc(dict( - doctype = 'Dashboard Chart', - chart_name = 'Test Single DocType In Dashboard Chart', - chart_type = 'Count', - document_type = 'System Settings', - group_by_based_on = 'Created On', - filters_json = '{}', - )) - - self.assertRaises(frappe.ValidationError, chart_doc.insert) +def create_new_communication(date, rating): + communication = { + 'doctype': 'Communication', + 'subject': 'Test Communication', + 'rating': rating, + 'communication_date': date + } + frappe.get_doc(communication).insert() From 77d2c3d8cfaaa61aa7c8984f6e5ec3640c6ab020 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 5 May 2020 13:43:34 +0530 Subject: [PATCH 04/83] feat: added patch for is unique --- frappe/patches.txt | 1 + .../patches/v13_0/set_unique_for_page_view.py | 6 +++++ .../doctype/web_page_view/web_page_view.json | 24 ++++++++++++++----- 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 frappe/patches/v13_0/set_unique_for_page_view.py diff --git a/frappe/patches.txt b/frappe/patches.txt index a086fa6f4a..3d8b0732d2 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -278,3 +278,4 @@ frappe.patches.v13_0.set_path_for_homepage_in_web_page_view frappe.patches.v13_0.migrate_translation_column_data frappe.patches.v13_0.set_read_times frappe.patches.v13_0.remove_web_view +frappe.patches.v13_0.set_unique_for_page_view \ No newline at end of file diff --git a/frappe/patches/v13_0/set_unique_for_page_view.py b/frappe/patches/v13_0/set_unique_for_page_view.py new file mode 100644 index 0000000000..2a084e52e3 --- /dev/null +++ b/frappe/patches/v13_0/set_unique_for_page_view.py @@ -0,0 +1,6 @@ +import frappe + +def execute(): + frappe.reload_doc('website', 'doctype', 'web_page_view', force=True) + site_url = frappe.utils.get_site_url(frappe.local.site) + frappe.db.sql("""UPDATE `tabWeb Page View` set is_unique=1 where referrer LIKE '%{0}%'""".format(site_url)) diff --git a/frappe/website/doctype/web_page_view/web_page_view.json b/frappe/website/doctype/web_page_view/web_page_view.json index 7a1a210d62..4243df39b1 100644 --- a/frappe/website/doctype/web_page_view/web_page_view.json +++ b/frappe/website/doctype/web_page_view/web_page_view.json @@ -1,5 +1,6 @@ { "actions": [], + "allow_import": 1, "creation": "2020-04-15 22:54:46.009703", "doctype": "DocType", "editable_grid": 1, @@ -9,7 +10,9 @@ "referrer", "browser", "browser_version", - "date" + "is_unique", + "time_zone", + "user_agent" ], "fields": [ { @@ -39,15 +42,24 @@ "set_only_once": 1 }, { - "fieldname": "date", - "fieldtype": "Datetime", - "label": "Date", - "set_only_once": 1 + "fieldname": "is_unique", + "fieldtype": "Data", + "label": "Is Unique" + }, + { + "fieldname": "time_zone", + "fieldtype": "Data", + "label": "Time Zone" + }, + { + "fieldname": "user_agent", + "fieldtype": "Data", + "label": "User Agent" } ], "in_create": 1, "links": [], - "modified": "2020-04-15 23:31:27.517793", + "modified": "2020-05-05 14:11:24.718770", "modified_by": "Administrator", "module": "Website", "name": "Web Page View", From ab9e596f627dc70dab273d1b8485796559ea23a3 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 5 May 2020 15:06:14 +0530 Subject: [PATCH 05/83] feat: do not track 404 --- frappe/www/404.html | 6 ++++-- frappe/www/website_script.js | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frappe/www/404.html b/frappe/www/404.html index 47685c45d0..dc178dbdc8 100644 --- a/frappe/www/404.html +++ b/frappe/www/404.html @@ -15,7 +15,9 @@ html, body { } {% include "templates/styles/card_style.css" %} - +
{{_("Page Missing or Moved")}} @@ -29,4 +31,4 @@ html, body { background-color: #f5f7fa; } -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/frappe/www/website_script.js b/frappe/www/website_script.js index 7fdc2e94d6..e31b6812d5 100644 --- a/frappe/www/website_script.js +++ b/frappe/www/website_script.js @@ -14,7 +14,7 @@ ga('send', 'pageview'); {%- endif %} {% if enable_view_tracking %} - if (navigator.doNotTrack != 1) { + if (navigator.doNotTrack != 1 && !window.is_404) { frappe.ready(() => { let browser = frappe.utils.get_browser(); frappe.call("frappe.website.doctype.web_page_view.web_page_view.make_view_log", { From fb66f85d6364f26b4d1cc4281aed6ac80ee8bdb7 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 5 May 2020 15:06:48 +0530 Subject: [PATCH 06/83] feat: update website analytics report --- .../website_analytics/website_analytics.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index 8b2d5b3806..b821cbff1f 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -17,8 +17,8 @@ class WebsiteAnalytics(object): def run(self): columns = self.get_columns() data = self.get_data() - summary = self.get_report_summary() chart = self.get_chart_data() + summary = self.get_report_summary() return columns, data, None, chart, summary def get_columns(self): @@ -56,7 +56,7 @@ class WebsiteAnalytics(object): field, date_format = _get_field_for_chart(self.filters.range) - data = frappe.db.sql(""" + self.chart_data = frappe.db.sql(""" SELECT DATE_FORMAT({0}, %s) as date, COUNT(*) as count, @@ -65,9 +65,9 @@ class WebsiteAnalytics(object): WHERE creation BETWEEN %s AND %s GROUP BY DATE_FORMAT({0}, %s) ORDER BY creation - """.format(field), (date_format, self.filters.from_date, self.filters.to_date, date_format), as_dict=1, debug=1) + """.format(field), (date_format, self.filters.from_date, self.filters.to_date, date_format), as_dict=1) - return self.prepare_chart_data(data) + return self.prepare_chart_data(self.chart_data) def prepare_chart_data(self, data): date_range = get_date_range(self.filters.from_date, self.filters.to_date, self.filters.range) @@ -121,14 +121,12 @@ class WebsiteAnalytics(object): def get_report_summary(self): - summary_data = frappe.get_all("Web Page View", fields=['is_unique', 'count(*) as count'], filters=self.query_filters, group_by="is_unique") - total_count = 0 unique_count = 0 - for data in summary_data: - if data.get('is_unique'): - unique_count = data.get('count') + for data in self.chart_data: + unique_count += data.get('unique_count') total_count += data.get('count') + report_summary = [ { "value": total_count, From f684756fba34fcd5ac1099ea7938f64fe08e574d Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 5 May 2020 18:22:04 +0530 Subject: [PATCH 07/83] feat: added website analytics report --- .../website_analytics/website_analytics.js | 2 - .../website_analytics/website_analytics.py | 116 +++++++++++++++--- 2 files changed, 98 insertions(+), 20 deletions(-) diff --git a/frappe/website/report/website_analytics/website_analytics.js b/frappe/website/report/website_analytics/website_analytics.js index b607a16eb4..7e051afa8c 100644 --- a/frappe/website/report/website_analytics/website_analytics.js +++ b/frappe/website/report/website_analytics/website_analytics.js @@ -9,14 +9,12 @@ frappe.query_reports["Website Analytics"] = { label: __("From Date"), fieldtype: "Date", default: frappe.datetime.add_days(frappe.datetime.now_date(true), -7), - reqd: 1 }, { fieldname:"to_date", label: __("To Date"), fieldtype: "Date", default: frappe.datetime.now_date(true), - reqd: 1 }, { fieldname: "range", diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index b821cbff1f..694bc9e797 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import frappe +from datetime import datetime from frappe.utils.dateutils import get_date_range def execute(filters=None): @@ -11,6 +12,16 @@ def execute(filters=None): class WebsiteAnalytics(object): def __init__(self, filters=None): self.filters = frappe._dict(filters or {}) + + if not self.filters.to_date: + self.filters.to_date = datetime.now() + + if not self.filters.from_date: + self.filters.from_date = frappe.utils.add_days(self.filters.to_date, -7) + + if not self.filters.range: + self.filters.range = "D" + self.filters.to_date = frappe.utils.add_days(self.filters.to_date, 1) self.query_filters = {'creation': ['between', [self.filters.from_date, self.filters.to_date]]} @@ -19,7 +30,8 @@ class WebsiteAnalytics(object): data = self.get_data() chart = self.get_chart_data() summary = self.get_report_summary() - return columns, data, None, chart, summary + + return columns, data[:250], None, chart, summary def get_columns(self): return [ @@ -34,38 +46,105 @@ class WebsiteAnalytics(object): "label": "Page Views", "fieldtype": "Int", "width": 150 + }, + { + "fieldname": "unique_count", + "label": "Unique Visitors", + "fieldtype": "Int", + "width": 150 } ] def get_data(self): - data = frappe.get_all("Web Page View", fields=['path', 'count(*) as count'], filters=self.query_filters, group_by="path", order_by='count desc') + pg_query = """ + SELECT + path, + COUNT(*) as count, + COUNT(CASE WHEN CAST(is_unique as Integer) = 1 THEN 1 END) as unique_count + FROM `tabWeb Page View` + WHERE coalesce("tabWeb Page View".creation, '0001-01-01') BETWEEN %s AND %s + GROUP BY path + ORDER BY count desc + """ + + mariadb_query = """ + SELECT + path, + COUNT(*) as count, + COUNT(CASE WHEN is_unique = 1 THEN 1 END) as unique_count + FROM `tabWeb Page View` + WHERE creation BETWEEN %s AND %s + GROUP BY path + ORDER BY count desc + """ + + data = frappe.db.multisql({ + "mariadb": mariadb_query, + "postgres": pg_query + }, (self.filters.from_date, self.filters.to_date)) return data - def get_chart_data(self): - def _get_field_for_chart(filters_range): - field = 'creation' - date_format = '%Y-%m-%d' + def _get_query_for_mariadb(self): + filters_range = self.filters.range + field = 'creation' + date_format = '%Y-%m-%d' - if filters_range == "W": - field = 'ADDDATE(creation, INTERVAL 1-DAYOFWEEK(creation) DAY)' + if filters_range == "W": + field = 'ADDDATE(creation, INTERVAL 1-DAYOFWEEK(creation) DAY)' - elif filters_range == "M": - date_format = '%Y-%m-01' + elif filters_range == "M": + date_format = '%Y-%m-01' - return field, date_format - - field, date_format = _get_field_for_chart(self.filters.range) - - self.chart_data = frappe.db.sql(""" + query = """ SELECT DATE_FORMAT({0}, %s) as date, COUNT(*) as count, - count(CASE WHEN is_unique = 1 THEN 1 END) as unique_count + COUNT(CASE WHEN is_unique = 1 THEN 1 END) as unique_count FROM `tabWeb Page View` WHERE creation BETWEEN %s AND %s GROUP BY DATE_FORMAT({0}, %s) ORDER BY creation - """.format(field), (date_format, self.filters.from_date, self.filters.to_date, date_format), as_dict=1) + """.format(field) + + values = (date_format, self.filters.from_date, self.filters.to_date, date_format) + + return query, values + + def _get_query_for_postgres(self): + filters_range = self.filters.range + field = 'creation' + granularity = 'day' + + if filters_range == "W": + granularity = 'week' + + elif filters_range == "M": + granularity = 'day' + + query = """ + SELECT + DATE_TRUNC(%s, {0}) as date, + COUNT(*) as count, + COUNT(CASE WHEN CAST(is_unique as Integer) = 1 THEN 1 END) as unique_count + FROM "tabWeb Page View" + WHERE coalesce("tabWeb Page View".{0}, '0001-01-01') BETWEEN %s AND %s + GROUP BY date_trunc(%s, {0}) + ORDER BY date + """.format(field) + + values = (granularity, self.filters.from_date, self.filters.to_date, granularity) + + return query, values + + def get_chart_data(self): + current_dialect = frappe.db.db_type or 'mariadb' + + if current_dialect == 'mariadb': + query, values = self._get_query_for_mariadb() + else: + query, values = self._get_query_for_postgres() + + self.chart_data = frappe.db.sql(query, values=values, as_dict=1) return self.prepare_chart_data(self.chart_data) @@ -114,7 +193,8 @@ class WebsiteAnalytics(object): }, 'axisOptions': { 'xIsSeries': 1 - } + }, + 'colors': ['#7cd6fd', '#5e64ff'] } return chart From 83380f25475edfb3bec8d79cad7013d78443b9b4 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 5 May 2020 18:22:47 +0530 Subject: [PATCH 08/83] feat: added website analytics chart as JSON fixture --- frappe/model/sync.py | 5 ++-- .../website_analytics/website_analytics.json | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 frappe/website/dashboard_chart/website_analytics/website_analytics.json diff --git a/frappe/model/sync.py b/frappe/model/sync.py index c2acb59f63..3dba994134 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -52,7 +52,8 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe ("desk", "desk_card"), ("desk", "desk_chart"), ("desk", "desk_shortcut"), - ("desk", "desk_page")): + ("desk", "desk_page"), + ("desk", "dashboard_chart")): files.append(os.path.join(frappe.get_app_path("frappe"), d[0], "doctype", d[1], d[1] + ".json")) @@ -82,7 +83,7 @@ def get_doc_files(files, start_path, force=0, sync_everything = False, verbose=F document_types = ['doctype', 'page', 'report', 'dashboard_chart_source', 'print_format', 'website_theme', 'web_form', 'web_template', 'notification', 'print_style', 'data_migration_mapping', 'data_migration_plan', 'desk_page', - 'onboarding_step', 'onboarding'] + 'onboarding_step', 'onboarding', 'dashboard_chart'] for doctype in document_types: doctype_path = os.path.join(start_path, doctype) diff --git a/frappe/website/dashboard_chart/website_analytics/website_analytics.json b/frappe/website/dashboard_chart/website_analytics/website_analytics.json new file mode 100644 index 0000000000..eeeb1a11f9 --- /dev/null +++ b/frappe/website/dashboard_chart/website_analytics/website_analytics.json @@ -0,0 +1,24 @@ +{ + "chart_name": "Website Analytics", + "chart_type": "Report", + "creation": "2020-05-05 18:14:19.369181", + "custom_options": "{\"type\": \"line\", \"lineOptions\": {\"regionFill\": 1}, \"axisOptions\": {\"shortenYAxisNumbers\": 1}, \"colors\": [\"#7cd6fd\", \"#5e64ff\"], \"tooltipOptions\": {}}", + "docstatus": 0, + "doctype": "Dashboard Chart", + "filters_json": "{}", + "group_by_type": "Count", + "idx": 0, + "is_custom": 1, + "is_public": 1, + "modified": "2020-05-05 18:16:47.383649", + "modified_by": "Administrator", + "name": "Website Analytics", + "number_of_groups": 0, + "owner": "Administrator", + "report_name": "Website Analytics", + "time_interval": "Yearly", + "timeseries": 0, + "timespan": "Last Year", + "type": "Line", + "y_axis": [] +} \ No newline at end of file From 06e4e6a3b933b28e29bcd7e01bf1027939a236a1 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 5 May 2020 18:29:42 +0530 Subject: [PATCH 09/83] feat: add analytics chart to desk --- frappe/website/desk_page/website/website.json | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/frappe/website/desk_page/website/website.json b/frappe/website/desk_page/website/website.json index 1c6066d21e..c42a17d404 100644 --- a/frappe/website/desk_page/website/website.json +++ b/frappe/website/desk_page/website/website.json @@ -27,7 +27,11 @@ } ], "category": "Modules", - "charts": [], + "charts": [ + { + "chart_name": "Website Analytics" + } + ], "creation": "2020-03-02 14:13:51.089373", "developer_mode_only": 0, "disable_user_customization": 0, @@ -37,7 +41,7 @@ "idx": 0, "is_standard": 1, "label": "Website", - "modified": "2020-04-26 13:03:49.094728", + "modified": "2020-05-05 18:17:13.232473", "modified_by": "Administrator", "module": "Website", "name": "Website", From 8a918fd54dce8c0ab4ad2326d2f6dc8257412d33 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Thu, 7 May 2020 19:18:08 +0530 Subject: [PATCH 10/83] feat: use new sync API --- frappe/model/sync.py | 5 ++- .../website_analytics/website_analytics.json | 24 ------------- frappe/website/dashboard_fixtures.py | 36 +++++++++++++++++++ 3 files changed, 38 insertions(+), 27 deletions(-) delete mode 100644 frappe/website/dashboard_chart/website_analytics/website_analytics.json create mode 100644 frappe/website/dashboard_fixtures.py diff --git a/frappe/model/sync.py b/frappe/model/sync.py index 7991d11d15..320cc24677 100644 --- a/frappe/model/sync.py +++ b/frappe/model/sync.py @@ -55,8 +55,7 @@ def sync_for(app_name, force=0, sync_everything = False, verbose=False, reset_pe ("desk", "desk_card"), ("desk", "desk_chart"), ("desk", "desk_shortcut"), - ("desk", "desk_page"), - ("desk", "dashboard_chart")): + ("desk", "desk_page")): files.append(os.path.join(frappe.get_app_path("frappe"), d[0], "doctype", d[1], d[1] + ".json")) @@ -86,7 +85,7 @@ def get_doc_files(files, start_path, force=0, sync_everything = False, verbose=F document_types = ['doctype', 'page', 'report', 'dashboard_chart_source', 'print_format', 'website_theme', 'web_form', 'web_template', 'notification', 'print_style', 'data_migration_mapping', 'data_migration_plan', 'desk_page', - 'onboarding_step', 'onboarding', 'dashboard_chart'] + 'onboarding_step', 'onboarding'] for doctype in document_types: doctype_path = os.path.join(start_path, doctype) diff --git a/frappe/website/dashboard_chart/website_analytics/website_analytics.json b/frappe/website/dashboard_chart/website_analytics/website_analytics.json deleted file mode 100644 index eeeb1a11f9..0000000000 --- a/frappe/website/dashboard_chart/website_analytics/website_analytics.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "chart_name": "Website Analytics", - "chart_type": "Report", - "creation": "2020-05-05 18:14:19.369181", - "custom_options": "{\"type\": \"line\", \"lineOptions\": {\"regionFill\": 1}, \"axisOptions\": {\"shortenYAxisNumbers\": 1}, \"colors\": [\"#7cd6fd\", \"#5e64ff\"], \"tooltipOptions\": {}}", - "docstatus": 0, - "doctype": "Dashboard Chart", - "filters_json": "{}", - "group_by_type": "Count", - "idx": 0, - "is_custom": 1, - "is_public": 1, - "modified": "2020-05-05 18:16:47.383649", - "modified_by": "Administrator", - "name": "Website Analytics", - "number_of_groups": 0, - "owner": "Administrator", - "report_name": "Website Analytics", - "time_interval": "Yearly", - "timeseries": 0, - "timespan": "Last Year", - "type": "Line", - "y_axis": [] -} \ No newline at end of file diff --git a/frappe/website/dashboard_fixtures.py b/frappe/website/dashboard_fixtures.py new file mode 100644 index 0000000000..4e5b5454bb --- /dev/null +++ b/frappe/website/dashboard_fixtures.py @@ -0,0 +1,36 @@ +import frappe + +def get_data(): + return frappe._dict({ + "dashboards": get_dashboards(), + "charts": get_charts(), + "number_cards": get_number_cards(), + }) + +def get_dashboards(): + return [{ + "name": "Website", + "dashboard_name": "Website", + "charts": [ + { "chart": "Website Analytics", "width": "Full" } + ] + }] + +def get_charts(): + return [{ + "chart_name": "Website Analytics", + "chart_type": "Report", + "custom_options": "{\"type\": \"line\", \"lineOptions\": {\"regionFill\": 1}, \"axisOptions\": {\"shortenYAxisNumbers\": 1}, \"colors\": [\"#7cd6fd\", \"#5e64ff\"], \"tooltipOptions\": {}}", + "doctype": "Dashboard Chart", + "filters_json": "{}", + "group_by_type": "Count", + "is_custom": 1, + "is_public": 1, + "name": "Website Analytics", + "number_of_groups": 0, + "report_name": "Website Analytics", + "time_interval": "Yearly", + "timeseries": 0, + "timespan": "Last Year", + "type": "Line" + }] \ No newline at end of file From 9e0b96c5c90239442a3240a74ae8d4378c1d9387 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 8 May 2020 18:21:18 +0530 Subject: [PATCH 11/83] fix: function not found --- frappe/website/dashboard_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/dashboard_fixtures.py b/frappe/website/dashboard_fixtures.py index 4e5b5454bb..01f1376d43 100644 --- a/frappe/website/dashboard_fixtures.py +++ b/frappe/website/dashboard_fixtures.py @@ -4,7 +4,7 @@ def get_data(): return frappe._dict({ "dashboards": get_dashboards(), "charts": get_charts(), - "number_cards": get_number_cards(), + "number_cards": None, }) def get_dashboards(): From 39ecc61b61d75bec0bfadf655de08e6a6c9dae69 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 13 May 2020 12:34:34 +0530 Subject: [PATCH 12/83] feat: added tour for web page --- frappe/website/doctype/web_page/web_page.js | 55 +++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/frappe/website/doctype/web_page/web_page.js b/frappe/website/doctype/web_page/web_page.js index c0a3bcdc20..c714b98241 100644 --- a/frappe/website/doctype/web_page/web_page.js +++ b/frappe/website/doctype/web_page/web_page.js @@ -48,3 +48,58 @@ frappe.ui.form.on('Web Page', { frappe.utils.set_meta_tag(frm.doc.route); } }); + +frappe.tour['Web Page'] = [ + { + fieldname: "title", + title: "Title of the page", + description: "This title will be used as the title of the webpage as well as in meta tags", + }, + { + fieldname: "published", + title: "Makes the page public", + description: "Checking this will publish the page on your website and it'll be visible to everyone.", + }, + { + fieldname: "route", + title: "URL of the page", + description: "This will be automatically generated when you publish the page, you can also enter a route yourself if you wish", + }, + { + fieldname: "content_type", + title: "Content type for building the page", + description: `You can select one from the following,
+
    +
  • Rich Text: Standard rich text editor with controls
    • +
  • Markdown: Github flavoured markdown syntax
    • +
  • HTML: HTML with jinja support
    • +
  • Page Builder: Frappe page builder using components
    • +
+ ` + }, + { + fieldname: "insert_code", + title: "Client Script", + description: "Checking this will show a text area where you can write custom javascript that will run on this page.", + }, + { + fieldname: "meta_title", + title: "Meta title for SEO", + description: "By default the title is used as meta title, adding a value here will override it.", + }, + { + fieldname: "meta_title", + title: "Meta Title", + description: "By default the title is used as meta title, adding a value here will override it.", + }, + { + fieldname: "meta_description", + title: "Meta Description", + description: "The meta description is an HTML attribute that provides a brief summary of a web page. Search engines such as Google often display the meta description in search results, which can influence click-through rates." + }, + { + fieldname: "meta_image", + title: "Meta Image", + description: "The meta image is unique image representing the content of the page. Images for this Card should be at least 280px in width, and at least 150px in height." + }, +]; \ No newline at end of file From aa76a03d266d085080dcfdc389badaea4f69ef07 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 13 May 2020 12:34:55 +0530 Subject: [PATCH 13/83] feat: added web page tour to onboarding --- .../module_onboarding/website/website.json | 5 ++++- .../add_blog_category/add_blog_category.json | 1 + .../create_blogger/create_blogger.json | 1 + .../enable_website_tracking.json | 1 + .../introduction_to_website.json | 1 + .../web_page_tour/web_page_tour.json | 17 +++++++++++++++++ 6 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 frappe/website/onboarding_step/web_page_tour/web_page_tour.json diff --git a/frappe/website/module_onboarding/website/website.json b/frappe/website/module_onboarding/website/website.json index b849a809ed..606ef09811 100644 --- a/frappe/website/module_onboarding/website/website.json +++ b/frappe/website/module_onboarding/website/website.json @@ -10,7 +10,7 @@ "documentation_url": "https://docs.erpnext.com/docs/user/manual/en/website", "idx": 0, "is_complete": 0, - "modified": "2020-04-30 20:23:06.438314", + "modified": "2020-05-13 12:32:35.269025", "modified_by": "Administrator", "module": "Website", "name": "Website", @@ -27,6 +27,9 @@ }, { "step": "Enable Website Tracking" + }, + { + "step": "Web Page Tour" } ], "subtitle": "Blogs, website view tracking, and more.", diff --git a/frappe/website/onboarding_step/add_blog_category/add_blog_category.json b/frappe/website/onboarding_step/add_blog_category/add_blog_category.json index a0d07c8464..5936e78c68 100644 --- a/frappe/website/onboarding_step/add_blog_category/add_blog_category.json +++ b/frappe/website/onboarding_step/add_blog_category/add_blog_category.json @@ -6,6 +6,7 @@ "idx": 0, "is_complete": 0, "is_mandatory": 0, + "is_single": 0, "is_skipped": 0, "modified": "2020-04-30 19:06:10.750976", "modified_by": "Administrator", diff --git a/frappe/website/onboarding_step/create_blogger/create_blogger.json b/frappe/website/onboarding_step/create_blogger/create_blogger.json index 5162e7e895..841325ca6a 100644 --- a/frappe/website/onboarding_step/create_blogger/create_blogger.json +++ b/frappe/website/onboarding_step/create_blogger/create_blogger.json @@ -6,6 +6,7 @@ "idx": 0, "is_complete": 0, "is_mandatory": 0, + "is_single": 0, "is_skipped": 0, "modified": "2020-04-30 19:06:10.694419", "modified_by": "Administrator", diff --git a/frappe/website/onboarding_step/enable_website_tracking/enable_website_tracking.json b/frappe/website/onboarding_step/enable_website_tracking/enable_website_tracking.json index 56a4fa58b6..b37a704b1e 100644 --- a/frappe/website/onboarding_step/enable_website_tracking/enable_website_tracking.json +++ b/frappe/website/onboarding_step/enable_website_tracking/enable_website_tracking.json @@ -7,6 +7,7 @@ "idx": 0, "is_complete": 0, "is_mandatory": 0, + "is_single": 0, "is_skipped": 0, "modified": "2020-04-30 20:22:50.778590", "modified_by": "Administrator", diff --git a/frappe/website/onboarding_step/introduction_to_website/introduction_to_website.json b/frappe/website/onboarding_step/introduction_to_website/introduction_to_website.json index 683d0a889e..f3c95657e8 100644 --- a/frappe/website/onboarding_step/introduction_to_website/introduction_to_website.json +++ b/frappe/website/onboarding_step/introduction_to_website/introduction_to_website.json @@ -6,6 +6,7 @@ "idx": 0, "is_complete": 0, "is_mandatory": 1, + "is_single": 0, "is_skipped": 0, "modified": "2020-04-30 19:06:10.578218", "modified_by": "Administrator", diff --git a/frappe/website/onboarding_step/web_page_tour/web_page_tour.json b/frappe/website/onboarding_step/web_page_tour/web_page_tour.json new file mode 100644 index 0000000000..1ee98d69ee --- /dev/null +++ b/frappe/website/onboarding_step/web_page_tour/web_page_tour.json @@ -0,0 +1,17 @@ +{ + "action": "Show Form Tour", + "creation": "2020-05-13 12:32:15.966570", + "docstatus": 0, + "doctype": "Onboarding Step", + "idx": 0, + "is_complete": 0, + "is_mandatory": 0, + "is_single": 0, + "is_skipped": 0, + "modified": "2020-05-13 12:32:15.966570", + "modified_by": "Administrator", + "name": "Web Page Tour", + "owner": "Administrator", + "reference_document": "Web Page", + "title": "Learn about Web Pages" +} \ No newline at end of file From 218e945b2614b0b2a72d59bfaf74ca2862a4794f Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 13 May 2020 19:56:00 +0530 Subject: [PATCH 14/83] feat: add get_last_day_of_week function --- frappe/utils/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index 58c74a905d..906b07fd08 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -190,6 +190,10 @@ def get_first_day(dt, d_years=0, d_months=0): def get_first_day_of_week(dt): return dt - datetime.timedelta(days=dt.weekday()) +def get_last_day_of_week(dt): + dt = get_first_day_of_week(dt) + return dt + datetime.timedelta(days=6) + def get_last_day(dt): """ Returns last day of the month using: From 8de7f7602e7aa6ba8ac10b60b5a83886fd77ab93 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 13 May 2020 19:56:16 +0530 Subject: [PATCH 15/83] refactor: replace pandas date range with native function --- frappe/utils/dateutils.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/frappe/utils/dateutils.py b/frappe/utils/dateutils.py index 907cab3d02..b98c1c12fc 100644 --- a/frappe/utils/dateutils.py +++ b/frappe/utils/dateutils.py @@ -6,6 +6,9 @@ import frappe import frappe.defaults import datetime from frappe.utils import get_datetime +from frappe.utils import nowdate, add_to_date, getdate, get_datetime +from frappe.utils.data import get_last_day_of_week +from frappe.desk.doctype.dashboard_chart.dashboard_chart import get_period_ending from six import string_types # global values -- used for caching @@ -74,8 +77,29 @@ def datetime_in_user_format(date_time): from frappe.utils import formatdate return formatdate(date_time.date()) + " " + date_time.strftime("%H:%M") -def get_date_range(from_date, to_date, frequency="M", normalize=True): - from pandas import date_range - date_range = date_range(start=from_date, end=to_date, freq=frequency, normalize=normalize).to_pydatetime().tolist() +def get_dates_from_timegrain(from_date, to_date, timegrain="Daily"): + from_date = getdate(from_date) + to_date = getdate(to_date) - return date_range \ No newline at end of file + days = months = years = 0 + if "Daily" == timegrain: + days = 1 + elif "Weekly" == timegrain: + days = 7 + elif "Monthly" == timegrain: + months = 1 + elif "Quarterly" == timegrain: + months = 3 + + if "Weekly" == timegrain: + dates = [get_last_day_of_week(from_date)] + else: + dates = [get_period_ending(from_date, timegrain)] + + while getdate(dates[-1]) < getdate(to_date): + if "Weekly" == timegrain: + date = get_last_day_of_week(add_to_date(dates[-1], years=years, months=months, days=days)) + else: + date = get_period_ending(add_to_date(dates[-1], years=years, months=months, days=days), timegrain) + dates.append(date) + return dates \ No newline at end of file From 0eaf778fe24911970f9e1b02e0e743fc63082103 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 13 May 2020 19:56:33 +0530 Subject: [PATCH 16/83] refactor: change values for report range options --- .../report/website_analytics/website_analytics.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frappe/website/report/website_analytics/website_analytics.js b/frappe/website/report/website_analytics/website_analytics.js index 7e051afa8c..9079949724 100644 --- a/frappe/website/report/website_analytics/website_analytics.js +++ b/frappe/website/report/website_analytics/website_analytics.js @@ -8,7 +8,7 @@ frappe.query_reports["Website Analytics"] = { fieldname: "from_date", label: __("From Date"), fieldtype: "Date", - default: frappe.datetime.add_days(frappe.datetime.now_date(true), -7), + default: frappe.datetime.add_days(frappe.datetime.now_date(true), -100), }, { fieldname:"to_date", @@ -21,11 +21,11 @@ frappe.query_reports["Website Analytics"] = { label: __("Range"), fieldtype: "Select", options: [ - { "value": "D", "label": __("Daily") }, - { "value": "W", "label": __("Weekly") }, - { "value": "M", "label": __("Monthly") }, + { "value": "Daily", "label": __("Daily") }, + { "value": "Weekly", "label": __("Weekly") }, + { "value": "Monthly", "label": __("Monthly") }, ], - default: "D", + default: "Daily", reqd: 1 } ] From 0bb10609989a71bcde291cb7cfaec43b428cb978 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 13 May 2020 19:57:30 +0530 Subject: [PATCH 17/83] feat: use new date range function --- .../website_analytics/website_analytics.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py index 694bc9e797..97c330fed9 100644 --- a/frappe/website/report/website_analytics/website_analytics.py +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -4,7 +4,8 @@ from __future__ import unicode_literals import frappe from datetime import datetime -from frappe.utils.dateutils import get_date_range +from frappe.utils import getdate +from frappe.utils.dateutils import get_dates_from_timegrain def execute(filters=None): return WebsiteAnalytics(filters).run() @@ -20,7 +21,7 @@ class WebsiteAnalytics(object): self.filters.from_date = frappe.utils.add_days(self.filters.to_date, -7) if not self.filters.range: - self.filters.range = "D" + self.filters.range = "Daily" self.filters.to_date = frappe.utils.add_days(self.filters.to_date, 1) self.query_filters = {'creation': ['between', [self.filters.from_date, self.filters.to_date]]} @@ -89,10 +90,10 @@ class WebsiteAnalytics(object): field = 'creation' date_format = '%Y-%m-%d' - if filters_range == "W": + if filters_range == "Weekly": field = 'ADDDATE(creation, INTERVAL 1-DAYOFWEEK(creation) DAY)' - elif filters_range == "M": + elif filters_range == "Monthly": date_format = '%Y-%m-01' query = """ @@ -115,10 +116,10 @@ class WebsiteAnalytics(object): field = 'creation' granularity = 'day' - if filters_range == "W": + if filters_range == "Weekly": granularity = 'week' - elif filters_range == "M": + elif filters_range == "Monthly": granularity = 'day' query = """ @@ -149,8 +150,8 @@ class WebsiteAnalytics(object): return self.prepare_chart_data(self.chart_data) def prepare_chart_data(self, data): - date_range = get_date_range(self.filters.from_date, self.filters.to_date, self.filters.range) - if self.filters.range == "M": + date_range = get_dates_from_timegrain(self.filters.from_date, self.filters.to_date, self.filters.range) + if self.filters.range == "Monthly": date_range = [frappe.utils.add_days(dd, 1) for dd in date_range] labels = [] @@ -159,8 +160,8 @@ class WebsiteAnalytics(object): def get_data_for_date(date): for item in data: - item_date = frappe.utils.get_datetime(item.get("date")).date() - if item_date == date.date(): + item_date = getdate(item.get("date")) + if item_date == date: return item return {'count': 0, 'unique_count': 0} From e2b5fcb6abe46d4b3da13263aca427d316d068ae Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 13 May 2020 19:58:33 +0530 Subject: [PATCH 18/83] chore: update fixtures --- frappe/website/dashboard_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/website/dashboard_fixtures.py b/frappe/website/dashboard_fixtures.py index 01f1376d43..1ac7ca60ec 100644 --- a/frappe/website/dashboard_fixtures.py +++ b/frappe/website/dashboard_fixtures.py @@ -20,7 +20,7 @@ def get_charts(): return [{ "chart_name": "Website Analytics", "chart_type": "Report", - "custom_options": "{\"type\": \"line\", \"lineOptions\": {\"regionFill\": 1}, \"axisOptions\": {\"shortenYAxisNumbers\": 1}, \"colors\": [\"#7cd6fd\", \"#5e64ff\"], \"tooltipOptions\": {}}", + "custom_options": "{\"type\": \"line\", \"lineOptions\": {\"regionFill\": 1}, \"axisOptions\": {\"shortenYAxisNumbers\": 1}, \"tooltipOptions\": {}}", "doctype": "Dashboard Chart", "filters_json": "{}", "group_by_type": "Count", From 9d4a4ef6f4c1a92ec5f0918f4773cb605586cbc9 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 15 May 2020 14:03:04 +0530 Subject: [PATCH 19/83] feat: collapse sections by default for page builder dialog --- frappe/public/js/frappe/utils/web_page_block.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frappe/public/js/frappe/utils/web_page_block.js b/frappe/public/js/frappe/utils/web_page_block.js index bcf821bcfa..acc7943671 100644 --- a/frappe/public/js/frappe/utils/web_page_block.js +++ b/frappe/public/js/frappe/utils/web_page_block.js @@ -4,7 +4,12 @@ frappe.ui.form.on('Web Page Block', { frappe.model.with_doc('Web Template', row.web_template).then(doc => { let d = new frappe.ui.Dialog({ title: __('Edit Values'), - fields: doc.fields, + fields: doc.fields.map(df => { + if (df.fieldtype == "Section Break") { + df.collapsible = 1; + } + return df + }), primary_action(values) { frappe.model.set_value( cdt, From a7b431465d127360f995f45acd6bb605b58e111f Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 15 May 2020 14:03:37 +0530 Subject: [PATCH 20/83] feat: allow 9 cards in card grid --- .../section_with_cards.html | 2 +- .../section_with_cards.json | 26 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/frappe/website/web_template/section_with_cards/section_with_cards.html b/frappe/website/web_template/section_with_cards/section_with_cards.html index 1f0f71a15b..431a64a86d 100644 --- a/frappe/website/web_template/section_with_cards/section_with_cards.html +++ b/frappe/website/web_template/section_with_cards/section_with_cards.html @@ -29,7 +29,7 @@ {%- set card_size = card_size or 'Small' -%}
- {%- for index in ['1', '2', '3', '4', '5', '6', '7', '8'] -%} + {%- for index in ['1', '2', '3', '4', '5', '6', '7', '8', '9'] -%} {%- set title = values['card_' + index + '_title'] -%} {%- set content = values['card_' + index + '_content'] -%} {%- set url = values['card_' + index + '_url'] -%} diff --git a/frappe/website/web_template/section_with_cards/section_with_cards.json b/frappe/website/web_template/section_with_cards/section_with_cards.json index 9ec430ae60..311cced829 100644 --- a/frappe/website/web_template/section_with_cards/section_with_cards.json +++ b/frappe/website/web_template/section_with_cards/section_with_cards.json @@ -213,10 +213,34 @@ "fieldtype": "Data", "label": "URL", "reqd": 0 + }, + { + "fieldname": "card_9", + "fieldtype": "Section Break", + "label": "Card 9", + "reqd": 0 + }, + { + "fieldname": "card_9_title", + "fieldtype": "Data", + "label": "Title", + "reqd": 0 + }, + { + "fieldname": "card_9_content", + "fieldtype": "Small Text", + "label": "Content", + "reqd": 0 + }, + { + "fieldname": "card_9_url", + "fieldtype": "Data", + "label": "URL", + "reqd": 0 } ], "idx": 0, - "modified": "2020-04-29 22:40:03.362229", + "modified": "2020-05-15 13:52:02.984001", "modified_by": "Administrator", "name": "Section with Cards", "owner": "Administrator", From 7721d540167d30f76308a2bc557afb29f021b33a Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 15 May 2020 14:15:59 +0530 Subject: [PATCH 21/83] feat: add hide login to website settings --- frappe/templates/includes/navbar/navbar_login.html | 2 +- .../doctype/website_settings/website_settings.json | 9 ++++++++- .../website/doctype/website_settings/website_settings.py | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/frappe/templates/includes/navbar/navbar_login.html b/frappe/templates/includes/navbar/navbar_login.html index 2a58efe039..4e2c6dc93b 100644 --- a/frappe/templates/includes/navbar/navbar_login.html +++ b/frappe/templates/includes/navbar/navbar_login.html @@ -1,5 +1,5 @@ -{% if not only_static %} +{% if not only_static and not hide_login %} {% if frappe.session.user != 'Guest' %}