diff --git a/frappe/patches.txt b/frappe/patches.txt index 7616ac65bc..d164258c42 100644 --- a/frappe/patches.txt +++ b/frappe/patches.txt @@ -278,6 +278,7 @@ frappe.patches.v13_0.set_path_for_homepage_in_web_page_view frappe.patches.v13_0.migrate_translation_column_data frappe.patches.v13_0.set_read_times frappe.patches.v13_0.remove_web_view +frappe.patches.v13_0.set_unique_for_page_view frappe.patches.v13_0.remove_tailwind_from_page_builder frappe.patches.v13_0.rename_onboarding frappe.patches.v13_0.email_unsubscribe diff --git a/frappe/patches/v13_0/set_unique_for_page_view.py b/frappe/patches/v13_0/set_unique_for_page_view.py new file mode 100644 index 0000000000..2a084e52e3 --- /dev/null +++ b/frappe/patches/v13_0/set_unique_for_page_view.py @@ -0,0 +1,6 @@ +import frappe + +def execute(): + frappe.reload_doc('website', 'doctype', 'web_page_view', force=True) + site_url = frappe.utils.get_site_url(frappe.local.site) + frappe.db.sql("""UPDATE `tabWeb Page View` set is_unique=1 where referrer LIKE '%{0}%'""".format(site_url)) diff --git a/frappe/utils/data.py b/frappe/utils/data.py index a0cd08770a..a0703c1465 100644 --- a/frappe/utils/data.py +++ b/frappe/utils/data.py @@ -190,6 +190,10 @@ def get_first_day(dt, d_years=0, d_months=0): def get_first_day_of_week(dt): return dt - datetime.timedelta(days=dt.weekday()) +def get_last_day_of_week(dt): + dt = get_first_day_of_week(dt) + return dt + datetime.timedelta(days=6) + def get_last_day(dt): """ Returns last day of the month using: diff --git a/frappe/utils/dateutils.py b/frappe/utils/dateutils.py index d5b7a3136b..90abdeb6cd 100644 --- a/frappe/utils/dateutils.py +++ b/frappe/utils/dateutils.py @@ -6,6 +6,9 @@ import frappe import frappe.defaults import datetime from frappe.utils import get_datetime +from frappe.utils import add_to_date, getdate +from frappe.utils.data import get_last_day_of_week +from frappe.desk.doctype.dashboard_chart.dashboard_chart import get_period_ending from six import string_types # global values -- used for caching @@ -73,3 +76,30 @@ def datetime_in_user_format(date_time): date_time = get_datetime(date_time) from frappe.utils import formatdate return formatdate(date_time.date()) + " " + date_time.strftime("%H:%M") + +def get_dates_from_timegrain(from_date, to_date, timegrain="Daily"): + from_date = getdate(from_date) + to_date = getdate(to_date) + + days = months = years = 0 + if "Daily" == timegrain: + days = 1 + elif "Weekly" == timegrain: + days = 7 + elif "Monthly" == timegrain: + months = 1 + elif "Quarterly" == timegrain: + months = 3 + + if "Weekly" == timegrain: + dates = [get_last_day_of_week(from_date)] + else: + dates = [get_period_ending(from_date, timegrain)] + + while getdate(dates[-1]) < getdate(to_date): + if "Weekly" == timegrain: + date = get_last_day_of_week(add_to_date(dates[-1], years=years, months=months, days=days)) + else: + date = get_period_ending(add_to_date(dates[-1], years=years, months=months, days=days), timegrain) + dates.append(date) + return dates \ No newline at end of file diff --git a/frappe/website/dashboard_fixtures.py b/frappe/website/dashboard_fixtures.py new file mode 100644 index 0000000000..1ac7ca60ec --- /dev/null +++ b/frappe/website/dashboard_fixtures.py @@ -0,0 +1,36 @@ +import frappe + +def get_data(): + return frappe._dict({ + "dashboards": get_dashboards(), + "charts": get_charts(), + "number_cards": None, + }) + +def get_dashboards(): + return [{ + "name": "Website", + "dashboard_name": "Website", + "charts": [ + { "chart": "Website Analytics", "width": "Full" } + ] + }] + +def get_charts(): + return [{ + "chart_name": "Website Analytics", + "chart_type": "Report", + "custom_options": "{\"type\": \"line\", \"lineOptions\": {\"regionFill\": 1}, \"axisOptions\": {\"shortenYAxisNumbers\": 1}, \"tooltipOptions\": {}}", + "doctype": "Dashboard Chart", + "filters_json": "{}", + "group_by_type": "Count", + "is_custom": 1, + "is_public": 1, + "name": "Website Analytics", + "number_of_groups": 0, + "report_name": "Website Analytics", + "time_interval": "Yearly", + "timeseries": 0, + "timespan": "Last Year", + "type": "Line" + }] \ No newline at end of file diff --git a/frappe/website/desk_page/website/website.json b/frappe/website/desk_page/website/website.json index 1c6066d21e..c42a17d404 100644 --- a/frappe/website/desk_page/website/website.json +++ b/frappe/website/desk_page/website/website.json @@ -27,7 +27,11 @@ } ], "category": "Modules", - "charts": [], + "charts": [ + { + "chart_name": "Website Analytics" + } + ], "creation": "2020-03-02 14:13:51.089373", "developer_mode_only": 0, "disable_user_customization": 0, @@ -37,7 +41,7 @@ "idx": 0, "is_standard": 1, "label": "Website", - "modified": "2020-04-26 13:03:49.094728", + "modified": "2020-05-05 18:17:13.232473", "modified_by": "Administrator", "module": "Website", "name": "Website", diff --git a/frappe/website/doctype/web_page/web_page.js b/frappe/website/doctype/web_page/web_page.js index c0a3bcdc20..437a86b5d0 100644 --- a/frappe/website/doctype/web_page/web_page.js +++ b/frappe/website/doctype/web_page/web_page.js @@ -48,3 +48,58 @@ frappe.ui.form.on('Web Page', { frappe.utils.set_meta_tag(frm.doc.route); } }); + +frappe.tour['Web Page'] = [ + { + fieldname: "title", + title: __("Title of the page"), + description: __("This title will be used as the title of the webpage as well as in meta tags"), + }, + { + fieldname: "published", + title: __("Makes the page public"), + description: __("Checking this will publish the page on your website and it'll be visible to everyone."), + }, + { + fieldname: "route", + title: __("URL of the page"), + description: __("This will be automatically generated when you publish the page, you can also enter a route yourself if you wish"), + }, + { + fieldname: "content_type", + title: __("Content type for building the page"), + description: `${__('You can select one from the following,')}
+ + ` + }, + { + fieldname: "insert_code", + title: __("Client Script"), + description: __("Checking this will show a text area where you can write custom javascript that will run on this page."), + }, + { + fieldname: "meta_title", + title: __("Meta title for SEO"), + description: __("By default the title is used as meta title, adding a value here will override it."), + }, + { + fieldname: "meta_title", + title: __("Meta Title"), + description: __("By default the title is used as meta title, adding a value here will override it."), + }, + { + fieldname: "meta_description", + title: __("Meta Description"), + description: __("The meta description is an HTML attribute that provides a brief summary of a web page. Search engines such as Google often display the meta description in search results, which can influence click-through rates.") + }, + { + fieldname: "meta_image", + title: __("Meta Image"), + description: __("The meta image is unique image representing the content of the page. Images for this Card should be at least 280px in width, and at least 150px in height.") + }, +]; \ No newline at end of file diff --git a/frappe/website/doctype/web_page_view/web_page_view.json b/frappe/website/doctype/web_page_view/web_page_view.json index 7a1a210d62..4243df39b1 100644 --- a/frappe/website/doctype/web_page_view/web_page_view.json +++ b/frappe/website/doctype/web_page_view/web_page_view.json @@ -1,5 +1,6 @@ { "actions": [], + "allow_import": 1, "creation": "2020-04-15 22:54:46.009703", "doctype": "DocType", "editable_grid": 1, @@ -9,7 +10,9 @@ "referrer", "browser", "browser_version", - "date" + "is_unique", + "time_zone", + "user_agent" ], "fields": [ { @@ -39,15 +42,24 @@ "set_only_once": 1 }, { - "fieldname": "date", - "fieldtype": "Datetime", - "label": "Date", - "set_only_once": 1 + "fieldname": "is_unique", + "fieldtype": "Data", + "label": "Is Unique" + }, + { + "fieldname": "time_zone", + "fieldtype": "Data", + "label": "Time Zone" + }, + { + "fieldname": "user_agent", + "fieldtype": "Data", + "label": "User Agent" } ], "in_create": 1, "links": [], - "modified": "2020-04-15 23:31:27.517793", + "modified": "2020-05-05 14:11:24.718770", "modified_by": "Administrator", "module": "Website", "name": "Web Page View", diff --git a/frappe/website/module_onboarding/website/website.json b/frappe/website/module_onboarding/website/website.json index b849a809ed..606ef09811 100644 --- a/frappe/website/module_onboarding/website/website.json +++ b/frappe/website/module_onboarding/website/website.json @@ -10,7 +10,7 @@ "documentation_url": "https://docs.erpnext.com/docs/user/manual/en/website", "idx": 0, "is_complete": 0, - "modified": "2020-04-30 20:23:06.438314", + "modified": "2020-05-13 12:32:35.269025", "modified_by": "Administrator", "module": "Website", "name": "Website", @@ -27,6 +27,9 @@ }, { "step": "Enable Website Tracking" + }, + { + "step": "Web Page Tour" } ], "subtitle": "Blogs, website view tracking, and more.", diff --git a/frappe/website/onboarding_step/add_blog_category/add_blog_category.json b/frappe/website/onboarding_step/add_blog_category/add_blog_category.json index a0d07c8464..5936e78c68 100644 --- a/frappe/website/onboarding_step/add_blog_category/add_blog_category.json +++ b/frappe/website/onboarding_step/add_blog_category/add_blog_category.json @@ -6,6 +6,7 @@ "idx": 0, "is_complete": 0, "is_mandatory": 0, + "is_single": 0, "is_skipped": 0, "modified": "2020-04-30 19:06:10.750976", "modified_by": "Administrator", diff --git a/frappe/website/onboarding_step/create_blogger/create_blogger.json b/frappe/website/onboarding_step/create_blogger/create_blogger.json index 5162e7e895..841325ca6a 100644 --- a/frappe/website/onboarding_step/create_blogger/create_blogger.json +++ b/frappe/website/onboarding_step/create_blogger/create_blogger.json @@ -6,6 +6,7 @@ "idx": 0, "is_complete": 0, "is_mandatory": 0, + "is_single": 0, "is_skipped": 0, "modified": "2020-04-30 19:06:10.694419", "modified_by": "Administrator", diff --git a/frappe/website/onboarding_step/enable_website_tracking/enable_website_tracking.json b/frappe/website/onboarding_step/enable_website_tracking/enable_website_tracking.json index 56a4fa58b6..b37a704b1e 100644 --- a/frappe/website/onboarding_step/enable_website_tracking/enable_website_tracking.json +++ b/frappe/website/onboarding_step/enable_website_tracking/enable_website_tracking.json @@ -7,6 +7,7 @@ "idx": 0, "is_complete": 0, "is_mandatory": 0, + "is_single": 0, "is_skipped": 0, "modified": "2020-04-30 20:22:50.778590", "modified_by": "Administrator", diff --git a/frappe/website/onboarding_step/introduction_to_website/introduction_to_website.json b/frappe/website/onboarding_step/introduction_to_website/introduction_to_website.json index 683d0a889e..f3c95657e8 100644 --- a/frappe/website/onboarding_step/introduction_to_website/introduction_to_website.json +++ b/frappe/website/onboarding_step/introduction_to_website/introduction_to_website.json @@ -6,6 +6,7 @@ "idx": 0, "is_complete": 0, "is_mandatory": 1, + "is_single": 0, "is_skipped": 0, "modified": "2020-04-30 19:06:10.578218", "modified_by": "Administrator", diff --git a/frappe/website/onboarding_step/web_page_tour/web_page_tour.json b/frappe/website/onboarding_step/web_page_tour/web_page_tour.json new file mode 100644 index 0000000000..1ee98d69ee --- /dev/null +++ b/frappe/website/onboarding_step/web_page_tour/web_page_tour.json @@ -0,0 +1,17 @@ +{ + "action": "Show Form Tour", + "creation": "2020-05-13 12:32:15.966570", + "docstatus": 0, + "doctype": "Onboarding Step", + "idx": 0, + "is_complete": 0, + "is_mandatory": 0, + "is_single": 0, + "is_skipped": 0, + "modified": "2020-05-13 12:32:15.966570", + "modified_by": "Administrator", + "name": "Web Page Tour", + "owner": "Administrator", + "reference_document": "Web Page", + "title": "Learn about Web Pages" +} \ No newline at end of file diff --git a/frappe/website/report/__init__.py b/frappe/website/report/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/website/report/website_analytics/__init__.py b/frappe/website/report/website_analytics/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frappe/website/report/website_analytics/website_analytics.js b/frappe/website/report/website_analytics/website_analytics.js new file mode 100644 index 0000000000..9079949724 --- /dev/null +++ b/frappe/website/report/website_analytics/website_analytics.js @@ -0,0 +1,32 @@ +// Copyright (c) 2016, Frappe Technologies and contributors +// For license information, please see license.txt +/* eslint-disable */ + +frappe.query_reports["Website Analytics"] = { + "filters": [ + { + fieldname: "from_date", + label: __("From Date"), + fieldtype: "Date", + default: frappe.datetime.add_days(frappe.datetime.now_date(true), -100), + }, + { + fieldname:"to_date", + label: __("To Date"), + fieldtype: "Date", + default: frappe.datetime.now_date(true), + }, + { + fieldname: "range", + label: __("Range"), + fieldtype: "Select", + options: [ + { "value": "Daily", "label": __("Daily") }, + { "value": "Weekly", "label": __("Weekly") }, + { "value": "Monthly", "label": __("Monthly") }, + ], + default: "Daily", + reqd: 1 + } + ] +}; diff --git a/frappe/website/report/website_analytics/website_analytics.json b/frappe/website/report/website_analytics/website_analytics.json new file mode 100644 index 0000000000..62c5751a5c --- /dev/null +++ b/frappe/website/report/website_analytics/website_analytics.json @@ -0,0 +1,27 @@ +{ + "add_total_row": 0, + "creation": "2020-04-17 13:04:45.770148", + "disable_prepared_report": 0, + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "idx": 0, + "is_standard": "Yes", + "modified": "2020-04-17 16:10:30.168312", + "modified_by": "Administrator", + "module": "Website", + "name": "Website Analytics", + "owner": "Administrator", + "prepared_report": 0, + "ref_doctype": "Web Page View", + "report_name": "Website Analytics", + "report_type": "Script Report", + "roles": [ + { + "role": "System Manager" + }, + { + "role": "Website Manager" + } + ] +} \ No newline at end of file diff --git a/frappe/website/report/website_analytics/website_analytics.py b/frappe/website/report/website_analytics/website_analytics.py new file mode 100644 index 0000000000..97c330fed9 --- /dev/null +++ b/frappe/website/report/website_analytics/website_analytics.py @@ -0,0 +1,224 @@ +# Copyright (c) 2013, Frappe Technologies and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from datetime import datetime +from frappe.utils import getdate +from frappe.utils.dateutils import get_dates_from_timegrain + +def execute(filters=None): + return WebsiteAnalytics(filters).run() + +class WebsiteAnalytics(object): + def __init__(self, filters=None): + self.filters = frappe._dict(filters or {}) + + if not self.filters.to_date: + self.filters.to_date = datetime.now() + + if not self.filters.from_date: + self.filters.from_date = frappe.utils.add_days(self.filters.to_date, -7) + + if not self.filters.range: + self.filters.range = "Daily" + + self.filters.to_date = frappe.utils.add_days(self.filters.to_date, 1) + self.query_filters = {'creation': ['between', [self.filters.from_date, self.filters.to_date]]} + + def run(self): + columns = self.get_columns() + data = self.get_data() + chart = self.get_chart_data() + summary = self.get_report_summary() + + return columns, data[:250], None, chart, summary + + def get_columns(self): + return [ + { + "fieldname": "path", + "label": "Page", + "fieldtype": "Data", + "width": 300 + }, + { + "fieldname": "count", + "label": "Page Views", + "fieldtype": "Int", + "width": 150 + }, + { + "fieldname": "unique_count", + "label": "Unique Visitors", + "fieldtype": "Int", + "width": 150 + } + ] + + def get_data(self): + pg_query = """ + SELECT + path, + COUNT(*) as count, + COUNT(CASE WHEN CAST(is_unique as Integer) = 1 THEN 1 END) as unique_count + FROM `tabWeb Page View` + WHERE coalesce("tabWeb Page View".creation, '0001-01-01') BETWEEN %s AND %s + GROUP BY path + ORDER BY count desc + """ + + mariadb_query = """ + SELECT + path, + COUNT(*) as count, + COUNT(CASE WHEN is_unique = 1 THEN 1 END) as unique_count + FROM `tabWeb Page View` + WHERE creation BETWEEN %s AND %s + GROUP BY path + ORDER BY count desc + """ + + data = frappe.db.multisql({ + "mariadb": mariadb_query, + "postgres": pg_query + }, (self.filters.from_date, self.filters.to_date)) + return data + + def _get_query_for_mariadb(self): + filters_range = self.filters.range + field = 'creation' + date_format = '%Y-%m-%d' + + if filters_range == "Weekly": + field = 'ADDDATE(creation, INTERVAL 1-DAYOFWEEK(creation) DAY)' + + elif filters_range == "Monthly": + date_format = '%Y-%m-01' + + query = """ + SELECT + DATE_FORMAT({0}, %s) as date, + COUNT(*) as count, + COUNT(CASE WHEN is_unique = 1 THEN 1 END) as unique_count + FROM `tabWeb Page View` + WHERE creation BETWEEN %s AND %s + GROUP BY DATE_FORMAT({0}, %s) + ORDER BY creation + """.format(field) + + values = (date_format, self.filters.from_date, self.filters.to_date, date_format) + + return query, values + + def _get_query_for_postgres(self): + filters_range = self.filters.range + field = 'creation' + granularity = 'day' + + if filters_range == "Weekly": + granularity = 'week' + + elif filters_range == "Monthly": + granularity = 'day' + + query = """ + SELECT + DATE_TRUNC(%s, {0}) as date, + COUNT(*) as count, + COUNT(CASE WHEN CAST(is_unique as Integer) = 1 THEN 1 END) as unique_count + FROM "tabWeb Page View" + WHERE coalesce("tabWeb Page View".{0}, '0001-01-01') BETWEEN %s AND %s + GROUP BY date_trunc(%s, {0}) + ORDER BY date + """.format(field) + + values = (granularity, self.filters.from_date, self.filters.to_date, granularity) + + return query, values + + def get_chart_data(self): + current_dialect = frappe.db.db_type or 'mariadb' + + if current_dialect == 'mariadb': + query, values = self._get_query_for_mariadb() + else: + query, values = self._get_query_for_postgres() + + self.chart_data = frappe.db.sql(query, values=values, as_dict=1) + + return self.prepare_chart_data(self.chart_data) + + def prepare_chart_data(self, data): + date_range = get_dates_from_timegrain(self.filters.from_date, self.filters.to_date, self.filters.range) + if self.filters.range == "Monthly": + date_range = [frappe.utils.add_days(dd, 1) for dd in date_range] + + labels = [] + total_dataset = [] + unique_dataset = [] + + def get_data_for_date(date): + for item in data: + item_date = getdate(item.get("date")) + if item_date == date: + return item + return {'count': 0, 'unique_count': 0} + + + for date in date_range: + labels.append(date.strftime("%b %d %Y")) + match = get_data_for_date(date) + total_dataset.append(match.get('count', 0)) + unique_dataset.append(match.get('unique_count', 0)) + + chart = { + "data": { + 'labels': labels, + 'datasets': [ + { + 'name': "Total Views", + 'type': 'line', + 'values': total_dataset + }, + { + 'name': "Unique Visits", + 'type': 'line', + 'values': unique_dataset + } + ] + }, + "type": "axis-mixed", + 'lineOptions': { + 'regionFill': 1, + }, + 'axisOptions': { + 'xIsSeries': 1 + }, + 'colors': ['#7cd6fd', '#5e64ff'] + } + + return chart + + + def get_report_summary(self): + total_count = 0 + unique_count = 0 + for data in self.chart_data: + unique_count += data.get('unique_count') + total_count += data.get('count') + + report_summary = [ + { + "value": total_count, + "label": "Total Page Views", + "datatype": "Int", + }, + { + "value": unique_count, + "label": "Unique Page Views", + "datatype": "Int", + }, + + ] + return report_summary \ No newline at end of file diff --git a/frappe/www/404.html b/frappe/www/404.html index 47685c45d0..dc178dbdc8 100644 --- a/frappe/www/404.html +++ b/frappe/www/404.html @@ -15,7 +15,9 @@ html, body { } {% include "templates/styles/card_style.css" %} - +
{{_("Page Missing or Moved")}} @@ -29,4 +31,4 @@ html, body { background-color: #f5f7fa; } -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/frappe/www/website_script.js b/frappe/www/website_script.js index 7fdc2e94d6..e31b6812d5 100644 --- a/frappe/www/website_script.js +++ b/frappe/www/website_script.js @@ -14,7 +14,7 @@ ga('send', 'pageview'); {%- endif %} {% if enable_view_tracking %} - if (navigator.doNotTrack != 1) { + if (navigator.doNotTrack != 1 && !window.is_404) { frappe.ready(() => { let browser = frappe.utils.get_browser(); frappe.call("frappe.website.doctype.web_page_view.web_page_view.make_view_log", {