From 4c483f0015d46feecc2836874885e92926b96906 Mon Sep 17 00:00:00 2001 From: Charles-Henri Decultot Date: Tue, 26 Mar 2019 09:51:10 +0000 Subject: [PATCH 1/2] fix: Google calendar connexion + data migration tool --- .../connectors/calendar_connector.py | 34 ++++++------- .../data_migration_run/data_migration_run.py | 30 ++++++------ .../event_to_gcalendar.json | 49 ++++++++++++++++++- .../gcalendar_to_event/__init__.py | 39 ++++++++------- .../gcalendar_to_event.json | 49 ++++++++++++++++++- .../gcalendar_sync/gcalendar_sync.json | 6 +-- 6 files changed, 150 insertions(+), 57 deletions(-) diff --git a/frappe/data_migration/doctype/data_migration_connector/connectors/calendar_connector.py b/frappe/data_migration/doctype/data_migration_connector/connectors/calendar_connector.py index ec62c69e72..578ac0fc37 100644 --- a/frappe/data_migration/doctype/data_migration_connector/connectors/calendar_connector.py +++ b/frappe/data_migration/doctype/data_migration_connector/connectors/calendar_connector.py @@ -7,6 +7,7 @@ from googleapiclient.errors import HttpError import time from datetime import datetime from frappe.utils import add_days, add_years +from frappe.desk.doctype.event.event import has_permission class CalendarConnector(BaseConnection): def __init__(self, connector): @@ -64,24 +65,21 @@ class CalendarConnector(BaseConnection): def insert(self, doctype, doc): if doctype == 'Events': - from frappe.desk.doctype.event.event import has_permission d = frappe.get_doc("Event", doc["name"]) if has_permission(d, self.account.name): - if doc["start_datetime"] >= datetime.now(): - try: - doctype = "Event" - e = self.insert_events(doctype, doc) - return e - except Exception: - frappe.log_error(frappe.get_traceback(), "GCalendar Synchronization Error") + try: + doctype = "Event" + e = self.insert_events(doctype, doc) + return e + except Exception: + frappe.log_error(frappe.get_traceback(), "GCalendar Synchronization Error") def update(self, doctype, doc, migration_id): if doctype == 'Events': - from frappe.desk.doctype.event.event import has_permission d = frappe.get_doc("Event", doc["name"]) if has_permission(d, self.account.name): - if doc["start_datetime"] >= datetime.now() and migration_id is not None: + if migration_id is not None: try: doctype = "Event" return self.update_events(doctype, doc, migration_id) @@ -217,23 +215,23 @@ class CalendarConnector(BaseConnection): day = [] if e.repeat_on == "Every Day": - if e.monday is not None: + if e.monday == 1: day.append("MO") - if e.tuesday is not None: + if e.tuesday == 1: day.append("TU") - if e.wednesday is not None: + if e.wednesday == 1: day.append("WE") - if e.thursday is not None: + if e.thursday == 1: day.append("TH") - if e.friday is not None: + if e.friday == 1: day.append("FR") - if e.saturday is not None: + if e.saturday == 1: day.append("SA") - if e.sunday is not None: + if e.sunday == 1: day.append("SU") day = "BYDAY=" + ",".join(str(d) for d in day) - frequency = "FREQ=DAILY" + frequency = "FREQ=WEEKLY" elif e.repeat_on == "Every Week": frequency = "FREQ=WEEKLY" diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py index 55e7dbe818..fb91ac551c 100644 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py @@ -6,7 +6,7 @@ from __future__ import unicode_literals import frappe, json, math from frappe.model.document import Document from frappe import _ -from frappe.utils import get_source_value +from frappe.utils import get_source_value, cstr class DataMigrationRun(Document): def run(self): @@ -213,19 +213,19 @@ class DataMigrationRun(Document): def get_deleted_local_data(self): '''Fetch local deleted data using `frappe.get_all`. Used during Push''' mapping = self.get_mapping(self.current_mapping) - or_filters = self.get_or_filters(mapping) - filters = dict( - deleted_doctype=mapping.local_doctype - ) + filters = self.get_last_modified_condition() + filters.update({ + "deleted_doctype": mapping.local_doctype + }) - data = frappe.get_all('Deleted Document', fields=['data'], - filters=filters, or_filters=or_filters) + data = frappe.get_all('Deleted Document', fields=['name', 'data'], + filters=filters) _data = [] for d in data: doc = json.loads(d.data) if doc.get(mapping.migration_id_field): - doc['_deleted_document_name'] = d.name + doc['_deleted_document_name'] = d["name"] _data.append(doc) return _data @@ -306,8 +306,8 @@ class DataMigrationRun(Document): self.update_log('push_insert', 1) # post process after insert self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception: - self.update_log('push_failed', d.name) + except Exception as e: + self.update_log('push_failed', {d.name: cstr(e)}) # update page_start self.db_set('current_mapping_start', @@ -338,8 +338,8 @@ class DataMigrationRun(Document): self.update_log('push_update', 1) # post process after update self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception: - self.update_log('push_failed', d.name) + except Exception as e: + self.update_log('push_failed', {d.name: cstr(e)}) # update page_start self.db_set('current_mapping_start', @@ -370,8 +370,8 @@ class DataMigrationRun(Document): self.update_log('push_delete', 1) # post process only when action is success self.post_process_doc(local_doc=d, remote_doc=response_doc) - except Exception: - self.update_log('push_failed', d.name) + except Exception as e: + self.update_log('push_failed', {d.name: cstr(e)}) # update page_start self.db_set('current_mapping_start', @@ -414,7 +414,7 @@ class DataMigrationRun(Document): self.post_process_doc(remote_doc=d, local_doc=local_doc) except Exception: # failed, append to log - self.update_log('pull_failed', migration_id_value) + self.update_log('push_failed', {migration_id_value: cstr(e)}) if len(data) < mapping.page_length: # last page, done with pull diff --git a/frappe/integrations/data_migration_mapping/event_to_gcalendar/event_to_gcalendar.json b/frappe/integrations/data_migration_mapping/event_to_gcalendar/event_to_gcalendar.json index b940237c8d..6d2a6020a5 100644 --- a/frappe/integrations/data_migration_mapping/event_to_gcalendar/event_to_gcalendar.json +++ b/frappe/integrations/data_migration_mapping/event_to_gcalendar/event_to_gcalendar.json @@ -33,6 +33,51 @@ "local_fieldname": "repeat_this_event", "remote_fieldname": "repeat_this_event" }, + { + "is_child_table": 0, + "local_fieldname": "repeat_on", + "remote_fieldname": "repeat_on" + }, + { + "is_child_table": 0, + "local_fieldname": "repeat_till", + "remote_fieldname": "repeat_till" + }, + { + "is_child_table": 0, + "local_fieldname": "monday", + "remote_fieldname": "monday" + }, + { + "is_child_table": 0, + "local_fieldname": "tuesday", + "remote_fieldname": "tuesday" + }, + { + "is_child_table": 0, + "local_fieldname": "wednesday", + "remote_fieldname": "wednesday" + }, + { + "is_child_table": 0, + "local_fieldname": "thursday", + "remote_fieldname": "thursday" + }, + { + "is_child_table": 0, + "local_fieldname": "friday", + "remote_fieldname": "friday" + }, + { + "is_child_table": 0, + "local_fieldname": "saturday", + "remote_fieldname": "saturday" + }, + { + "is_child_table": 0, + "local_fieldname": "sunday", + "remote_fieldname": "sunday" + }, { "is_child_table": 0, "local_fieldname": "name", @@ -45,8 +90,8 @@ "mapping_name": "Event to GCalendar", "mapping_type": "Push", "migration_id_field": "gcalendar_sync_id", - "modified": "2018-05-18 14:38:43.658069", - "modified_by": "chdecultot@dokos.io", + "modified": "2019-03-26 10:16:45.400150", + "modified_by": "Administrator", "name": "Event to GCalendar", "owner": "Administrator", "page_length": 10, diff --git a/frappe/integrations/data_migration_mapping/gcalendar_to_event/__init__.py b/frappe/integrations/data_migration_mapping/gcalendar_to_event/__init__.py index 441d2c3797..5518871c97 100644 --- a/frappe/integrations/data_migration_mapping/gcalendar_to_event/__init__.py +++ b/frappe/integrations/data_migration_mapping/gcalendar_to_event/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals import frappe -from datetime import datetime +from datetime import datetime, timedelta from dateutil.parser import parse from pytz import timezone from frappe.utils import add_days @@ -30,7 +30,7 @@ def pre_process(events): event_tz = events["start"]["timeZone"] else: event_tz = events["calendar_tz"] - start_dt= timezone(event_tz).localize(start_dt) + start_dt = timezone(event_tz).localize(start_dt) if end_dt.tzinfo is None or end_dt.tzinfo.utcoffset(end_dt) is None: if "timeZone" in events["end"]: @@ -79,32 +79,37 @@ def get_recurrence_event_fields_value(recur_rule, starts_on): else: repeat_on = "Every Year" elif "UNTIL" in _str: # get repeat till - date = datetime.strptime(_str.split("=")[1], "%Y%m%dT%H%M%SZ") + date = parse(_str.split("=")[1]) repeat_till = get_repeat_till_date(date) elif "COUNT" in _str: # get repeat till - date = datetime.strptime(starts_on, "%Y-%m-%d %H:%M:%S") + date = parse(starts_on) repeat_till = get_repeat_till_date(date, count=_str.split("=")[1], repeat_on=repeat_on) elif "BYDAY" in _str: days = _str.split("=")[1] - if repeat_on == "DAILY": - repeat_days.update({ - "sunday": 1 if "SU" in days else 0, - "monday": 1 if "MO" in days else 0, - "tuesday": 1 if "TU" in days else 0, - "wednesday": 1 if "WD" in days else 0, - "thursday": 1 if "TU" in days else 0, - "friday": 1 if "TU" in days else 0, - "saturday": 1 if "TU" in days else 0, - }) + repeat_days.update({ + "sunday": 1 if "SU" in days else 0, + "monday": 1 if "MO" in days else 0, + "tuesday": 1 if "TU" in days else 0, + "wednesday": 1 if "WD" in days else 0, + "thursday": 1 if "TH" in days else 0, + "friday": 1 if "FR" in days else 0, + "saturday": 1 if "SA" in days else 0, + }) + repeat_on = "Every Day" - return { + recurrence = { "repeat_on": repeat_on, "repeat_till": repeat_till, - "repeat_this_event": 1, - "repeat_days": repeat_days + "repeat_this_event": 1 } + if repeat_days: + recurrence.update(repeat_days) + + return recurrence + + def get_repeat_till_date(date, count=None, repeat_on=None): if count: if repeat_on == "Every Day": diff --git a/frappe/integrations/data_migration_mapping/gcalendar_to_event/gcalendar_to_event.json b/frappe/integrations/data_migration_mapping/gcalendar_to_event/gcalendar_to_event.json index ec5bfc5f00..a4ca740ce5 100644 --- a/frappe/integrations/data_migration_mapping/gcalendar_to_event/gcalendar_to_event.json +++ b/frappe/integrations/data_migration_mapping/gcalendar_to_event/gcalendar_to_event.json @@ -33,6 +33,51 @@ "local_fieldname": "repeat_this_event", "remote_fieldname": "repeat_this_event" }, + { + "is_child_table": 0, + "local_fieldname": "repeat_on", + "remote_fieldname": "repeat_on" + }, + { + "is_child_table": 0, + "local_fieldname": "repeat_till", + "remote_fieldname": "repeat_till" + }, + { + "is_child_table": 0, + "local_fieldname": "monday", + "remote_fieldname": "monday" + }, + { + "is_child_table": 0, + "local_fieldname": "tuesday", + "remote_fieldname": "tuesday" + }, + { + "is_child_table": 0, + "local_fieldname": "wednesday", + "remote_fieldname": "wednesday" + }, + { + "is_child_table": 0, + "local_fieldname": "thursday", + "remote_fieldname": "thursday" + }, + { + "is_child_table": 0, + "local_fieldname": "friday", + "remote_fieldname": "friday" + }, + { + "is_child_table": 0, + "local_fieldname": "saturday", + "remote_fieldname": "saturday" + }, + { + "is_child_table": 0, + "local_fieldname": "sunday", + "remote_fieldname": "sunday" + }, { "is_child_table": 0, "local_fieldname": "gcalendar_sync_id", @@ -50,8 +95,8 @@ "mapping_name": "GCalendar to Event", "mapping_type": "Pull", "migration_id_field": "gcalendar_sync_id", - "modified": "2018-05-18 14:38:43.694867", - "modified_by": "chdecultot@dokos.io", + "modified": "2019-03-26 10:16:45.426138", + "modified_by": "Administrator", "name": "GCalendar to Event", "owner": "Administrator", "page_length": 250, diff --git a/frappe/integrations/data_migration_plan/gcalendar_sync/gcalendar_sync.json b/frappe/integrations/data_migration_plan/gcalendar_sync/gcalendar_sync.json index 2fac63854f..9eef669203 100644 --- a/frappe/integrations/data_migration_plan/gcalendar_sync/gcalendar_sync.json +++ b/frappe/integrations/data_migration_plan/gcalendar_sync/gcalendar_sync.json @@ -2,7 +2,7 @@ "creation": "2018-03-23 19:10:23.715161", "docstatus": 0, "doctype": "Data Migration Plan", - "idx": 4, + "idx": 22, "mappings": [ { "enabled": 1, @@ -13,8 +13,8 @@ "mapping": "GCalendar to Event" } ], - "modified": "2018-05-18 14:38:43.559026", - "modified_by": "chdecultot@dokos.io", + "modified": "2019-03-26 10:16:45.369037", + "modified_by": "Administrator", "module": "Integrations", "name": "GCalendar Sync", "owner": "Administrator", From e8ff32edfcdc127fc38580dd277fbf3e5a2be1d6 Mon Sep 17 00:00:00 2001 From: Charles-Henri Decultot Date: Tue, 26 Mar 2019 10:57:20 +0000 Subject: [PATCH 2/2] Typo correction --- .../doctype/data_migration_run/data_migration_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py index fb91ac551c..93c0d8254d 100644 --- a/frappe/data_migration/doctype/data_migration_run/data_migration_run.py +++ b/frappe/data_migration/doctype/data_migration_run/data_migration_run.py @@ -414,7 +414,7 @@ class DataMigrationRun(Document): self.post_process_doc(remote_doc=d, local_doc=local_doc) except Exception: # failed, append to log - self.update_log('push_failed', {migration_id_value: cstr(e)}) + self.update_log('pull_failed', {migration_id_value: cstr(e)}) if len(data) < mapping.page_length: # last page, done with pull