I am currently receiving error:
Leaf Error (leaf1:3306): Cannot compile pipeline plan for `incoming_sfdc_v2`.`contact` because reference db is out of sync on this leaf
It seems I can run my data pipeline about one time before MemSQL starts complaining about subsequent runs.
The SQL statement looks as follows:
CREATE DATABASE IF NOT EXISTS incoming_sfdc_v2;
DROP PIPELINE IF EXISTS contact;
DROP TABLE IF EXISTS contact;
CREATE TABLE `incoming_sfdc_v2`.`contact` ( `attributes` TEXT , `id` TEXT , `isdeleted` bigint , `masterrecordid` TEXT , `accountid` TEXT , `lastname` TEXT , `firstname` TEXT , `salutation` TEXT , `middlename` TEXT , `suffix` TEXT , `name` TEXT , `mailingstreet` TEXT , `mailingcity` TEXT , `mailingstate` TEXT , `mailingpostalcode` TEXT , `mailingcountry` TEXT , `mailingstatecode` TEXT , `mailingcountrycode` TEXT , `mailinglatitude` double , `mailinglongitude` double , `mailinggeocodeaccuracy` TEXT , `mailingaddress` TEXT , `phone` TEXT , `fax` TEXT , `mobilephone` TEXT , `reportstoid` TEXT , `email` TEXT , `title` TEXT , `department` TEXT , `currencyisocode` TEXT , `ownerid` TEXT , `hasoptedoutofemail` bigint , `createddate` TEXT , `createdbyid` TEXT , `lastmodifieddate` TEXT , `lastmodifiedbyid` TEXT , `systemmodstamp` TEXT , `lastactivitydate` TEXT , `lastcurequestdate` TEXT , `lastcuupdatedate` TEXT , `lastvieweddate` TEXT , `lastreferenceddate` TEXT , `emailbouncedreason` TEXT , `emailbounceddate` TEXT , `isemailbounced` bigint , `photourl` TEXT , `jigsaw` TEXT , `jigsawcontactid` TEXT , `ip_country__c` TEXT , `ip_state__c` TEXT , `lid__linkedin_company_id__c` TEXT , `lid__linkedin_member_token__c` TEXT , `last_email_name__c` TEXT , `last_registered_event__c` TEXT , `ip_address__c` TEXT , `ip_city__c` TEXT , `email_opt_out_marketing_information__c` bigint , `culture_amp_email_subscription__c` TEXT , `outreach_persona__c` TEXT , `outbound_personalised_note__c` TEXT , `chat_result__c` TEXT , `network_interested__c` TEXT , `hubspot_score__c` double , `email_opt_out_people_geek_events__c` bigint , `original_source_drill_down_2__c` TEXT , `original_source_drill_down_1__c` TEXT , `first_page_seen__c` TEXT , `last_page_seen__c` TEXT , `first_conversion__c` TEXT , `team_for_views__c` TEXT , `became_a_marketing_qualified_lead_date__c` TEXT , `net_promoter_score_nps__c` double , `last_registered_event_date__c` TEXT , `first_conversion_date__c` TEXT , `last_email_click_date__c` TEXT , `contact_id_18_char__c` TEXT , `eventbritesync__eventbriteid__c` TEXT , `eventbritesync__eventbritetype__c` TEXT , `eventbritesync__uniquepersonid__c` TEXT , `time_first_seen_contact__c` TEXT , `converted_campaign__c` TEXT , `date_of_last_chat__c` TEXT , `outreach_stage__c` TEXT , `clearbit_company_metrics_employees__c` double , `recent_campaign_touch__c` TEXT , `recent_campaign_source_contact__c` TEXT , `outreach_notes__c` TEXT , `campaign_influence_lead__c` TEXT , `most_recent_campaign_touch_date__c` TEXT , `contact_previous_campaign_touch__c` TEXT , `previous_campaign_touch_date_contact__c` TEXT , `account_abm_stage__c` TEXT , `account_owner_for_views__c` TEXT , `contact_tweet__c` TEXT , `recent_campaign__c` double , `became_a_demo_date_contact__c` TEXT , `recent_campaign_id_contact__c` TEXT , `campaign_member__c` TEXT , `leandata__ld_segment__c` TEXT , `leandata__matched_buyer_persona__c` TEXT , `leandata__routing_action__c` TEXT , `linkedin_page__c` TEXT , `persona__c` TEXT , `last_nps_date__c` TEXT , `master_class_date__c` TEXT , `people_geekly_subscriber__c` bigint , `abm_source__c` TEXT , `level_of_influence__c` TEXT , `meeting_type_cp__c` TEXT , `email_opt_out_newsletter__c` bigint , `product_admin_status__c` TEXT , `recent_conversion_date__c` TEXT , `individual_view__c` TEXT , `employee_size_segment__c` TEXT , `country_calling_code__c` TEXT , `actively_being_sequenced__c` bigint , `job_change__c` TEXT , `email_product_events_opt_in__c` bigint , `email_receives_events__c` bigint , `account_owner_email__c` TEXT , `account_owner_logged_in_user__c` bigint , `sdr_owner_manager__c` TEXT , `logged_in_user_sdr_owner_manager__c` bigint , `logged_in_user_leadership_role__c` bigint , `account_owner_lookup__c` TEXT , `champion__c` TEXT , `email_product_newsletter_opt_in__c` bigint , `email_product_insights_opt_in__c` bigint , `email_receives_insights__c` bigint , `email_receives_newsletter__c` bigint , `date_of_last_geekup__c` TEXT , `date_of_last_workshop__c` TEXT , `zendesk__create_in_zendesk__c` bigint , `zendesk__last_sync_date__c` TEXT , `zendesk__last_sync_status__c` TEXT , `zendesk__result__c` TEXT , `zendesk__tags__c` TEXT , `zendesk__zendesk_outofsync__c` bigint , `zendesk__zendesk_oldtags__c` TEXT , `zendesk__iscreatedupdatedflag__c` bigint , `zendesk__notes__c` TEXT , `zendesk__zendesk_id__c` TEXT , `last_tc_lead__c` TEXT , `tc_clock_active__c` bigint , `contact_status__c` TEXT , `inactive_reason__c` TEXT , `import_notes__c` TEXT , `cloudingoagent__ces__c` double , `cloudingoagent__mar__c` TEXT , `cloudingoagent__mas__c` double , `cloudingoagent__mav__c` TEXT , `cloudingoagent__mrdi__c` TEXT , `cloudingoagent__mtz__c` TEXT , `cloudingoagent__oar__c` TEXT , `cloudingoagent__oas__c` double , `cloudingoagent__oav__c` TEXT , `cloudingoagent__ordi__c` TEXT , `cloudingoagent__otz__c` TEXT , `became_mql_date_time__c` TEXT , `leandata_inactive_merge__c` TEXT , `bizible2__ad_campaign_name_ft__c` TEXT , `bizible2__ad_campaign_name_lc__c` TEXT , `bizible2__bizibleid__c` TEXT , `bizible2__landing_page_ft__c` TEXT , `bizible2__landing_page_lc__c` TEXT , `bizible2__marketing_channel_ft__c` TEXT , `bizible2__marketing_channel_lc__c` TEXT , `bizible2__touchpoint_date_ft__c` TEXT , `bizible2__touchpoint_date_lc__c` TEXT , `bizible2__touchpoint_source_ft__c` TEXT , `bizible2__touchpoint_source_lc__c` TEXT , `slack_notification_text__c` TEXT , `customer_status__c` TEXT , `recent_conversion_event__c` TEXT , `marketing_qualified_reason__c` TEXT , `last_outreach_sequence__c` TEXT , `clearbit_timestamp__c` TEXT , `converted_from_lead__c` bigint , `account_address__c` TEXT , `address_validation__c` TEXT , `last_salesforce_campaign__c` TEXT , `wm4sf3__walkme_engagement_score__c` TEXT , `sourced_by__c` TEXT , `current_sequence_task_due_date__c` TEXT , `current_outreach_sequence_new__c` TEXT , `dfox__account__c` TEXT , `dfox__company_alexa_rank__c` double , `dfox__company_city__c` TEXT , `dfox__company_country__c` TEXT , `dfox__company_crunchbase_url__c` TEXT , `dfox__company_datafox_id__c` TEXT , `dfox__company_datafox_score__c` double , `dfox__company_datafox_url__c` TEXT , `dfox__company_finance_score__c` double , `dfox__company_growth_score__c` double , `dfox__company_hr_score__c` double , `dfox__company_influence_score__c` double , `dfox__company_investors__c` TEXT , `dfox__company_last_founding_type__c` TEXT , `dfox__company_last_funding_amount__c` double , `dfox__company_last_funding_date__c` TEXT , `dfox__company_last_funding_type__c` TEXT , `dfox__company_linkedin_url__c` TEXT , `dfox__company_long_description__c` TEXT , `dfox__company_name__c` TEXT , `dfox__company_number_of_employees__c` double , `dfox__company_phone_number__c` TEXT , `dfox__company_private_funding__c` double , `dfox__company_revenue_estimate__c` double , `dfox__company_short_description__c` TEXT , `dfox__company_stage__c` TEXT , `dfox__company_state__c` TEXT , `dfox__company_stock_ticker__c` TEXT , `dfox__company_street_address__c` TEXT , `dfox__company_techstack__c` TEXT , `dfox__company_top_keywords__c` TEXT , `dfox__company_twitter_followers__c` double , `dfox__company_twitter_handle__c` TEXT , `dfox__company_website__c` TEXT , `dfox__company_zip_code__c` TEXT , `dfox__id__c` TEXT , `dfox__last_synced_date__c` TEXT , `dfox__middle_name__c` TEXT , `dfox__sourced_from_datafox__c` bigint , `brighttalk__user_id__c` double , `community_score_engaging__c` double , `account_owner_pipeline_pod__c` TEXT , `last_salesforce_campaign_status__c` TEXT , `cbit__clearbitready__c` bigint , `cbit__clearbit__c` TEXT , `cbit__createdbyclearbit__c` bigint , `cbit__facebook__c` TEXT , `cbit__linkedin__c` TEXT , `cbit__twitter__c` TEXT , `glid__c` TEXT , `xuid__c` TEXT , `community_score_learning__c` double , `community_score_listening__c` double , `community_score_overall__c` double , `hubspot_sync_inclusion_list__c` bigint , `hubspot_sfdc_sync_master__c` bigint , `how_did_you_hear__c` TEXT , `how_did_you_hear_details__c` TEXT , `date_of_qualified_opportunity__c` TEXT , `date_last_contact_status_change__c` TEXT , `time_in_contact_status_days__c` double , `slack_notification_text_backup__c` TEXT , `region_contact__c` TEXT , `qualified_ntc_opp__c` bigint , `qualified_tc_opp__c` bigint , `customer_mql__c` bigint , `lead_notes__c` TEXT , `count_of_activities_last_90_days__c` double , `abm_sourced_by__c` TEXT , `website_interest__c` TEXT , `sdr_owner_locked__c` bigint , `sdr_owner__c` TEXT , `logged_in_user_sdr_owner__c` bigint , `unique_contacts__c` double , `count_of_activities_since_mql_date__c` double , `hubspot_sync_testing__c` TEXT , `recent_campaign_history__c` TEXT , `website_interest_sales_multiselect__c` TEXT , `original_source_drill_down_1_del__c` TEXT , `description` TEXT , `pathfactory_last_topic_seen__c` TEXT , `account_owner_id__c` TEXT , `sdr_owner_s_manager_team__c` TEXT , `time` bigint , KEY(`time`) USING CLUSTERED COLUMNSTORE );;
CREATE OR REPLACE PIPELINE contact
AS LOAD DATA S3 "<bucket-name>/31155e34-96b8-400c-9477-d442ef4dbc5d/incoming_sfdc_v2/contact/"
CONFIG "{\"region\": \"us-west-2\"}"
SKIP DUPLICATE KEY ERRORS
INTO TABLE contact
FIELDS TERMINATED BY "\t"
LINES TERMINATED BY "\r\n"
IGNORE 1 LINES
( attributes,id,isdeleted,masterrecordid,accountid,lastname,firstname,salutation,middlename,suffix,name,mailingstreet,mailingcity,mailingstate,mailingpostalcode,mailingcountry,mailingstatecode,mailingcountrycode,mailinglatitude,mailinglongitude,mailinggeocodeaccuracy,mailingaddress,phone,fax,mobilephone,reportstoid,email,title,department,currencyisocode,ownerid,hasoptedoutofemail,createddate,createdbyid,lastmodifieddate,lastmodifiedbyid,systemmodstamp,lastactivitydate,lastcurequestdate,lastcuupdatedate,lastvieweddate,lastreferenceddate,emailbouncedreason,emailbounceddate,isemailbounced,photourl,jigsaw,jigsawcontactid,ip_country__c,ip_state__c,lid__linkedin_company_id__c,lid__linkedin_member_token__c,last_email_name__c,last_registered_event__c,ip_address__c,ip_city__c,email_opt_out_marketing_information__c,culture_amp_email_subscription__c,outreach_persona__c,outbound_personalised_note__c,chat_result__c,network_interested__c,hubspot_score__c,email_opt_out_people_geek_events__c,original_source_drill_down_2__c,original_source_drill_down_1__c,first_page_seen__c,last_page_seen__c,first_conversion__c,team_for_views__c,became_a_marketing_qualified_lead_date__c,net_promoter_score_nps__c,last_registered_event_date__c,first_conversion_date__c,last_email_click_date__c,contact_id_18_char__c,eventbritesync__eventbriteid__c,eventbritesync__eventbritetype__c,eventbritesync__uniquepersonid__c,time_first_seen_contact__c,converted_campaign__c,date_of_last_chat__c,outreach_stage__c,clearbit_company_metrics_employees__c,recent_campaign_touch__c,recent_campaign_source_contact__c,outreach_notes__c,campaign_influence_lead__c,most_recent_campaign_touch_date__c,contact_previous_campaign_touch__c,previous_campaign_touch_date_contact__c,account_abm_stage__c,account_owner_for_views__c,contact_tweet__c,recent_campaign__c,became_a_demo_date_contact__c,recent_campaign_id_contact__c,campaign_member__c,leandata__ld_segment__c,leandata__matched_buyer_persona__c,leandata__routing_action__c,linkedin_page__c,persona__c,last_nps_date__c,master_class_date__c,people_geekly_subscriber__c,abm_source__c,level_of_influence__c,meeting_type_cp__c,email_opt_out_newsletter__c,product_admin_status__c,recent_conversion_date__c,individual_view__c,employee_size_segment__c,country_calling_code__c,actively_being_sequenced__c,job_change__c,email_product_events_opt_in__c,email_receives_events__c,account_owner_email__c,account_owner_logged_in_user__c,sdr_owner_manager__c,logged_in_user_sdr_owner_manager__c,logged_in_user_leadership_role__c,account_owner_lookup__c,champion__c,email_product_newsletter_opt_in__c,email_product_insights_opt_in__c,email_receives_insights__c,email_receives_newsletter__c,date_of_last_geekup__c,date_of_last_workshop__c,zendesk__create_in_zendesk__c,zendesk__last_sync_date__c,zendesk__last_sync_status__c,zendesk__result__c,zendesk__tags__c,zendesk__zendesk_outofsync__c,zendesk__zendesk_oldtags__c,zendesk__iscreatedupdatedflag__c,zendesk__notes__c,zendesk__zendesk_id__c,last_tc_lead__c,tc_clock_active__c,contact_status__c,inactive_reason__c,import_notes__c,cloudingoagent__ces__c,cloudingoagent__mar__c,cloudingoagent__mas__c,cloudingoagent__mav__c,cloudingoagent__mrdi__c,cloudingoagent__mtz__c,cloudingoagent__oar__c,cloudingoagent__oas__c,cloudingoagent__oav__c,cloudingoagent__ordi__c,cloudingoagent__otz__c,became_mql_date_time__c,leandata_inactive_merge__c,bizible2__ad_campaign_name_ft__c,bizible2__ad_campaign_name_lc__c,bizible2__bizibleid__c,bizible2__landing_page_ft__c,bizible2__landing_page_lc__c,bizible2__marketing_channel_ft__c,bizible2__marketing_channel_lc__c,bizible2__touchpoint_date_ft__c,bizible2__touchpoint_date_lc__c,bizible2__touchpoint_source_ft__c,bizible2__touchpoint_source_lc__c,slack_notification_text__c,customer_status__c,recent_conversion_event__c,marketing_qualified_reason__c,last_outreach_sequence__c,clearbit_timestamp__c,converted_from_lead__c,account_address__c,address_validation__c,last_salesforce_campaign__c,wm4sf3__walkme_engagement_score__c,sourced_by__c,current_sequence_task_due_date__c,current_outreach_sequence_new__c,dfox__account__c,dfox__company_alexa_rank__c,dfox__company_city__c,dfox__company_country__c,dfox__company_crunchbase_url__c,dfox__company_datafox_id__c,dfox__company_datafox_score__c,dfox__company_datafox_url__c,dfox__company_finance_score__c,dfox__company_growth_score__c,dfox__company_hr_score__c,dfox__company_influence_score__c,dfox__company_investors__c,dfox__company_last_founding_type__c,dfox__company_last_funding_amount__c,dfox__company_last_funding_date__c,dfox__company_last_funding_type__c,dfox__company_linkedin_url__c,dfox__company_long_description__c,dfox__company_name__c,dfox__company_number_of_employees__c,dfox__company_phone_number__c,dfox__company_private_funding__c,dfox__company_revenue_estimate__c,dfox__company_short_description__c,dfox__company_stage__c,dfox__company_state__c,dfox__company_stock_ticker__c,dfox__company_street_address__c,dfox__company_techstack__c,dfox__company_top_keywords__c,dfox__company_twitter_followers__c,dfox__company_twitter_handle__c,dfox__company_website__c,dfox__company_zip_code__c,dfox__id__c,dfox__last_synced_date__c,dfox__middle_name__c,dfox__sourced_from_datafox__c,brighttalk__user_id__c,community_score_engaging__c,account_owner_pipeline_pod__c,last_salesforce_campaign_status__c,cbit__clearbitready__c,cbit__clearbit__c,cbit__createdbyclearbit__c,cbit__facebook__c,cbit__linkedin__c,cbit__twitter__c,glid__c,xuid__c,community_score_learning__c,community_score_listening__c,community_score_overall__c,hubspot_sync_inclusion_list__c,hubspot_sfdc_sync_master__c,how_did_you_hear__c,how_did_you_hear_details__c,date_of_qualified_opportunity__c,date_last_contact_status_change__c,time_in_contact_status_days__c,slack_notification_text_backup__c,region_contact__c,qualified_ntc_opp__c,qualified_tc_opp__c,customer_mql__c,lead_notes__c,count_of_activities_last_90_days__c,abm_sourced_by__c,website_interest__c,sdr_owner_locked__c,sdr_owner__c,logged_in_user_sdr_owner__c,unique_contacts__c,count_of_activities_since_mql_date__c,hubspot_sync_testing__c,recent_campaign_history__c,website_interest_sales_multiselect__c,original_source_drill_down_1_del__c,description,pathfactory_last_topic_seen__c,account_owner_id__c,sdr_owner_s_manager_team__c,time );
START PIPELINE contact FOREGROUND;