kalipso/racalmas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite Export to Google Calendar

Browse Source 
In the past Google Calendar exports were done by first removing all
events of a day and then create new ones. This takes a lot of time for
export and runs into Google Calendar usage limits after some time.

By now content will be compared before removing/creating a single event
one. To be able to do so, all other sync sources and targets have been
removed, so its only possible to export from database to Google Calendar
by this change.

To trigger an export you need to create a trigger file.
run_jobs.pl runs periodically e.g. started by cron and checks if a
trigger file exists and start sync_cms.pl to export the selected events
to the Google Calendar. Trigger files and jobs are configured at
jobs.config. Each job has a source and target file containing the access
data for calcms and the calendar.

Configuration files have been cleaned up. Old Accounts and passwords have
been removed. They hopefully should have been not active for a long time ;-)
This commit is contained in:
Milan
2018-09-22 22:47:00 +02:00
parent 4134ffb278
commit 56881b92d0
34 changed files with 935 additions and 2649 deletions
Download Patch File Download Diff File Expand all files Collapse all files

671
tools/sync_cms/sync_cms.pl
View File

@@ -1,438 +1,236 @@
#!/usr/bin/perl
BEGIN{
my $dir='';
$ENV{SCRIPT_FILENAME}||'' if ($dir eq'');
$dir=~s/(.*\/)[^\/]+/$1/ if ($dir ne '');
$dir=$ENV{PWD} if ($dir eq'');
$dir=`pwd` if ($dir eq'');
#add calcms libs
unshift(@INC,$dir.'/../calcms/');
}
use strict;
use warnings;
use Data::Dumper;
use Getopt::Long;
use Config::General;
use time;
use DateTime;
use DateTime::Duration;
use strict;
use warnings;
use IO::Socket::INET;
use Fcntl ':flock';
check_running_processes();
use FindBin;
use lib "$FindBin::Bin/lib";
use lib "$FindBin::Bin/../../calcms";
my $read_mode='';
my $update_mode='';
my $all_events='';
my $modified_events='';
my $source_config_file='';
my $target_config_file='';
my $block_number=0;
my $block_size=2000;
our $from='';
our $till='';
my $read_only=0;
our $output_type='text';
our $debug=0;
use Common ( 'info', 'error' );
use GoogleCalendar;
use CalcmsEvents;
GetOptions(
"read" => \$read_mode,
"update" => \$update_mode,
"all" => \$all_events,
"modified" => \$modified_events,
"from=s" => \$from,
"till=s" => \$till,
"source=s" => \$source_config_file,
"target=s" => \$target_config_file,
"block_number:i" => \$block_number,
"block_size:i" => \$block_size,
"output_type=s" => \$output_type,
);
$|=1;
Common::checkSingleInstance();
BEGIN {
our $utf8dbi=1;
$ENV{LANG}="en_US.UTF-8";
$ENV{LANG} = "en_US.UTF-8";
}
$| = 1;
my $sourceConfigFile = '';
my $targetConfigFile = '';
my $debug = 1;
my $from = undef;
my $till = undef;
GetOptions(
"from=s" => \$from,
"till=s" => \$till,
"source=s" => \$sourceConfigFile,
"target=s" => \$targetConfigFile,
);
#source and taget settings are loaded from config files
our $settings={
};
my $settings = {};
#user interface
our $ask_before_insert=0;
our $ask_before_update=0;
# end of configuration
if ($update_mode){
$db::write=1;
# print_info("enter update mode");
}elsif($read_mode){
#default
$db::write=0;
# print_info("enter read-only mode");
}else{
print_error("set parameter >read< or >update<");
}
unless ($modified_events || $all_events || $from || $till){
print_error("set one of folling parameters: --modified, --from, --till");
}
error "set one of folling parameters: --from, --till" unless $from || $till;
init();
sync();
print_info("$0 done.");
info "$0 done.";
exit 0;
#sync all events, splitting multi-day-requests into multiple 1-day-requests to avoid large result sets
sub sync{
#prepare target
print_info("$0 inited");
print_info("last update: $settings->{source}->{last_update}");
sub sync {
my $timeZone = CalcmsEvents::get('date')->{time_zone};
my $from = CalcmsEvents::get('start_min');
my $till = CalcmsEvents::get('start_max');
if (my $days=source::split_request()){
#set 1-day start-min and start-max parameters, requires --from and --till values
for my $date (@$days){
for my $key(keys %$date){
$settings->{source}->{$key}=$date->{$key};
}
#print "\nrequest ".$settings->{source}->{"start_min"}." to ".$settings->{source}->{"start_max"}."\n";
sync_timespan();
}
}else{
#update without time span (e.g. --modified)
sync_timespan();
}
info "sync from $from till $till at $timeZone";
print_info("\nclean up old database entries...");
target::clean_up();
#prepare target
info "last update: " . ( CalcmsEvents::get('last_update') || '' );
print_info("\nset last-update time: $settings->{event}->{update_start}");
set_last_update_time($source_config_file,$target_config_file,$settings->{event}->{update_start});
if ( my $days = CalcmsEvents::splitRequest( $from, $till, $timeZone ) ) {
for my $date (@$days) {
syncTimespan( $date->{from}, $date->{till} );
}
} else {
syncTimespan( $from, $till );
}
info "\nset last-update time: $settings->{event}->{update_start}";
setLastUpdateTime( $sourceConfigFile, $targetConfigFile, $settings->{event}->{update_start} );
}
#sync all events of a given source timespan
sub sync_timespan{
#get a list of all days and their events
#print Dumper($settings->{source});
my $source_events=source::get_events($settings->{source},$settings->{target});
my @dates=(keys %$source_events);
sub syncTimespan {
my $from = shift;
my $till = shift;
#print "2\n";
if (@dates==0){
my $more='';
if ((defined $settings->{source}->{block_number}) && ($settings->{source}->{block_number} ne '0')){
$more='more ';
}elsif ($modified_events){
$more.='modified ';
}
print_info("\n".'no '.$more."entries found.");
}else{
print "<table>" if ($output_type eq 'html');
#sort lists of date and time (same time events should be preserved)
for my $date(sort {$a cmp $b} @dates){
# for my $date(@dates){
# print "\n$date:\n";
sync_events($source_events->{$date}, $settings);
}
print "</table>" if ($output_type eq 'html');
}
#get a list of all days and their events
my $sourceEvents = CalcmsEvents::getEvents( $from, $till );
my @dates = keys %$sourceEvents;
if ( scalar @dates == 0 ) {
info "\nno entries found.";
return;
}
#sort lists of date and time (same time events should be preserved)
for my $date ( sort { $a cmp $b } @dates ) {
syncEvents( $sourceEvents->{$date} );
}
}
#syncronize a list of source events to target events
sub sync_events{
my $source_events=shift;
my $settings=shift;
sub syncEvents($) {
my $sourceEvents = shift;
# my $source_settings =$settings->{source};
# my $target_settings =$settings->{target};
my $event_settings =$settings->{event};
my @sourceEvents = sort { $a->{calcms_start} cmp $b->{calcms_start} } @$sourceEvents;
$sourceEvents = \@sourceEvents;
my $start = $sourceEvents->[0]->{start};
my $end = $sourceEvents->[-1]->{end};
my $c=0;
$c=$source::settings->{start_index}+0 if (defined $source::settings->{start_index});
# print "<events>\n";
print html_table_header() if ($output_type eq 'html');
#order processing by start time (TODO: order by last-modified date)
for my $event (sort{$a->{calcms_start} cmp $b->{calcms_start}} @$source_events){
target::pre_sync({
start =>$source_events->[0]->{start},
end =>$source_events->[-1]->{end}
});
my $targetEvents = GoogleCalendar::getEvents(
{
start => $start,
end => $end
}
);
$targetEvents = $targetEvents->{items};
info "google:" . scalar(@$targetEvents) . " vs " . scalar(@$sourceEvents);
print "<tr><td>"if ($output_type eq 'html');
# mark all known target events
my $targetEventsByKey = {};
for my $event (@$targetEvents) {
#print Dumper($event);
next if $event->{status} eq 'canceled';
my $key = getGoogleEventToString($event);
$targetEventsByKey->{$key} = $event;
}
#read event
$event=source::get_event_attributes($event);
# mark all knwon source events
my $sourceEventsByKey = {};
for my $event (@$sourceEvents) {
$event = CalcmsEvents::mapToSchema($event);
$event = GoogleCalendar::mapToSchema($event);
my $key = getCalcmsEventToString($event);
$sourceEventsByKey->{$key} = $event;
}
#convert to calcms schema
$event=source::map_to_schema($event);
# delete target entries without matching source entries
for my $key ( keys %$targetEventsByKey ) {
next if defined $sourceEventsByKey->{$key};
my $event = $targetEventsByKey->{$key};
info "delete $key ";
print Dumper($event);
GoogleCalendar::deleteEvent($event);
}
#map event to target schema
$event=target::map_to_schema($event);
#deprecated: override defined attributes by configuration
if ((defined $source::settings->{override}) && (ref($source::settings->{override})eq 'HASH')){
for my $key (keys %{$source::settings->{override}}){
my $value=$source::settings->{override}->{$key};
if ($source::settings->{override} ne ''){
print_info("override '$key'='$value'");
$event->{event}->{$key}=$value;
}
}
}
if ($output_type eq'html'){
print_event_html("[".($c+1)."]",$event);
}else{
print_event_text("[".($c+1)."]",$event);
}
if ($event->{event}->{start} eq '' || $event->{event}->{end} eq ''){
print ('WARNING: Cannot read start or end of event');
print "\n";
}else{
# print Dumper($event);
sync_event($event);
}
# last;
$event=undef;
$c++;
print "</td></tr>"if ($output_type eq 'html');
}
# print "\n</events>\n";
# insert source entries without matching target entries
for my $key ( keys %$sourceEventsByKey ) {
if ( defined $targetEventsByKey->{$key} ) {
info "$key is up to date";
next;
}
my $event = $sourceEventsByKey->{$key};
info "insert $key";
GoogleCalendar::insertEvent($event);
}
}
#syncronize a single source event with target
sub sync_event{
my $event=shift;
sub getGoogleEventToString {
my $event = shift;
my $result = "\n";
$result .= "start: " . substr( $event->{start}->{dateTime}, 0, 19 ) . "\n";
$result .= "end : " . substr( $event->{end}->{dateTime}, 0, 19 ) . "\n";
$result .= "title: $event->{summary}\n";
$result .= "desc : $event->{description}\n";
return $result;
}
#look if target_event exists by reference id incl. recurrence counter
#print Dumper($event);
my $target_event=target::get_event_by_reference_id($event->{event}->{reference});
#if target_event exists
if (defined $target_event){
#delete canceled events
if ($event->{event}->{status}eq'canceled'){
print cell("delete canceled event:".qq{$target_event});
# target::delete($target_event->{id});
return;
}
$event->{event_id}=$target_event->{id};
target::update_event($event,$target_event);
print cell("(ref. update)");
}else{
#find by date, time and title
$target_event=target::find_event($event);
if (defined $target_event){
target::update_event($event,$target_event);
#print Dumper($event);
$event->{event_id}=$target_event->{id};
print cell("(update)");
}else{
target::insert_event($event);
#print Dumper($event);
$target_event=target::get_event_by_reference_id($event->{event}->{reference});
#print Dumper($target_event);
$event->{event_id}=$target_event->{id};
print cell("(new)");
}
}
print "\n";
for my $category (@{$event->{categories}}){
target::assign_category_to_event($category,$event);
}
for my $meta (@{$event->{meta}}){
target::assign_meta_to_event($meta,$event);
}
# print Dumper($event);
sub getCalcmsEventToString {
my $event = shift;
my $result = "\n";
$result .= "start: " . substr( $event->{event}->{start_datetime}, 0, 19 ) . "\n";
$result .= "end : " . substr( $event->{event}->{end_datetime}, 0, 19 ) . "\n";
$result .= "title: $event->{event}->{title}\n";
$result .= "desc : $event->{event}->{content}\n";
return $result;
}
#import requested source and target libs
sub init{
binmode STDOUT, ":utf8";
sub init {
binmode STDOUT, ":utf8";
#require source config file
print_error ("missing source parameter!") unless ($source_config_file=~/\S/);
print_error ("source file: '$source_config_file' does not exist") unless (-e $source_config_file);
print_error ("cannot read source file: '$source_config_file'") unless (-r $source_config_file);
#$settings->{source}=require $source_config_file;
my $configuration = new Config::General($source_config_file);
$settings->{source}=$configuration->{DefaultConfig}->{source};
{
#require target config file
error "missing target parameter!" unless $targetConfigFile =~ /\S/;
error "target file: '$targetConfigFile' does not exist" unless -e $targetConfigFile;
error "cannot read target file: '$targetConfigFile'" unless -r $targetConfigFile;
my $config = new Config::General($targetConfigFile);
$config = $config->{DefaultConfig}->{target};
GoogleCalendar::init($config);
#require source import lib from config file
my $source_import_lib='lib/source/'.$settings->{source}->{type}.'.pl';
print_error ("missing 'type' in 'source' config ") unless ($settings->{source}->{type}=~/\S/);
print_error ("cannot read source type import lib: '$source_import_lib'")unless (-r $source_import_lib);
require $source_import_lib;
#require target config file
print_error ("missing target parameter!") unless ($target_config_file=~/\S/);
print_error ("target file: '$target_config_file' does not exist") unless (-e $target_config_file);
print_error ("cannot read target file: '$target_config_file'") unless (-r $target_config_file);
$configuration = new Config::General($target_config_file);
$settings->{target}=$configuration->{DefaultConfig}->{target};
#$settings->{target}=require $target_config_file;
#require target import lib from config file
my $target_import_lib='lib/target/'.$settings->{target}->{type}.'.pl';
print_error ("missing 'type' in 'target' config ") unless ($settings->{target}->{type}=~/\S/);
print_error ("cannot read target type import lib: '$target_import_lib'")unless (-r $target_import_lib);
require $target_import_lib;
#print Dumper($settings);
if ((defined $settings->{source}->{read_blocks}) && ($settings->{source}->{read_blocks}==1)){
$settings->{source}->{block_number} =$block_number;
$settings->{source}->{block_size} =$block_size;
}
$settings->{source}->{last_update} =get_last_update_time($source_config_file,$target_config_file);
$settings->{source}->{modified_events} =$modified_events;
if ($from=~/^\d\d\d\d\-\d\d\-\d\d$/){
$from.='T00:00';
# print "from:$from\t";
}
if ($till=~/^\d\d\d\d\-\d\d\-\d\d$/){
$till.='T23:59';
# print "till:$till\t";
}
if ($from=~/^([-+]?\d+$)/){
my $days=$1;
my $duration=new DateTime::Duration(days=>$days);
$from=DateTime->today->add_duration($duration);
# print "from:$from\t";
}
if ($till=~/^([-+]?\d+$)/){
my $days=$1+1;
my $duration=new DateTime::Duration(days=>$days);
$till=DateTime->today->add_duration($duration);
# print "till:$till\t";
}
$settings->{source}->{start_min} =$from if defined ($from);
$settings->{source}->{start_max} =$till if defined ($till);
my $gmt_difference =0;#*=3600;
my $now =time();
my $now_gmt =$now-$gmt_difference;
$now =time::time_to_datetime($now);
$now_gmt =time::time_to_datetime($now_gmt);
$settings->{event}={
update_start => time::time_to_datetime(time()),
modified_at => $now,
modified_at_gmt => $now_gmt
};
source::init($settings->{source});
target::init($settings->{target});
}
# print date/time, title and excerpt of an calendar event
# TODO: replace by output filter (text, html)
sub print_event_text{
my $header=shift;
my $event=shift;
my $s=$header;
$s=$s." "x (8-length($s));
my $start=$event->{event}->{start}||'';
$start=~s/T/ /g;
$start=~s/\:00$//g;
if (defined $event->{event}->{program}){
$s.="$start $event->{event}->{program}";
$s=$s." "x (45-length($s));
}
if (defined $event->{event}->{series_name}){
$s.=" : $event->{event}->{series_name}";
$s=$s." "x (75-length($s));
{
#require source config file
error "missing source parameter!" unless $sourceConfigFile =~ /\S/;
error "source file: '$sourceConfigFile' does not exist" unless -e $sourceConfigFile;
error "cannot read source file: '$sourceConfigFile'" unless -r $sourceConfigFile;
my $config = new Config::General($sourceConfigFile);
$config = $config->{DefaultConfig}->{source};
$config->{last_update} = getLastUpdateTime( $sourceConfigFile, $targetConfigFile );
CalcmsEvents::init($config);
}
if (defined $event->{event}->{title}){
$s.=" - $event->{event}->{title}";
$s=$s." "x (110-length($s));
$from .= 'T00:00' if $from =~ /^\d\d\d\d\-\d\d\-\d\d$/;
$till .= 'T23:59' if $till =~ /^\d\d\d\d\-\d\d\-\d\d$/;
if ( $from =~ /^([-+]?\d+$)/ ) {
my $days = $1;
my $duration = new DateTime::Duration( days => $days );
$from = DateTime->today->add_duration($duration);
}
if ($event->{categories}){
$s.= "(".join(", ",(@{$event->{categories}})).")";
}
$s=$s." "x (135-length($s));
if ( $till =~ /^([-+]?\d+$)/ ) {
my $days = $1 + 1;
my $duration = new DateTime::Duration( days => $days );
$till = DateTime->today->add_duration($duration);
}
my $status=$event->{event}->{status};
$s.=$status.' ' if (defined $status);
$s=$s." "x (140-length($s));
CalcmsEvents::set( 'start_min', $from ) if defined $from;
CalcmsEvents::set( 'start_max', $till ) if defined $till;
my $reference=$event->{event}->{reference};
$s.=substr($reference,length($reference)-25) if (defined $reference);
my $now = time();
$now = time::time_to_datetime($now);
$settings->{event} = {
update_start => time::time_to_datetime( time() ),
modified_at => $now,
};
print $s;
}
sub print_event_html{
my $header=shift;
my $event=shift;
#close error block
my $s='</td>';
my $start=$event->{event}->{start}||'';
$start=~s/T/ /g;
$start=~s/\:00$//g;
$s.=cell($start);
$s.=cell($event->{event}->{program});
$s.=cell($event->{event}->{series_name});
$s.=cell($event->{event}->{title});
if ($event->{categories}){
$s.=cell( join(", " , ( @{$event->{categories}} ) ) );
}
my $status=$event->{event}->{status};
$s.=cell($status) if (defined $status);
my $reference=$event->{event}->{reference};
$reference=substr($reference,length($reference)-25) if (defined $reference);
$s.=cell($reference);
$s.="<td>";
print $s;
}
sub cell{
if ($output_type eq 'html'){
return "<td>$_[0]</td>";
}else{
return "\t".$_[0];
};
}
#output usage on error or --help parameter
sub print_usage{
print qq{
sub usage {
print qq{
update all/modified events from source at target.
USAGE: sync_cms.pl [--read,--update] [--modified,--all] --source s --target t [--block_number b] [--block_size s]
USAGE: sync_cms.pl [--read,--update] [--modified,--all] --source s --target t
on using --from and --till requests will be processed as multiple single-day-requests.
@@ -440,128 +238,55 @@ parameters:
--read show all events without updating database
--update update target database with source events
--modified process only modified events.
--all' process all events
--source source configuration file
--target target configuration file
--from start of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow)
--till end of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow)
--output_type log output format [text,html]
--block_number which block is to be syncronized [0..n]. To split up processing into multiple blocks (for machines with small memory resources).
--block_size size of a block, default=20 events
examples:
update modified
perl sync_cms.pl --update --modified --source=config/source/program.cfg --target=config/target/calcms.cfg
perl sync_cms.pl --update --source=config/source/program.cfg --target=config/target/calcms.cfg
update a given time range
perl sync_cms.pl --update --all --from=2009-09-01T00:00:00 --till=2009-11-22T23:59:59 --source=config/source/program.cfg --target=config/target/calcms.cfg
update from last 2 days until next 3 days
perl sync_cms.pl --update --all --from=-2 --till=+3 --source=config/source/program.cfg --target=config/target/calcms.cfg
perl sync_cms.pl --update --from=-2 --till=+3 --source=config/source/program.cfg --target=config/target/calcms.cfg
};
exit 1;
};
#default error handling
sub print_error{
print "\nERROR: $_[0]\n" ;
print_usage();
exit 1;
}
sub print_info{
my $message=shift;
if ($message=~/^\n/){
$message=~s/^\n//g;
print "\n";
}
if ($output_type eq 'html'){
print "$message<br/>";
}else{
print "INFO:\t$message\n";
}
}
sub html_table_header{
return qq{
<tr>
<th> </th>
<th>start date</th>
<th>project</th>
<th>series</th>
<th>title</th>
<th>category</th>
<th>status</th>
<th>id</th>
<th> </th>
<th>action</th>
</tr>
};
};
#load last update time out of sync.data
sub get_last_update_time{
my $source=shift;
my $target=shift;
sub getLastUpdateTime {
my $source = shift;
my $target = shift;
my $date=undef;
return undef unless(-r "sync.data");
open my $DATA, "<:utf8","sync.data" || die ('cannot read update timestamp');
while (<$DATA>){
my $line=$_;
if ($line=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){
$date=$1;
last;
}
}
close $DATA;
return $date;
my $date = undef;
return undef unless -r "sync.data";
my $content = Common::loadFile("sync.data");
if ( $content =~ /$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/ ) {
$date = $1;
}
return $date;
}
#save last update time to sync.data
sub set_last_update_time{
my $source =shift;
my $target =shift;
my $date =shift;
sub setLastUpdateTime {
my $source = shift;
my $target = shift;
my $date = shift;
my $data='';
if (-r "sync.data"){
open my $DATA, "<:utf8","sync.data";
$data=join("\n",(<$DATA>));
close $DATA;
}
my $data = '';
if ( -r "sync.data" ) {
$data = Common::loadFile("sync.data");
}
if ($data=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){
$data=~s/($source\s+\->\s+$target\s+:)\s+\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}/$1\t$date/gi;
}else{
$data.="$source\t\->\t$target\t:\t$date\n";
}
if ( $data =~ /$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/ ) {
$data =~ s/($source\s+\->\s+$target\s+:)\s+\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}/$1\t$date/gi;
} else {
$data .= "$source\t\->\t$target\t:\t$date\n";
}
$data=~s/[\r\n]+/\n/g;
open my $DATA2, ">:utf8","sync.data" || die ('cannot write update timestamp');
print $DATA2 $data;
close $DATA2;
# print $data;
$data =~ s/[\r\n]+/\n/g;
Common::saveFile( "sync.data", $data );
}
#avoid to run more than one sync process in parallel
sub check_running_processes{
my $cmd="ps -afex 2>/dev/null | grep sync_cms.pl | grep -v nice | grep -v grep ";
my $ps=`$cmd`;
# print "$ps";
my @lines=(split(/\n/,$ps));
if (@lines>1){
print "ERROR: another ".@lines." synchronization processes 'sync_cms.pl' instances are running!".qq{
$cmd
$ps
-> program will exit
};
exit;
}
}