diff --git a/tools/sync_cms/INSTALL b/tools/sync_cms/INSTALL deleted file mode 100755 index ce7bea7..0000000 --- a/tools/sync_cms/INSTALL +++ /dev/null @@ -1,90 +0,0 @@ -#install libxml2, libxml2-dev (for headers) to use XML::Atom (required by Net::Google::Calendar) -#install perl modules: DateTime, DateTime::TimeZone, XML::Atom, XML::Atom::Feed, Net::Google::Calendar -#if reading calendar fails, patch Entry line 184, from -# if ($elem->hasAttribute($key)) { -# to -# if (defined $elem && $elem->hasAttribute($key)) { -# -#patch Entry before line 184, insert -# return unless ($tmp); - - -#patch Entry line 176, modify - $val =~ s!^http://schemas.google.com/g/2005#event\.!! if (defined $val); - -#admin,admin - - #all available google calendar definitions, replace in url 'basic' by 'full' to get calendar entries!!! (basic covers feed content only, but no calendar data...) -# google_calendars => { -# programm => 'http://www.google.com/calendar/feeds/58ei894fakpf84hj0u7o6el4sc%40group.calendar.google.com/public/full', -# programm_intern => 'http://www.google.com/calendar/feeds/lin4mscfdld2eiv22qda82t478%40group.calendar.google.com/public/full', -# planung => 'http://www.google.com/calendar/feeds/0is4ruq5thsb6ndsqr5gicff2k%40group.calendar.google.com/public/full', -# termine_intern => 'http://www.google.com/calendar/feeds/1n762hqutnsocd46h6nji3i2l4%40group.calendar.google.com/public/full', -# termine => 'http://www.google.com/calendar/feeds/f29rqfutlkub911i8u0eerusb0%40group.calendar.google.com/public/full' -# }, - -GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, FILE, INDEX, ALTER ON * . * TO 'root'@'localhost' IDENTIFIED BY 'calcms' WITH MAX_QUERIES_PER_HOUR 0 MAX_CONNECTIONS_PER_HOUR 0 MAX_UPDATES_PER_HOUR 0 MAX_USER_CONNECTIONS 0; - -Query OK, 0 rows affected (0.02 sec) - -flush previleges -https://www.google.com/calendar/dav/peter_retep@gmx.de/events - - -create database calcms-herbstradio; -mysql calcms_herbstradio -u root -p < calcms/calcms.sql - - -GRANT SELECT ON * . * TO 'root'@'localhost' IDENTIFIED BY 'calcms-agenda' WITH MAX_QUERIES_PER_HOUR 0 MAX_CONNECTIONS_PER_HOUR 0 MAX_UPDATES_PER_HOUR 0 MAX_USER_CONNECTIONS 0; - - -flush previleges - - -CREATE USER 'milan'@'localhost' IDENTIFIED BY 'eeGei3Yo'; -GRANT SELECT ON calcms_agenda.* TO 'calcms'@'localhost'; - -chmod 777 cache - -GRANT SELECT ON calcms_herbstradio.* TO 'calcms_agenda'@'localhost' IDENTIFIED BY 'eeGei3Yo' -GRANT SELECT, INSERT, UPDATE ON calcms_herbstradio.* TO 'calcms'@'localhost' IDENTIFIED BY 'CheiBai8' - - -ERROR: Can't create '/usr/local/lib/perl5/5.8.6/man/man3' -Do not have write permissions on '/usr/local/lib/perl5/5.8.6/man/man3' -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - at /home/milan/perl/lib/Module/Build/Base.pm line 2975 - SIMONW/Net-Google-Calendar-0.97.tar.gz - ./Build install -- NOT OK ----- - You may have to su to root to install the package - (Or you may want to run something like - o conf make_install_make_command 'sudo make' - to raise your permissions.Failed during this command: - DROLSKY/DateTime-Locale-0.43.tar.gz : install NO - DROLSKY/DateTime-TimeZone-0.91.tar.gz : install NO - DROLSKY/DateTime-0.50.tar.gz : make_test NO - SIMONW/Net-Google-AuthSub-0.5.tar.gz : install NO - GRANTM/XML-SAX-0.96.tar.gz : make_test NO - SIMONW/Net-Google-Calendar-0.97.tar.gz : install NO - -See perldoc ExtUtils::MakeMaker for full details. For Module::Build -modules, you need to create a ~/.modulebuildrc file containing -bindoc=~/man/man1 libdoc=~/man/man3 - -o conf makepl_arg "PREFIX=/home/milan/perl/ LIB=/home/milan/perl/lib INST_LIB=/home/milan/perl/lib INSTALLSITELIB=/home/milan/perl/lib INSTALLMAN1DIR=/home/milan/perl/man/man1 INSTALLSITEMAN1DIR=/home/milan/perl/man/man1 INSTALLMAN3DIR=~/home/milan/perl/man/man3 INSTALLSITEMAN3DIR=/home/milan/perl/man/man3 INSTALLDIRS=/home/milan/perl/ SITEPREFIX=/home/milan/perl/ VENDORPREFIX=/home/milan/perl/" - -LIB=$PREFIX/lib INST_LIB=$PREFIX/lib PREFIX=$PREFIX SITEPREFIX=$PREFIX VENDORPREFIX=$PREFIX - - - - -o conf make_arg -I/home/twiki/lib/CPAN -o conf make_install_arg -I/home/twiki/lib/CPAN -o conf makepl_arg "install_base=/home/twiki/lib/CPAN LIB=/home/twiki/lib/CPAN/lib INSTALLPRIVLIB=/home/twiki/lib/CPAN/lib INSTALLARCHLIB=/home/twiki/lib/CPAN/lib/arch INSTALLSITEARCH=/home/twiki/lib/CPAN/lib/arch INSTALLSITELIB=/home/twiki/lib/CPAN/lib INSTALLSCRIPT=/home/twiki/lib/CPAN/bin INSTALLBIN=/home/twiki/lib/CPAN/bin INSTALLSITEBIN=/home/twiki/lib/CPAN/bin INSTALLMAN1DIR=/home/twiki/lib/CPAN/man/man1 INSTALLSITEMAN1DIR=/home/twiki/lib/CPAN/man/man1 INSTALLMAN3DIR=/home/twiki/lib/CPAN/man/man3 INSTALLSITEMAN3DIR=/home/twiki/lib/CPAN/man/man3 " -o conf commit -q - - - - diff --git a/tools/sync_cms/config/jobs.config b/tools/sync_cms/config/jobs.config new file mode 100644 index 0000000..110b3c5 --- /dev/null +++ b/tools/sync_cms/config/jobs.config @@ -0,0 +1,18 @@ + + +start_dir /home/calcms/agenda/admin/jobs/start/ +log_dir /home/calcms/agenda/admin/jobs/logs/ + + + title potsdam : sender berlin -> 88vier.de + name potsdam_to_88vier.de + command /home/radio/calcms/sync_cms/sync_jobs/calcms_to_google.sh -7 31 potsdam + + + + title frb : sender berlin -> 88vier.de + name frb_to_88vier.de + command /home/radio/calcms/sync_cms/sync_jobs/calcms_to_google.sh -7 31 frb + + + diff --git a/tools/sync_cms/config/source/calcms_ansage.cfg b/tools/sync_cms/config/source/calcms_ansage.cfg index a404f68..3083ea4 100644 --- a/tools/sync_cms/config/source/calcms_ansage.cfg +++ b/tools/sync_cms/config/source/calcms_ansage.cfg @@ -1,12 +1,10 @@ - type calcms_i - hostname localhost port 3306 - database calcms_herbstradio - username calcms - password CheiBai8 + database calcms + username calcms_read + password password @@ -19,7 +17,7 @@ name 88vier title 88vier Studio Ansage start_date 2010-05-01 - end_date 2016-06-01 + end_date 2020-06-01 location ansage diff --git a/tools/sync_cms/config/source/calcms_colabo.cfg b/tools/sync_cms/config/source/calcms_colabo.cfg index 06e2018..4be6dcd 100644 --- a/tools/sync_cms/config/source/calcms_colabo.cfg +++ b/tools/sync_cms/config/source/calcms_colabo.cfg @@ -1,12 +1,10 @@ - type calcms_i - hostname localhost port 3306 - database calcms_herbstradio - username calcms - password CheiBai8 + database calcms + username calcms_read + password password @@ -19,7 +17,7 @@ name 88vier title 88vier Colaboradio start_date 2010-05-01 - end_date 2016-06-01 + end_date 2020-06-01 location colabo diff --git a/tools/sync_cms/config/source/calcms.cfg b/tools/sync_cms/config/source/calcms_frb.cfg similarity index 54% rename from tools/sync_cms/config/source/calcms.cfg rename to tools/sync_cms/config/source/calcms_frb.cfg index 2540478..5d89ead 100644 --- a/tools/sync_cms/config/source/calcms.cfg +++ b/tools/sync_cms/config/source/calcms_frb.cfg @@ -1,12 +1,10 @@ - type calcms_i - hostname localhost port 3306 - database calcms_herbstradio - username calcms - password CheiBai8 + database calcms + username calcms_read + password password @@ -17,15 +15,15 @@ <88vier> name 88vier - title 88vier PI-Radio 2010 + title 88vier FRB start_date 2010-05-01 - end_date 2013-08-31 + end_date 2020-06-01 - location piradio + location frb - event_details_url http://piradio.de/programm/sendung/.html + event_details_url http://senderberlin.org/programm/sendung/.html diff --git a/tools/sync_cms/config/source/calcms_piradio.cfg b/tools/sync_cms/config/source/calcms_piradio.cfg index 7a391c5..1f689ed 100644 --- a/tools/sync_cms/config/source/calcms_piradio.cfg +++ b/tools/sync_cms/config/source/calcms_piradio.cfg @@ -1,12 +1,10 @@ - type calcms_i - hostname localhost port 3306 - database calcms_herbstradio - username calcms - password CheiBai8 + database calcms + username calcms_read + password password @@ -19,7 +17,7 @@ name 88vier title 88vier PI-Radio start_date 2010-05-01 - end_date 2016-06-01 + end_date 2020-06-01 location piradio diff --git a/tools/sync_cms/config/source/calcms_potsdam.cfg b/tools/sync_cms/config/source/calcms_potsdam.cfg index d2452c4..28d49c1 100644 --- a/tools/sync_cms/config/source/calcms_potsdam.cfg +++ b/tools/sync_cms/config/source/calcms_potsdam.cfg @@ -1,12 +1,10 @@ - type calcms_i - hostname localhost port 3306 - database calcms_herbstradio - username calcms - password CheiBai8 + database calcms + username calcms_read + password password @@ -19,7 +17,7 @@ name 88vier title 88vier Frrapo start_date 2010-05-01 - end_date 2016-06-01 + end_date 2020-06-01 location potsdam diff --git a/tools/sync_cms/config/target/88vier_ansage.cfg b/tools/sync_cms/config/target/88vier_ansage.cfg index 67be6dc..bb7991e 100644 --- a/tools/sync_cms/config/target/88vier_ansage.cfg +++ b/tools/sync_cms/config/target/88vier_ansage.cfg @@ -1,10 +1,8 @@ - type google_calendar2 - - calendarId info@studioansage.de - serviceAccount 433089473368-bv26eveq03b7nhb9p62nu3ts7htgb4g3@developer.gserviceaccount.com - serviceAccountKeyFile /home/radio/googleApi.key + calendarId your-id + serviceAccount your-accounr@developer.gserviceaccount.com + serviceAccountKeyFile googleApi.key diff --git a/tools/sync_cms/config/target/88vier_colabo.cfg b/tools/sync_cms/config/target/88vier_colabo.cfg index 42df591..fbd2c13 100644 --- a/tools/sync_cms/config/target/88vier_colabo.cfg +++ b/tools/sync_cms/config/target/88vier_colabo.cfg @@ -1,10 +1,8 @@ - type google_calendar2 - - calendarId colaboradio@gmail.com - serviceAccount 433089473368-bv26eveq03b7nhb9p62nu3ts7htgb4g3@developer.gserviceaccount.com - serviceAccountKeyFile /home/radio/googleApi.key + calendarId your-calendar-id + serviceAccount your-account@developer.gserviceaccount.com + serviceAccountKeyFile googleApi.key diff --git a/tools/sync_cms/config/target/88vier_frb.cfg b/tools/sync_cms/config/target/88vier_frb.cfg new file mode 100644 index 0000000..a02aa9d --- /dev/null +++ b/tools/sync_cms/config/target/88vier_frb.cfg @@ -0,0 +1,21 @@ + + + calendarId your-id@group.calendar.google.com + serviceAccount your-account@developer.gserviceaccount.com + serviceAccountKeyFile googleApi.key + + + + time_zone Europe/Berlin + + + + title : - + content mehr zur Sendung + + + + debug 1 + + + diff --git a/tools/sync_cms/config/target/88vier_piradio.cfg b/tools/sync_cms/config/target/88vier_piradio.cfg index 7cc1aa7..8359efb 100644 --- a/tools/sync_cms/config/target/88vier_piradio.cfg +++ b/tools/sync_cms/config/target/88vier_piradio.cfg @@ -1,10 +1,8 @@ - type google_calendar2 - - calendarId li6if8drs373kf9ttot7er6suc@group.calendar.google.com - serviceAccount 433089473368-bv26eveq03b7nhb9p62nu3ts7htgb4g3@developer.gserviceaccount.com - serviceAccountKeyFile /home/radio/googleApi.key + calendarId your-id@group.calendar.google.com + serviceAccount your-account@developer.gserviceaccount.com + serviceAccountKeyFile googleApi.key diff --git a/tools/sync_cms/export_db.pl b/tools/sync_cms/export_db.pl deleted file mode 100644 index 79c3016..0000000 --- a/tools/sync_cms/export_db.pl +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/perl - -use strict; -use warnings; -use lib "../calcms"; -use utf8; - -use Data::Dumper; -use Config::General; -use Storable qw(nstore); - -use db; -use config; - -our $default={ - configFile => '/home/radio/piradio.de/agenda/config/config.cgi', - timezone => 'Europe/Berlin', - local_media_url => 'http://piradio.de/agenda_files/media/', - project => '88vier', - location => 'piradio', -}; - -my $config = config::get($default->{configFile}); -print Dumper($config); - -my $dbh=db::connect($config); -my $query=q{ - select * from calcms_events - order by start -}; - -my $events=db::get($dbh, $query); -nstore($events, 'event_export.dat'); diff --git a/tools/sync_cms/ical_html_to_ical_creole.pl b/tools/sync_cms/ical_html_to_ical_creole.pl deleted file mode 100644 index a512d49..0000000 --- a/tools/sync_cms/ical_html_to_ical_creole.pl +++ /dev/null @@ -1,17 +0,0 @@ -require '../lib/text_markup.pl'; - -open FILE,"<$ARGV[0]"; -while (){ - my $line=$_; - if ($line=~/^DESCRIPTION:/){ - my $description=substr($line,length('DESCRIPTION:')); - my $html=markup::ical_to_plain($description); - my $creole=markup::html_to_creole($html); - my $ical=markup::plain_to_ical($creole); - $line= 'DESCRIPTION:'.$ical."\n"; - } - print $line; -} -close FILE; - - diff --git a/tools/sync_cms/import_ical.pl b/tools/sync_cms/import_ical.pl deleted file mode 100755 index ec25c76..0000000 --- a/tools/sync_cms/import_ical.pl +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/perl - -use strict; -use warnings; -use lib "../calcms"; -use utf8; - -use DateTime; -use Net::Google::Calendar; -use DateTime::Format::ICal; -use Data::Dumper; -use Config::General; - -use db; -use config; -use creole_wiki; -use markup; -use events; - -my $filename=$ARGV[0]; -die("USAGE: $0 filename") unless defined $filename; -die("cannot read from '$filename'") unless -e $filename; - -our $default={ - configFile => '/home/radio/piradio.de/agenda/config/config.cgi', - timezone => 'Europe/Berlin', - local_media_url => 'http://piradio.de/agenda_files/media/', - project => '88vier', - location => 'piradio', -}; - -my $config = config::get($default->{configFile}); -print Dumper($config); - -parseICalFile($config, $filename); -our $active=0; -sub parseICalFile{ - my $config=shift; - my $filename=shift; - - print "open $filename\n"; - open my $file, "<:encoding(UTF-8)", $filename; - my $parse=0; - my $event=undef; - my $lastKey=undef; - - while (<$file>){ - my $line=$_; - #print $parse." ".$line; - if ($line=~/^BEGIN\:VEVENT/){ - $event={}; - $parse=1; - #print "start event\n"; - next; - } - if ($line=~/^END\:VEVENT/){ - $parse=0; - processEvent($config, $event) if defined $event; - #print "end event\n"; - next; - } - if ($line=~/^\s/){ - my $key = $lastKey; - my $value = substr($line, 1); - $value=~s/[\r\n]+$//; - $event->{$key}.=$value; - $lastKey=$key; - next; - }else{ - my ($key,$value)=split(/\:/,$line,2); - $value=~s/[\r\n]+$//; - $event->{$key}=$value; - $lastKey=$key; - } - - } - close $file; -} - -sub processEvent{ - my $config=shift; - my $source=shift; - - my $event={}; - - $event->{title} = $source->{SUMMARY}; - $event->{content} = $source->{DESCRIPTION}; - $event->{title} = markup::ical_to_plain($event->{title}); - $event->{content} = markup::ical_to_plain($event->{content}); - - unless (defined $source->{DTSTART}){ - print STDERR "missing DTSTART in ".Dumper($source); - return; - } - unless (defined $source->{DTEND}){ - print STDERR "missing DTEND in ".Dumper($source); - return; - } - my $start = DateTime::Format::ICal->parse_datetime($source->{DTSTART}); - $start=$start->set_time_zone($default->{timezone}); - $event->{start} = $start->datetime(); - - my $end = DateTime::Format::ICal->parse_datetime($source->{DTEND}); - $end = $end->set_time_zone($default->{timezone}); - $event->{end} = $end->datetime(); - - my $params={ - title => $event->{title}, - content => $event->{content}, - local_media_url => $default->{local_media_url} - }; - - #$params->{content}=~s/\x0A\x20/\n/g; - $event=creole_wiki::extractEventFromWikiText($params, $event); - - $event->{project} = $default->{project}; - $event->{location} = $default->{location}; - - return unless ($event->{start} ge '2015-09-01'); - - $active=1 if ($event->{series_name}=~/Brainwashed/); - print "$active $event->{start} $event->{series_name} - $event->{title}\n"; - #saveEvent($config, $event); - #exit; -} - -sub saveEvent{ - my $config = shift; - my $event = shift; - - $config->{access}->{write}=1; - my $dbh=db::connect($config); - - $event->{'html_content'}=markup::creole_to_html($event->{'content'}); - - # set start date - my $day_start=$config->{date}->{day_starting_hour}; - $event->{start_date} = time::add_hours_to_datetime($event->{start}, -$day_start); - $event->{start_date} = time::datetime_to_date($event->{start_date}); - - # set end date - $event->{end_date} = time::add_hours_to_datetime($event->{end}, -$day_start); - $event->{end_date} = time::datetime_to_date($event->{end_date}); - - delete $event->{categories} if defined $event->{categories}; - - # set time of day - my $day_times=$config->{date}->{time_of_day}; - my $event_hour=int((split(/[\-\:\sT]/,$event->{start}))[3]); - for my $hour(sort {$a <=> $b} (keys %$day_times)){ - if ($event_hour >= $hour){ - $event->{time_of_day}=$day_times->{$hour}; - }else{ - last; - }; - } - $event->{published}=0; - $event->{modified_by}='sync_cms'; - print Dumper($event); - #db::insert($dbh,'calcms_events', $event); -} - diff --git a/tools/sync_cms/lib/CalcmsEvents.pm b/tools/sync_cms/lib/CalcmsEvents.pm new file mode 100644 index 0000000..9f9d45e --- /dev/null +++ b/tools/sync_cms/lib/CalcmsEvents.pm @@ -0,0 +1,135 @@ +package CalcmsEvents; + +use strict; +use warnings; + +use Common ('info','error'); +use DateTime; +use Data::Dumper; + +use creole_wiki; +use events; +use time; +#use config; + +my $settings = {}; + +sub init($) { + $settings = shift || {}; +} + +sub set($$) { + my $key = shift; + my $value = shift; + $settings->{$key} = $value; +} + +sub get($) { + my $key = shift; + return $settings->{$key}; +} + +# return a list of start_min, start_max request parameters. +sub splitRequest($$$) { + my $from = shift; + my $till = shift; + my $timeZone = shift; + + return undef unless defined $from; + return undef unless defined $till; + return undef if $from eq ''; + return undef if $till eq ''; + + my $dates = []; + + my $start = time::get_datetime( $from, $timeZone ); + my $end = time::get_datetime( $till, $timeZone ); + + #build a list of dates + my $date = $start; + my @dates = (); + while ( $date < $end ) { + push @dates, $date; + $date = $date->clone->add( days => 7 ); + } + my $duration = $end - $date; + + push @dates, $end->clone if $duration->delta_seconds <= 0; + + #build a list of parameters from dates + $start = shift @dates; + for my $end (@dates) { + push @$dates, + { + from => $start, + till => $end + }; + $start = $end; + } + + return $dates; + +} + +#get a hash with per-day-lists days of a google calendar, given by its url defined at $calendar_name +sub getEvents($$) { + my $from = shift; + my $till = shift; + + my $last_update = get('last_update'); + info "getEvents from $from till $till"; + + my $request_parameters = { + from_date => $from, + till_date => $till, + project => get('project'), + archive => 'all', + template => 'no' + }; + my $location = get('location') || ''; + $request_parameters->{location} = $location if $location ne ''; + + my $config = $settings; + my %params = (); + my $request = { + url => $ENV{QUERY_STRING}, + params => { + original => \%params, + checked => events::check_params( $config, $request_parameters, $settings ), + }, + }; + + my $sourceEvents = events::get( $config, $request, $settings ); + + #return events by date + my $eventsByDate = {}; + for my $source (@$sourceEvents) { + $source->{calcms_start} = $source->{start}; + my $key = substr( $source->{start}, 0, 10 ); + push @{ $eventsByDate->{$key} }, $source; + } + return $eventsByDate; +} + +sub mapToSchema { + my $event = shift; + + #override settings by source map filter + for my $key ( keys %{ get('mapping') } ) { + $event->{$key} = get('mapping')->{$key}; + } + + #resolve variables set in mapped values + for my $mkey ( keys %{ get('mapping') } ) { + for my $key ( keys %{$event} ) { + my $val = $event->{$key}; + $val = $event->{$key} if ( $mkey eq $key ); + $event->{$mkey} =~ s//$val/g; + } + } + + return $event; +} + +#do not delete last line +1; diff --git a/tools/sync_cms/lib/Common.pm b/tools/sync_cms/lib/Common.pm new file mode 100644 index 0000000..90b78ca --- /dev/null +++ b/tools/sync_cms/lib/Common.pm @@ -0,0 +1,75 @@ +package Common; +use warnings; +use strict; + +use Fcntl ':flock'; + +use base 'Exporter'; +our @EXPORT_OK = ( 'info', 'error' ); + +sub checkSingleInstance() { + open my $self, '<', $0 or die "Couldn't open self: $!"; + flock $self, LOCK_EX | LOCK_NB or die "This script $0 is already running"; +} + +sub loadFile($) { + my $filename = shift; + + my $content = ''; + open my $file, '<', $filename || die("cannot load $filename"); + while (<$file>) { + $content .= $_; + } + close $file; + return $content; +} + +sub saveFile($$) { + my $filename = shift; + my $content = shift; + open my $file, ">:utf8", $filename || die("cannot write $filename"); + print $file $content; + close $file; + +} + +sub getModifiedAt { + my $file = shift; + my @stats = stat $file; + return 0 if scalar @stats == 0; + my $modifiedAt = $stats[9]; + return $modifiedAt; +} + +sub execute($) { + my $command = shift; + print "EXEC:\t$command\n"; + my $result = `$command`; + my $exitCode = ( $? >> 8 ); + print "ERROR! exitCode=$?\n" if $exitCode > 0; + return ( $exitCode, $result ); +} + +my $debug = 0; + +sub debug($$) { + my $level = shift; + my $message = shift; + print $message. "\n" if $debug > $level; +} + +sub error ($) { + print "\nERROR: $_[0]\nsee $0 --help for help"; + exit 1; +} + +sub info ($) { + my $message = shift; + if ( $message =~ /^\n/ ) { + $message =~ s/^\n//g; + print "\n"; + } + print "INFO:\t$message\n"; +} + +return 1; diff --git a/tools/sync_cms/lib/GoogleCalendar.pm b/tools/sync_cms/lib/GoogleCalendar.pm new file mode 100644 index 0000000..57df031 --- /dev/null +++ b/tools/sync_cms/lib/GoogleCalendar.pm @@ -0,0 +1,197 @@ +package GoogleCalendar; + +use strict; +use warnings; + +use Data::Dumper; + +use lib '../calcms/'; +use Common ( 'info', 'error' ); +use GoogleCalendarApi(); +use time(); + +my $settings = {}; +my $cal = undef; +my $debug = 1; + +sub set($$) { + my $key = shift; + my $value = shift; + $settings->{$key} = $value; +} + +sub get($) { + my $key = shift; + return $settings->{$key}; +} + +sub init($) { + $settings = shift || {}; + + my $access = get('access'); + + # 1. create service account at https://console.developers.google.com/ + # 2. enable Calendar API + # 3. share calendar with service account for update permissions + + # see http://search.cpan.org/~shigeta/Google-API-Client-0.13/lib/Google/API/Client.pm + + my $serviceAccount = $access->{serviceAccount}; + my $serviceAccountKeyFile = $access->{serviceAccountKeyFile}; + my $calendarId = $access->{calendarId}; + + my $serviceAccountKey = Common::loadFile($serviceAccountKeyFile); + my $calendar = GoogleCalendarApi->new( + { + 'serviceAccount' => $serviceAccount, + 'privateKey' => $serviceAccountKey, + 'calendarId' => $calendarId, + 'debug' => 0 + } + ); + + $cal = $calendar; +} + +#map event schema to target schema +sub mapToSchema { + my $event = shift; + + #clone event + my $targetEvent = {}; + for my $key ( keys %{$event} ) { + $targetEvent->{$key} = $event->{$key}; + } + + if ( defined $event->{recurrence} && ref( $event->{recurrence} ) eq 'HASH' ) { + $targetEvent->{reference} .= '[' . $event->{recurrence}->{number} . ']' if ( $event->{recurrence}->{number} > 0 ); + $targetEvent->{recurrence} = $event->{recurrence}->{number} + 0; + } + $targetEvent->{rating} = 0; + $targetEvent->{visibility} = 0; + + #set project by project's date range + my $projects = get('projects'); + if ( ref($projects) eq 'HASH' ) { + for my $projectName ( keys %$projects ) { + my $project = get('projects')->{$projectName}; + my $start = substr( $event->{start}, 0, 10 ); + if ( $start ge $project->{start_date} && $start le $project->{end_date} ) { + $targetEvent->{project} = $project->{name}; + } + } + } + + #override settings by target map filter + for my $key ( keys %{ get('mapping') } ) { + $targetEvent->{$key} = get('mapping')->{$key}; + } + + #resolve variables set in mapped values + for my $mkey ( keys %{ get('mapping') } ) { + for my $key ( sort keys %{$targetEvent} ) { + my $val = $targetEvent->{$key}; + $val = $event->{$key} if $mkey eq $key; + $targetEvent->{$mkey} =~ s//$val/g; + } + } + $targetEvent->{title} =~ s/\s+$//g; + $targetEvent->{title} =~ s/\s*\#$//g; + $targetEvent->{title} =~ s/\s*\-\s*$//g; + + my $schema = { event => $targetEvent }; + + return $schema; +} + +#this is done before sync and allows to delete old events before adding new +sub getEvents { + my $event = shift; + + return undef if get('date')->{'time_zone'} eq ''; + return undef if $event->{start} eq ''; + return undef if $event->{end} eq ''; + + #delete a span of dates + my $timeZone = get('date')->{'time_zone'}; + my $start = time::get_datetime( $event->{start}, $timeZone ); + my $end = time::get_datetime( $event->{end}, $timeZone ); + + info( "search target for events from " . $start . " to " . $end ); + + #search datetime with same timezone + my $events = $cal->getEvents( + { + timeMin => $cal->getDateTime( $start->datetime, $timeZone ), + timeMax => $cal->getDateTime( $end->datetime, $timeZone ), + maxResults => 50, + singleEvents => 'true', + orderBy => 'startTime' + } + ); + + return $events; +} + +# insert a new event +sub insertEvent { + my $event = shift; + my $entity = $event->{event}; + + $entity->{'html_content'} = markup::creole_to_html( $entity->{'content'} ); + + my $timeZone = get('date')->{'time_zone'}; + + my $start = $cal->getDateTime( $entity->{start}, $timeZone ); + my $end = $cal->getDateTime( $entity->{end}, $timeZone ); + + #info "insert event\t$start\t$entity->{title}"; + my $entry = { + start => $start, + end => $end, + summary => $entity->{title}, + description => $entity->{content}, + location => $entity->{location}, + transparency => 'transparent', + status => 'confirmed' + }; + + my $result = $cal->insertEvent($entry); + my $id = $result->{id}; +} + +sub deleteEvent { + my $event = shift; + + #info "delete event"; + $cal->deleteEvent( $event->{id} ); +} + +sub fixFields { + my $event = shift; + + #lower case for upper case titles longer than 4 characters + for my $attr ( 'series_name', 'title' ) { + my $val = $event->{$attr}; + my $c = 0; + while ( $val =~ /\b([A-Z]{5,99})\b/ && $c < 10 ) { + my $word = $1; + my $lower = lc $word; + $lower =~ s/^([a-z])/\u$1/gi; + $val =~ s/$word/$lower/g; + $c++; + } + $event->{$attr} = $val if $event->{$attr} ne $val; + } + + for my $attr ( 'series_name', 'title', 'excerpt', 'content' ) { + my $val = $event->{$attr}; + $val =~ s/^\s*(.*?)\s*$/$1/g; + $val =~ s/^[ \t]/ /g; + $event->{$attr} = $val if $event->{$attr} ne $val; + } + return $event; +} + +#do not delete last line +1; diff --git a/tools/sync_cms/lib/GoogleCalendarApi.pm b/tools/sync_cms/lib/GoogleCalendarApi.pm index aec1490..4d690c0 100644 --- a/tools/sync_cms/lib/GoogleCalendarApi.pm +++ b/tools/sync_cms/lib/GoogleCalendarApi.pm @@ -2,6 +2,7 @@ package GoogleCalendarApi; use strict; use warnings; + use JSON; use JSON::WebToken; use LWP::UserAgent; @@ -9,34 +10,38 @@ use HTML::Entities; use URI::Escape; use Data::Dumper; use DateTime; +use Time::HiRes qw(sleep); +use Common ( 'info', 'error' ); sub new { my $class = shift; my $params = shift; - - my $self={}; - for my $attr ('calendarId','debug'){ - $self->{$attr}=$params->{$attr} if defined $params->{$attr}; + + #print Dumper($class); + my $self = {}; + for my $attr ( 'calendarId', 'debug' ) { + $self->{$attr} = $params->{$attr} if defined $params->{$attr}; } - - my $instance=bless $self, $class; - if ((defined $params->{serviceAccount}) && (defined $params->{privateKey})){ - $instance->login($params->{serviceAccount}, $params->{privateKey}); + $self->{debug} = 1; + + my $instance = bless $self, $class; + if ( ( defined $params->{serviceAccount} ) && ( defined $params->{privateKey} ) ) { + $instance->login( $params->{serviceAccount}, $params->{privateKey} ); } - + return $instance; } -sub setCalendar{ - my $self=shift; - my $calendarId=shift; - $self->{calendarId}=$calendarId; +sub setCalendar { + my $self = shift; + my $calendarId = shift; + $self->{calendarId} = $calendarId; } -sub getBasicUrl{ - my $self=shift; - return 'https://www.googleapis.com/calendar/v3/calendars/'.encode_entities($self->{calendarId}); +sub getBasicUrl { + my $self = shift; + return 'https://www.googleapis.com/calendar/v3/calendars/' . encode_entities( $self->{calendarId} ); } #https://developers.google.com/google-apps/calendar/v3/reference/events/list @@ -52,149 +57,161 @@ sub getBasicUrl{ # 'updated' => '2014-11-12T19:46:22.086Z', # 'items' => [...] # } -sub getEvents{ - my $self=shift; - my $params=shift; +sub getEvents { + my $self = shift; + my $params = shift; - my $url='/events?'; - for my $param ('iCalUID','alwaysIncludeEmail','maxAttendees','maxResults','orderBy','pageToken','privateExtendedProperty', - 'q','sharedExtendedProperty','showDeleted','showHiddenInvitations','singleEvents','syncToken','timeZone' - ){ - $url.='&'.$param.'='.uri_escape($params->{$param}) if defined $params->{$param}; + my $url = '/events?'; + for my $param ( + 'iCalUID', 'alwaysIncludeEmail', 'maxAttendees', 'maxResults', + 'orderBy', 'pageToken', 'privateExtendedProperty', 'q', + 'sharedExtendedProperty', 'showDeleted', 'showHiddenInvitations', 'singleEvents', + 'syncToken', 'timeZone' + ) + { + $url .= '&' . $param . '=' . uri_escape( $params->{$param} ) if defined $params->{$param}; } - for my $param ('timeMin','timeMax','updatedMin'){ - $url.='&'.$param.'='.uri_escape($self->formatDateTime($params->{$param})) if defined $params->{$param}; + for my $param ( 'timeMin', 'timeMax', 'updatedMin' ) { + $url .= '&' . $param . '=' . uri_escape( $self->formatDateTime( $params->{$param} ) ) if defined $params->{$param}; } - my $result=$self->httpRequest('GET', $url); + my $result = $self->httpRequest( 'GET', $url ); return $result; } # sleep 0.25 seconds to prevent hitting the 5.0 requests/second/user rate -sub sleep{ - my $this=shift; - my $duration=shift || 0.25; - select(undef, undef, undef, $duration); -} +#sub sleep{ +# my $this=shift; +# my $duration=shift; +# $duration=1 unless defined $duration; +# select(undef, undef, undef, $duration); +#} #https://developers.google.com/google-apps/calendar/v3/reference/events/delete -sub deleteEvent{ - my $self=shift; - my $eventId=shift; - my $url='/events/'.$eventId; +sub deleteEvent { + my $self = shift; + my $eventId = shift; + my $url = '/events/' . $eventId; + #DELETE https://www.googleapis.com/calendar/v3/calendars/calendarId/events/eventId - my $result=$self->httpRequest('DELETE', $url); - $self->sleep(); + my $result = $self->httpRequest( 'DELETE', $url ); + + #$self->sleep(); return $result; } #https://developers.google.com/google-apps/calendar/v3/reference/events/insert -sub insertEvent{ - my $self=shift; - my $params=shift; +sub insertEvent { + my $self = shift; + my $params = shift; - my $event={ + my $event = { start => { - dateTime => $self->formatDateTime($params->{start}) + dateTime => $self->formatDateTime( $params->{start} ) }, - end => { - dateTime => $self->formatDateTime($params->{end}) + end => { + dateTime => $self->formatDateTime( $params->{end} ) }, - summary => $params->{summary}||'', - description => $params->{description}||'', - location => $params->{location}||'', - status => $params->{confirmed}||'confirmed' + summary => $params->{summary} || '', + description => $params->{description} || '', + location => $params->{location} || '', + status => $params->{confirmed} || 'confirmed' }; - $event= encode_json $event; + $event = encode_json $event; #POST https://www.googleapis.com/calendar/v3/calendars/calendarId/events - my $url='/events'; - my $result=$self->httpRequest('POST', $url, $event); - $self->sleep(); + my $url = '/events'; + my $result = $self->httpRequest( 'POST', $url, $event ); + + #$self->sleep(); return $result; } # send a HTTP request -sub httpRequest{ - my $self=shift; - my $method=shift; - my $url=shift; - my $content=shift||''; - - print STDERR "$method ".$url."\n" if $self->{debug}; +sub httpRequest { + my $self = shift; + my $method = shift; + my $url = shift; + my $content = shift || ''; - die ("missing url") unless defined $url; - die ("calendarId not set") unless defined $self->{calendarId}; - die ("not logged in ") unless defined $self->{api}; + sleep 0.3; + print STDERR "$method " . $url . "\n" if $self->{debug}; - #prepend basic url including calendar id - $url=$self->getBasicUrl().$url; - print STDERR "$method ".$url."\n" if $self->{debug}; + die("missing url") unless defined $url; + die("calendarId not set") unless defined $self->{calendarId}; + die("not logged in ") unless defined $self->{api}; - my $response=undef; - if($method eq 'GET'){ + #prepend basic url including calendar id + $url = $self->getBasicUrl() . $url; + print STDERR "$method " . $url . "\n" if $self->{debug}; + + my $response = undef; + if ( $method eq 'GET' ) { $response = $self->{api}->get($url); - }elsif(($method eq 'POST')||($method eq 'PUT')){ - print STDERR $content."\n" if $self->{debug}; + } elsif ( ( $method eq 'POST' ) || ( $method eq 'PUT' ) ) { + #return; + print STDERR $content . "\n" if $self->{debug}; my $request = HTTP::Request->new( $method, $url ); $request->header( 'Content-Type' => 'application/json' ); - $request->content( $content ); - $response=$self->{api}->request( $request ); - }elsif($method eq 'DELETE'){ + $request->content($content); + $response = $self->{api}->request($request); + } elsif ( $method eq 'DELETE' ) { + #return; $response = $self->{api}->delete($url); } - if($response->is_success) { + if ( $response->is_success ) { my $content = $response->content; return {} if $content eq ''; return decode_json($content); } else { print "ERROR:\n"; - print "Code: ".$response->code."\n"; - print "Message: ".$response->message."\n"; - print $response->content."\n"; + print "Code: " . $response->code . "\n"; + print "Message: " . $response->message . "\n"; + print $response->content . "\n"; die; } } # write datetime object to string -sub formatDateTime{ - my $self=shift; - my $dt=shift; +sub formatDateTime { + my $self = shift; + my $dt = shift; - my $datetime= $dt->format_cldr("yyyy-MM-ddTHH:mm:ssZZZZZ"); + my $datetime = $dt->format_cldr("yyyy-MM-ddTHH:mm:ssZZZZZ"); print STDERR "$dt -> $datetime\n" if $self->{debug}; return $datetime; } # parse datetime from string to object -sub getDateTime{ - my $self=shift; - my $datetime=shift; - my $timezone=shift; +sub getDateTime { + my $self = shift; + my $datetime = shift; + my $timezone = shift; - return if((!defined $datetime) or ($datetime eq '')); - my @l=split /[\-\;T\s\:\+\.]/,$datetime; + return if ( !defined $datetime ) or ( $datetime eq '' ); + my @l = split /[\-\;T\s\:\+\.]/, $datetime; - $datetime=DateTime->new( - year => $l[0], - month => $l[1], - day => $l[2], - hour => $l[3], - minute => $l[4], - second => $l[5], - time_zone => $timezone - ); - return $datetime; + $datetime = DateTime->new( + year => $l[0], + month => $l[1], + day => $l[2], + hour => $l[3], + minute => $l[4], + second => $l[5], + time_zone => $timezone + ); + return $datetime; } # login with serviceAccount and webToken (from privateKey) -sub login{ - my $self=shift; - my $serviceAccount=shift; - my $privateKey=shift; +sub login { + my $self = shift; + my $serviceAccount = shift; + my $privateKey = shift; # https://developers.google.com/accounts/docs/OAuth2ServiceAccount my $time = time; + #create JSON Web Token my $jwt = JSON::WebToken->encode( { @@ -203,10 +220,10 @@ sub login{ aud => 'https://accounts.google.com/o/oauth2/token', exp => $time + 3600, iat => $time, - }, - $privateKey, - 'RS256', - {typ => 'JWT'} + }, + $privateKey, + 'RS256', + { typ => 'JWT' } ); #send JSON web token to authentication service @@ -215,16 +232,16 @@ sub login{ 'https://accounts.google.com/o/oauth2/token', { grant_type => encode_entities('urn:ietf:params:oauth:grant-type:jwt-bearer'), - assertion => $jwt + assertion => $jwt } ); - - die($response->code, "\n", $response->content, "\n") unless $response->is_success(); - my $data= decode_json($response->content); - + + die( $response->code, "\n", $response->content, "\n" ) unless $response->is_success(); + my $data = decode_json( $response->content ); + #create a new user agent and set token to bearer $self->{api} = LWP::UserAgent->new(); - $self->{api}->default_header(Authorization => 'Bearer ' . $data->{access_token}); + $self->{api}->default_header( Authorization => 'Bearer ' . $data->{access_token} ); print STDERR "login successful\n" if $self->{debug}; return $data; diff --git a/tools/sync_cms/lib/source/calcms_i.pl b/tools/sync_cms/lib/source/calcms_i.pl deleted file mode 100644 index 962c25d..0000000 --- a/tools/sync_cms/lib/source/calcms_i.pl +++ /dev/null @@ -1,133 +0,0 @@ -#use markup; -use creole_wiki; -use DateTime; -use events; -use time; -use config; -#use DateTime::Format::ICal; - -package source; -use Data::Dumper; - -my $settings={}; - -sub init{ - $source::settings=shift; - #print STDERR Dumper($source::settings); -} - -#return a list of start_min, start_max request parameters. list is defined as timespan given by start_min and start_max in source_options -sub split_request{ - - return undef if ( - (!(defined $source::settings->{start_min})) || ($source::settings->{start_min} eq'') - ||(!(defined $source::settings->{start_max})) || ($source::settings->{start_max} eq'') - ); - - #print Dumper($source_options); - my $dates=[]; - - my $start =time::get_datetime($source::settings->{start_min},$source::settings->{date}->{time_zone}); - my $end =time::get_datetime($source::settings->{start_max},$source::settings->{date}->{time_zone}); - my $date =$start; - - #build a list of dates - my @dates=(); - while ($date < $end){ - push @dates,$date; - $date=$date->clone->add(days=>7); - } - my $duration=$end-$date; -# print "sec:".($duration->delta_seconds/(60*60))."\n"; - if ($duration->delta_seconds <= 0){ -# pop @dates; - push @dates,$end->clone; - } - - #build a list of parameters from dates - my $start=shift @dates; - for my $end (@dates){ - push @$dates,{ - start_min => $start, - start_max => $end - }; - $start=$end; - } - -# for $day(@$dates){print "$day->{start_min} - $day->{start_max}\n";} - return $dates; - -} - -#get a hash with per-day-lists days of a google calendar, given by its url defined at $calendar_name -sub get_events{ - my $block_number =$source::settings->{block_number}; - my $block_size =$source::settings->{block_size}; - my $last_update =$source::settings->{last_update}; - - #print Dumper($request); - - my $request_parameters={ - from_date => $source::settings->{start_min}, - till_date => $source::settings->{start_max}, - archive => 'all', - project => $source::settings->{project}, - template => 'no' - }; - $request_parameters->{location}=$source::settings->{location} if ($source::settings->{location}ne''); - - my $config = $source::settings; - my $request={ - url => $ENV{QUERY_STRING}, - params => { - original => \%params, - checked => events::check_params($config, - $request_parameters, - $source::settings - ), - }, - }; - #print Dumper($request); - - my $source_events=events::get($config, $request, $source::settings); - #print Dumper($source_events); - - #return events by date - my $sources_by_date={}; - my $old_start=''; - for my $source (@$source_events){ - $source->{calcms_start}=$source->{start}; - my $key=substr($source->{start},0,10); - push @{$sources_by_date->{$key}},$source; - } - return $sources_by_date; -} - -sub get_event_attributes{ - my $source=shift; - return $source; -} - -sub map_to_schema{ - my $event=shift; -# print Dumper($source_options); -# exit; - - #override settings by source map filter - for my $key (keys %{$source::settings->{mapping}}){ - $event->{$key}=$source::settings->{mapping}->{$key}; - } - - #resolve variables set in mapped values - for my $mkey (keys %{$source::settings->{mapping}}){ - for my $key (keys %{$event}){ - my $val=$event->{$key}; - $val=$event->{$key} if($mkey eq $key); - $event->{$mkey}=~s//$val/g; - } - } - - return $event; -} - -eof; diff --git a/tools/sync_cms/lib/source/google_calendar.pl b/tools/sync_cms/lib/source/google_calendar.pl deleted file mode 100755 index 4ce739a..0000000 --- a/tools/sync_cms/lib/source/google_calendar.pl +++ /dev/null @@ -1,339 +0,0 @@ -#use markup; -use creole_wiki; -use DateTime; -use Net::Google::Calendar; -use DateTime::Format::ICal; - -package source; -#do 'time.pl'; -use Data::Dumper; - -my $settings={}; - -sub init{ - $source::settings=shift; -} - -#return a list of start_min, start_max request parameters. -#list is defined as timespan given by start_min and start_max in source::settings -sub split_request{ - - return undef if ( - (!(defined $source::settings->{start_min})) || ($source::settings->{start_min} eq'') - ||(!(defined $source::settings->{start_max})) || ($source::settings->{start_max} eq'') - ); - - my $dates=[]; - - my $start =get_datetime($source::settings->{start_min},$source::settings->{date}->{time_zone}); - my $end =get_datetime($source::settings->{start_max},$source::settings->{date}->{time_zone}); - my $date =$start; - - #build a list of dates - my @dates=(); - while ($date < $end){ - push @dates,$date; - $date=$date->clone->add(days=>7); - } - my $duration=$end-$date; -# print "sec:".($duration->delta_seconds/(60*60))."\n"; - if ($duration->delta_seconds <= 0){ -# pop @dates; - push @dates,$end->clone; - } - - #build a list of parameters from dates - my $start=shift @dates; - for my $end (@dates){ - push @$dates,{ - start_min => $start, - start_max => $end - }; - $start=$end; - } - -# for $day(@$dates){print "$day->{start_min} - $day->{start_max}\n";} - return $dates; - -} - -#get a hash with per-day-lists days of a google calendar, given by its url defined at $calendar_name -sub get_events{ - -# print Dumper($source::settings); - my $url =$source::settings->{access}->{url}; - my $email =$source::settings->{access}->{email}; - my $password =$source::settings->{access}->{password}; - - my $block_number =$source::settings->{block_number}; - my $block_size =$source::settings->{block_size}; - my $last_update =$source::settings->{last_update}; - - my $parameters={}; - my $start_index=undef; - my $stop_index=undef; - if ($source::settings->{read_blocks}==1){ - my $start_index=$block_number*$block_size+1 ; - my $stop_index=$start_index+$block_size-1; - $parameters->{"start-index"} = $start_index; - $parameters->{"max-results"} = $block_size; - $source::settings->{start_index}=$start_index; - $source::settings->{stop_index}=$stop_index; - }else{ - $parameters->{"max-results"} = 10000; - } - - #see http://code.google.com/intl/de/apis/calendar/data/2.0/reference.html - $parameters->{singleevents}='true'; - $parameters->{orderby}='lastmodified'; - - my $more='modified' if (defined $last_update && $source::settings->{modified_events}==1); - main::print_info("read $more events from google calendar: '".substr($url,0,40)."...".substr($url,length($url)-8)."'"); - - # print "\nblock '$block_number' (events ".$start_index."..".$stop_index.") \n" if (defined $block_number || defined $start_index || defined $stop_index); - - # http://search.cpan.org/~simonw/Net-Google-Calendar-0.97/lib/Net/Google/Calendar.pm#get_events_[_%opts_] - - my $cal = Net::Google::Calendar->new( url => $url ); - #main::print_info("new\n"); - if ($email ne'' && $password ne''){ - $cal->login($email, $password) ; - # $cal->auth($email, $password) if ($email ne'' && $password ne''); - # main::print_info("login $email $password"); - } - #print Dumper($cal); - - #set UTF-8 - $XML::Atom::ForceUnicode = 1; - $XML::Atom::DefaultVersion = "1.0"; - -# my $xml=$cal->get_xml(); -# $xml=~s/{modified_events}==1)){ - my $datetime=$last_update; - $datetime=source::get_datetime($datetime,$source::settings->{date}->{time_zone}) if (ref($datetime)eq''); - $datetime->set_time_zone('UTC'); - $parameters->{"updated-min"} = $datetime->datetime; - #print "last update\n"; - } - #set start min (using UTC) - if ((defined $source::settings->{start_min}) && ($source::settings->{start_min}ne'')){ - my $datetime=$source::settings->{start_min}; - $datetime=source::get_datetime($datetime,$source::settings->{date}->{time_zone}) if (ref($datetime)eq''); - $datetime->set_time_zone('UTC'); - $parameters->{"start-min"} = $datetime->datetime; - $parameters->{"recurrence-expansion-start"}= $datetime->datetime; - } - #set start max (using UTC) - if ((defined $source::settings->{start_max})&&($source::settings->{start_max} ne'')){ - my $datetime=$source::settings->{start_max}; - $datetime=source::get_datetime($datetime,$source::settings->{date}->{time_zone}) if (ref($datetime)eq''); - $datetime->set_time_zone('UTC'); - $parameters->{"start-max"} = $datetime->datetime; - $parameters->{"recurrence-expansion-end"}= $datetime->datetime; - } - - -# print Dumper($parameters); - my @events=(); - my @source_events=$cal->get_events(%$parameters); - main::print_info("found ".@source_events." events"); - -# print Dumper($parameters); -# print Dumper($source::settings); -# exit; - - for my $source(@source_events) { - (my $start,my $end)=$source->when; - $start= $start->set_time_zone($source::settings->{date}->{time_zone})->datetime if (defined $start); - $end= $end->set_time_zone ($source::settings->{date}->{time_zone})->datetime if (defined $end); - $source->{calcms_start} = $start; - $source->{calcms_end} = $end; - $source->{status} = $source->status; - } - - #return events by date - my $sources_by_date={}; - my $old_start=''; -# for my $source (sort{$a->{calcms_start} cmp $b->{calcms_start} }@source_events){ - for my $source (@source_events){ -# if ($source->{status}eq'confirmed'){ - my $key=substr($source->{calcms_start},0,10); -# if ($old_start eq $source->{calcms_start}){ -# my $source=pop (@{$sources_by_date->{$key}}); -# print STDERR "WARNING: ignore canceled entry in google calendar: ".$source->{calcms_start}."\t".$source->{title}."\t".$source->{id}."\n"; -# } -# - push @{$sources_by_date->{$key}},$source; -# -# $old_start=$source->{calcms_start}; -# } - } - return $sources_by_date; -} - -sub map_to_schema{ - my $event=shift; - - my $params={ - title => $event->{title}, - content => $event->{content}, - local_media_url => '' - }; - $params->{content}=~s/\x0A\x20/\n/g; - #print Dumper($params); - #open FILE,">/tmp/test"; - #print FILE Dumper($params); - #close FILE; - - #decode event - $event=creole_wiki::extractEventFromWikiText($params, $event); - #exit; - - #override settings by source map filter - for my $key (keys %{$source::settings->{mapping}}){ - $event->{$key}=$source::settings->{mapping}->{$key}; - } - - #resolve variables set in mapped values - for my $mkey (keys %{$source::settings->{mapping}}){ - for my $key (keys %{$event}){ - my $val=$event->{$key}; - $val=$event->{$key} if($mkey eq $key); - $event->{$mkey}=~s//$val/g; - } - } - #print Dumper($event); - - return $event; -} - -sub get_event_attributes{ - my $source=shift; - - #print @source_events." ".Dumper($source)."\n"; - #use Data::Dumper;print Dumper($source->when); - - #create an hash with calendar event settings - my $event={ - start => $source->{calcms_start}, - end => $source->{calcms_end}, - status => $source->{status}, -# recurrence => $source->{recurrence}, - reference => $source->id, -# program => $program, -# series_name => $series_name, - title => $source->title, - content => $source->content->body, - author_name => $source->author->name, - author_uri => $source->author->uri, - author_email => $source->author->email, - transparency => $source->transparency, - visibility => $source->visibility, - location => $source->location, -# podcast_url => $podcast_url, -# media_url => $media_url, -# comments => $source->comments -# who_name => $source->who->name, -# who_email => $source->who->email, -# who_attendee_status => $source->who->attendee_status, - }; - #print Dumper($event); - -# if ($source->recurrence){ -# $event->{recurrence}=get_event_recurrence($source,$event); -# } - - return $event; -} - -sub get_event_recurrence{ - my $source=shift; - my $event=shift; - #print Dumper(); - - my $event_recurrence=$source->recurrence; - my $properties = $event_recurrence->properties; -# print Dumper($properties); - - my $dtstart = $properties->{dtstart}->[0]->{value}; - my $timezone = $properties->{dtstart}->[0]->{_parameters}->{TZID}; - my $dtend = $properties->{dtend}->[0]->{value}; - my $rrule = $properties->{rrule}->[0]->{value}; -# print $rrule."\n"; - - #convert timezone from "until=" to same datetime as in dtstart - if ($rrule=~/UNTIL=([\dT]+Z?)/){ - my $ical=$1; - - #convert timezone at ical format - my $datetime= DateTime::Format::ICal->parse_datetime($ical); - $datetime=$datetime->set_time_zone($timezone); - $ical=DateTime::Format::ICal->format_datetime($datetime); - - #remove TZID=... from ical, since not implemented at format_datetime - $ical=~s/[^\:]+\://; - $rrule=~s/(UNTIL\=)([\dT]+Z?)/$1$ical/g; -# print $datetime->datetime." --> $ical --> $rrule\n"; - } - - $dtstart = DateTime::Format::ICal->parse_datetime($dtstart); - $dtend = DateTime::Format::ICal->parse_datetime($dtend);#->add(seconds=>3600)->set_time_zone('UTC'); - - my $recurrence={ - dtstart => $dtstart, - dtend => $dtend, - rrule => $rrule - }; - - #calc duration of the event - my $duration=$dtend-$dtstart; - my $duration_min=$duration->delta_minutes; - -# print Dumper($duration_min); - #print Dumper($recurrence); - my $recurrence_start = DateTime::Format::ICal->parse_recurrence( - recurrence =>$rrule, - dtstart =>$dtstart - ); - - #step through recurrent events and mark if event matchs - my $start_iter = $recurrence_start->iterator; - $c=1; - while (my $start = $start_iter->next ){ -# print "$start eq $event->{start}, $end\n"; - $recurrence->{number}=$c if ($start eq $event->{start}); -# push @dates,{ -# start => $start->set_time_zone($source::settings->{time_zone})->datetime, -# end => $start->set_time_zone($source::settings->{time_zone})->add(minutes=>$duration_min)->datetime -# }; - $c++; - } - $event->{recurrence}=$recurrence; - #print Dumper($event->{recurrence}); - -} - -sub get_datetime{ - my $datetime=shift; - my $timezone=shift; - - return if((!defined $datetime) or ($datetime eq '')); - my @l=@{time::datetime_to_array($datetime)}; - $datetime=DateTime->new( - year =>$l[0], - month =>$l[1], - day =>$l[2], - hour =>$l[3], - minute =>$l[4], - second =>$l[5], - time_zone=> $timezone - ); - return $datetime; -} - -eof; diff --git a/tools/sync_cms/lib/target/google_calendar.pl b/tools/sync_cms/lib/target/google_calendar.pl deleted file mode 100644 index 84b705d..0000000 --- a/tools/sync_cms/lib/target/google_calendar.pl +++ /dev/null @@ -1,233 +0,0 @@ -#require 'db.pl'; -#use db; -#use markup; - -package target; -use Data::Dumper; -use Net::Google::Calendar; -use time; - -my $settings={}; -my $cal = undef; -#my $op_count=0; - -sub init{ - $target::settings=shift; - my $access=$target::settings->{access}; - $target::cal = Net::Google::Calendar->new( url => $access->{url} ); - #main::print_info("init\n"); - #main::print_info("new\n"); - #print Dumper($access); - - my $email=$access->{email}; - my $password=$access->{password}; - if ($email ne'' && $password ne''){ - $target::cal->login($email, $password) ; - # $target::cal->auth($email, $password) if ($email ne'' && $password ne''); - main::print_info("loged in"); - } - #print Dumper($target::cal); - -# for my $c($target::cal->get_calendars) { -# print "'".$c->title."'\n"; -# print $c->id."\n\n"; -# if ($c->title eq 'petra poss'){ -# $target::cal->set_calendar($c); -# main::print_info("found matching calendar!"); -# } -# } -# exit; - - #set UTF-8 - $XML::Atom::ForceUnicode = 1; - $XML::Atom::DefaultVersion = "1.0"; - -} - -#map event schema to target schema -sub map_to_schema{ - my $event=shift; - - #clone event - my $target_event={}; - for my $key (keys %{$event}){ - $target_event->{$key}=$event->{$key}; - } - - $target_event->{reference}.='['.$event->{recurrence}->{number}.']' if ($event->{recurrence}->{number}>0); - $target_event->{recurrence} => $event->{recurrence}->{number}+0; - $target_event->{rating} => 0; - $target_event->{visibility} => 0; -# $target_event->{transparency} => $event->{transparency}; - - #set project by project's date range - for my $project_name (keys %{$target::settings->{projects}}){ - my $project=$target::settings->{projects}->{$project_name}; - my $start=substr($event->{start},0,10); - if ($start ge $project->{start_date} && $start le $project->{end_date}){ - $target_event->{project}=$project->{name}; - } -# print "$event->{start} gt $project->{start_date} $target_event->{project}\n"; - } - - #override settings by target map filter - for my $key (keys %{$target::settings->{mapping}}){ - $target_event->{$key}=$target::settings->{mapping}->{$key}; - } - #use Data::Dumper;print Dumper($target_event); - - #resolve variables set in mapped values - for my $mkey (keys %{$target::settings->{mapping}}){ - my $mval=$target_event->{$mkey}; - for my $key (sort keys %{$target_event}){ - my $val=$target_event->{$key}; - $val=$event->{$key} if($mkey eq $key); - #print $target_event->{$mkey}."\t".$key."-> $val\n"; - $target_event->{$mkey}=~s//$val/g; - } - } - #use Data::Dumper;print Dumper($target_event);#exit; - - #$schema->{event}=fix_fields($schema->{event}); - - my $schema={ - event => $target_event - }; - - return $schema; -} - -# get a event by an existing google id, e.g. to check if the event exists in target -sub get_event_by_reference_id{ - return undef; -} - -#try to find a event, matching to $event from google calendar -sub find_event{ - my $event=shift; - return undef; - -} - -sub pre_sync{ - my $event=shift; - - $debug=1; - return undef if(($target::settings->{date}->{'time_zone'} eq '') || ($event->{start} eq '' ) || ($event->{end} eq '')); - - #delete a span of dates - print "\n" if ($debug eq '1'); - my $time_zone=$target::settings->{date}->{'time_zone'}; - my $start=time::get_datetime($event->{start},$time_zone); - $start->set_time_zone('UTC'); - $parameters->{"start-min"} = $start->datetime; - #$parameters->{"recurrence-expansion-start"}= $start->datetime; - - my $end=time::get_datetime($event->{end},$time_zone); - $end->set_time_zone('UTC'); - $parameters->{"start-max"} = $end->datetime; - #$parameters->{"recurrence-expansion-end"}= $end->datetime; - - main::print_info("search target for events from ".$start." to ".$end) if ($debug eq '1'); - - my @events=$target::cal->get_events(%$parameters); - - for my $event(@events){ - main::print_info("delete ".$event->title) if ($debug eq '1'); - $target::cal->delete_entry($event); - }; -} - - -# insert a new event -sub insert_event{ - my $event=shift; - my $entity=$event->{event}; - - $entity->{'html_content'}=markup::creole_to_html($entity->{'content'}); - - my $time_zone =$target::settings->{date}->{'time_zone'}; - my $start =time::get_datetime($entity->{start},$time_zone); - my $end =time::get_datetime($entity->{end},$time_zone); - #print Dumper($start)."\n"; - #print Dumper($end)."\n"; - print "\n" if ($debug eq '1'); - - main::print_info("insert event") if ($debug eq '1'); - my $entry = Net::Google::Calendar::Entry->new(); - - #print Dumper($entity); - $entry->title($entity->{title}); - $entry->content($entity->{content}); - $entry->location($entity->{location}); - $entry->transparency('transparent'); - $entry->status('confirmed'); - $entry->when($start, $end); - #print Dumper($entry); - - $target::cal->add_entry($entry); - #exit; -} - - -# update an existing event -sub update_event{ - return; -} -### end of interface implementation ### - - -sub print_event{ - my $header=shift; - my $event=shift; - - if ($header eq'google'){ - print "\n===== $header ====="; - }else{ - print "$header\n" if $header ne ''; - } -# print qq!$event->{start} $event->{program} : $event->{series_name} - $event->{title}!."\n"; - #content: >$event->{content}< -}; - -sub delete_event{ - return; - -} - -sub fix_fields{ - my $event=shift; - #lower case for upper case titles longer than 4 characters - for my $attr qw(program series_name title){ - my $val=$event->{$attr}; - my $c=0; - while ($val=~/\b([A-Z]{5,99})\b/ && $c<10){ - my $word=$1; - my $lower=lc $word; - $lower=~s/^([a-z])/\u$1/gi; - $val=~s/$word/$lower/g; - $c++; - } - if ($event->{$attr} ne $val){ - $event->{$attr}=$val; -# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ; - } - } - - for my $attr qw(program series_name title excerpt content ){ - my $val=$event->{$attr}; - $val=~s/^\s*(.*?)\s*$/$1/g; - $val=~s/^[ \t]/ /g; - if ($event->{$attr} ne $val){ - $event->{$attr}=$val; -# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ; - } - } - return $event; -} - -sub clean_up{ - return; -} - -1; diff --git a/tools/sync_cms/lib/target/google_calendar2.pl b/tools/sync_cms/lib/target/google_calendar2.pl deleted file mode 100644 index ada99a8..0000000 --- a/tools/sync_cms/lib/target/google_calendar2.pl +++ /dev/null @@ -1,254 +0,0 @@ -#require 'db.pl'; -#use db; -#use markup; - -package target; -use lib '/home/radio/calcms/sync_cms/lib/'; -use Data::Dumper; -#use Net::Google::Calendar; -use GoogleCalendarApi; -use time; - -my $settings={}; -my $cal = undef; -#my $op_count=0; - -sub init{ - $target::settings=shift; - my $access=$target::settings->{access}; - - # 1. create service account at https://console.developers.google.com/ - # 2. enable Calendar API - # 3. share calendar with service account for update permissions - - # see http://search.cpan.org/~shigeta/Google-API-Client-0.13/lib/Google/API/Client.pm - - my $serviceAccount = $access->{serviceAccount}; - my $serviceAccountKeyFile = $access->{serviceAccountKeyFile}; - my $calendarId = $access->{calendarId}; - - my $serviceAccountKey = loadFile($serviceAccountKeyFile); - - #print "connect...\n"; - my $calendar = new GoogleCalendarApi({ - 'serviceAccount' => $serviceAccount, - 'privateKey' => $serviceAccountKey, - 'calendarId' => $calendarId, - 'debug' => 0 - }); - #print Dumper($calendar); - $target::cal = $calendar; - -} - -#map event schema to target schema -sub map_to_schema{ - my $event=shift; - - #clone event - my $target_event={}; - for my $key (keys %{$event}){ - $target_event->{$key}=$event->{$key}; - } - - $target_event->{reference}.='['.$event->{recurrence}->{number}.']' if ($event->{recurrence}->{number}>0); - $target_event->{recurrence} => $event->{recurrence}->{number}+0; - $target_event->{rating} => 0; - $target_event->{visibility} => 0; -# $target_event->{transparency} => $event->{transparency}; - - #set project by project's date range - for my $project_name (keys %{$target::settings->{projects}}){ - my $project=$target::settings->{projects}->{$project_name}; - my $start=substr($event->{start},0,10); - if ($start ge $project->{start_date} && $start le $project->{end_date}){ - $target_event->{project}=$project->{name}; - } -# print "$event->{start} gt $project->{start_date} $target_event->{project}\n"; - } - - #override settings by target map filter - for my $key (keys %{$target::settings->{mapping}}){ - $target_event->{$key}=$target::settings->{mapping}->{$key}; - } - #use Data::Dumper;print Dumper($target_event); - - #resolve variables set in mapped values - for my $mkey (keys %{$target::settings->{mapping}}){ - my $mval=$target_event->{$mkey}; - for my $key (sort keys %{$target_event}){ - my $val=$target_event->{$key}; - $val=$event->{$key} if($mkey eq $key); - #print $target_event->{$mkey}."\t".$key."-> $val\n"; - $target_event->{$mkey}=~s//$val/g; - } - } - #use Data::Dumper;print Dumper($target_event);#exit; - - #$schema->{event}=fix_fields($schema->{event}); - - my $schema={ - event => $target_event - }; - - return $schema; -} - -# get a event by an existing google id, e.g. to check if the event exists in target -sub get_event_by_reference_id{ - return undef; -} - -#try to find a event, matching to $event from google calendar -sub find_event{ - my $event=shift; - return undef; - -} - -#this is done before sync and allows to delete old events before adding new -sub pre_sync{ - my $event=shift; - - $debug=1; - return undef if(($target::settings->{date}->{'time_zone'} eq '') || ($event->{start} eq '' ) || ($event->{end} eq '')); - - #delete a span of dates - print "\n" if ($debug eq '1'); - - my $timeZone=$target::settings->{date}->{'time_zone'}; - - #get datetime in timezone - my $start = time::get_datetime($event->{start}, $timeZone); - my $end = time::get_datetime($event->{end}, $timeZone); - - main::print_info("search target for events from ".$start." to ".$end) if ($debug eq '1'); - - my $events=$target::cal->getEvents({ - #search datetime with same timezone - timeMin => $target::cal->getDateTime($start->datetime, $timeZone), - timeMax => $target::cal->getDateTime($end->datetime, $timeZone), - maxResults => 50, - singleEvents => 'true', - orderBy => 'startTime' - }); - - my $now=DateTime->now()->set_time_zone('UTC')->epoch(); - #print Dumper($now->datetime); - #exit; - - for my $event(@{$events->{items}}){ - main::print_info("delete\t$event->{start}->{dateTime}\t".$event->{summary}) if ($debug eq '1'); - #my $updated = $target::cal->getDateTime($event->{updated},'UTC')->epoch(); - #my $delta = $now-$updated; - #print $delta." seconds old\n"; - $target::cal->deleteEvent($event->{id}) - }; - #exit; -} - - -# insert a new event -sub insert_event{ - my $event=shift; - my $entity=$event->{event}; - - $entity->{'html_content'}=markup::creole_to_html($entity->{'content'}); - - my $timeZone = $target::settings->{date}->{'time_zone'}; - #print Dumper($timeZone); - #print Dumper($entity); - my $start = $target::cal->getDateTime($entity->{start}, $timeZone); - my $end = $target::cal->getDateTime($entity->{end}, $timeZone); - print "\n" if ($debug eq '1'); - #exit; - main::print_info("insert event\t$start\t$entity->{title}") if ($debug eq '1'); - my $entry = { - start => $start, - end => $end, - summary => $entity->{title}, - description => $entity->{content}, - location => $entity->{location}, - transparency => 'transparent', - status => 'confirmed' - }; - - my $result=$target::cal->insertEvent($entry); - my $id=$result->{id}; - - #exit; -} - -sub loadFile{ - my $filename=shift; - my $content=''; - - open my $file, '<', $filename || die("cannot load $filename"); - while(<$file>){ - $content.=$_; - } - close $file; - return $content; -} - -# update an existing event -sub update_event{ - return; -} -### end of interface implementation ### - - -sub print_event{ - my $header=shift; - my $event=shift; - - if ($header eq'google'){ - print "\n===== $header ====="; - }else{ - print "$header\n" if $header ne ''; - } -# print qq!$event->{start} $event->{program} : $event->{series_name} - $event->{title}!."\n"; - #content: >$event->{content}< -}; - -sub delete_event{ - return; - -} - -sub fix_fields{ - my $event=shift; - #lower case for upper case titles longer than 4 characters - for my $attr qw(program series_name title){ - my $val=$event->{$attr}; - my $c=0; - while ($val=~/\b([A-Z]{5,99})\b/ && $c<10){ - my $word=$1; - my $lower=lc $word; - $lower=~s/^([a-z])/\u$1/gi; - $val=~s/$word/$lower/g; - $c++; - } - if ($event->{$attr} ne $val){ - $event->{$attr}=$val; -# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ; - } - } - - for my $attr qw(program series_name title excerpt content ){ - my $val=$event->{$attr}; - $val=~s/^\s*(.*?)\s*$/$1/g; - $val=~s/^[ \t]/ /g; - if ($event->{$attr} ne $val){ - $event->{$attr}=$val; -# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ; - } - } - return $event; -} - -sub clean_up{ - return; -} - -1; diff --git a/tools/sync_cms/lib/target/playlist_csv.pl b/tools/sync_cms/lib/target/playlist_csv.pl deleted file mode 100644 index 2456688..0000000 --- a/tools/sync_cms/lib/target/playlist_csv.pl +++ /dev/null @@ -1,195 +0,0 @@ -package target; -use Data::Dumper; -use time; -use warnings; -use strict; - -my $settings={}; -my $cal = undef; - -sub init{ - $target::settings=shift; - my $access=$target::settings->{access}; - $cal = []; -} - -#map event schema to target schema -sub map_to_schema{ - my $event=shift; - #clone event - my $target_event={}; - for my $key (keys %{$event}){ - $target_event->{$key}=$event->{$key}; - } - - $event->{recurrence}->{number}=0 unless (defined $event->{recurrence} || defined $event->{recurrence}->{number}); - $target_event->{reference}.='['.$event->{recurrence}->{number}.']' if ($event->{recurrence}->{number}>0); - $target_event->{recurrence} => $event->{recurrence}->{number}; - $target_event->{rating} => 0; - $target_event->{visibility} => 0; -# $target_event->{transparency} => $event->{transparency}; - - #set project by project's date range - for my $project_name (keys %{$target::settings->{projects}}){ - my $project=$target::settings->{projects}->{$project_name}; - my $start=substr($event->{start},0,10); - if ($start ge $project->{start_date} && $start le $project->{end_date}){ - $target_event->{project}=$project->{name}; - } - } - - #override settings by target map filter - for my $key (keys %{$target::settings->{mapping}}){ - $target_event->{$key}=$target::settings->{mapping}->{$key}; - } - - #resolve variables set in mapped values - for my $mkey (keys %{$target::settings->{mapping}}){ - my $mval=$target_event->{$mkey}; - for my $key (keys %{$target_event}){ - my $val=$target_event->{$key}; - $val=$event->{$key} if($mkey eq $key); - $target_event->{$mkey}=~s//$val/g; - } - } - - my $schema={ - event => $target_event - }; - - return $schema; -} - -# get a event by an existing reference id, e.g. to check if the event exists in target -sub get_event_by_reference_id{ - my $event_id=shift; - my $event={}; - return undef; -} - -#try to find a event -sub find_event{ - my $event=shift; - return undef; -} - -# insert a new event -sub insert_event{ - my $event=shift; - my $entity=$event->{event}; - - my $time_zone =$target::settings->{date}->{'time_zone'}; - my $start =time::get_datetime($entity->{start},$time_zone); - my $end =time::get_datetime($entity->{end},$time_zone); - print "\n" if ($main::debug eq '1'); - - main::print_info("insert event") if ($main::debug eq '1'); - push @$cal,{ - start => $start, - end => $end, - title => $entity->{title} - } - #exit; -} - - -# update an existing event -sub update_event{ - my $event=shift; - my $entity=shift; -} -### end of interface implementation ### - - -sub print_event{ - my $header=shift; - my $event=shift; - - if ($header eq'google'){ - print "\n===== $header ====="; - }else{ - print "$header\n" if $header ne ''; - } -# print qq!$event->{start} $event->{program} : $event->{series_name} - $event->{title}!."\n"; - #content: >$event->{content}< -}; - -sub delete_event{ - my $event_id=shift; - -} - -sub fix_fields{ - my $event=shift; - - for my $attr qw(title){ - my $val=$event->{$attr}; - $val=~s/^\s*(.*?)\s*$/$1/g; - $val=~s/^[ \t]/ /g; - if ($event->{$attr} ne $val){ - $event->{$attr}=$val; -# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ; - } - } - return $event; -} - -sub pre_sync{ -} - -sub clean_up{ - my $content=''; - - my @cal=sort {$a->{start} cmp $b->{end}} @$cal; - my @cal2=(); - #print Dumper(\@cal); - #fill in default - if (defined $target::settings->{date}->{default_entry}){ - my $from=$main::from; - if ($from=~/^\d\d\d\d\-\d\d\-\d\dT\d\d$/){ - $from.=':00'; - } - my $till=$main::till; - if ($till=~/^\d\d\d\d\-\d\d\-\d\dT\d\d$/){ - $till.=':59'; - } - - my $default=$target::settings->{date}->{default_entry}; - if ($cal[0]->{start} gt $from){ - unshift @cal,{ - start => $from, - end => $cal[0]->{start}, - title => $default - } - } - if ($cal[-1]->{end} lt $till){ - push @cal,{ - start => $cal[-1]->{end}, - end => $till, - title => $default - } - } - my $old_event={end=>$from}; - for my $event (@cal){ - if ($event->{start} gt $old_event->{end}){ - push @cal2,{ - start => $old_event->{end}, - end => $event->{start}, - title => $default - } - } - push @cal2,$event; - $old_event=$event; - - } - } - - - for my $event(@cal2){ - $content.= $event->{start}.";\t".$event->{end}.";\t".$event->{title}."\n"; - } - log::save_file($target::settings->{access}->{file},$content); - return; -} - -1; diff --git a/tools/sync_cms/run_jobs.pl b/tools/sync_cms/run_jobs.pl new file mode 100755 index 0000000..b61df03 --- /dev/null +++ b/tools/sync_cms/run_jobs.pl @@ -0,0 +1,81 @@ +#! /usr/bin/perl + +use strict; +use warnings; + +use Data::Dumper; +use FindBin; +use lib "$FindBin::Bin/lib"; +use lib "$FindBin::Bin/../calcms"; + +use Common ( 'info', 'error' ); + +use config(); +use time(); +use log(); + +$| = 1; + +sub runJobs { + my $jobs = shift; + my $startDir = shift; + my $logDir = shift; + + for my $job (@$jobs) { + + my $startFile = $startDir . '/' . $job->{name} . '.start.txt'; + my $startAge = Common::getModifiedAt($startFile); + next if $startAge == 0; + + my $logFile = $logDir . '/' . $job->{name} . '.log'; + my $logAge = Common::getModifiedAt($logFile); + next if $startAge < $logAge; + + # read parameters form start file + my $content = log::load_file($startFile); + + #execute command + my $command = $job->{command} . ' 2>&1 > ' . $logFile; + my ( $exitCode, $result ) = Common::execute($command); + error "exitCode=$exitCode on $command" if $exitCode != 0; + } +} + +sub check() { + my $configFile = shift @ARGV; + error qq{cannot read $configFile "$configFile"} unless -e $configFile; + + my $config = config::get($configFile); + + my $startDir = $config->{start_dir} || ''; + error 'missing configuration of jobs/start_dir!' if $startDir eq ''; + error "job dir does not exist '$startDir'" unless -e $startDir; + error "cannot read from job dir '$startDir'. Please check permissions!" unless -w $startDir; + + my $logDir = $config->{log_dir} || ''; + error 'missing configuration of jobs/log_dir' if $logDir eq ''; + error "job log dir does not exist '$logDir'" unless -e $logDir; + error "cannot read from job log dir '$logDir'. Please check permissions!" unless -r $logDir; + error "cannot write to job log dir '$logDir'. Please check permissions!" unless -w $logDir; + + my $jobs = $config->{job}; + error "no jobs defined!" if scalar @$jobs == 0; + + return ( $jobs, $startDir, $logDir ); +} + +sub main() { + + info "INIT\t" . time::time_to_datetime(); + Common::checkSingleInstance(); + my ( $jobs, $startDir, $logDir ) = check(); + + #exit after a at most 10 minute timeout in case of hanging process + local $SIG{ALRM} = sub { die "ERROR: exit due to synchronization hangs\n" }; + alarm 10 * 60; + + runJobs( $jobs, $startDir, $logDir ); + info "DONE\t" . time::time_to_datetime(); +} + +main(); diff --git a/tools/sync_cms/sync_cms.pl b/tools/sync_cms/sync_cms.pl index 981a025..a5fa3aa 100755 --- a/tools/sync_cms/sync_cms.pl +++ b/tools/sync_cms/sync_cms.pl @@ -1,438 +1,236 @@ #!/usr/bin/perl -BEGIN{ - my $dir=''; - $ENV{SCRIPT_FILENAME}||'' if ($dir eq''); - $dir=~s/(.*\/)[^\/]+/$1/ if ($dir ne ''); - $dir=$ENV{PWD} if ($dir eq''); - $dir=`pwd` if ($dir eq''); - - #add calcms libs - unshift(@INC,$dir.'/../calcms/'); -} +use strict; +use warnings; use Data::Dumper; use Getopt::Long; use Config::General; -use time; use DateTime; use DateTime::Duration; -use strict; -use warnings; +use IO::Socket::INET; +use Fcntl ':flock'; -check_running_processes(); +use FindBin; +use lib "$FindBin::Bin/lib"; +use lib "$FindBin::Bin/../../calcms"; -my $read_mode=''; -my $update_mode=''; -my $all_events=''; -my $modified_events=''; -my $source_config_file=''; -my $target_config_file=''; -my $block_number=0; -my $block_size=2000; -our $from=''; -our $till=''; -my $read_only=0; -our $output_type='text'; -our $debug=0; +use Common ( 'info', 'error' ); +use GoogleCalendar; +use CalcmsEvents; -GetOptions( - "read" => \$read_mode, - "update" => \$update_mode, - "all" => \$all_events, - "modified" => \$modified_events, - "from=s" => \$from, - "till=s" => \$till, - "source=s" => \$source_config_file, - "target=s" => \$target_config_file, - "block_number:i" => \$block_number, - "block_size:i" => \$block_size, - "output_type=s" => \$output_type, -); - -$|=1; +Common::checkSingleInstance(); BEGIN { - our $utf8dbi=1; - $ENV{LANG}="en_US.UTF-8"; + $ENV{LANG} = "en_US.UTF-8"; } +$| = 1; + +my $sourceConfigFile = ''; +my $targetConfigFile = ''; +my $debug = 1; + +my $from = undef; +my $till = undef; +GetOptions( + "from=s" => \$from, + "till=s" => \$till, + "source=s" => \$sourceConfigFile, + "target=s" => \$targetConfigFile, +); + #source and taget settings are loaded from config files -our $settings={ -}; +my $settings = {}; -#user interface -our $ask_before_insert=0; -our $ask_before_update=0; - -# end of configuration - -if ($update_mode){ - $db::write=1; -# print_info("enter update mode"); -}elsif($read_mode){ - #default - $db::write=0; -# print_info("enter read-only mode"); -}else{ - print_error("set parameter >read< or >update<"); -} - -unless ($modified_events || $all_events || $from || $till){ - print_error("set one of folling parameters: --modified, --from, --till"); -} +error "set one of folling parameters: --from, --till" unless $from || $till; init(); sync(); -print_info("$0 done."); +info "$0 done."; exit 0; #sync all events, splitting multi-day-requests into multiple 1-day-requests to avoid large result sets -sub sync{ - #prepare target - print_info("$0 inited"); - print_info("last update: $settings->{source}->{last_update}"); +sub sync { + my $timeZone = CalcmsEvents::get('date')->{time_zone}; + my $from = CalcmsEvents::get('start_min'); + my $till = CalcmsEvents::get('start_max'); - if (my $days=source::split_request()){ - #set 1-day start-min and start-max parameters, requires --from and --till values - for my $date (@$days){ - for my $key(keys %$date){ - $settings->{source}->{$key}=$date->{$key}; - } - #print "\nrequest ".$settings->{source}->{"start_min"}." to ".$settings->{source}->{"start_max"}."\n"; - sync_timespan(); - } - }else{ - #update without time span (e.g. --modified) - sync_timespan(); - } + info "sync from $from till $till at $timeZone"; - print_info("\nclean up old database entries..."); - target::clean_up(); + #prepare target + info "last update: " . ( CalcmsEvents::get('last_update') || '' ); - print_info("\nset last-update time: $settings->{event}->{update_start}"); - set_last_update_time($source_config_file,$target_config_file,$settings->{event}->{update_start}); + if ( my $days = CalcmsEvents::splitRequest( $from, $till, $timeZone ) ) { + for my $date (@$days) { + syncTimespan( $date->{from}, $date->{till} ); + } + } else { + syncTimespan( $from, $till ); + } + + info "\nset last-update time: $settings->{event}->{update_start}"; + setLastUpdateTime( $sourceConfigFile, $targetConfigFile, $settings->{event}->{update_start} ); } #sync all events of a given source timespan -sub sync_timespan{ - #get a list of all days and their events - #print Dumper($settings->{source}); - my $source_events=source::get_events($settings->{source},$settings->{target}); - my @dates=(keys %$source_events); +sub syncTimespan { + my $from = shift; + my $till = shift; - #print "2\n"; - if (@dates==0){ - my $more=''; - if ((defined $settings->{source}->{block_number}) && ($settings->{source}->{block_number} ne '0')){ - $more='more '; - }elsif ($modified_events){ - $more.='modified '; - } - print_info("\n".'no '.$more."entries found."); - }else{ - print "" if ($output_type eq 'html'); - #sort lists of date and time (same time events should be preserved) - for my $date(sort {$a cmp $b} @dates){ -# for my $date(@dates){ -# print "\n$date:\n"; - sync_events($source_events->{$date}, $settings); - } - print "
" if ($output_type eq 'html'); - } + #get a list of all days and their events + my $sourceEvents = CalcmsEvents::getEvents( $from, $till ); + + my @dates = keys %$sourceEvents; + if ( scalar @dates == 0 ) { + info "\nno entries found."; + return; + } + + #sort lists of date and time (same time events should be preserved) + for my $date ( sort { $a cmp $b } @dates ) { + syncEvents( $sourceEvents->{$date} ); + } } #syncronize a list of source events to target events -sub sync_events{ - my $source_events=shift; - my $settings=shift; +sub syncEvents($) { + my $sourceEvents = shift; -# my $source_settings =$settings->{source}; -# my $target_settings =$settings->{target}; - my $event_settings =$settings->{event}; + my @sourceEvents = sort { $a->{calcms_start} cmp $b->{calcms_start} } @$sourceEvents; + $sourceEvents = \@sourceEvents; + my $start = $sourceEvents->[0]->{start}; + my $end = $sourceEvents->[-1]->{end}; - my $c=0; - $c=$source::settings->{start_index}+0 if (defined $source::settings->{start_index}); - -# print "\n"; - print html_table_header() if ($output_type eq 'html'); - #order processing by start time (TODO: order by last-modified date) - for my $event (sort{$a->{calcms_start} cmp $b->{calcms_start}} @$source_events){ - target::pre_sync({ - start =>$source_events->[0]->{start}, - end =>$source_events->[-1]->{end} - }); + my $targetEvents = GoogleCalendar::getEvents( + { + start => $start, + end => $end + } + ); + $targetEvents = $targetEvents->{items}; + info "google:" . scalar(@$targetEvents) . " vs " . scalar(@$sourceEvents); - print ""if ($output_type eq 'html'); + # mark all known target events + my $targetEventsByKey = {}; + for my $event (@$targetEvents) { + #print Dumper($event); + next if $event->{status} eq 'canceled'; + my $key = getGoogleEventToString($event); + $targetEventsByKey->{$key} = $event; + } - #read event - $event=source::get_event_attributes($event); + # mark all knwon source events + my $sourceEventsByKey = {}; + for my $event (@$sourceEvents) { + $event = CalcmsEvents::mapToSchema($event); + $event = GoogleCalendar::mapToSchema($event); + my $key = getCalcmsEventToString($event); + $sourceEventsByKey->{$key} = $event; + } - #convert to calcms schema - $event=source::map_to_schema($event); + # delete target entries without matching source entries + for my $key ( keys %$targetEventsByKey ) { + next if defined $sourceEventsByKey->{$key}; + my $event = $targetEventsByKey->{$key}; + info "delete $key "; + print Dumper($event); + GoogleCalendar::deleteEvent($event); + } - #map event to target schema - $event=target::map_to_schema($event); - - #deprecated: override defined attributes by configuration - if ((defined $source::settings->{override}) && (ref($source::settings->{override})eq 'HASH')){ - for my $key (keys %{$source::settings->{override}}){ - my $value=$source::settings->{override}->{$key}; - if ($source::settings->{override} ne ''){ - print_info("override '$key'='$value'"); - $event->{event}->{$key}=$value; - } - } - } - - if ($output_type eq'html'){ - print_event_html("[".($c+1)."]",$event); - }else{ - print_event_text("[".($c+1)."]",$event); - } - - if ($event->{event}->{start} eq '' || $event->{event}->{end} eq ''){ - print ('WARNING: Cannot read start or end of event'); - print "\n"; - }else{ -# print Dumper($event); - sync_event($event); - } - - # last; - $event=undef; - $c++; - print ""if ($output_type eq 'html'); - } -# print "\n\n"; + # insert source entries without matching target entries + for my $key ( keys %$sourceEventsByKey ) { + if ( defined $targetEventsByKey->{$key} ) { + info "$key is up to date"; + next; + } + my $event = $sourceEventsByKey->{$key}; + info "insert $key"; + GoogleCalendar::insertEvent($event); + } } -#syncronize a single source event with target -sub sync_event{ - my $event=shift; +sub getGoogleEventToString { + my $event = shift; + my $result = "\n"; + $result .= "start: " . substr( $event->{start}->{dateTime}, 0, 19 ) . "\n"; + $result .= "end : " . substr( $event->{end}->{dateTime}, 0, 19 ) . "\n"; + $result .= "title: $event->{summary}\n"; + $result .= "desc : $event->{description}\n"; + return $result; +} - #look if target_event exists by reference id incl. recurrence counter - #print Dumper($event); - my $target_event=target::get_event_by_reference_id($event->{event}->{reference}); - - #if target_event exists - if (defined $target_event){ - #delete canceled events - if ($event->{event}->{status}eq'canceled'){ - print cell("delete canceled event:".qq{$target_event}); -# target::delete($target_event->{id}); - return; - } - - $event->{event_id}=$target_event->{id}; - - target::update_event($event,$target_event); - print cell("(ref. update)"); - - }else{ - #find by date, time and title - $target_event=target::find_event($event); - - if (defined $target_event){ - target::update_event($event,$target_event); - #print Dumper($event); - $event->{event_id}=$target_event->{id}; - print cell("(update)"); - }else{ - target::insert_event($event); - #print Dumper($event); - $target_event=target::get_event_by_reference_id($event->{event}->{reference}); - #print Dumper($target_event); - $event->{event_id}=$target_event->{id}; - print cell("(new)"); - } - } - print "\n"; - - for my $category (@{$event->{categories}}){ - target::assign_category_to_event($category,$event); - } - - for my $meta (@{$event->{meta}}){ - target::assign_meta_to_event($meta,$event); - } -# print Dumper($event); +sub getCalcmsEventToString { + my $event = shift; + my $result = "\n"; + $result .= "start: " . substr( $event->{event}->{start_datetime}, 0, 19 ) . "\n"; + $result .= "end : " . substr( $event->{event}->{end_datetime}, 0, 19 ) . "\n"; + $result .= "title: $event->{event}->{title}\n"; + $result .= "desc : $event->{event}->{content}\n"; + return $result; } #import requested source and target libs -sub init{ - binmode STDOUT, ":utf8"; +sub init { + binmode STDOUT, ":utf8"; - #require source config file - print_error ("missing source parameter!") unless ($source_config_file=~/\S/); - print_error ("source file: '$source_config_file' does not exist") unless (-e $source_config_file); - print_error ("cannot read source file: '$source_config_file'") unless (-r $source_config_file); - #$settings->{source}=require $source_config_file; - my $configuration = new Config::General($source_config_file); - $settings->{source}=$configuration->{DefaultConfig}->{source}; + { + #require target config file + error "missing target parameter!" unless $targetConfigFile =~ /\S/; + error "target file: '$targetConfigFile' does not exist" unless -e $targetConfigFile; + error "cannot read target file: '$targetConfigFile'" unless -r $targetConfigFile; + my $config = new Config::General($targetConfigFile); + $config = $config->{DefaultConfig}->{target}; + GoogleCalendar::init($config); - #require source import lib from config file - my $source_import_lib='lib/source/'.$settings->{source}->{type}.'.pl'; - print_error ("missing 'type' in 'source' config ") unless ($settings->{source}->{type}=~/\S/); - print_error ("cannot read source type import lib: '$source_import_lib'")unless (-r $source_import_lib); - require $source_import_lib; - - #require target config file - print_error ("missing target parameter!") unless ($target_config_file=~/\S/); - print_error ("target file: '$target_config_file' does not exist") unless (-e $target_config_file); - print_error ("cannot read target file: '$target_config_file'") unless (-r $target_config_file); - $configuration = new Config::General($target_config_file); - $settings->{target}=$configuration->{DefaultConfig}->{target}; - #$settings->{target}=require $target_config_file; - - #require target import lib from config file - my $target_import_lib='lib/target/'.$settings->{target}->{type}.'.pl'; - print_error ("missing 'type' in 'target' config ") unless ($settings->{target}->{type}=~/\S/); - print_error ("cannot read target type import lib: '$target_import_lib'")unless (-r $target_import_lib); - require $target_import_lib; - - #print Dumper($settings); - if ((defined $settings->{source}->{read_blocks}) && ($settings->{source}->{read_blocks}==1)){ - $settings->{source}->{block_number} =$block_number; - $settings->{source}->{block_size} =$block_size; - } - $settings->{source}->{last_update} =get_last_update_time($source_config_file,$target_config_file); - $settings->{source}->{modified_events} =$modified_events; - - if ($from=~/^\d\d\d\d\-\d\d\-\d\d$/){ - $from.='T00:00'; -# print "from:$from\t"; - } - - if ($till=~/^\d\d\d\d\-\d\d\-\d\d$/){ - $till.='T23:59'; -# print "till:$till\t"; - } - - if ($from=~/^([-+]?\d+$)/){ - my $days=$1; - my $duration=new DateTime::Duration(days=>$days); - $from=DateTime->today->add_duration($duration); -# print "from:$from\t"; - } - if ($till=~/^([-+]?\d+$)/){ - my $days=$1+1; - my $duration=new DateTime::Duration(days=>$days); - $till=DateTime->today->add_duration($duration); -# print "till:$till\t"; - - } - - $settings->{source}->{start_min} =$from if defined ($from); - $settings->{source}->{start_max} =$till if defined ($till); - - my $gmt_difference =0;#*=3600; - my $now =time(); - my $now_gmt =$now-$gmt_difference; - $now =time::time_to_datetime($now); - $now_gmt =time::time_to_datetime($now_gmt); - - $settings->{event}={ - update_start => time::time_to_datetime(time()), - modified_at => $now, - modified_at_gmt => $now_gmt - }; - - source::init($settings->{source}); - target::init($settings->{target}); - -} - -# print date/time, title and excerpt of an calendar event -# TODO: replace by output filter (text, html) -sub print_event_text{ - my $header=shift; - my $event=shift; - - my $s=$header; - $s=$s." "x (8-length($s)); - - my $start=$event->{event}->{start}||''; - $start=~s/T/ /g; - $start=~s/\:00$//g; - - if (defined $event->{event}->{program}){ - $s.="$start $event->{event}->{program}"; - $s=$s." "x (45-length($s)); } - if (defined $event->{event}->{series_name}){ - $s.=" : $event->{event}->{series_name}"; - $s=$s." "x (75-length($s)); + { + #require source config file + error "missing source parameter!" unless $sourceConfigFile =~ /\S/; + error "source file: '$sourceConfigFile' does not exist" unless -e $sourceConfigFile; + error "cannot read source file: '$sourceConfigFile'" unless -r $sourceConfigFile; + my $config = new Config::General($sourceConfigFile); + $config = $config->{DefaultConfig}->{source}; + $config->{last_update} = getLastUpdateTime( $sourceConfigFile, $targetConfigFile ); + CalcmsEvents::init($config); } - if (defined $event->{event}->{title}){ - $s.=" - $event->{event}->{title}"; - $s=$s." "x (110-length($s)); + $from .= 'T00:00' if $from =~ /^\d\d\d\d\-\d\d\-\d\d$/; + $till .= 'T23:59' if $till =~ /^\d\d\d\d\-\d\d\-\d\d$/; + + if ( $from =~ /^([-+]?\d+$)/ ) { + my $days = $1; + my $duration = new DateTime::Duration( days => $days ); + $from = DateTime->today->add_duration($duration); } - if ($event->{categories}){ - $s.= "(".join(", ",(@{$event->{categories}})).")"; - } - $s=$s." "x (135-length($s)); + if ( $till =~ /^([-+]?\d+$)/ ) { + my $days = $1 + 1; + my $duration = new DateTime::Duration( days => $days ); + $till = DateTime->today->add_duration($duration); + } - my $status=$event->{event}->{status}; - $s.=$status.' ' if (defined $status); - $s=$s." "x (140-length($s)); + CalcmsEvents::set( 'start_min', $from ) if defined $from; + CalcmsEvents::set( 'start_max', $till ) if defined $till; - my $reference=$event->{event}->{reference}; - $s.=substr($reference,length($reference)-25) if (defined $reference); + my $now = time(); + $now = time::time_to_datetime($now); + $settings->{event} = { + update_start => time::time_to_datetime( time() ), + modified_at => $now, + }; - print $s; -} - -sub print_event_html{ - my $header=shift; - my $event=shift; - - #close error block - my $s=''; - - my $start=$event->{event}->{start}||''; - $start=~s/T/ /g; - $start=~s/\:00$//g; - $s.=cell($start); - $s.=cell($event->{event}->{program}); - $s.=cell($event->{event}->{series_name}); - $s.=cell($event->{event}->{title}); - - if ($event->{categories}){ - $s.=cell( join(", " , ( @{$event->{categories}} ) ) ); - } - - my $status=$event->{event}->{status}; - $s.=cell($status) if (defined $status); - - my $reference=$event->{event}->{reference}; - $reference=substr($reference,length($reference)-25) if (defined $reference); - $s.=cell($reference); - - $s.=""; - - print $s; -} - -sub cell{ - if ($output_type eq 'html'){ - return "$_[0]"; - }else{ - return "\t".$_[0]; - }; } #output usage on error or --help parameter -sub print_usage{ - print qq{ +sub usage { + print qq{ update all/modified events from source at target. -USAGE: sync_cms.pl [--read,--update] [--modified,--all] --source s --target t [--block_number b] [--block_size s] +USAGE: sync_cms.pl [--read,--update] [--modified,--all] --source s --target t on using --from and --till requests will be processed as multiple single-day-requests. @@ -440,128 +238,55 @@ parameters: --read show all events without updating database --update update target database with source events - --modified process only modified events. - --all' process all events - --source source configuration file --target target configuration file --from start of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow) --till end of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow) - --output_type log output format [text,html] - - --block_number which block is to be syncronized [0..n]. To split up processing into multiple blocks (for machines with small memory resources). - --block_size size of a block, default=20 events examples: update modified - perl sync_cms.pl --update --modified --source=config/source/program.cfg --target=config/target/calcms.cfg + perl sync_cms.pl --update --source=config/source/program.cfg --target=config/target/calcms.cfg update a given time range perl sync_cms.pl --update --all --from=2009-09-01T00:00:00 --till=2009-11-22T23:59:59 --source=config/source/program.cfg --target=config/target/calcms.cfg update from last 2 days until next 3 days - perl sync_cms.pl --update --all --from=-2 --till=+3 --source=config/source/program.cfg --target=config/target/calcms.cfg + perl sync_cms.pl --update --from=-2 --till=+3 --source=config/source/program.cfg --target=config/target/calcms.cfg }; - exit 1; -}; - -#default error handling -sub print_error{ - print "\nERROR: $_[0]\n" ; - print_usage(); + exit 1; } -sub print_info{ - my $message=shift; - if ($message=~/^\n/){ - $message=~s/^\n//g; - print "\n"; - } - if ($output_type eq 'html'){ - print "$message
"; - }else{ - print "INFO:\t$message\n"; - } -} -sub html_table_header{ - return qq{ - - - start date - project - series - title - category - status - id - - action - - }; -}; - - #load last update time out of sync.data -sub get_last_update_time{ - my $source=shift; - my $target=shift; +sub getLastUpdateTime { + my $source = shift; + my $target = shift; - my $date=undef; - return undef unless(-r "sync.data"); - - open my $DATA, "<:utf8","sync.data" || die ('cannot read update timestamp'); - while (<$DATA>){ - my $line=$_; - if ($line=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){ - $date=$1; - last; - } - } - close $DATA; - return $date; + my $date = undef; + return undef unless -r "sync.data"; + my $content = Common::loadFile("sync.data"); + if ( $content =~ /$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/ ) { + $date = $1; + } + return $date; } #save last update time to sync.data -sub set_last_update_time{ - my $source =shift; - my $target =shift; - my $date =shift; +sub setLastUpdateTime { + my $source = shift; + my $target = shift; + my $date = shift; - my $data=''; - if (-r "sync.data"){ - open my $DATA, "<:utf8","sync.data"; - $data=join("\n",(<$DATA>)); - close $DATA; - } + my $data = ''; + if ( -r "sync.data" ) { + $data = Common::loadFile("sync.data"); + } - if ($data=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){ - $data=~s/($source\s+\->\s+$target\s+:)\s+\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}/$1\t$date/gi; - }else{ - $data.="$source\t\->\t$target\t:\t$date\n"; - } + if ( $data =~ /$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/ ) { + $data =~ s/($source\s+\->\s+$target\s+:)\s+\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}/$1\t$date/gi; + } else { + $data .= "$source\t\->\t$target\t:\t$date\n"; + } - $data=~s/[\r\n]+/\n/g; - - open my $DATA2, ">:utf8","sync.data" || die ('cannot write update timestamp'); - print $DATA2 $data; - close $DATA2; - -# print $data; + $data =~ s/[\r\n]+/\n/g; + Common::saveFile( "sync.data", $data ); } -#avoid to run more than one sync process in parallel -sub check_running_processes{ - my $cmd="ps -afex 2>/dev/null | grep sync_cms.pl | grep -v nice | grep -v grep "; - my $ps=`$cmd`; -# print "$ps"; - my @lines=(split(/\n/,$ps)); - if (@lines>1){ - print "ERROR: another ".@lines." synchronization processes 'sync_cms.pl' instances are running!".qq{ - -$cmd -$ps --> program will exit -}; - exit; - } - -} diff --git a/tools/sync_cms/sync_days.pl b/tools/sync_cms/sync_days.pl deleted file mode 100644 index 1f2ea48..0000000 --- a/tools/sync_cms/sync_days.pl +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/perl -I ../lib #-w - -BEGIN{ - my $dir=''; - $ENV{SCRIPT_FILENAME} if ($dir eq''); - $dir=~s/(.*\/)[^\/]+/$1/; - $dir=$ENV{PWD} if ($dir eq''); - $dir=`pwd` if ($dir eq''); - - #local perl installation libs - unshift(@INC,$dir.'/../../perl/lib/'); - - #calcms libs + configuration - unshift(@INC,$dir.'/../calcms/'); -} - -#use utf8; -use warnings "all"; -use strict; -use Data::Dumper; - -#use CGI; -#use HTML::Template; -use Date::Calc; -#use calendar; -#use time; -#use log; - -if(@ARGV<2){ - print qq{ERROR: $0 yyyy-mm-dd yyyy-mm-dd -syncronize from given start date to end date, day by day -}; - exit 1; -} - -my $start =$ARGV[0]; -my $end =$ARGV[1]; - -(my $start_year,my $start_month,my $start_day)=split(/\-/,$start); -my $last_day=Date::Calc::Days_in_Month($start_year,$start_month); -$start_day = 1 if ($start_day<1); -$start_day = $last_day if ($start_day gt $last_day); - -(my $end_year,my $end_month,my $end_day)=split(/\-/,$end); -$last_day=Date::Calc::Days_in_Month($end_year,$end_month); -$end_day = 1 if ($end_day<1); -$end_day = $last_day if ($end_day gt $last_day); - - - -for my $year($start_year..$end_year){ - my $m1=1; - my $m2=12; - $m1=$start_month if($year eq $start_year); - $m2=$end_month if($year eq $end_year); - - for my $month($m1..$m2){ - $month='0'.$month if (length($month)==1); - my $d1=1; - my $d2=Date::Calc::Days_in_Month($year,$month); - $d1=$start_day if($month eq $start_month); - $d2=$end_day if($month eq $end_month); - - for my $day($d1..$d2){ - $day='0'.$day if (length($day)==1); - my $date=join('-',($year,$month,$day)); - my $cmd="perl sync_cms.pl --update --all --source config/source/program.cfg --target config/target/calcms.cfg --from ".$date."T00:00:00 --till ".$date."T23:59:59"; - #print "$cmd\n"; - print `nice -n 10 $cmd`; - } - } - -} - diff --git a/tools/sync_cms/sync_jobs/calcms_to_google.sh b/tools/sync_cms/sync_jobs/calcms_to_google.sh new file mode 100755 index 0000000..70c0d40 --- /dev/null +++ b/tools/sync_cms/sync_jobs/calcms_to_google.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +from=$1 +till=$2 +project=$3 + +export LC_ALL="de_DE.utf8" +export LANGUAGE="de_DE.utf8" + +set -x +cd /home/radio/calcms/sync_cms +nice -n 10 perl sync_cms.pl --from=$from --till=$till --source=config/source/calcms_$project.cfg --target=config/target/88vier_$project.cfg 2>&1 + diff --git a/tools/sync_cms/sync_jobs/sync.sh b/tools/sync_cms/sync_jobs/sync.sh new file mode 100755 index 0000000..43b3cf0 --- /dev/null +++ b/tools/sync_cms/sync_jobs/sync.sh @@ -0,0 +1,7 @@ +#/bin/sh + +./sync_project.sh "$1" "$2" piradio +./sync_project.sh "$1" "$2" potsdam +./sync_project.sh "$1" "$2" ansage +./sync_project.sh "$1" "$2" collabo +./sync_project.sh "$1" "$2" frb diff --git a/tools/sync_cms/sync_jobs/update_ansage.sh b/tools/sync_cms/sync_jobs/update_ansage.sh new file mode 100755 index 0000000..fe37ceb --- /dev/null +++ b/tools/sync_cms/sync_jobs/update_ansage.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +file=/home/radio/piradio.de/agenda/admin/jobs/start/ansage_to_88vier.de.start.txt +touch $file +chown radio:www-data $file + diff --git a/tools/sync_cms/sync_jobs/update_colabo.sh b/tools/sync_cms/sync_jobs/update_colabo.sh new file mode 100755 index 0000000..904b5e1 --- /dev/null +++ b/tools/sync_cms/sync_jobs/update_colabo.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +file=/home/radio/piradio.de/agenda/admin/jobs/start/colabo_to_88vier.de.start.txt +touch $file +chown radio:www-data $file + diff --git a/tools/sync_cms/sync_jobs/update_frb.sh b/tools/sync_cms/sync_jobs/update_frb.sh new file mode 100755 index 0000000..0139861 --- /dev/null +++ b/tools/sync_cms/sync_jobs/update_frb.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +file=/home/radio/piradio.de/agenda/admin/jobs/start/frb_to_88vier.de.start.txt +touch $file +chown radio:www-data $file + diff --git a/tools/sync_cms/sync_jobs/update_piradio.sh b/tools/sync_cms/sync_jobs/update_piradio.sh new file mode 100755 index 0000000..ad7c170 --- /dev/null +++ b/tools/sync_cms/sync_jobs/update_piradio.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +file=/home/radio/piradio.de/agenda/admin/jobs/start/piradio_to_88vier.de.start.txt +touch $file +chown radio:www-data $file + diff --git a/tools/sync_cms/sync_jobs/update_potsdam.sh b/tools/sync_cms/sync_jobs/update_potsdam.sh new file mode 100755 index 0000000..86d30d8 --- /dev/null +++ b/tools/sync_cms/sync_jobs/update_potsdam.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +file=/home/radio/piradio.de/agenda/admin/jobs/start/potsdam_to_88vier.de.start.txt +touch $file +chown radio:www-data $file + diff --git a/tools/sync_cms/time_gate.pl b/tools/sync_cms/time_gate.pl deleted file mode 100644 index b83bc2b..0000000 --- a/tools/sync_cms/time_gate.pl +++ /dev/null @@ -1,487 +0,0 @@ -#!/usr/bin/perl -I ../lib #-w - -BEGIN{ - my $dir=''; - $ENV{SCRIPT_FILENAME}||'' if ($dir eq''); - $dir=~s/(.*\/)[^\/]+/$1/ if ($dir ne ''); - $dir=$ENV{PWD} if ($dir eq''); - $dir=`pwd` if ($dir eq''); - - #local perl installation libs - unshift(@INC,$dir.'/../../perl/lib/'); - - #calcms libs + configuration - unshift(@INC,$dir.'/../calcms/'); -} - -#use utf8; -use Data::Dumper; -#require 'time.pl'; -use Getopt::Long; -use time; -use DateTime; -use DateTime::Duration; -use strict; -use warnings; - -check_running_processes(); - - -my $read_mode=''; -my $update_mode=''; -my $all_events=''; -my $modified_events=''; -my $source_config_file=''; -my $target_config_file=''; -my $block_number=0; -my $block_size=2000; -my $from=''; -my $till=''; -my $read_only=0; -my $project=''; - -GetOptions( - "read" => \$read_mode, - "update" => \$update_mode, - "all" => \$all_events, - "modified" => \$modified_events, - "from=s" => \$from, - "till=s" => \$till, - "source=s" => \$source_config_file, - "target=s" => \$target_config_file, - "project=s" => \$project, - "block_number:i" => \$block_number, - "block_size:i" => \$block_size -); - -$|=1; - -BEGIN { - our $utf8dbi=1; - $ENV{LANG}="en_US.UTF-8"; -# print Dumper(\%ENV); -} - -#source and taget settings are loaded from config files -our $settings={ -}; - -#user interface -our $ask_before_insert=0; -our $ask_before_update=0; - -# end of configuration - -if ($update_mode){ - $db::write=1; -# print "enter update mode\n"; -}elsif($read_mode){ - #default - $db::write=0; -# print "enter read-only mode\n"; -}else{ - print_error("set parameter >read< or >update<"); -} - -unless ($modified_events || $all_events || $from || $till){ - print_error("set one of folling parameters: --modified, --from, --till"); -} - -init(); -my $project_target=$source::settings->{sources}->{$project}; -unless (defined $project){ - print_error("missing parameter --project") unless(defined $project_target); - print_error("cant find project configuration '$project_target'") unless (-f $project_target); - print_error("cant read project configuration '$project_target'") unless (-r $project_target); -} - -my $events=[]; -print "TIME_GATE: READ ALL CALENDARS\n"; -sync(); -$events=compress_events($events); -my $c=0; -if ($project eq ''){ - for my $event (@$events){ - print_event("[".($c+1)."]",$event); - print "\n"; - $c++; - } -}else{ - my $source=$source::settings->{sources}->{$project}; - my $target='config/target/calcms.cfg'; - - for my $event (@$events){ - my $from=$event->{start}; - #print Dumper($event->{end}); - #remove a second - my $till=source::get_datetime($event->{end}, $source::settings->{date}->{time_zone})->add(seconds=>-1)->datetime(); - print_event("STATION TIMESLOT [".($c+1)."]\t",$event); - print "\n"; - $c++; - my $command="perl sync_cms.pl --update --all --from=$from --till=$till --source $source --target $target "; - print_info($command); - print `$command`; - #exit; - } - -} -print "\ndone.\n"; -exit 0; - -sub compress_events{ - my $events=shift; - - my @results=(); - my $old_event={end=>'', start=>'', title=>''}; - for my $event(sort {$a->{start} gt $b->{start}} @$events){ -# print "$event->{start}\t$event->{end}\t$event->{title}\n"; - if ( -# (defined $event) && (defined $event->{start}) && (defined $event->{end}) && (defined $event->{title}) - ( #station continues - ($event->{start} eq $old_event->{end}) - || (#multiple entries for same event - ($event->{start} ge $old_event->{start}) - && ($event->{end} eq $old_event->{end}) - ) - ) - && ($event->{title} eq $old_event->{title}) - && (@results>0) - ){ - $results[-1]->{end}=$event->{end}; -# print @results."\tmerge \n"; - }else{ - push @results,{ - start => $event->{start}, - end => $event->{end}, - title => $event->{title}, - }; -# print @results."\tinsert \n"; - } - $old_event=$results[-1]; - } -# print Dumper(\@results); - - return \@results; -} - -#sync all events, splitting multi-day-requests into multiple 1-day-requests to avoid large result sets -sub sync{ - #prepare target - target::init($settings->{target}); - print_info("last update: $settings->{source}->{last_update}"); - - if (my $days=source::split_request($settings->{source})){ - #set 1-day start-min and start-max parameters, requires --from and --till values - for my $date (@$days){ - for my $key(keys %$date){ - $settings->{source}->{$key}=$date->{$key}; - } - print "\nrequest ".$settings->{source}->{"start_min"}." to ".$settings->{source}->{"start_max"}."\n"; - sync_timespan(); - } - }else{ - #update without time span (e.g. --modified) - sync_timespan(); - } - - print_info("\nset last-update time: $settings->{event}->{update_start}"); - set_last_update_time($source_config_file,$target_config_file,$settings->{event}->{update_start}); -} - -#sync all events of a given source timespan -sub sync_timespan{ - #get a list of all days and their events - #print Dumper($settings->{source}); - my $source_events=source::get_events($settings->{source},$settings->{target}); - #print Dumper($source_events); - my @dates=(keys %$source_events); - - if (@dates==0){ - my $more=''; - if ((defined $settings->{source}->{block_number}) && ($settings->{source}->{block_number} ne '0')){ - $more='more '; - }elsif ($modified_events){ - $more.='modified '; - } - print_info("\n".'no '.$more."entries found."); - }else{ - #sort lists of date and time (same time events should be preserved) - for my $date(sort {$a cmp $b} @dates){ -# for my $date(@dates){ -# print "\n$date:\n"; - sync_events($source_events->{$date}, $settings); - } - } - -} - -#syncronize a list of source events to target events -sub sync_events{ - my $source_events=shift; - my $settings=shift; - - my $c=0; - $c=$source::settings->{start_index}+0 if (defined $source::settings->{start_index}); - -# print "\n"; - - #order processing by start time (TODO: order by last-modified date) - for my $event (sort{$a->{calcms_start} cmp $b->{calcms_start}} @$source_events){ - #read event attributes - $event=source::get_event_attributes($event); - - $event->{title}=~s/\s//g; - - $event->{event}={ - title => $event->{title}, - start => $event->{start}, - end => $event->{end}, - status => $event->{status}, - }; - -# print "\n"; - #print_event("[".($c+1)."]",$event); - #print "\n".$event->{event}->{title}." ".$project."\n"; - - if ($event->{event}->{status}eq'canceled'){ - print "canceled event:".qq{$event}; - }elsif ($event->{event}->{start} eq ''){ - print ('WARNING: Cannot read start of event'."\n"); - }elsif ($event->{event}->{end} eq ''){ - print ('WARNING: Cannot read start of end'."\n"); - }elsif ($event->{event}->{title} eq ''){ - print ('WARNING: Cannot read start of title'."\n"); - }elsif ($project ne ''){ - if ($event->{event}->{title} eq $project){ - push @$events, $event->{event}; - } - }else{ - push @$events, $event->{event}; - } - $event=undef; - $c++; - } -} - - -#import requested source and target libs -sub init{ - binmode STDOUT, ":utf8"; - - #require source config file - print_error ("missing source parameter!") unless ($source_config_file=~/\S/); - print_error ("source file: '$source_config_file' does not exist") unless (-e $source_config_file); - print_error ("cannot read source file: '$source_config_file'") unless (-r $source_config_file); - #$settings->{source}=require $source_config_file; - my $configuration = new Config::General($source_config_file); - $settings->{source}=$configuration->{DefaultConfig}->{source}; - - #require source import lib from config file - my $source_import_lib='lib/source/'.$settings->{source}->{type}.'.pl'; - print_error ("missing 'type' in 'source' config ") unless ($settings->{source}->{type}=~/\S/); - print_error ("cannot read source type import lib: '$source_import_lib'")unless (-r $source_import_lib); - require $source_import_lib; - - #require target config file - print_error ("missing target parameter!") unless ($target_config_file=~/\S/); - print_error ("target file: '$target_config_file' does not exist") unless (-e $target_config_file); - print_error ("cannot read target file: '$target_config_file'") unless (-r $target_config_file); - #$settings->{target}=require $target_config_file; - $configuration = new Config::General($target_config_file); - $settings->{target}=$configuration->{DefaultConfig}->{target}; - - #require target import lib from config file - my $target_import_lib='lib/target/'.$settings->{target}->{type}.'.pl'; - print_error ("missing 'type' in 'target' config ") unless ($settings->{target}->{type}=~/\S/); - print_error ("cannot read target type import lib: '$target_import_lib'")unless (-r $target_import_lib); - require $target_import_lib; - - #print Dumper($settings); - if ((defined $settings->{source}->{read_blocks}) && ($settings->{source}->{read_blocks}==1)){ - $settings->{source}->{block_number} =$block_number; - $settings->{source}->{block_size} =$block_size; - } - $settings->{source}->{last_update} =get_last_update_time($source_config_file,$target_config_file); - $settings->{source}->{modified_events} =$modified_events; - - if ($from=~/^\d\d\d\d\-\d\d\-\d\d$/){ - $from.='T00:00'; - } - - if ($till=~/^\d\d\d\d\-\d\d\-\d\d$/){ - $till.='T23:59'; - } - - if ($from=~/^([-+]?\d+$)/){ - my $days=$1; - my $duration=new DateTime::Duration(days=>$days); - $from=DateTime->today->add_duration($duration); -# print "from:$from\t"; - } - if ($till=~/^([-+]?\d+$)/){ - my $days=$1+1; - my $duration=new DateTime::Duration(days=>$days); - $till=DateTime->today->add_duration($duration); -# print "till:$till\t"; - - } - - - $settings->{source}->{start_min} =$from if defined ($from); - $settings->{source}->{start_max} =$till if defined ($till); - - my $gmt_difference =0;#*=3600; - my $now =time(); - my $now_gmt =$now-$gmt_difference; - $now =time::time_to_datetime($now); - $now_gmt =time::time_to_datetime($now_gmt); - - $settings->{event}={ - update_start => time::time_to_datetime(time()), - modified_at => $now, - modified_at_gmt => $now_gmt - }; - source::init($settings->{source}); - -} - -# print date/time, title and excerpt of an calendar event -# TODO: replace by output filter (text, html) -sub print_event{ - my $header=shift; - my $event=shift; - - my $s=$header; - $s=$s." "x (8-length($s)); - -# print Dumper($event); - my $start=$event->{start}||''; - $start=~s/T/ /g; - $start=~s/\:00$//g; - - my $end=$event->{end}||''; - $end=~s/T/ /g; - $end=~s/\:00$//g; - - $s.="$start\t$end\t'$event->{title}'"; - -# print Dumper($event->{event}); - print $s; -#excerpt: >$event->{excerpt}< -#content: >$event->{content}< -#content: >$event->{content}< - -} - -#output usage on error or --help parameter -sub print_usage{ - print qq{ -update all/modified events from source at target. - -USAGE: $0 [--read,--update] [--modified,--all] --source s --target t [--block_number b] [--block_size s] - -on using --from and --till requests will be processed as multiple single-day-requests. - -parameters: - --read show all events without updating database - --update update target database with source events - - --modified process only modified events. - --all' process all events - - --source source configuration file - --target target configuration file - - --from start of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow) - --till end of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow) - - --block_number which block is to be syncronized [0..n]. To split up processing into multiple blocks (for machines with small memory resources). - --block_size size of a block, default=20 events - -examples: - perl $0 --update --modified --source=config/source/einheit.cfg --target=config/target/calcms.cfg - perl $0 --update --all --from=2009-09-01T00:00:00 --till=2009-11-22T23:59:59 --source=config/source/einheit.cfg --target=config/target/calcms.cfg -}; - exit 1; -}; - -#load last update time out of sync.data -sub get_last_update_time{ - my $source=shift; - my $target=shift; - - my $date=undef; - return undef unless(-r "sync.data"); - - open my $DATA, "<:utf8","sync.data" || die ('cannot read update timestamp'); - while (<$DATA>){ - my $line=$_; - if ($line=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){ - $date=$1; - last; - } - } - close $DATA; - return $date; -} - -#save last update time to sync.data -sub set_last_update_time{ - my $source =shift; - my $target =shift; - my $date =shift; - - my $data=''; - if (-r "sync.data"){ - open my $DATA, "<:utf8","sync.data"; - $data=join("\n",(<$DATA>)); - close $DATA; - } - - if ($data=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){ - $data=~s/($source\s+\->\s+$target\s+:)\s+\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}/$1\t$date/gi; - }else{ - $data.="$source\t\->\t$target\t:\t$date\n"; - } - - $data=~s/[\r\n]+/\n/g; - - open my $DATA2, ">:utf8","sync.data" || die ('cannot write update timestamp'); - print $DATA2 $data; - close $DATA2; - -# print $data; -} - -#default error handling -sub print_error{ - print "\nERROR:\t$_[0]\n" ; - print_usage(); -} - -sub print_info{ - my $message=shift; - if ($message=~/^\n/){ - $message=~s/^\n//g; - print "\n"; - } - print "INFO:\t$message\n"; -} - -#avoid to run more than one sync process simultaniously -sub check_running_processes{ - my $cmd="ps -afex 2>/dev/null | grep $0.pl | grep -v nice | grep -v grep "; - my $ps=`$cmd`; -# print "$ps"; - my @lines=(split(/\n/,$ps)); - if (@lines>1){ - print "ERROR:\tanother ".@lines." synchronization processes '$0.pl' instances are running!".qq{ - -$cmd -$ps --> program will exit -}; - exit; - } - -}