kalipso/racalmas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite Export to Google Calendar

Browse Source 
In the past Google Calendar exports were done by first removing all
events of a day and then create new ones. This takes a lot of time for
export and runs into Google Calendar usage limits after some time.

By now content will be compared before removing/creating a single event
one. To be able to do so, all other sync sources and targets have been
removed, so its only possible to export from database to Google Calendar
by this change.

To trigger an export you need to create a trigger file.
run_jobs.pl runs periodically e.g. started by cron and checks if a
trigger file exists and start sync_cms.pl to export the selected events
to the Google Calendar. Trigger files and jobs are configured at
jobs.config. Each job has a source and target file containing the access
data for calcms and the calendar.

Configuration files have been cleaned up. Old Accounts and passwords have
been removed. They hopefully should have been not active for a long time ;-)
This commit is contained in:
Milan
2018-09-22 22:47:00 +02:00
parent 4134ffb278
commit 56881b92d0
34 changed files with 935 additions and 2649 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
tools/sync_cms/INSTALL
View File

@@ -1,90 +0,0 @@
#install libxml2, libxml2-dev (for headers) to use XML::Atom (required by Net::Google::Calendar)
#install perl modules: DateTime, DateTime::TimeZone, XML::Atom, XML::Atom::Feed, Net::Google::Calendar
#if reading calendar fails, patch Entry line 184, from
# if ($elem->hasAttribute($key)) {
# to
# if (defined $elem && $elem->hasAttribute($key)) {
#
#patch Entry before line 184, insert
# return unless ($tmp);
#patch Entry line 176, modify
$val =~ s!^http://schemas.google.com/g/2005#event\.!! if (defined $val);
#admin,admin
#all available google calendar definitions, replace in url 'basic' by 'full' to get calendar entries!!! (basic covers feed content only, but no calendar data...)
# google_calendars => {
# programm => 'http://www.google.com/calendar/feeds/58ei894fakpf84hj0u7o6el4sc%40group.calendar.google.com/public/full',
# programm_intern => 'http://www.google.com/calendar/feeds/lin4mscfdld2eiv22qda82t478%40group.calendar.google.com/public/full',
# planung => 'http://www.google.com/calendar/feeds/0is4ruq5thsb6ndsqr5gicff2k%40group.calendar.google.com/public/full',
# termine_intern => 'http://www.google.com/calendar/feeds/1n762hqutnsocd46h6nji3i2l4%40group.calendar.google.com/public/full',
# termine => 'http://www.google.com/calendar/feeds/f29rqfutlkub911i8u0eerusb0%40group.calendar.google.com/public/full'
# },
GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, FILE, INDEX, ALTER ON * . * TO 'root'@'localhost' IDENTIFIED BY 'calcms' WITH MAX_QUERIES_PER_HOUR 0 MAX_CONNECTIONS_PER_HOUR 0 MAX_UPDATES_PER_HOUR 0 MAX_USER_CONNECTIONS 0;
Query OK, 0 rows affected (0.02 sec)
flush previleges
https://www.google.com/calendar/dav/peter_retep@gmx.de/events
create database calcms-herbstradio;
mysql calcms_herbstradio -u root -p < calcms/calcms.sql
GRANT SELECT ON * . * TO 'root'@'localhost' IDENTIFIED BY 'calcms-agenda' WITH MAX_QUERIES_PER_HOUR 0 MAX_CONNECTIONS_PER_HOUR 0 MAX_UPDATES_PER_HOUR 0 MAX_USER_CONNECTIONS 0;
flush previleges
CREATE USER 'milan'@'localhost' IDENTIFIED BY 'eeGei3Yo';
GRANT SELECT ON calcms_agenda.* TO 'calcms'@'localhost';
chmod 777 cache
GRANT SELECT ON calcms_herbstradio.* TO 'calcms_agenda'@'localhost' IDENTIFIED BY 'eeGei3Yo'
GRANT SELECT, INSERT, UPDATE ON calcms_herbstradio.* TO 'calcms'@'localhost' IDENTIFIED BY 'CheiBai8'
ERROR: Can't create '/usr/local/lib/perl5/5.8.6/man/man3'
Do not have write permissions on '/usr/local/lib/perl5/5.8.6/man/man3'
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
at /home/milan/perl/lib/Module/Build/Base.pm line 2975
SIMONW/Net-Google-Calendar-0.97.tar.gz
./Build install -- NOT OK
----
You may have to su to root to install the package
(Or you may want to run something like
o conf make_install_make_command 'sudo make'
to raise your permissions.Failed during this command:
DROLSKY/DateTime-Locale-0.43.tar.gz : install NO
DROLSKY/DateTime-TimeZone-0.91.tar.gz : install NO
DROLSKY/DateTime-0.50.tar.gz : make_test NO
SIMONW/Net-Google-AuthSub-0.5.tar.gz : install NO
GRANTM/XML-SAX-0.96.tar.gz : make_test NO
SIMONW/Net-Google-Calendar-0.97.tar.gz : install NO
See perldoc ExtUtils::MakeMaker for full details. For Module::Build
modules, you need to create a ~/.modulebuildrc file containing
bindoc=~/man/man1 libdoc=~/man/man3
o conf makepl_arg "PREFIX=/home/milan/perl/ LIB=/home/milan/perl/lib INST_LIB=/home/milan/perl/lib INSTALLSITELIB=/home/milan/perl/lib INSTALLMAN1DIR=/home/milan/perl/man/man1 INSTALLSITEMAN1DIR=/home/milan/perl/man/man1 INSTALLMAN3DIR=~/home/milan/perl/man/man3 INSTALLSITEMAN3DIR=/home/milan/perl/man/man3 INSTALLDIRS=/home/milan/perl/ SITEPREFIX=/home/milan/perl/ VENDORPREFIX=/home/milan/perl/"
LIB=$PREFIX/lib INST_LIB=$PREFIX/lib PREFIX=$PREFIX SITEPREFIX=$PREFIX VENDORPREFIX=$PREFIX
o conf make_arg -I/home/twiki/lib/CPAN
o conf make_install_arg -I/home/twiki/lib/CPAN
o conf makepl_arg "install_base=/home/twiki/lib/CPAN LIB=/home/twiki/lib/CPAN/lib INSTALLPRIVLIB=/home/twiki/lib/CPAN/lib INSTALLARCHLIB=/home/twiki/lib/CPAN/lib/arch INSTALLSITEARCH=/home/twiki/lib/CPAN/lib/arch INSTALLSITELIB=/home/twiki/lib/CPAN/lib INSTALLSCRIPT=/home/twiki/lib/CPAN/bin INSTALLBIN=/home/twiki/lib/CPAN/bin INSTALLSITEBIN=/home/twiki/lib/CPAN/bin INSTALLMAN1DIR=/home/twiki/lib/CPAN/man/man1 INSTALLSITEMAN1DIR=/home/twiki/lib/CPAN/man/man1 INSTALLMAN3DIR=/home/twiki/lib/CPAN/man/man3 INSTALLSITEMAN3DIR=/home/twiki/lib/CPAN/man/man3 "
o conf commit
q

18
tools/sync_cms/config/jobs.config Normal file
View File

@@ -0,0 +1,18 @@
<config>
start_dir /home/calcms/agenda/admin/jobs/start/
log_dir /home/calcms/agenda/admin/jobs/logs/
<job>
title potsdam : sender berlin -> 88vier.de
name potsdam_to_88vier.de
command /home/radio/calcms/sync_cms/sync_jobs/calcms_to_google.sh -7 31 potsdam
</job>
<job>
title frb : sender berlin -> 88vier.de
name frb_to_88vier.de
command /home/radio/calcms/sync_cms/sync_jobs/calcms_to_google.sh -7 31 frb
</job>
</config>

10
tools/sync_cms/config/source/calcms_ansage.cfg
View File

@@ -1,12 +1,10 @@
<source>
type calcms_i
<access>
hostname localhost
port 3306
database calcms_herbstradio
username calcms
password CheiBai8
database calcms
username calcms_read
password password
</access>
<date>
@@ -19,7 +17,7 @@
name 88vier
title 88vier Studio Ansage
start_date 2010-05-01
end_date 2016-06-01
end_date 2020-06-01
</88vier>
</projects>
location ansage

10
tools/sync_cms/config/source/calcms_colabo.cfg
View File

@@ -1,12 +1,10 @@
<source>
type calcms_i
<access>
hostname localhost
port 3306
database calcms_herbstradio
username calcms
password CheiBai8
database calcms
username calcms_read
password password
</access>
<date>
@@ -19,7 +17,7 @@
name 88vier
title 88vier Colaboradio
start_date 2010-05-01
end_date 2016-06-01
end_date 2020-06-01
</88vier>
</projects>
location colabo

16
tools/sync_cms/config/source/calcms.cfg → tools/sync_cms/config/source/calcms_frb.cfg
View File

@@ -1,12 +1,10 @@
<source>
type calcms_i
<access>
hostname localhost
port 3306
database calcms_herbstradio
username calcms
password CheiBai8
database calcms
username calcms_read
password password
</access>
<date>
@@ -17,15 +15,15 @@
<projects>
<88vier>
name 88vier
title 88vier PI-Radio 2010
title 88vier FRB
start_date 2010-05-01
end_date 2013-08-31
end_date 2020-06-01
</88vier>
</projects>
location piradio
location frb
<mapping>
event_details_url http://piradio.de/programm/sendung/<TMPL_VAR event_id>.html
event_details_url http://senderberlin.org/programm/sendung/<TMPL_VAR event_id>.html
</mapping>
<system>

10
tools/sync_cms/config/source/calcms_piradio.cfg
View File

@@ -1,12 +1,10 @@
<source>
type calcms_i
<access>
hostname localhost
port 3306
database calcms_herbstradio
username calcms
password CheiBai8
database calcms
username calcms_read
password password
</access>
<date>
@@ -19,7 +17,7 @@
name 88vier
title 88vier PI-Radio
start_date 2010-05-01
end_date 2016-06-01
end_date 2020-06-01
</88vier>
</projects>
location piradio

10
tools/sync_cms/config/source/calcms_potsdam.cfg
View File

@@ -1,12 +1,10 @@
<source>
type calcms_i
<access>
hostname localhost
port 3306
database calcms_herbstradio
username calcms
password CheiBai8
database calcms
username calcms_read
password password
</access>
<date>
@@ -19,7 +17,7 @@
name 88vier
title 88vier Frrapo
start_date 2010-05-01
end_date 2016-06-01
end_date 2020-06-01
</88vier>
</projects>
location potsdam

8
tools/sync_cms/config/target/88vier_ansage.cfg
View File

@@ -1,10 +1,8 @@
<target>
type google_calendar2
<access>
calendarId info@studioansage.de
serviceAccount 433089473368-bv26eveq03b7nhb9p62nu3ts7htgb4g3@developer.gserviceaccount.com
serviceAccountKeyFile /home/radio/googleApi.key
calendarId your-id
serviceAccount your-accounr@developer.gserviceaccount.com
serviceAccountKeyFile googleApi.key
</access>
<date>

8
tools/sync_cms/config/target/88vier_colabo.cfg
View File

@@ -1,10 +1,8 @@
<target>
type google_calendar2
<access>
calendarId colaboradio@gmail.com
serviceAccount 433089473368-bv26eveq03b7nhb9p62nu3ts7htgb4g3@developer.gserviceaccount.com
serviceAccountKeyFile /home/radio/googleApi.key
calendarId your-calendar-id
serviceAccount your-account@developer.gserviceaccount.com
serviceAccountKeyFile googleApi.key
</access>
<date>

21
tools/sync_cms/config/target/88vier_frb.cfg Normal file
View File

@@ -0,0 +1,21 @@
<target>
<access>
calendarId your-id@group.calendar.google.com
serviceAccount your-account@developer.gserviceaccount.com
serviceAccountKeyFile googleApi.key
</access>
<date>
time_zone Europe/Berlin
</date>
<mapping>
title <TMPL_VAR location> : <TMPL_VAR series_name> - <TMPL_VAR title>
content <TMPL_VAR excerpt> <a href="<TMPL_VAR event_details_url>">mehr zur Sendung</a>
</mapping>
<system>
debug 1
</system>
</target>

8
tools/sync_cms/config/target/88vier_piradio.cfg
View File

@@ -1,10 +1,8 @@
<target>
type google_calendar2
<access>
calendarId li6if8drs373kf9ttot7er6suc@group.calendar.google.com
serviceAccount 433089473368-bv26eveq03b7nhb9p62nu3ts7htgb4g3@developer.gserviceaccount.com
serviceAccountKeyFile /home/radio/googleApi.key
calendarId your-id@group.calendar.google.com
serviceAccount your-account@developer.gserviceaccount.com
serviceAccountKeyFile googleApi.key
</access>
<date>

33
tools/sync_cms/export_db.pl
View File

@@ -1,33 +0,0 @@
#!/usr/bin/perl
use strict;
use warnings;
use lib "../calcms";
use utf8;
use Data::Dumper;
use Config::General;
use Storable qw(nstore);
use db;
use config;
our $default={
configFile => '/home/radio/piradio.de/agenda/config/config.cgi',
timezone => 'Europe/Berlin',
local_media_url => 'http://piradio.de/agenda_files/media/',
project => '88vier',
location => 'piradio',
};
my $config = config::get($default->{configFile});
print Dumper($config);
my $dbh=db::connect($config);
my $query=q{
select * from calcms_events
order by start
};
my $events=db::get($dbh, $query);
nstore($events, 'event_export.dat');

17
tools/sync_cms/ical_html_to_ical_creole.pl
View File

@@ -1,17 +0,0 @@
require '../lib/text_markup.pl';
open FILE,"<$ARGV[0]";
while (<FILE>){
my $line=$_;
if ($line=~/^DESCRIPTION:/){
my $description=substr($line,length('DESCRIPTION:'));
my $html=markup::ical_to_plain($description);
my $creole=markup::html_to_creole($html);
my $ical=markup::plain_to_ical($creole);
$line= 'DESCRIPTION:'.$ical."\n";
}
print $line;
}
close FILE;

162
tools/sync_cms/import_ical.pl
View File

@@ -1,162 +0,0 @@
#!/usr/bin/perl
use strict;
use warnings;
use lib "../calcms";
use utf8;
use DateTime;
use Net::Google::Calendar;
use DateTime::Format::ICal;
use Data::Dumper;
use Config::General;
use db;
use config;
use creole_wiki;
use markup;
use events;
my $filename=$ARGV[0];
die("USAGE: $0 filename") unless defined $filename;
die("cannot read from '$filename'") unless -e $filename;
our $default={
configFile => '/home/radio/piradio.de/agenda/config/config.cgi',
timezone => 'Europe/Berlin',
local_media_url => 'http://piradio.de/agenda_files/media/',
project => '88vier',
location => 'piradio',
};
my $config = config::get($default->{configFile});
print Dumper($config);
parseICalFile($config, $filename);
our $active=0;
sub parseICalFile{
my $config=shift;
my $filename=shift;
print "open $filename\n";
open my $file, "<:encoding(UTF-8)", $filename;
my $parse=0;
my $event=undef;
my $lastKey=undef;
while (<$file>){
my $line=$_;
#print $parse." ".$line;
if ($line=~/^BEGIN\:VEVENT/){
$event={};
$parse=1;
#print "start event\n";
next;
}
if ($line=~/^END\:VEVENT/){
$parse=0;
processEvent($config, $event) if defined $event;
#print "end event\n";
next;
}
if ($line=~/^\s/){
my $key = $lastKey;
my $value = substr($line, 1);
$value=~s/[\r\n]+$//;
$event->{$key}.=$value;
$lastKey=$key;
next;
}else{
my ($key,$value)=split(/\:/,$line,2);
$value=~s/[\r\n]+$//;
$event->{$key}=$value;
$lastKey=$key;
}
}
close $file;
}
sub processEvent{
my $config=shift;
my $source=shift;
my $event={};
$event->{title} = $source->{SUMMARY};
$event->{content} = $source->{DESCRIPTION};
$event->{title} = markup::ical_to_plain($event->{title});
$event->{content} = markup::ical_to_plain($event->{content});
unless (defined $source->{DTSTART}){
print STDERR "missing DTSTART in ".Dumper($source);
return;
}
unless (defined $source->{DTEND}){
print STDERR "missing DTEND in ".Dumper($source);
return;
}
my $start = DateTime::Format::ICal->parse_datetime($source->{DTSTART});
$start=$start->set_time_zone($default->{timezone});
$event->{start} = $start->datetime();
my $end = DateTime::Format::ICal->parse_datetime($source->{DTEND});
$end = $end->set_time_zone($default->{timezone});
$event->{end} = $end->datetime();
my $params={
title => $event->{title},
content => $event->{content},
local_media_url => $default->{local_media_url}
};
#$params->{content}=~s/\x0A\x20/\n/g;
$event=creole_wiki::extractEventFromWikiText($params, $event);
$event->{project} = $default->{project};
$event->{location} = $default->{location};
return unless ($event->{start} ge '2015-09-01');
$active=1 if ($event->{series_name}=~/Brainwashed/);
print "$active $event->{start} $event->{series_name} - $event->{title}\n";
#saveEvent($config, $event);
#exit;
}
sub saveEvent{
my $config = shift;
my $event = shift;
$config->{access}->{write}=1;
my $dbh=db::connect($config);
$event->{'html_content'}=markup::creole_to_html($event->{'content'});
# set start date
my $day_start=$config->{date}->{day_starting_hour};
$event->{start_date} = time::add_hours_to_datetime($event->{start}, -$day_start);
$event->{start_date} = time::datetime_to_date($event->{start_date});
# set end date
$event->{end_date} = time::add_hours_to_datetime($event->{end}, -$day_start);
$event->{end_date} = time::datetime_to_date($event->{end_date});
delete $event->{categories} if defined $event->{categories};
# set time of day
my $day_times=$config->{date}->{time_of_day};
my $event_hour=int((split(/[\-\:\sT]/,$event->{start}))[3]);
for my $hour(sort {$a <=> $b} (keys %$day_times)){
if ($event_hour >= $hour){
$event->{time_of_day}=$day_times->{$hour};
}else{
last;
};
}
$event->{published}=0;
$event->{modified_by}='sync_cms';
print Dumper($event);
#db::insert($dbh,'calcms_events', $event);
}

135
tools/sync_cms/lib/CalcmsEvents.pm Normal file
View File

@@ -0,0 +1,135 @@
package CalcmsEvents;
use strict;
use warnings;
use Common ('info','error');
use DateTime;
use Data::Dumper;
use creole_wiki;
use events;
use time;
#use config;
my $settings = {};
sub init($) {
$settings = shift || {};
}
sub set($$) {
my $key = shift;
my $value = shift;
$settings->{$key} = $value;
}
sub get($) {
my $key = shift;
return $settings->{$key};
}
# return a list of start_min, start_max request parameters.
sub splitRequest($$$) {
my $from = shift;
my $till = shift;
my $timeZone = shift;
return undef unless defined $from;
return undef unless defined $till;
return undef if $from eq '';
return undef if $till eq '';
my $dates = [];
my $start = time::get_datetime( $from, $timeZone );
my $end = time::get_datetime( $till, $timeZone );
#build a list of dates
my $date = $start;
my @dates = ();
while ( $date < $end ) {
push @dates, $date;
$date = $date->clone->add( days => 7 );
}
my $duration = $end - $date;
push @dates, $end->clone if $duration->delta_seconds <= 0;
#build a list of parameters from dates
$start = shift @dates;
for my $end (@dates) {
push @$dates,
{
from => $start,
till => $end
};
$start = $end;
}
return $dates;
}
#get a hash with per-day-lists days of a google calendar, given by its url defined at $calendar_name
sub getEvents($$) {
my $from = shift;
my $till = shift;
my $last_update = get('last_update');
info "getEvents from $from till $till";
my $request_parameters = {
from_date => $from,
till_date => $till,
project => get('project'),
archive => 'all',
template => 'no'
};
my $location = get('location') || '';
$request_parameters->{location} = $location if $location ne '';
my $config = $settings;
my %params = ();
my $request = {
url => $ENV{QUERY_STRING},
params => {
original => \%params,
checked => events::check_params( $config, $request_parameters, $settings ),
},
};
my $sourceEvents = events::get( $config, $request, $settings );
#return events by date
my $eventsByDate = {};
for my $source (@$sourceEvents) {
$source->{calcms_start} = $source->{start};
my $key = substr( $source->{start}, 0, 10 );
push @{ $eventsByDate->{$key} }, $source;
}
return $eventsByDate;
}
sub mapToSchema {
my $event = shift;
#override settings by source map filter
for my $key ( keys %{ get('mapping') } ) {
$event->{$key} = get('mapping')->{$key};
}
#resolve variables set in mapped values
for my $mkey ( keys %{ get('mapping') } ) {
for my $key ( keys %{$event} ) {
my $val = $event->{$key};
$val = $event->{$key} if ( $mkey eq $key );
$event->{$mkey} =~ s/<TMPL_VAR $key>/$val/g;
}
}
return $event;
}
#do not delete last line
1;

75
tools/sync_cms/lib/Common.pm Normal file
View File

@@ -0,0 +1,75 @@
package Common;
use warnings;
use strict;
use Fcntl ':flock';
use base 'Exporter';
our @EXPORT_OK = ( 'info', 'error' );
sub checkSingleInstance() {
open my $self, '<', $0 or die "Couldn't open self: $!";
flock $self, LOCK_EX | LOCK_NB or die "This script $0 is already running";
}
sub loadFile($) {
my $filename = shift;
my $content = '';
open my $file, '<', $filename || die("cannot load $filename");
while (<$file>) {
$content .= $_;
}
close $file;
return $content;
}
sub saveFile($$) {
my $filename = shift;
my $content = shift;
open my $file, ">:utf8", $filename || die("cannot write $filename");
print $file $content;
close $file;
}
sub getModifiedAt {
my $file = shift;
my @stats = stat $file;
return 0 if scalar @stats == 0;
my $modifiedAt = $stats[9];
return $modifiedAt;
}
sub execute($) {
my $command = shift;
print "EXEC:\t$command\n";
my $result = `$command`;
my $exitCode = ( $? >> 8 );
print "ERROR! exitCode=$?\n" if $exitCode > 0;
return ( $exitCode, $result );
}
my $debug = 0;
sub debug($$) {
my $level = shift;
my $message = shift;
print $message. "\n" if $debug > $level;
}
sub error ($) {
print "\nERROR: $_[0]\nsee $0 --help for help";
exit 1;
}
sub info ($) {
my $message = shift;
if ( $message =~ /^\n/ ) {
$message =~ s/^\n//g;
print "\n";
}
print "INFO:\t$message\n";
}
return 1;

197
tools/sync_cms/lib/GoogleCalendar.pm Normal file
View File

@@ -0,0 +1,197 @@
package GoogleCalendar;
use strict;
use warnings;
use Data::Dumper;
use lib '../calcms/';
use Common ( 'info', 'error' );
use GoogleCalendarApi();
use time();
my $settings = {};
my $cal = undef;
my $debug = 1;
sub set($$) {
my $key = shift;
my $value = shift;
$settings->{$key} = $value;
}
sub get($) {
my $key = shift;
return $settings->{$key};
}
sub init($) {
$settings = shift || {};
my $access = get('access');
# 1. create service account at https://console.developers.google.com/
# 2. enable Calendar API
# 3. share calendar with service account for update permissions
# see http://search.cpan.org/~shigeta/Google-API-Client-0.13/lib/Google/API/Client.pm
my $serviceAccount = $access->{serviceAccount};
my $serviceAccountKeyFile = $access->{serviceAccountKeyFile};
my $calendarId = $access->{calendarId};
my $serviceAccountKey = Common::loadFile($serviceAccountKeyFile);
my $calendar = GoogleCalendarApi->new(
{
'serviceAccount' => $serviceAccount,
'privateKey' => $serviceAccountKey,
'calendarId' => $calendarId,
'debug' => 0
}
);
$cal = $calendar;
}
#map event schema to target schema
sub mapToSchema {
my $event = shift;
#clone event
my $targetEvent = {};
for my $key ( keys %{$event} ) {
$targetEvent->{$key} = $event->{$key};
}
if ( defined $event->{recurrence} && ref( $event->{recurrence} ) eq 'HASH' ) {
$targetEvent->{reference} .= '[' . $event->{recurrence}->{number} . ']' if ( $event->{recurrence}->{number} > 0 );
$targetEvent->{recurrence} = $event->{recurrence}->{number} + 0;
}
$targetEvent->{rating} = 0;
$targetEvent->{visibility} = 0;
#set project by project's date range
my $projects = get('projects');
if ( ref($projects) eq 'HASH' ) {
for my $projectName ( keys %$projects ) {
my $project = get('projects')->{$projectName};
my $start = substr( $event->{start}, 0, 10 );
if ( $start ge $project->{start_date} && $start le $project->{end_date} ) {
$targetEvent->{project} = $project->{name};
}
}
}
#override settings by target map filter
for my $key ( keys %{ get('mapping') } ) {
$targetEvent->{$key} = get('mapping')->{$key};
}
#resolve variables set in mapped values
for my $mkey ( keys %{ get('mapping') } ) {
for my $key ( sort keys %{$targetEvent} ) {
my $val = $targetEvent->{$key};
$val = $event->{$key} if $mkey eq $key;
$targetEvent->{$mkey} =~ s/<TMPL_VAR $key>/$val/g;
}
}
$targetEvent->{title} =~ s/\s+$//g;
$targetEvent->{title} =~ s/\s*\#$//g;
$targetEvent->{title} =~ s/\s*\-\s*$//g;
my $schema = { event => $targetEvent };
return $schema;
}
#this is done before sync and allows to delete old events before adding new
sub getEvents {
my $event = shift;
return undef if get('date')->{'time_zone'} eq '';
return undef if $event->{start} eq '';
return undef if $event->{end} eq '';
#delete a span of dates
my $timeZone = get('date')->{'time_zone'};
my $start = time::get_datetime( $event->{start}, $timeZone );
my $end = time::get_datetime( $event->{end}, $timeZone );
info( "search target for events from " . $start . " to " . $end );
#search datetime with same timezone
my $events = $cal->getEvents(
{
timeMin => $cal->getDateTime( $start->datetime, $timeZone ),
timeMax => $cal->getDateTime( $end->datetime, $timeZone ),
maxResults => 50,
singleEvents => 'true',
orderBy => 'startTime'
}
);
return $events;
}
# insert a new event
sub insertEvent {
my $event = shift;
my $entity = $event->{event};
$entity->{'html_content'} = markup::creole_to_html( $entity->{'content'} );
my $timeZone = get('date')->{'time_zone'};
my $start = $cal->getDateTime( $entity->{start}, $timeZone );
my $end = $cal->getDateTime( $entity->{end}, $timeZone );
#info "insert event\t$start\t$entity->{title}";
my $entry = {
start => $start,
end => $end,
summary => $entity->{title},
description => $entity->{content},
location => $entity->{location},
transparency => 'transparent',
status => 'confirmed'
};
my $result = $cal->insertEvent($entry);
my $id = $result->{id};
}
sub deleteEvent {
my $event = shift;
#info "delete event";
$cal->deleteEvent( $event->{id} );
}
sub fixFields {
my $event = shift;
#lower case for upper case titles longer than 4 characters
for my $attr ( 'series_name', 'title' ) {
my $val = $event->{$attr};
my $c = 0;
while ( $val =~ /\b([A-Z]{5,99})\b/ && $c < 10 ) {
my $word = $1;
my $lower = lc $word;
$lower =~ s/^([a-z])/\u$1/gi;
$val =~ s/$word/$lower/g;
$c++;
}
$event->{$attr} = $val if $event->{$attr} ne $val;
}
for my $attr ( 'series_name', 'title', 'excerpt', 'content' ) {
my $val = $event->{$attr};
$val =~ s/^\s*(.*?)\s*$/$1/g;
$val =~ s/^[ \t]/ /g;
$event->{$attr} = $val if $event->{$attr} ne $val;
}
return $event;
}
#do not delete last line
1;

219
tools/sync_cms/lib/GoogleCalendarApi.pm
View File

@@ -2,6 +2,7 @@ package GoogleCalendarApi;
use strict;
use warnings;
use JSON;
use JSON::WebToken;
use LWP::UserAgent;
@@ -9,34 +10,38 @@ use HTML::Entities;
use URI::Escape;
use Data::Dumper;
use DateTime;
use Time::HiRes qw(sleep);
use Common ( 'info', 'error' );
sub new {
my $class = shift;
my $params = shift;
my $self={};
for my $attr ('calendarId','debug'){
$self->{$attr}=$params->{$attr} if defined $params->{$attr};
#print Dumper($class);
my $self = {};
for my $attr ( 'calendarId', 'debug' ) {
$self->{$attr} = $params->{$attr} if defined $params->{$attr};
}
$self->{debug} = 1;
my $instance=bless $self, $class;
if ((defined $params->{serviceAccount}) && (defined $params->{privateKey})){
$instance->login($params->{serviceAccount}, $params->{privateKey});
my $instance = bless $self, $class;
if ( ( defined $params->{serviceAccount} ) && ( defined $params->{privateKey} ) ) {
$instance->login( $params->{serviceAccount}, $params->{privateKey} );
}
return $instance;
}
sub setCalendar{
my $self=shift;
my $calendarId=shift;
$self->{calendarId}=$calendarId;
sub setCalendar {
my $self = shift;
my $calendarId = shift;
$self->{calendarId} = $calendarId;
}
sub getBasicUrl{
my $self=shift;
return 'https://www.googleapis.com/calendar/v3/calendars/'.encode_entities($self->{calendarId});
sub getBasicUrl {
my $self = shift;
return 'https://www.googleapis.com/calendar/v3/calendars/' . encode_entities( $self->{calendarId} );
}
#https://developers.google.com/google-apps/calendar/v3/reference/events/list
@@ -52,149 +57,161 @@ sub getBasicUrl{
# 'updated' => '2014-11-12T19:46:22.086Z',
# 'items' => [...]
# }
sub getEvents{
my $self=shift;
my $params=shift;
sub getEvents {
my $self = shift;
my $params = shift;
my $url='/events?';
for my $param ('iCalUID','alwaysIncludeEmail','maxAttendees','maxResults','orderBy','pageToken','privateExtendedProperty',
'q','sharedExtendedProperty','showDeleted','showHiddenInvitations','singleEvents','syncToken','timeZone'
){
$url.='&'.$param.'='.uri_escape($params->{$param}) if defined $params->{$param};
my $url = '/events?';
for my $param (
'iCalUID', 'alwaysIncludeEmail', 'maxAttendees', 'maxResults',
'orderBy', 'pageToken', 'privateExtendedProperty', 'q',
'sharedExtendedProperty', 'showDeleted', 'showHiddenInvitations', 'singleEvents',
'syncToken', 'timeZone'
)
{
$url .= '&' . $param . '=' . uri_escape( $params->{$param} ) if defined $params->{$param};
}
for my $param ('timeMin','timeMax','updatedMin'){
$url.='&'.$param.'='.uri_escape($self->formatDateTime($params->{$param})) if defined $params->{$param};
for my $param ( 'timeMin', 'timeMax', 'updatedMin' ) {
$url .= '&' . $param . '=' . uri_escape( $self->formatDateTime( $params->{$param} ) ) if defined $params->{$param};
}
my $result=$self->httpRequest('GET', $url);
my $result = $self->httpRequest( 'GET', $url );
return $result;
}
# sleep 0.25 seconds to prevent hitting the 5.0 requests/second/user rate
sub sleep{
my $this=shift;
my $duration=shift || 0.25;
select(undef, undef, undef, $duration);
}
#sub sleep{
# my $this=shift;
# my $duration=shift;
# $duration=1 unless defined $duration;
# select(undef, undef, undef, $duration);
#}
#https://developers.google.com/google-apps/calendar/v3/reference/events/delete
sub deleteEvent{
my $self=shift;
my $eventId=shift;
my $url='/events/'.$eventId;
sub deleteEvent {
my $self = shift;
my $eventId = shift;
my $url = '/events/' . $eventId;
#DELETE https://www.googleapis.com/calendar/v3/calendars/calendarId/events/eventId
my $result=$self->httpRequest('DELETE', $url);
$self->sleep();
my $result = $self->httpRequest( 'DELETE', $url );
#$self->sleep();
return $result;
}
#https://developers.google.com/google-apps/calendar/v3/reference/events/insert
sub insertEvent{
my $self=shift;
my $params=shift;
sub insertEvent {
my $self = shift;
my $params = shift;
my $event={
my $event = {
start => {
dateTime => $self->formatDateTime($params->{start})
dateTime => $self->formatDateTime( $params->{start} )
},
end => {
dateTime => $self->formatDateTime($params->{end})
end => {
dateTime => $self->formatDateTime( $params->{end} )
},
summary => $params->{summary}||'',
description => $params->{description}||'',
location => $params->{location}||'',
status => $params->{confirmed}||'confirmed'
summary => $params->{summary} || '',
description => $params->{description} || '',
location => $params->{location} || '',
status => $params->{confirmed} || 'confirmed'
};
$event= encode_json $event;
$event = encode_json $event;
#POST https://www.googleapis.com/calendar/v3/calendars/calendarId/events
my $url='/events';
my $result=$self->httpRequest('POST', $url, $event);
$self->sleep();
my $url = '/events';
my $result = $self->httpRequest( 'POST', $url, $event );
#$self->sleep();
return $result;
}
# send a HTTP request
sub httpRequest{
my $self=shift;
my $method=shift;
my $url=shift;
my $content=shift||'';
sub httpRequest {
my $self = shift;
my $method = shift;
my $url = shift;
my $content = shift || '';
print STDERR "$method ".$url."\n" if $self->{debug};
sleep 0.3;
print STDERR "$method " . $url . "\n" if $self->{debug};
die ("missing url") unless defined $url;
die ("calendarId not set") unless defined $self->{calendarId};
die ("not logged in ") unless defined $self->{api};
die("missing url") unless defined $url;
die("calendarId not set") unless defined $self->{calendarId};
die("not logged in ") unless defined $self->{api};
#prepend basic url including calendar id
$url=$self->getBasicUrl().$url;
print STDERR "$method ".$url."\n" if $self->{debug};
$url = $self->getBasicUrl() . $url;
print STDERR "$method " . $url . "\n" if $self->{debug};
my $response=undef;
if($method eq 'GET'){
my $response = undef;
if ( $method eq 'GET' ) {
$response = $self->{api}->get($url);
}elsif(($method eq 'POST')||($method eq 'PUT')){
print STDERR $content."\n" if $self->{debug};
} elsif ( ( $method eq 'POST' ) || ( $method eq 'PUT' ) ) {
#return;
print STDERR $content . "\n" if $self->{debug};
my $request = HTTP::Request->new( $method, $url );
$request->header( 'Content-Type' => 'application/json' );
$request->content( $content );
$response=$self->{api}->request( $request );
}elsif($method eq 'DELETE'){
$request->content($content);
$response = $self->{api}->request($request);
} elsif ( $method eq 'DELETE' ) {
#return;
$response = $self->{api}->delete($url);
}
if($response->is_success) {
if ( $response->is_success ) {
my $content = $response->content;
return {} if $content eq '';
return decode_json($content);
} else {
print "ERROR:\n";
print "Code: ".$response->code."\n";
print "Message: ".$response->message."\n";
print $response->content."\n";
print "Code: " . $response->code . "\n";
print "Message: " . $response->message . "\n";
print $response->content . "\n";
die;
}
}
# write datetime object to string
sub formatDateTime{
my $self=shift;
my $dt=shift;
sub formatDateTime {
my $self = shift;
my $dt = shift;
my $datetime= $dt->format_cldr("yyyy-MM-ddTHH:mm:ssZZZZZ");
my $datetime = $dt->format_cldr("yyyy-MM-ddTHH:mm:ssZZZZZ");
print STDERR "$dt -> $datetime\n" if $self->{debug};
return $datetime;
}
# parse datetime from string to object
sub getDateTime{
my $self=shift;
my $datetime=shift;
my $timezone=shift;
sub getDateTime {
my $self = shift;
my $datetime = shift;
my $timezone = shift;
return if((!defined $datetime) or ($datetime eq ''));
my @l=split /[\-\;T\s\:\+\.]/,$datetime;
return if ( !defined $datetime ) or ( $datetime eq '' );
my @l = split /[\-\;T\s\:\+\.]/, $datetime;
$datetime=DateTime->new(
year => $l[0],
month => $l[1],
day => $l[2],
hour => $l[3],
minute => $l[4],
second => $l[5],
time_zone => $timezone
);
return $datetime;
$datetime = DateTime->new(
year => $l[0],
month => $l[1],
day => $l[2],
hour => $l[3],
minute => $l[4],
second => $l[5],
time_zone => $timezone
);
return $datetime;
}
# login with serviceAccount and webToken (from privateKey)
sub login{
my $self=shift;
my $serviceAccount=shift;
my $privateKey=shift;
sub login {
my $self = shift;
my $serviceAccount = shift;
my $privateKey = shift;
# https://developers.google.com/accounts/docs/OAuth2ServiceAccount
my $time = time;
#create JSON Web Token
my $jwt = JSON::WebToken->encode(
{
@@ -206,7 +223,7 @@ sub login{
},
$privateKey,
'RS256',
{typ => 'JWT'}
{ typ => 'JWT' }
);
#send JSON web token to authentication service
@@ -215,16 +232,16 @@ sub login{
'https://accounts.google.com/o/oauth2/token',
{
grant_type => encode_entities('urn:ietf:params:oauth:grant-type:jwt-bearer'),
assertion => $jwt
assertion => $jwt
}
);
die($response->code, "\n", $response->content, "\n") unless $response->is_success();
my $data= decode_json($response->content);
die( $response->code, "\n", $response->content, "\n" ) unless $response->is_success();
my $data = decode_json( $response->content );
#create a new user agent and set token to bearer
$self->{api} = LWP::UserAgent->new();
$self->{api}->default_header(Authorization => 'Bearer ' . $data->{access_token});
$self->{api}->default_header( Authorization => 'Bearer ' . $data->{access_token} );
print STDERR "login successful\n" if $self->{debug};
return $data;

133
tools/sync_cms/lib/source/calcms_i.pl
View File

@@ -1,133 +0,0 @@
#use markup;
use creole_wiki;
use DateTime;
use events;
use time;
use config;
#use DateTime::Format::ICal;
package source;
use Data::Dumper;
my $settings={};
sub init{
$source::settings=shift;
#print STDERR Dumper($source::settings);
}
#return a list of start_min, start_max request parameters. list is defined as timespan given by start_min and start_max in source_options
sub split_request{
return undef if (
(!(defined $source::settings->{start_min})) || ($source::settings->{start_min} eq'')
||(!(defined $source::settings->{start_max})) || ($source::settings->{start_max} eq'')
);
#print Dumper($source_options);
my $dates=[];
my $start =time::get_datetime($source::settings->{start_min},$source::settings->{date}->{time_zone});
my $end =time::get_datetime($source::settings->{start_max},$source::settings->{date}->{time_zone});
my $date =$start;
#build a list of dates
my @dates=();
while ($date < $end){
push @dates,$date;
$date=$date->clone->add(days=>7);
}
my $duration=$end-$date;
# print "sec:".($duration->delta_seconds/(60*60))."\n";
if ($duration->delta_seconds <= 0){
# pop @dates;
push @dates,$end->clone;
}
#build a list of parameters from dates
my $start=shift @dates;
for my $end (@dates){
push @$dates,{
start_min => $start,
start_max => $end
};
$start=$end;
}
# for $day(@$dates){print "$day->{start_min} - $day->{start_max}\n";}
return $dates;
}
#get a hash with per-day-lists days of a google calendar, given by its url defined at $calendar_name
sub get_events{
my $block_number =$source::settings->{block_number};
my $block_size =$source::settings->{block_size};
my $last_update =$source::settings->{last_update};
#print Dumper($request);
my $request_parameters={
from_date => $source::settings->{start_min},
till_date => $source::settings->{start_max},
archive => 'all',
project => $source::settings->{project},
template => 'no'
};
$request_parameters->{location}=$source::settings->{location} if ($source::settings->{location}ne'');
my $config = $source::settings;
my $request={
url => $ENV{QUERY_STRING},
params => {
original => \%params,
checked => events::check_params($config,
$request_parameters,
$source::settings
),
},
};
#print Dumper($request);
my $source_events=events::get($config, $request, $source::settings);
#print Dumper($source_events);
#return events by date
my $sources_by_date={};
my $old_start='';
for my $source (@$source_events){
$source->{calcms_start}=$source->{start};
my $key=substr($source->{start},0,10);
push @{$sources_by_date->{$key}},$source;
}
return $sources_by_date;
}
sub get_event_attributes{
my $source=shift;
return $source;
}
sub map_to_schema{
my $event=shift;
# print Dumper($source_options);
# exit;
#override settings by source map filter
for my $key (keys %{$source::settings->{mapping}}){
$event->{$key}=$source::settings->{mapping}->{$key};
}
#resolve variables set in mapped values
for my $mkey (keys %{$source::settings->{mapping}}){
for my $key (keys %{$event}){
my $val=$event->{$key};
$val=$event->{$key} if($mkey eq $key);
$event->{$mkey}=~s/<TMPL_VAR $key>/$val/g;
}
}
return $event;
}
eof;

339
tools/sync_cms/lib/source/google_calendar.pl
View File

@@ -1,339 +0,0 @@
#use markup;
use creole_wiki;
use DateTime;
use Net::Google::Calendar;
use DateTime::Format::ICal;
package source;
#do 'time.pl';
use Data::Dumper;
my $settings={};
sub init{
$source::settings=shift;
}
#return a list of start_min, start_max request parameters.
#list is defined as timespan given by start_min and start_max in source::settings
sub split_request{
return undef if (
(!(defined $source::settings->{start_min})) || ($source::settings->{start_min} eq'')
||(!(defined $source::settings->{start_max})) || ($source::settings->{start_max} eq'')
);
my $dates=[];
my $start =get_datetime($source::settings->{start_min},$source::settings->{date}->{time_zone});
my $end =get_datetime($source::settings->{start_max},$source::settings->{date}->{time_zone});
my $date =$start;
#build a list of dates
my @dates=();
while ($date < $end){
push @dates,$date;
$date=$date->clone->add(days=>7);
}
my $duration=$end-$date;
# print "sec:".($duration->delta_seconds/(60*60))."\n";
if ($duration->delta_seconds <= 0){
# pop @dates;
push @dates,$end->clone;
}
#build a list of parameters from dates
my $start=shift @dates;
for my $end (@dates){
push @$dates,{
start_min => $start,
start_max => $end
};
$start=$end;
}
# for $day(@$dates){print "$day->{start_min} - $day->{start_max}\n";}
return $dates;
}
#get a hash with per-day-lists days of a google calendar, given by its url defined at $calendar_name
sub get_events{
# print Dumper($source::settings);
my $url =$source::settings->{access}->{url};
my $email =$source::settings->{access}->{email};
my $password =$source::settings->{access}->{password};
my $block_number =$source::settings->{block_number};
my $block_size =$source::settings->{block_size};
my $last_update =$source::settings->{last_update};
my $parameters={};
my $start_index=undef;
my $stop_index=undef;
if ($source::settings->{read_blocks}==1){
my $start_index=$block_number*$block_size+1 ;
my $stop_index=$start_index+$block_size-1;
$parameters->{"start-index"} = $start_index;
$parameters->{"max-results"} = $block_size;
$source::settings->{start_index}=$start_index;
$source::settings->{stop_index}=$stop_index;
}else{
$parameters->{"max-results"} = 10000;
}
#see http://code.google.com/intl/de/apis/calendar/data/2.0/reference.html
$parameters->{singleevents}='true';
$parameters->{orderby}='lastmodified';
my $more='modified' if (defined $last_update && $source::settings->{modified_events}==1);
main::print_info("read $more events from google calendar: '".substr($url,0,40)."...".substr($url,length($url)-8)."'");
# print "\nblock '$block_number' (events ".$start_index."..".$stop_index.") \n" if (defined $block_number || defined $start_index || defined $stop_index);
# http://search.cpan.org/~simonw/Net-Google-Calendar-0.97/lib/Net/Google/Calendar.pm#get_events_[_%opts_]
my $cal = Net::Google::Calendar->new( url => $url );
#main::print_info("new\n");
if ($email ne'' && $password ne''){
$cal->login($email, $password) ;
# $cal->auth($email, $password) if ($email ne'' && $password ne'');
# main::print_info("login $email $password");
}
#print Dumper($cal);
#set UTF-8
$XML::Atom::ForceUnicode = 1;
$XML::Atom::DefaultVersion = "1.0";
# my $xml=$cal->get_xml();
# $xml=~s/<content/\n<content/gi;
# print $xml."\n";
# exit;
#set updated-min (using UTC)
if ((defined $last_update) && ($source::settings->{modified_events}==1)){
my $datetime=$last_update;
$datetime=source::get_datetime($datetime,$source::settings->{date}->{time_zone}) if (ref($datetime)eq'');
$datetime->set_time_zone('UTC');
$parameters->{"updated-min"} = $datetime->datetime;
#print "last update\n";
}
#set start min (using UTC)
if ((defined $source::settings->{start_min}) && ($source::settings->{start_min}ne'')){
my $datetime=$source::settings->{start_min};
$datetime=source::get_datetime($datetime,$source::settings->{date}->{time_zone}) if (ref($datetime)eq'');
$datetime->set_time_zone('UTC');
$parameters->{"start-min"} = $datetime->datetime;
$parameters->{"recurrence-expansion-start"}= $datetime->datetime;
}
#set start max (using UTC)
if ((defined $source::settings->{start_max})&&($source::settings->{start_max} ne'')){
my $datetime=$source::settings->{start_max};
$datetime=source::get_datetime($datetime,$source::settings->{date}->{time_zone}) if (ref($datetime)eq'');
$datetime->set_time_zone('UTC');
$parameters->{"start-max"} = $datetime->datetime;
$parameters->{"recurrence-expansion-end"}= $datetime->datetime;
}
# print Dumper($parameters);
my @events=();
my @source_events=$cal->get_events(%$parameters);
main::print_info("found ".@source_events." events");
# print Dumper($parameters);
# print Dumper($source::settings);
# exit;
for my $source(@source_events) {
(my $start,my $end)=$source->when;
$start= $start->set_time_zone($source::settings->{date}->{time_zone})->datetime if (defined $start);
$end= $end->set_time_zone ($source::settings->{date}->{time_zone})->datetime if (defined $end);
$source->{calcms_start} = $start;
$source->{calcms_end} = $end;
$source->{status} = $source->status;
}
#return events by date
my $sources_by_date={};
my $old_start='';
# for my $source (sort{$a->{calcms_start} cmp $b->{calcms_start} }@source_events){
for my $source (@source_events){
# if ($source->{status}eq'confirmed'){
my $key=substr($source->{calcms_start},0,10);
# if ($old_start eq $source->{calcms_start}){
# my $source=pop (@{$sources_by_date->{$key}});
# print STDERR "WARNING: ignore canceled entry in google calendar: ".$source->{calcms_start}."\t".$source->{title}."\t".$source->{id}."\n";
# }
#
push @{$sources_by_date->{$key}},$source;
#
# $old_start=$source->{calcms_start};
# }
}
return $sources_by_date;
}
sub map_to_schema{
my $event=shift;
my $params={
title => $event->{title},
content => $event->{content},
local_media_url => '<TMPL_VAR local_media_url>'
};
$params->{content}=~s/\x0A\x20/\n/g;
#print Dumper($params);
#open FILE,">/tmp/test";
#print FILE Dumper($params);
#close FILE;
#decode event
$event=creole_wiki::extractEventFromWikiText($params, $event);
#exit;
#override settings by source map filter
for my $key (keys %{$source::settings->{mapping}}){
$event->{$key}=$source::settings->{mapping}->{$key};
}
#resolve variables set in mapped values
for my $mkey (keys %{$source::settings->{mapping}}){
for my $key (keys %{$event}){
my $val=$event->{$key};
$val=$event->{$key} if($mkey eq $key);
$event->{$mkey}=~s/<TMPL_VAR $key>/$val/g;
}
}
#print Dumper($event);
return $event;
}
sub get_event_attributes{
my $source=shift;
#print @source_events." ".Dumper($source)."\n";
#use Data::Dumper;print Dumper($source->when);
#create an hash with calendar event settings
my $event={
start => $source->{calcms_start},
end => $source->{calcms_end},
status => $source->{status},
# recurrence => $source->{recurrence},
reference => $source->id,
# program => $program,
# series_name => $series_name,
title => $source->title,
content => $source->content->body,
author_name => $source->author->name,
author_uri => $source->author->uri,
author_email => $source->author->email,
transparency => $source->transparency,
visibility => $source->visibility,
location => $source->location,
# podcast_url => $podcast_url,
# media_url => $media_url,
# comments => $source->comments
# who_name => $source->who->name,
# who_email => $source->who->email,
# who_attendee_status => $source->who->attendee_status,
};
#print Dumper($event);
# if ($source->recurrence){
# $event->{recurrence}=get_event_recurrence($source,$event);
# }
return $event;
}
sub get_event_recurrence{
my $source=shift;
my $event=shift;
#print Dumper();
my $event_recurrence=$source->recurrence;
my $properties = $event_recurrence->properties;
# print Dumper($properties);
my $dtstart = $properties->{dtstart}->[0]->{value};
my $timezone = $properties->{dtstart}->[0]->{_parameters}->{TZID};
my $dtend = $properties->{dtend}->[0]->{value};
my $rrule = $properties->{rrule}->[0]->{value};
# print $rrule."\n";
#convert timezone from "until=<datetime>" to same datetime as in dtstart
if ($rrule=~/UNTIL=([\dT]+Z?)/){
my $ical=$1;
#convert timezone at ical format
my $datetime= DateTime::Format::ICal->parse_datetime($ical);
$datetime=$datetime->set_time_zone($timezone);
$ical=DateTime::Format::ICal->format_datetime($datetime);
#remove TZID=... from ical, since not implemented at format_datetime
$ical=~s/[^\:]+\://;
$rrule=~s/(UNTIL\=)([\dT]+Z?)/$1$ical/g;
# print $datetime->datetime." --> $ical --> $rrule\n";
}
$dtstart = DateTime::Format::ICal->parse_datetime($dtstart);
$dtend = DateTime::Format::ICal->parse_datetime($dtend);#->add(seconds=>3600)->set_time_zone('UTC');
my $recurrence={
dtstart => $dtstart,
dtend => $dtend,
rrule => $rrule
};
#calc duration of the event
my $duration=$dtend-$dtstart;
my $duration_min=$duration->delta_minutes;
# print Dumper($duration_min);
#print Dumper($recurrence);
my $recurrence_start = DateTime::Format::ICal->parse_recurrence(
recurrence =>$rrule,
dtstart =>$dtstart
);
#step through recurrent events and mark if event matchs
my $start_iter = $recurrence_start->iterator;
$c=1;
while (my $start = $start_iter->next ){
# print "$start eq $event->{start}, $end\n";
$recurrence->{number}=$c if ($start eq $event->{start});
# push @dates,{
# start => $start->set_time_zone($source::settings->{time_zone})->datetime,
# end => $start->set_time_zone($source::settings->{time_zone})->add(minutes=>$duration_min)->datetime
# };
$c++;
}
$event->{recurrence}=$recurrence;
#print Dumper($event->{recurrence});
}
sub get_datetime{
my $datetime=shift;
my $timezone=shift;
return if((!defined $datetime) or ($datetime eq ''));
my @l=@{time::datetime_to_array($datetime)};
$datetime=DateTime->new(
year =>$l[0],
month =>$l[1],
day =>$l[2],
hour =>$l[3],
minute =>$l[4],
second =>$l[5],
time_zone=> $timezone
);
return $datetime;
}
eof;

233
tools/sync_cms/lib/target/google_calendar.pl
View File

@@ -1,233 +0,0 @@
#require 'db.pl';
#use db;
#use markup;
package target;
use Data::Dumper;
use Net::Google::Calendar;
use time;
my $settings={};
my $cal = undef;
#my $op_count=0;
sub init{
$target::settings=shift;
my $access=$target::settings->{access};
$target::cal = Net::Google::Calendar->new( url => $access->{url} );
#main::print_info("init\n");
#main::print_info("new\n");
#print Dumper($access);
my $email=$access->{email};
my $password=$access->{password};
if ($email ne'' && $password ne''){
$target::cal->login($email, $password) ;
# $target::cal->auth($email, $password) if ($email ne'' && $password ne'');
main::print_info("loged in");
}
#print Dumper($target::cal);
# for my $c($target::cal->get_calendars) {
# print "'".$c->title."'\n";
# print $c->id."\n\n";
# if ($c->title eq 'petra poss'){
# $target::cal->set_calendar($c);
# main::print_info("found matching calendar!");
# }
# }
# exit;
#set UTF-8
$XML::Atom::ForceUnicode = 1;
$XML::Atom::DefaultVersion = "1.0";
}
#map event schema to target schema
sub map_to_schema{
my $event=shift;
#clone event
my $target_event={};
for my $key (keys %{$event}){
$target_event->{$key}=$event->{$key};
}
$target_event->{reference}.='['.$event->{recurrence}->{number}.']' if ($event->{recurrence}->{number}>0);
$target_event->{recurrence} => $event->{recurrence}->{number}+0;
$target_event->{rating} => 0;
$target_event->{visibility} => 0;
# $target_event->{transparency} => $event->{transparency};
#set project by project's date range
for my $project_name (keys %{$target::settings->{projects}}){
my $project=$target::settings->{projects}->{$project_name};
my $start=substr($event->{start},0,10);
if ($start ge $project->{start_date} && $start le $project->{end_date}){
$target_event->{project}=$project->{name};
}
# print "$event->{start} gt $project->{start_date} $target_event->{project}\n";
}
#override settings by target map filter
for my $key (keys %{$target::settings->{mapping}}){
$target_event->{$key}=$target::settings->{mapping}->{$key};
}
#use Data::Dumper;print Dumper($target_event);
#resolve variables set in mapped values
for my $mkey (keys %{$target::settings->{mapping}}){
my $mval=$target_event->{$mkey};
for my $key (sort keys %{$target_event}){
my $val=$target_event->{$key};
$val=$event->{$key} if($mkey eq $key);
#print $target_event->{$mkey}."\t".$key."-> $val\n";
$target_event->{$mkey}=~s/<TMPL_VAR $key>/$val/g;
}
}
#use Data::Dumper;print Dumper($target_event);#exit;
#$schema->{event}=fix_fields($schema->{event});
my $schema={
event => $target_event
};
return $schema;
}
# get a event by an existing google id, e.g. to check if the event exists in target
sub get_event_by_reference_id{
return undef;
}
#try to find a event, matching to $event from google calendar
sub find_event{
my $event=shift;
return undef;
}
sub pre_sync{
my $event=shift;
$debug=1;
return undef if(($target::settings->{date}->{'time_zone'} eq '') || ($event->{start} eq '' ) || ($event->{end} eq ''));
#delete a span of dates
print "\n" if ($debug eq '1');
my $time_zone=$target::settings->{date}->{'time_zone'};
my $start=time::get_datetime($event->{start},$time_zone);
$start->set_time_zone('UTC');
$parameters->{"start-min"} = $start->datetime;
#$parameters->{"recurrence-expansion-start"}= $start->datetime;
my $end=time::get_datetime($event->{end},$time_zone);
$end->set_time_zone('UTC');
$parameters->{"start-max"} = $end->datetime;
#$parameters->{"recurrence-expansion-end"}= $end->datetime;
main::print_info("search target for events from ".$start." to ".$end) if ($debug eq '1');
my @events=$target::cal->get_events(%$parameters);
for my $event(@events){
main::print_info("delete ".$event->title) if ($debug eq '1');
$target::cal->delete_entry($event);
};
}
# insert a new event
sub insert_event{
my $event=shift;
my $entity=$event->{event};
$entity->{'html_content'}=markup::creole_to_html($entity->{'content'});
my $time_zone =$target::settings->{date}->{'time_zone'};
my $start =time::get_datetime($entity->{start},$time_zone);
my $end =time::get_datetime($entity->{end},$time_zone);
#print Dumper($start)."\n";
#print Dumper($end)."\n";
print "\n" if ($debug eq '1');
main::print_info("insert event") if ($debug eq '1');
my $entry = Net::Google::Calendar::Entry->new();
#print Dumper($entity);
$entry->title($entity->{title});
$entry->content($entity->{content});
$entry->location($entity->{location});
$entry->transparency('transparent');
$entry->status('confirmed');
$entry->when($start, $end);
#print Dumper($entry);
$target::cal->add_entry($entry);
#exit;
}
# update an existing event
sub update_event{
return;
}
### end of interface implementation ###
sub print_event{
my $header=shift;
my $event=shift;
if ($header eq'google'){
print "\n===== $header =====";
}else{
print "$header\n" if $header ne '';
}
# print qq!$event->{start} $event->{program} : $event->{series_name} - $event->{title}!."\n";
#content: >$event->{content}<
};
sub delete_event{
return;
}
sub fix_fields{
my $event=shift;
#lower case for upper case titles longer than 4 characters
for my $attr qw(program series_name title){
my $val=$event->{$attr};
my $c=0;
while ($val=~/\b([A-Z]{5,99})\b/ && $c<10){
my $word=$1;
my $lower=lc $word;
$lower=~s/^([a-z])/\u$1/gi;
$val=~s/$word/$lower/g;
$c++;
}
if ($event->{$attr} ne $val){
$event->{$attr}=$val;
# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ;
}
}
for my $attr qw(program series_name title excerpt content ){
my $val=$event->{$attr};
$val=~s/^\s*(.*?)\s*$/$1/g;
$val=~s/^[ \t]/ /g;
if ($event->{$attr} ne $val){
$event->{$attr}=$val;
# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ;
}
}
return $event;
}
sub clean_up{
return;
}
1;

254
tools/sync_cms/lib/target/google_calendar2.pl
View File

@@ -1,254 +0,0 @@
#require 'db.pl';
#use db;
#use markup;
package target;
use lib '/home/radio/calcms/sync_cms/lib/';
use Data::Dumper;
#use Net::Google::Calendar;
use GoogleCalendarApi;
use time;
my $settings={};
my $cal = undef;
#my $op_count=0;
sub init{
$target::settings=shift;
my $access=$target::settings->{access};
# 1. create service account at https://console.developers.google.com/
# 2. enable Calendar API
# 3. share calendar with service account for update permissions
# see http://search.cpan.org/~shigeta/Google-API-Client-0.13/lib/Google/API/Client.pm
my $serviceAccount = $access->{serviceAccount};
my $serviceAccountKeyFile = $access->{serviceAccountKeyFile};
my $calendarId = $access->{calendarId};
my $serviceAccountKey = loadFile($serviceAccountKeyFile);
#print "connect...\n";
my $calendar = new GoogleCalendarApi({
'serviceAccount' => $serviceAccount,
'privateKey' => $serviceAccountKey,
'calendarId' => $calendarId,
'debug' => 0
});
#print Dumper($calendar);
$target::cal = $calendar;
}
#map event schema to target schema
sub map_to_schema{
my $event=shift;
#clone event
my $target_event={};
for my $key (keys %{$event}){
$target_event->{$key}=$event->{$key};
}
$target_event->{reference}.='['.$event->{recurrence}->{number}.']' if ($event->{recurrence}->{number}>0);
$target_event->{recurrence} => $event->{recurrence}->{number}+0;
$target_event->{rating} => 0;
$target_event->{visibility} => 0;
# $target_event->{transparency} => $event->{transparency};
#set project by project's date range
for my $project_name (keys %{$target::settings->{projects}}){
my $project=$target::settings->{projects}->{$project_name};
my $start=substr($event->{start},0,10);
if ($start ge $project->{start_date} && $start le $project->{end_date}){
$target_event->{project}=$project->{name};
}
# print "$event->{start} gt $project->{start_date} $target_event->{project}\n";
}
#override settings by target map filter
for my $key (keys %{$target::settings->{mapping}}){
$target_event->{$key}=$target::settings->{mapping}->{$key};
}
#use Data::Dumper;print Dumper($target_event);
#resolve variables set in mapped values
for my $mkey (keys %{$target::settings->{mapping}}){
my $mval=$target_event->{$mkey};
for my $key (sort keys %{$target_event}){
my $val=$target_event->{$key};
$val=$event->{$key} if($mkey eq $key);
#print $target_event->{$mkey}."\t".$key."-> $val\n";
$target_event->{$mkey}=~s/<TMPL_VAR $key>/$val/g;
}
}
#use Data::Dumper;print Dumper($target_event);#exit;
#$schema->{event}=fix_fields($schema->{event});
my $schema={
event => $target_event
};
return $schema;
}
# get a event by an existing google id, e.g. to check if the event exists in target
sub get_event_by_reference_id{
return undef;
}
#try to find a event, matching to $event from google calendar
sub find_event{
my $event=shift;
return undef;
}
#this is done before sync and allows to delete old events before adding new
sub pre_sync{
my $event=shift;
$debug=1;
return undef if(($target::settings->{date}->{'time_zone'} eq '') || ($event->{start} eq '' ) || ($event->{end} eq ''));
#delete a span of dates
print "\n" if ($debug eq '1');
my $timeZone=$target::settings->{date}->{'time_zone'};
#get datetime in timezone
my $start = time::get_datetime($event->{start}, $timeZone);
my $end = time::get_datetime($event->{end}, $timeZone);
main::print_info("search target for events from ".$start." to ".$end) if ($debug eq '1');
my $events=$target::cal->getEvents({
#search datetime with same timezone
timeMin => $target::cal->getDateTime($start->datetime, $timeZone),
timeMax => $target::cal->getDateTime($end->datetime, $timeZone),
maxResults => 50,
singleEvents => 'true',
orderBy => 'startTime'
});
my $now=DateTime->now()->set_time_zone('UTC')->epoch();
#print Dumper($now->datetime);
#exit;
for my $event(@{$events->{items}}){
main::print_info("delete\t$event->{start}->{dateTime}\t".$event->{summary}) if ($debug eq '1');
#my $updated = $target::cal->getDateTime($event->{updated},'UTC')->epoch();
#my $delta = $now-$updated;
#print $delta." seconds old\n";
$target::cal->deleteEvent($event->{id})
};
#exit;
}
# insert a new event
sub insert_event{
my $event=shift;
my $entity=$event->{event};
$entity->{'html_content'}=markup::creole_to_html($entity->{'content'});
my $timeZone = $target::settings->{date}->{'time_zone'};
#print Dumper($timeZone);
#print Dumper($entity);
my $start = $target::cal->getDateTime($entity->{start}, $timeZone);
my $end = $target::cal->getDateTime($entity->{end}, $timeZone);
print "\n" if ($debug eq '1');
#exit;
main::print_info("insert event\t$start\t$entity->{title}") if ($debug eq '1');
my $entry = {
start => $start,
end => $end,
summary => $entity->{title},
description => $entity->{content},
location => $entity->{location},
transparency => 'transparent',
status => 'confirmed'
};
my $result=$target::cal->insertEvent($entry);
my $id=$result->{id};
#exit;
}
sub loadFile{
my $filename=shift;
my $content='';
open my $file, '<', $filename || die("cannot load $filename");
while(<$file>){
$content.=$_;
}
close $file;
return $content;
}
# update an existing event
sub update_event{
return;
}
### end of interface implementation ###
sub print_event{
my $header=shift;
my $event=shift;
if ($header eq'google'){
print "\n===== $header =====";
}else{
print "$header\n" if $header ne '';
}
# print qq!$event->{start} $event->{program} : $event->{series_name} - $event->{title}!."\n";
#content: >$event->{content}<
};
sub delete_event{
return;
}
sub fix_fields{
my $event=shift;
#lower case for upper case titles longer than 4 characters
for my $attr qw(program series_name title){
my $val=$event->{$attr};
my $c=0;
while ($val=~/\b([A-Z]{5,99})\b/ && $c<10){
my $word=$1;
my $lower=lc $word;
$lower=~s/^([a-z])/\u$1/gi;
$val=~s/$word/$lower/g;
$c++;
}
if ($event->{$attr} ne $val){
$event->{$attr}=$val;
# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ;
}
}
for my $attr qw(program series_name title excerpt content ){
my $val=$event->{$attr};
$val=~s/^\s*(.*?)\s*$/$1/g;
$val=~s/^[ \t]/ /g;
if ($event->{$attr} ne $val){
$event->{$attr}=$val;
# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ;
}
}
return $event;
}
sub clean_up{
return;
}
1;

195
tools/sync_cms/lib/target/playlist_csv.pl
View File

@@ -1,195 +0,0 @@
package target;
use Data::Dumper;
use time;
use warnings;
use strict;
my $settings={};
my $cal = undef;
sub init{
$target::settings=shift;
my $access=$target::settings->{access};
$cal = [];
}
#map event schema to target schema
sub map_to_schema{
my $event=shift;
#clone event
my $target_event={};
for my $key (keys %{$event}){
$target_event->{$key}=$event->{$key};
}
$event->{recurrence}->{number}=0 unless (defined $event->{recurrence} || defined $event->{recurrence}->{number});
$target_event->{reference}.='['.$event->{recurrence}->{number}.']' if ($event->{recurrence}->{number}>0);
$target_event->{recurrence} => $event->{recurrence}->{number};
$target_event->{rating} => 0;
$target_event->{visibility} => 0;
# $target_event->{transparency} => $event->{transparency};
#set project by project's date range
for my $project_name (keys %{$target::settings->{projects}}){
my $project=$target::settings->{projects}->{$project_name};
my $start=substr($event->{start},0,10);
if ($start ge $project->{start_date} && $start le $project->{end_date}){
$target_event->{project}=$project->{name};
}
}
#override settings by target map filter
for my $key (keys %{$target::settings->{mapping}}){
$target_event->{$key}=$target::settings->{mapping}->{$key};
}
#resolve variables set in mapped values
for my $mkey (keys %{$target::settings->{mapping}}){
my $mval=$target_event->{$mkey};
for my $key (keys %{$target_event}){
my $val=$target_event->{$key};
$val=$event->{$key} if($mkey eq $key);
$target_event->{$mkey}=~s/<TMPL_VAR $key>/$val/g;
}
}
my $schema={
event => $target_event
};
return $schema;
}
# get a event by an existing reference id, e.g. to check if the event exists in target
sub get_event_by_reference_id{
my $event_id=shift;
my $event={};
return undef;
}
#try to find a event
sub find_event{
my $event=shift;
return undef;
}
# insert a new event
sub insert_event{
my $event=shift;
my $entity=$event->{event};
my $time_zone =$target::settings->{date}->{'time_zone'};
my $start =time::get_datetime($entity->{start},$time_zone);
my $end =time::get_datetime($entity->{end},$time_zone);
print "\n" if ($main::debug eq '1');
main::print_info("insert event") if ($main::debug eq '1');
push @$cal,{
start => $start,
end => $end,
title => $entity->{title}
}
#exit;
}
# update an existing event
sub update_event{
my $event=shift;
my $entity=shift;
}
### end of interface implementation ###
sub print_event{
my $header=shift;
my $event=shift;
if ($header eq'google'){
print "\n===== $header =====";
}else{
print "$header\n" if $header ne '';
}
# print qq!$event->{start} $event->{program} : $event->{series_name} - $event->{title}!."\n";
#content: >$event->{content}<
};
sub delete_event{
my $event_id=shift;
}
sub fix_fields{
my $event=shift;
for my $attr qw(title){
my $val=$event->{$attr};
$val=~s/^\s*(.*?)\s*$/$1/g;
$val=~s/^[ \t]/ /g;
if ($event->{$attr} ne $val){
$event->{$attr}=$val;
# print Dumper($event->{$attr}).'<>'.Dumper($val)."\n" ;
}
}
return $event;
}
sub pre_sync{
}
sub clean_up{
my $content='';
my @cal=sort {$a->{start} cmp $b->{end}} @$cal;
my @cal2=();
#print Dumper(\@cal);
#fill in default
if (defined $target::settings->{date}->{default_entry}){
my $from=$main::from;
if ($from=~/^\d\d\d\d\-\d\d\-\d\dT\d\d$/){
$from.=':00';
}
my $till=$main::till;
if ($till=~/^\d\d\d\d\-\d\d\-\d\dT\d\d$/){
$till.=':59';
}
my $default=$target::settings->{date}->{default_entry};
if ($cal[0]->{start} gt $from){
unshift @cal,{
start => $from,
end => $cal[0]->{start},
title => $default
}
}
if ($cal[-1]->{end} lt $till){
push @cal,{
start => $cal[-1]->{end},
end => $till,
title => $default
}
}
my $old_event={end=>$from};
for my $event (@cal){
if ($event->{start} gt $old_event->{end}){
push @cal2,{
start => $old_event->{end},
end => $event->{start},
title => $default
}
}
push @cal2,$event;
$old_event=$event;
}
}
for my $event(@cal2){
$content.= $event->{start}.";\t".$event->{end}.";\t".$event->{title}."\n";
}
log::save_file($target::settings->{access}->{file},$content);
return;
}
1;

81
tools/sync_cms/run_jobs.pl Executable file
View File

@@ -0,0 +1,81 @@
#! /usr/bin/perl
use strict;
use warnings;
use Data::Dumper;
use FindBin;
use lib "$FindBin::Bin/lib";
use lib "$FindBin::Bin/../calcms";
use Common ( 'info', 'error' );
use config();
use time();
use log();
$| = 1;
sub runJobs {
my $jobs = shift;
my $startDir = shift;
my $logDir = shift;
for my $job (@$jobs) {
my $startFile = $startDir . '/' . $job->{name} . '.start.txt';
my $startAge = Common::getModifiedAt($startFile);
next if $startAge == 0;
my $logFile = $logDir . '/' . $job->{name} . '.log';
my $logAge = Common::getModifiedAt($logFile);
next if $startAge < $logAge;
# read parameters form start file
my $content = log::load_file($startFile);
#execute command
my $command = $job->{command} . ' 2>&1 > ' . $logFile;
my ( $exitCode, $result ) = Common::execute($command);
error "exitCode=$exitCode on $command" if $exitCode != 0;
}
}
sub check() {
my $configFile = shift @ARGV;
error qq{cannot read $configFile "$configFile"} unless -e $configFile;
my $config = config::get($configFile);
my $startDir = $config->{start_dir} || '';
error 'missing configuration of jobs/start_dir!' if $startDir eq '';
error "job dir does not exist '$startDir'" unless -e $startDir;
error "cannot read from job dir '$startDir'. Please check permissions!" unless -w $startDir;
my $logDir = $config->{log_dir} || '';
error 'missing configuration of jobs/log_dir' if $logDir eq '';
error "job log dir does not exist '$logDir'" unless -e $logDir;
error "cannot read from job log dir '$logDir'. Please check permissions!" unless -r $logDir;
error "cannot write to job log dir '$logDir'. Please check permissions!" unless -w $logDir;
my $jobs = $config->{job};
error "no jobs defined!" if scalar @$jobs == 0;
return ( $jobs, $startDir, $logDir );
}
sub main() {
info "INIT\t" . time::time_to_datetime();
Common::checkSingleInstance();
my ( $jobs, $startDir, $logDir ) = check();
#exit after a at most 10 minute timeout in case of hanging process
local $SIG{ALRM} = sub { die "ERROR: exit due to synchronization hangs\n" };
alarm 10 * 60;
runJobs( $jobs, $startDir, $logDir );
info "DONE\t" . time::time_to_datetime();
}
main();

671
tools/sync_cms/sync_cms.pl
View File

@@ -1,438 +1,236 @@
#!/usr/bin/perl
BEGIN{
my $dir='';
$ENV{SCRIPT_FILENAME}||'' if ($dir eq'');
$dir=~s/(.*\/)[^\/]+/$1/ if ($dir ne '');
$dir=$ENV{PWD} if ($dir eq'');
$dir=`pwd` if ($dir eq'');
#add calcms libs
unshift(@INC,$dir.'/../calcms/');
}
use strict;
use warnings;
use Data::Dumper;
use Getopt::Long;
use Config::General;
use time;
use DateTime;
use DateTime::Duration;
use strict;
use warnings;
use IO::Socket::INET;
use Fcntl ':flock';
check_running_processes();
use FindBin;
use lib "$FindBin::Bin/lib";
use lib "$FindBin::Bin/../../calcms";
my $read_mode='';
my $update_mode='';
my $all_events='';
my $modified_events='';
my $source_config_file='';
my $target_config_file='';
my $block_number=0;
my $block_size=2000;
our $from='';
our $till='';
my $read_only=0;
our $output_type='text';
our $debug=0;
use Common ( 'info', 'error' );
use GoogleCalendar;
use CalcmsEvents;
GetOptions(
"read" => \$read_mode,
"update" => \$update_mode,
"all" => \$all_events,
"modified" => \$modified_events,
"from=s" => \$from,
"till=s" => \$till,
"source=s" => \$source_config_file,
"target=s" => \$target_config_file,
"block_number:i" => \$block_number,
"block_size:i" => \$block_size,
"output_type=s" => \$output_type,
);
$|=1;
Common::checkSingleInstance();
BEGIN {
our $utf8dbi=1;
$ENV{LANG}="en_US.UTF-8";
$ENV{LANG} = "en_US.UTF-8";
}
$| = 1;
my $sourceConfigFile = '';
my $targetConfigFile = '';
my $debug = 1;
my $from = undef;
my $till = undef;
GetOptions(
"from=s" => \$from,
"till=s" => \$till,
"source=s" => \$sourceConfigFile,
"target=s" => \$targetConfigFile,
);
#source and taget settings are loaded from config files
our $settings={
};
my $settings = {};
#user interface
our $ask_before_insert=0;
our $ask_before_update=0;
# end of configuration
if ($update_mode){
$db::write=1;
# print_info("enter update mode");
}elsif($read_mode){
#default
$db::write=0;
# print_info("enter read-only mode");
}else{
print_error("set parameter >read< or >update<");
}
unless ($modified_events || $all_events || $from || $till){
print_error("set one of folling parameters: --modified, --from, --till");
}
error "set one of folling parameters: --from, --till" unless $from || $till;
init();
sync();
print_info("$0 done.");
info "$0 done.";
exit 0;
#sync all events, splitting multi-day-requests into multiple 1-day-requests to avoid large result sets
sub sync{
#prepare target
print_info("$0 inited");
print_info("last update: $settings->{source}->{last_update}");
sub sync {
my $timeZone = CalcmsEvents::get('date')->{time_zone};
my $from = CalcmsEvents::get('start_min');
my $till = CalcmsEvents::get('start_max');
if (my $days=source::split_request()){
#set 1-day start-min and start-max parameters, requires --from and --till values
for my $date (@$days){
for my $key(keys %$date){
$settings->{source}->{$key}=$date->{$key};
}
#print "\nrequest ".$settings->{source}->{"start_min"}." to ".$settings->{source}->{"start_max"}."\n";
sync_timespan();
}
}else{
#update without time span (e.g. --modified)
sync_timespan();
}
info "sync from $from till $till at $timeZone";
print_info("\nclean up old database entries...");
target::clean_up();
#prepare target
info "last update: " . ( CalcmsEvents::get('last_update') || '' );
print_info("\nset last-update time: $settings->{event}->{update_start}");
set_last_update_time($source_config_file,$target_config_file,$settings->{event}->{update_start});
if ( my $days = CalcmsEvents::splitRequest( $from, $till, $timeZone ) ) {
for my $date (@$days) {
syncTimespan( $date->{from}, $date->{till} );
}
} else {
syncTimespan( $from, $till );
}
info "\nset last-update time: $settings->{event}->{update_start}";
setLastUpdateTime( $sourceConfigFile, $targetConfigFile, $settings->{event}->{update_start} );
}
#sync all events of a given source timespan
sub sync_timespan{
#get a list of all days and their events
#print Dumper($settings->{source});
my $source_events=source::get_events($settings->{source},$settings->{target});
my @dates=(keys %$source_events);
sub syncTimespan {
my $from = shift;
my $till = shift;
#print "2\n";
if (@dates==0){
my $more='';
if ((defined $settings->{source}->{block_number}) && ($settings->{source}->{block_number} ne '0')){
$more='more ';
}elsif ($modified_events){
$more.='modified ';
}
print_info("\n".'no '.$more."entries found.");
}else{
print "<table>" if ($output_type eq 'html');
#sort lists of date and time (same time events should be preserved)
for my $date(sort {$a cmp $b} @dates){
# for my $date(@dates){
# print "\n$date:\n";
sync_events($source_events->{$date}, $settings);
}
print "</table>" if ($output_type eq 'html');
}
#get a list of all days and their events
my $sourceEvents = CalcmsEvents::getEvents( $from, $till );
my @dates = keys %$sourceEvents;
if ( scalar @dates == 0 ) {
info "\nno entries found.";
return;
}
#sort lists of date and time (same time events should be preserved)
for my $date ( sort { $a cmp $b } @dates ) {
syncEvents( $sourceEvents->{$date} );
}
}
#syncronize a list of source events to target events
sub sync_events{
my $source_events=shift;
my $settings=shift;
sub syncEvents($) {
my $sourceEvents = shift;
# my $source_settings =$settings->{source};
# my $target_settings =$settings->{target};
my $event_settings =$settings->{event};
my @sourceEvents = sort { $a->{calcms_start} cmp $b->{calcms_start} } @$sourceEvents;
$sourceEvents = \@sourceEvents;
my $start = $sourceEvents->[0]->{start};
my $end = $sourceEvents->[-1]->{end};
my $c=0;
$c=$source::settings->{start_index}+0 if (defined $source::settings->{start_index});
my $targetEvents = GoogleCalendar::getEvents(
{
start => $start,
end => $end
}
);
$targetEvents = $targetEvents->{items};
info "google:" . scalar(@$targetEvents) . " vs " . scalar(@$sourceEvents);
# print "<events>\n";
print html_table_header() if ($output_type eq 'html');
#order processing by start time (TODO: order by last-modified date)
for my $event (sort{$a->{calcms_start} cmp $b->{calcms_start}} @$source_events){
target::pre_sync({
start =>$source_events->[0]->{start},
end =>$source_events->[-1]->{end}
});
# mark all known target events
my $targetEventsByKey = {};
for my $event (@$targetEvents) {
#print Dumper($event);
next if $event->{status} eq 'canceled';
my $key = getGoogleEventToString($event);
$targetEventsByKey->{$key} = $event;
}
print "<tr><td>"if ($output_type eq 'html');
# mark all knwon source events
my $sourceEventsByKey = {};
for my $event (@$sourceEvents) {
$event = CalcmsEvents::mapToSchema($event);
$event = GoogleCalendar::mapToSchema($event);
my $key = getCalcmsEventToString($event);
$sourceEventsByKey->{$key} = $event;
}
#read event
$event=source::get_event_attributes($event);
# delete target entries without matching source entries
for my $key ( keys %$targetEventsByKey ) {
next if defined $sourceEventsByKey->{$key};
my $event = $targetEventsByKey->{$key};
info "delete $key ";
print Dumper($event);
GoogleCalendar::deleteEvent($event);
}
#convert to calcms schema
$event=source::map_to_schema($event);
#map event to target schema
$event=target::map_to_schema($event);
#deprecated: override defined attributes by configuration
if ((defined $source::settings->{override}) && (ref($source::settings->{override})eq 'HASH')){
for my $key (keys %{$source::settings->{override}}){
my $value=$source::settings->{override}->{$key};
if ($source::settings->{override} ne ''){
print_info("override '$key'='$value'");
$event->{event}->{$key}=$value;
}
}
}
if ($output_type eq'html'){
print_event_html("[".($c+1)."]",$event);
}else{
print_event_text("[".($c+1)."]",$event);
}
if ($event->{event}->{start} eq '' || $event->{event}->{end} eq ''){
print ('WARNING: Cannot read start or end of event');
print "\n";
}else{
# print Dumper($event);
sync_event($event);
}
# last;
$event=undef;
$c++;
print "</td></tr>"if ($output_type eq 'html');
}
# print "\n</events>\n";
# insert source entries without matching target entries
for my $key ( keys %$sourceEventsByKey ) {
if ( defined $targetEventsByKey->{$key} ) {
info "$key is up to date";
next;
}
my $event = $sourceEventsByKey->{$key};
info "insert $key";
GoogleCalendar::insertEvent($event);
}
}
#syncronize a single source event with target
sub sync_event{
my $event=shift;
sub getGoogleEventToString {
my $event = shift;
my $result = "\n";
$result .= "start: " . substr( $event->{start}->{dateTime}, 0, 19 ) . "\n";
$result .= "end : " . substr( $event->{end}->{dateTime}, 0, 19 ) . "\n";
$result .= "title: $event->{summary}\n";
$result .= "desc : $event->{description}\n";
return $result;
}
#look if target_event exists by reference id incl. recurrence counter
#print Dumper($event);
my $target_event=target::get_event_by_reference_id($event->{event}->{reference});
#if target_event exists
if (defined $target_event){
#delete canceled events
if ($event->{event}->{status}eq'canceled'){
print cell("delete canceled event:".qq{$target_event});
# target::delete($target_event->{id});
return;
}
$event->{event_id}=$target_event->{id};
target::update_event($event,$target_event);
print cell("(ref. update)");
}else{
#find by date, time and title
$target_event=target::find_event($event);
if (defined $target_event){
target::update_event($event,$target_event);
#print Dumper($event);
$event->{event_id}=$target_event->{id};
print cell("(update)");
}else{
target::insert_event($event);
#print Dumper($event);
$target_event=target::get_event_by_reference_id($event->{event}->{reference});
#print Dumper($target_event);
$event->{event_id}=$target_event->{id};
print cell("(new)");
}
}
print "\n";
for my $category (@{$event->{categories}}){
target::assign_category_to_event($category,$event);
}
for my $meta (@{$event->{meta}}){
target::assign_meta_to_event($meta,$event);
}
# print Dumper($event);
sub getCalcmsEventToString {
my $event = shift;
my $result = "\n";
$result .= "start: " . substr( $event->{event}->{start_datetime}, 0, 19 ) . "\n";
$result .= "end : " . substr( $event->{event}->{end_datetime}, 0, 19 ) . "\n";
$result .= "title: $event->{event}->{title}\n";
$result .= "desc : $event->{event}->{content}\n";
return $result;
}
#import requested source and target libs
sub init{
binmode STDOUT, ":utf8";
sub init {
binmode STDOUT, ":utf8";
#require source config file
print_error ("missing source parameter!") unless ($source_config_file=~/\S/);
print_error ("source file: '$source_config_file' does not exist") unless (-e $source_config_file);
print_error ("cannot read source file: '$source_config_file'") unless (-r $source_config_file);
#$settings->{source}=require $source_config_file;
my $configuration = new Config::General($source_config_file);
$settings->{source}=$configuration->{DefaultConfig}->{source};
{
#require target config file
error "missing target parameter!" unless $targetConfigFile =~ /\S/;
error "target file: '$targetConfigFile' does not exist" unless -e $targetConfigFile;
error "cannot read target file: '$targetConfigFile'" unless -r $targetConfigFile;
my $config = new Config::General($targetConfigFile);
$config = $config->{DefaultConfig}->{target};
GoogleCalendar::init($config);
#require source import lib from config file
my $source_import_lib='lib/source/'.$settings->{source}->{type}.'.pl';
print_error ("missing 'type' in 'source' config ") unless ($settings->{source}->{type}=~/\S/);
print_error ("cannot read source type import lib: '$source_import_lib'")unless (-r $source_import_lib);
require $source_import_lib;
#require target config file
print_error ("missing target parameter!") unless ($target_config_file=~/\S/);
print_error ("target file: '$target_config_file' does not exist") unless (-e $target_config_file);
print_error ("cannot read target file: '$target_config_file'") unless (-r $target_config_file);
$configuration = new Config::General($target_config_file);
$settings->{target}=$configuration->{DefaultConfig}->{target};
#$settings->{target}=require $target_config_file;
#require target import lib from config file
my $target_import_lib='lib/target/'.$settings->{target}->{type}.'.pl';
print_error ("missing 'type' in 'target' config ") unless ($settings->{target}->{type}=~/\S/);
print_error ("cannot read target type import lib: '$target_import_lib'")unless (-r $target_import_lib);
require $target_import_lib;
#print Dumper($settings);
if ((defined $settings->{source}->{read_blocks}) && ($settings->{source}->{read_blocks}==1)){
$settings->{source}->{block_number} =$block_number;
$settings->{source}->{block_size} =$block_size;
}
$settings->{source}->{last_update} =get_last_update_time($source_config_file,$target_config_file);
$settings->{source}->{modified_events} =$modified_events;
if ($from=~/^\d\d\d\d\-\d\d\-\d\d$/){
$from.='T00:00';
# print "from:$from\t";
}
if ($till=~/^\d\d\d\d\-\d\d\-\d\d$/){
$till.='T23:59';
# print "till:$till\t";
}
if ($from=~/^([-+]?\d+$)/){
my $days=$1;
my $duration=new DateTime::Duration(days=>$days);
$from=DateTime->today->add_duration($duration);
# print "from:$from\t";
}
if ($till=~/^([-+]?\d+$)/){
my $days=$1+1;
my $duration=new DateTime::Duration(days=>$days);
$till=DateTime->today->add_duration($duration);
# print "till:$till\t";
}
$settings->{source}->{start_min} =$from if defined ($from);
$settings->{source}->{start_max} =$till if defined ($till);
my $gmt_difference =0;#*=3600;
my $now =time();
my $now_gmt =$now-$gmt_difference;
$now =time::time_to_datetime($now);
$now_gmt =time::time_to_datetime($now_gmt);
$settings->{event}={
update_start => time::time_to_datetime(time()),
modified_at => $now,
modified_at_gmt => $now_gmt
};
source::init($settings->{source});
target::init($settings->{target});
}
# print date/time, title and excerpt of an calendar event
# TODO: replace by output filter (text, html)
sub print_event_text{
my $header=shift;
my $event=shift;
my $s=$header;
$s=$s." "x (8-length($s));
my $start=$event->{event}->{start}||'';
$start=~s/T/ /g;
$start=~s/\:00$//g;
if (defined $event->{event}->{program}){
$s.="$start $event->{event}->{program}";
$s=$s." "x (45-length($s));
}
if (defined $event->{event}->{series_name}){
$s.=" : $event->{event}->{series_name}";
$s=$s." "x (75-length($s));
{
#require source config file
error "missing source parameter!" unless $sourceConfigFile =~ /\S/;
error "source file: '$sourceConfigFile' does not exist" unless -e $sourceConfigFile;
error "cannot read source file: '$sourceConfigFile'" unless -r $sourceConfigFile;
my $config = new Config::General($sourceConfigFile);
$config = $config->{DefaultConfig}->{source};
$config->{last_update} = getLastUpdateTime( $sourceConfigFile, $targetConfigFile );
CalcmsEvents::init($config);
}
if (defined $event->{event}->{title}){
$s.=" - $event->{event}->{title}";
$s=$s." "x (110-length($s));
$from .= 'T00:00' if $from =~ /^\d\d\d\d\-\d\d\-\d\d$/;
$till .= 'T23:59' if $till =~ /^\d\d\d\d\-\d\d\-\d\d$/;
if ( $from =~ /^([-+]?\d+$)/ ) {
my $days = $1;
my $duration = new DateTime::Duration( days => $days );
$from = DateTime->today->add_duration($duration);
}
if ($event->{categories}){
$s.= "(".join(", ",(@{$event->{categories}})).")";
}
$s=$s." "x (135-length($s));
if ( $till =~ /^([-+]?\d+$)/ ) {
my $days = $1 + 1;
my $duration = new DateTime::Duration( days => $days );
$till = DateTime->today->add_duration($duration);
}
my $status=$event->{event}->{status};
$s.=$status.' ' if (defined $status);
$s=$s." "x (140-length($s));
CalcmsEvents::set( 'start_min', $from ) if defined $from;
CalcmsEvents::set( 'start_max', $till ) if defined $till;
my $reference=$event->{event}->{reference};
$s.=substr($reference,length($reference)-25) if (defined $reference);
my $now = time();
$now = time::time_to_datetime($now);
$settings->{event} = {
update_start => time::time_to_datetime( time() ),
modified_at => $now,
};
print $s;
}
sub print_event_html{
my $header=shift;
my $event=shift;
#close error block
my $s='</td>';
my $start=$event->{event}->{start}||'';
$start=~s/T/ /g;
$start=~s/\:00$//g;
$s.=cell($start);
$s.=cell($event->{event}->{program});
$s.=cell($event->{event}->{series_name});
$s.=cell($event->{event}->{title});
if ($event->{categories}){
$s.=cell( join(", " , ( @{$event->{categories}} ) ) );
}
my $status=$event->{event}->{status};
$s.=cell($status) if (defined $status);
my $reference=$event->{event}->{reference};
$reference=substr($reference,length($reference)-25) if (defined $reference);
$s.=cell($reference);
$s.="<td>";
print $s;
}
sub cell{
if ($output_type eq 'html'){
return "<td>$_[0]</td>";
}else{
return "\t".$_[0];
};
}
#output usage on error or --help parameter
sub print_usage{
print qq{
sub usage {
print qq{
update all/modified events from source at target.
USAGE: sync_cms.pl [--read,--update] [--modified,--all] --source s --target t [--block_number b] [--block_size s]
USAGE: sync_cms.pl [--read,--update] [--modified,--all] --source s --target t
on using --from and --till requests will be processed as multiple single-day-requests.
@@ -440,128 +238,55 @@ parameters:
--read show all events without updating database
--update update target database with source events
--modified process only modified events.
--all' process all events
--source source configuration file
--target target configuration file
--from start of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow)
--till end of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow)
--output_type log output format [text,html]
--block_number which block is to be syncronized [0..n]. To split up processing into multiple blocks (for machines with small memory resources).
--block_size size of a block, default=20 events
examples:
update modified
perl sync_cms.pl --update --modified --source=config/source/program.cfg --target=config/target/calcms.cfg
perl sync_cms.pl --update --source=config/source/program.cfg --target=config/target/calcms.cfg
update a given time range
perl sync_cms.pl --update --all --from=2009-09-01T00:00:00 --till=2009-11-22T23:59:59 --source=config/source/program.cfg --target=config/target/calcms.cfg
update from last 2 days until next 3 days
perl sync_cms.pl --update --all --from=-2 --till=+3 --source=config/source/program.cfg --target=config/target/calcms.cfg
perl sync_cms.pl --update --from=-2 --till=+3 --source=config/source/program.cfg --target=config/target/calcms.cfg
};
exit 1;
};
#default error handling
sub print_error{
print "\nERROR: $_[0]\n" ;
print_usage();
exit 1;
}
sub print_info{
my $message=shift;
if ($message=~/^\n/){
$message=~s/^\n//g;
print "\n";
}
if ($output_type eq 'html'){
print "$message<br/>";
}else{
print "INFO:\t$message\n";
}
}
sub html_table_header{
return qq{
<tr>
<th> </th>
<th>start date</th>
<th>project</th>
<th>series</th>
<th>title</th>
<th>category</th>
<th>status</th>
<th>id</th>
<th> </th>
<th>action</th>
</tr>
};
};
#load last update time out of sync.data
sub get_last_update_time{
my $source=shift;
my $target=shift;
sub getLastUpdateTime {
my $source = shift;
my $target = shift;
my $date=undef;
return undef unless(-r "sync.data");
open my $DATA, "<:utf8","sync.data" || die ('cannot read update timestamp');
while (<$DATA>){
my $line=$_;
if ($line=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){
$date=$1;
last;
}
}
close $DATA;
return $date;
my $date = undef;
return undef unless -r "sync.data";
my $content = Common::loadFile("sync.data");
if ( $content =~ /$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/ ) {
$date = $1;
}
return $date;
}
#save last update time to sync.data
sub set_last_update_time{
my $source =shift;
my $target =shift;
my $date =shift;
sub setLastUpdateTime {
my $source = shift;
my $target = shift;
my $date = shift;
my $data='';
if (-r "sync.data"){
open my $DATA, "<:utf8","sync.data";
$data=join("\n",(<$DATA>));
close $DATA;
}
my $data = '';
if ( -r "sync.data" ) {
$data = Common::loadFile("sync.data");
}
if ($data=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){
$data=~s/($source\s+\->\s+$target\s+:)\s+\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}/$1\t$date/gi;
}else{
$data.="$source\t\->\t$target\t:\t$date\n";
}
if ( $data =~ /$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/ ) {
$data =~ s/($source\s+\->\s+$target\s+:)\s+\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}/$1\t$date/gi;
} else {
$data .= "$source\t\->\t$target\t:\t$date\n";
}
$data=~s/[\r\n]+/\n/g;
open my $DATA2, ">:utf8","sync.data" || die ('cannot write update timestamp');
print $DATA2 $data;
close $DATA2;
# print $data;
$data =~ s/[\r\n]+/\n/g;
Common::saveFile( "sync.data", $data );
}
#avoid to run more than one sync process in parallel
sub check_running_processes{
my $cmd="ps -afex 2>/dev/null | grep sync_cms.pl | grep -v nice | grep -v grep ";
my $ps=`$cmd`;
# print "$ps";
my @lines=(split(/\n/,$ps));
if (@lines>1){
print "ERROR: another ".@lines." synchronization processes 'sync_cms.pl' instances are running!".qq{
$cmd
$ps
-> program will exit
};
exit;
}
}

74
tools/sync_cms/sync_days.pl
View File

@@ -1,74 +0,0 @@
#!/usr/bin/perl -I ../lib #-w
BEGIN{
my $dir='';
$ENV{SCRIPT_FILENAME} if ($dir eq'');
$dir=~s/(.*\/)[^\/]+/$1/;
$dir=$ENV{PWD} if ($dir eq'');
$dir=`pwd` if ($dir eq'');
#local perl installation libs
unshift(@INC,$dir.'/../../perl/lib/');
#calcms libs + configuration
unshift(@INC,$dir.'/../calcms/');
}
#use utf8;
use warnings "all";
use strict;
use Data::Dumper;
#use CGI;
#use HTML::Template;
use Date::Calc;
#use calendar;
#use time;
#use log;
if(@ARGV<2){
print qq{ERROR: $0 yyyy-mm-dd yyyy-mm-dd
syncronize from given start date to end date, day by day
};
exit 1;
}
my $start =$ARGV[0];
my $end =$ARGV[1];
(my $start_year,my $start_month,my $start_day)=split(/\-/,$start);
my $last_day=Date::Calc::Days_in_Month($start_year,$start_month);
$start_day = 1 if ($start_day<1);
$start_day = $last_day if ($start_day gt $last_day);
(my $end_year,my $end_month,my $end_day)=split(/\-/,$end);
$last_day=Date::Calc::Days_in_Month($end_year,$end_month);
$end_day = 1 if ($end_day<1);
$end_day = $last_day if ($end_day gt $last_day);
for my $year($start_year..$end_year){
my $m1=1;
my $m2=12;
$m1=$start_month if($year eq $start_year);
$m2=$end_month if($year eq $end_year);
for my $month($m1..$m2){
$month='0'.$month if (length($month)==1);
my $d1=1;
my $d2=Date::Calc::Days_in_Month($year,$month);
$d1=$start_day if($month eq $start_month);
$d2=$end_day if($month eq $end_month);
for my $day($d1..$d2){
$day='0'.$day if (length($day)==1);
my $date=join('-',($year,$month,$day));
my $cmd="perl sync_cms.pl --update --all --source config/source/program.cfg --target config/target/calcms.cfg --from ".$date."T00:00:00 --till ".$date."T23:59:59";
#print "$cmd\n";
print `nice -n 10 $cmd`;
}
}
}

13
tools/sync_cms/sync_jobs/calcms_to_google.sh Executable file
View File

@@ -0,0 +1,13 @@
#!/bin/sh
from=$1
till=$2
project=$3
export LC_ALL="de_DE.utf8"
export LANGUAGE="de_DE.utf8"
set -x
cd /home/radio/calcms/sync_cms
nice -n 10 perl sync_cms.pl --from=$from --till=$till --source=config/source/calcms_$project.cfg --target=config/target/88vier_$project.cfg 2>&1

7
tools/sync_cms/sync_jobs/sync.sh Executable file
View File

@@ -0,0 +1,7 @@
#/bin/sh
./sync_project.sh "$1" "$2" piradio
./sync_project.sh "$1" "$2" potsdam
./sync_project.sh "$1" "$2" ansage
./sync_project.sh "$1" "$2" collabo
./sync_project.sh "$1" "$2" frb

6
tools/sync_cms/sync_jobs/update_ansage.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/sh
file=/home/radio/piradio.de/agenda/admin/jobs/start/ansage_to_88vier.de.start.txt
touch $file
chown radio:www-data $file

6
tools/sync_cms/sync_jobs/update_colabo.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/sh
file=/home/radio/piradio.de/agenda/admin/jobs/start/colabo_to_88vier.de.start.txt
touch $file
chown radio:www-data $file

6
tools/sync_cms/sync_jobs/update_frb.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/sh
file=/home/radio/piradio.de/agenda/admin/jobs/start/frb_to_88vier.de.start.txt
touch $file
chown radio:www-data $file

6
tools/sync_cms/sync_jobs/update_piradio.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/sh
file=/home/radio/piradio.de/agenda/admin/jobs/start/piradio_to_88vier.de.start.txt
touch $file
chown radio:www-data $file

6
tools/sync_cms/sync_jobs/update_potsdam.sh Executable file
View File

@@ -0,0 +1,6 @@
#!/bin/sh
file=/home/radio/piradio.de/agenda/admin/jobs/start/potsdam_to_88vier.de.start.txt
touch $file
chown radio:www-data $file

487
tools/sync_cms/time_gate.pl
View File

@@ -1,487 +0,0 @@
#!/usr/bin/perl -I ../lib #-w
BEGIN{
my $dir='';
$ENV{SCRIPT_FILENAME}||'' if ($dir eq'');
$dir=~s/(.*\/)[^\/]+/$1/ if ($dir ne '');
$dir=$ENV{PWD} if ($dir eq'');
$dir=`pwd` if ($dir eq'');
#local perl installation libs
unshift(@INC,$dir.'/../../perl/lib/');
#calcms libs + configuration
unshift(@INC,$dir.'/../calcms/');
}
#use utf8;
use Data::Dumper;
#require 'time.pl';
use Getopt::Long;
use time;
use DateTime;
use DateTime::Duration;
use strict;
use warnings;
check_running_processes();
my $read_mode='';
my $update_mode='';
my $all_events='';
my $modified_events='';
my $source_config_file='';
my $target_config_file='';
my $block_number=0;
my $block_size=2000;
my $from='';
my $till='';
my $read_only=0;
my $project='';
GetOptions(
"read" => \$read_mode,
"update" => \$update_mode,
"all" => \$all_events,
"modified" => \$modified_events,
"from=s" => \$from,
"till=s" => \$till,
"source=s" => \$source_config_file,
"target=s" => \$target_config_file,
"project=s" => \$project,
"block_number:i" => \$block_number,
"block_size:i" => \$block_size
);
$|=1;
BEGIN {
our $utf8dbi=1;
$ENV{LANG}="en_US.UTF-8";
# print Dumper(\%ENV);
}
#source and taget settings are loaded from config files
our $settings={
};
#user interface
our $ask_before_insert=0;
our $ask_before_update=0;
# end of configuration
if ($update_mode){
$db::write=1;
# print "enter update mode\n";
}elsif($read_mode){
#default
$db::write=0;
# print "enter read-only mode\n";
}else{
print_error("set parameter >read< or >update<");
}
unless ($modified_events || $all_events || $from || $till){
print_error("set one of folling parameters: --modified, --from, --till");
}
init();
my $project_target=$source::settings->{sources}->{$project};
unless (defined $project){
print_error("missing parameter --project") unless(defined $project_target);
print_error("cant find project configuration '$project_target'") unless (-f $project_target);
print_error("cant read project configuration '$project_target'") unless (-r $project_target);
}
my $events=[];
print "TIME_GATE: READ ALL CALENDARS\n";
sync();
$events=compress_events($events);
my $c=0;
if ($project eq ''){
for my $event (@$events){
print_event("[".($c+1)."]",$event);
print "\n";
$c++;
}
}else{
my $source=$source::settings->{sources}->{$project};
my $target='config/target/calcms.cfg';
for my $event (@$events){
my $from=$event->{start};
#print Dumper($event->{end});
#remove a second
my $till=source::get_datetime($event->{end}, $source::settings->{date}->{time_zone})->add(seconds=>-1)->datetime();
print_event("STATION TIMESLOT [".($c+1)."]\t",$event);
print "\n";
$c++;
my $command="perl sync_cms.pl --update --all --from=$from --till=$till --source $source --target $target ";
print_info($command);
print `$command`;
#exit;
}
}
print "\ndone.\n";
exit 0;
sub compress_events{
my $events=shift;
my @results=();
my $old_event={end=>'', start=>'', title=>''};
for my $event(sort {$a->{start} gt $b->{start}} @$events){
# print "$event->{start}\t$event->{end}\t$event->{title}\n";
if (
# (defined $event) && (defined $event->{start}) && (defined $event->{end}) && (defined $event->{title})
( #station continues
($event->{start} eq $old_event->{end})
|| (#multiple entries for same event
($event->{start} ge $old_event->{start})
&& ($event->{end} eq $old_event->{end})
)
)
&& ($event->{title} eq $old_event->{title})
&& (@results>0)
){
$results[-1]->{end}=$event->{end};
# print @results."\tmerge \n";
}else{
push @results,{
start => $event->{start},
end => $event->{end},
title => $event->{title},
};
# print @results."\tinsert \n";
}
$old_event=$results[-1];
}
# print Dumper(\@results);
return \@results;
}
#sync all events, splitting multi-day-requests into multiple 1-day-requests to avoid large result sets
sub sync{
#prepare target
target::init($settings->{target});
print_info("last update: $settings->{source}->{last_update}");
if (my $days=source::split_request($settings->{source})){
#set 1-day start-min and start-max parameters, requires --from and --till values
for my $date (@$days){
for my $key(keys %$date){
$settings->{source}->{$key}=$date->{$key};
}
print "\nrequest ".$settings->{source}->{"start_min"}." to ".$settings->{source}->{"start_max"}."\n";
sync_timespan();
}
}else{
#update without time span (e.g. --modified)
sync_timespan();
}
print_info("\nset last-update time: $settings->{event}->{update_start}");
set_last_update_time($source_config_file,$target_config_file,$settings->{event}->{update_start});
}
#sync all events of a given source timespan
sub sync_timespan{
#get a list of all days and their events
#print Dumper($settings->{source});
my $source_events=source::get_events($settings->{source},$settings->{target});
#print Dumper($source_events);
my @dates=(keys %$source_events);
if (@dates==0){
my $more='';
if ((defined $settings->{source}->{block_number}) && ($settings->{source}->{block_number} ne '0')){
$more='more ';
}elsif ($modified_events){
$more.='modified ';
}
print_info("\n".'no '.$more."entries found.");
}else{
#sort lists of date and time (same time events should be preserved)
for my $date(sort {$a cmp $b} @dates){
# for my $date(@dates){
# print "\n$date:\n";
sync_events($source_events->{$date}, $settings);
}
}
}
#syncronize a list of source events to target events
sub sync_events{
my $source_events=shift;
my $settings=shift;
my $c=0;
$c=$source::settings->{start_index}+0 if (defined $source::settings->{start_index});
# print "<events>\n";
#order processing by start time (TODO: order by last-modified date)
for my $event (sort{$a->{calcms_start} cmp $b->{calcms_start}} @$source_events){
#read event attributes
$event=source::get_event_attributes($event);
$event->{title}=~s/\s//g;
$event->{event}={
title => $event->{title},
start => $event->{start},
end => $event->{end},
status => $event->{status},
};
# print "\n";
#print_event("[".($c+1)."]",$event);
#print "\n".$event->{event}->{title}." ".$project."\n";
if ($event->{event}->{status}eq'canceled'){
print "canceled event:".qq{$event};
}elsif ($event->{event}->{start} eq ''){
print ('WARNING: Cannot read start of event'."\n");
}elsif ($event->{event}->{end} eq ''){
print ('WARNING: Cannot read start of end'."\n");
}elsif ($event->{event}->{title} eq ''){
print ('WARNING: Cannot read start of title'."\n");
}elsif ($project ne ''){
if ($event->{event}->{title} eq $project){
push @$events, $event->{event};
}
}else{
push @$events, $event->{event};
}
$event=undef;
$c++;
}
}
#import requested source and target libs
sub init{
binmode STDOUT, ":utf8";
#require source config file
print_error ("missing source parameter!") unless ($source_config_file=~/\S/);
print_error ("source file: '$source_config_file' does not exist") unless (-e $source_config_file);
print_error ("cannot read source file: '$source_config_file'") unless (-r $source_config_file);
#$settings->{source}=require $source_config_file;
my $configuration = new Config::General($source_config_file);
$settings->{source}=$configuration->{DefaultConfig}->{source};
#require source import lib from config file
my $source_import_lib='lib/source/'.$settings->{source}->{type}.'.pl';
print_error ("missing 'type' in 'source' config ") unless ($settings->{source}->{type}=~/\S/);
print_error ("cannot read source type import lib: '$source_import_lib'")unless (-r $source_import_lib);
require $source_import_lib;
#require target config file
print_error ("missing target parameter!") unless ($target_config_file=~/\S/);
print_error ("target file: '$target_config_file' does not exist") unless (-e $target_config_file);
print_error ("cannot read target file: '$target_config_file'") unless (-r $target_config_file);
#$settings->{target}=require $target_config_file;
$configuration = new Config::General($target_config_file);
$settings->{target}=$configuration->{DefaultConfig}->{target};
#require target import lib from config file
my $target_import_lib='lib/target/'.$settings->{target}->{type}.'.pl';
print_error ("missing 'type' in 'target' config ") unless ($settings->{target}->{type}=~/\S/);
print_error ("cannot read target type import lib: '$target_import_lib'")unless (-r $target_import_lib);
require $target_import_lib;
#print Dumper($settings);
if ((defined $settings->{source}->{read_blocks}) && ($settings->{source}->{read_blocks}==1)){
$settings->{source}->{block_number} =$block_number;
$settings->{source}->{block_size} =$block_size;
}
$settings->{source}->{last_update} =get_last_update_time($source_config_file,$target_config_file);
$settings->{source}->{modified_events} =$modified_events;
if ($from=~/^\d\d\d\d\-\d\d\-\d\d$/){
$from.='T00:00';
}
if ($till=~/^\d\d\d\d\-\d\d\-\d\d$/){
$till.='T23:59';
}
if ($from=~/^([-+]?\d+$)/){
my $days=$1;
my $duration=new DateTime::Duration(days=>$days);
$from=DateTime->today->add_duration($duration);
# print "from:$from\t";
}
if ($till=~/^([-+]?\d+$)/){
my $days=$1+1;
my $duration=new DateTime::Duration(days=>$days);
$till=DateTime->today->add_duration($duration);
# print "till:$till\t";
}
$settings->{source}->{start_min} =$from if defined ($from);
$settings->{source}->{start_max} =$till if defined ($till);
my $gmt_difference =0;#*=3600;
my $now =time();
my $now_gmt =$now-$gmt_difference;
$now =time::time_to_datetime($now);
$now_gmt =time::time_to_datetime($now_gmt);
$settings->{event}={
update_start => time::time_to_datetime(time()),
modified_at => $now,
modified_at_gmt => $now_gmt
};
source::init($settings->{source});
}
# print date/time, title and excerpt of an calendar event
# TODO: replace by output filter (text, html)
sub print_event{
my $header=shift;
my $event=shift;
my $s=$header;
$s=$s." "x (8-length($s));
# print Dumper($event);
my $start=$event->{start}||'';
$start=~s/T/ /g;
$start=~s/\:00$//g;
my $end=$event->{end}||'';
$end=~s/T/ /g;
$end=~s/\:00$//g;
$s.="$start\t$end\t'$event->{title}'";
# print Dumper($event->{event});
print $s;
#excerpt: >$event->{excerpt}<
#content: >$event->{content}<
#content: >$event->{content}<
}
#output usage on error or --help parameter
sub print_usage{
print qq{
update all/modified events from source at target.
USAGE: $0 [--read,--update] [--modified,--all] --source s --target t [--block_number b] [--block_size s]
on using --from and --till requests will be processed as multiple single-day-requests.
parameters:
--read show all events without updating database
--update update target database with source events
--modified process only modified events.
--all' process all events
--source source configuration file
--target target configuration file
--from start of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow)
--till end of date range: datetime (YYYY-MM-DDTHH:MM::SS) or days from today (e.g. -1 for yesterday, +1 for tomorrow)
--block_number which block is to be syncronized [0..n]. To split up processing into multiple blocks (for machines with small memory resources).
--block_size size of a block, default=20 events
examples:
perl $0 --update --modified --source=config/source/einheit.cfg --target=config/target/calcms.cfg
perl $0 --update --all --from=2009-09-01T00:00:00 --till=2009-11-22T23:59:59 --source=config/source/einheit.cfg --target=config/target/calcms.cfg
};
exit 1;
};
#load last update time out of sync.data
sub get_last_update_time{
my $source=shift;
my $target=shift;
my $date=undef;
return undef unless(-r "sync.data");
open my $DATA, "<:utf8","sync.data" || die ('cannot read update timestamp');
while (<$DATA>){
my $line=$_;
if ($line=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){
$date=$1;
last;
}
}
close $DATA;
return $date;
}
#save last update time to sync.data
sub set_last_update_time{
my $source =shift;
my $target =shift;
my $date =shift;
my $data='';
if (-r "sync.data"){
open my $DATA, "<:utf8","sync.data";
$data=join("\n",(<$DATA>));
close $DATA;
}
if ($data=~/$source\s+\->\s+$target\s+:\s+(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2})/){
$data=~s/($source\s+\->\s+$target\s+:)\s+\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}/$1\t$date/gi;
}else{
$data.="$source\t\->\t$target\t:\t$date\n";
}
$data=~s/[\r\n]+/\n/g;
open my $DATA2, ">:utf8","sync.data" || die ('cannot write update timestamp');
print $DATA2 $data;
close $DATA2;
# print $data;
}
#default error handling
sub print_error{
print "\nERROR:\t$_[0]\n" ;
print_usage();
}
sub print_info{
my $message=shift;
if ($message=~/^\n/){
$message=~s/^\n//g;
print "\n";
}
print "INFO:\t$message\n";
}
#avoid to run more than one sync process simultaniously
sub check_running_processes{
my $cmd="ps -afex 2>/dev/null | grep $0.pl | grep -v nice | grep -v grep ";
my $ps=`$cmd`;
# print "$ps";
my @lines=(split(/\n/,$ps));
if (@lines>1){
print "ERROR:\tanother ".@lines." synchronization processes '$0.pl' instances are running!".qq{
$cmd
$ps
-> program will exit
};
exit;
}
}