Working active/passive mode

cluster_cron.pl

@@ -1,11 +1,11 @@
 #!/usr/bin/env perl
+use bignum;
 use warnings;
 use strict;
-use bignum;
-use Sys::Hostname;
 use File::Compare;
 use File::Copy;
 use File::Temp;
+use Sys::Hostname;
 
 my $num_args = $#ARGV + 1;
 if ($num_args < 2) {
@@ -104,9 +104,13 @@ sub is_active {
 				print "I am not active\n";
 			} else {
 				print "I am active\n";
+				my $outfile = "$sharedcrondir/active";
+				open(OUTFILE,"+>$outfile");
+				print OUTFILE $hostname;
+				close OUTFILE;
 			}
 		} else {
-			print "Node: $node->{'name'} was not active last 120 sec\n";
+			print "Node: $node->{'name'} was not active last $failovertimeout sec\n";
 
 		}
 	}
@@ -187,28 +191,41 @@ sub run {
 	print "Host: $hostname started run at: " . time . "\n";
 	print "Mode: $mode, user: $user, shareddir: $shareddir and spooldir: $spooldir\n";
 
+	if(-e $activesharedcronfile) {
+		unless(-e $oldactivesharedcronfile) {
+			copy($activesharedcronfile, $oldactivesharedcronfile);
+		}
+	}
 
-	# If we are running in active/passive mode
-	if($mode == 0) {
 	unless (-d $electiondir) {
 		mkdir $electiondir;
 	}
-		my $tempfile = tmpnam();
+	# If we are running in active/passive mode
-
+	if($mode == 0) {
+		if(-e $passivesharedcronfile) {
+			unless(-e $oldpassivesharedcronfile) {
+				copy($passivesharedcronfile, $oldpassivesharedcronfile);
+			}
+		}
 		# Update timestamp
 		write_timestamp;
 
+		# If we have an active shared cronfile, we should also have an old shared
 		# See if I am the one
 		if (is_active(get_nodes) ){
+			my $tempfile = tmpnam();
 			my $compare = cron_compare($cronfile, $activesharedcronfile, $oldactivesharedcronfile );
 			# The other node has changed the file
+			# This would mean that we are in some kind of multi master mode which is not invented yet, should not happen
 			if($compare == 1) {
 				copy($activesharedcronfile, $cronfile);
 				copy($activesharedcronfile, $oldactivesharedcronfile);
 			# This node has changed the file 
 			} elsif ($compare == 2) {
 				copy($cronfile, $activesharedcronfile);
-			#comment_out($cronfile, $passivesharedcronfile);
+				# Since we are the only master we need to update the old one as well
+				copy($activesharedcronfile, $oldactivesharedcronfile);
+				comment_out($cronfile, $passivesharedcronfile);
 			}
 			comment_out($cronfile, $tempfile);
 			$compare = cron_compare($tempfile, $passivesharedcronfile, $oldpassivesharedcronfile);
@@ -223,6 +240,7 @@ sub run {
 				copy($cronfile, $activesharedcronfile);
 				comment_out($cronfile, $passivesharedcronfile);
 			}
+			unlink $tempfile;
 		# We are passive node
 		} else {
 			my $compare = cron_compare($cronfile, $passivesharedcronfile, $oldpassivesharedcronfile );
@@ -236,7 +254,6 @@ sub run {
 				copy($passivesharedcronfile, $cronfile);
 			}
 		}
-		unlink $tempfile;
 	}
 
 	# If we are running in active/active mode
@@ -244,8 +261,8 @@ sub run {
 		my $compare = cron_compare($cronfile, $activesharedcronfile, $oldactivesharedcronfile );
 		# The other node has changed the file
 		if($compare == 1) {
-				copy($activesharedcronfile, $oldactivesharedcronfile);
 			copy($activesharedcronfile, $cronfile);
+			copy($activesharedcronfile, $oldactivesharedcronfile);
 		# We have changed things
 		} elsif ($compare == 2) {
 			copy($cronfile, $activesharedcronfile);

cluster_cron2.pl

@@ -0,0 +1,188 @@
+#!/usr/bin/env perl
+use warnings;
+use strict;
+use bignum;
+use Sys::Hostname;
+use File::Compare;
+use File::Copy;
+use File::Temp;
+
+my $num_args = $#ARGV + 1;
+if ($num_args < 2) {
+    print "\nUsage: $0 <user> <shared directory> [cron spool directory]\n";
+    exit;
+}
+
+# We need a common directory for our nodes to write and read state from plus a couple of cronfiles
+my ($user, $shareddir, $spooldir)  = @ARGV;
+
+# Set some defaults if we didn't get them
+unless ($spooldir) { 
+	$spooldir = "/var/spool/cron/crontabs";
+}
+
+my $sharedcrondir = "$shareddir/crontab";
+my $cronfile = "$spooldir/$user";
+my $activesharedcronfile = "$sharedcrondir/$user";
+my $oldactivesharedcronfile = "$sharedcrondir/$user.old";
+my $electiondir = "$sharedcrondir/election";
+my $hostname = hostname;
+my $failovertimeout = 40;
+
+
+# Create directories 
+unless (-d $sharedcrondir) {
+	mkdir $sharedcrondir or die "Can not create shared directory: $!";
+}
+
+
+
+# Get the nodes in the cluster
+sub get_nodes {
+	# Our data type is a hash with the following fields
+	# name, time, numeric
+	my @nodes = ();
+	my @files = <$electiondir/*>;
+	foreach my $filename (@files) {
+		chomp $filename;
+		my $nodename = $filename;
+		$nodename =~ s/^$electiondir\///;
+		# Don't add this host
+		unless ($nodename eq $hostname) {
+			my $node = {};
+			open(FILE, "<$filename") or die "Could not open file: $!";
+			$node->{'name'} = $nodename;
+			my $content = <FILE>;
+			chomp $content;
+			$node->{'time'} = $content;
+			$node->{'numeric'} = get_numeric($nodename);
+			close FILE;
+			push @nodes, $node;
+		}
+	}
+	return @nodes;
+	
+
+}
+
+
+# Simply write a unix timestamp to a file with our name on it
+sub write_timestamp {
+	my $time = time;
+	open(FILE,"+>$electiondir/$hostname");
+	print FILE $time;
+	close FILE;
+
+}
+
+# Lets see if we are the active node
+sub is_active {
+
+	my @nodes = @_;
+
+	# Assume you are active
+	my $state = 1;
+	# Loop all nodes
+	foreach my $node (@nodes) {
+		#  If it has been active in the last 120 seconds it is eligable
+		my $diff = time - $node->{'time'};
+		if( $diff < $failovertimeout ) {
+			print "Node: $node->{'name'} was active last $failovertimeout sec\n";
+			# Go to passive mode if the numeric of that host is less than that of this host
+			my $this = get_numeric($hostname);
+			my $that = $node->{'numeric'};
+			print "Numeric for $hostname is $this and for $node->{'name'} is $that\n";
+			if( $this > $that ) {
+				$state = 0 ;
+				print "I am not active\n";
+			} else {
+				print "I am active\n";
+				my $outfile = "/var/lib/thruk/crontab/active";
+				open(OUTFILE,"+>$outfile");
+				print OUTFILE $hostname;
+				close OUTFILE;
+			}
+		} else {
+			print "Node: $node->{'name'} was not active last 120 sec\n";
+
+		}
+	}
+	return $state;
+}
+
+# Turn a hostname in to a decimal number used for comparison, lowest numer gets to be active if it is working
+sub get_numeric {
+	my ($host) = @_;
+	my @ascii = unpack 'C*', $host;
+	my $numeric = '';
+	foreach my $val (@ascii) {
+		$numeric .=  $val;
+	}
+	return $numeric + 0;
+}
+
+sub cron_compare {
+	my ($one, $two, $three) = @_;
+
+	# Current cronfile and shared cronfile is not same
+	if(compare($one, $two) != 0) {
+
+		print "$one and $two is not the same\n";
+
+		# This means that the other node has changed the cron file
+		if(compare($one, $three) == 0) {
+
+			print "Some other node has changed the file\n";
+			return 1;
+		# This means that my node has changed the cron file
+		} else {
+
+			print "This node has changed the cron file\n";
+			return 2;
+		}
+	}
+
+	return 0;
+
+}
+
+sub run {
+	print "Host: $hostname started run at: " . time . "\n";
+	print "User: $user, shareddir: $shareddir and spooldir: $spooldir\n";
+
+	if(-e $activesharedcronfile) {
+		unless(-e $oldactivesharedcronfile) {
+			copy($activesharedcronfile, $oldactivesharedcronfile);
+		}
+	}
+
+	unless (-d $electiondir) {
+		mkdir $electiondir;
+	}
+	# Update timestamp
+	write_timestamp;
+	is_active(get_nodes);
+	my $compare = cron_compare($cronfile, $activesharedcronfile, $oldactivesharedcronfile );
+	# The other node has changed the file
+	if($compare == 1) {
+		copy($activesharedcronfile, $cronfile);
+		copy($activesharedcronfile, $oldactivesharedcronfile);
+	# We have changed things
+	} elsif ($compare == 2) {
+		copy($cronfile, $activesharedcronfile);
+	}
+
+	print "Host: $hostname finished run at: " . time . "\n\n\n";
+}
+
+for (my $i = 0; $i < 6; $i++) {
+	my $start = time;
+	$| = 1;
+	run;
+	if ((my $remaining = 10 - (time - $start)) > 0) {
+		sleep $remaining;
+	}
+}
+
+exit;
+