Articles‎ > ‎

SSH redirection based on load

posted Dec 28, 2008, 7:18 AM by Philip Rinehart   [ updated Dec 28, 2008, 7:22 AM by Greg Neagle ]

by Timothy O'Keefe

If you have a compute cluster and wish to have SSH users automatically "redirected" from the login node to a child node based on load average, this Perl script (and directions for its use) may be of interest.

What's most special about this script is, the node to which a user is redirected is chosen based on load statistics as returned by Sun Grid Engine with a fallback on choosing a node at random.

The way I got the script to invoke without the chance that a user could circumvent it was to put this line in /etc/bashrc and /etc/csh.login

--- /etc/bashrc
x=`id | grep 80\(admin\)`
if [ -z "$x" ] && [ ${TERM} != "dumb" ] 
then
/bin/sshsh
exec /bin/exitshell
fi
---

-- /etc/csh.login (and variants)
if ( "$x" == ""  && ${TERM} != "dumb") then
/bin/sshsh
exec /bin/exitshell
endif
---
 
These lines will run the redirection utility as soon as a non-administrator attempts to log in; such a user has no choice in the matter. Also, dumb terminals will not be redirected (e.g., SFTP). The user also cannot terminate the script prematurely in that it implements an interruption handler.
 
Here is the actual script. Only a couple of parameters need to be setup at the top:
 
 
#!/usr/bin/perl
#
# Secure Shell Redirection v1.0
# Uses Sun Grid Engine Load Statistics
# Fallback on random node selection
#
# Created by: tim okeefe 7.31.07
#       modified: 9.19.07 - fix qstat com
#

# ----------------------------------------------------------------------
my $queue = 'queueName';
my $domain = 'domain.name';
my $broadcast = '192.168.1.255';
my $admin = 'administrator@email.com';

my @excludeHosts = ("machines","not","to","be","considered","for","redirection");
# ----------------------------------------------------------------------

## interruption handler
$SIG{INT} = \&INT_handler;

print STDERR "####################################\n\n";
## try querying SGE
&getQstatHosts;

## if SGE query fails (returns nothing), try choosing randomly
if(@hostsSorted == 0) {
        &getRandomHost
} else {
        print "Load statistics are in...\n\n";
}

## if we still don't have any hosts, fail entirely.
if(@hostsSorted == 0) {
        print "Cannot redirect you to a host...\n  contact: $admin\n\n";
        print STDERR "####################################\n\n";
        exit;
}

&redirect;

exit;

######################################################################
sub redirect {
        $machine = $hostsSorted[0];

        print STDERR "Redirecting you to $machine\n\n";
        print STDERR "####################################\n\n";
        system("/usr/bin/ssh", "-t", $machine);
}

sub getQstatHosts {

        ## Return a list of hosts sorted by `qstat` load average statistics

        # -- get load stats
        open(QSTAT, "qstat -f | grep $queue 2> /dev/null |");
        while(<QSTAT>) {
                @a = split(/\s+/,$_);
                @tmp = split("@",@a[0]);
                if(@a[3] ne "-NA-") {
                        $hosts{@tmp[1]} = @a[3];
                }
        }
        close QSTAT;

        # -- sort hash
        $i=0;
        foreach $value (sort { $hosts{$a} <=> $hosts{$b} } keys %hosts)
        {
                @hostsSorted[$i] = $value;
                $i++;
        }
}

sub getRandomHost {

        ## Query a list of hosts on local subnet by `arp` and permute

        use Net::Ping;
        # --- update dns cache
        $p = Net::Ping->new("tcp",1,64);
        $p->ping($broadcast);
        $p->close();

        # -- get host names
        open(ARP, "arp -a | grep -vir incomplete | grep -vir ? | grep $domain |");
        $i=1;
        $k=0;
        while(<ARP>) {
                @a = split(/\s+/,$_);
                $exclude = 0;
                for($j=0; $j<=@excludeHosts-1; $j++) {
                        if(@a[0] eq @excludeHosts[$j].".".$domain) {
                                $exclude = 1;
                        }
                }
                if($exclude == 0) {
                        @hosts[$k] = @a[0];
                        $k++;
                }
                $i++;
        }

        # -- permute host name list
        $nHosts = @hosts;
        for($i=0; $i<=$nHosts-1; $i++) {
                $random_n = int(rand(@hosts-1));
                @hostsSorted[$i] = @hosts[$random_n];
                splice(@hosts,$random_n,1);
        }

}

sub INT_handler {
        print "It's rude to interrupt.";
        exit(0);
}
ClosePrint
Comments