#!/bin/sh

# xymon-rclient.sh
#
# Version 0.3, Jeremy Laidman, December 2011
#
# This script gets xymon client data over a remote connection, typically ssh
# but could be anything that provides a shell prompt.
#
# It typically runs on a display server (but could be any server that
# has a Xymon server or client installed.  It works by pushing the
# xymonclient-<ostype>.sh script to the target server and running it,
# getting the output, enhancing it, and feeding it into the Xymon display
# server using $XYMON.  It's designed to be run from within Xymon, but could
# be run separately, as long as some XYM* environment variables are defined.
#
# This is a replacement for the "xymonclient.sh" script that normally
# calls xymonclient-<osname>.sh on the local server.  It requires no
# Xymon installation on the target server.
#
# Tested on a Linux display server, connecting to Solaris target servers.
# Tested using bash as /bin/sh on the display server.

# TODO:
# - implement logfetch
# - allow default settings in .default. hostname

# Release History:
# - v0.1 - Jeremy Laidman
#          initial release
# - v0.2 - Jeremy Laidman
#          modified to work with hobbit
# - v0.3 - Jeremy Laidman
#          adjusted name of client OS script for hobbit

# Installation:
#
# 1. Copy the script to a useful location on your display server
#    such as /usr/lib/xymon/server/ext/
#
# 2. Add a section into tasks.cfg (or create an include file in tasks.d
#    containing the following (adjust as appropriate):
#
#        [xymon-rclient]
#            ENVFILE $XYMONHOME/etc/xymonserver.cfg
#            CMD /$XYMONHOME/ext/xymon-rclient.sh
#            LOGFILE $XYMONSERVERLOGS/xymon-rclient.log
#            INTERVAL 5m
#
#    You can add "-d 1" to the CMD to show debug output.
#
# 3. Add "RCLIENT" definitions to your hosts.cfg as appropriate, such as:
#
#        10.99.1.1 remserver1.example.com # testip noping dialup "RCLIENT:cmd(ssh -T otheruser@%{H}),ostype(sunos)"
#        10.99.1.2 remserver2.example.com # testip noping dialup "RCLIENT:cmd(ssh -T user1@gateway ssh -T -l user1 %{H}),ostype(sunos)"
#        10.99.1.3 remserver3.example.com # testip noping dialup "RCLIENT:cmd(rsh),ostype(linux)"
#        10.99.1.4 remserver4.example.com # testip noping dialup "RCLIENT:cmd(ssh -T remserver4.local),ostype(linux)"
#
#    The %{H} will be substituted with the servername (remserver1.example.com.
#    in the first example).  If the cmd() definition has no spaces, then the
#    servername will be appended regardless.  There are no defaults.
#
#    The first example connects with ssh to a server and gets its data.
#    The second connects to one server as a jump point to another server.
#    The third connects to a server using rsh.
#    The fourth connects to a server by using a different hostname.
#
# 4. If using ssh, establish key authentication, add host keys, etc. 
#    If using rsh, you're an idiot, nevertheless, setup your trust.

#---------------------------------------------------------------------------#

# adjust if necessary
REM_XYMONTMP=/tmp			# where the target server stores its temp files

# these defaults can be overridden on command-line
LOCALMODE="no"		# if "yes", uses xymond_client instead of xymon
DEBUG=0			# if >0, enables debug levels
DRYRUN=""		# if defined, does everything except send the update
TIMEOUT=60		# how long we wait for command to complete

#---------------------------------------------------------------------------#
# functions
die() { echo "`date`: $*">&2; exit 1; }
warn() { echo "`date`: $*">&2; }

do_usage() {
	echo "$0 [options] [hostname ...]"
	echo ""
	echo "Connects to a remote server and gathers host data"
	echo "for feeding into Xymon."
	echo ""
	echo "If hostname is not specified, connects to"
	echo "all hosts in hosts.cfg with 'RCLIENT' defined"
	echo ""
	echo "Options:"
	echo "  -e             : default client proxy command, if not defined in hosts.cfg"
	echo "                   (eg -e 'ssh user1@server1 ssh -T user2@server2')"
	echo "  -l             : local mode"
	echo "  -o <ostype>    : define OS type (eg sunos, or linux)"
	echo "  -m <hostmatch> : only on matching hosts or IPs"
	echo "  -d <level>     : debug level"
	echo "  -t <seconds>   : timeout (defaults to $TIMEOUT)"
	echo "  -y             : dry-run"
	echo "  -h             : help (what you're reading now)"
	exit 0
}

send_client_script() {
	# skip the exit so we can add other commands
	# assumes the only matching exit is at the end
	sed '/^exit$/d' $SCRIPTNAME
}

send_clock_commands() {
	# TODO: adjust epoch command (or its preference) based on OS
	cat <<- -EOF-
		# assumes our date is GNU date
		EPOCH=\`date +%s\`
		# assumes we have nawk
		[ "\$EPOCH" = "+%s" ] && EPOCH=\`nawk 'BEGIN{print srand()}' 2>/dev/null\`
		# assumes we have perl
		[ "\$EPOCH" ] && EPOCH=\`perl -e 'print time,"\n"' 2>/dev/null\`
		if [ "\$EPOCH" ]; then
			echo
			echo [clock]
			echo epoch: \${EPOCH}.000000
			echo local: `date    "+%Y-%m-%d %H:%M:%S %Z"`
			echo local: `date -u "+%Y-%m-%d %H:%M:%S %Z"`
		fi
-EOF-
}

do_host() {
	# connect to host with command and return results on stdout
	HN="$1"
	OS="$2"
	TO="$3"
	RSH_CMD="$4"

	# check if there's a space, requiring a hostname
	if echo "$RSH_CMD" | grep " " >/dev/null; then
		# substitute with hostname (if exists)
		CMD=`echo "$RSH_CMD" | sed 's/%{H}/'$HN'/'`
	else
		# leave alone
		CMD="$RSH_CMD $HN"
	fi
	[ 0$DEBUG -gt 0 ] && echo "Command: $CMD" >&2

	# $XYMONTMP and $MACHINEDOTS need to be defined on the remote server before
	# running the script (used for constructing tempfile name and location).
	# Clock values are appended after running the script.

	SCRIPTNAME=$XYMONSERVERROOT/client/bin/${XYMONCLIENT}-${OS}.sh
	[ -f "$SCRIPTNAME" ] || die "No matching script for OS $OS: $SCRIPTNAME"

	# we run the command pipeline in background and then wait for timeout period
	(
		echo "XYMONTMP=$REM_XYMONTMP"
		echo "MACHINEDOTS=$HN"
		# clean up any preamble
		echo "echo"
		echo "echo ---START---"
		send_client_script
		send_clock_commands
	) | eval $CMD 2>/dev/null | sed '1,/---START---/d;$a[endmarker]\ndummy entry' &
	PROCPID=$!
	[ "$PROCPID" ] || die "Failed to fork command $CMD"

	TIMER=0
	while kill -0 $PROCPID 2>/dev/null && [ $TIMER -lt $TO ]; do
		TIMER=`expr $TIMER + 1`
		[ 0$DEBUG -gt 1 ] && echo "tick $TIMER" >&2
		sleep 1
	done


	if kill -0 $PROCPID 2>/dev/null; then
		[ 0$DEBUG -gt 1 ] && echo "command timed out after $TIMER seconds" >&2 ||
		warn "Process $PROCPID timed out after $TO seconds: $CMD"
		[ 0$DEBUG -gt 2 ] && ps -fp $PROCPID >&2
		kill -15 $PROCPID 2>/dev/null
		sleep 1
		kill -0 $PROCPID 2>/dev/null && kill -9 $PROCPID
	else
		[ 0$DEBUG -gt 1 ] && echo "command completed after $TIMER seconds" >&2
	fi
}

parse_hosts() {
	TMPFILE="$1"
	# get hosts using xymongrep
	# build hashes of hostname/ostype and hostname/cmd
	# RCLIENT format is "RCLIENT:cmd(command),ostype(sunos)"
	$XYMONHOME/bin/$XYMONGREP "RCLIENT*" | sed 's/^[0-9\.]* //;s/ *# */ /;s/ dialup//;s/ testip//' > $TMPFILE
	while read HN RCLIENT; do
		[ "$HOSTMATCH" = "" ] || echo "$HN" | egrep "$HOSTMATCH" >/dev/null ||
			{ [ 0$DEBUG -gt 0 ] && echo "Skipping host $HN, doesn't match /$HOSTMATCH/"; continue; }
		RCLIENT=`echo "$RCLIENT" | sed 's/^"//;s/"$//'`	 # strip off quotes
		case $RCLIENT in RCLIENT:*);; *) die "RCLIENT malformed: $HN $RCLIENT";; esac
		[ 0$DEBUG -gt 1 ] && echo RCLIENT is $RCLIENT
		CMD=`echo "$RCLIENT" | grep "cmd(" | sed 's/^.*cmd(//;s/).*$//'`
		[ 0$DEBUG -gt 1 ] && echo CMD is "$CMD"
		OSTYPE=`echo "$RCLIENT" | grep "ostype(" | sed 's/^.*ostype(//;s/).*$//'`
		[ 0$DEBUG -gt 1 ] && echo OSTYPE is $OSTYPE
		TMOUT=`echo "$RCLIENT" | grep "timeout(" | sed 's/^.*timeout(//;s/).*$//'`
		[ 0$DEBUG -gt 1 ] && echo TMOUT is $TMOUT
		eval RCLIENT_HN_$COUNTER="$HN"
		eval RCLIENT_CMD_$COUNTER=\'"$CMD"\'
		eval RCLIENT_OS_$COUNTER="$OSTYPE"
		eval RCLIENT_TO_$COUNTER="$TMOUT"
		COUNTER=`expr $COUNTER + 1`
	done < $TMPFILE
}

add_host() {
	# populate hashes from command-line
	CMD="$1"
	OSTYPE="$2"
	TO="$3"
	HN="$4"
	eval RCLIENT_HN_$COUNTER="$HN"
	eval RCLIENT_CMD_$COUNTER=\'"$CMD"\'
	eval RCLIENT_TO_$COUNTER=$TO
	eval RCLIENT_OS_$COUNTER=$OSTYPE
}

#---------------------------------------------------------------------------#

[ "$1" ] || exec 2>&1	# send STDERR to STDOUT if being run from xymonlaunch

# mainline
while [ "$1" ]; do
	case $1 in
		-e)		shift; RSH_CMD="$1";;
		-h|--h*)	do_usage; exit;;
		-o|--os*)	shift; DEFOSTYPE="$1";;
		-l|--local*)	LOCALMODE=yes;;
		-d|--debug)	shift; DEBUG="$1";;
		-y|--dryrun)	DRYRUN=yes;;
		-t|--timeout)	shift; TIMEOUT="$1";;
		-m|--match)	shift; HOSTMATCH="$1";;
		-q|--quiet)	shift; QUIET=yes;;
		--)		shift; break;;
		-*)		die "Invalid option: $1"; exit;;
		*)		break;
	esac
	shift
done

[ "$XYMON" -o "$BB" ] || die "XYMON environment not set"

# xymon binaries
XYMONDCLIENT="xymond_client"
XYMONCLIENT="xymonclient"
XYMONGREP="xymongrep"

if [ "$XYMON" = "" ]; then
	# must be hobbit, we'll emulate hobbit using xymon vars
	XYMON="$BB"
	XYMONHOME="$BBHOME"
	XYMSRV="$BBDISP"
	XYMONSERVERROOT="$BBSERVERROOT"
	# hobbit binaries and filenames
	XYMONDCLIENT="hobbitd_client"
	XYMONCLIENT="hobbitclient"
	XYMONGREP="bbhostgrep"
fi

[ "$QUIET" ] || echo "`date`: starting $0"
[ "`test "$TIMEOUT" -gt 0 2>&1`" = "" ] || die "Invalid timeout specified"
[ $TIMEOUT -gt 0 ] || die "Timeout must be a positive integer"

[ "$1" -a "$RSH_CMD" = "" ] && die "No proxy command specified, argv: $0 $@"
if [ "$LOCALMODE" = "yes" ]; then
	die "Local mode is not supported (yet)"
	[ -x $XYMONHOME/bin/$XYMONDCLIENT ] || die "Unable to locate $XYMONHOME/bin/$XYMONDCLIENT"
else
	[ -x $XYMON ] || die "Unable to locate $XYMON"
fi

TMPFILE=`mktemp /tmp/xymon-client-remote-XXXXX`
TMPFILE2=`mktemp /tmp/xymon-client-remote-XXXXX`
trap "rm -f $TMPFILE $TMPFILE2" exit	# clean-up on exit

[ 0$DEBUG -gt 0 ] && date

COUNTER=1	# incremented by the following parse/add functions
if [ "$1" ]; then
	# it doesn't make a lot of sense hitting
	# multiple hosts using the same proxy command 
	# but we support it anyway
	[ 0$DEBUG -gt 0 ] && echo "Adding hosts from CLI"
	while [ "$1" ]; do
		add_host "$RSH_CMD" "$DEFOSTYPE" "$TIMEOUT" "$1"
		COUNTER=`expr $COUNTER + 1`
		shift
	done
else
	[ 0$DEBUG -gt 0 ] && echo "Adding hosts from hosts.cfg"
	parse_hosts $TMPFILE
fi

[ "$DRYRUN" -a 0$DEBUG -gt 0 ] && echo "Dry-run mode enabled"

# Now we do the work

INDEX=1
OKCOUNT=0
while [ $INDEX -le $COUNTER ]; do
	eval MACHINEDOTS="\$RCLIENT_HN_${INDEX}"
	[ "$MACHINEDOTS" ] || break

	eval OSTYPE="\$RCLIENT_OS_${INDEX}"
	[ "$OSTYPE" ] || die "Unable to get OS type for $MACHINEDOTS"

	eval CMD="\$RCLIENT_CMD_${INDEX}"
	[ "$CMD" ] || die "Unable to get proxy for $MACHINEDOTS"

	eval TO="\$RCLIENT_TO_${INDEX}"
	[ "$TO" ] || TO=$TIMEOUT
	[ "$TO" ] || die "Unable to get timeout for $MACHINEDOTS"

	[ 0$DEBUG -gt 2 ] && echo "Host $MACHINEDOTS using ostype=$OSTYPE, timeout=$TO, cmd=$CMD" >&2

	MACHINE=`echo "$MACHINEDOTS" | sed 's/\./,/g'`
	CONFIGCLASS=$OSTYPE

	[ 0$DEBUG -gt 0 ] && echo "Server $INDEX $MACHINEDOTS($OSTYPE)"

	INDEX=`expr $INDEX + 1`

	do_host $MACHINEDOTS $OSTYPE $TO "$CMD" > $TMPFILE
	[ -s $TMPFILE ] || {
		warn "Failed to collect data for $MACHINEDOTS"
		continue
	}
	grep "^\[endmarker\]$" $TMPFILE >/dev/null || {
		warn "Failed to collect complete data for $MACHINEDOTS"
		continue
	}

	OKCOUNT=`expr $OKCOUNT + 1`

	if [ "$LOCALMODE" = "yes" ]; then
		(
			echo "@@client#1|0|127.0.0.1|$MACHINEDOTS|$SERVEROSTYPE"
			echo "client $MACHINE.$OSTYPE $CONFIGCLASS"
			cat $TMPFILE
		) > $TMPFILE2
		# this doesn't seem to work
		if [ "$DRYRUN" = "" ]; then
			$XYMONHOME/bin/$XYMONDCLIENT --local --config=$XYMONHOME/etc/localclient.cfg < $TMPFILE2
		else
			echo $XYMONHOME/bin/$XYMONDCLIENT --local --config=$XYMONHOME/etc/localclient.cfg \< $TMPFILE2
		fi
	else
		(
			echo "client $MACHINE.$OSTYPE $CONFIGCLASS"
			cat $TMPFILE
		) > $TMPFILE2
		if [ "$DRYRUN" = "" ]; then
			$XYMON $XYMSRV "@" < $TMPFILE2 >/dev/null
		else
			echo $XYMON $XYMSRV "@" \< $TMPFILE2
		fi
	fi
done

INDEX=`expr $INDEX - 1`
[ "$QUIET" ] || echo "`date`: finished $0 (completed $OKCOUNT out of $INDEX)"