#!/bin/sh

# xymon-rclient.sh
#
# Version 0.4, Jeremy Laidman, February 2012
#
# This script gets xymon client data over a remote connection, typically ssh
# but could be anything that provides a shell prompt.
#
# It typically runs on a display server (but could be any host that
# has a Xymon server or client installed.  It works by pushing the
# xymonclient-<ostype>.sh script to the target server and running it,
# getting the output, enhancing, and feeding it into the Xymon display
# server using $XYMON.  It's designed to be run from within Xymon, but could
# be run separately, as long as some XYM* environment variables are defined.
#
# This is a replacement for the "xymonclient.sh" script that normally
# calls xymonclient-<osname>.sh on the local server.  It requires no
# Xymon installation on the target server.
#
# Tested on a Linux display server, connecting to Solaris target servers.
# Tested using bash as /bin/sh on the display server.

# TODO:
# - implement logfetch
# - allow default settings in .default. hostname

# Release History:
# - v0.1 - Jeremy Laidman
#          initial release
# - v0.2 - Jeremy Laidman
#          modified to work with hobbit
# - v0.3 - Jeremy Laidman
#          adjusted name of client OS script for hobbit
# - v0.4 - Jeremy Laidman
#          more debug info, including showing the whole script to be sent
#          can specify IP address to connect to with %{I} or --ip
#          can specify environment vars to set (can be a security risk)
#          add TOP=$TOP to script, allowing env var to define
#          command-line ostype overrides configured ostype
#          can now specify os of script separately to os of client host

# Installation:
#
# 1. Copy the script to a useful location on your display server
#    such as /usr/lib/xymon/server/ext/
#
# 2. Add a section into tasks.cfg (or create an include file in tasks.d
#    containing the following (adjust as appropriate):
#
#        [xymon-rclient]
#            ENVFILE $XYMONHOME/etc/xymonserver.cfg
#            CMD /$XYMONHOME/ext/xymon-rclient.sh
#            LOGFILE $XYMONSERVERLOGS/xymon-rclient.log
#            INTERVAL 5m
#
#    You can add "-d 1" to the CMD to show debug output.
#
# 3. Add "RCLIENT" definitions to your hosts.cfg as appropriate, such as:
#
#        10.99.1.1 remserver1.example.com # testip noping dialup "RCLIENT:cmd(ssh -T otheruser@%{H}),ostype(sunos)"
#        10.99.1.2 remserver2.example.com # testip noping dialup "RCLIENT:cmd(ssh -T user1@gateway ssh -T -l user1 %{I}),ostype(sunos)"
#        10.99.1.3 remserver3.example.com # testip noping dialup "RCLIENT:cmd(rsh),ostype(linux)"
#        10.99.1.4 remserver4.example.com # testip noping dialup "RCLIENT:cmd(ssh -T remserver4.local),ostype(linux)"
#
#    The %{H} will be substituted with the servername (remserver1.example.com.
#    in the first example).  If the cmd() definition has no spaces, then the
#    servername will be appended regardless.  There are no defaults for cmd()
#    and ostype() - they must be defined.  If %{I} is found, it is replaced
#    by the IP address in the first field of hosts.cfg (see second example).
#
#    The first example connects with ssh to a server and gets its data.
#    The second connects to one server as a jump point to another server, and
#      uses the IP address to connect.
#    The third connects to a server using rsh.
#    The fourth connects to a server by using a different hostname.
#
# 4. If using ssh, establish key authentication, add host keys, etc. 
#    If using rsh, you're an idiot; nevertheless, setup your trust first.

#---------------------------------------------------------------------------#

# adjust if necessary
REM_XYMONTMP=/tmp			# where the target server stores its temp files

# these defaults can be overridden on command-line
LOCALMODE="no"		# if "yes", uses xymond_client instead of xymon
DEBUG=0			# if >0, enables debug levels
DRYRUN=""		# if defined, does everything except send the update
TIMEOUT=60		# how long we wait for command to complete

#---------------------------------------------------------------------------#
# functions
die() { echo "`date`: $*">&2; exit 1; }
warn() { echo "`date`: $*">&2; }

do_usage() {
	echo "$0 [options] [hostname ...]"
	echo ""
	echo "Connects to a remote server and gathers host data"
	echo "for feeding into Xymon."
	echo ""
	echo "If hostname is not specified, connects to"
	echo "all hosts in hosts.cfg with 'RCLIENT' defined"
	echo ""
	echo "Options:"
	echo "  -e             : default client proxy command, if not defined in hosts.cfg"
	echo "                   (eg -e 'ssh user1@server1 ssh -T user2@server2')"
	echo "  -l             : local mode"
	echo "  -o <ostype>    : define or override OS type (eg sunos, or linux)"
	echo "  -s <scriptos>  : define or override OS script name (eg sunos, or linux)"
	echo "  -m <hostmatch> : only on matching hosts or IPs"
	echo "  -i <ip>        : use specified IP to connect"
	echo "  -d <level>     : debug level"
	echo "  -t <seconds>   : timeout (defaults to $TIMEOUT)"
	echo "  -y             : dry-run"
	echo "  -h             : help (what you're reading now)"
	exit 0
}

send_client_script() {
	# skip the exit so we can add other commands
	# assumes the only matching exit is at the end
	[ "$1" ] || return
	TOP=/usr/local/bin/top
	if [ "$TOP" ]; then
		sed '2s,^.*$,TOP='${TOP}',;/^exit$/d' $1
	else
		sed '/^exit$/d' $1
	fi
}

send_clock_commands() {
	# TODO: adjust epoch command (or its preference) based on OS
	cat <<- -EOF-
		# assumes our date is GNU date
		EPOCH=\`date +%s\`
		# assumes we have nawk
		[ "\$EPOCH" = "+%s" ] && EPOCH=\`nawk 'BEGIN{print srand()}' 2>/dev/null\`
		# assumes we have perl
		[ "\$EPOCH" ] && EPOCH=\`perl -e 'print time,"\n"' 2>/dev/null\`
		if [ "\$EPOCH" ]; then
			echo
			echo [clock]
			echo epoch: \${EPOCH}.000000
			echo local: `date    "+%Y-%m-%d %H:%M:%S %Z"`
			echo local: `date -u "+%Y-%m-%d %H:%M:%S %Z"`
		fi
-EOF-
}

do_host() {
	# connect to host with command and return results on stdout
	HN="$1"
	IP="$2"
	OS="$3"
	SC="$4"
	TO="$5"
	RSH_CMD="$6"
	EN="$7"

	[ "$EN" ] && eval $EN

	# check if there's a space, requiring a hostname
	if echo "$RSH_CMD" | grep " " >/dev/null; then
		# substitute with hostname (if exists)
		CMD=`echo "$RSH_CMD" | sed 's/%{H}/'$HN'/g;s/%{I}/'$IP'/g'`
	else
		# leave alone
		CMD="$RSH_CMD $HN"
	fi
	[ 0$DEBUG -gt 0 ] && echo "Command: $CMD" >&2

	# $XYMONTMP and $MACHINEDOTS need to be defined on the remote server before
	# running the script (used for constructing tempfile name and location).
	# Clock values are appended after running the script.

	SCRIPTNAME=$XYMONSERVERROOT/client/bin/${XYMONCLIENT}-${SC}.sh
	[ -f "$SCRIPTNAME" ] || die "No matching script for OS $OS: $SCRIPTNAME"

	# we run the command pipeline in background and then wait for timeout period
	(
		echo "XYMONTMP=$REM_XYMONTMP"
		echo "MACHINEDOTS=$HN"
		# clean up any preamble
		echo "echo"
		echo "echo ---START---"
		send_client_script $SCRIPTNAME
		send_clock_commands
	) | eval $CMD 2>/dev/null | sed '1,/---START---/d;$a[endmarker]\ndummy entry' &
	PROCPID=$!
	[ "$PROCPID" ] || die "Failed to fork command $CMD"

	TIMER=0
	while kill -0 $PROCPID 2>/dev/null && [ $TIMER -lt $TO ]; do
		TIMER=`expr $TIMER + 1`
		[ 0$DEBUG -gt 1 ] && echo "tick $TIMER" >&2
		sleep 1
	done


	if kill -0 $PROCPID 2>/dev/null; then
		[ 0$DEBUG -gt 1 ] && echo "command timed out after $TIMER seconds" >&2 ||
		warn "Process $PROCPID timed out after $TO seconds: $CMD"
		[ 0$DEBUG -gt 2 ] && ps -fp $PROCPID >&2
		kill -15 $PROCPID 2>/dev/null
		sleep 1
		kill -0 $PROCPID 2>/dev/null && kill -9 $PROCPID
	else
		[ 0$DEBUG -gt 1 ] && echo "command completed after $TIMER seconds" >&2
	fi
}

parse_hosts() {
	TMPFILE="$1"
	# get hosts using xymongrep
	# build hashes of hostname/ostype and hostname/cmd
	# RCLIENT format is "RCLIENT:cmd(command),ostype(sunos)"
	$XYMONHOME/bin/$XYMONGREP "RCLIENT*" | sed 's/ *# */ /;s/ dialup//;s/ testip//' > $TMPFILE
	while read IP HN RCLIENT; do
		[ "$HOSTMATCH" = "" ] || echo "$HN" | egrep "$HOSTMATCH" >/dev/null ||
			{ [ 0$DEBUG -gt 0 ] && echo "Skipping host $HN, doesn't match /$HOSTMATCH/"; continue; }
		RCLIENT=`echo "$RCLIENT" | sed 's/^"//;s/"$//'`	 # strip off quotes
		case $RCLIENT in RCLIENT:*);; *) die "RCLIENT malformed: $HN $RCLIENT";; esac
		[ 0$DEBUG -gt 1 ] && echo RCLIENT is $RCLIENT

		CMD=`echo "$RCLIENT" | grep "cmd(" | sed 's/^.*cmd(//;s/).*$//'`
		[ 0$DEBUG -gt 1 ] && echo CMD is "$CMD"

		OSTYPE=`echo "$RCLIENT" | grep "ostype(" | sed 's/^.*ostype(//;s/).*$//'`
		[ 0$DEBUG -gt 1 ] && echo OSTYPE is $OSTYPE
		if [ "$DEFOSTYPE" -a "$OSTYPE" ]; then
			[ 0$DEBUG -gt 1 ] && echo "Overriding configured OS type $OSTYPE with $DEFOSTYPE"
			OSTYPE=$DEFOSTYPE
		fi

		SCRIPTOS=`echo "$RCLIENT" | grep "scriptos(" | sed 's/^.*scriptos(//;s/).*$//'`
		if [ "$DEFSCRIPTOS" -a "$SCRIPTOS" ]; then
			[ 0$DEBUG -gt 1 ] && echo "Overriding configured script OS $SCRIPTOS with $DEFSCRIPTOS"
			SCRIPTOS=$DEFSCRIPTOS
		fi
		[ "$SCRIPTOS" ] || SCRIPTOS=$OSTYPE
		[ 0$DEBUG -gt 1 ] && echo SCRIPTOS is $SCRIPTOS

		TMOUT=`echo "$RCLIENT" | grep "timeout(" | sed 's/^.*timeout(//;s/).*$//'`
		EN=`echo "$RCLIENT" | grep "env(" | sed 's/^.*env(//;s/).*$//'`
		[ 0$DEBUG -gt 1 ] && echo TMOUT is $TMOUT

		eval RCLIENT_HN_$COUNTER="$HN"
		eval RCLIENT_CMD_$COUNTER=\'"$CMD"\'
		eval RCLIENT_OS_$COUNTER="$OSTYPE"
		eval RCLIENT_TO_$COUNTER="$TMOUT"
		eval RCLIENT_IP_$COUNTER=\'"$IP"\'
		eval RCLIENT_EN_$COUNTER=\'"$EN"\'
		eval RCLIENT_SC_$COUNTER=\'"$SCRIPTOS"\'
		COUNTER=`expr $COUNTER + 1`
	done < $TMPFILE
}

add_host() {
	# populate hashes from command-line
	CMD="$1"
	OSTYPE="$2"
	SCRIPTOS="$3"
	TO="$4"
	HN="$5"
	HN="$6"
	eval RCLIENT_HN_$COUNTER="$HN"
	eval RCLIENT_CMD_$COUNTER=\'"$CMD"\'
	eval RCLIENT_TO_$COUNTER=$TO
	eval RCLIENT_OS_$COUNTER=$OSTYPE
	eval RCLIENT_IP_$COUNTER=$IP
	eval RCLIENT_EN_$COUNTER=$EN
	eval RCLIENT_SC_$COUNTER=$SCRIPTOS
}

#---------------------------------------------------------------------------#

[ "$1" ] || exec 2>&1	# send STDERR to STDOUT if being run from xymonlaunch

# mainline
while [ "$1" ]; do
	case $1 in
		-e)		shift; RSH_CMD="$1";;
		-h|--h*)	do_usage; exit;;
		-o|--os*)	shift; DEFOSTYPE="$1";;
		-s|--script*)	shift; DEFSCRIPTOS="$1";;
		-l|--local*)	LOCALMODE=yes;;
		-d|--debug)	shift; DEBUG="$1";;
		-y|--dryrun)	DRYRUN=yes;;
		-t|--timeout)	shift; TIMEOUT="$1";;
		-i|--ip)	shift; IP="$1";;
		-m|--match)	shift; HOSTMATCH="$1";;
		-q|--quiet)	shift; QUIET=yes;;
		--)		shift; break;;
		-*)		die "Invalid option: $1"; exit;;
		*)		break;
	esac
	shift
done

[ "$XYMON" -o "$BB" ] || die "XYMON environment not set"

# xymon binaries
XYMONDCLIENT="xymond_client"
XYMONCLIENT="xymonclient"
XYMONGREP="xymongrep"

if [ "$XYMON" = "" ]; then
	# must be hobbit, we'll emulate a xymon server using hobbit vars
	XYMON="$BB"
	XYMONHOME="$BBHOME"
	XYMSRV="$BBDISP"
	XYMONSERVERROOT="$BBSERVERROOT"
	# hobbit binaries and filenames
	XYMONDCLIENT="hobbitd_client"
	XYMONCLIENT="hobbitclient"
	XYMONGREP="bbhostgrep"
fi

[ "$QUIET" ] || echo "`date`: starting $0"
[ "`test "$TIMEOUT" -gt 0 2>&1`" = "" ] || die "Invalid timeout specified"
[ $TIMEOUT -gt 0 ] || die "Timeout must be a positive integer"

[ "$1" -a "$RSH_CMD" = "" ] && die "No proxy command specified, argv: $0 $@"
if [ "$LOCALMODE" = "yes" ]; then
	die "Local mode is not supported (yet)"
	[ -x $XYMONHOME/bin/$XYMONDCLIENT ] || die "Unable to locate $XYMONHOME/bin/$XYMONDCLIENT"
else
	[ -x $XYMON ] || die "Unable to locate $XYMON"
fi

TMPFILE=`mktemp /tmp/xymon-client-remote-XXXXX`
TMPFILE2=`mktemp /tmp/xymon-client-remote-XXXXX`
trap "rm -f $TMPFILE $TMPFILE2" exit	# clean-up on exit

[ 0$DEBUG -gt 0 ] && date

COUNTER=1	# incremented by the following parse/add functions
if [ "$1" ]; then
	# it doesn't make a lot of sense hitting
	# multiple hosts using the same proxy command 
	# but we support it anyway
	[ 0$DEBUG -gt 0 ] && echo "Adding hosts from CLI"
	while [ "$1" ]; do
		add_host "$RSH_CMD" "$DEFOSTYPE" "$DEFSCRIPTOS" "$TIMEOUT" "$1" "$IP"
		COUNTER=`expr $COUNTER + 1`
		shift
	done
else
	[ 0$DEBUG -gt 0 ] && echo "Adding hosts from hosts.cfg"
	parse_hosts $TMPFILE
fi

[ "$DRYRUN" -a 0$DEBUG -gt 0 ] && echo "Dry-run mode enabled"

# Now we do the work

INDEX=1
OKCOUNT=0
while [ $INDEX -le $COUNTER ]; do
	eval MACHINEDOTS="\$RCLIENT_HN_${INDEX}"
	[ "$MACHINEDOTS" ] || break

	eval OSTYPE="\$RCLIENT_OS_${INDEX}"
	[ "$OSTYPE" ] || die "Unable to get OS type for $MACHINEDOTS"

	eval SCRIPTOS="\$RCLIENT_SC_${INDEX}"
	[ "$SCRIPTOS" ] || die "Unable to get OS script name for $MACHINEDOTS"

	eval CMD="\$RCLIENT_CMD_${INDEX}"
	[ "$CMD" ] || die "Unable to get proxy for $MACHINEDOTS"

	eval TO="\$RCLIENT_TO_${INDEX}"
	[ "$TO" ] || TO=$TIMEOUT
	[ "$TO" ] || die "Unable to get timeout for $MACHINEDOTS"

	eval IP="\$RCLIENT_IP_${INDEX}"
	[ "$IP" ] || IP="0.0.0.0"

	eval EN="\$RCLIENT_EN_${INDEX}"

	[ 0$DEBUG -gt 2 ] && echo "Host $MACHINEDOTS with IP=$IP using ostype=$OSTYPE, scriptos=$SCRIPTOS, timeout=$TO, cmd=$CMD, env=$EN" >&2

	MACHINE=`echo "$MACHINEDOTS" | sed 's/\./,/g'`
	CONFIGCLASS=$OSTYPE

	[ 0$DEBUG -gt 0 ] && echo "Server $INDEX $MACHINEDOTS($OSTYPE)"

	INDEX=`expr $INDEX + 1`

 	[ 0$DEBUG -gt 3 ] && echo "Client script:" && send_client_script $XYMONSERVERROOT/client/bin/${XYMONCLIENT}-${SCRIPTOS}.sh | sed 's/^/  /'

	do_host $MACHINEDOTS $IP $OSTYPE $SCRIPTOS $TO "$CMD" "$EN" > $TMPFILE
	[ -s $TMPFILE ] || {
		warn "Failed to collect data for $MACHINEDOTS"
		continue
	}
	[ 0$DEBUG -gt 3 ] && echo "Client data:" && sed 's/^/  /' $TMPFILE
	grep "^\[endmarker\]$" $TMPFILE >/dev/null || {
		warn "Failed to collect complete data for $MACHINEDOTS"
		continue
	}

	OKCOUNT=`expr $OKCOUNT + 1`

	if [ "$LOCALMODE" = "yes" ]; then
		(
			echo "@@client#1|0|127.0.0.1|$MACHINEDOTS|$SERVEROSTYPE"
			echo "client $MACHINE.$OSTYPE $CONFIGCLASS"
			cat $TMPFILE
		) > $TMPFILE2
		# this doesn't seem to work
		if [ "$DRYRUN" = "" ]; then
			$XYMONHOME/bin/$XYMONDCLIENT --local --config=$XYMONHOME/etc/localclient.cfg < $TMPFILE2
		else
			echo $XYMONHOME/bin/$XYMONDCLIENT --local --config=$XYMONHOME/etc/localclient.cfg \< $TMPFILE2
		fi
	else
		(
			echo "client $MACHINE.$OSTYPE $CONFIGCLASS"
			cat $TMPFILE
		) > $TMPFILE2
		if [ "$DRYRUN" = "" ]; then
			$XYMON $XYMSRV "@" < $TMPFILE2 >/dev/null
		else
			echo $XYMON $XYMSRV "@" \< $TMPFILE2
		fi
	fi
done

INDEX=`expr $INDEX - 1`
[ "$QUIET" ] || echo "`date`: finished $0 (completed $OKCOUNT out of $INDEX)"