#!/bin/sh
# PCP QA Test No. 856
# Check PMCD state change notification, aka PMCD_ADD_AGENT
#
# Copyright (c) 2016 Red Hat, Inc.
#

seq=`basename $0`
echo "QA output created by $seq"

# get standard filters
. ./common.product
. ./common.filter
. ./common.check

signal=$PCP_BINADM_DIR/pmsignal
status=1
done_clean=false
LOCALHOST=`hostname`
LOGGING_DIR="$PCP_ARCHIVE_DIR"

_check_local_primary_archive || _notrun "primary pmlogger not writing locally"

if [ ! -f $LOGGING_DIR/$LOCALHOST/Latest ]
then
    _notrun "no Latest folio for primary pmlogger (logpush?)"
    # NOTREACHED
fi

_cleanup()
{
    if $done_clean
    then
	:
    else
	echo "Restore pmcd.conf and restart PMCD ..."
	if [ -f $tmp.pmcd.conf ]
	then
	    $sudo cp $tmp.pmcd.conf $PCP_PMCDCONF_PATH
	    rm -f $tmp.pmcd.conf
	fi
	_restore_primary_logger
	_service pmcd restart 2>&1 | _filter_pcp_restart
	_wait_for_pmcd
	_restore_auto_restart pmcd
	_service pmlogger restart 2>&1 | _filter_pcp_restart \
	| sed -e '/pmlogger not running/d'
	_wait_for_pmlogger
	_restore_auto_restart pmlogger
	done_clean=true
    fi
    rm -f $tmp.*
    exit $status
}

trap "_cleanup" 0 1 2 3 15

# real QA test starts here
PMDA_PMCD_PATH=$PCP_PMDAS_DIR/pmcd/pmda_pmcd.$DSO_SUFFIX

# copy the pmcd config file to restore state later.
cp $PCP_PMCDCONF_PATH $tmp.pmcd.conf

# start from a known starting point
_stop_auto_restart pmcd
_stop_auto_restart pmlogger
if ! _service pmlogger stop 2>&1; then _exit 1; fi \
| _filter_pcp_stop
_wait_pmlogger_end || _exit 1
if ! _service pmcd stop 2>&1; then _exit 1; fi \
| _filter_pcp_stop
_wait_pmcd_end || _exit 1

cat <<End-of-File >$tmp.tmp
# Installed by PCP QA test $seq on `date`
pmcd	2	dso	pmcd_init	$PMDA_PMCD_PATH
sample	29	pipe	binary 		$PCP_PMDAS_DIR/sample/pmdasample -d 29 
simple	253	pipe	binary 		/var/lib/pcp/pmdas/simple/pmdasimple -d 253 
End-of-File
$sudo cp $tmp.tmp $PCP_PMCDCONF_PATH

_writable_primary_logger
if ! _service pmcd restart 2>&1; then _exit 1; fi \
| _filter_pcp_restart
_wait_for_pmcd || _exit 1
if ! _service pmlogger restart 2>&1; then _exit 1; fi \
| _filter_pcp_restart
_wait_for_pmlogger || _exit 1

# Reset pmlogger
echo "log sample.dynamic.meta.metric"
# Log the sample pmda's sample.dynamic.meta.metric
# ... may need to retry to get connected to the primary pmlogger
#
can_wait=10
i=0
while [ $i -lt $can_wait ]
do
    echo 'connect primary
log mandatory on 500 msec { sample.dynamic.meta.metric }' \
    | pmlc >$tmp.out 2>&1
    if [ -s $tmp.out ]
    then
	:
    else
	# expect no output if all OK
	#
	break
    fi
    sleep 1
    i=`expr $i + 1`
done
[ -s $tmp.out ] && cat $tmp.out

# log a few samples
sleep 2

echo "archive contents before pmda simple restart..."
echo "expect no <mark> record"
echo "---------------------------------------------------------------"
pmafm $LOGGING_DIR/$LOCALHOST/Latest pmdumplog >$tmp.out 2>&1
grep -E '(<mark>)|(sample.dynamic.meta.metric)' $tmp.out \
| _filter_pmdumplog \
| uniq
echo "---------------------------------------------------------------"

# Get the instance domain of sample.dynamic.meta.metric by fetching 
# sample.dynamic.meta.pmdesc.indom using pminfo.
echo "pminfo -f sample.dynamic.meta.pmdesc.indom" | tee -a $seq_full
indom=`pminfo -f sample.dynamic.meta.pmdesc.indom | tee -a $seq_full \
    | awk '/value/ { print $2 }'`

# Change the instance domain of sample.dynamic.meta.metric by using pmstore to store into
# sample.dynamic.meta.pmdesc.indom
echo "changing instance domain from $indom to $((indom - 1))"
pmstore sample.dynamic.meta.pmdesc.indom $((indom - 1))

# Restart the simple pmda by killing it and the getting pmcd to restart it (HUP).
# pmlogger should receive PMCD_ADD_AGENT, which should cause it to re-check
# all of it's configured metrics. It should then flag the change to
# sample.dynamic.meta.pmdesc.indom and exit.
echo "restart the simple pmda"
$sudo $signal -a -s TERM pmdasimple
sleep 1 # let the signal arrive and let pmdasimple exit
$sudo $signal -a -s HUP pmcd   # ensure pmcd restarts the PMDA
sleep 1 # let the signal arrive and pmcd think about it

echo "archive contents after pmda simple restart..."
echo "expect no <mark> record"
echo "---------------------------------------------------------------"
pmafm $LOGGING_DIR/$LOCALHOST/Latest pmdumplog >$tmp.out 2>&1
grep -E '(<mark>)|(sample.dynamic.meta.metric)' $tmp.out \
| _filter_pmdumplog \
| uniq
echo "---------------------------------------------------------------"

# pmlogger should have stopped. Check for the messages indicating that
# the re-check of the configured metrics failed and that it stopped.
echo >>$seq_full
echo "pmlogger log ..." >>$seq_full
cat $LOGGING_DIR/$LOCALHOST/pmlogger.log | tee -a $seq_full | \
grep \
 -e 'Instance domain of metric "sample.dynamic.meta.metric" has changed from PM_INDOM_NULL to 511.4194302' \
 -e 'One or more configured metrics have changed after pmcd state change. Exiting'

status=0
exit
