Back to home page

Enduro/X

 
 

    


0001 #!/bin/bash
0002 ##
0003 ## @brief @(#) Test025, Client Process Monitor tests
0004 ##   Currently will test for
0005 ##   - Auto startup
0006 ##   - Manual startup
0007 ##   - Automatic restart
0008 ##   - Shutdown
0009 ##
0010 ## @file run.sh
0011 ##
0012 ## -----------------------------------------------------------------------------
0013 ## Enduro/X Middleware Platform for Distributed Transaction Processing
0014 ## Copyright (C) 2009-2016, ATR Baltic, Ltd. All Rights Reserved.
0015 ## Copyright (C) 2017-2023, Mavimax, Ltd. All Rights Reserved.
0016 ## This software is released under one of the following licenses:
0017 ## AGPL (with Java and Go exceptions) or Mavimax's license for commercial use.
0018 ## See LICENSE file for full text.
0019 ## -----------------------------------------------------------------------------
0020 ## AGPL license:
0021 ##
0022 ## This program is free software; you can redistribute it and/or modify it under
0023 ## the terms of the GNU Affero General Public License, version 3 as published
0024 ## by the Free Software Foundation;
0025 ##
0026 ## This program is distributed in the hope that it will be useful, but WITHOUT ANY
0027 ## WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
0028 ## PARTICULAR PURPOSE. See the GNU Affero General Public License, version 3
0029 ## for more details.
0030 ##
0031 ## You should have received a copy of the GNU Affero General Public License along 
0032 ## with this program; if not, write to the Free Software Foundation, Inc.,
0033 ## 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
0034 ##
0035 ## -----------------------------------------------------------------------------
0036 ## A commercial use license is available from Mavimax, Ltd
0037 ## contact@mavimax.com
0038 ## -----------------------------------------------------------------------------
0039 ##
0040 
0041 export TESTNO="025"
0042 export TESTNAME_SHORT="cpmsrv"
0043 export TESTNAME="test${TESTNO}_${TESTNAME_SHORT}"
0044 
0045 PWD=`pwd`
0046 if [ `echo $PWD | grep $TESTNAME ` ]; then
0047         # Do nothing 
0048         echo > /dev/null
0049 else
0050         # started from parent folder
0051         pushd .
0052         echo "Doing cd"
0053         cd $TESTNAME
0054 fi;
0055 
0056 . ../testenv.sh
0057 
0058 export TESTDIR="$NDRX_APPHOME/atmitest/$TESTNAME"
0059 export PATH=$PATH:$TESTDIR
0060 # Configure the runtime - override stuff here!
0061 export NDRX_CONFIG=$TESTDIR/ndrxconfig.xml
0062 export NDRX_DMNLOG=$TESTDIR/ndrxd.log
0063 export NDRX_LOG=$TESTDIR/ndrx.log
0064 export NDRX_DEBUG_CONF=$TESTDIR/debug.conf
0065 # Override timeout!
0066 export NDRX_TOUT=30
0067 export NDRX_SILENT=Y
0068 export NDRX_ULOG=$TESTDIR
0069 # Test process count
0070 PROC_COUNT=100
0071 #
0072 # Having some bash forks... (now with read from pipe - no forks... - thus the count must be exact...)
0073 # when all bins have started...
0074 #
0075 PROC_COUNT_DIFFALLOW=100
0076 
0077 #
0078 # Generic exit function
0079 #
0080 function go_out {
0081     echo "Test exiting with: $1"
0082     xadmin stop -y
0083     xadmin down -y
0084     xadmin killall chld1.sh chld2.sh chld3.sh chld4.sh chld5.sh chld6.sh
0085     rm test25_sleep.fifo
0086 
0087     popd 2>/dev/null
0088     exit $1
0089 }
0090 
0091 #
0092 # Test process count with given name
0093 #
0094 function test_proc_cnt {
0095     proc=$1
0096     cnt=$2
0097     go=$3
0098 
0099     echo ">>> Testing $proc to be $cnt of count. If fail, error $go"
0100     $PSCMD | grep $proc | grep -v grep
0101     CNT=`$PSCMD | grep $proc | grep -v grep | wc | awk '{print $1}'`
0102     XPROC_COUNT=$cnt
0103     echo ">>> $PSCMD procs: $CNT"
0104     if [[ "X$CNT" != "X$XPROC_COUNT" ]]; then 
0105         echo "****** grep"
0106         $PSCMD | grep $proc | grep -v grep
0107         echo "****** wc"
0108         $PSCMD | grep $proc | grep -v grep | wc
0109         echo "******"
0110 
0111         echo "TESTERROR! $XPROC_COUNT $proc not booted (according to $PSCMD )!"
0112         go_out $go
0113     fi
0114 
0115     echo "$proc ok cnt $cnt ok"
0116 
0117 }  
0118 
0119 rm *.log
0120 
0121 #
0122 # Kill the children test processes if any
0123 #
0124 xadmin killall chld1.sh chld2.sh chld3.sh chld4.sh chld5.sh chld6.sh ndrxbatchmode whileproc.sh
0125 rm test25_sleep.fifo 2>/dev/null
0126 mkfifo test25_sleep.fifo || exit 99
0127 
0128 xadmin down -y
0129 xadmin start -y || go_out 1
0130 
0131 ################################################################################
0132 echo "Run some tests of the batch mode"
0133 ################################################################################
0134 
0135 test_proc_cnt "ndrxbatchmode.sh" "0" "29"
0136 
0137 echo "Batch start"
0138 
0139 #
0140 # Wait time shall be less than time out...
0141 #
0142 xadmin bc -t BATCH% -s% -w 15000
0143 
0144 xadmin pc
0145 
0146 # xadmin killall cpmsrv does not work, as our test name is test025_cpmsrv
0147 # thus it will kill the test
0148 echo "Kill CPMSRV, test shared memory recovery...."
0149 CPMPID=`xadmin ppm | grep cpmsrv | awk '{print $3}'`
0150 
0151 echo "CPMSRV pid to kill=$CPMPID"
0152 
0153 kill -9 $CPMPID
0154 xadmin ppm
0155 
0156 echo "Wait for cpmsrv reboot"
0157 sleep 20
0158 
0159 xadmin ppm
0160 xadmin pc
0161 
0162 echo "CONTINUE..."
0163 
0164 test_proc_cnt "ndrxbatchmode.sh" "3" "30"
0165 
0166 echo "Batch stop (no subsect)"
0167 xadmin sc -t BATCH%
0168 
0169 test_proc_cnt "ndrxbatchmode.sh" "2" "31"
0170 
0171 echo "with subsect"
0172 xadmin sc -t BATCH% -sB%
0173 test_proc_cnt "ndrxbatchmode.sh" "0" "32"
0174 
0175 echo "Batch start"
0176 xadmin bc -t BATCH% -s%
0177 sleep 15
0178 
0179 test_proc_cnt "ndrxbatchmode.sh" "3" "33"
0180 
0181 
0182 echo "Testing batch reload"
0183 OUT1=`xadmin pc`
0184 
0185 echo "Before reload [$OUT1]"
0186 
0187 # shall pass on w/out problem...
0188 xadmin rc NOT_EXISTS_TEST
0189 
0190 xadmin rc -t BATCH% -s% -w 15000
0191 
0192 echo "***** PC *****"
0193 xadmin pc
0194 echo "***** PC, END *****"
0195 
0196 # wait for processes to boot, seems on some machines
0197 # the processes are still booting...
0198 sleep 5
0199 test_proc_cnt "ndrxbatchmode.sh" "3" "34"
0200 OUT2=`xadmin pc`
0201 
0202 echo "After reload [$OUT2]"
0203 
0204 if [ "X$OUT1" == "X$OUT2" ]; then
0205     echo "TESTERROR! [$OUT1]==[$OUT2] -> FAIL, pid must be changed..."
0206     go_out 35
0207 fi
0208 
0209 xadmin bc -t TESTENV
0210 xadmin bc -t TESTENV2
0211 
0212 sleep 10
0213 
0214 xadmin pc
0215 
0216 
0217 ################################################################################
0218 # Child cleanup... testing
0219 ################################################################################
0220 
0221 # At this point we should have all child processes
0222 
0223 test_proc_cnt "chld1.sh" "1" "21"
0224 test_proc_cnt "chld2.sh" "1" "22"
0225 test_proc_cnt "chld3.sh" "1" "23"
0226 test_proc_cnt "chld4.sh" "1" "24"
0227 test_proc_cnt "chld5.sh" "1" "25"
0228 test_proc_cnt "chld6.sh" "1" "26"
0229 
0230 echo "Stopping CHLD 1,3,5 ..."
0231 
0232 xadmin sc -t CHLD1
0233 test_proc_cnt "chld1.sh" "0" "27"
0234 test_proc_cnt "chld2.sh" "0" "28"
0235 
0236 xadmin sc -t CHLD3
0237 test_proc_cnt "chld3.sh" "0" "23"
0238 test_proc_cnt "chld4.sh" "0" "24"
0239 
0240 #
0241 # here "chld3.log" should contain some messages about shutdown...
0242 #
0243 OUT=`grep "Doing exit of chld3" chld3.log`
0244 
0245 if [[ "X$OUT" == "X" ]]; then
0246     echo "chld3.log does not contain the message!!!!"
0247     go_out 25
0248 fi
0249 
0250 xadmin sc -t CHLD5
0251 test_proc_cnt "chld5.sh" "0" "26"
0252 test_proc_cnt "chld6.sh" "1" "27"
0253 
0254 ################################################################################
0255 # Child cleanup... testing, end
0256 ################################################################################
0257 
0258 # Test for section/subsection passing
0259 OUT=`grep '\-t TAG1 \-s SUBSECTION1' testbin1-1.log`
0260 if [[ "X$OUT" == "X" ]]; then
0261         echo "TESTERROR! invalid testbin1-1.log content!"
0262         go_out 1
0263 fi
0264 
0265 OUT=`grep '\-t TAG2 \-s SUBSECTION2' testbin1-2.log`
0266 if [[ "X$OUT" == "X" ]]; then
0267         echo "TESTERROR! invalid testbin1-2.log content!"
0268         go_out 2
0269 fi
0270 
0271 # Test for env override processing
0272 OUT=`grep ___ENV_OVER__1  env1.log`
0273 if [[ "X$OUT" == "X" ]]; then
0274         echo "TESTERROR! invalid env1.log content!"
0275         go_out 3
0276 fi
0277 
0278 OUT=`grep ___ENV_OVER__2  env2.log`
0279 if [[ "X$OUT" == "X" ]]; then
0280         echo "TESTERROR! invalid env2.log content!"
0281         go_out 4
0282 fi
0283 
0284 # Test for automatic restarts...
0285 CNT=`wc testbin1-1.log | awk '{print $1}'`
0286 echo "Process restarts: $CNT"
0287 if [[ "$CNT" -lt "2" ]]; then 
0288         echo "TESTERROR! Automatic process reboots does not work!"
0289         go_out 5
0290 fi
0291 
0292 # Start some tons of processes
0293 #for i in `seq 1 $PROC_COUNT`; do
0294 for ((i=1;i<=100;i++)); do
0295         xadmin bc -t WHILE -s $i
0296 done
0297 
0298 sleep 60
0299 
0300 # have some sync (wait for startup to complete, print the output)
0301 xadmin pc
0302 
0303 CNT=`xadmin pc | grep "WHILE" | grep "running pid" | wc | awk '{print $1}'`
0304 echo "xadmin procs: $CNT"
0305 if [[ "$CNT" != "$PROC_COUNT" ]]; then 
0306         echo "TESTERROR! $PROC_COUNT procs not booted (according to xadmin pc)!"
0307         go_out 6
0308 fi
0309 
0310 echo "Processes in system: "
0311 $PSCMD
0312 
0313 #
0314 # Having some issues when bash is doing forks inside the test script -> whileproc.sh
0315 # Thus filter by cpmsrv pid in ps line...
0316 #
0317 CPM_PID=`xadmin ppm | grep cpmsrv | awk '{print $3}'`
0318 
0319 echo "CPM_PID=$CPM_PID"
0320 xadmin ps -a whileproc.sh
0321 CNT=`xadmin ps -a whileproc.sh | grep $CPM_PID | wc | awk '{print $1}' `
0322 echo "procs: $CNT"
0323 
0324 if [ "$CNT" -lt "$PROC_COUNT" ] || [ "$CNT" -gt "$PROC_COUNT_DIFFALLOW"  ]; then 
0325         echo "TESTERROR! $PROC_COUNT procs not booted (according to $PSCMD )!"
0326         go_out 7
0327 fi
0328 
0329 # test signle binary shutdown 
0330 xadmin sc -t IGNORE
0331 # We should have a binary which does not react on kill signals
0332 OUT=`grep SIGINT ignore.log`
0333 if [[ "X$OUT" == "X" ]]; then
0334         echo "WARNING! no SIGINT in ignore.log!"
0335 fi
0336 
0337 #boot it back for next test
0338 > ignore.log
0339 
0340 xadmin bc -t IGNORE
0341 sleep 10
0342 
0343 echo ">>> BEFORE STOP"
0344 date
0345 # Stop the cpmsrv (will do automatic process shutdown)...
0346 xadmin stop -s cpmsrv
0347 echo ">>> AFTER STOP"
0348 
0349 # let init to collect the childs, as cpmsrv on shutdown does kill -9 on
0350 # the processes, and it might not yet collected childs
0351 sleep 5
0352 
0353 # We should have 0 now
0354 CNT=`$PSCMD | grep whileproc.sh | grep -v grep | wc | awk '{print $1}'`
0355 echo "$PSCMD procs: $CNT"
0356 echo ">>> whileproc.sh listing start"
0357 $PSCMD | grep whileproc.sh | grep -v grep
0358 echo ">>> whileproc.sh listing end"
0359 if [[ "$CNT" -ne "0" ]]; then 
0360         echo "TESTERROR! not all whileproc.sh stopped!"
0361         go_out 8
0362 fi
0363 
0364 # We should have a binary which does not react on kill signals
0365 OUT=`grep SIGINT ignore.log`
0366 if [[ "X$OUT" == "X" ]]; then
0367         echo "WARNING! no SIGINT in ignore.log!"
0368 fi
0369 
0370 OUT=`grep SIGTERM ignore.log`
0371 if [[ "X$OUT" == "X" ]]; then
0372         echo "WARNING! no SIGTERM in ignore.log!"
0373 fi
0374 
0375 # process must be killed
0376 CNT=`$PSCMD | grep ignore.sh | grep -v grep | wc | awk '{print $1}'`
0377 echo "$PSCMD procs: $CNT"
0378 if [[ "$CNT" -ne "0" ]]; then 
0379         echo "TESTERROR! ignore.sh not stopped!"
0380         go_out 11
0381 fi
0382 
0383 go_out 0
0384 
0385 # vim: set ts=4 sw=4 et smartindent: