atmitest/test103_singlegrp/run.sh

0001 #!/bin/bash
0002 ##
0003 ## @brief Singleton group tests - test launcher
0004 ##
0005 ## @file run.sh
0006 ##
0007 ## -----------------------------------------------------------------------------
0008 ## Enduro/X Middleware Platform for Distributed Transaction Processing
0009 ## Copyright (C) 2009-2016, ATR Baltic, Ltd. All Rights Reserved.
0010 ## Copyright (C) 2017-2023, Mavimax, Ltd. All Rights Reserved.
0011 ## This software is released under one of the following licenses:
0012 ## AGPL (with Java and Go exceptions) or Mavimax's license for commercial use.
0013 ## See LICENSE file for full text.
0014 ## -----------------------------------------------------------------------------
0015 ## AGPL license:
0016 ##
0017 ## This program is free software; you can redistribute it and/or modify it under
0018 ## the terms of the GNU Affero General Public License, version 3 as published
0019 ## by the Free Software Foundation;
0020 ##
0021 ## This program is distributed in the hope that it will be useful, but WITHOUT ANY
0022 ## WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
0023 ## PARTICULAR PURPOSE. See the GNU Affero General Public License, version 3
0024 ## for more details.
0025 ##
0026 ## You should have received a copy of the GNU Affero General Public License along
0027 ## with this program; if not, write to the Free Software Foundation, Inc.,
0028 ## 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
0029 ##
0030 ## -----------------------------------------------------------------------------
0031 ## A commercial use license is available from Mavimax, Ltd
0032 ## contact@mavimax.com
0033 ## -----------------------------------------------------------------------------
0034 ##
0035
0036 export TESTNAME="test103_singlegrp"
0037
0038 PWD=`pwd`
0039 if [ `echo $PWD | grep $TESTNAME ` ]; then
0040     # Do nothing
0041     echo > /dev/null
0042 else
0043     # started from parent folder
0044     pushd .
0045     echo "Doing cd"
0046     cd $TESTNAME
0047 fi;
0048
0049 . ../testenv.sh
0050
0051 export TESTDIR="$NDRX_APPHOME/atmitest/$TESTNAME"
0052 export PATH=$PATH:$TESTDIR
0053 export NDRX_ULOG=$TESTDIR
0054 export NDRX_TOUT=10
0055 export NDRX_SILENT=Y
0056
0057 # This gives:
0058 # 3 sec check time for exsinglesv
0059 # 20 sec lock take over time
0060 export NDRX_SGREFRESH=10
0061
0062 export NDRX_TEST_CASE=0
0063
0064 #
0065 # Domain 1 - here client will live
0066 #
0067 set_dom1() {
0068     echo "Setting domain 1"
0069     . ../dom1.sh
0070
0071     if [ "$NDRX_TEST_CASE" == "1" ]; then
0072         export NDRX_CONFIG=$TESTDIR/ndrxconfig-dom1-niv.xml
0073     else
0074         export NDRX_CONFIG=$TESTDIR/ndrxconfig-dom1.xml
0075     fi
0076     export NDRX_CCONFIG=$TESTDIR/app-dom1.ini
0077     export NDRX_DMNLOG=$TESTDIR/ndrxd-dom1.log
0078     export NDRX_LOG=$TESTDIR/ndrx-dom1.log
0079     #export NDRX_DEBUG_CONF=$TESTDIR/debug-dom1.conf
0080 }
0081
0082
0083 #
0084 # Domain 2 - here server will live
0085 #
0086 set_dom2() {
0087     echo "Setting domain 2"
0088     . ../dom2.sh
0089
0090     if [ "$NDRX_TEST_CASE" == "1" ]; then
0091         export NDRX_CONFIG=$TESTDIR/ndrxconfig-dom2-niv.xml
0092     else
0093         export NDRX_CONFIG=$TESTDIR/ndrxconfig-dom2.xml
0094     fi
0095
0096     export NDRX_CCONFIG=$TESTDIR/app-dom2.ini
0097     export NDRX_DMNLOG=$TESTDIR/ndrxd-dom2.log
0098     export NDRX_LOG=$TESTDIR/ndrx-dom2.log
0099     #export NDRX_DEBUG_CONF=$TESTDIR/debug-dom2.conf
0100 }
0101
0102 #
0103 # Generic exit function
0104 #
0105 function go_out {
0106     echo "Test exiting with: $1"
0107
0108     set_dom1;
0109     xadmin stop -y
0110     xadmin down -y
0111
0112
0113     set_dom2;
0114     xadmin stop -y
0115     xadmin down -y
0116
0117     # If some alive stuff left...
0118     xadmin killall atmiclt103
0119
0120     popd 2>/dev/null
0121     exit $1
0122 }
0123
0124 function validate_OK1_lock_loss {
0125
0126     # have some debug:
0127     xadmin ppm
0128     xadmin psg
0129
0130     CNT=`xadmin ppm | grep atmi.sv1 | grep 'wait  runok' | wc | awk '{print $1}'`
0131     if [ "$CNT" -ne "11" ]; then
0132         echo "Expected 11 atmi.sv103 processes in wait state, got [$CNT] (after the lock lost)"
0133         go_out -1
0134     fi
0135
0136     # validate the xadmin pc output
0137     CMD="xadmin pc"
0138     echo "$CMD"
0139     OUT=`$CMD 2>&1`
0140
0141     PATTERN="TAG1\/SUBSECTION1 - waiting on group lock \(process group OK1 \(no 1\), .*
0142 TAG2\/SUBSECTION2 - waiting on group lock \(process group OK1 \(no 1\), .*
0143 TAG3\/- - running pid [0-9]+ .*"
0144
0145     echo "got output [$OUT]"
0146
0147     if ! [[ "$OUT" =~ $PATTERN ]]; then
0148         echo "Expected to wait on group lock by the clients."
0149         go_out -1
0150     fi
0151
0152 }
0153
0154 #
0155 # Test recover after lock loss
0156 #
0157 function validate_OK1_recovery {
0158
0159     # the lock is not returned immediately, but instead of 20 sec...
0160     echo "Wait 5 to check locked_wait"
0161     sleep 5
0162     xadmin ppm
0163     xadmin psg
0164
0165     # we shall still wait on lock, as exsignlesv was restarted
0166     CNT=`xadmin ppm | grep atmi.sv1 | grep 'wait  runok' | wc | awk '{print $1}'`
0167     if [ "$CNT" -ne "11" ]; then
0168         echo "Expected 11 atmi.sv103 processes in wait state, got [$CNT] (after the recovery lost)"
0169         go_out -1
0170     fi
0171
0172     echo "Wait 65 (lock regain + wait) check locked_wait + boot order booted all processes..."
0173     sleep 65
0174     xadmin ppm
0175     xadmin psg
0176
0177     CNT=`xadmin ppm | grep atmi.sv1 | grep 'runok runok' | wc | awk '{print $1}'`
0178     if [ "$CNT" -ne "11" ]; then
0179         echo "Expected 11 atmi.sv103 processes in runok state, got [$CNT] (after the lock lost)"
0180         go_out -1
0181     fi
0182
0183     # validate the xadmin pc output
0184     CMD="xadmin pc"
0185     echo "$CMD"
0186     OUT=`$CMD 2>&1`
0187
0188     PATTERN="TAG1\/SUBSECTION1 - running pid [0-9]+ .*
0189 TAG2\/SUBSECTION2 - running pid [0-9]+ .*
0190 TAG3\/- - running pid [0-9]+ .*"
0191
0192     echo "got output [$OUT]"
0193
0194     if ! [[ "$OUT" =~ $PATTERN ]]; then
0195         echo "Expected to wait on group lock by the clients."
0196         go_out -1
0197     fi
0198 }
0199
0200 # lets start with Domain 2. While the Domain 1 is in cold state...
0201
0202 #
0203 # first test is without sg_verify_nodes="Y"
0204 #
0205 for test_no in 0 1; do
0206
0207 ################################################################################
0208 echo ">>> TEST LOOP $test_no"
0209 ################################################################################
0210 NDRX_TEST_CASE=$test_no
0211
0212 # clean up some old stuff
0213 rm *.log 2>/dev/null
0214 rm lock_* 2>/dev/null
0215 rm ULOG* 2>/dev/null
0216 rm -f *.status 2>/dev/null
0217 xadmin killall tpbridge
0218
0219 set_dom1;
0220 xadmin down -y
0221 set |grep NDRX_CONFIG
0222
0223 set_dom2;
0224 xadmin down -y
0225 set |grep NDRX_CONFIG
0226
0227 ################################################################################
0228 echo ">>> Basic tests of no lock at the boot"
0229 ################################################################################
0230 CMD="xadmin start -y 2>&1"
0231 echo "$CMD"
0232 OUT=`$CMD 2>&1`
0233
0234 #
0235 # Validate that we wait on group lock. And group 2 starts OK
0236 #
0237 PATTERN="exec exsingleckl -k nZ22K8K7kewKo -i 15 -e .*\/atmitest\/test103_singlegrp\/exsingleckl-dom2.log -r --  :
0238 [[:space:]]process id=[0-9]+ ... Started.
0239 exec exsingleckr -k nZ22K8K7kewKo -i 20 -e .*\/atmitest\/test103_singlegrp\/exsingleckr-dom2.log -r --  :
0240 [[:space:]]process id=[0-9]+ ... Started.
0241 exec atmi.sv103 -k nZ22K8K7kewKo -i 50 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r -- -w10 --  :
0242 [[:space:]]process id=0 ... Waiting on group lock.
0243 exec atmi.sv103 -k nZ22K8K7kewKo -i 100 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0244 [[:space:]]process id=0 ... Waiting on group lock.
0245 exec atmi.sv103 -k nZ22K8K7kewKo -i 101 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0246 [[:space:]]process id=0 ... Waiting on group lock.
0247 exec atmi.sv103 -k nZ22K8K7kewKo -i 102 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0248 [[:space:]]process id=0 ... Waiting on group lock.
0249 exec atmi.sv103 -k nZ22K8K7kewKo -i 103 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0250 [[:space:]]process id=0 ... Waiting on group lock.
0251 exec atmi.sv103 -k nZ22K8K7kewKo -i 104 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0252 [[:space:]]process id=0 ... Waiting on group lock.
0253 exec atmi.sv103 -k nZ22K8K7kewKo -i 105 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0254 [[:space:]]process id=0 ... Waiting on group lock.
0255 exec atmi.sv103 -k nZ22K8K7kewKo -i 106 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0256 [[:space:]]process id=0 ... Waiting on group lock.
0257 exec atmi.sv103 -k nZ22K8K7kewKo -i 107 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0258 [[:space:]]process id=0 ... Waiting on group lock.
0259 exec atmi.sv103 -k nZ22K8K7kewKo -i 108 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0260 [[:space:]]process id=0 ... Waiting on group lock.
0261 exec atmi.sv103 -k nZ22K8K7kewKo -i 109 -e .*\/atmitest\/test103_singlegrp\/atmisv-dom2.log -r  --  :
0262 [[:space:]]process id=0 ... Waiting on group lock.
0263 exec tpbridge -k nZ22K8K7kewKo -i 2300 -e .*\/atmitest\/test103_singlegrp\/bridge-dom2.log -r -- -f -n1 -r -i 0.0.0.0 -p 20003 -tP -z30 -P0 :
0264 [[:space:]]process id=[0-9]+ ... Started.
0265 exec exsinglesv -k nZ22K8K7kewKo -i 3000 -e .*\/atmitest\/test103_singlegrp\/exsinglesv2-dom2.log -r --  :
0266 [[:space:]]process id=[0-9]+ ... Started.
0267 exec exsingleckl -k nZ22K8K7kewKo -i 3005 -e .*\/atmitest\/test103_singlegrp\/exsingleckl2-dom2.log -r --  :
0268 [[:space:]]process id=[0-9]+ ... Started.
0269 exec exsingleckr -k nZ22K8K7kewKo -i 3010 -e .*\/atmitest\/test103_singlegrp\/exsingleckr2-dom2.log -r --  :
0270 [[:space:]]process id=[0-9]+ ... Started.
0271 exec atmi103_v2 -k nZ22K8K7kewKo -i 4000 -e .*\/atmitest\/test103_singlegrp\/atmi103_v2-dom2.log -r  --  :
0272 [[:space:]]process id=[0-9]+ ... Started.
0273 exec atmi103_v2 -k nZ22K8K7kewKo -i 4001 -e .*\/atmitest\/test103_singlegrp\/atmi103_v2-dom2.log -r  --  :
0274 [[:space:]]process id=[0-9]+ ... Started.
0275 exec atmi103_v2 -k nZ22K8K7kewKo -i 4002 -e .*\/atmitest\/test103_singlegrp\/atmi103_v2-dom2.log -r  --  :
0276 [[:space:]]process id=[0-9]+ ... Started.
0277 exec atmi103_v2 -k nZ22K8K7kewKo -i 4003 -e .*\/atmitest\/test103_singlegrp\/atmi103_v2-dom2.log -r  --  :
0278 [[:space:]]process id=[0-9]+ ... Started.
0279 exec atmi103_v2 -k nZ22K8K7kewKo -i 4004 -e .*\/atmitest\/test103_singlegrp\/atmi103_v2-dom2.log -r  --  :
0280 [[:space:]]process id=[0-9]+ ... Started.
0281 exec cpmsrv -k nZ22K8K7kewKo -i 9999 -e .*\/atmitest\/test103_singlegrp\/cpmsrv-dom2.log -r -- -k3 -i1 --  :
0282 [[:space:]]process id=[0-9]+ ... Started.
0283 Startup finished. 8 processes started."
0284
0285 echo "got output [$OUT]"
0286
0287 #if ! [[ "$OUT" =~ $PATTERN ]]; then
0288 #    echo "Unexpected output from xadmin start -y"
0289 #    go_out -1
0290 #fi
0291
0292 date
0293 xadmin psc
0294 date
0295 xadmin ppm
0296
0297 date
0298 ################################################################################
0299 echo ">>> Basic tests of singleton groups -> group start, check order"
0300 ################################################################################
0301
0302 date
0303 xadmin psg -a
0304 date
0305 # this might race with respawn checks and we might get all booted:
0306 # this above shall be documented.
0307 # however, wolloing might let to bypass the tests:
0308 xadmin start -i  10 &
0309 date
0310 # Let processes to lock and boot
0311 sleep 7
0312
0313 date
0314 CMD="xadmin ppm"
0315 echo "$CMD"
0316 OUT=`$CMD 2>&1`
0317
0318 date
0319 echo "got output [$OUT]"
0320
0321 date
0322 # which this is ?
0323 CNT=`xadmin ppm | grep atmi.sv1 | grep 'start runok' | wc | awk '{print $1}'`
0324 if [ "$CNT" -ne "1" ]; then
0325     echo "Expected 1 in start state, got [$CNT]"
0326     go_out -1
0327 fi
0328
0329 CNT=`xadmin ppm | grep atmi.sv1 | grep 'wait  runok' | wc | awk '{print $1}'`
0330 if [ "$CNT" -ne "10" ]; then
0331     echo "Expected 10 atmi.sv103 processes in wait state, got [$CNT]"
0332     go_out -1
0333 fi
0334
0335 xadmin ps -a atmi.sv103
0336 CNT=`xadmin ps -a atmi.sv103 -b nZ22K8K7kewKo | wc | awk '{print $1}'`
0337 if [ "$CNT" -ne "1" ]; then
0338     echo "Expected 1 atmi.sv103 running, got [$CNT]"
0339     go_out -1
0340 fi
0341
0342 # check the bootlock flag
0343 if [ ! -f ${TESTDIR}/exec_on_bootlocked.status ]; then
0344     echo "Expected exec_on_bootlocked.status file"
0345     go_out -1
0346 fi
0347
0348 if [ -f ${TESTDIR}/exec_on_locked.status ]; then
0349     echo "Not expected exec_on_locked.status file"
0350     go_out -1
0351 fi
0352
0353 # Ensure that PSG is locked but server & client are not booted
0354 CMD="xadmin psg"
0355 echo "$CMD"
0356 OUT=`$CMD 2>&1`
0357
0358 PATTERN="SGID LCKD MMON SBOOT CBOOT LPSRVID    LPPID LPPROCNM          REFRESH RSN FLAGS
0359 ---- ---- ---- ----- ----- ------- -------- ---------------- -------- --- -----
0360    1 Y    N    N     N          10[[:space:]]+[0-9]+ exsinglesv[[:space:]]+.*   0 i"
0361
0362 echo "got output [$OUT]"
0363
0364 if ! [[ "$OUT" =~ $PATTERN ]]; then
0365     echo "Expected group to be locked, but not client/servers booted"
0366     go_out -1
0367 fi
0368
0369 CMD="xadmin pc"
0370 echo "$CMD"
0371 OUT=`$CMD 2>&1`
0372
0373 PATTERN="TAG1/SUBSECTION1 - waiting on group lock \(process group OK1 \(no 1\), .*
0374 TAG2/SUBSECTION2 - waiting on group lock \(process group OK1 \(no 1\), .*
0375 TAG3/- - running pid [0-9]+ .*"
0376
0377 echo "got output [$OUT]"
0378
0379 if ! [[ "$OUT" =~ $PATTERN ]]; then
0380     echo "Expected to wait on group lock by the clients."
0381     go_out -1
0382 fi
0383
0384 echo ">>> wait 45 for full boot..."
0385 sleep 45
0386 xadmin ppm
0387
0388 CNT=`xadmin ppm | grep atmi.sv1 | grep 'runok runok' | wc | awk '{print $1}'`
0389 if [ "$CNT" -ne "11" ]; then
0390     echo "Expected 11 atmi.sv103 processes in start state, got [$CNT]"
0391     go_out -1
0392 fi
0393
0394 xadmin ps -a atmi.sv103
0395 CNT=`xadmin ps -a atmi.sv103 -b nZ22K8K7kewKo | wc | awk '{print $1}'`
0396 if [ "$CNT" -ne "11" ]; then
0397     echo "Expected 11 atmi.sv103 running, got [$CNT]"
0398     go_out -1
0399 fi
0400
0401 # Ensure that group is fully booted.
0402 CMD="xadmin psg"
0403 echo "$CMD"
0404 OUT=`$CMD 2>&1`
0405
0406 PATTERN="SGID LCKD MMON SBOOT CBOOT LPSRVID    LPPID LPPROCNM          REFRESH RSN FLAGS
0407 ---- ---- ---- ----- ----- ------- -------- ---------------- -------- --- -----
0408    1 Y    N    Y     Y          10[[:space:]]+[0-9]+ exsinglesv[[:space:]]+.*   0 i"
0409
0410 echo "got output [$OUT]"
0411
0412 if ! [[ "$OUT" =~ $PATTERN ]]; then
0413     echo "Expected group to be locked, but not client/servers booted"
0414     go_out -1
0415 fi
0416
0417 CMD="xadmin pc"
0418 echo "$CMD"
0419 OUT=`$CMD 2>&1`
0420
0421 PATTERN="TAG1/SUBSECTION1 - running pid .*
0422 TAG2/SUBSECTION2 - running pid .*"
0423
0424 echo "got output [$OUT]"
0425
0426 if ! [[ "$OUT" =~ $PATTERN ]]; then
0427     echo "Expected TAG1/SUBSECTION1 and TAG2/SUBSECTION2 to be running"
0428     go_out -1
0429 fi
0430
0431 ################################################################################
0432 echo ">>> Lock loss: ping failure"
0433 ################################################################################
0434
0435 atmiclt103 lock_file ${TESTDIR}/lock_OK1_2 &
0436
0437 # the exsinglesv interval is 3 sec, so ping shall detect that it cannot lock
0438 # anymore, and group will be unlocked and processes would get killed
0439 # and would result in waiting for lock again
0440 # ----- V2 -----
0441 # we get the write lock WAIT on ping file.
0442 # thus ndrxd shall detect that refresh has not happended in time
0443 # and unlock the group / kill binaryies
0444 sleep 15
0445 validate_OK1_lock_loss;
0446 xadmin killall atmiclt103
0447 validate_OK1_recovery;
0448
0449 if [ ! -f ${TESTDIR}/exec_on_locked.status ]; then
0450     echo "Expected exec_on_locked.status file"
0451     go_out -1
0452 fi
0453
0454 ################################################################################
0455 echo ">>> Lock loss: due to exsinglesv crash"
0456 ################################################################################
0457
0458 LOCK_PID=`xadmin ps -a "exsinglesv -k nZ22K8K7kewKo -i 10" -p`
0459 kill -9 $LOCK_PID
0460
0461 # let ndrxd to collect the fact
0462 # on freebsd for some reason at points it slow...
0463 sleep 8
0464 validate_OK1_lock_loss;
0465 validate_OK1_recovery;
0466
0467 ################################################################################
0468 echo ">>> Lock loss: exsinglesv freeze (lock loss)"
0469 ################################################################################
0470
0471 LOCK_PID=`xadmin ps -a "exsinglesv -k nZ22K8K7kewKo -i 10" -p`
0472 kill -SIGSTOP $LOCK_PID
0473 # wait for loss detect:
0474 # -> group timeout => 10 sec
0475 # -> killall processes -> 1sec
0476 # -> try boot processes -> 1 sec
0477 # -> + 2 sec buffer
0478 sleep 17
0479 validate_OK1_lock_loss;
0480 kill -SIGCONT $LOCK_PID
0481 validate_OK1_recovery;
0482
0483 ################################################################################
0484 echo ">>> Node 1 boot -> groups not locked"
0485 ################################################################################
0486
0487 set_dom1;
0488 xadmin down -y
0489 xadmin start -y || go_out 1
0490 xadmin ppm
0491 xadmin pc
0492 xadmin psg
0493
0494 # let CPM to cycle on...
0495 sleep 5
0496
0497 # verify that no group is booted...
0498 CNT=`xadmin ppm | grep atmi | grep 'wait  runok' | wc | awk '{print $1}'`
0499 if [ "$CNT" -ne "16" ]; then
0500     echo "Expected 16 atmi.sv103/atmi103_v2 processes in wait state, got [$CNT]"
0501     go_out -1
0502 fi
0503
0504 CNT=`xadmin pc | grep 'waiting on group lock' | wc | awk '{print $1}'`
0505 if [ "$CNT" -ne "3" ]; then
0506     echo "Expected 3 clients in waiting state, got [$CNT]"
0507     go_out -1
0508 fi
0509
0510 ################################################################################
0511 echo ">>> Test maintenance mode (no lock takover)"
0512 ################################################################################
0513
0514 xadmin mmon
0515
0516 # node2 goes down, first does not take over as in maintenace mode...
0517 set_dom2;
0518 xadmin stop -y
0519
0520 set_dom1;
0521 # let exsinglesv to cycle..
0522 sleep 5
0523
0524 xadmin ppm
0525 xadmin pc
0526 xadmin psg
0527
0528 CNT=`xadmin ppm | grep atmi | grep 'wait  runok' | wc | awk '{print $1}'`
0529 if [ "$CNT" -ne "16" ]; then
0530     echo "Expected 16 atmi.sv103/atmi103_v2 processes in wait state, got [$CNT]"
0531     go_out -1
0532 fi
0533
0534 CNT=`xadmin pc | grep 'waiting on group lock' | wc | awk '{print $1}'`
0535 if [ "$CNT" -ne "3" ]; then
0536     echo "Expected 3 clients in waiting state, got [$CNT]"
0537     go_out -1
0538 fi
0539
0540 # release maintenance mode
0541 xadmin mmoff
0542 ################################################################################
0543 echo ">>> Node 2 shutdown: exsinglesv waits for reporting lock"
0544 ################################################################################
0545
0546 echo "sleep 5, check that we do not report the lock yet"
0547 sleep 5
0548 xadmin ppm
0549 xadmin pc
0550 xadmin psg
0551
0552 CNT=`xadmin ppm | grep atmi. | grep 'wait  runok' | wc | awk '{print $1}'`
0553 if [ "$CNT" -ne "16" ]; then
0554     echo "Expected 16 atmi.sv103/atmi103_v2 processes in wait state, got [$CNT]"
0555     go_out -1
0556 fi
0557
0558 CNT=`xadmin pc | grep 'waiting on group lock' | wc | awk '{print $1}'`
0559 if [ "$CNT" -ne "3" ]; then
0560     echo "Expected 3 clients in waiting state, got [$CNT]"
0561     go_out -1
0562 fi
0563
0564 ################################################################################
0565 echo ">>> Node 2 shutdown: no-order group wait long wait server booted fully"
0566 ################################################################################
0567
0568 # wait for locks to establish
0569 # seems needs longer time for AIX...
0570 # however having issues with boot order checks...
0571 sleep 35
0572
0573 # all process shall be running, expect that long booting, shall be still in start
0574 # state
0575
0576 xadmin ppm
0577 xadmin psg
0578 xadmin pc
0579
0580 CNT=`xadmin ppm | grep atmi.sv1 | grep 'runok runok' | wc | awk '{print $1}'`
0581 if [ "$CNT" -ne "10" ]; then
0582     echo "Expected 10 atmi.sv103 processes in runok state, got [$CNT]"
0583     go_out -1
0584 fi
0585
0586 CNT=`xadmin ppm | grep atmi.sv1 | grep 'start runok' | wc | awk '{print $1}'`
0587 if [ "$CNT" -ne "1" ]; then
0588     echo "Expected 1 atmi.sv103 processes in start state, got [$CNT]"
0589     go_out -1
0590 fi
0591
0592 CNT=`xadmin pc | grep 'running pid' | wc | awk '{print $1}'`
0593 if [ "$CNT" -ne "3" ]; then
0594     echo "Expected 3 clients in running state, got [$CNT]"
0595     go_out -1
0596 fi
0597
0598 # Ensure that PSG is locked but server & client are not booted
0599 CMD="xadmin psg"
0600 echo "$CMD"
0601 OUT=`$CMD 2>&1`
0602
0603 flag="ni"
0604 if [ "$test_no" == "1" ]; then
0605     flag="niv"
0606 fi
0607
0608 PATTERN="SGID LCKD MMON SBOOT CBOOT LPSRVID    LPPID LPPROCNM          REFRESH RSN FLAGS
0609 ---- ---- ---- ----- ----- ------- -------- ---------------- -------- --- -----
0610    1 Y    N    Y     Y          10[[:space:]]+[0-9]+ exsinglesv[[:space:]]+.*   0 $flag[[:space:]]*
0611    2 Y    N    Y     Y        3000[[:space:]]+[0-9]+ exsinglesv[[:space:]]+.*   0 $flag[[:space:]]*"
0612
0613 echo "got output [$OUT]"
0614
0615 if ! [[ "$OUT" =~ $PATTERN ]]; then
0616     echo "Expected both groups locked..."
0617     go_out -1
0618 fi
0619
0620
0621 ################################################################################
0622 echo ">>> bootlock script not working..."
0623 ################################################################################
0624
0625 chmod 000 $TESTDIR/exec_on_bootlocked.status
0626 chmod 000 $TESTDIR/exec_on_locked.status
0627
0628 xadmin stop -y
0629 xadmin start -y
0630
0631 # let cpmsrv to gather stats..
0632 sleep 5
0633
0634 # let the lock to fail...
0635 xadmin ppm
0636 xadmin psg
0637 xadmin pc
0638
0639 CNT=`xadmin ppm | grep atmi.sv1 | grep 'wait  runok' | wc | awk '{print $1}'`
0640 if [ "$CNT" -ne "11" ]; then
0641     echo "Expected 11 atmi.sv103 processes in wait state, got [$CNT]"
0642     go_out -1
0643 fi
0644
0645 # wait by cpmsrv too...
0646 CNT=`xadmin pc | grep 'waiting on group lock' | wc | awk '{print $1}'`
0647 if [ "$CNT" -ne "2" ]; then
0648     echo "Expected 2 clients in waiting state, got [$CNT]"
0649     go_out -1
0650 fi
0651
0652 # we are done!
0653
0654 RET=0
0655
0656 if [[ "X$RET" != "X0" ]]; then
0657     go_out $RET
0658 fi
0659
0660 # Catch is there is test error!!!
0661 if [ "X`grep TESTERROR *.log`" != "X" ]; then
0662     echo "Test error detected!"
0663     RET=-2
0664 fi
0665
0666 # master loop of the test case
0667 done
0668
0669 go_out $RET
0670
0671 # vim: set ts=4 sw=4 et smartindent:
0672