Back to home page

Enduro/X

 
 

    


0001 #!/bin/bash
0002 ##
0003 ## @brief @(#) Test086 - testing on MQ limitations
0004 ##
0005 ## @file run-dom.sh
0006 ##
0007 ## -----------------------------------------------------------------------------
0008 ## Enduro/X Middleware Platform for Distributed Transaction Processing
0009 ## Copyright (C) 2009-2016, ATR Baltic, Ltd. All Rights Reserved.
0010 ## Copyright (C) 2017-2023, Mavimax, Ltd. All Rights Reserved.
0011 ## This software is released under one of the following licenses:
0012 ## AGPL (with Java and Go exceptions) or Mavimax's license for commercial use.
0013 ## See LICENSE file for full text.
0014 ## -----------------------------------------------------------------------------
0015 ## AGPL license:
0016 ##
0017 ## This program is free software; you can redistribute it and/or modify it under
0018 ## the terms of the GNU Affero General Public License, version 3 as published
0019 ## by the Free Software Foundation;
0020 ##
0021 ## This program is distributed in the hope that it will be useful, but WITHOUT ANY
0022 ## WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
0023 ## PARTICULAR PURPOSE. See the GNU Affero General Public License, version 3
0024 ## for more details.
0025 ##
0026 ## You should have received a copy of the GNU Affero General Public License along 
0027 ## with this program; if not, write to the Free Software Foundation, Inc.,
0028 ## 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
0029 ##
0030 ## -----------------------------------------------------------------------------
0031 ## A commercial use license is available from Mavimax, Ltd
0032 ## contact@mavimax.com
0033 ## -----------------------------------------------------------------------------
0034 ##
0035 
0036 export TESTNO="086"
0037 export TESTNAME_SHORT="tmqlimit"
0038 export TESTNAME="test${TESTNO}_${TESTNAME_SHORT}"
0039 
0040 PWD=`pwd`
0041 if [ `echo $PWD | grep $TESTNAME ` ]; then
0042         # Do nothing 
0043         echo > /dev/null
0044 else
0045         # started from parent folder
0046         pushd .
0047         echo "Doing cd"
0048         cd $TESTNAME
0049 fi;
0050 
0051 . ../testenv.sh
0052 
0053 export TESTDIR="$NDRX_APPHOME/atmitest/$TESTNAME"
0054 export PATH=$PATH:$TESTDIR
0055 # Override timeout!
0056 export NDRX_TOUT=90
0057 export NDRX_LIBEXT="so"
0058 export NDRX_ULOG=$TESTDIR
0059 export NDRX_SILENT=Y
0060 
0061 # minimum performance enq/deq to proceed with test
0062 # where the absolute values are used
0063 TEST_PERF_MIN=70
0064 
0065 
0066 #
0067 # Domain 1 - here client will live
0068 #
0069 function set_dom1 {
0070     echo "Setting domain 1"
0071     . ../dom1.sh
0072     export NDRX_CONFIG=$TESTDIR/ndrxconfig-dom1.xml
0073     export NDRX_DMNLOG=$TESTDIR/ndrxd-dom1.log
0074     export NDRX_LOG=$TESTDIR/ndrx-dom1.log
0075     export NDRX_DEBUG_CONF=$TESTDIR/debug-dom1.conf
0076 
0077 # XA config, mandatory for TMQ:
0078     export NDRX_XA_RES_ID=1
0079     export NDRX_XA_OPEN_STR="datadir=./QSPACE1,qspace=MYSPACE"
0080     export NDRX_XA_CLOSE_STR=$NDRX_XA_OPEN_STR
0081 # Used from parent
0082     export NDRX_XA_DRIVERLIB=$NDRX_XA_DRIVERLIB_FILENAME
0083 
0084     export NDRX_XA_RMLIB=libndrxxaqdisk.so
0085 if [ "$(uname)" == "Darwin" ]; then
0086     export NDRX_XA_RMLIB=libndrxxaqdisk.dylib
0087     export NDRX_LIBEXT="dylib"
0088 fi
0089     export NDRX_XA_LAZY_INIT=0
0090 }
0091 
0092 #
0093 # Generic exit function
0094 #
0095 function go_out {
0096     echo "Test exiting with: $1"
0097     
0098     set_dom1;
0099     xadmin stop -y
0100     xadmin down -y
0101 
0102     # If some alive stuff left...
0103     xadmin killall atmiclt86
0104 
0105     popd 2>/dev/null
0106     exit $1
0107 }
0108 
0109 function clean_logs {
0110 
0111     # clean-up the logs for debbuging at the error.
0112     for f in `ls *.log`; do
0113          echo > $f
0114     done
0115 
0116 }
0117 
0118 UNAME=`uname`
0119 
0120 #
0121 # export the library path.
0122 #
0123 case $UNAME in
0124 
0125   Darwin)
0126     export NDRX_PLUGINS=libt86_lcf.dylib
0127     export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$TESTDIR
0128     ;;
0129 
0130   AIX)
0131     export NDRX_PLUGINS=libt86_lcf.so
0132     export LIBPATH=$LIBPATH:$TESTDIR
0133     ;;
0134 
0135   *)
0136     export NDRX_PLUGINS=libt86_lcf.so
0137     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TESTDIR
0138     ;;
0139 esac
0140 
0141 
0142 # Where to store TM logs
0143 rm -rf ./RM1
0144 mkdir RM1
0145 
0146 rm -rf ./RM2
0147 mkdir RM2
0148 
0149 # Where to store Q messages (QSPACE1)
0150 rm -rf ./QSPACE1
0151 mkdir QSPACE1
0152 
0153 set_dom1;
0154 # clean up anything left from prevoius tests...
0155 xadmin down -y
0156 # let ndrxd to finish
0157 sleep 2
0158 xadmin start -y || go_out 1
0159 
0160 # Run the client test...
0161 xadmin psc
0162 xadmin psvc
0163 xadmin ppm
0164 clean_logs;
0165 rm ULOG*
0166 
0167 if [[ "${NDRX_XA_DRIVERLIB_FILENAME}" == "libndrxxaqdisks"* ]]; then
0168 
0169 ################################################################################
0170 echo "Empty prep Q restart - completes OK (static only)"
0171 ################################################################################
0172 
0173     xadmin stop -y
0174     xadmin start -y
0175     xadmin lcf
0176 
0177     # slow respawn pls, so that tmq gets time to reboot and see the dummy files..
0178     xadmin appconfig restart_min 10
0179 
0180     (./atmiclt86 emptyq_commit_crash 2>&1) >> ./atmiclt-dom1.log
0181     RET=$?
0182     if [[ "X$RET" != "X0" ]]; then
0183         echo "emptyq_commit_restart failed"
0184         go_out $RET
0185     fi
0186 
0187     # prep Q folder must be empty...
0188 
0189     ls -l ./QSPACE1/prepared
0190 
0191     CNT=`ls -1 ./QSPACE1/prepared | wc | awk '{print $1}'`
0192 
0193     if [ "X$CNT" != "X0" ]; then
0194         echo "emptyq_commit_restart - empty transactions are not completed"
0195         go_out -1
0196     fi
0197 
0198 fi
0199 
0200 ################################################################################
0201 echo "Transaction timeouts"
0202 ################################################################################
0203 # use custom timeout
0204 export NDRX_TOUT=10
0205 xadmin stop -y
0206 xadmin start -y
0207 
0208 # Load message in each Q to finish up
0209 exbenchcl -n1 -P -t30 -b "{}" -f EX_DATA -S1 -QMYSPACE -sT_OK -E -R1
0210 RET=$?
0211 if [[ "X$RET" != "X0" ]]; then
0212     echo "exbenchcl failed T_OK load fail"
0213     go_out $RET
0214 fi
0215 
0216 exbenchcl -n1 -P -t30 -b "{}" -f EX_DATA -S1 -QMYSPACE -sT_NOK -E -R1
0217 RET=$?
0218 if [[ "X$RET" != "X0" ]]; then
0219     echo "exbenchcl failed T_NOK load fail"
0220     go_out $RET
0221 fi
0222 
0223 exbenchcl -n1 -P -t30 -b "{}" -f EX_DATA -S1 -QMYSPACE -sNS_OK -E -R1
0224 RET=$?
0225 if [[ "X$RET" != "X0" ]]; then
0226     echo "exbenchcl failed NS_OK load fail"
0227     go_out $RET
0228 fi
0229 
0230 exbenchcl -n1 -P -t30 -b "{}" -f EX_DATA -S1 -QMYSPACE -sND_OK -E -R1
0231 RET=$?
0232 if [[ "X$RET" != "X0" ]]; then
0233     echo "exbenchcl failed ND_OK load fail"
0234     go_out $RET
0235 fi
0236 
0237 exbenchcl -n1 -P -t30 -b "{}" -f EX_DATA -S1 -QMYSPACE -sND_NOK -E -R1
0238 RET=$?
0239 if [[ "X$RET" != "X0" ]]; then
0240     echo "exbenchcl failed ND_NOK load fail"
0241     go_out $RET
0242 fi
0243 
0244 echo "Wait 60, all tout Qs shall complete..."
0245 sleep 60
0246 xadmin mqlq
0247 xadmin psc
0248 
0249 STATS=`xadmin mqlq | grep "MYSPACE   T_OK          0     0     1     1     1     0"`
0250 echo "Stats: [$STATS]"
0251 if [ "X$STATS" == "X" ]; then
0252     echo "Invalid T_OK stats!"
0253     go_out -1
0254 fi
0255 
0256 STATS=`xadmin mqlq | grep "MYSPACE   T_NOK         0     0     1     1     0     1"`
0257 echo "Stats: [$STATS]"
0258 if [ "X$STATS" == "X" ]; then
0259     echo "Invalid T_NOK stats!"
0260     go_out -1
0261 fi
0262 
0263 STATS=`xadmin mqlq | grep "MYSPACE   NS_OK         0     0     1     1     1     0"`
0264 echo "Stats: [$STATS]"
0265 if [ "X$STATS" == "X" ]; then
0266     echo "Invalid NS_OK stats!"
0267     go_out -1
0268 fi
0269 
0270 STATS=`xadmin mqlq | grep "MYSPACE   ND_OK         0     0     1     1     1     0"`
0271 echo "Stats: [$STATS]"
0272 if [ "X$STATS" == "X" ]; then
0273     echo "Invalid ND_OK stats!"
0274     go_out -1
0275 fi
0276 
0277 STATS=`xadmin mqlq | grep "MYSPACE   ND_NOK        0     0     1     1     0     1"`
0278 echo "Stats: [$STATS]"
0279 if [ "X$STATS" == "X" ]; then
0280     echo "Invalid ND_NOK stats!"
0281     go_out -1
0282 fi
0283 
0284 clean_logs;
0285 rm ULOG*
0286 # restore tout:
0287 export NDRX_TOUT=90
0288 xadmin stop -y
0289 xadmin start -y
0290 
0291 ################################################################################
0292 echo "Forward thread wakup test"
0293 ################################################################################
0294 rm wakeup.out 2>/dev/null
0295 export NDRX_BENCH_FILE="wakeup.out"
0296 #
0297 # Check the system performance, if we can do 40 enq/deq sec, then continue with the
0298 # given test
0299 #
0300 exbenchcl -n1 -P -t30 -b "{}" -f EX_DATA -S1 -QMYSPACE -sTEST1
0301 
0302 NR_CALLS=`tail -1 wakeup.out  | cut -d ' '  -f3`
0303 # remember the performance indication
0304 # as on slow systems, say slower than 70 tps we will not be able to complete
0305 # test in time
0306 
0307 TEST_PERF=$NR_CALLS
0308 
0309 if [ "$TEST_PERF" -ge "$TEST_PERF_MIN" ]; then
0310 
0311     rm wakeup.out 2>/dev/null
0312     exbenchcl -n1 -P -t30 -b "{}" -f EX_DATA -S1 -QMYSPACE -sWAKEUP -E -I
0313 
0314     RET=$?
0315     if [[ "X$RET" != "X0" ]]; then
0316         echo "exbenchcl failed"
0317         go_out $RET
0318     fi
0319 
0320     # validate that there is more messages than say 65
0321     cat wakeup.out
0322 
0323     NR_CALLS=`tail -1 wakeup.out  | cut -d ' '  -f3`
0324 
0325     if [ "$NR_CALLS" -lt "50" ]; then
0326         echo "Expected more calls than 50 got $NR_CALLS"
0327         go_out -1
0328     fi
0329 
0330 else
0331     echo "Skipping thread wakup test as disk performance $TEST_PERF < $TEST_PERF_MIN"
0332 fi
0333 
0334 ################################################################################
0335 # Crash loop of autoq=T
0336 ################################################################################
0337 
0338 if [ `xadmin poller` != "emq" ]; then
0339     echo "Testing crashloop_t"
0340     # use custom timeout
0341     export NDRX_TOUT=30
0342     xadmin stop -y
0343     xadmin start -y
0344     (./atmiclt86 crashloop_t 2>&1) >> ./atmiclt-dom1.log
0345     RET=$?
0346     if [[ "X$RET" != "X0" ]]; then
0347         xadmin psc
0348         go_out $RET
0349     fi
0350 
0351     # print what's left in q...
0352     xadmin mqlq
0353     xadmin pt
0354 
0355     STATS=`xadmin mqlq | grep "ERROR         0     0"`
0356 
0357     echo "Stats: [$STATS]"
0358 
0359     if [[ "X$STATS" == "X" ]]; then
0360         echo "Expecting ERROR queue to be fully empty!"
0361         go_out -1
0362     fi
0363 
0364     # restore tout:
0365     export NDRX_TOUT=90
0366     xadmin stop -y
0367     xadmin start -y
0368 
0369     clean_logs;
0370     rm ULOG*
0371 fi
0372 
0373 
0374 ################################################################################
0375 echo "Validate transactional sequence..."
0376 ################################################################################
0377 
0378 (./atmiclt86 seqvalid 2>&1) >> ./atmiclt-dom1.log
0379 RET=$?
0380 if [[ "X$RET" != "X0" ]]; then
0381     echo "seqvalid failed"
0382     go_out $RET
0383 fi
0384 
0385 xadmin mqch -n1 -i 100 -qSEQVALID,autoq=T
0386 
0387 RET=$?
0388 if [[ "X$RET" != "X0" ]]; then
0389     echo "mqch failed"
0390     go_out $RET
0391 fi
0392 
0393 i=0
0394 # wait to complete...
0395 while [[ $i -lt 90 ]]
0396 do
0397     echo "Wait completion ($i)"
0398     sleep 1
0399 
0400     ((i++))
0401 
0402     xadmin mqlq | grep "SEQVALID      0     0    1K    1K    1K     0"
0403     STATS=`xadmin mqlq | grep "SEQVALID      0     0    1K    1K    1K     0"`
0404     if [ "X$STATS" != "X" ]; then
0405         break;
0406     fi
0407 done
0408 
0409 # validate the result finally.
0410 xadmin mqlq | grep SEQVALID
0411 STATS=`xadmin mqlq | grep "SEQVALID      0     0    1K    1K    1K     0"`
0412 echo "Stats: [$STATS]"
0413 if [ "X$STATS" == "X" ]; then
0414     echo "Expecting SEQVALID to be completed!"
0415     go_out -1
0416 fi
0417 
0418 ################################################################################
0419 echo "QoS test... last Q slow"
0420 ################################################################################
0421 
0422 # Load first fast.. (just to get forward order)
0423 exbenchcl -n1 -P -t9999 -b "{}" -f EX_DATA -S1 -QMYSPACE -sLASTOK -R5 -E
0424 
0425 RET=$?
0426 if [[ "X$RET" != "X0" ]]; then
0427     echo "Failed to load QoS messages..."
0428     go_out $RET
0429 fi
0430 
0431 # Load second slow...
0432 exbenchcl -n1 -P -t9999 -b "{}" -f EX_DATA -S1 -QMYSPACE -sLASTSLOW -R60 -E
0433 RET=$?
0434 if [[ "X$RET" != "X0" ]]; then
0435     echo "Failed to load QoS messages..."
0436     go_out $RET
0437 fi
0438 
0439 echo "Current situation:"
0440 xadmin mqlq
0441 
0442 # Load fast queue, fast load
0443 exbenchcl -n40 -P -t9999 -b "{}" -f EX_DATA -S1 -QMYSPACE -sLASTOK -R5 -E
0444 
0445 RET=$?
0446 if [[ "X$RET" != "X0" ]]; then
0447     echo "Failed to load QoS messages..."
0448     go_out $RET
0449 fi
0450 
0451 echo "Wait 20..."
0452 sleep 20
0453 
0454 echo "Current situation:"
0455 xadmin mqlq
0456 
0457 # Fast queue shall be done with 200 msgs...
0458 STATS=`xadmin mqlq | grep "LASTOK        0     0   205   205   205     0"`
0459 echo "Stats: [$STATS]"
0460 if [ "X$STATS" == "X" ]; then
0461     echo "Expecting LASTOK to be completed!"
0462     go_out -1
0463 fi
0464 
0465 echo "Wait 60 for slow to complete..."
0466 sleep 60
0467 xadmin mqlq
0468 STATS=`xadmin mqlq | grep "LASTSLOW      0     0    60    60    60     0"`
0469 echo "Stats: [$STATS]"
0470 if [ "X$STATS" == "X" ]; then
0471     echo "Expecting LASTSLOW to be completed!"
0472     go_out -1
0473 fi
0474 
0475 ################################################################################
0476 echo "TPETRAN join on timed out transaction ..."
0477 ################################################################################
0478 (NDRX_CCONFIG=${TESTDIR}/nulltm.ini NDRX_CCTAG=NULL ./atmiclt86 txtout 2>&1) >> ./atmiclt-dom1.log
0479 RET=$?
0480 if [[ "X$RET" != "X0" ]]; then
0481     xadmin psc
0482     go_out $RET
0483 fi
0484 
0485 ################################################################################
0486 echo "Massive global transaction load ..."
0487 ################################################################################
0488 
0489 # test cases checks for any possiblity of deadlock due to TM sending
0490 # completions to QSP, but QSP requests join to TM.
0491 
0492 #export NDRX_XA_FLAGS="FSYNC;DSYNC"
0493 # Do we need to fsync?
0494 #xadmin stop -y
0495 #xadmin start -y
0496 
0497 # Run the load...
0498 NDRX_CCONFIG=${TESTDIR}/nulltm.ini NDRX_CCTAG=NULL exbenchcl -n60 -P -t160 -b "{}" -f EX_DATA -S1024 -QMYSPACE -sEXB -N5 -T60
0499 
0500 RET=$?
0501 if [ "X$RET" != "X0" ]; then
0502     echo "Global transaction load test..."
0503     go_out $RET
0504 fi
0505 
0506 #unset NDRX_XA_FLAGS
0507 #xadmin stop -y
0508 #xadmin start -y
0509 
0510 ################################################################################
0511 echo "QoS test... (slow queue does not slow down all other Qs)"
0512 ################################################################################
0513 
0514 if [ "$TEST_PERF" -ge "$TEST_PERF_MIN" ]; then
0515 
0516 
0517 exbenchcl -n5 -P -t9999 -b "{}" -f EX_DATA -S1 -QMYSPACE -sQOS -R100 -E -N5
0518 
0519 RET=$?
0520 if [[ "X$RET" != "X0" ]]; then
0521     echo "Failed to load QoS messages..."
0522     go_out $RET
0523 fi
0524 
0525 # load some more to single q, so that not all Qs finish in the same time
0526 # which might introduce some issues with busy all/any flag
0527 exbenchcl -n1 -P -t9999 -b "{}" -f EX_DATA -S1 -QMYSPACE -sQOS003 -R100 -E
0528 
0529 RET=$?
0530 if [[ "X$RET" != "X0" ]]; then
0531     echo "Failed to load QoS messages..."
0532     go_out $RET
0533 fi
0534 
0535 # Check the queue stats... QOS000 is slow queue, shall be ready 200ms*100
0536 # all other queues shall be flushed immediatlly, say 10 sec should be sufficient
0537 # OK?
0538 
0539 # all shall complete within 10 sec, not?
0540 echo "Wait 30..."
0541 sleep 30
0542 xadmin mqlq
0543 
0544 
0545 STATS=`xadmin mqlq | grep "QOS000        0     0   100   100   100     0"`
0546 echo "Stats: [$STATS]"
0547 if [ "X$STATS" != "X" ]; then
0548     echo "Expecting QOS000 to be in-completed!"
0549     go_out -1
0550 fi
0551 
0552 STATS=`xadmin mqlq | grep "QOS001        0     0   100   100   100     0"`
0553 echo "Stats: [$STATS]"
0554 if [ "X$STATS" == "X" ]; then
0555     echo "Expecting QOS001 to be completed!"
0556     go_out -1
0557 fi
0558 
0559 STATS=`xadmin mqlq | grep "QOS002        0     0   100   100   100     0"`
0560 echo "Stats: [$STATS]"
0561 if [ "X$STATS" == "X" ]; then
0562     echo "Expecting QOS002 to be completed!"
0563     go_out -1
0564 fi
0565 
0566 STATS=`xadmin mqlq | grep "QOS003        0     0   200   200   200     0"`
0567 echo "Stats: [$STATS]"
0568 if [ "X$STATS" == "X" ]; then
0569     echo "Expecting QOS003 to be completed!"
0570     go_out -1
0571 fi
0572 
0573 STATS=`xadmin mqlq | grep "QOS004        0     0   100   100   100     0"`
0574 echo "Stats: [$STATS]"
0575 if [ "X$STATS" == "X" ]; then
0576     echo "Expecting QOS004 to be completed!"
0577     go_out -1
0578 fi
0579 
0580 
0581 # give some 30 sec total for other to complete... 
0582 echo "Wait 65..."
0583 sleep 65
0584 xadmin mqlq
0585 
0586 STATS=`xadmin mqlq | grep "QOS000        0     0   100   100   100     0"`
0587 echo "Stats: [$STATS]"
0588 if [ "X$STATS" == "X" ]; then
0589     echo "Expecting QOS000 to be completed!"
0590     go_out -1
0591 fi
0592 
0593 else
0594     echo "System too slow... skipping the test"
0595 fi
0596 
0597 ################################################################################
0598 echo "AutoQ performance test..."
0599 ################################################################################
0600 (./atmiclt86 autoperf 2>&1) >> ./atmiclt-dom1.log
0601 RET=$?
0602 if [[ "X$RET" != "X0" ]]; then
0603     xadmin psc
0604     go_out $RET
0605 fi
0606 
0607 ################################################################################
0608 echo "Testing rmrollback - rollback due to internal timeout / no activity"
0609 ################################################################################
0610 (./atmiclt86 rmrollback 2>&1) >> ./atmiclt-dom1.log
0611 RET=$?
0612 if [[ "X$RET" != "X0" ]]; then
0613     xadmin psc
0614     go_out $RET
0615 fi
0616 
0617 ################################################################################
0618 echo "Testing rmnorollback - no rollback, due to having slow, but actvity"
0619 ################################################################################
0620 (./atmiclt86 rmnorollback 2>&1) >> ./atmiclt-dom1.log
0621 RET=$?
0622 if [[ "X$RET" != "X0" ]]; then
0623     xadmin psc
0624     go_out $RET
0625 fi
0626 
0627 ################################################################################
0628 # Forward crash / commit fails, thus timeout rollback and when new tmsrv
0629 # is available, messages shall go normally to errorq and all msgs shall be
0630 # downloable
0631 ################################################################################
0632 
0633 echo "Testing fwdcrash"
0634 (./atmiclt86 fwdcrash 2>&1) >> ./atmiclt-dom1.log
0635 RET=$?
0636 if [[ "X$RET" != "X0" ]]; then
0637     xadmin psc
0638     go_out $RET
0639 fi
0640 
0641 ################################################################################
0642 # Test the fsync flags...
0643 ################################################################################
0644 
0645 export NDRX_XA_FLAGS="FSYNC;DSYNC"
0646 
0647 xadmin stop -y
0648 xadmin start -y
0649 
0650 echo "Testing FSYNC;DSYNC"
0651 (./atmiclt86 enqdeq 2>&1) >> ./atmiclt-dom1.log
0652 RET=$?
0653 if [[ "X$RET" != "X0" ]]; then
0654     go_out $RET
0655 fi
0656 
0657 export NDRX_XA_FLAGS="FDATASYNC;DSYNC"
0658 
0659 xadmin stop -y
0660 xadmin start -y
0661 
0662 echo "Testing FDATASYNC;DSYNC"
0663 (./atmiclt86 enqdeq 2>&1) >> ./atmiclt-dom1.log
0664 RET=$?
0665 if [[ "X$RET" != "X0" ]]; then
0666     go_out $RET
0667 fi
0668 
0669 unset NDRX_XA_FLAGS
0670 
0671 xadmin stop -y
0672 xadmin start -y
0673 
0674 ################################################################################
0675 
0676 ###################################################
0677 # for darwin /emq missing robost locks may stall the testing
0678 # thus it is not possible to test this func here
0679 ###################################################
0680 
0681 if [ `xadmin poller` != "emq" ]; then
0682     echo "Testing crashloop"
0683     # use custom timeout
0684     export NDRX_TOUT=30
0685     xadmin stop -y
0686     xadmin start -y
0687     (./atmiclt86 crashloop 2>&1) >> ./atmiclt-dom1.log
0688     RET=$?
0689     if [[ "X$RET" != "X0" ]]; then
0690         xadmin psc
0691         go_out $RET
0692     fi
0693 
0694     # print what's left in q...
0695     xadmin mqlq
0696     xadmin pt
0697 
0698     STATS=`xadmin mqlq | grep "ERROR         0     0"`
0699 
0700     echo "Stats: [$STATS]"
0701 
0702     if [[ "X$STATS" == "X" ]]; then
0703         echo "Expecting ERROR queue to be fully empty!"
0704         go_out -1
0705     fi
0706 
0707     # restore tout:
0708     export NDRX_TOUT=90
0709     xadmin stop -y
0710     xadmin start -y
0711 
0712     clean_logs;
0713     rm ULOG*
0714 fi
0715 ###################################################
0716 
0717 echo "Testing errorq"
0718 (./atmiclt86 errorq 2>&1) >> ./atmiclt-dom1.log
0719 RET=$?
0720 if [[ "X$RET" != "X0" ]]; then
0721     xadmin psc
0722     go_out $RET
0723 fi
0724 
0725 echo "Testing abortrules"
0726 (./atmiclt86 abortrules 2>&1) >> ./atmiclt-dom1.log
0727 RET=$?
0728 if [[ "X$RET" != "X0" ]]; then
0729     go_out $RET
0730 fi
0731 
0732 echo "Testing loadprep"
0733 (./atmiclt86 loadprep 2>&1) >> ./atmiclt-dom1.log
0734 RET=$?
0735 if [[ "X$RET" != "X0" ]]; then
0736     go_out $RET
0737 fi
0738 
0739 # after loadprep log folders shall be cleaned up...
0740 xadmin stop -y
0741 
0742 # Where to store TM logs
0743 rm -rf ./RM1
0744 mkdir RM1
0745 
0746 rm -rf ./RM2
0747 mkdir RM2
0748 
0749 # Where to store Q messages (QSPACE1)
0750 rm -rf ./QSPACE1
0751 mkdir QSPACE1
0752 
0753 xadmin start -y
0754 
0755 echo "Testing diskfull"
0756 clean_logs;
0757 xadmin help lcf 
0758 
0759 (./atmiclt86 diskfull 2>&1) >> ./atmiclt-dom1.log
0760 RET=$?
0761 if [[ "X$RET" != "X0" ]]; then
0762     go_out $RET
0763 fi
0764 
0765 echo "Testing tmsrvdiskerr"
0766 (./atmiclt86 tmsrvdiskerr 2>&1) >> ./atmiclt-dom1.log
0767 RET=$?
0768 if [[ "X$RET" != "X0" ]]; then
0769     go_out $RET
0770 fi
0771 
0772 echo "Testing commit_crash"
0773 (./atmiclt86 commit_crash 2>&1) >> ./atmiclt-dom1.log
0774 RET=$?
0775 if [[ "X$RET" != "X0" ]]; then
0776     go_out $RET
0777 fi
0778 
0779 echo "Testing badmsg"
0780 (./atmiclt86 badmsg 2>&1) >> ./atmiclt-dom1.log
0781 RET=$?
0782 if [[ "X$RET" != "X0" ]]; then
0783     go_out $RET
0784 fi
0785 
0786 echo "Testing tmqrestart"
0787 (./atmiclt86 tmqrestart 2>&1) >> ./atmiclt-dom1.log
0788 RET=$?
0789 if [[ "X$RET" != "X0" ]]; then
0790     go_out $RET
0791 fi
0792 
0793 echo "Testing tmsrvrestart"
0794 (./atmiclt86 tmsrvrestart 2>&1) >> ./atmiclt-dom1.log
0795 RET=$?
0796 if [[ "X$RET" != "X0" ]]; then
0797     go_out $RET
0798 fi
0799 
0800 echo "Testing commit_shut"
0801 (./atmiclt86 commit_shut 2>&1) >> ./atmiclt-dom1.log
0802 RET=$?
0803 if [[ "X$RET" != "X0" ]]; then
0804     go_out $RET
0805 fi
0806 
0807 echo "Testing deqwriteerr"
0808 (./atmiclt86  deqwriteerr 2>&1) >> ./atmiclt-dom1.log
0809 RET=$?
0810 if [[ "X$RET" != "X0" ]]; then
0811     go_out $RET
0812 fi
0813 
0814 echo "Cannot start as damaged prepared files... (manual resolve)"
0815 dd if=/dev/zero of=./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-010 count=1024 bs=1
0816 touch ./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-011
0817 touch ./QSPACE1/prepared/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-012
0818 
0819 xadmin stop -s tmqueue
0820 xadmin start -s tmqueue
0821 sleep 5
0822 
0823 if [[ ! -f ./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-010 ]]; then
0824     echo "./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-010 must not be removed!"
0825     go_out -1
0826 fi
0827 
0828 if [[ ! -f ./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-011 ]]; then
0829     echo "./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-011 must not be removed!"
0830     go_out -1
0831 fi
0832 
0833 if [[ ! -f ./QSPACE1/prepared/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-012 ]]; then
0834     echo "./QSPACE1/prepared/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-012 must not be removed!"
0835     go_out -1
0836 fi
0837 
0838 echo "Remove damaged prepare file, shall abort all ok"
0839 rm ./QSPACE1/prepared/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-012
0840 xadmin stop -s tmqueue
0841 xadmin start -s tmqueue
0842 sleep 5
0843 
0844 if [[ -f ./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-010 ]]; then
0845     echo "./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-010 must be removed!"
0846     go_out -1
0847 fi
0848 
0849 if [[ -f ./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-011 ]]; then
0850     echo "./QSPACE1/active/YZT3oUBAwBhirg3IRi2wqkSZp6IitwEAAQAy-011 must be removed!"
0851     go_out -1
0852 fi
0853 
0854 go_out $RET
0855 
0856 # vim: set ts=4 sw=4 et smartindent: