Back to home page

Enduro/X
 
 

     

0001 /**
0002  * @brief Build process model structure
0003  *
0004  * @file pmodel.c
0005  */
0006 /* -----------------------------------------------------------------------------
0007  * Enduro/X Middleware Platform for Distributed Transaction Processing
0008  * Copyright (C) 2009-2016, ATR Baltic, Ltd. All Rights Reserved.
0009  * Copyright (C) 2017-2023, Mavimax, Ltd. All Rights Reserved.
0010  * This software is released under one of the following licenses:
0011  * AGPL (with Java and Go exceptions) or Mavimax's license for commercial use.
0012  * See LICENSE file for full text.
0013  * -----------------------------------------------------------------------------
0014  * AGPL license:
0015  *
0016  * This program is free software; you can redistribute it and/or modify it under
0017  * the terms of the GNU Affero General Public License, version 3 as published
0018  * by the Free Software Foundation;
0019  *
0020  * This program is distributed in the hope that it will be useful, but WITHOUT ANY
0021  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
0022  * PARTICULAR PURPOSE. See the GNU Affero General Public License, version 3
0023  * for more details.
0024  *
0025  * You should have received a copy of the GNU Affero General Public License along 
0026  * with this program; if not, write to the Free Software Foundation, Inc.,
0027  * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
0028  *
0029  * -----------------------------------------------------------------------------
0030  * A commercial use license is available from Mavimax, Ltd
0031  * contact@mavimax.com
0032  * -----------------------------------------------------------------------------
0033  */
0034 #include <ndrx_config.h>
0035 #include <string.h>
0036 #include <stdio.h>
0037 #include <stdlib.h>
0038 #include <memory.h>
0039 #include <utlist.h>
0040 #include <errno.h>
0041 #include <sys/resource.h>
0042 #include <sys/wait.h>
0043 #include <unistd.h>
0044 #include <signal.h>
0045 #include <libgen.h>
0046 #include <fcntl.h>
0047 
0048 #include <ndrstandard.h>
0049 #include <ndebug.h>
0050 #include <ndrxd.h>
0051 #include <ndrxdcmn.h>
0052 #include <userlog.h>
0053 
0054 #include <nstopwatch.h>
0055 #include <cmd_processor.h>
0056 #include <pthread.h>
0057 #include <nstdutil.h>
0058 #include <bridge_int.h>
0059 #include <atmi_shm.h>
0060 #include <sys_unix.h>
0061 #include <libndrxconf.h>
0062 #include <ndrxdiag.h>
0063 #include <lcfint.h>
0064 #include <singlegrp.h>
0065 
0066 /*---------------------------Externs------------------------------------*/
0067 /*---------------------------Macros-------------------------------------*/
0068 
0069 /** Step to realloc the CMD line */
0070 #define REALLOC_CMD_STEP    10
0071 #define REALLOC_CMD alloc_args+=REALLOC_CMD_STEP; \
0072     if (NULL==cmd) \
0073             cmd = NDRX_MALLOC(sizeof(char *)*alloc_args); \
0074     else \
0075             cmd = NDRX_REALLOC(cmd, sizeof(char *)*alloc_args); \
0076         if (NULL==cmd) \
0077         {\
0078             int err = errno;\
0079             fprintf(stderr, "%s: failed to realloc %ld bytes: %s\n", __func__, \
0080                 (long)sizeof(char *)*alloc_args, strerror(err));\
0081             userlog("%s: failed to realloc %ld bytes: %s\n", __func__, \
0082                 (long)sizeof(char *)*alloc_args, strerror(err));\
0083             exit(1);\
0084         }
0085 
0086 #define NDRX_PM_SET_ENV(ENV__, VAL__) if (EXSUCCEED!=setenv(ENV__, VAL__, EXTRUE))\
0087         {\
0088            int err = errno;\
0089             userlog("%s: failed to set %s=[%s]: %s", __func__, \
0090                 ENV__, VAL__, strerror(err));\
0091             exit(1);\
0092         }
0093 
0094 /**
0095  * Support #459 
0096  * locked debug use signal-thread
0097  * Fact is if we forking, we might get lock on localtime_r
0098  * by signal thread, and the forked process will try to debug
0099  * but will be unable because there will be left over lock on localtime_r
0100  * from the signal thread which no more exists after the fork.
0101  */
0102 #define LOCKED_DEBUG(lev, fmt, ...) MUTEX_LOCK_V(M_forklock);\
0103                                     NDRX_LOG(lev, fmt, ##__VA_ARGS__);\
0104                                     MUTEX_UNLOCK_V(M_forklock);
0105 
0106 /*---------------------------Enums--------------------------------------*/
0107 /*---------------------------Typedefs-----------------------------------*/
0108 /*---------------------------Globals------------------------------------*/
0109 exprivate pthread_t M_signal_thread; /* Signalled thread */
0110 exprivate int M_signal_thread_set = EXFALSE; /* Signal thread is set */
0111 exprivate volatile int M_shutdown = EXFALSE; /**< Doing shutdown    */
0112 
0113 exprivate MUTEX_LOCKDECL(M_forklock);       /**< forking lock, no q ops during fork!  */
0114 
0115 /*---------------------------Statics------------------------------------*/
0116 /*---------------------------Prototypes---------------------------------*/
0117 
0118 /**
0119  * Initiate process reload
0120  * @return
0121  */
0122 expublic int self_sreload(pm_node_t *p_pm)
0123 {
0124     int ret=EXSUCCEED;
0125     command_startstop_t call;
0126     
0127     memset(&call, 0, sizeof(call));
0128     
0129     call.srvid = EXFAIL;
0130     NDRX_STRCPY_SAFE(call.binary_name, p_pm->binary_name);
0131     
0132     NDRX_LOG(log_debug, "Sending sreload command that [%s] must be reloaded, rq [%s]",
0133             call.binary_name, call.call.reply_queue);
0134     
0135     /* 
0136      * Send the notification that process needs to be reloaded
0137      */
0138     ret=cmd_generic_callfl(NDRXD_COM_SRELOADI_RQ, NDRXD_SRC_NDRXD,
0139                         NDRXD_CALL_TYPE_GENERIC,
0140                         (command_call_t *)&call, sizeof(call),
0141                         G_command_state.listenq_str,
0142                         (mqd_t)EXFAIL,
0143                         (mqd_t)EXFAIL,
0144                         G_command_state.listenq_str,
0145                         0, 
0146                         NULL,
0147                         NULL,
0148                         NULL,
0149                         NULL,
0150                         EXFALSE, TPNOBLOCK);
0151     
0152 out:
0153     return ret;
0154 }
0155 
0156 
0157 /**
0158  * Report to ndrxd, that process had sigchld
0159  * @return
0160  */
0161 expublic int self_notify(srv_status_t *status, int block)
0162 {
0163     int ret=EXSUCCEED;
0164     size_t  send_size = sizeof(srv_status_t);
0165 
0166     LOCKED_DEBUG(log_debug, "About to send: %d bytes/%d svcs",
0167                         send_size, status->svc_count);
0168     
0169     /* we want new q/open + close here,
0170      * so that we do not interference with our main queue blocked/non blocked flags.
0171      */
0172     MUTEX_LOCK_V(M_forklock);
0173     ret=cmd_generic_callfl(NDRXD_COM_PMNTIFY_RQ, NDRXD_SRC_NDRXD,
0174                         NDRXD_CALL_TYPE_PM_INFO,
0175                         (command_call_t *)status, send_size,
0176                         G_command_state.listenq_str,
0177                         (mqd_t)EXFAIL,
0178                         (mqd_t)EXFAIL,
0179                         G_command_state.listenq_str,
0180                         0, 
0181                         NULL,
0182                         NULL,
0183                         NULL,
0184                         NULL,
0185                         EXFALSE, TPNOBLOCK);
0186     MUTEX_UNLOCK_V(M_forklock);
0187     
0188 out:
0189     return ret;
0190 }
0191 
0192 /**
0193  * Handle the dead child detected either by sigwait or signal.
0194  * @param chldpid
0195  * @param stat_loc
0196  */
0197 exprivate void handle_child(pid_t chldpid, int stat_loc)
0198 {
0199     srv_status_t status;
0200     memset(&status, 0, sizeof(status));
0201     
0202     if (chldpid>0)
0203     {
0204         LOCKED_DEBUG(log_warn, "sigchld: PID: %d exit status: %d",
0205                                            chldpid, stat_loc);
0206         if (WIFSTOPPED(stat_loc))
0207         {
0208             LOCKED_DEBUG(log_warn, "Process is stopped - ignore..");
0209             return;
0210         }
0211         status.srvinfo.pid = chldpid;
0212         
0213         if (WIFEXITED(stat_loc) && 0==WEXITSTATUS(stat_loc))
0214         {
0215             LOCKED_DEBUG(log_error, "Process normal shutdown!");
0216             status.srvinfo.state = NDRXD_PM_EXIT;
0217         }
0218         else if (WIFEXITED(stat_loc) && TPEXIT_ENOENT == WEXITSTATUS(stat_loc))
0219         {
0220             LOCKED_DEBUG(log_error, "Binary not found!");
0221             status.srvinfo.state = NDRXD_PM_ENOENT;
0222         }
0223         else
0224         {
0225             LOCKED_DEBUG(log_error, "Process abnormal shutdown!");
0226             status.srvinfo.state = NDRXD_PM_DIED;
0227         }
0228         /* NDRX_LOG(log_warn, "Sending notification"); */
0229         self_notify(&status, EXFALSE);
0230     }
0231 }
0232 
0233 /**
0234  * Checks for child exit.
0235  * We will let mainthread to do all internal struct related work!
0236  * @return Got child exit
0237  */
0238 exprivate void * check_child_exit(void *arg)
0239 {
0240     pid_t chldpid;
0241     int stat_loc;
0242     sigset_t blockMask;
0243     int sig;
0244     struct rusage rusage;
0245     int old;
0246     int ret;
0247     
0248     pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, &old);
0249     
0250     sigemptyset(&blockMask);
0251     sigaddset(&blockMask, SIGCHLD);
0252     
0253     /* seems like needed for osx */
0254 
0255     LOCKED_DEBUG(log_debug, "check_child_exit - enter...");
0256     while (!M_shutdown)
0257     {
0258         
0259 /* seems not working on darwin ... thus just wait for pid.
0260  * if we do not have any childs, then sleep for 1 sec.
0261  */
0262 #if !(EX_OS_DARWIN && EX_LSB_RELEASE_VER_MAJOR < 23)
0263         LOCKED_DEBUG(log_debug, "about to sigwait()");
0264         
0265         /* Wait for notification signal */
0266         ret=sigwait(&blockMask, &sig);
0267         
0268         if (EXSUCCEED!=ret)
0269         {
0270             LOCKED_DEBUG(log_warn, "sigwait failed:(%s)", strerror(errno));
0271         }        
0272 #endif
0273         
0274         if (M_shutdown)
0275         {
0276             break;
0277         }
0278         
0279         LOCKED_DEBUG(log_debug, "about to wait()");
0280         
0281 #if (EX_OS_DARWIN && EX_LSB_RELEASE_VER_MAJOR < 23)
0282         /* waitpid cancel enabled...
0283          * sleep if no childs
0284          */
0285         while (1)
0286         {
0287             chldpid = (pid_t)waitpid(-1, &stat_loc, WUNTRACED);
0288             int err;
0289             
0290             if (EXFAIL==chldpid)
0291             {
0292                 if (err!=ECHILD)
0293                 {
0294                     userlog("waitpid failed: %s", tpstrerror(err));
0295                 }
0296                 sleep(1);
0297             }
0298             else
0299             {
0300                 pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old);
0301                 handle_child(chldpid, stat_loc);
0302                 pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old);
0303             }
0304         }       
0305 #else
0306         while ((chldpid = wait3(&stat_loc, WNOHANG|WUNTRACED, &rusage)) > 0)
0307         {
0308             handle_child(chldpid, stat_loc);
0309         }
0310 #endif
0311         
0312     }
0313    
0314     LOCKED_DEBUG(log_debug, "check_child_exit terminated");
0315     
0316     return NULL;
0317 }
0318 
0319 /**
0320  * Initialize polling lib
0321  * not thread safe.
0322  * @return EXSUCCED/EXFAIL
0323  */
0324 expublic int ndrxd_sigchld_init(void)
0325 {
0326     int ret = EXSUCCEED;
0327     pthread_attr_t pthread_custom_attr;
0328     char *fn = "ndrxd_sigchld_init";
0329 
0330     NDRX_LOG(log_debug, "%s - enter", fn);
0331     
0332     pthread_attr_init(&pthread_custom_attr);
0333     
0334     /* set some small stacks size, 1M should be fine! */
0335     ndrx_platf_stack_set(&pthread_custom_attr);
0336     
0337     if (EXSUCCEED!=pthread_create(&M_signal_thread, &pthread_custom_attr, 
0338             check_child_exit, NULL))
0339     {
0340         NDRX_PLATF_DIAG(NDRX_DIAG_PTHREAD_CREATE, errno, "ndrxd_sigchld_init");
0341         EXFAIL_OUT(ret);
0342     }
0343 
0344     M_signal_thread_set = EXTRUE;
0345     
0346 out:
0347     return ret;
0348 }
0349 
0350 /**
0351  * Un-initialize sigchild monitor thread
0352  * @return
0353  */
0354 expublic void ndrxd_sigchld_uninit(void)
0355 {
0356     char *fn = "ndrxd_sigchld_uninit";
0357     int err;
0358     NDRX_LOG(log_debug, "%s - enter", fn);
0359     
0360     if (!M_signal_thread_set)
0361     {
0362         NDRX_LOG(log_debug, "Signal thread was not initialised, nothing todo...");
0363         goto out;
0364     }
0365 
0366     NDRX_LOG(log_debug, "About to cancel signal thread");
0367     
0368     /* TODO: have a counter for number of sets, so that we can do 
0369      * un-init...
0370      */
0371     M_shutdown = EXTRUE;
0372     
0373 #if (EX_OS_DARWIN && EX_LSB_RELEASE_VER_MAJOR<23)
0374     if (EXSUCCEED!=pthread_cancel(M_signal_thread))
0375     {
0376         NDRX_LOG(log_error, "Failed to kill poll signal thread: %s", strerror(errno));
0377     }
0378 #else
0379     if (EXSUCCEED!=(err=pthread_kill(M_signal_thread, SIGCHLD)))
0380     {
0381         NDRX_LOG(log_error, "Failed to kill poll signal thread: %s", strerror(err));
0382     }
0383 #endif
0384     else
0385     {
0386         if (EXSUCCEED!=pthread_join(M_signal_thread, NULL))
0387         {
0388             NDRX_LOG(log_error, "Failed to join pthread_join() signal thread: %s", 
0389                     strerror(errno));
0390         }
0391     }
0392     
0393     M_signal_thread_set = EXFALSE;
0394     NDRX_LOG(log_debug, "finished ok");
0395 out:
0396     return;
0397 }
0398 
0399 /**
0400  * Add PID to PID hash
0401  * @param pid_hash
0402  * @return SUCCEED/FAIL
0403  */
0404 expublic int add_to_pid_hash(pm_pidhash_t **pid_hash, pm_node_t *p_pm)
0405 {
0406     int hash_key = p_pm->pid % ndrx_get_G_atmi_env()->max_servers;
0407     int ret=EXSUCCEED;
0408     pm_pidhash_t *pm_pid  = NDRX_MALLOC(sizeof(pm_pidhash_t));
0409     memset(pm_pid, 0, sizeof(pm_pidhash_t));
0410 
0411     NDRX_LOG(log_debug, "About to add pid %d", p_pm->pid);
0412 
0413     /* check error */
0414     if (NULL==pm_pid)
0415     {
0416         NDRXD_set_error_fmt(NDRXD_EOS, "failed to allocate pm_pidhash_t (%d bytes): %s",
0417                             sizeof(pm_pidhash_t), strerror(errno));
0418         EXFAIL_OUT(ret);
0419     }
0420 
0421     pm_pid->p_pm = p_pm;
0422     pm_pid->pid = p_pm->pid;
0423     NDRX_LOG(log_debug, "Added pid %d to hash with key %d", pm_pid->pid, hash_key);
0424     DL_APPEND(pid_hash[hash_key], pm_pid);
0425 
0426 out:
0427     return ret;
0428 }
0429 
0430 /**
0431  * Delete from PIDhash
0432  * @return SUCCEED/FAIL
0433  */
0434 expublic int delete_from_pid_hash(pm_pidhash_t **pid_hash, pm_pidhash_t *pm_pid)
0435 {
0436     int ret=EXSUCCEED;
0437 
0438     if (NULL!=pm_pid)
0439     {
0440         int hash_key = pm_pid->pid % ndrx_get_G_atmi_env()->max_servers;
0441 
0442         if (NULL!=pm_pid)
0443         {
0444             /* delete it */
0445             NDRX_LOG(log_error, "Removing pid %d from pidhash", pm_pid->pid);
0446             DL_DELETE(pid_hash[hash_key], pm_pid);
0447             /* here was memory leak!! */
0448             NDRX_FREE(pm_pid);
0449         }
0450     }
0451 out:
0452     return ret;
0453 }
0454 
0455 /**
0456  * Compares two field defs and returns equality result
0457  * @param a
0458  * @param b
0459  * @return 0 - equal/ -1 - not equal
0460  */
0461 exprivate int pid_hash_cmp(pm_pidhash_t *a, pm_pidhash_t *b)
0462 {
0463     return (a->pid==b->pid?EXSUCCEED:EXFAIL);
0464 }
0465 
0466 /**
0467  * Get field entry from int hash
0468  */
0469 expublic pm_pidhash_t *pid_hash_get(pm_pidhash_t **pid_hash, pid_t pid)
0470 {
0471     /* Get the linear array key */
0472     int hash_key = pid % ndrx_get_G_atmi_env()->max_servers; /* Simple mod based hash */
0473     pm_pidhash_t *ret=NULL;
0474     pm_pidhash_t tmp;
0475     
0476     tmp.pid=pid;
0477     
0478     if (NULL!=pid_hash && NULL!=pid_hash[hash_key])
0479     {
0480         DL_SEARCH(pid_hash[hash_key], ret, &tmp, pid_hash_cmp);
0481         NDRX_LOG(log_debug, "Search for pid %d to hash with key %d, result: 0x%lx",
0482                             tmp.pid, hash_key, ret);
0483     }
0484     else
0485     {
0486        NDRX_LOG(log_warn, "Empty PID hashes. Cannot process pid %d", pid);
0487     }
0488 
0489     return ret;
0490 }
0491 
0492 /**
0493  * This will build initial process model
0494  * So this is linked list with all processes.
0495  *
0496  * Shall this support re-load?
0497  * In that case we should check that servers with existing ID's are not running,
0498  * but configuration is different?
0499  * - That probably could be done in another stange, there we should check existing
0500  * ones, if some process names are different, then reject!?!?
0501  */
0502 expublic int build_process_model(conf_server_node_t *p_server_conf,
0503                                 pm_node_t **p_pm_model, /* proces model linked list */
0504                                 pm_node_t **p_pm_hash/* Hash table models */)
0505 {
0506     int ret=EXSUCCEED;
0507     conf_server_node_t *p_conf;
0508     pm_node_t   *p_pm=NULL;
0509     char tmp[PATH_MAX+1];
0510     int cnt;
0511     char *p;
0512     NDRX_LOG(log_debug, "build_process_model enter");
0513 
0514     DL_FOREACH(p_server_conf, p_conf)
0515     {
0516         for (cnt=0; cnt<p_conf->max; cnt++)
0517         {
0518             /* Now prepare add node to list + hash table */
0519             p_pm = NDRX_CALLOC(1, sizeof(pm_node_t));
0520             if (NULL==p_pm)
0521             {
0522                 /* Set return error code? */
0523                 NDRXD_set_error_fmt(NDRXD_EOS, "Failed to allocate pm_node_t: `%s'",
0524                                                     strerror(errno));
0525                 ret=EXFAIL;
0526                 goto out;
0527             }
0528             
0529             p_pm->conf = p_conf; /* keep the reference to config entry */
0530 
0531             /* format the process model entry */
0532             NDRX_STRCPY_SAFE(p_pm->binary_name, p_conf->binary_name);
0533             /*
0534             NDRX_STRCPY_SAFE(p_pm->binary_name_real, p_conf->binary_name_real);
0535             */
0536             /* get the path of the binary... */
0537             if (p_conf->reloadonchange)
0538             {
0539                 
0540                 if (EXEOS!=p_pm->conf->fullpath[0])
0541                 {
0542                     NDRX_LOG(log_debug, "Reusing full path from config...");
0543                     NDRX_STRCPY_SAFE(p_pm->binary_path, p_pm->conf->fullpath);
0544                 }
0545                 else if (NULL==ndrx_get_executable_path(p_pm->binary_path, 
0546                         sizeof(p_pm->binary_path), p_pm->binary_name))
0547                 {
0548                     NDRX_LOG(log_error, "Failed to get path for executable [%s] "
0549                             "`reloadonchange' will not be able to monitor it!",
0550                             p_pm->binary_name);
0551                 }
0552                 else
0553                 {
0554                     NDRX_LOG(log_info, "Got binary path: [%s]",
0555                             p_pm->binary_path);
0556                 }
0557             }
0558             
0559             p_pm->srvid = p_conf->srvid+cnt;
0560             /* Request state is stopped */
0561             p_pm->reqstate = NDRXD_PM_NOT_STARTED;
0562             
0563             /* This must autostart! */
0564             if (p_conf->min > cnt)
0565             {
0566                 p_pm->autostart=EXTRUE;
0567             }
0568             p_pm->autokill = p_conf->autokill;
0569             
0570             snprintf(p_pm->clopt, sizeof(p_pm->clopt), "-k %s -i %d %s", 
0571                     ndrx_get_G_atmi_env()->rnd_key, p_pm->srvid, p_conf->clopt);
0572             
0573             /* substitute env... - Feature #331 */
0574             
0575             snprintf(tmp, sizeof(tmp), "%d", (int)p_pm->srvid);
0576             NDRX_PM_SET_ENV(CONF_NDRX_SVSRVID, tmp);
0577             
0578             ndrx_str_env_subs_len(p_pm->clopt, sizeof(p_pm->clopt));
0579             
0580             /* build process real name  */
0581             if (EXEOS!=p_pm->conf->cmdline[0])
0582             {
0583                 NDRX_STRCPY_SAFE(tmp, p_pm->conf->cmdline);
0584 
0585                 /* substitute env */
0586 
0587                 if (EXSUCCEED!=setenv(CONF_NDRX_SVPROCNAME, p_pm->conf->binary_name, EXTRUE))
0588                 {
0589                     NDRX_LOG(log_error, "%s: failed to set %s=[%s]: %s", __func__, 
0590                         CONF_NDRX_SVPROCNAME, p_pm->conf->binary_name, strerror(errno));
0591                     EXFAIL_OUT(ret);
0592                 }
0593 
0594                 if (EXSUCCEED!=setenv(CONF_NDRX_SVCLOPT, p_pm->conf->clopt, EXTRUE))
0595                 {
0596                     NDRX_LOG(log_error, "%s: failed to set %s=[%s]: %s", __func__, 
0597                         CONF_NDRX_SVCLOPT, p_pm->conf->clopt, strerror(errno));
0598 
0599                     EXFAIL_OUT(ret);
0600                 }
0601 
0602                 /* format the cmdline */
0603                 ndrx_str_env_subs_len(tmp, sizeof(tmp));
0604 
0605                 /* extract binary name for the path... */
0606                 p = strtok(tmp, " \t");
0607 
0608                 if (NULL==p)
0609                 {
0610                     NDRX_LOG(log_error, "Missing process name in server's <cmdline> tag for [%s]",
0611                             p_pm->conf->binary_name);
0612                     EXFAIL_OUT(ret);
0613                 }
0614 
0615                 /* get the base name */
0616                 p = basename(tmp);
0617                 NDRX_STRCPY_SAFE(p_pm->binary_name_real, p);
0618 
0619                 NDRX_LOG(log_debug, "Extracted real binary name [%s] from [%s]", 
0620                         p_pm->binary_name_real, p);
0621 
0622                 /* unset variables */
0623                 unsetenv(CONF_NDRX_SVPROCNAME);
0624                 unsetenv(CONF_NDRX_SVCLOPT);
0625             }
0626             else
0627             {
0628                 NDRX_STRCPY_SAFE(p_pm->binary_name_real, p_pm->binary_name);
0629             }
0630 
0631             unsetenv(CONF_NDRX_SVSRVID);
0632             
0633             /* substitute env... - Feature #331, END */
0634 
0635             /* now check the hash table for server instance entry */
0636             if (p_pm->srvid < 1 || p_pm->srvid > ndrx_get_G_atmi_env()->max_servers-1)
0637             {
0638                 /* Invalid srvid  */
0639                 NDRXD_set_error_fmt(NDRXD_ESRVCIDINV, "(%s) Invalid server id `%d'", 
0640                         G_sys_config.config_file_short, p_pm->srvid);
0641                 ret = EXFAIL;
0642                 goto out;
0643             }
0644             else if (NULL!=p_pm_hash[p_pm->srvid])
0645             {
0646                 /* Duplicate srvid */
0647                 NDRXD_set_error_fmt(NDRXD_ESRVCIDDUP, "(%s) Duplicate server id `%d'", 
0648                         G_sys_config.config_file_short, p_pm->srvid);
0649                 ret = EXFAIL;
0650                 goto out;
0651             }
0652             else
0653             {
0654                 NDRX_LOG(log_debug, "adding %s:%d", p_pm->binary_name, 
0655                         p_pm->srvid);
0656                 /* Add it to has & model? */
0657                 DL_APPEND(*p_pm_model, p_pm);
0658                 /* Add it to the hash */
0659                 p_pm_hash[p_pm->srvid] = p_pm;
0660                 p_pm=NULL;
0661             }
0662         }/* for */
0663     } /* DL_FOREACH */
0664 
0665 out:
0666     if (NULL!=p_pm)
0667     {
0668         NDRX_FREE(p_pm);
0669     }
0670     NDRX_LOG(log_debug, "build_process_model return %d", ret);
0671     return ret;
0672 }
0673 
0674 /**
0675  *
0676  * @param srvid
0677  * @return
0678  */
0679 expublic pm_node_t * get_pm_from_srvid(int srvid)
0680 {
0681     if (srvid>=0 && srvid<ndrx_get_G_atmi_env()->max_servers)
0682     {
0683         return G_process_model_hash[srvid];
0684     }
0685     else
0686     {
0687         NDRX_LOG(log_error, "Got invalid srvid %d", srvid);
0688         return NULL;
0689     }
0690 }
0691 
0692 /**
0693  * Returns server admin q formatted.
0694  * @param p_pm
0695  * @return 
0696  */
0697 expublic char * get_srv_admin_q(pm_node_t * p_pm)
0698 {
0699     static char ret[NDRX_MAX_Q_SIZE+1];
0700     
0701     snprintf(ret, sizeof(ret), NDRX_ADMIN_FMT, G_sys_config.qprefix, 
0702             p_pm->binary_name_real, p_pm->srvid, p_pm->svpid);
0703     
0704     return ret;
0705 }
0706 
0707 /**
0708  * Remove resources used by process which failed to start.
0709  * I.e. this should remove process from PIDhash & and any services advertised.
0710  * @param p_pm
0711  * @param svcnm - single service to un-register, optional
0712  * @param pm_pid - Optional, if provided then PIDs of p_pm & pm_pid ir tested.
0713  *                 If they are different then only record from pidhash is removed
0714  *                 with PID of pm_pid.
0715  * @return EXSUCCEED/EXFAIL
0716  */
0717 expublic int remove_startfail_process(pm_node_t *p_pm, char *svcnm, 
0718         pm_pidhash_t *pm_pid)
0719 {
0720     int ret=EXSUCCEED;
0721     pm_node_svc_t *elt, *tmp;
0722     
0723     if (NULL==p_pm)
0724         goto out;
0725 
0726     if (NULL!=pm_pid && p_pm->pid!=pm_pid->pid)
0727     {
0728         NDRX_LOG(log_warn, "Process Model SRV/PID=%d/%d but given "
0729                 "PID Hash SRV/PID=%d/%d - thus remove later from pidhash only!",
0730                 p_pm->srvid, p_pm->pid, pm_pid->p_pm->srvid, pm_pid->pid);
0731         
0732         /* Remove only pidhash entry.
0733          * This might happen in cases if binary was externally started or some 
0734          * kind of delays in startup caused two instances to be started?
0735          */
0736         if (EXFAIL!=pm_pid->pid && 0!=pm_pid->pid)
0737         {
0738             delete_from_pid_hash(G_process_model_pid_hash,
0739                                 pid_hash_get(G_process_model_pid_hash, pm_pid->pid));
0740         }
0741         
0742         goto out;
0743         
0744     }
0745     
0746     if (NULL==svcnm)
0747     {
0748         /* Remote from pidhash */
0749         if (EXFAIL!=p_pm->pid && 0!=p_pm->pid)
0750         {
0751             delete_from_pid_hash(G_process_model_pid_hash,
0752                                 pid_hash_get(G_process_model_pid_hash, p_pm->pid));
0753         }
0754         /* Reset signal counter */
0755         p_pm->num_term_sigs=0;
0756      
0757         /* Reset kill request */
0758         p_pm->killreq = EXFALSE;
0759         
0760         /* Remove any queues used... */
0761         remove_server_queues(p_pm->binary_name_real, p_pm->pid, p_pm->srvid, NULL);
0762         
0763     }
0764     
0765     /* delete any services allocated */
0766     if (p_pm->flags&SRV_KEY_FLAGS_BRIDGE)
0767     {
0768         brd_del_bridge(p_pm->nodeid);
0769         /* Remove bridge related flags. */
0770         p_pm->flags&=~SRV_KEY_FLAGS_BRIDGE;
0771         p_pm->flags&=~SRV_KEY_FLAGS_SENDREFERSH;
0772         p_pm->flags&=~SRV_KEY_FLAGS_CONNECTED;
0773         p_pm->nodeid = 0;
0774     }
0775     
0776     brd_begin_diff();
0777     /* Delete the stuff out !!! Warning ! if server did add to shm but did not notify us
0778      * there is chance that some services will be left hanging in SHM */
0779     DL_FOREACH_SAFE(p_pm->svcs,elt,tmp)
0780     {
0781         int last;
0782         if (NULL==svcnm || 0==strcmp(elt->svc.svc_nm, svcnm))
0783         {
0784             NDRX_LOG(log_warn, "Removing pid's %d service [%s] resid %d", p_pm->pid, 
0785                     elt->svc.svc_nm, p_pm->resid);
0786             /* Remove from shm */
0787             
0788             /* ###################### CRITICAL SECTION ###################### */
0789             /* So we make this part critical... */
0790             if (EXSUCCEED!=ndrx_lock_svc_op(__func__))
0791             {
0792                 ret=EXFAIL;
0793                 goto out;
0794             }
0795             
0796             /* TODO: We need to uninstall by RQADDR! */
0797             ndrxd_shm_uninstall_svc(elt->svc.svc_nm, &last, p_pm->resid);
0798 
0799 #if defined(EX_USE_SYSVQ)
0800             
0801             /* nothing todo here! As we do not have per service queues */
0802             if (last)
0803             {
0804                 NDRX_LOG(log_debug, "Service [%s] will be zapped by "
0805                         "RQADDR sanity checks", elt->svc.svc_nm);
0806             }
0807             
0808 #elif defined(EX_USE_POLL)
0809             /* for poll() queues must be always removed. */
0810             remove_service_q(elt->svc.svc_nm, p_pm->srvid, (mqd_t)EXFAIL, NULL);
0811 #else
0812             if (last)
0813             {
0814                 remove_service_q(elt->svc.svc_nm, p_pm->srvid, (mqd_t)EXFAIL, NULL);
0815             }
0816 #endif
0817             
0818             /* Remove the lock! */
0819             ndrx_unlock_svc_op(__func__);
0820             /* ###################### CRITICAL SECTION, END ################# */
0821             
0822             /*  Delete it from our bridge view */
0823             brd_del_svc_from_hash(elt->svc.svc_nm);
0824             
0825             /* Delete out it from list */
0826             DL_DELETE(p_pm->svcs,elt);
0827             /* Fee up resources */
0828             NDRX_FREE(elt);
0829             
0830             if (NULL!=svcnm)
0831                 break; /* This was one we wanted to remove! */
0832         }
0833     }
0834     brd_end_diff();
0835     
0836 out:
0837     return ret;
0838 }
0839 
0840 /**
0841  * Wait for startup status of the program
0842  * @param p_pm PM node
0843  * @param p_processes_started number of processes started
0844  * @param doabort abort requested by command
0845  * @return EXSUCCEED/EXFAIL
0846  */
0847 exprivate int wait_for_status(pm_node_t *p_pm, long *p_processes_started, int *doabort)
0848 {
0849     int ret = EXSUCCEED;
0850     ndrx_stopwatch_t timer;
0851     int finished = EXFALSE;
0852 
0853     /* this is parent for child - sleep some seconds, then check for PID... */
0854     /* TODO: Replace sleep with wait call from service - wait for message? */
0855     /*usleep(250000);  250 milli seconds */
0856     ndrx_stopwatch_reset(&timer);
0857 
0858     do
0859     {
0860         NDRX_LOG(log_debug, "Waiting for response from srv...");
0861         /* do command processing for now */
0862         command_wait_and_run(&finished, doabort);
0863         /* check the status? */
0864     } while (ndrx_stopwatch_get_delta(&timer) < p_pm->conf->srvstartwait && 
0865                     NDRXD_PM_STARTING==p_pm->state && !(*doabort));
0866 
0867     if (NDRXD_PM_RUNNING_OK==p_pm->state && p_pm->conf->sleep_after)
0868     {
0869         ndrx_stopwatch_t sleep_timer;
0870         ndrx_stopwatch_reset(&sleep_timer);
0871 
0872         do
0873         {
0874             NDRX_LOG(log_debug, "In process after start sleep...");
0875             command_wait_and_run(&finished, doabort);
0876         } while (ndrx_stopwatch_get_delta_sec(&sleep_timer) < p_pm->conf->sleep_after);
0877 
0878     }
0879 
0880     /* Check for process name & pid */
0881     if (ndrx_sys_is_process_running(p_pm->svpid, p_pm->binary_name_real))
0882     {
0883         /*Should be set at info: p_pm->state = NDRXD_PM_RUNNING;*/
0884         NDRX_LOG(log_debug, "binary %s/%s, srvid %d started with pid %d/%d",
0885                     p_pm->binary_name,p_pm->binary_name_real, p_pm->srvid, 
0886                 p_pm->pid, p_pm->svpid);
0887         (*p_processes_started)++;
0888     }
0889     else if (NDRXD_PM_NOT_STARTED==p_pm->state)
0890     {
0891         /* Assume died? 
0892          * we should normally get self notification...
0893          */
0894         p_pm->state = NDRXD_PM_DIED;
0895         p_pm->state_changed = SANITY_CNT_START;
0896         NDRX_LOG(log_debug, "binary %s, srvid %d failed to start",
0897                     p_pm->binary_name, p_pm->srvid, p_pm->pid);
0898     }
0899     
0900 out:
0901     return ret;
0902 }
0903 
0904 /**
0905  * Start single process...
0906  * TODO: Add SIGCHLD handler here!
0907  * @param pm
0908  * @param do_wait - wait for process startup results
0909  * @param sg_snapshoot - snapshoot of the singleton groups
0910  * @param is_respawn - did process restart after the crash?
0911  * @return SUCCEED/FAIL
0912  */
0913 expublic int start_process(command_startstop_t *cmd_call, pm_node_t *p_pm,
0914             void (*p_startup_progress)(command_startstop_t *call, pm_node_t *p_pm, int calltype),
0915             long *p_processes_started,
0916             int do_wait,
0917             int *doabort,
0918             int *sg_snapshoot,
0919             int is_respawn)
0920 {
0921     int ret=EXSUCCEED;
0922     pid_t pid;
0923     /* prepare args for execution... */
0924     char cmd_str[PATH_MAX];
0925     char **cmd = NULL; /* split pointers.. */
0926     char *token;
0927     int numargs;
0928     int was_starting=EXFALSE;
0929     int alloc_args;
0930     char tmp[256];
0931     shm_srvinfo_t* srv;
0932 
0933     NDRX_LOG(log_warn, "*********processing for startup %s/%d (state: %ld)*********",
0934                                     p_pm->binary_name, p_pm->srvid, p_pm->state);
0935     
0936     if (NDRXD_PM_RUNNING_OK==p_pm->state)
0937     {
0938 
0939         NDRX_LOG(log_warn, "Not starting %s/%d, already in "
0940                                       "running/starting state!",
0941                                       p_pm->binary_name, p_pm->srvid);
0942         goto out;
0943     }
0944     else if (NDRXD_PM_STOPPING==p_pm->state)
0945     {
0946         /* Bug #798 */
0947         NDRX_LOG(log_warn, "Not starting %s/%d as it is in still shutting down, "
0948                                     "requesting restart!",
0949                                     p_pm->binary_name, p_pm->srvid);
0950 
0951         p_pm->reqstate = NDRXD_PM_RESTART;
0952         goto out;
0953     }
0954 
0955    /* Send notification, that we are about to start? */
0956     
0957     if (NDRXD_PM_STARTING==p_pm->state)
0958     {
0959         /* background did respawn the process, but yet no results,
0960          * so just sync on this for CLI...
0961          */
0962         was_starting=EXTRUE;
0963     }
0964     else if (   p_pm->conf->procgrp_no > 0
0965                 /* group is singleton: */
0966                 && ndrx_ndrxconf_procgroups_is_singleton(G_app_config->procgroups, 
0967                     p_pm->conf->procgrp_no)
0968                 /* and group is not locked by snapshoot or current data */
0969         &&  (  (NULL!=sg_snapshoot && !sg_snapshoot[p_pm->conf->procgrp_no-1])
0970             || EXTRUE!=ndrx_sg_is_locked(p_pm->conf->procgrp_no, NULL, 0)
0971             ))
0972     {
0973         NDRX_LOG(log_warn, " %s/%d Not staring as singleton process group %d is not locked",
0974                                     p_pm->binary_name, p_pm->srvid, p_pm->conf->procgrp_no);
0975 
0976         if (NULL!=p_startup_progress)
0977             p_startup_progress(cmd_call, p_pm, NDRXD_CALL_TYPE_PM_STARTING);
0978 
0979         if (NDRXD_PM_WAIT!=p_pm->state)
0980         {
0981             p_pm->state = NDRXD_PM_WAIT;
0982             p_pm->reqstate = NDRXD_PM_RUNNING_OK;
0983             p_pm->state_changed = SANITY_CNT_START;
0984         }
0985         goto progress_out;
0986     }
0987     else
0988     {
0989         p_pm->state = NDRXD_PM_STARTING;
0990     }
0991 
0992     p_pm->state_changed = SANITY_CNT_START;
0993 
0994     if (NULL!=p_startup_progress)
0995         p_startup_progress(cmd_call, p_pm, NDRXD_CALL_TYPE_PM_STARTING);
0996     
0997     if (was_starting)
0998     {
0999         NDRX_LOG(log_info, "Binary was restarted and startup was requested, sync");
1000         
1001         if (do_wait && EXSUCCEED!=wait_for_status(p_pm, p_processes_started, doabort))
1002         {
1003             EXFAIL_OUT(ret);
1004         }
1005         goto progress_out;
1006     }
1007     
1008     /* continue normal startup here... */
1009     
1010     /* calculate the checksum of the process */
1011     if (p_pm->conf->reloadonchange && EXEOS!=p_pm->binary_path[0])
1012     {
1013         roc_mark_as_reloaded(p_pm->binary_path, G_sanity_cycle);
1014     }
1015     
1016     /* clone our self */
1017     
1018     /*
1019      * During the fork, no queue ops shall be done! This can cause
1020      * System V event thread corruption.
1021      */
1022     MUTEX_LOCK_V(M_forklock);
1023     pid = ndrx_fork();
1024     
1025     /* No error */
1026     srv=ndrxd_shm_getsrv(p_pm->srvid);
1027     
1028     if (NULL!=srv)
1029     {
1030         srv->execerr=0;
1031     }
1032     
1033     if( pid == 0)
1034     {
1035         char sysflags_str[30];
1036         long sysflags = 0;
1037         int fd;
1038         
1039         /* reset signal handler so that for new processes there is scratch start */
1040         signal(SIGCHLD, SIG_DFL);
1041         signal(SIGINT, SIG_DFL);
1042         signal(SIGTERM, SIG_DFL);
1043         signal(SIGHUP, SIG_DFL);
1044         sigprocmask(SIG_SETMASK, &ndrx_G_org_mask, NULL);
1045 
1046         /* Bug #176: close parent resources - not needed any more... */
1047         ndrxd_shm_close_all();
1048         
1049         /* Export startup mode */
1050         if (G_sys_config.fullstart)
1051         {
1052             sysflags |= NDRX_PRC_SYSFLAGS_FULLSTART;
1053         }
1054         
1055         snprintf(sysflags_str, sizeof(sysflags_str), "%ld", sysflags);
1056         
1057         if (EXSUCCEED!=setenv(CONF_NDRX_SYSFLAGS, sysflags_str, EXTRUE))
1058         {
1059             userlog("Failed to set env: %s", strerror(errno));
1060             ndrxd_shm_srv_fork_status(p_pm->srvid, NDRXD_PM_EENV);
1061             exit(1);
1062         }
1063 
1064         /* for System V child event thread is not resumed
1065          * but it is not a big deal, this just deallocates the resources
1066          * after exec they will be free'd anyway.
1067          */
1068         if (EXSUCCEED!=ndrx_mq_close(G_command_state.listenq))
1069         {
1070             userlog("Failed to close: [%s] err: %s",
1071                                      G_command_state.listenq_str, strerror(errno));
1072         }
1073         
1074         /* some small delay so that parent gets time for PIDhash setup! */
1075         usleep(9000);
1076         
1077         /* this is child - start EnduroX back-end*/
1078         /*fprintf(stderr, "starting with: [%s]", p_pm->clopt);*/
1079         
1080         /* export intermediate variables 
1081          * CONF_NDRX_SVPROCNAME -> binary_name
1082          * CONF_NDRX_SVCLOPT-> dynamic clopt built
1083          */
1084         
1085         NDRX_PM_SET_ENV(CONF_NDRX_SVPROCNAME, p_pm->binary_name);
1086         NDRX_PM_SET_ENV(CONF_NDRX_SVCLOPT, p_pm->clopt);        
1087         /* export server id... */
1088         snprintf(tmp, sizeof(tmp), "%d", (int)p_pm->srvid);
1089         NDRX_PM_SET_ENV(CONF_NDRX_SVSRVID, tmp);
1090         
1091         /* ... and export PID 
1092          * This might be a parent process of the script
1093          * thus we need to pass it down to th client process.
1094          */
1095         snprintf(tmp, sizeof(tmp), "%d", (int)getpid());
1096         NDRX_PM_SET_ENV(CONF_NDRX_SVPPID, tmp);
1097         
1098         if (p_pm->conf->mindispatchthreads > 1)
1099         {
1100             snprintf(tmp, sizeof(tmp), "%d", p_pm->conf->mindispatchthreads);
1101             NDRX_PM_SET_ENV(CONF_NDRX_MINDISPATCHTHREADS, tmp);
1102         }
1103         
1104         if (p_pm->conf->maxdispatchthreads > 1)
1105         {
1106             snprintf(tmp, sizeof(tmp), "%d", p_pm->conf->maxdispatchthreads);
1107             NDRX_PM_SET_ENV(CONF_NDRX_MAXDISPATCHTHREADS, tmp);
1108         }
1109         
1110         if (p_pm->conf->threadstacksize > 0)
1111         {
1112             snprintf(tmp, sizeof(tmp), "%d", p_pm->conf->threadstacksize);
1113             NDRX_PM_SET_ENV(CONF_NDRX_THREADSTACKSIZE, tmp);
1114         }
1115 
1116         /* export process groups */
1117         if (p_pm->conf->procgrp_lp_no > 0)
1118         {
1119             snprintf(tmp, sizeof(tmp), "%d", p_pm->conf->procgrp_lp_no);
1120             NDRX_PM_SET_ENV(CONF_NDRX_PROCGRP_LP_NO, tmp);
1121         }
1122 
1123         if (p_pm->conf->procgrp_no > 0)
1124         {
1125             snprintf(tmp, sizeof(tmp), "%d", p_pm->conf->procgrp_no);
1126             NDRX_PM_SET_ENV(CONF_NDRX_PROCGRP_NO, tmp);
1127         }
1128 
1129         if (is_respawn)
1130         {
1131             NDRX_PM_SET_ENV(CONF_NDRX_RESPAWN, "1");
1132         }
1133 
1134         alloc_args = 0;
1135         REALLOC_CMD;
1136         
1137         if (EXEOS!=p_pm->conf->cmdline[0])
1138         {
1139             NDRX_STRCPY_SAFE(cmd_str, p_pm->conf->cmdline);
1140             
1141             /* format the cmdline */
1142             ndrx_str_env_subs_len(cmd_str, sizeof(cmd_str));
1143             
1144             numargs=0;
1145             
1146             /* have strtok which respects quoted strings... */
1147             token = ndrx_strtokblk(cmd_str, NDRX_CMDLINE_SEP, NDRX_CMDLINE_QUOTES);
1148             while( token != NULL )
1149             {
1150                 /* the alloc args are zero based index.. */
1151                 if (numargs+1 >= alloc_args)
1152                 {
1153                     /* realloc the cmd storage */
1154                     REALLOC_CMD;
1155                 }
1156                 cmd[numargs] = token;
1157                 token = ndrx_strtokblk( NULL, NDRX_CMDLINE_SEP, NDRX_CMDLINE_QUOTES);
1158                 numargs++;
1159             }
1160             cmd[numargs] = NULL;
1161             
1162         }
1163         else
1164         {
1165             NDRX_STRCPY_SAFE(cmd_str, p_pm->clopt);
1166 
1167             numargs=1;
1168                 
1169             if (EXEOS!=p_pm->conf->fullpath[0])
1170             {
1171                 cmd[0] = p_pm->conf->fullpath;
1172             }
1173             else
1174             {
1175                 cmd[0] = p_pm->binary_name;
1176             }
1177             
1178             token = ndrx_strtokblk(cmd_str, NDRX_CMDLINE_SEP, NDRX_CMDLINE_QUOTES);
1179             while( token != NULL )
1180             {
1181                 /* the alloc args are zero based index.. */
1182                 if (numargs+1 >= alloc_args)
1183                 {
1184                     /* realloc the cmd storage */
1185                     REALLOC_CMD;
1186                 }
1187                 
1188                 cmd[numargs] = token;
1189                 token = ndrx_strtokblk( NULL, NDRX_CMDLINE_SEP, NDRX_CMDLINE_QUOTES);
1190                 numargs++;
1191             }
1192             cmd[numargs] = NULL;
1193         }
1194         
1195         /*  Override environment, if there is such thing */
1196         if (EXEOS!=p_pm->conf->env[0])
1197         {
1198             if (EXSUCCEED!=ndrx_load_new_env(p_pm->conf->env))
1199             {
1200                 userlog("Failed to load custom env from: %s!", 
1201                         p_pm->conf->env);
1202                 ndrxd_shm_srv_fork_status(p_pm->srvid, NDRXD_PM_EENV);
1203                 exit(1);
1204             }
1205         }
1206         
1207         if (EXEOS!=p_pm->conf->cctag[0])
1208         {
1209             if (EXSUCCEED!=setenv(NDRX_CCTAG, p_pm->conf->cctag, EXTRUE))
1210             {
1211                 userlog("Cannot set [%s] to [%s]: %s", 
1212                         NDRX_CCTAG, p_pm->conf->cctag, strerror(errno));
1213                 exit(1);
1214             }
1215         }
1216 
1217         /* process env variables defined in ndrxconfig.xml */
1218         if (EXSUCCEED!=ndrx_ndrxconf_envs_applyall(p_pm->conf->envgrouplist,
1219                 p_pm->conf->envlist))
1220         {
1221             userlog("Failed to load config from ndrxconfig.xml!");
1222             ndrxd_shm_srv_fork_status(p_pm->srvid, NDRXD_PM_EENV);
1223             exit(1);
1224         }
1225         
1226         /* special processing for svapoll
1227          * if CONF_NDRX_NOPOLLEXCL is not set, then
1228          * apply polling flags
1229          */  
1230 #ifdef EX_USE_SVAPOLL
1231         if (NULL==getenv(CONF_NDRX_NOPOLLEXCL) && NULL==getenv(NDRX_POLLEXCL_POLICY))
1232         {
1233             setenv(NDRX_POLLEXCL_POLICY, NDRX_POLLEXCL_POLICY_DFLT, EXTRUE);
1234         }
1235 #endif
1236         /* free up the allocate resources */
1237         /* close stdin... */
1238         if (EXFAIL!=(fd = open("/dev/null", O_RDWR)))
1239         {
1240             dup2(fd, 0);
1241             close(fd);
1242         }
1243 
1244         if (EXSUCCEED != execvp (cmd[0], cmd))
1245         {
1246             int err = errno;
1247             int errcode = 0;
1248             
1249             userlog("Failed to start server [%s], error: %d, %s",
1250                                 cmd[0], err, strerror(err));
1251             
1252             switch (err)
1253             {
1254                 case E2BIG:
1255                     errcode=E2BIG;
1256                     break;
1257                 case EACCES:
1258                 case EPERM:
1259                     errcode=NDRXD_PM_EACCESS;
1260                     break;
1261                 case EFAULT:
1262                 case EIO:
1263                 case ELOOP:
1264                 case ENOMEM:
1265                     errcode=NDRXD_PM_ESYSTEM;
1266                     break;
1267                 case EINVAL:
1268                 case EISDIR:
1269                 case ENOEXEC:
1270                     errcode=NDRXD_PM_EBADFILE;
1271                     break;
1272                 case EMFILE:
1273                 case ENAMETOOLONG:
1274                     errcode=NDRXD_PM_ELIMIT;
1275                     break;
1276                 case ENOENT:
1277                 case ENOTDIR:
1278                     errcode=NDRXD_PM_ENOENT;
1279                     break;
1280             }
1281             
1282             if (errcode > 0)
1283             {
1284                 ndrxd_shm_srv_fork_status(p_pm->srvid, errcode);
1285             }
1286             
1287             /* free up the list, so that we do not report memory leak... 
1288              * in case if binary not started.
1289              */
1290             NDRX_FREE(cmd);
1291             
1292             if (ENOENT==err)
1293                 exit(TPEXIT_ENOENT);
1294             else
1295                 exit(1);
1296         }
1297     }
1298     else if (EXFAIL!=pid)
1299     {
1300         /* parent unlock */
1301         MUTEX_UNLOCK_V(M_forklock);
1302         
1303         /* Add stuff to PIDhash */
1304         p_pm->pid = pid;
1305         /* currently assume they are the same */
1306         p_pm->svpid = pid;
1307         add_to_pid_hash(G_process_model_pid_hash, p_pm);
1308         
1309         /* Reset PM timer */
1310         p_pm->rspstwatch = SANITY_CNT_START;
1311         
1312         /* Set requested state to started */
1313         p_pm->reqstate=NDRXD_PM_RUNNING_OK;
1314         
1315         /* Each time we try to start it we increment the try counter: */
1316         if (p_pm->exec_seq_try+1 < RESPAWN_CNTR_MAX)
1317         {
1318             p_pm->exec_seq_try++;
1319         }
1320         
1321         /* Do bellow stuff only if we can wait! */
1322         if (do_wait && EXSUCCEED!=wait_for_status(p_pm, p_processes_started, doabort))
1323         {
1324             EXFAIL_OUT(ret);
1325         }
1326 
1327         /* if was started, and given process provides singleton group lock,
1328          * update the snapshoot, as normally lock provider would come first
1329          */
1330         if (NULL!=sg_snapshoot 
1331             && p_pm->procgrp_lp_no > 0
1332             && EXTRUE==ndrx_sg_is_locked(p_pm->procgrp_lp_no, NULL, 0))
1333         {
1334             sg_snapshoot[p_pm->procgrp_lp_no-1] = EXTRUE;
1335         }
1336     }
1337     else
1338     {
1339         /* parent unlock */
1340         MUTEX_UNLOCK_V(M_forklock);
1341         
1342         NDRXD_set_error_fmt(NDRXD_EOS, "Fork failed: %s", strerror(errno));
1343         p_pm->state = NDRXD_PM_ELIMIT;
1344         p_pm->state_changed = SANITY_CNT_START;
1345         /*
1346         ret=FAIL;
1347          */
1348     }
1349 
1350     NDRX_LOG(log_debug, "PID of started process is %d", p_pm->pid);
1351 
1352 progress_out:    
1353     if (NULL!=p_startup_progress)
1354         p_startup_progress(cmd_call, p_pm, NDRXD_CALL_TYPE_PM_STARTED);
1355 
1356 out:
1357     return ret;
1358 }
1359 
1360 /**
1361  * Stop single process
1362  * @param p_pm
1363  * @return SUCCEED/FAIL
1364  */
1365 expublic int stop_process(command_startstop_t *cmd_call, pm_node_t *p_pm,
1366             void (*p_shutdown_progress)(command_call_t *call, pm_node_t *pm, int calltype),
1367             long *p_processes_shutdown,
1368             int *abort)
1369 {
1370     int ret=EXSUCCEED;
1371     command_call_t call;
1372     ndrx_stopwatch_t timer;
1373     int finished = EXFALSE;
1374     int first_shutdown;
1375     long start_state;
1376     char *srv_queue;
1377     int retry;
1378     char fn[] = "stop_process";
1379 
1380     memset(&call, 0, sizeof(call));
1381     NDRX_LOG(log_debug, "%s: Enter", fn);
1382 
1383 
1384     /* Check process initially, do it really requires shutdown? */
1385     NDRX_LOG(log_warn, "processing shutdown %s/%d",
1386                                 p_pm->binary_name, p_pm->srvid);
1387 
1388     /* Request that we stay stopped! */
1389     p_pm->reqstate = NDRXD_PM_NOT_STARTED;
1390     
1391     if (NDRXD_PM_RUNNING_OK!=p_pm->state &&
1392             NDRXD_PM_STOPPING!=p_pm->state &&
1393             NDRXD_PM_STARTING!=p_pm->state)
1394     {
1395         NDRX_LOG(log_debug, "Process already in non-runnable "
1396                                     "state: %d", p_pm->state);
1397         /* set the state to shutdown! */
1398         p_pm->state = NDRXD_PM_EXIT;
1399         p_pm->state_changed = SANITY_CNT_START;
1400         goto out;
1401     }
1402 
1403     /* Send notification, that we are about to stop? */
1404     p_pm->state = NDRXD_PM_STOPPING;
1405     p_pm->state_changed = SANITY_CNT_START;
1406     /* Reset response timer */
1407     p_pm->rspstwatch = SANITY_CNT_START;
1408     /* Reset ping too */
1409     p_pm->pingstwatch = SANITY_CNT_IDLE;
1410     
1411 
1412     if (NULL!=cmd_call && NULL!=p_shutdown_progress)
1413         p_shutdown_progress((command_call_t*)cmd_call, p_pm, NDRXD_CALL_TYPE_PM_STOPPING);
1414 
1415     /* Form a call queue, probably we need move all stuff to atmienv!  
1416     snprintf(srv_queue, sizeof(srv_queue), NDRX_ADMIN_FMT, G_sys_config.qprefix, 
1417             p_pm->binary_name, p_pm->srvid, p_pm->pid);*/
1418 
1419     ndrx_stopwatch_reset(&timer);
1420     first_shutdown=EXTRUE;
1421     retry=EXFALSE;
1422     start_state = p_pm->state;
1423     do
1424     {
1425         /* Request the shutdown 
1426          * in case if first request
1427          * or if server slipped initial shutdown request
1428          * if states are changed that means that still we are in running range
1429          * otherwise the bellow while should terminate
1430          */
1431         if (first_shutdown || start_state!=p_pm->state || retry)
1432         {
1433             srv_queue = get_srv_admin_q(p_pm);
1434             NDRX_LOG(log_debug, "%s: calling up: [%s] (retry: %d start_state: %ld cur_state: %ld)", 
1435                     fn, srv_queue, retry, start_state, p_pm->state);
1436             if (EXSUCCEED!=(ret=cmd_generic_call_2(NDRXD_COM_SRVSTOP_RQ, NDRXD_SRC_ADMIN,
1437                             NDRXD_CALL_TYPE_GENERIC,
1438                             &call, sizeof(call),
1439                             G_command_state.listenq_str,
1440                             G_command_state.listenq,
1441                             (mqd_t)EXFAIL,
1442                             srv_queue,
1443                             0, NULL,
1444                             NULL,
1445                             NULL,
1446                             NULL,
1447                             EXFALSE,
1448                             EXFALSE,
1449                             NULL, NULL, TPNOTIME|TPNOBLOCK, NULL)))
1450             {
1451                 /*goto out; Ignore this condition, just get the status of binary... */
1452                 /* if server was starting, the admin possibly q was not open */
1453                 retry=EXTRUE;
1454             }
1455             else
1456             {
1457                 start_state=p_pm->state;
1458                 retry=EXFALSE;
1459             }
1460         }
1461         
1462         NDRX_LOG(log_debug, "Waiting for response from srv... state: %d",
1463                                         p_pm->state); 
1464         /* do command processing for now */
1465         command_wait_and_run(&finished, abort);
1466         
1467         
1468         /* check the status? 
1469          * in case if during the shutdown process have changed state
1470          * i.e. from one started state switched to another started
1471          * state, that could indicate that process was in slow
1472          * startup and did ignore our shutdown request..
1473          * Thus if state is changed we should restart the call
1474          */
1475         first_shutdown=EXFALSE;
1476         
1477     } while (ndrx_stopwatch_get_delta(&timer) < p_pm->conf->srvstopwait &&
1478                     !PM_NOT_RUNNING(p_pm->state) &&
1479                     !(*abort));
1480     
1481     /* grab some stats... */
1482     if (PM_NOT_RUNNING(p_pm->state))
1483     {
1484         if (NULL!=p_processes_shutdown)
1485             (*p_processes_shutdown)++;
1486     }
1487     
1488     /* Send the progress of the shtudown (detail) */
1489     if (NULL!=cmd_call && NULL!=p_shutdown_progress)
1490         p_shutdown_progress((command_call_t*)cmd_call, p_pm, NDRXD_CALL_TYPE_PM_STOPPED);
1491 
1492 out:
1493     NDRX_LOG(log_debug, "%s: Exit (%d)", fn, ret);
1494     return ret;
1495 }
1496 
1497 /**
1498  * Mark group as booted
1499  * @param nrgrps number of groups
1500  * @param sg_groups array of groups
1501  */
1502 expublic void ndrx_mark_singlegrp_srv_booted(int nrgrps, int *sg_groups)
1503 {
1504     int i;
1505     for (i=0; i<nrgrps; i++)
1506     {
1507         if (sg_groups[i] && !ndrx_sg_bootflag_srv_get(i+1))
1508         {
1509             NDRX_LOG(log_debug, "Marking singleton group %d as servers booted", i);
1510             ndrx_sg_bootflag_srv_set(i+1);
1511         }
1512     }
1513 }
1514 
1515 /**
1516  * Start whole application. If configuration is not loaded, then this will
1517  * initiate configuration load.
1518  * @return SUCCEED/FAIL
1519  */
1520 expublic int app_startup(command_startstop_t *call,
1521         void (*p_startup_progress)(command_startstop_t *call, pm_node_t *pm, int calltype),
1522         long *p_processes_started) /* have some progress feedback */
1523 {
1524     int ret=EXSUCCEED;
1525     pm_node_t *p_pm;
1526     ndrx_procgroup_t *p_procgrp;
1527     int abort = EXFALSE;
1528     NDRX_LOG(log_warn, "Starting application domain");
1529 
1530     /*
1531     if (NULL==G_app_config && EXSUCCEED!=load_active_config(&G_app_config,
1532                 &G_process_model, &G_process_model_hash, &G_process_model_pid_hash))
1533      */
1534     if (NULL==G_app_config && EXSUCCEED!=load_active_config_live())
1535     {
1536         ret=EXFAIL;
1537         goto out;
1538     }
1539 
1540     /* OK, now loop through the stuff */
1541     G_sys_config.stat_flags |= NDRXD_STATE_DOMSTART;
1542 
1543     if (EXEOS!=call->procgrp[0])
1544     {
1545         p_procgrp =  ndrx_ndrxconf_procgroups_resolvenm(G_app_config->procgroups, call->procgrp);
1546         if (NULL==p_procgrp)
1547         {
1548             NDRX_LOG(log_warn, "Process group [%s] is not defined!", call->procgrp);
1549             NDRXD_set_error_fmt(NDRXD_ENOENT, "Process group [%s] is not defined!", call->procgrp);
1550             EXFAIL_OUT(ret);
1551         }
1552     }
1553 
1554     if (EXFAIL!=call->srvid)
1555     {
1556         /* Check the servid... */
1557         if (call->srvid>=0 && call->srvid<ndrx_get_G_atmi_env()->max_servers)
1558         {
1559             pm_node_t *p_pm_srvid = G_process_model_hash[call->srvid];
1560 
1561             if (NULL!=p_pm_srvid)
1562             {
1563                 start_process(call, p_pm_srvid, p_startup_progress,
1564                                     p_processes_started, EXTRUE, &abort, NULL, EXFALSE);
1565             }
1566             else
1567             {
1568                 NDRX_LOG(log_error, "Srvid: %d not initialized", call->srvid);
1569             }
1570         }
1571         else
1572         {
1573             NDRX_LOG(log_error, "Invalid srvid: %d", call->srvid);
1574         }
1575     }
1576     else /* process this if request for srvnm or full startup... */
1577     {
1578         int nrgrps = ndrx_G_libnstd_cfg.pgmax;
1579         int sg_groups[nrgrps];
1580 
1581         if (EXEOS==call->binary_name[0])
1582         {
1583             G_sys_config.fullstart=EXTRUE;
1584         }
1585 
1586         ndrx_sg_get_lock_snapshoot(sg_groups, &nrgrps, 0);
1587         
1588         DL_FOREACH(G_process_model, p_pm)
1589         {
1590             /* if particular binary shutdown requested (probably we could add some index!?) */
1591             
1592             /* Bug #306 auto start affects also server boot by name
1593              * Only autostart instances needs to be booted.
1594              */
1595             if (p_pm->autostart &&
1596                     ( (EXEOS!=call->binary_name[0] && 0==strcmp(call->binary_name, p_pm->binary_name)) ||
1597                       (EXEOS!=call->procgrp[0] && p_pm->conf->procgrp_no == p_procgrp->grpno) ||
1598                       (EXEOS!=call->procgrp[0] && (call->flags & NDRXD_CALL_FLAGS_LP2GRP) && 
1599                         p_pm->conf->procgrp_lp_no == p_procgrp->grpno) ||
1600                       /* Do full startup if requested autostart! */
1601                       (EXEOS==call->binary_name[0] && EXEOS==call->procgrp[0])
1602                     )) /* or If full shutdown requested */
1603             {
1604                 start_process(call, p_pm, p_startup_progress, 
1605                         p_processes_started, EXTRUE, &abort, sg_groups, EXFALSE);
1606                 
1607                 if (abort)
1608                 {
1609                     NDRX_LOG(log_warn, "Aborting app domain startup!");
1610                     NDRXD_set_error_fmt(NDRXD_EABORT, "App domain startup aborted!");
1611                     EXFAIL_OUT(ret);
1612                 }
1613             }
1614         } /* DL_FORACH pm. */
1615 
1616         ndrx_mark_singlegrp_srv_booted(nrgrps, sg_groups);
1617         
1618         if (G_sys_config.fullstart)
1619         {
1620             /* downgrade back to partial start */
1621             G_sys_config.fullstart=EXFALSE;
1622         }
1623     }
1624 out:
1625     return ret;
1626 }
1627 
1628 /**
1629  * Stop whole application.
1630  * App domain must be started in order to do shutdown.
1631  * @return SUCCEED/FAIL
1632  */
1633 expublic int app_shutdown(command_startstop_t *call,
1634         /* have some progress feedback */
1635         void (*p_shutdown_progress)(command_call_t *call, pm_node_t *pm, int calltype),
1636         long *p_processes_shutdown)
1637 {
1638     int ret=EXSUCCEED;
1639     int abort=EXFALSE;
1640     pm_node_t *p_pm;
1641     ndrx_procgroup_t *p_procgrp;
1642     
1643     NDRX_LOG(log_warn, "Stopping application domain");
1644 
1645     if (EXEOS!=call->procgrp[0])
1646     {
1647         if (NULL==G_app_config)
1648         {
1649             NDRX_LOG(log_error, "Configuration not loaded!");
1650             NDRXD_set_error_fmt(NDRXD_ENOCFGLD, "Configuration not loaded!");
1651             ret=EXFAIL;
1652             goto out;
1653         }
1654 
1655         p_procgrp =  ndrx_ndrxconf_procgroups_resolvenm(G_app_config->procgroups, call->procgrp);
1656         if (NULL==p_procgrp)
1657         {
1658             NDRX_LOG(log_warn, "Process group [%s] is not defined!", call->procgrp);
1659             NDRXD_set_error_fmt(NDRXD_ENOENT, "Process group [%s] is not defined!", call->procgrp);
1660             EXFAIL_OUT(ret);
1661         }
1662     }
1663     
1664     /* OK, now loop throught the stuff 
1665     G_sys_config.stat_flags |= NDRXD_STATE_SHUTDOWN;
1666 */
1667     if (EXFAIL!=call->srvid)
1668     {
1669         /* Check the servid... */
1670         if (call->srvid>=0 && call->srvid<ndrx_get_G_atmi_env()->max_servers)
1671         {
1672             pm_node_t *p_pm_srvid = G_process_model_hash[call->srvid];
1673             
1674             if (NULL!=p_pm_srvid)
1675             {
1676                 stop_process(call, p_pm_srvid, p_shutdown_progress,
1677                                     p_processes_shutdown, &abort);
1678             }
1679             else
1680             {
1681                 NDRX_LOG(log_error, "Srvid: %d not initialized", call->srvid);
1682             }
1683         }
1684         else
1685         {
1686             NDRX_LOG(log_error, "Invalid srvid: %d", call->srvid);
1687         }
1688     }
1689     else if (G_process_model) /* process this if request for srvnm or full shutdown... */
1690     {
1691         int i;
1692         DL_REVFOREACH(G_process_model, p_pm, i)
1693         {
1694             /* if particular binary shutdown requested (probably we could add some index!?) */
1695             if (  
1696                     (EXEOS!=call->binary_name[0] && 0==strcmp(call->binary_name, p_pm->binary_name)) ||
1697                     (EXEOS!=call->procgrp[0] && (p_pm->conf->procgrp_no==p_procgrp->grpno)) ||
1698                     (EXEOS!=call->procgrp[0] && (call->flags & NDRXD_CALL_FLAGS_LP2GRP) && 
1699                         p_pm->conf->procgrp_lp_no == p_procgrp->grpno) ||
1700                     (EXEOS==call->binary_name[0] && EXEOS==call->procgrp[0] &&  
1701                     /* or If full shutdown requested */
1702                     /* is if binary is not protected, or we run complete shutdown... */
1703                     (!p_pm->conf->isprotected || call->complete_shutdown))) 
1704             {
1705                 stop_process(call, p_pm, p_shutdown_progress, 
1706                         p_processes_shutdown, &abort);
1707                 
1708                 if (abort)
1709                 {
1710                     NDRX_LOG(log_warn, "Aborting app domain shutdown!");
1711                     NDRXD_set_error_fmt(NDRXD_EABORT, "App domain shutdown aborted!");
1712                     ret=EXFAIL;
1713                     goto out;
1714                 }
1715             }
1716         } /* For each pm. */
1717     }
1718     
1719     /* set state that we are shutdowned...
1720      * Hmm we should reply back and at next loop go down?
1721      * Or stay in idle?
1722      Not now, somehow needs to sync...
1723     G_sys_config.stat_flags &= ~NDRXD_STATE_SHUTDOWN;
1724     G_sys_config.stat_flags |= NDRXD_STATE_SHUTDOWNED;
1725     */
1726 out:
1727     return ret;
1728 }
1729 
1730 /**
1731  * Test process mode, it should be shutdown...!
1732  * @return 
1733  */
1734 expublic int is_srvs_down(void)
1735 {
1736     pm_node_t *p_pm;
1737     int is_down = EXTRUE;
1738     
1739     DL_FOREACH(G_process_model, p_pm)
1740     {
1741         if (PM_RUNNING(p_pm->state))
1742         {
1743             NDRX_LOG(6, "All servers not down...");
1744             is_down=EXFALSE;
1745         }
1746     } /* DL_FORACH pm. */
1747     
1748     return is_down;
1749 }
1750 
1751 /**
1752  * Reload services...
1753  * @return SUCCEED/FAIL
1754  */
1755 expublic int app_sreload(command_startstop_t *call,
1756         void (*p_startup_progress)(command_startstop_t *call, pm_node_t *pm, int calltype),
1757         void (*p_shutdown_progress)(command_call_t *call, pm_node_t *pm, int calltype),
1758         long *p_processes_started) /* have some progress feedback */
1759 {
1760     int ret=EXSUCCEED;
1761     pm_node_t *p_pm;
1762     int abort = EXFALSE;
1763     ndrx_procgroup_t *p_procgrp;
1764     NDRX_LOG(log_warn, "Reloading application domain");
1765     
1766     if (NULL==G_app_config)
1767     {
1768         NDRX_LOG(log_error, "Configuration not loaded!");
1769         NDRXD_set_error_fmt(NDRXD_ENOCFGLD, "Configuration not loaded!");
1770         ret=EXFAIL;
1771         goto out;
1772     }
1773 
1774     if (EXEOS!=call->procgrp[0])
1775     {
1776         p_procgrp = ndrx_ndrxconf_procgroups_resolvenm(G_app_config->procgroups, call->procgrp);
1777         if (NULL==p_procgrp)
1778         {
1779             NDRX_LOG(log_warn, "Process group [%s] is not defined!", call->procgrp);
1780             NDRXD_set_error_fmt(NDRXD_ENOENT, "Process group [%s] is not defined!", call->procgrp);
1781             EXFAIL_OUT(ret);
1782         }
1783     }
1784 
1785     /* OK, now loop throught the stuff 
1786     G_sys_config.stat_flags |= NDRXD_STATE_DOMSTART;
1787      * */
1788 
1789     if (EXFAIL!=call->srvid)
1790     {
1791         /* Check the servid... */
1792         if (call->srvid>=0 && call->srvid<ndrx_get_G_atmi_env()->max_servers)
1793         {
1794             pm_node_t *p_pm_srvid = G_process_model_hash[call->srvid];
1795 
1796             if (NULL!=p_pm_srvid
1797                 /* Only if requested state runnable (Bug #202) 
1798                  * in case if process was put in restart state (non runnable)
1799                  * it will be booted back when killed & respawned and finally
1800                  * it will be in running state. Thus we can skip it here from this
1801                  * particular reload.
1802                  * Thesame applies to bellow where restart is done by name
1803                  */
1804                 && (PM_RUNNING(p_pm_srvid->reqstate))
1805                 )
1806             {
1807                 /* Firstly stop it */
1808                 stop_process(call, p_pm_srvid, p_shutdown_progress, 
1809                         NULL, &abort);
1810 
1811                 /* Then start it */
1812                 if (!abort)
1813                 {
1814                     start_process(call, p_pm_srvid, p_startup_progress,
1815                                         p_processes_started, EXTRUE, &abort, NULL, EXFALSE);
1816                 }
1817                 
1818                 if (abort)
1819                 {
1820                     NDRX_LOG(log_warn, "Aborting app domain startup!");
1821                     NDRXD_set_error_fmt(NDRXD_EABORT, "App domain startup aborted!");
1822                     ret=EXFAIL;
1823                     goto out;
1824                 }
1825             }
1826             else
1827             {
1828                 NDRX_LOG(log_error, "Srvid: %d not initialized or "
1829                     "not requested to be run", call->srvid);
1830             }
1831         }
1832         else
1833         {
1834             NDRX_LOG(log_error, "Invalid srvid: %d", call->srvid);
1835         }
1836     }
1837     else /* process this if request for srvnm or full startup... */
1838     {
1839         DL_FOREACH(G_process_model, p_pm)
1840         {
1841             if ( (  (EXEOS!=call->binary_name[0] && 0==strcmp(call->binary_name, p_pm->binary_name)) ||
1842                     (EXEOS!=call->procgrp[0] && p_pm->conf->procgrp_no==p_procgrp->grpno) ||
1843                     (EXEOS!=call->procgrp[0] && (call->flags & NDRXD_CALL_FLAGS_LP2GRP) && 
1844                         p_pm->conf->procgrp_lp_no == p_procgrp->grpno) ||
1845                     (EXEOS==call->binary_name[0] && EXEOS==call->procgrp[0] && p_pm->autostart)
1846                 )
1847                 /* start only those binaries which were requested for start: */
1848                 && (PM_RUNNING(p_pm->reqstate))
1849                 )
1850             {
1851                 
1852                 stop_process(call, p_pm, p_shutdown_progress, 
1853                         NULL, &abort);
1854                 
1855                 if (!abort)
1856                 {
1857                     start_process(call, p_pm, p_startup_progress, 
1858                             p_processes_started, EXTRUE, &abort, NULL, EXFALSE);
1859                 }
1860                 
1861                 if (abort)
1862                 {
1863                     NDRX_LOG(log_warn, "Aborting app domain startup!");
1864                     NDRXD_set_error_fmt(NDRXD_EABORT, "App domain startup aborted!");
1865                     ret=EXFAIL;
1866                     goto out;
1867                 }
1868             }
1869         } /* DL_FORACH pm. */
1870     }
1871     
1872 out:
1873     return ret;
1874 }
1875 
1876 /* vim: set ts=4 sw=4 et smartindent: */
[ Source navigation ] [ Diff markup ] [ Identifier search ] [ general search ]

This page was automatically generated by the 2.1.0 LXR engine.

The LXR team
Valid CSS 2.1! Valid HTML 4.01!