source/libatmisrv/srvmain.c

0001 /**
0002  * @brief Enduro/X server main entry point
0003  *
0004  * @file srvmain.c
0005  */
0006 /* -----------------------------------------------------------------------------
0007  * Enduro/X Middleware Platform for Distributed Transaction Processing
0008  * Copyright (C) 2009-2016, ATR Baltic, Ltd. All Rights Reserved.
0009  * Copyright (C) 2017-2023, Mavimax, Ltd. All Rights Reserved.
0010  * This software is released under one of the following licenses:
0011  * AGPL (with Java and Go exceptions) or Mavimax's license for commercial use.
0012  * See LICENSE file for full text.
0013  * -----------------------------------------------------------------------------
0014  * AGPL license:
0015  *
0016  * This program is free software; you can redistribute it and/or modify it under
0017  * the terms of the GNU Affero General Public License, version 3 as published
0018  * by the Free Software Foundation;
0019  *
0020  * This program is distributed in the hope that it will be useful, but WITHOUT ANY
0021  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
0022  * PARTICULAR PURPOSE. See the GNU Affero General Public License, version 3
0023  * for more details.
0024  *
0025  * You should have received a copy of the GNU Affero General Public License along
0026  * with this program; if not, write to the Free Software Foundation, Inc.,
0027  * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
0028  *
0029  * -----------------------------------------------------------------------------
0030  * A commercial use license is available from Mavimax, Ltd
0031  * contact@mavimax.com
0032  * -----------------------------------------------------------------------------
0033  */
0034
0035 /*---------------------------Includes-----------------------------------*/
0036 #include <stdio.h>
0037 #include <stdlib.h>
0038 #include <errno.h>
0039 #include <fcntl.h>
0040
0041 #include <ndrstandard.h>
0042 #include <ndebug.h>
0043 #include <utlist.h>
0044 #include <string.h>
0045 #include <unistd.h>
0046 #include <tperror.h>
0047
0048 #include <atmi.h>
0049 #include "srv_int.h"
0050 #include "userlog.h"
0051 #include <atmi_int.h>
0052 #include <typed_buf.h>
0053 #include <atmi_tls.h>
0054 #include <nstd_int.h>
0055 /*---------------------------Externs------------------------------------*/
0056 /*---------------------------Macros-------------------------------------*/
0057
0058
0059 /** Alloc the CLOPTS */
0060 #define REALLOC_CLOPT_STEP    10
0061 #define REALLOC_CLOPT alloc_args+=REALLOC_CLOPT_STEP; \
0062     if (NULL==argv) \
0063         argv = NDRX_MALLOC(sizeof(char *)*alloc_args); \
0064     else \
0065         argv = NDRX_REALLOC(argv, sizeof(char *)*alloc_args); \
0066     if (NULL==argv) \
0067     {\
0068         int err = errno;\
0069         fprintf(stderr, "%s: failed to realloc %ld bytes: %s\n", __func__, \
0070             (long)sizeof(char *)*alloc_args, strerror(err));\
0071         userlog("%s: failed to realloc %ld bytes: %s\n", __func__, \
0072             (long)sizeof(char *)*alloc_args, strerror(err));\
0073         exit(1);\
0074     }
0075
0076 /*---------------------------Enums--------------------------------------*/
0077 /*---------------------------Typedefs-----------------------------------*/
0078 /*---------------------------Globals------------------------------------*/
0079 srv_conf_t G_server_conf={.service_array=NULL};
0080
0081 /**
0082  * Do not advertise these particular services
0083  */
0084 ndrx_svchash_t *ndrx_G_svchash_skip = NULL;
0085
0086 /**
0087  * Advertised functions in case if -N is used
0088  * Convert ndrx_skipsvc_add to exception_lists
0089  * for ndrx_G_svchash_skip and ndrx_G_svchash_funcs
0090  * -SNEWSVC:FUNC shall be added to exception list services, as otherwise -N would suppress them
0091  * -T new flag would suppress service advertises from tmdsptchtbl_t, but exception
0092  * list is still needed, because we want -S to be present even if -N is used.
0093  */
0094 ndrx_svchash_t *ndrx_G_svchash_funcs = NULL;
0095
0096 /**
0097  * List of defer messages so that we can call self services during the
0098  * startup...
0099  */
0100 exprivate ndrx_tpacall_defer_t *M_deferred_tpacalls = NULL;
0101
0102 /*---------------------------Statics------------------------------------*/
0103 /*---------------------------Prototypes---------------------------------*/
0104 exprivate int ndrx_tpacall_noservice_hook_defer(char *svc, char *data, long len, long flags);
0105 /**
0106  * Add service to skip advertise list
0107  * @param hash hash handle
0108  * @param svc_nm service name to add to hash
0109  * @return EXSUCCEED/EXFAIL
0110  */
0111 expublic int ndrx_svchash_add(ndrx_svchash_t **hash, char *svc_nm)
0112 {
0113     int ret = EXSUCCEED;
0114     ndrx_svchash_t *el = NULL;
0115
0116     if (NULL==(el = NDRX_MALLOC(sizeof(ndrx_svchash_t))))
0117     {
0118         NDRX_LOG(log_error, "%s: Failed to malloc: %s",
0119                 __func__, strerror(errno));
0120         userlog("%s: Failed to malloc: %s",
0121                 __func__, strerror(errno));
0122         EXFAIL_OUT(ret);
0123     }
0124
0125     NDRX_STRCPY_SAFE(el->svc_nm, svc_nm);
0126     EXHASH_ADD_STR( *hash, svc_nm, el);
0127
0128 out:
0129     return ret;
0130 }
0131
0132 /**
0133  * Check is service in the list
0134  * Be aware we could lookup for group names, but names with out @<GRP> are then
0135  * list.
0136  * @param hash hash handle to check service presence there
0137  * @param svcnm
0138  * @return EXFALSE/EXTRUE
0139  */
0140 expublic int ndrx_svchash_chk(ndrx_svchash_t **hash, char *svc_nm)
0141 {
0142     ndrx_svchash_t *el = NULL;
0143     char tmp[XATMI_SERVICE_NAME_LENGTH+1];
0144     char *p;
0145
0146     NDRX_STRCPY_SAFE(tmp, svc_nm);
0147
0148     p = strchr(tmp, NDRX_SYS_SVC_PFXC);
0149
0150     if (NULL!=p)
0151     {
0152         *p=EXEOS;
0153     }
0154
0155     /* search for name with out suffix... */
0156     EXHASH_FIND_STR( *hash, tmp, el);
0157
0158     if (NULL!=el)
0159     {
0160         return EXTRUE;
0161     }
0162
0163     return EXFALSE;
0164 }
0165
0166 /**
0167  * Delete hash list (un-init)
0168  * @param hash hash list to clean up
0169  */
0170 expublic void ndrx_svchash_cleanup(ndrx_svchash_t **hash)
0171 {
0172     ndrx_svchash_t *el = NULL, *elt = NULL;
0173
0174     EXHASH_ITER(hh, *hash, el, elt)
0175     {
0176         EXHASH_DEL(*hash, el);
0177         NDRX_FREE(el);
0178     }
0179 }
0180
0181 /**
0182  * Parse service argument (-s)
0183  * The format is following:
0184  * -s<New Service1>[,|/]<New Service2>[,|/]..[,|/]<New Service N>:<existing service>
0185  * e.g.
0186  * -sNEWSVC1/NEWSVC2:EXISTINGSVC
0187  *  TODO: if we are in routing group, then add @grp automatically for each alias
0188  *  - Also check the length of the new service. IF does not fit, then return error.
0189  * @param msg1 debug msg1
0190  * @param usegrp use groups if available
0191  * @return
0192  */
0193 expublic int ndrx_parse_svc_arg_cmn(char *msg1,
0194         svc_entry_t **root_svc_list, char *arg, int usegrp)
0195 {
0196     char alias_name[XATMI_SERVICE_NAME_LENGTH+1]={EXEOS};
0197     char grpsvc[MAXTIDENT*2]={EXEOS};
0198     char *p;
0199     svc_entry_t *entry=NULL;
0200     char *grparr[3]={NULL, NULL, NULL};
0201     int i;
0202     int len;
0203
0204     NDRX_LOG(log_debug, "Parsing %s entry: [%s]", msg1, arg);
0205
0206     if (NULL!=(p=strchr(arg, ':')))
0207     {
0208         NDRX_LOG(log_debug, "Aliasing requested");
0209         /* extract alias name out */
0210         NDRX_STRCPY_SAFE(alias_name, p+1);
0211         /* Put the EOS in place of : */
0212         *p=EXEOS;
0213     }
0214
0215     /* Now loop through services and add them to the list.
0216      * Seems that , is eat up by shell.. thus another symbol could be /
0217      */
0218     p = strtok(arg, ",/");
0219     while (NULL!=p)
0220     {
0221         grparr[0]=p;
0222         if (usegrp && G_atmi_env.rtgrp[0])
0223         {
0224             NDRX_STRCPY_SAFE(grpsvc, p);
0225             NDRX_STRCAT_S(grpsvc, sizeof(grpsvc), NDRX_SYS_SVC_PFX);
0226             NDRX_STRCAT_S(grpsvc, sizeof(grpsvc), G_atmi_env.rtgrp);
0227             grparr[1]=grpsvc;
0228         }
0229         else
0230         {
0231             grparr[1]=NULL;
0232         }
0233
0234         for (i=0; NULL!=grparr[i]; i++)
0235         {
0236             len = strlen(grparr[i]);
0237             if (len>XATMI_SERVICE_NAME_LENGTH)
0238             {
0239                 ndrx_TPset_error_fmt(TPEINVAL,
0240                         "Invalid service name [%s] too long %d, max allowed %d",
0241                         grparr[i], len, XATMI_SERVICE_NAME_LENGTH);
0242                 return EXFAIL; /* <<< return FAIL! */
0243             }
0244
0245             /* allocate memory for entry */
0246             if ( (entry = (svc_entry_t*)NDRX_MALLOC(sizeof(svc_entry_t))) == NULL)
0247             {
0248                 ndrx_TPset_error_fmt(TPEOS,
0249                         "Failed to allocate %d bytes while parsing -s",
0250                         sizeof(svc_entry_t));
0251                 return EXFAIL; /* <<< return FAIL! */
0252             }
0253
0254             NDRX_STRCPY_SAFE(entry->svc_nm, grparr[i]);
0255             entry->svc_aliasof[0]=EXEOS;
0256
0257             if (EXEOS!=alias_name[0])
0258             {
0259                 NDRX_STRCPY_SAFE(entry->svc_aliasof, alias_name);
0260             }
0261
0262             /*
0263              * Should we check duplicate names here?
0264              */
0265             DL_APPEND((*root_svc_list), entry);
0266
0267             NDRX_LOG(log_debug, "%s [%s]:[%s]", msg1, entry->svc_nm, entry->svc_aliasof);
0268         }
0269
0270         p = strtok(NULL, ",/");
0271     }
0272
0273     return EXSUCCEED;
0274 }
0275
0276 /**
0277  * Process -s CLI flag / fill array, service:service mapping flag
0278  * @param arg -s flag value
0279  * @return
0280  */
0281 expublic int ndrx_parse_svc_arg(char *arg)
0282 {
0283     return ndrx_parse_svc_arg_cmn("-s", &G_server_conf.svc_list, arg, EXTRUE);
0284 }
0285
0286 /**
0287  * parse -S service:function mapping flag
0288  * Note that function aliases are replied much later via help of tpadveritse
0289  * thus at that point group is added to the name (if having any DDR group set)
0290  * @param root_svc_list
0291  * @param arg -S flag value
0292  */
0293 expublic int ndrx_parse_func_arg(char *arg)
0294 {
0295     return ndrx_parse_svc_arg_cmn("-S", &G_server_conf.funcsvc_list, arg, EXFALSE);
0296 }
0297
0298 /*
0299  * Lookup conversion function registered for hash
0300  */
0301 expublic long ndrx_xcvt_lookup(char *fn_nm)
0302 {
0303     xbufcvt_entry_t *entry=NULL;
0304
0305     EXHASH_FIND_STR( G_server_conf.xbufcvt_tab, fn_nm, entry);
0306
0307     if (NULL!=entry)
0308     {
0309         return entry->xcvtflags;
0310     }
0311
0312     return 0;
0313 }
0314
0315 /**
0316  * Parse flags
0317  * @param argc
0318  * @return
0319  */
0320 int ndrx_parse_xcvt_arg(char *arg)
0321 {
0322     char cvtfunc[XATMI_SERVICE_NAME_LENGTH+1]={EXEOS};
0323     char *p;
0324     xbufcvt_entry_t *entry=NULL;
0325     int ret = EXSUCCEED;
0326     long flags = 0;
0327     NDRX_LOG(log_debug, "Parsing function buffer convert entry: [%s]", arg);
0328
0329     if (NULL!=(p=strchr(arg, ':')))
0330     {
0331         /* Conversion function name */
0332         NDRX_STRCPY_SAFE(cvtfunc, p+1);
0333         *p=EXEOS;
0334
0335         /* Verify that function is correct */
0336         if (0==strcmp(cvtfunc, BUF_CVT_INCOMING_JSON2UBF_STR))
0337         {
0338             flags|=SYS_SRV_CVT_JSON2UBF;
0339         }
0340         else if (0==strcmp(cvtfunc, BUF_CVT_INCOMING_UBF2JSON_STR))
0341         {
0342             flags|=SYS_SRV_CVT_UBF2JSON;
0343         }
0344         if (0==strcmp(cvtfunc, BUF_CVT_INCOMING_JSON2VIEW_STR))
0345         {
0346             flags|=SYS_SRV_CVT_JSON2VIEW;
0347         }
0348         else if (0==strcmp(cvtfunc, BUF_CVT_INCOMING_VIEW2JSON_STR))
0349         {
0350             flags|=SYS_SRV_CVT_VIEW2JSON;
0351         }
0352
0353         if (0==flags)
0354         {
0355             NDRX_LOG(log_error, "Invalid automatic buffer conversion function (%s)!",
0356                     cvtfunc);
0357             EXFAIL_OUT(ret);
0358         }
0359     }
0360     else
0361     {
0362         NDRX_LOG(log_error, "Invalid argument for -x (%s) missing `:'", arg);
0363         EXFAIL_OUT(ret);
0364     }
0365
0366     p = strtok(arg, ",");
0367     while (NULL!=p)
0368     {
0369         /* allocate memory for entry */
0370         if ( (entry = (xbufcvt_entry_t*)NDRX_MALLOC(sizeof(xbufcvt_entry_t))) == NULL)
0371         {
0372             ndrx_TPset_error_fmt(TPMINVAL, "Failed to allocate %d bytes while parsing -s",
0373                                 sizeof(svc_entry_t));
0374             return EXFAIL; /* <<< return FAIL! */
0375         }
0376
0377         NDRX_STRCPY_SAFE(entry->fn_nm, p);
0378         entry->xcvtflags = flags;
0379
0380
0381         NDRX_LOG(log_debug, "Added have automatic convert option [%s] "
0382                 "for function [%s] (-x)", cvtfunc, entry->fn_nm);
0383
0384         EXHASH_ADD_STR( G_server_conf.xbufcvt_tab, fn_nm, entry );
0385
0386         p = strtok(NULL, ",");
0387     }
0388
0389 out:
0390     return ret;
0391 }
0392
0393
0394 /**
0395  * Configure server stderr / stdout and initial logging
0396  */
0397 exprivate void configure_outputs(void)
0398 {
0399     FILE *f_stderr = NULL;
0400     FILE *f_stdout = NULL;
0401     int stdout_ok = EXFALSE;
0402     long sync_flags = NDRX_LOG_FSYNCSTDERR;
0403
0404     /* open stdout, if any set... */
0405     if (EXEOS!=G_server_conf.std_output[0] &&
0406         NULL!=(f_stdout=NDRX_FOPEN(G_server_conf.std_output, "a")))
0407     {
0408         if (EXSUCCEED!=fcntl(fileno(f_stdout), F_SETFD, FD_CLOEXEC))
0409         {
0410             userlog("WARNING: Failed to set FD_CLOEXEC (1): %s",
0411                 strerror(errno));
0412         }
0413
0414         if (EXFAIL==dup2(fileno(f_stdout), STDOUT_FILENO))
0415         {
0416             userlog("%s: Failed to dup2(1): %s", __func__, strerror(errno));
0417         }
0418         else
0419         {
0420             stdout_ok=EXTRUE;
0421         }
0422
0423         if (0==strcmp(G_server_conf.std_output, G_server_conf.err_output))
0424         {
0425             sync_flags |= NDRX_LOG_FSYNCSTDOUT;
0426         }
0427     }
0428     else if (EXEOS!=G_server_conf.std_output[0])
0429     {
0430         userlog("Failed to open stdout file [%s]: %s",
0431                 G_server_conf.std_output, strerror(errno));
0432     }
0433
0434     /* Open error log, OK? */
0435     /* Do we need to close this on exec? */
0436     if (EXEOS!=G_server_conf.err_output[0] &&
0437         NULL!=(f_stderr=NDRX_FOPEN(G_server_conf.err_output, "a")))
0438     {
0439         /* Bug #176 */
0440         if (EXSUCCEED!=fcntl(fileno(f_stderr), F_SETFD, FD_CLOEXEC))
0441         {
0442             userlog("WARNING: Failed to set FD_CLOEXEC (2): %s",
0443                 strerror(errno));
0444         }
0445
0446         if (!stdout_ok)
0447         {
0448             if (EXFAIL==dup2(fileno(f_stderr), STDOUT_FILENO))
0449             {
0450                 userlog("%s: Failed to dup2(1): %s", __func__, strerror(errno));
0451             }
0452             else
0453             {
0454                 sync_flags |= NDRX_LOG_FSYNCSTDOUT;
0455             }
0456         }
0457
0458         if (EXFAIL==dup2(fileno(f_stderr), STDERR_FILENO))
0459         {
0460             userlog("%s: Failed to dup2(2): %s", __func__, strerror(errno));
0461         }
0462
0463         /* ALSO! reconfigure ndrx/tp/ubf to user this log!
0464          * if stderr currently is used...
0465          * However not sure what we could do with threads?
0466          * MQ or others if they have chosen to work with stderr
0467          * then those will not rotate...
0468          * Anyway we will do the best.
0469          */
0470         if (ndrx_debug_is_proc_stderr())
0471         {
0472             /* if we do logrotate, we should re-open stdout/stderr */
0473             if (EXSUCCEED!=tplogconfig(LOG_CODE_UBF|LOG_CODE_NDRX|LOG_CODE_TP, EXFAIL,
0474                     NULL, NULL, G_server_conf.err_output))
0475             {
0476                 NDRX_LOG(log_debug, "Failed to re-open logger");
0477             }
0478             else
0479             {
0480                 ndrx_debug_proc_link_ndrx(sync_flags);
0481             }
0482         }
0483     }
0484     else if (EXEOS!=G_server_conf.err_output[0])
0485     {
0486         userlog("Failed to open error file [%s]: %s", G_server_conf.err_output, strerror(errno));
0487     }
0488
0489     if (NULL!=f_stderr)
0490     {
0491         NDRX_FCLOSE(f_stderr);
0492     }
0493
0494     if (NULL!=f_stdout)
0495     {
0496         NDRX_FCLOSE(f_stdout);
0497     }
0498 }
0499
0500 /**
0501  * Internal initialization.
0502  * Here we will:
0503  * - Determine server name (binary)
0504  * - Determine client ID (1,2,3,4,5, etc...) (flag -i <num>)
0505  * @param argc
0506  * @param argv
0507  * @return  SUCCEED/FAIL
0508  */
0509 expublic int ndrx_init(int argc, char** argv)
0510 {
0511     int ret=EXSUCCEED;
0512     int c;
0513     int dbglev;
0514     char *p;
0515     char key[NDRX_MAX_KEY_SIZE]={EXEOS};
0516     char rqaddress[NDRX_MAX_Q_SIZE+1] = "";
0517     char tmp[NDRX_MAX_Q_SIZE+1];
0518     int was_grp_used=EXFALSE;
0519
0520     /* reply aliases later, as maybe there is -sSVC -g GRP
0521      * thus if performing advertise first, we do not yet know that we are part of the
0522      * group
0523      */
0524     string_list_t *svcalias=NULL, *svciter;
0525
0526     /* Create ATMI context */
0527     ATMI_TLS_ENTRY;
0528
0529     /* set pre-check values */
0530     memset(&G_server_conf, 0, sizeof(G_server_conf));
0531     G_shutdown_req=EXFALSE;
0532     /* Set default advertise all */
0533     G_server_conf.advertise_all = 1;
0534     G_server_conf.time_out = EXFAIL;
0535     G_server_conf.mindispatchthreads = 1;
0536     G_server_conf.maxdispatchthreads = 1;
0537
0538     /* Load common atmi library environment variables */
0539     if (EXSUCCEED!=ndrx_load_common_env())
0540     {
0541         NDRX_LOG(log_error, "Failed to load common env");
0542         ret=EXFAIL;
0543         goto out;
0544     }
0545
0546 #ifdef __GNU_LIBRARY__
0547     optind=0; /* reset lib, so that we can scan again. */
0548 #else
0549     optind=1; /* reset lib, so that we can scan again. */
0550 #endif
0551
0552     /* Parse command line, will use simple getopt */
0553     while ((c = getopt(argc, argv, "h?:D:i:k:e:R:rs:t:x:Nn:S:g:GBo:--")) != EXFAIL)
0554     {
0555         switch(c)
0556         {
0557             case 'g':
0558
0559                 /* routing group... override here whatever we have in env... */
0560                 if (EXEOS!=G_atmi_env.rtgrp[0])
0561                 {
0562                     was_grp_used=EXTRUE;
0563                 }
0564
0565                 NDRX_STRCPY_SAFE(G_atmi_env.rtgrp, optarg);
0566                 NDRX_LOG(log_info, "Routing group %s to [%s]",
0567                         (was_grp_used?"cli override":"set"), G_atmi_env.rtgrp);
0568                 break;
0569             case 'G':
0570                 /* Send @GPR encoded in service TPSVCINFO.name*/
0571                 G_server_conf.ddr_keep_grp=EXTRUE;
0572                 NDRX_LOG(log_info, "Keeping DDR group name in service names");
0573                 break;
0574             case 'k':
0575                 /* just ignore the key... */
0576                 NDRX_STRCPY_SAFE(key, optarg);
0577                 break;
0578             case 'R':
0579                 /* just ignore the key... */
0580
0581                 if (NDRX_SYS_SVC_PFXC==optarg[0])
0582                 {
0583                     NDRX_LOG(log_error, "-R request address cannot start with [%c]",
0584                             NDRX_SYS_SVC_PFXC);
0585                     userlog("-R request address cannot start with [%c]",
0586                             NDRX_SYS_SVC_PFXC);
0587                     ndrx_TPset_error_fmt(TPEINVAL, "-R request address cannot start with [%c]",
0588                             NDRX_SYS_SVC_PFXC);
0589                     EXFAIL_OUT(ret);
0590                 }
0591
0592                 NDRX_STRCPY_SAFE(rqaddress, optarg);
0593                 break;
0594
0595             case 's':
0596
0597                 if (EXSUCCEED!=ndrx_string_list_add(&svcalias, optarg))
0598                 {
0599                     /* probably OOM */
0600                     NDRX_LOG(log_error, "Failed to populate svcalias list");
0601                     EXFAIL_OUT(ret);
0602                 }
0603
0604                 break;
0605             case 'S':
0606                 ret=ndrx_parse_func_arg(optarg);
0607                 break;
0608
0609             case 'x':
0610                 ret=ndrx_parse_xcvt_arg(optarg);
0611                 break;
0612             case 'D': /* Not used. */
0613                 dbglev = atoi(optarg);
0614                 tplogconfig(LOG_FACILITY_NDRX, dbglev, NULL, NULL, NULL);
0615                 break;
0616             case 'i': /* server id */
0617                 /*fprintf(stderr, "got -i: %s\n", optarg);*/
0618                 G_server_conf.srv_id = atoi(optarg);
0619                 break;
0620             case 'N':
0621                 /* Do not advertise all services */
0622                 G_server_conf.advertise_all = 0;
0623                 break;
0624             case 'B':
0625                 /* Do not advertise services provided by buildserver...
0626                  * basically this is opposite to -A of Tuxedo
0627                  */
0628                 G_server_conf.no_built_advertise = EXTRUE;
0629                 break;
0630             case 'n':
0631                 /* Do not advertise single service */
0632                 if (EXSUCCEED!=ndrx_svchash_add(&ndrx_G_svchash_skip, optarg))
0633                 {
0634                     ndrx_TPset_error_msg(TPESYSTEM, "Malloc failed");
0635                     EXFAIL_OUT(ret);
0636                 }
0637                 break;
0638             case 'r': /* Not used. */
0639                 /* Not sure actually what does this mean, but ok, lets have it. */
0640                 G_server_conf.log_work = 1;
0641                 break;
0642             case 'e':
0643                 NDRX_STRCPY_SAFE(G_server_conf.err_output, optarg);
0644                 break;
0645             case 'o':
0646                 NDRX_STRCPY_SAFE(G_server_conf.std_output, optarg);
0647                 break;
0648             case 't':
0649                 /* Override timeout settings for communications with ndrxd
0650                  * i.e. for ndrxd resposne waiting & other msg requeuing...
0651                  */
0652                 G_server_conf.time_out = atoi(optarg);
0653                 break;
0654             case 'h': case '?':
0655                 fprintf(stderr, "usage: %s [-D dbglev] -i server_id [-N - do "
0656                         "not advertise servers]"
0657                         " [-sSERVER:ALIAS] [-sSERVER]\n",
0658                                 argv[0]);
0659                 goto out;
0660                 break;
0661             /* add support for s */
0662         }
0663     }
0664
0665     configure_outputs();
0666
0667     /* reply the aliases as we have now groups set ... */
0668     LL_FOREACH(svcalias, svciter)
0669     {
0670         if (EXSUCCEED!=ndrx_parse_svc_arg(svciter->qname))
0671         {
0672             EXFAIL_OUT(ret);
0673         }
0674     }
0675
0676     /* Override the timeout with system value, if FAIL i.e. -t was not present */
0677     if (EXFAIL==G_server_conf.time_out)
0678     {
0679         /* Get timeout */
0680         if (NULL!=(p=getenv(CONF_NDRX_TOUT)))
0681         {
0682             G_server_conf.time_out = atoi(p);
0683         }
0684         else
0685         {
0686             ndrx_TPset_error_msg(TPEINVAL, "Error: Missing evn param: NDRX_TOUT, "
0687                     "cannot determine default timeout!");
0688             ret=EXFAIL;
0689             goto out;
0690         }
0691     }
0692
0693     NDRX_LOG(log_debug, "Using comms timeout: %d",
0694                                     G_server_conf.time_out);
0695
0696     /* Validate the configuration */
0697     if (G_server_conf.srv_id<1)
0698     {
0699         ndrx_TPset_error_msg(TPEINVAL, "Error: server ID (-i) must be >= 1");
0700         ret=EXFAIL;
0701         goto out;
0702     }
0703
0704     /*
0705      * Extract the binary name
0706      */
0707     p=strrchr(argv[0], '/');
0708     if (NULL!=p)
0709     {
0710         NDRX_STRCPY_SAFE(G_server_conf.binary_name, p+1);
0711     }
0712     else
0713     {
0714         NDRX_STRCPY_SAFE(G_server_conf.binary_name, argv[0]);
0715     }
0716
0717     /*
0718      * Read queue prefix (This is mandatory to have)
0719      */
0720     if (NULL==(p=getenv(CONF_NDRX_QPREFIX)))
0721     {
0722         ndrx_TPset_error_fmt(TPEINVAL, "Env [%s] not set", CONF_NDRX_QPREFIX);
0723         ret=EXFAIL;
0724         goto out;
0725     }
0726     else
0727     {
0728         NDRX_STRCPY_SAFE(G_server_conf.q_prefix, p);
0729     }
0730
0731     /* configure server threads... */
0732     if (NULL==(p=getenv(CONF_NDRX_QPREFIX)))
0733     {
0734         ndrx_TPset_error_fmt(TPEINVAL, "Env [%s] not set", CONF_NDRX_QPREFIX);
0735         ret=EXFAIL;
0736         goto out;
0737     }
0738     else
0739     {
0740         NDRX_STRCPY_SAFE(G_server_conf.q_prefix, p);
0741     }
0742
0743     if (NULL!=(p=getenv(CONF_NDRX_MINDISPATCHTHREADS)))
0744     {
0745         G_server_conf.mindispatchthreads = atoi(p);
0746     }
0747
0748     if (NULL!=(p=getenv(CONF_NDRX_MAXDISPATCHTHREADS)))
0749     {
0750         G_server_conf.maxdispatchthreads = atoi(p);
0751     }
0752
0753     if (G_server_conf.maxdispatchthreads < G_server_conf.mindispatchthreads)
0754     {
0755         NDRX_LOG(log_error, "Error ! MAXDISPATCHTHREADS(=%d) < MINDISPATCHTHREADS(=%d)",
0756                 G_server_conf.maxdispatchthreads,
0757                 G_server_conf.mindispatchthreads
0758                 );
0759         userlog("Error ! MAXDISPATCHTHREADS(=%d) < MINDISPATCHTHREADS(=%d)",
0760                 G_server_conf.maxdispatchthreads,
0761                 G_server_conf.mindispatchthreads);
0762
0763         ndrx_TPset_error_fmt(TPEINVAL, "Error ! MAXDISPATCHTHREADS(=%d) < MINDISPATCHTHREADS(=%d)",
0764                 G_server_conf.maxdispatchthreads,
0765                 G_server_conf.mindispatchthreads);
0766         EXFAIL_OUT(ret);
0767     }
0768
0769     /* check thread option.. */
0770     if (!_tmbuilt_with_thread_option && G_server_conf.maxdispatchthreads > 1)
0771     {
0772         NDRX_LOG(log_error, "Warning ! Server not built for multi-threading, "
0773                 "but MINDISPATCHTHREADS=%d MAXDISPATCHTHREADS=%d, falling back to single thread mode",
0774                 G_server_conf.mindispatchthreads,
0775                 G_server_conf.maxdispatchthreads
0776                 );
0777         userlog("Warning ! Server not built for multi-threading, "
0778                 "but MINDISPATCHTHREADS=%d MAXDISPATCHTHREADS=%d, falling back to single thread mode",
0779                 G_server_conf.mindispatchthreads,
0780                 G_server_conf.maxdispatchthreads);
0781     }
0782
0783     if (G_server_conf.mindispatchthreads <=0 )
0784     {
0785         NDRX_LOG(log_error, "Error ! MINDISPATCHTHREADS(=%d) <=0",
0786                 G_server_conf.mindispatchthreads);
0787         userlog("Error ! MINDISPATCHTHREADS(=%d) <=0",
0788                 G_server_conf.mindispatchthreads);
0789         ndrx_TPset_error_fmt(TPEINVAL, "Error ! MINDISPATCHTHREADS(=%d) <=0",
0790                 G_server_conf.mindispatchthreads);
0791         EXFAIL_OUT(ret);
0792     }
0793
0794     if (G_server_conf.maxdispatchthreads <=0)
0795     {
0796         NDRX_LOG(log_error, "Error ! MAXDISPATCHTHREADS(=%d) <=0",
0797                 G_server_conf.maxdispatchthreads);
0798         userlog("Error ! MAXDISPATCHTHREADS(=%d) <=0",
0799                 G_server_conf.maxdispatchthreads);
0800         ndrx_TPset_error_fmt(TPEINVAL, "Error ! MAXDISPATCHTHREADS(=%d) <=0",
0801                 G_server_conf.maxdispatchthreads);
0802         EXFAIL_OUT(ret);
0803     }
0804
0805     /* start as multi-threaded
0806      * thus we can run the MT mode operates with single thread too..
0807      * i.e. maxdispatchthreads=2, mindispatchthreads=1
0808      */
0809     if (G_server_conf.maxdispatchthreads > 1 && _tmbuilt_with_thread_option)
0810     {
0811         G_server_conf.is_threaded = EXTRUE;
0812         NDRX_SPIN_INIT_V(G_server_conf.mt_lock);
0813     }
0814
0815     G_srv_id = G_server_conf.srv_id;
0816
0817     /* Default number of events supported by e-poll */
0818     G_server_conf.max_events = 1;
0819
0820 #ifdef EX_USE_SYSVQ
0821     /* format the request queue - only for system v*/
0822     if (EXEOS==rqaddress[0])
0823     {
0824         /* so name not set, lets build per binary request address... */
0825         snprintf(rqaddress, sizeof(rqaddress), NDRX_SVR_SVADDR_FMT,
0826                 G_server_conf.q_prefix, G_server_conf.binary_name, G_srv_id);
0827
0828         ndrx_epoll_mainq_set(rqaddress);
0829         NDRX_STRCPY_SAFE(G_server_conf.rqaddress, rqaddress);
0830     }
0831     else
0832     {
0833         snprintf(tmp, sizeof(tmp), NDRX_SVR_RQADDR_FMT,
0834                 G_server_conf.q_prefix, rqaddress);
0835         ndrx_epoll_mainq_set(tmp);
0836         NDRX_STRCPY_SAFE(G_server_conf.rqaddress, tmp);
0837     }
0838 #endif
0839
0840 out:
0841
0842     ndrx_string_list_free(svcalias);
0843
0844     return ret;
0845 }
0846
0847 /**
0848  * terminate server session after fork in child process
0849  * as it is not valid there.
0850  */
0851 exprivate void childsrvuninit(void)
0852 {
0853     NDRX_LOG(log_debug, "Server un-init in forked child thread...");
0854     atmisrv_un_initialize(EXTRUE);
0855 }
0856
0857 /**
0858  * Wrapper for server thread done.
0859  * This calls the users' tpsvrthrdone. Additionally tpterm() is called
0860  * to terminate the thread ATMI session (basically a client)
0861  */
0862 exprivate void ndrx_call_tpsvrthrdone(void)
0863 {
0864     if (NULL!=ndrx_G_tpsvrthrdone)
0865     {
0866         ndrx_G_tpsvrthrdone();
0867     }
0868
0869     /* terminate the session */
0870     tpterm();
0871 }
0872
0873
0874 /**
0875  * Thread basic init. Performs initial tpinit.
0876  * @param argc command line argument count
0877  * @param argv cli arguments
0878  */
0879 exprivate int ndrx_call_tpsvrthrinit(int argc, char ** argv)
0880 {
0881     int ret = EXSUCCEED;
0882     int init_ok = EXFALSE;
0883
0884     /* start the session... */
0885     NDRX_LOG(log_info, "Starting new server dispatched thread");
0886     userlog("Starting new server dispatched thread");
0887
0888     if (EXSUCCEED!=tpinit(NULL))
0889     {
0890         EXFAIL_OUT(ret);
0891     }
0892
0893     init_ok = EXTRUE;
0894
0895     G_atmi_tls->pf_tpacall_noservice_hook = &ndrx_tpacall_noservice_hook_defer;
0896
0897     if (NULL!=ndrx_G_tpsvrthrinit
0898             && ndrx_G_tpsvrthrinit(argc, argv) < 0)
0899     {
0900         EXFAIL_OUT(ret);
0901     }
0902
0903     /* remove handler.. */
0904     G_atmi_tls->pf_tpacall_noservice_hook = NULL;
0905 out:
0906
0907     if (EXSUCCEED!=ret && init_ok)
0908     {
0909         tpterm();
0910     }
0911
0912     return ret;
0913 }
0914
0915 /**
0916  * Enqueue messages for later send
0917  * firstly search the service.
0918  * We need locking so that if doing advertise / unadvertise from server
0919  * threads
0920  * @param svc service name to call
0921  * @param data XATMI allocated buffer
0922  * @param len data len
0923  * @param flags tpacall flags
0924  * @return EXSUCCEED/EXFAIL
0925  */
0926 exprivate int ndrx_tpacall_noservice_hook_defer(char *svc, char *data, long len, long flags)
0927 {
0928     int ret = EXSUCCEED;
0929     svc_entry_fn_t *existing=NULL, eltmp;
0930     ndrx_tpacall_defer_t *call = NULL;
0931     int err;
0932
0933     NDRX_STRCPY_SAFE(eltmp.svc_nm, svc);
0934
0935     /* use the same advertise lock, the may not advertise, if msg is being defered */
0936     ndrx_sv_advertise_lock();
0937
0938     DL_SEARCH(G_server_conf.service_raw_list, existing, &eltmp, ndrx_svc_entry_fn_cmp);
0939
0940     /* OK add to linked list for reply... */
0941     if (!existing)
0942     {
0943         /* generate error as service not found */
0944         ndrx_TPset_error_fmt(TPENOENT, "%s: Service is not available %s by %s",
0945         __func__, svc, "server_init");
0946         EXFAIL_OUT(ret);
0947     }
0948
0949     /* duplicate the data and add to queue */
0950     call = NDRX_FPMALLOC(sizeof(ndrx_tpacall_defer_t), 0);
0951
0952     if (NULL==call)
0953     {
0954         err=errno;
0955         NDRX_LOG(log_error, "Failed to malloc %d bytes: %s", tpstrerror(err));
0956         ndrx_TPset_error_fmt(TPEOS, "%s: Service is not available %s by %s",
0957         __func__, svc, "server_init");
0958         EXFAIL_OUT(ret);
0959
0960     }
0961
0962     call->flags=flags;
0963     call->len=len;
0964     NDRX_STRCPY_SAFE(call->svcnm, svc);
0965
0966     if (NULL!=data)
0967     {
0968         char type[16+1]={EXEOS};
0969         char subtype[XATMI_SUBTYPE_LEN]={EXEOS};
0970         long xatmi_len;
0971         /* call->data */
0972
0973         xatmi_len=tptypes(data, type, subtype);
0974
0975         if (EXFAIL==xatmi_len)
0976         {
0977             NDRX_LOG(log_error, "Failed to get data type for defered tpacall buffer");
0978             EXFAIL_OUT(ret);
0979         }
0980
0981         call->data = tpalloc(type, subtype, xatmi_len);
0982
0983         if (NULL==call->data)
0984         {
0985             NDRX_LOG(log_error, "Failed to alloc defered msg data buf");
0986             EXFAIL_OUT(ret);
0987         }
0988
0989         /* copy full msg, no interpretation, for UBF we could make shorter
0990          * copy...
0991          */
0992         memcpy(call->data, data, xatmi_len);
0993
0994     }
0995     else
0996     {
0997         call->data = NULL;
0998     }
0999
1000     /* Add the MSG */
1001     NDRX_LOG(log_info, "Enqueue deferred tpacall svcnm=[%s] org_buf=%p "
1002             "buf=%p (copy) len=%ld flags=%ld",
1003             call->svcnm, data, call->data, call->len, call->flags);
1004
1005     DL_APPEND(M_deferred_tpacalls, call);
1006
1007 out:
1008
1009     if (EXSUCCEED!=ret)
1010     {
1011         /* delete any left overs... */
1012         if (NULL!=call)
1013         {
1014             if (NULL!=call->data)
1015             {
1016                 tpfree(call->data);
1017             }
1018
1019             NDRX_FPFREE(call);
1020         }
1021     }
1022
1023     ndrx_sv_advertise_unlock();
1024
1025     return ret;
1026 }
1027
1028 /**
1029  * Perform the sending of tpacall messages, once services are open...
1030  * and we are about to poll
1031  * @return EXSUCCEED/EXFAIL
1032  */
1033 exprivate int ndrx_tpacall_noservice_hook_send(void)
1034 {
1035     int ret=EXSUCCEED;
1036     ndrx_tpacall_defer_t *el, *elt;
1037
1038     DL_FOREACH_SAFE(M_deferred_tpacalls, el, elt)
1039     {
1040         NDRX_LOG(log_info, "Performing deferred tpacall svcnm=[%s] buf=%p len=%ld flags=%ld",
1041             el->svcnm, el->data, el->len, el->flags);
1042
1043         if (EXFAIL==tpacall(el->svcnm, el->data, el->len, el->flags))
1044         {
1045             NDRX_LOG(log_info, "Deferred tpacall failed (svcnm=[%s] buf=%p len=%ld flags=%ld): %s",
1046                 el->svcnm, el->data, el->len, el->flags, tpstrerror(tperrno));
1047             userlog("Deferred tpacall failed (svcnm=[%s] buf=%p len=%ld flags=%ld): %s",
1048                 el->svcnm, el->data, el->len, el->flags, tpstrerror(tperrno));
1049             EXFAIL_OUT(ret);
1050         }
1051
1052         if (NULL!=el->data)
1053         {
1054             tpfree(el->data);
1055         }
1056
1057         DL_DELETE(M_deferred_tpacalls, el);
1058         NDRX_FPFREE(el);
1059     }
1060
1061 out:
1062
1063     /* delete messages if for error something have left here */
1064     if (EXSUCCEED!=ret)
1065     {
1066         DL_FOREACH_SAFE(M_deferred_tpacalls, el, elt)
1067         {
1068             if (NULL!=el->data)
1069             {
1070                 tpfree(el->data);
1071             }
1072
1073             DL_DELETE(M_deferred_tpacalls, el);
1074             NDRX_FPFREE(el);
1075         }
1076     }
1077
1078     return ret;
1079 }
1080
1081 /**
1082  * Real processing starts here.
1083  * @param argc
1084  * @param argv
1085  * @return
1086  */
1087 int ndrx_main(int argc, char** argv)
1088 {
1089     int ret=EXSUCCEED;
1090     char *env_procname;
1091     char *env_clopt = NULL;
1092
1093     /* in case of argc/argv are empty, we shall attempt  */
1094
1095     if (argc<=1 || NULL==argv)
1096     {
1097         char *p;
1098         char *tok;
1099         int alloc_args = 0;
1100         /* try to lookup env variables */
1101
1102         /* well, for server process we need a real binary name
1103          * the env is just logical server process name
1104          * thus we have to use our macros here
1105          */
1106         env_procname = (char *)EX_PROGNAME;/* getenv(CONF_NDRX_SVPROCNAME); */
1107
1108         p = getenv(CONF_NDRX_SVCLOPT);
1109
1110         if (NULL==p)
1111         {
1112             NDRX_LOG(log_error, "%s: argc/argv are empty and %s/%s env vars not "
1113                     "present - missing server params", __func__,
1114                     CONF_NDRX_SVPROCNAME, CONF_NDRX_SVCLOPT);
1115             userlog("%s: argc/argv are empty and %s/%s env vars not "
1116                     "present - missing server params", __func__,
1117                     CONF_NDRX_SVPROCNAME, CONF_NDRX_SVCLOPT);
1118             ndrx_TPset_error_fmt(TPEINVAL, "%s: argc/argv are empty and %s/%s env vars not "
1119                     "present - missing server params", __func__,
1120                     CONF_NDRX_SVPROCNAME, CONF_NDRX_SVCLOPT);
1121             EXFAIL_OUT(ret);
1122         }
1123
1124         if (NULL==(env_clopt=NDRX_STRDUP(p)))
1125         {
1126             int err=errno;
1127
1128             NDRX_LOG(log_error, "%s: Failed to strdup: %s", __func__,
1129                     strerror(err));
1130             userlog("%s: Failed to strdup: %s", __func__,
1131                     strerror(err));
1132             ndrx_TPset_error_fmt(TPEOS, "%s: Failed to strdup: %s", __func__,
1133                     strerror(err));
1134             EXFAIL_OUT(ret);
1135         }
1136
1137         /* realloc some space */
1138         argv = NULL;
1139         REALLOC_CLOPT;
1140
1141         argc=1;
1142         argv[0] = env_procname;
1143
1144         tok = ndrx_strtokblk(env_clopt, NDRX_CMDLINE_SEP, NDRX_CMDLINE_QUOTES);
1145         while (NULL!=tok)
1146         {
1147             argc++;
1148
1149             if (argc > alloc_args)
1150             {
1151                 REALLOC_CLOPT;
1152             }
1153
1154             argv[argc-1] = tok;
1155
1156             /* Get next */
1157             tok = ndrx_strtokblk(NULL, NDRX_CMDLINE_SEP, NDRX_CMDLINE_QUOTES);
1158         }
1159
1160     }
1161
1162     /* do internal initialization, get configuration, request for admin q */
1163     if (EXSUCCEED!=ndrx_init(argc, argv))
1164     {
1165         NDRX_LOG(log_error, "ndrx_init() fail");
1166         userlog("ndrx_init() fail");
1167         EXFAIL_OUT(ret);
1168     }
1169
1170     /*
1171      * Initialize polling subsystem
1172      */
1173     if (EXSUCCEED!=ndrx_epoll_sys_init())
1174     {
1175         NDRX_LOG(log_error, "ndrx_epoll_sys_init() fail");
1176         userlog("ndrx_epoll_sys_init() fail");
1177         EXFAIL_OUT(ret);
1178     }
1179
1180
1181     /*
1182      * Initialize services, system..
1183      */
1184     if (NULL!=ndrx_G_tpsvrinit_sys && EXSUCCEED!=ndrx_G_tpsvrinit_sys(argc, argv))
1185     {
1186         NDRX_LOG(log_error, "tpsvrinit_sys() fail");
1187         userlog("tpsvrinit_sys() fail");
1188         EXFAIL_OUT(ret);
1189     }
1190
1191     /* hook up atmitls with tpacall() service not found callback */
1192
1193     G_atmi_tls->pf_tpacall_noservice_hook=&ndrx_tpacall_noservice_hook_defer;
1194
1195     /*
1196      * Initialize services
1197      */
1198     if (NULL!=G_tpsvrinit__ && EXSUCCEED!=G_tpsvrinit__(argc, argv))
1199     {
1200         NDRX_LOG(log_error, "tpsvrinit() fail");
1201         userlog("tpsvrinit() fail");
1202         EXFAIL_OUT(ret);
1203     }
1204
1205     /* unset hook...  */
1206     G_atmi_tls->pf_tpacall_noservice_hook = NULL;
1207
1208
1209     /* initialize the library - switch main thread to server ... */
1210     if (EXSUCCEED!=atmisrv_initialise_atmi_library())
1211     {
1212         NDRX_LOG(log_error, "initialise_atmi_library() fail");
1213         userlog("initialise_atmi_library() fail");
1214         EXFAIL_OUT(ret);
1215     }
1216
1217     /*
1218      * Run off thread init if any
1219      * We could provide noservice hook here too..
1220      * Needs to configure hook handler, but then somehow remove it..
1221      */
1222     if (G_server_conf.is_threaded)
1223     {
1224         NDRX_LOG(log_debug, "About to init dispatch thread pool");
1225         G_server_conf.dispthreads = ndrx_thpool_init(G_server_conf.mindispatchthreads,
1226                 &ret, ndrx_call_tpsvrthrinit, ndrx_call_tpsvrthrdone, argc, argv);
1227
1228         if (EXSUCCEED!=ret)
1229         {
1230             NDRX_LOG(log_error, "Thread pool init failure");
1231             EXFAIL_OUT(ret);
1232         }
1233     }
1234
1235     /*
1236      * Push the services out!
1237      */
1238     if (EXSUCCEED!=atmisrv_build_advertise_list())
1239     {
1240         NDRX_LOG(log_error, "tpsvrinit() fail");
1241         userlog("tpsvrinit() fail");
1242         EXFAIL_OUT(ret);
1243     }
1244
1245     /*
1246      * Open the queues
1247      */
1248     if (EXSUCCEED!=sv_open_queue())
1249     {
1250         NDRX_LOG(log_error, "sv_open_queue() fail");
1251         userlog("sv_open_queue() fail");
1252         EXFAIL_OUT(ret);
1253     }
1254
1255     /* Do lib updates after Q open... */
1256     if (EXSUCCEED!=tp_internal_init_upd_replyq(G_server_conf.service_array[1]->q_descr,
1257                 G_server_conf.service_array[1]->listen_q))
1258     {
1259         NDRX_LOG(log_error, "tp_internal_init_upd_replyq() fail");
1260         userlog("tp_internal_init_upd_replyq() fail");
1261         EXFAIL_OUT(ret);
1262     }
1263
1264     /* <Test point...> */
1265     if (NULL!=G_atmi_env.test_advertise_crash)
1266     {
1267         G_atmi_env.test_advertise_crash();
1268     }
1269     /* </Test point...> */
1270
1271     /* As we can run even without ndrxd, then we ignore the result of send op */
1272     report_to_ndrxd();
1273
1274     if (EXSUCCEED!=ndrx_atfork(NULL, NULL, childsrvuninit))
1275     {
1276         NDRX_LOG(log_error, "Failed to add atfork handler!");
1277         userlog("Failed to add atfork handler!");
1278         EXFAIL_OUT(ret);
1279     }
1280
1281     /* reply the linked list of any internal tpacall("X", TPNOREPLY)
1282      * so that infinite servers may start...
1283      */
1284     if (EXSUCCEED!=(ret=ndrx_tpacall_noservice_hook_send()))
1285     {
1286         NDRX_LOG(log_error, "ndrx_tpacall_noservice_hook_send() fail %d", ret);
1287         userlog("ndrx_tpacall_noservice_hook_send() fail %d", ret);
1288         goto out;
1289     }
1290
1291     /* run process here! */
1292     if (EXSUCCEED!=(ret=sv_wait_for_request()))
1293     {
1294         NDRX_LOG(log_error, "sv_wait_for_request() fail %d", ret);
1295         userlog("sv_wait_for_request() fail %d", ret);
1296         goto out;
1297     }
1298
1299 out:
1300
1301     /* finish up. */
1302     if (NULL!=G_tpsvrdone__)
1303     {
1304         G_tpsvrdone__();
1305     }
1306
1307     /* if thread pool was in place, then perform de-init... */
1308     if (NULL!=G_server_conf.dispthreads)
1309     {
1310         ndrx_thpool_destroy(G_server_conf.dispthreads);
1311     }
1312
1313     /*
1314      * un-initalize polling sub-system
1315      */
1316     ndrx_epoll_sys_uninit();
1317
1318     atmisrv_un_initialize(EXFALSE);
1319     /*
1320      * Print error message on exit.
1321      */
1322     if (EXSUCCEED!=ret)
1323     {
1324         printf("Error: %s\n", tpstrerror(tperrno));
1325     }
1326
1327     fprintf(stderr, "Server exit: %d, id: %d\n", ret, G_srv_id);
1328
1329     if (NULL!=env_clopt)
1330     {
1331         NDRX_FREE(env_clopt);
1332
1333         /* all pointers comes from other variables */
1334         if (NULL!=argv)
1335         {
1336             NDRX_FREE(argv);
1337         }
1338     }
1339
1340     return ret;
1341 }
1342
1343 /* vim: set ts=4 sw=4 et smartindent: */