geoip模块引起的status的waiting持续增加问题

41346471

我在网上找到一段代码，附加到ngx_http_geoip_module.c中，能够正常解析地理信息，但是在高并发的时候，查看nginx的status模块，在半小时的时候waiting值一直保持在3000左右，大约50分钟后，waiting将会持续上升，不会释放链接，哪怕lvs关掉这个waiting值也会一直那在不会降低。如果我把getIpAddress中的index屏蔽掉，给定一个值，waiting值一直保持在3000左右；如果我把getIpAddress中的所有readArea和readSp屏蔽掉，waiting也值一直保持在3000左右。麻烦大家给看看到底哪块出了问题。Ip库用的是纯真IP库qqwry.dat
static GeoIPRecord *
ngx_http_geoip_get_city_record(ngx_http_request_t *r)
{
     GeoIPRecord    *record;

    char country[20]="西安";

    char area[50]="北京";

    ngx_http_geoip_conf_t *gcf;

    gcf = ngx_http_get_module_main_conf(r, ngx_http_geoip_module);

    if (gcf->city) {
            record = malloc(sizeof(GeoIPRecord));
            memset(record, 0, sizeof(GeoIPRecord));


            iplocater("/usr/echo1/qqwry0929.dat");
            getIpAddress(ngx_http_geoip_addr(r), country, area);

            record->region = malloc(strlen((char *)country)+1);

            strncpy(record->region, (char *) country, strlen((char *)country) + 1);


    if (strlen((char *)area)>0){
            record->city = malloc(strlen((char *)area)+1);
            strncpy(record->city, (char *) area, strlen((char *)area) + 1);
    }
     return record;
        }

    return NULL;
}
void iplocater(const char *ipFileName)
{

    if(mdq==0)
    {char sFirstIndex[5] = {0};
    char sLastIndex[5] = {0};
    int iShmSize;
    struct stat file_info;


    int fd = open(ipFileName, O_RDONLY);
    if (fd < 0)
    {
    printf("Error stating file\n");
    }

    int iRet = stat(ipFileName, &file_info);
    if (iRet != 0)
    {
    printf("Error stating file\n");
    }

    iShmSize = file_info.st_size;


    m_ipbuff= (char*) malloc(iShmSize);
    int nbyte = read(fd, (void*)m_ipbuff, iShmSize);
    if (nbyte != iShmSize)
    {
        printf( "stat[%d]|read[%d]读到的字符数与文件大小不一致", iShmSize, nbyte);
    }

    memcpy((void*)&m_iFirstIndex, m_ipbuff, 4);
    memcpy((void*)&m_iLastIndex, m_ipbuff + 4, 4);
    mdq=1;
    close(fd);}

}

int binary_find(unsigned int ip, int left, int right)
{
    if ( right-left == 1)
    {
        return left;
    }
    else
    {
        int middle = (left + right) / 2;

        int offset = m_iFirstIndex + middle * INDEX_RECORD_SIZE;
        unsigned int new_ip = 0;
        memcpy((void*)&new_ip, m_ipbuff + offset, 4);

        if ( ip >= new_ip ) {
            return binary_find(ip,middle,right);
        }else{
            return binary_find(ip,left,middle);
        }
    }
}

int getIpAddress(unsigned int uip, char *area, char *sp)
{


    int record_num = (m_iLastIndex - m_iFirstIndex) / INDEX_RECORD_SIZE + 1;
    int index = binary_find(uip, 0, record_num - 1);
    int index_offset = m_iFirstIndex + index * INDEX_RECORD_SIZE + 4;
    int addr_offset = 0;
//    printf("index=%d\n",index);
    memcpy((void*)&addr_offset, m_ipbuff + index_offset, 3);

    int iRedirectMode = 0;
    char cRedirectMode;
    cRedirectMode = *(m_ipbuff + addr_offset + 4);

    int iCountryOffset = 0;
    if (cRedirectMode == REDIRECT_MODE_1)
    {
        memcpy((void*)&iCountryOffset, m_ipbuff+addr_offset + 5, 3);

        char byte = *(m_ipbuff + iCountryOffset);
        if (byte == REDIRECT_MODE_2)
        {
            int iCtOf = 0;
            memcpy((void*)&iCtOf, m_ipbuff + iCountryOffset + 1, 3);
            readArea(iCtOf, area);
            readSp(iCountryOffset + 4, sp);
            printf("1\n");
        }
        else
        {
            readArea(iCountryOffset, area);
            int newoffset = iCountryOffset + strlen(area) + 1;
            readSp(newoffset, sp);
printf("2\n");
        }
    }
    else if (cRedirectMode == REDIRECT_MODE_2)
    {
        memcpy((void*)&iCountryOffset, m_ipbuff + addr_offset + 5, 3);
        readArea(iCountryOffset, area);
        readSp(addr_offset + 8, sp);
printf("3\n");
    }
    else
    {
        readArea(addr_offset + 4, area);
        int newoffset = addr_offset + 4 + strlen(area) + 1;
        readSp(newoffset, sp);
    printf("4\n");
    }
    //free(m_ipbuff);
    return 0;
}

int readArea(const int offset, char *sArea)
{
    int len = 0;
    char sCountry[50] = {0};

    char ch = *(m_ipbuff + offset);
    while ( ch != 0 && ch != EOF )
    {
        ch = *(m_ipbuff + offset + len);
        sCountry[len] = ch;
        len++;
    }
    strncpy(sArea, sCountry, len);

    return 0;
}

int readSp(const int offset, char *sSp)
{
    char b = *(m_ipbuff + offset);

    if ( b == REDIRECT_MODE_1 || b == REDIRECT_MODE_2)
    {
        int iSpOffset=0;
        memcpy((void*)&iSpOffset, m_ipbuff + offset + 1, 3);

        if ( iSpOffset)
        {
            readArea(iSpOffset, sSp);
        }
        else
        {
            strncpy(sSp, "Unkown", 6);
        }
    }
    else
    {
        readArea(offset, sSp);
    }

    return 0;
}

agentzh

Hello!

2012/10/11 lanxin:
> 我在网上找到一段代码，附加到ngx_http_geoip_module.c中，能够正常解析地理信息，但是在高并发的时候，查看nginx的status模块，在半小时的时候waiting值一直保持在3000左右，大约50分钟后，waiting将会持续上升，

waiting 持续无续走高的一个常见情形是你的 nginx worker 进程持续崩溃，从而导致 waiting
等全局计数器失准。你从网上找的代码很可能是导致崩溃的主要原因，但你需要用 gdb 或者 valgrind 等工具加以确认。

建议首先通过 ps 这样的工具简单检查一下你的各个 nginx worker 进程的生命期是否正常。

Best regards,
-agentzh

41346471

查看error.log发现经常出现 29559#0：worker process 30811 exited on signal 11

我想让系统产生core文件，来查找原因。在网上搜了下设置：

ulimit -c unlimited

echo "1" > /proc/sys/kernel/core_uses_pid

echo "/tmp/core-%e-%p" > core_pattern

当error.log又出现worker process 30811 exited on signal 11这样错误的时候，

用命令 find / -name core.*

但是却找不到对应的core文件

在 2012年10月12日星期五UTC+8上午2时13分14秒，agentzh写道：

Hello!
2012/10/11 lanxin:
> 我在网上找到一段代码，附加到ngx_http_geoip_module.c中，能够正常解析地理信息，但是在高并发的时候，查看nginx的status模块，在半小时的时候waiting值一直保持在3000左右，大约50分钟后，waiting将会持续上升，
waiting 持续无续走高的一个常见情形是你的 nginx worker 进程持续崩溃，从而导致 waiting
等全局计数器失准。你从网上找的代码很可能是导致崩溃的主要原因，但你需要用 gdb 或者 valgrind 等工具加以确认。
建议首先通过 ps 这样的工具简单检查一下你的各个 nginx worker 进程的生命期是否正常。
Best regards,
-agentzh

41346471

之前的问题已经解决了，但是现在使用gdb ./nginx core.4367的时候，总是无法给出真正引起错误的地方，上面总是提示缺少很多debugging symbols，最后总是“？？“
可是我明明在nginx编译的时候，已经在/auto/cc/conf中加了-g了啊

GNU gdb (GDB) Red Hat Enterprise Linux (7.0.1-32.el5)
Copyright (C) 2009 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /data/nginx/sbin/nginx...done.
BFD: Warning: /data/nginx/sbin/core.7367 is truncated: expected core file size >= 68366336, found: 13443072.
Reading symbols from /lib64/libpthread.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib64/libpthread.so.0
Reading symbols from /lib64/libcrypt.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcrypt.so.1
Reading symbols from /lib64/libm.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libm.so.6
Reading symbols from /lib64/libssl.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libssl.so.6
Reading symbols from /lib64/libcrypto.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcrypto.so.6
Reading symbols from /lib64/libdl.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libdl.so.2
Reading symbols from /usr/lib64/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libz.so.1
Reading symbols from /usr/local/lib/libGeoIP.so.1...done.
Loaded symbols for /usr/local/lib/libGeoIP.so.1
Reading symbols from /lib64/libc.so.6...(no debugging symbols found)...done.
Loaded symbols for /lib64/libc.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/lib64/libgssapi_krb5.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libgssapi_krb5.so.2
Reading symbols from /usr/lib64/libkrb5.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libkrb5.so.3
Reading symbols from /lib64/libcom_err.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libcom_err.so.2
Reading symbols from /usr/lib64/libk5crypto.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libk5crypto.so.3
Reading symbols from /usr/lib64/libkrb5support.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib64/libkrb5support.so.0
Reading symbols from /lib64/libkeyutils.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libkeyutils.so.1
Reading symbols from /lib64/libresolv.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libresolv.so.2
Reading symbols from /lib64/libselinux.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libselinux.so.1
Reading symbols from /lib64/libsepol.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib64/libsepol.so.1
Reading symbols from /lib64/libnss_files.so.2...(no debugging symbols found)...done.
Loaded symbols for /lib64/libnss_files.so.2
Failed to read a valid object file image from memory.
Core was generated by `nginx: w'.
Program terminated with signal 11, Segmentation fault.
#0 0x000000000000b88f in ?? ()

zhuzhaoyuan

Hi，

你的代码里好多内存泄露。

尽量不要动态分配内存；如果确实要动态分配内存，尽量使用nginx的内存池；如果没法使用nginx的内存池，记得malloc出来的内存要及时free。

2012/10/11 lanxin <413...@qq.com>

我在网上找到一段代码，附加到ngx_http_geoip_module.c中，能够正常解析地理信息，但是在高并发的时候，查看nginx的status模块，在半小时的时候waiting值一直保持在3000左右，大约50分钟后，waiting将会持续上升，不会释放链接，哪怕lvs关掉这个waiting值也会一直那在不会降低。如果我把getIpAddress中的index屏蔽掉，给定一个值，waiting值一直保持在3000左右；如果我把getIpAddress中的所有readArea和readSp屏蔽掉，waiting也值一直保持在3000左右。麻烦大家给看看到底哪块出了问题。Ip库用的是纯真IP库qqwry.dat

Regards，

--
Joshua Zhu
Senior Software Engineer
Server Platforms Team at Taobao