用resty-logger模块在log_by介段，为什么收集到的日志结果数据少很多呢？和压测的请求数异同

winlia520

local cjson = require "cjson.safe"
local logger = require "resty.logger.socket"
local common = require "atomic.common"
local config = require "atomic.config"
local uri = ngx.var.uri
local get_req = ngx.req.get_uri_args()
local req_tvinfo = {}
 

--收集日志数据
config.collect_log_service_1 = {
    host = "127.0.0.1", --部署时,需要填写logserver服务器ip
    port = 6678, --部署时,需要填写logserver服务器port
    sock_type = "tcp", --采用tcp,udp传输
    flush_limit = 1, --1就是时实传输,
    --drop_limit = 5678, --默认1mb 超过会自动删除
    timeout = 10000, --超时设置
    pool_size = 200 --连接池大小
}


if uri == '/v1/dl' or uri == '/v1/dl/' then
    if not logger.initted() then
        local ok, err = logger.init(config.collect_log_service_1)
        if not ok then
            ngx.log(ngx.ERR, "failed to initialize the logger: ", err)
            return
        end
    end
    req_info['version'] = get_req["version"]

    local logstr = cjson.encode(req_info).."\n"
    local bytes, err = logger.log(logstr)
    if err then
        ngx.log(ngx.ERR, "failed to log message: ", err)
        return
    end

end

└─(10:58:01)──> wrk -t 32 -c 500 -d1m --latency "http://localhost:8083/v1/dl?version=2342353"

Running 1m test @ http://localhost:8083/v1/dl?version=2342353

32 threads and 500 connections

Thread Stats Avg Stdev Max +/- Stdev

Latency 51.57ms 76.84ms 763.69ms 85.67%

Req/Sec 722.67 421.50 18.52k 82.65%

Latency Distribution

50% 11.72ms

75% 71.42ms

90% 157.29ms

99% 326.16ms

1332575 requests in 1.00m, 219.79MB read 这里按道理我的日志文件应该有1332575条记录啊

Socket errors: connect 0, read 70, write 0, timeout 0

Requests/sec: 22170.55

Transfer/sec: 3.66MB

接收日志处理代码

local sock = assert(ngx.req.socket(true))

sock:settimeout(1000) -- one second timout

local data = "">

if not common.is_empty(data) then

ngx.log(ngx.ALERT, "服务端接收数据：", data) ，把日志数据写到日志文件中，实现发现日志文件并没有上面1332575 个请求日志，难道是哪里发包错吗？

local req_log_info = common.url_decode(data)

if req_log_info == nil then

--ngx.log(ngx.ERR, "服务端接收数据解析错误：", data)

return 0

end

return 1

else

ngx.log(ngx.ERR, "发包错误", data)

end

qleex001

```

--drop_limit = 5678, --默认1mb 超过会自动删除
```

logger模块不保证日志完整。超过1mb时日志会丢掉

在 2017年3月8日星期三 UTC+8上午11:13:24，tan winfan写道：

local cjson = require "cjson.safe" local logger = require "resty.logger.socket" local common = require "atomic.common" local config = require "atomic.config" local uri = ngx.var.uri local get_req = ngx.req.get_uri_args() local req_tvinfo = {} --收集日志数据 config.collect_log_service_1 = { host = "127.0.0.1", --部署时,需要填写logserver服务器ip port = 6678, --部署时,需要填写logserver服务器port sock_type = "tcp", --采用tcp,udp传输 flush_limit = 1, --1就是时实传输, --drop_limit = 5678, --默认1mb 超过会自动删除 timeout = 10000, --超时设置 pool_size = 200 --连接池大小 } if uri == '/v1/dl' or uri == '/v1/dl/' then if not logger.initted() then local ok, err = logger.init(config.collect_log_service_1) if not ok then ngx.log(ngx.ERR, "failed to initialize the logger: ", err) return end end req_info['version'] = get_req["version"] local logstr = cjson.encode(req_info).."\n" local bytes, err = logger.log(logstr) if err then ngx.log(ngx.ERR, "failed to log message: ", err) return end end

└─(10:58:01)──> wrk -t 32 -c 500 -d1m --latency "http://localhost:8083/v1/dl?version=2342353"
Running 1m test @ http://localhost:8083/v1/dl?version=2342353
32 threads and 500 connections
Thread Stats Avg Stdev Max +/- Stdev
Latency 51.57ms 76.84ms 763.69ms 85.67%
Req/Sec 722.67 421.50 18.52k 82.65%
Latency Distribution
50% 11.72ms
75% 71.42ms
90% 157.29ms
99% 326.16ms
1332575 requests in 1.00m, 219.79MB read 这里按道理我的日志文件应该有1332575条记录啊
Socket errors: connect 0, read 70, write 0, timeout 0
Requests/sec: 22170.55
Transfer/sec: 3.66MB

接收日志处理代码

local sock = assert(ngx.req.socket(true))
sock:settimeout(1000) -- one second timout
local data = "">

if not common.is_empty(data) then
ngx.log(ngx.ALERT, "服务端接收数据：", data) ，把日志数据写到日志文件中，实现发现日志文件并没有上面1332575 个请求日志，难道是哪里发包错吗？
local req_log_info = common.url_decode(data)
if req_log_info == nil then
--ngx.log(ngx.ERR, "服务端接收数据解析错误：", data)
return 0
end
return 1
else
ngx.log(ngx.ERR, "发包错误", data)
end

winlia520

怎么让它不丢掉呢？

在 2017年3月8日星期三 UTC+8上午11:13:24，tan winfan写道：

local cjson = require "cjson.safe" local logger = require "resty.logger.socket" local common = require "atomic.common" local config = require "atomic.config" local uri = ngx.var.uri local get_req = ngx.req.get_uri_args() local req_tvinfo = {} --收集日志数据 config.collect_log_service_1 = { host = "127.0.0.1", --部署时,需要填写logserver服务器ip port = 6678, --部署时,需要填写logserver服务器port sock_type = "tcp", --采用tcp,udp传输 flush_limit = 1, --1就是时实传输, --drop_limit = 5678, --默认1mb 超过会自动删除 timeout = 10000, --超时设置 pool_size = 200 --连接池大小 } if uri == '/v1/dl' or uri == '/v1/dl/' then if not logger.initted() then local ok, err = logger.init(config.collect_log_service_1) if not ok then ngx.log(ngx.ERR, "failed to initialize the logger: ", err) return end end req_info['version'] = get_req["version"] local logstr = cjson.encode(req_info).."\n" local bytes, err = logger.log(logstr) if err then ngx.log(ngx.ERR, "failed to log message: ", err) return end end

└─(10:58:01)──> wrk -t 32 -c 500 -d1m --latency "http://localhost:8083/v1/dl?version=2342353"
Running 1m test @ http://localhost:8083/v1/dl?version=2342353
32 threads and 500 connections
Thread Stats Avg Stdev Max +/- Stdev
Latency 51.57ms 76.84ms 763.69ms 85.67%
Req/Sec 722.67 421.50 18.52k 82.65%
Latency Distribution
50% 11.72ms
75% 71.42ms
90% 157.29ms
99% 326.16ms
1332575 requests in 1.00m, 219.79MB read 这里按道理我的日志文件应该有1332575条记录啊
Socket errors: connect 0, read 70, write 0, timeout 0
Requests/sec: 22170.55
Transfer/sec: 3.66MB

接收日志处理代码

local sock = assert(ngx.req.socket(true))
sock:settimeout(1000) -- one second timout
local data = "">

if not common.is_empty(data) then
ngx.log(ngx.ALERT, "服务端接收数据：", data) ，把日志数据写到日志文件中，实现发现日志文件并没有上面1332575 个请求日志，难道是哪里发包错吗？
local req_log_info = common.url_decode(data)
if req_log_info == nil then
--ngx.log(ngx.ERR, "服务端接收数据解析错误：", data)
return 0
end
return 1
else
ngx.log(ngx.ERR, "发包错误", data)
end

doujiang24

Hello

这个设计是有取舍的，要保证一定不丢就得允许返回发送失败，由更上层来处理

具体你可以看下代码的实现

2017-03-08 14:48 GMT+08:00 tan winfan <winl...@gmail.com>:

怎么让它不丢掉呢？

在 2017年3月8日星期三 UTC+8上午11:13:24，tan winfan写道：

local cjson = require "cjson.safe" local logger = require "resty.logger.socket" local common = require "atomic.common" local config = require "atomic.config" local uri = ngx.var.uri local get_req = ngx.req.get_uri_args() local req_tvinfo = {} --收集日志数据 config.collect_log_service_1 = { host = "127.0.0.1", --部署时,需要填写logserver服务器ip port = 6678, --部署时,需要填写logserver服务器port sock_type = "tcp", --采用tcp,udp传输 flush_limit = 1, --1就是时实传输, --drop_limit = 5678, --默认1mb 超过会自动删除 timeout = 10000, --超时设置 pool_size = 200 --连接池大小 } if uri == '/v1/dl' or uri == '/v1/dl/' then if not logger.initted() then local ok, err = logger.init(config.collect_log_service_1) if not ok then ngx.log(ngx.ERR, "failed to initialize the logger: ", err) return end end req_info['version'] = get_req["version"] local logstr = cjson.encode(req_info).."\n" local bytes, err = logger.log(logstr) if err then ngx.log(ngx.ERR, "failed to log message: ", err) return end end

└─(10:58:01)──> wrk -t 32 -c 500 -d1m --latency "http://localhost:8083/v1/dl?version=2342353"
Running 1m test @ http://localhost:8083/v1/dl?version=2342353
32 threads and 500 connections
Thread Stats Avg Stdev Max +/- Stdev
Latency 51.57ms 76.84ms 763.69ms 85.67%
Req/Sec 722.67 421.50 18.52k 82.65%
Latency Distribution
50% 11.72ms
75% 71.42ms
90% 157.29ms
99% 326.16ms
1332575 requests in 1.00m, 219.79MB read 这里按道理我的日志文件应该有1332575条记录啊
Socket errors: connect 0, read 70, write 0, timeout 0
Requests/sec: 22170.55
Transfer/sec: 3.66MB

接收日志处理代码

local sock = assert(ngx.req.socket(true))
sock:settimeout(1000) -- one second timout
local data = "">

if not common.is_empty(data) then
ngx.log(ngx.ALERT, "服务端接收数据：", data) ，把日志数据写到日志文件中，实现发现日志文件并没有上面1332575 个请求日志，难道是哪里发包错吗？
local req_log_info = common.url_decode(data)
if req_log_info == nil then
--ngx.log(ngx.ERR, "服务端接收数据解析错误：", data)
return 0
end
return 1
else
ngx.log(ngx.ERR, "发包错误", data)
end

--