lua-resty-upload接收音频文件，出现每个音频文件丢失几百KB的现象

jzmskygm · 2016-10-28T14:16:32+00:00

更新：增加了proxy_cache_valid 200 304 60m; 指令后，debug看到确实命...

lua-resty-upload接收音频文件，出现每个音频文件丢失几百KB的现象

jzmskygm

我在使用lua-resty-upload模块实现文件上传的时候，为了确保服务端接收到的文件的完整性，需要客户端在上传文件的时候带上该文件的md5值（对整个文件计算md5值），然后服务端接收到文件后再计算一次md5，与客户端传过来的md5做比较看是否一致，以此来确保接收到的文件的完整性。现在是测试环境未发现问题，而生产环境高频出现md5不一致的现象。生产环境是日均50万次的请求。以下是我上传函数的代码：

local upload = require "resty.upload"

local json = require "cjson.safe"

local config = require("lapis.config").get()

local chunk_size = config.chunk_size

local start_upload = false

local data_upload = false

local content_type = ''

local _M = {}

function _M:uploadfile(s3, filepath, file_md5)

local form, err = upload:new(chunk_size)

if not form then

ngx.log(ngx.ERR, "failed to new upload: ", err)

return false

end

form:set_timeout(config.timeout) --ms

local appender = {}

local n = 1

local url, object_err, filename

while true do

local typ, res, err = form:read()

if not typ then

ngx.log(ngx.ERR, "failed to read: ", err)

return false

end

if typ == "header" then

ngx.log(ngx.NOTICE, "header: ", json.encode(res))

content_type = res[2]

ngx.log(ngx.NOTICE, "content_type: ", content_type)

if content_type then

start_upload = true

end

local inputFile = [[name="input"]]

if res[1]=='Content-Disposition' and ngx.re.find(res[2] or '', inputFile) then

data_upload = true

end

elseif typ == "body" then

if start_upload == true and res then

appender[n] = res

n = n+1

end

if data_upload == true and res then

data_upload = false

end

elseif typ == "part_end" then

local content = table.concat(appender)

if content ~= '' then

local data_len = string.len(content)

ngx.log(ngx.NOTICE,"start to upload ", content_type, " len=" , data_len)

if data_len>20 then

filename = ngx.md5(content)

if filename ~= file_md5 then

ngx.log(ngx.ERR, "filename ~= file_md5, filename ="..filename, "failed file length="..data_len)

return false

end

url, object_err = s3:upload(content, content_type, filepath..filename)

if url then

local show = json.encode({status = "0"; url = "">

ngx.log(ngx.NOTICE, "success : ", show)

else

ngx.log(ngx.ERR, "request return : ", object_err)

return false

end

else

local reason = " body too short " .. content_type

ngx.log(ngx.NOTICE, reason)

return false

end

content = nil

appender = nil

n = 1

start_upload = false

end

elseif typ == "eof" then

break

end

return true, url, filename

end

return _M

此问题已经困扰很久，不胜感激。

jzmskygm

服务器位于新加坡，客户端为安卓应用。上传的均为音频文件，大概会出现三分之一以上上传失败的情况。

在 2016年10月28日星期五 UTC+8下午2:16:35，Tom Cluse写道：

我在使用lua-resty-upload模块实现文件上传的时候，为了确保服务端接收到的文件的完整性，需要客户端在上传文件的时候带上该文件的md5值（对整个文件计算md5值），然后服务端接收到文件后再计算一次md5，与客户端传过来的md5做比较看是否一致，以此来确保接收到的文件的完整性。现在是测试环境未发现问题，而生产环境高频出现md5不一致的现象。生产环境是日均50万次的请求。以下是我上传函数的代码：
local upload = require "resty.upload"
local json = require "cjson.safe"
local config = require("lapis.config").get()
local chunk_size = config.chunk_size
local start_upload = false
local data_upload = false
local content_type = ''

local _M = {}

function _M:uploadfile(s3, filepath, file_md5)
local form, err = upload:new(chunk_size)
if not form then
ngx.log(ngx.ERR, "failed to new upload: ", err)
return false
end
form:set_timeout(config.timeout) --ms

local appender = {}
local n = 1
local url, object_err, filename

while true do
local typ, res, err = form:read()
if not typ then
ngx.log(ngx.ERR, "failed to read: ", err)
return false
end
if typ == "header" then
ngx.log(ngx.NOTICE, "header: ", json.encode(res))
content_type = res[2]
ngx.log(ngx.NOTICE, "content_type: ", content_type)
if content_type then
start_upload = true
end
local inputFile = [[name="input"]]
if res[1]=='Content-Disposition' and ngx.re.find(res[2] or '', inputFile) then
data_upload = true
end
elseif typ == "body" then
if start_upload == true and res then
appender[n] = res
n = n+1
end
if data_upload == true and res then
data_upload = false
end
elseif typ == "part_end" then
local content = table.concat(appender)
if content ~= '' then
local data_len = string.len(content)
ngx.log(ngx.NOTICE,"start to upload ", content_type, " len=" , data_len)
if data_len>20 then
filename = ngx.md5(content)
if filename ~= file_md5 then
ngx.log(ngx.ERR, "filename ~= file_md5, filename ="..filename, "failed file length="..data_len)
return false
end
url, object_err = s3:upload(content, content_type, filepath..filename)
if url then
local show = json.encode({status = "0"; url = "">
ngx.log(ngx.NOTICE, "success : ", show)
else
ngx.log(ngx.ERR, "request return : ", object_err)
return false
end
else
local reason = " body too short " .. content_type
ngx.log(ngx.NOTICE, reason)
return false
end
content = nil
appender = nil
n = 1
start_upload = false
end
elseif typ == "eof" then
break
end
end
return true, url, filename
end

return _M

此问题已经困扰很久，不胜感激。

jzmskygm

上传文件时用的是HTTP POST，基于TCP协议，为可靠传输，丢包会有重发机制，为什么还会出现接收到的文件不完整的现象呢？

在 2016年10月28日星期五 UTC+8下午2:16:35，Tom Cluse写道：

我在使用lua-resty-upload模块实现文件上传的时候，为了确保服务端接收到的文件的完整性，需要客户端在上传文件的时候带上该文件的md5值（对整个文件计算md5值），然后服务端接收到文件后再计算一次md5，与客户端传过来的md5做比较看是否一致，以此来确保接收到的文件的完整性。现在是测试环境未发现问题，而生产环境高频出现md5不一致的现象。生产环境是日均50万次的请求。以下是我上传函数的代码：
local upload = require "resty.upload"
local json = require "cjson.safe"
local config = require("lapis.config").get()
local chunk_size = config.chunk_size
local start_upload = false
local data_upload = false
local content_type = ''

local _M = {}

function _M:uploadfile(s3, filepath, file_md5)
local form, err = upload:new(chunk_size)
if not form then
ngx.log(ngx.ERR, "failed to new upload: ", err)
return false
end
form:set_timeout(config.timeout) --ms

local appender = {}
local n = 1
local url, object_err, filename

while true do
local typ, res, err = form:read()
if not typ then
ngx.log(ngx.ERR, "failed to read: ", err)
return false
end
if typ == "header" then
ngx.log(ngx.NOTICE, "header: ", json.encode(res))
content_type = res[2]
ngx.log(ngx.NOTICE, "content_type: ", content_type)
if content_type then
start_upload = true
end
local inputFile = [[name="input"]]
if res[1]=='Content-Disposition' and ngx.re.find(res[2] or '', inputFile) then
data_upload = true
end
elseif typ == "body" then
if start_upload == true and res then
appender[n] = res
n = n+1
end
if data_upload == true and res then
data_upload = false
end
elseif typ == "part_end" then
local content = table.concat(appender)
if content ~= '' then
local data_len = string.len(content)
ngx.log(ngx.NOTICE,"start to upload ", content_type, " len=" , data_len)
if data_len>20 then
filename = ngx.md5(content)
if filename ~= file_md5 then
ngx.log(ngx.ERR, "filename ~= file_md5, filename ="..filename, "failed file length="..data_len)
return false
end
url, object_err = s3:upload(content, content_type, filepath..filename)
if url then
local show = json.encode({status = "0"; url = "">
ngx.log(ngx.NOTICE, "success : ", show)
else
ngx.log(ngx.ERR, "request return : ", object_err)
return false
end
else
local reason = " body too short " .. content_type
ngx.log(ngx.NOTICE, reason)
return false
end
content = nil
appender = nil
n = 1
start_upload = false
end
elseif typ == "eof" then
break
end
end
return true, url, filename
end

return _M

此问题已经困扰很久，不胜感激。

doujiang24

Hello

你有检查过 nginx 的 error_log ？

既然失败率这么高，用 tcpdump 抓包看看？

在 2016年10月28日下午2:38， <jzm...@gmail.com>写道：

上传文件时用的是HTTP POST，基于TCP协议，为可靠传输，丢包会有重发机制，为什么还会出现接收到的文件不完整的现象呢？

在 2016年10月28日星期五 UTC+8下午2:16:35，Tom Cluse写道：
我在使用lua-resty-upload模块实现文件上传的时候，为了确保服务端接收到的文件的完整性，需要客户端在上传文件的时候带上该文件的md5值（对整个文件计算md5值），然后服务端接收到文件后再计算一次md5，与客户端传过来的md5做比较看是否一致，以此来确保接收到的文件的完整性。现在是测试环境未发现问题，而生产环境高频出现md5不一致的现象。生产环境是日均50万次的请求。以下是我上传函数的代码：
local upload = require "resty.upload"
local json = require "cjson.safe"
local config = require("lapis.config").get()
local chunk_size = config.chunk_size
local start_upload = false
local data_upload = false
local content_type = ''

local _M = {}

function _M:uploadfile(s3, filepath, file_md5)
local form, err = upload:new(chunk_size)
if not form then
ngx.log(ngx.ERR, "failed to new upload: ", err)
return false
end
form:set_timeout(config.timeout) --ms

local appender = {}
local n = 1
local url, object_err, filename

while true do
local typ, res, err = form:read()
if not typ then
ngx.log(ngx.ERR, "failed to read: ", err)
return false
end
if typ == "header" then
ngx.log(ngx.NOTICE, "header: ", json.encode(res))
content_type = res[2]
ngx.log(ngx.NOTICE, "content_type: ", content_type)
if content_type then
start_upload = true
end
local inputFile = [[name="input"]]
if res[1]=='Content-Disposition' and ngx.re.find(res[2] or '', inputFile) then
data_upload = true
end
elseif typ == "body" then
if start_upload == true and res then
appender[n] = res
n = n+1
end
if data_upload == true and res then
data_upload = false
end
elseif typ == "part_end" then
local content = table.concat(appender)
if content ~= '' then
local data_len = string.len(content)
ngx.log(ngx.NOTICE,"start to upload ", content_type, " len=" , data_len)
if data_len>20 then
filename = ngx.md5(content)
if filename ~= file_md5 then
ngx.log(ngx.ERR, "filename ~= file_md5, filename ="..filename, "failed file length="..data_len)
return false
end
url, object_err = s3:upload(content, content_type, filepath..filename)
if url then
local show = json.encode({status = "0"; url = "">
ngx.log(ngx.NOTICE, "success : ", show)
else
ngx.log(ngx.ERR, "request return : ", object_err)
return false
end
else
local reason = " body too short " .. content_type
ngx.log(ngx.NOTICE, reason)
return false
end
content = nil
appender = nil
n = 1
start_upload = false
end
elseif typ == "eof" then
break
end
end
return true, url, filename
end

return _M

此问题已经困扰很久，不胜感激。

--