we have implemented nginx caching of static (large) files using openresty bundle by leverage lua to to divide the request into sequnce of chunks (4MB) for efficient caching and storing these chunks in the cache, everything worked fine but i noticed the memory usage for nginx keeps growing until the server is running out of memory, i tried using some optimizations but still facing the problem, the key issue in my belief is that the users are downloading really large files (up to 20GB) which makes each request divided into large itterations number which might reserve alot of memory, but the requests are not high (up to 2000) but the download speed in not very high, anyone can point out what is causing this !
-- includes
local http = require("resty.http") -- https://github.com/liseen/lua-resty-http
local cjson = require("cjson")
local bslib = require("bitset") -- https://github.com/bsm/bitset.lua
-- basic configuration
local block_size = 4*1024*1024 -- Block size 256k
local backend = "http://127.0.0.1:8080/" -- backend
local fcttl = 300 -- Time to cache HEAD requests
local bypass_headers = {
["expires"] = "Expires",
["content-type"] = "Content-Type",
["last-modified"] = "Last-Modified",
["expires"] = "Expires",
["cache-control"] = "Cache-Control",
["server"] = "Server",
["content-length"] = "Content-Length",
["p3p"] = "P3P",
["accept-ranges"] = "Accept-Ranges",
["content-disposition"] = "Content-Disposition"
}
local httpc = http.new()
local file_dict = ngx.shared.file_dict
local query = ""
if ngx.var.query_string == nil then
query = ""
else
query = ngx.var.query_string
end
local start = 0
local stop = -1
ngx.status = 206 -- Default HTTP status
-- register on_abort callback
local ok, err = ngx.on_abort(function ()
ngx.log(ngx.INFO, "aborting" )
file_dict = nill
httpc = nil
collectgarbage("collect")
ngx.exit(499)
end)
if not ok then
ngx.err(ngx.LOG, "Can't register on_abort function.")
ngx.exit(500)
end
-- try reading values from dict, if not issue a HEAD request and save the value
local updating, flags = file_dict:get(ngx.var.uri .. "-update")
local retry = 0
repeat
updating, flags = file_dict:get(ngx.var.uri .. "-update")
ngx.sleep(0.1)
retry = retry +1
until not updating or retry > 1000
local origin_headers = {}
local origin_info = file_dict:get(ngx.var.uri .. "-info")
if not origin_info then
file_dict:set(ngx.var.uri .. "-update", true, 5)
local ok, code, headers, status, body = httpc:request {
url = "" .. ngx.var.uri .. "?" .. query ,
method = 'HEAD'
}
if code == 200 then
--ngx.log(ngx.INFO, "storing data" .. code )
for key, value in pairs(bypass_headers) do
origin_headers[value] = headers[key]
end
origin_info = cjson.encode(origin_headers)
file_dict:set(ngx.var.uri .. "-info", origin_info, fcttl)
else
ngx.exit(500)
end
file_dict:delete(ngx.var.uri .. "-update")
end
origin_headers = cjson.decode(origin_info)
-- parse range header
local range_header = ngx.req.get_headers()["Range"] or "bytes=0-"
local matches, err = ngx.re.match(range_header, "^bytes=(\\d+)?-([^\\\\]\\d+)?", "joi")
if matches then
if matches[1] == nil and matches[2] then
stop = (origin_headers["Content-Length"] - 1)
start = (stop - matches[2]) + 1
else
start = matches[1] or 0
stop = matches[2] or (origin_headers["Content-Length"] - 1)
end
else
stop = (origin_headers["Content-Length"] - 1)
end
for header, value in pairs(origin_headers) do
ngx.header[header] = value
end
local cl = origin_headers["Content-Length"]
ngx.header["Content-Length"] = (stop - (start - 1))
ngx.header["Content-Range"] = "bytes " .. start .. "-" .. stop .. "/" .. cl
local block_stop = (math.ceil(stop / block_size) * block_size)
local block_start = (math.floor(start / block_size) * block_size)
ngx.send_headers()
origin_headers = nil
bypass_headers = nil
local indx = 1
-- fetch the content from the backend
for block_range_start = block_start, stop, block_size do
local block_range_stop = (block_range_start + block_size) - 1
local block_id = (math.floor(block_range_start / block_size))
local content_start = 0
local content_stop = -1
local req_params = {
url = "" .. ngx.var.uri .. "?" .. query ,
method = 'GET',
headers = {
Range = "bytes=" .. block_range_start .. "-" .. block_range_stop,
}
}
req_params["body_callback"] = function(data, chunked_header, ...)
if chunked_header then ngx.log(ngx.INFO, "ending" )
return end
ngx.print(data)
ngx.flush(true)
data = "">
end
if block_range_start == block_start then
req_params["body_callback"] = nil
content_start = (start - block_range_start)
end
if (block_range_stop + 1) == block_stop then
req_params["body_callback"] = nil
content_stop = (stop - block_range_start) +1
end
local ok, code, headers, status, body = httpc:request(req_params)
if body then
ngx.print(string.sub(body, (content_start) +1, content_stop)) -- lua count from 1
ngx.flush(true) -- not sure if this is equired
body = nil -- trying to reduce memory usage
end
indx = indx + 1 --for each 10 parts server clean the garbage
if indx%10 == 0 then collectgarbage("collect") end
end
file_dict = nill
httpc = nil
collectgarbage("collect")
ngx.eof()
return ngx.exit(206)
