log data expires now

2 files changed, 53 insertions, 18 deletions

diff --git a/src/daq/nginx-lua/s5-nginx-init.lua b/src/daq/nginx-lua/s5-nginx-init.lua
index 9d6db84..d5bc6a7 100644
--- a/src/daq/nginx-lua/s5-nginx-init.lua
+++ b/src/daq/nginx-lua/s5-nginx-init.lua
@@ -33,7 +33,8 @@
 -- Install this by adding the following to your nginx.conf
 --
 -- http {
---   lua_shared_dict sfive 64m;
+--   lua_shared_dict sfive 64k;
+--   lua_shared_dict sfive_log 64m;
 --   init_by_lua_file '/path/to/s5-nginx-init.lua';
 -- }
 --
@@ -42,14 +43,13 @@ ngx.log(ngx.DEBUG, "SFive: loaded")
 local sfive = ngx.shared.sfive;
 sfive:flush_all()
 sfive:flush_expired()
-local succ, err, forc = sfive:set("sem", 0)
-if not succ then
-   ngx.log(ngx.ERR, "SFive: creating semaphore failed: " .. err)
+local sfive_log = ngx.shared.sfive_log;
+sfive_log:flush_all()
+sfive_log:flush_expired()
+
+local ok, err, force = sfive:set("log:idx", 0)
+if not ok then
+   ngx.log(ngx.ERR, "SFive: creating log index counter failed: " .. err)
 else
-   local succ, err, forc = sfive:set("log_idx", 0)
-   if not succ then
-      ngx.log(ngx.ERR, "SFive: creating index failed: " .. err)
-   else
-      ngx.log(ngx.INFO, "SFive: initalized successfully!")
-   end
+   ngx.log(ngx.INFO, "SFive: initialized successfully!")
 end
diff --git a/src/daq/nginx-lua/s5-nginx-log.lua b/src/daq/nginx-lua/s5-nginx-log.lua
index f81bb8e..862abee 100644
--- a/src/daq/nginx-lua/s5-nginx-log.lua
+++ b/src/daq/nginx-lua/s5-nginx-log.lua
@@ -37,16 +37,49 @@
 -- }
 --
 
+local cleanup_delay = 1200
+local log_exptime = 600
+
+local log_cleanup = function(premature)
+   local sfive = ngx.shared.sfive;
+   sfive:flush_expired()
+   if premature then
+      return
+   end
+   local ok, err = ngx.timer.at(cleanup_delay, log_cleanup)
+   if not ok then
+      ngx.log(ngx.ERR, "SFive: failed to reenqueue log_cleanup: ", err)
+      sfive:delete("log:cleanup_running")
+      return
+   end
+end
+
+local start_cleanup_thread = function(sfive)
+   local ok, err, force = sfive:add("log:cleanup_running", 1)
+   if not ok then
+      if err == "exists" then
+         return
+      end
+      ngx.log(ngx.ERR, "SFive: failed to set cleanup_running flag: " .. err)
+   end
+   ok, err = ngx.timer.at(cleanup_delay, log_cleanup)
+   if not ok then
+      ngx.log(ngx.ERR, "SFive: failed to enqueue log_cleanup: ", err)
+      sfive:delete("log:cleanup_running")
+      return
+   end
+end
+
 local status = ngx.var.status
 
 if status == '200' or status == '206' then
    local sfive = ngx.shared.sfive;
-   local idx, err = sfive:incr("log_idx", 1)
+   local idx, err = sfive:incr("log:idx", 1)
    if not idx then
-      ngx.log(ngx.ERR, "SFive: incrementing index failed: " .. err)
+      ngx.log(ngx.ERR, "SFive: incrementing log index failed: " .. err)
    else
       local json = '{'
-      json = json .. '"time": "' .. string.gsub(ngx.utctime(), " ", "T", 1) .. '",'
+      json = json .. '"time": "' .. string.gsub(ngx.utctime(), " ", "T", 1) .. 'Z",'
       json = json .. '"client": "' .. ngx.var.remote_addr .. '",'
       json = json .. '"port": ' .. ngx.var.remote_port .. ','
       json = json .. '"ua": "' .. ngx.var.http_user_agent .. '",'
@@ -55,11 +88,13 @@ if status == '200' or status == '206' then
       json = json .. '"bytes_sent": ' .. ngx.var.bytes_sent
       json = json .. '}'
 
-      local succ, err, forc = sfive:add("log_" .. idx, json, 600)
-      if not succ then
-         ngx.log(ngx.ERR, "SFive: adding log line (log_" .. idx ") to shared dict: " .. err)
-      elseif forc then
-         ngx.log(ngx.WARN, "SFive: adding log line has overwritten other log lines - consider increasing the shared dict")
+      local sfive_log = ngx.shared.sfive_log;
+      local ok, err, force = sfive_log:add(idx, json, log_exptime)
+      if not ok then
+         ngx.log(ngx.ERR, "SFive: adding log line (".. idx .. ") to log store failed: " .. err)
+      elseif force then
+         ngx.log(ngx.WARN, "SFive: adding log line has overwritten other log lines - consider increasing the log store!")
       end
+      start_cleanup_thread(sfive)
    end
 end