[Rspamd] Fix metadata_exporter

[Web] Show subjet in quarantine
[Compose] Update Rspamd image

data/Dockerfiles/rspamd/Dockerfile

@@ -21,7 +21,7 @@ RUN apt-get update && apt-get install -y \
 
 COPY settings.conf /etc/rspamd/settings.conf
 COPY docker-entrypoint.sh /docker-entrypoint.sh
-COPY ratelimit.lua /usr/share/rspamd/lua/ratelimit.lua
+COPY metadata_exporter.lua /usr/share/rspamd/lua/metadata_exporter.lua
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 

data/Dockerfiles/rspamd/metadata_exporter.lua

@@ -0,0 +1,722 @@
+--[[
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+Copyright (c) 2016, Vsevolod Stakhov <vsevolod@highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+if confighelp then
+  return
+end
+
+-- A plugin that pushes metadata (or whole messages) to external services
+
+local redis_params
+local lua_util = require "lua_util"
+local rspamd_http = require "rspamd_http"
+local rspamd_tcp = require "rspamd_tcp"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local ucl = require "ucl"
+local E = {}
+local N = 'metadata_exporter'
+
+local settings = {
+  pusher_enabled = {},
+  pusher_format = {},
+  pusher_select = {},
+  mime_type = 'text/plain',
+  defer = false,
+  mail_from = '',
+  mail_to = 'postmaster@localhost',
+  helo = 'rspamd',
+  email_template = [[From: "Rspamd" <$mail_from>
+To: $mail_to
+Subject: Spam alert
+Date: $date
+MIME-Version: 1.0
+Message-ID: <$our_message_id>
+Content-type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+Authenticated username: $user
+IP: $ip
+Queue ID: $qid
+SMTP FROM: $from
+SMTP RCPT: $rcpt
+MIME From: $header_from
+MIME To: $header_to
+MIME Date: $header_date
+Subject: $header_subject
+Message-ID: $message_id
+Action: $action
+Score: $score
+Symbols: $symbols]],
+}
+
+local function get_general_metadata(task, flatten, no_content)
+  local r = {}
+  local ip = task:get_from_ip()
+  if ip and ip:is_valid() then
+    r.ip = tostring(ip)
+  else
+    r.ip = 'unknown'
+  end
+  r.user = task:get_user() or 'unknown'
+  r.qid = task:get_queue_id() or 'unknown'
+  r.subject = task:get_subject() or 'unknown'
+  r.action = task:get_metric_action('default')
+
+  local s = task:get_metric_score('default')[1]
+  r.score = flatten and string.format('%.2f', s) or s
+
+  local rcpt = task:get_recipients('smtp')
+  if rcpt then
+    local l = {}
+    for _, a in ipairs(rcpt) do
+      table.insert(l, a['addr'])
+    end
+    if not flatten then
+      r.rcpt = l
+    else
+      r.rcpt = table.concat(l, ', ')
+    end
+  else
+    r.rcpt = 'unknown'
+  end
+  local from = task:get_from('smtp')
+  if ((from or E)[1] or E).addr then
+    r.from = from[1].addr
+  else
+    r.from = 'unknown'
+  end
+  local syminf = task:get_symbols_all()
+  if flatten then
+    local l = {}
+    for _, sym in ipairs(syminf) do
+      local txt
+      if sym.options then
+        local topt = table.concat(sym.options, ', ')
+        txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')' .. ' [' .. topt .. ']'
+      else
+        txt = sym.name .. '(' .. string.format('%.2f', sym.score) .. ')'
+      end
+      table.insert(l, txt)
+    end
+    r.symbols = table.concat(l, '\n\t')
+  else
+    r.symbols = syminf
+  end
+  local function process_header(name)
+    local hdr = task:get_header_full(name)
+    if hdr then
+      local l = {}
+      for _, h in ipairs(hdr) do
+        table.insert(l, h.decoded)
+      end
+      if not flatten then
+        return l
+      else
+        return table.concat(l, '\n')
+      end
+    else
+      return 'unknown'
+    end
+  end
+  if not no_content then
+    r.header_from = process_header('from')
+    r.header_to = process_header('to')
+    r.header_subject = process_header('subject')
+    r.header_date = process_header('date')
+    r.message_id = task:get_message_id()
+  end
+  return r
+end
+
+local formatters = {
+  default = function(task)
+    return task:get_content()
+  end,
+  email_alert = function(task, rule, extra)
+    local meta = get_general_metadata(task, true)
+    local display_emails = {}
+    meta.mail_from = rule.mail_from or settings.mail_from
+    local mail_targets = rule.mail_to or settings.mail_to
+    if type(mail_targets) ~= 'table' then
+      table.insert(display_emails, string.format('<%s>', mail_targets))
+      mail_targets = {[mail_targets] = true}
+    else
+      for _, e in ipairs(mail_targets) do
+        table.insert(display_emails, string.format('<%s>', e))
+      end
+    end
+    if rule.email_alert_sender then
+      local x = task:get_from('smtp')
+      if x and string.len(x[1].addr) > 0 then
+        mail_targets[x] = true
+        table.insert(display_emails, string.format('<%s>', x[1].addr))
+      end
+    end
+    if rule.email_alert_user then
+      local x = task:get_user()
+      if x then
+        mail_targets[x] = true
+        table.insert(display_emails, string.format('<%s>', x))
+      end
+    end
+    if rule.email_alert_recipients then
+      local x = task:get_recipients('smtp')
+      if x then
+        for _, e in ipairs(x) do
+          if string.len(e.addr) > 0 then
+            mail_targets[e.addr] = true
+            table.insert(display_emails, string.format('<%s>', e.addr))
+          end
+        end
+      end
+    end
+    meta.mail_to = table.concat(display_emails, ', ')
+    meta.our_message_id = rspamd_util.random_hex(12) .. '@rspamd'
+    meta.date = rspamd_util.time_to_string(rspamd_util.get_time())
+    return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets}
+  end,
+  json = function(task)
+    return ucl.to_format(get_general_metadata(task), 'json-compact')
+  end
+}
+
+local function is_spam(action)
+  return (action == 'reject' or action == 'add header' or action == 'rewrite subject')
+end
+
+local selectors = {
+  default = function(task)
+    return true
+  end,
+  is_spam = function(task)
+    local action = task:get_metric_action('default')
+    return is_spam(action)
+  end,
+  is_spam_authed = function(task)
+    if not task:get_user() then
+      return false
+    end
+    local action = task:get_metric_action('default')
+    return is_spam(action)
+  end,
+  is_reject = function(task)
+    local action = task:get_metric_action('default')
+    return (action == 'reject')
+  end,
+  is_reject_authed = function(task)
+    if not task:get_user() then
+      return false
+    end
+    local action = task:get_metric_action('default')
+    return (action == 'reject')
+  end,
+}
+
+local function maybe_defer(task, rule)
+  if rule.defer then
+    rspamd_logger.warnx(task, 'deferring message')
+    task:set_pre_result('soft reject', 'deferred', N)
+  end
+end
+
+local pushers = {
+  redis_pubsub = function(task, formatted, rule)
+    local _,ret,upstream
+    local function redis_pub_cb(err)
+      if err then
+        rspamd_logger.errx(task, 'got error %s when publishing on server %s',
+            err, upstream:get_addr())
+        return maybe_defer(task, rule)
+      end
+      return true
+    end
+    ret,_,upstream = rspamd_redis_make_request(task,
+      redis_params, -- connect params
+      nil, -- hash key
+      true, -- is write
+      redis_pub_cb, --callback
+      'PUBLISH', -- command
+      {rule.channel, formatted} -- arguments
+    )
+    if not ret then
+      rspamd_logger.errx(task, 'error connecting to redis')
+      maybe_defer(task, rule)
+    end
+  end,
+  http = function(task, formatted, rule)
+    local function http_callback(err, code)
+      if err then
+        rspamd_logger.errx(task, 'got error %s in http callback', err)
+        return maybe_defer(task, rule)
+      end
+      if code ~= 200 then
+        rspamd_logger.errx(task, 'got unexpected http status: %s', code)
+        return maybe_defer(task, rule)
+      end
+      return true
+    end
+    local hdrs = {}
+    if rule.meta_headers then
+      local gm = get_general_metadata(task, false, true)
+      local pfx = rule.meta_header_prefix or 'X-Rspamd-'
+      for k, v in pairs(gm) do
+        if type(v) == 'table' then
+          hdrs[pfx .. k] = ucl.to_format(v, 'json-compact')
+        else
+          hdrs[pfx .. k] = v
+        end
+      end
+    end
+    rspamd_http.request({
+      task=task,
+      url=rule.url,
+      body=formatted,
+      callback=http_callback,
+      mime_type=rule.mime_type or settings.mime_type,
+      headers=hdrs,
+    })
+  end,
+  send_mail = function(task, formatted, rule, extra)
+    local function mail_cb(err, data, conn)
+      local function no_error(merr, mdata, wantcode)
+        wantcode = wantcode or '2'
+        if merr then
+          rspamd_logger.errx(task, 'got error in tcp callback: %s', merr)
+          if conn then
+            conn:close()
+          end
+          maybe_defer(task, rule)
+          return false
+        end
+        if mdata then
+          if type(mdata) ~= 'string' then
+            mdata = tostring(mdata)
+            end
+          if string.sub(mdata, 1, 1) ~= wantcode then
+            rspamd_logger.errx(task, 'got bad smtp response: %s', mdata)
+            if conn then
+              conn:close()
+            end
+            maybe_defer(task, rule)
+            return false
+            end
+        else
+            rspamd_logger.errx(task, 'no data')
+          if conn then
+            conn:close()
+          end
+          maybe_defer(task, rule)
+          return false
+        end
+        return true
+      end
+      local function all_done_cb(merr, mdata)
+        if conn then
+          conn:close()
+        end
+        return true
+      end
+      local function quit_done_cb(merr, mdata)
+        conn:add_read(all_done_cb, '\r\n')
+      end
+      local function quit_cb(merr, mdata)
+        if no_error(merr, mdata) then
+          conn:add_write(quit_done_cb, 'QUIT\r\n')
+        end
+      end
+      local function pre_quit_cb(merr, mdata)
+        if no_error(merr, '2') then
+          conn:add_read(quit_cb, '\r\n')
+        end
+      end
+      local function data_done_cb(merr, mdata)
+        if no_error(merr, mdata, '3') then
+          conn:add_write(pre_quit_cb, {formatted, '\r\n.\r\n'})
+        end
+      end
+      local function data_cb(merr, mdata)
+        if no_error(merr, '2') then
+          conn:add_read(data_done_cb, '\r\n')
+        end
+      end
+      local from_done_cb
+      local function rcpt_done_cb(merr, mdata)
+        if no_error(merr, mdata) then
+          local k = next(extra.mail_targets)
+          if not k then
+            conn:add_write(data_cb, 'DATA\r\n')
+          else
+            from_done_cb('2', '2')
+          end
+        end
+      end
+      local function rcpt_cb(merr, mdata)
+        if no_error(merr, '2') then
+          conn:add_read(rcpt_done_cb, '\r\n')
+        end
+      end
+      from_done_cb = function(merr, mdata)
+        local k
+        if extra then
+          k = next(extra.mail_targets)
+        else
+          extra = {mail_targets = {}}
+          if type(rule.mail_to) == 'string' then
+            extra = {mail_targets = {}}
+            k = rule.mail_to
+          elseif type(rule.mail_to) == 'table' then
+            for _, r in ipairs(rule.mail_to) do
+              extra.mail_targets[r] = true
+            end
+            k = next(extra.mail_targets)
+          end
+        end
+        extra.mail_targets[k] = nil
+        conn:add_write(rcpt_cb, {'RCPT TO: <', k, '>\r\n'})
+      end
+      local function from_cb(merr, mdata)
+        if no_error(merr, '2') then
+          conn:add_read(from_done_cb, '\r\n')
+        end
+      end
+        local function hello_done_cb(merr, mdata)
+        if no_error(merr, mdata) then
+          conn:add_write(from_cb, {'MAIL FROM: <', rule.mail_from or settings.mail_from, '>\r\n'})
+        end
+      end
+      local function hello_cb(merr)
+        if no_error(merr, '2') then
+          conn:add_read(hello_done_cb, '\r\n')
+        end
+      end
+      if no_error(err, data) then
+        conn:add_write(hello_cb, {'HELO ', rule.helo or settings.helo, '\r\n'})
+      end
+    end
+    rspamd_tcp.request({
+      task = task,
+      callback = mail_cb,
+      stop_pattern = '\r\n',
+      host = rule.smtp,
+      port = rule.smtp_port or settings.smtp_port or 25,
+    })
+  end,
+}
+
+local opts = rspamd_config:get_all_opt(N)
+if not opts then return end
+local process_settings = {
+  select = function(val)
+    selectors.custom = assert(load(val))()
+  end,
+  format = function(val)
+    formatters.custom = assert(load(val))()
+  end,
+  push = function(val)
+    pushers.custom = assert(load(val))()
+  end,
+  custom_push = function(val)
+    if type(val) == 'table' then
+      for k, v in pairs(val) do
+        pushers[k] = assert(load(v))()
+      end
+    end
+  end,
+  custom_select = function(val)
+    if type(val) == 'table' then
+      for k, v in pairs(val) do
+        selectors[k] = assert(load(v))()
+      end
+    end
+  end,
+  custom_format = function(val)
+    if type(val) == 'table' then
+      for k, v in pairs(val) do
+        formatters[k] = assert(load(v))()
+      end
+    end
+  end,
+  pusher_enabled = function(val)
+    if type(val) == 'string' then
+      if pushers[val] then
+        settings.pusher_enabled[val] = true
+      else
+        rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val)
+      end
+    elseif type(val) == 'table' then
+      for _, v in ipairs(val) do
+        if pushers[v] then
+          settings.pusher_enabled[v] = true
+        else
+          rspamd_logger.errx(rspamd_config, 'Pusher type: %s is invalid', val)
+        end
+      end
+    end
+  end,
+}
+for k, v in pairs(opts) do
+  local f = process_settings[k]
+  if f then
+    f(opts[k])
+  else
+    settings[k] = v
+  end
+end
+if type(settings.rules) ~= 'table' then
+  -- Legacy config
+  settings.rules = {}
+  if not next(settings.pusher_enabled) then
+    if pushers.custom then
+      rspamd_logger.infox(rspamd_config, 'Custom pusher implicitly enabled')
+      settings.pusher_enabled.custom = true
+    else
+      -- Check legacy options
+      if settings.url then
+        rspamd_logger.warnx(rspamd_config, 'HTTP pusher implicitly enabled')
+        settings.pusher_enabled.http = true
+      end
+      if settings.channel then
+        rspamd_logger.warnx(rspamd_config, 'Redis Pubsub pusher implicitly enabled')
+        settings.pusher_enabled.redis_pubsub = true
+      end
+      if settings.smtp and settings.mail_to then
+        rspamd_logger.warnx(rspamd_config, 'SMTP pusher implicitly enabled')
+        settings.pusher_enabled.send_mail = true
+      end
+    end
+  end
+  if not next(settings.pusher_enabled) then
+    rspamd_logger.errx(rspamd_config, 'No push backend enabled')
+    return
+  end
+  if settings.formatter then
+    settings.format = formatters[settings.formatter]
+    if not settings.format then
+      rspamd_logger.errx(rspamd_config, 'No such formatter: %s', settings.formatter)
+      return
+    end
+  end
+  if settings.selector then
+    settings.select = selectors[settings.selector]
+    if not settings.select then
+      rspamd_logger.errx(rspamd_config, 'No such selector: %s', settings.selector)
+      return
+    end
+  end
+  for k in pairs(settings.pusher_enabled) do
+    local formatter = settings.pusher_format[k]
+    local selector = settings.pusher_select[k]
+    if not formatter then
+      settings.pusher_format[k] = settings.formatter or 'default'
+      rspamd_logger.infox(rspamd_config, 'Using default formatter for %s pusher', k)
+    else
+      if not formatters[formatter] then
+        rspamd_logger.errx(rspamd_config, 'No such formatter: %s - disabling %s', formatter, k)
+        settings.pusher_enabled.k = nil
+      end
+    end
+    if not selector then
+      settings.pusher_select[k] = settings.selector or 'default'
+      rspamd_logger.infox(rspamd_config, 'Using default selector for %s pusher', k)
+    else
+      if not selectors[selector] then
+        rspamd_logger.errx(rspamd_config, 'No such selector: %s - disabling %s', selector, k)
+        settings.pusher_enabled.k = nil
+      end
+    end
+  end
+  if settings.pusher_enabled.redis_pubsub then
+    redis_params = rspamd_parse_redis_server(N)
+    if not redis_params then
+      rspamd_logger.errx(rspamd_config, 'No redis servers are specified')
+      settings.pusher_enabled.redis_pubsub = nil
+    else
+      local r = {}
+      r.backend = 'redis_pubsub'
+      r.channel = settings.channel
+      r.defer = settings.defer
+      r.selector = settings.pusher_select.redis_pubsub
+      r.formatter = settings.pusher_format.redis_pubsub
+      settings.rules[r.backend:upper()] = r
+    end
+  end
+  if settings.pusher_enabled.http then
+    if not settings.url then
+      rspamd_logger.errx(rspamd_config, 'No URL is specified')
+      settings.pusher_enabled.http = nil
+    else
+      local r = {}
+      r.backend = 'http'
+      r.url = settings.url
+      r.mime_type = settings.mime_type
+      r.defer = settings.defer
+      r.selector = settings.pusher_select.http
+      r.formatter = settings.pusher_format.http
+      settings.rules[r.backend:upper()] = r
+    end
+  end
+  if settings.pusher_enabled.send_mail then
+    if not (settings.mail_to and settings.smtp) then
+      rspamd_logger.errx(rspamd_config, 'No mail_to and/or smtp setting is specified')
+      settings.pusher_enabled.send_mail = nil
+    else
+      local r = {}
+      r.backend = 'send_mail'
+      r.mail_to = settings.mail_to
+      r.mail_from = settings.mail_from
+      r.helo = settings.hello
+      r.smtp = settings.smtp
+      r.smtp_port = settings.smtp_port
+      r.email_template = settings.email_template
+      r.defer = settings.defer
+      r.selector = settings.pusher_select.send_mail
+      r.formatter = settings.pusher_format.send_mail
+      settings.rules[r.backend:upper()] = r
+    end
+  end
+  if not next(settings.pusher_enabled) then
+    rspamd_logger.errx(rspamd_config, 'No push backend enabled')
+    return
+  end
+elseif not next(settings.rules) then
+  lua_util.debugm(N, rspamd_config, 'No rules enabled')
+  return
+end
+if not settings.rules or not next(settings.rules) then
+  rspamd_logger.errx(rspamd_config, 'No rules enabled')
+  return
+end
+local backend_required_elements = {
+  http = {
+    'url',
+  },
+  smtp = {
+    'mail_to',
+    'smtp',
+  },
+  redis_pubsub = {
+    'channel',
+  },
+}
+local check_element = {
+  selector = function(k, v)
+    if not selectors[v] then
+      rspamd_logger.errx(rspamd_config, 'Rule %s has invalid selector %s', k, v)
+      return false
+    else
+      return true
+    end
+  end,
+  formatter = function(k, v)
+    if not formatters[v] then
+      rspamd_logger.errx(rspamd_config, 'Rule %s has invalid formatter %s', k, v)
+      return false
+    else
+      return true
+    end
+  end,
+}
+local backend_check = {
+  default = function(k, rule)
+    local reqset = backend_required_elements[rule.backend]
+    if reqset then
+      for _, e in ipairs(reqset) do
+        if not rule[e] then
+          rspamd_logger.errx(rspamd_config, 'Rule %s misses required setting %s', k, e)
+          settings.rules[k] = nil
+        end
+      end
+    end
+    for sett, v in pairs(rule) do
+      local f = check_element[sett]
+      if f then
+        if not f(sett, v) then
+          settings.rules[k] = nil
+        end
+      end
+    end
+  end,
+}
+backend_check.redis_pubsub = function(k, rule)
+  if not redis_params then
+    redis_params = rspamd_parse_redis_server(N)
+  end
+  if not redis_params then
+    rspamd_logger.errx(rspamd_config, 'No redis servers are specified')
+    settings.rules[k] = nil
+  else
+    backend_check.default(k, rule)
+  end
+end
+setmetatable(backend_check, {
+  __index = function()
+    return backend_check.default
+  end,
+})
+for k, v in pairs(settings.rules) do
+  if type(v) == 'table' then
+    local backend = v.backend
+    if not backend then
+      rspamd_logger.errx(rspamd_config, 'Rule %s has no backend', k)
+      settings.rules[k] = nil
+    elseif not pushers[backend] then
+      rspamd_logger.errx(rspamd_config, 'Rule %s has invalid backend %s', k, backend)
+      settings.rules[k] = nil
+    else
+      local f = backend_check[backend]
+      f(k, v)
+    end
+  else
+    rspamd_logger.errx(rspamd_config, 'Rule %s has bad type: %s', k, type(v))
+    settings.rules[k] = nil
+  end
+end
+
+local function gen_exporter(rule)
+  return function (task)
+    if task:has_flag('skip') then return end
+    local selector = rule.selector or 'default'
+    local selected = selectors[selector](task)
+    if selected then
+      lua_util.debugm(N, task, 'Message selected for processing')
+      local formatter = rule.formatter or 'default'
+      local formatted, extra = formatters[formatter](task, rule)
+      if formatted then
+        pushers[rule.backend](task, formatted, rule, extra)
+      else
+        lua_util.debugm(N, task, 'Formatter [%s] returned non-truthy value [%s]', formatter, formatted)
+      end
+    else
+      lua_util.debugm(N, task, 'Selector [%s] returned non-truthy value [%s]', selector, selected)
+    end
+  end
+end
+
+if not next(settings.rules) then
+  rspamd_logger.errx(rspamd_config, 'No rules enabled')
+  lua_util.disable_module(N, "config")
+end
+for k, r in pairs(settings.rules) do
+  rspamd_config:register_symbol({
+    name = 'EXPORT_METADATA_' .. k,
+    type = 'postfilter,idempotent',
+    callback = gen_exporter(r),
+    priority = 10,
+    flags = 'empty',
+  })
+end

data/Dockerfiles/rspamd/ratelimit.lua

@@ -1,864 +0,0 @@
---[[
-Copyright (c) 2011-2017, Vsevolod Stakhov <vsevolod@highsecure.ru>
-Copyright (c) 2016-2017, Andrew Lewis <nerf@judo.za.org>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-]]--
-
-if confighelp then
-  return
-end
-
-local rspamd_logger = require "rspamd_logger"
-local rspamd_util = require "rspamd_util"
-local rspamd_lua_utils = require "lua_util"
-local lua_redis = require "lua_redis"
-local fun = require "fun"
-local lua_maps = require "lua_maps"
-local lua_util = require "lua_util"
-local rspamd_hash = require "rspamd_cryptobox_hash"
-local lua_selectors = require "lua_selectors"
-local ts = require("tableshape").types
-
--- A plugin that implements ratelimits using redis
-
-local E = {}
-local N = 'ratelimit'
-local redis_params
--- Senders that are considered as bounce
-local settings = {
-  bounce_senders = { 'postmaster', 'mailer-daemon', '', 'null', 'fetchmail-daemon', 'mdaemon' },
--- Do not check ratelimits for these recipients
-  whitelisted_rcpts = { 'postmaster', 'mailer-daemon' },
-  prefix = 'RL',
-  ham_factor_rate = 1.01,
-  spam_factor_rate = 0.99,
-  ham_factor_burst = 1.02,
-  spam_factor_burst = 0.98,
-  max_rate_mult = 5,
-  max_bucket_mult = 10,
-  expire = 60 * 60 * 24 * 2, -- 2 days by default
-  limits = {},
-  allow_local = false,
-}
-
--- Checks bucket, updating it if needed
--- KEYS[1] - prefix to update, e.g. RL_<triplet>_<seconds>
--- KEYS[2] - current time in milliseconds
--- KEYS[3] - bucket leak rate (messages per millisecond)
--- KEYS[4] - bucket burst
--- KEYS[5] - expire for a bucket
--- return 1 if message should be ratelimited and 0 if not
--- Redis keys used:
---   l - last hit
---   b - current burst
---   dr - current dynamic rate multiplier (*10000)
---   db - current dynamic burst multiplier (*10000)
-local bucket_check_script = [[
-  local last = redis.call('HGET', KEYS[1], 'l')
-  local now = tonumber(KEYS[2])
-  local dynr, dynb, leaked = 0, 0, 0
-  if not last then
-    -- New bucket
-    redis.call('HSET', KEYS[1], 'l', KEYS[2])
-    redis.call('HSET', KEYS[1], 'b', '0')
-    redis.call('HSET', KEYS[1], 'dr', '10000')
-    redis.call('HSET', KEYS[1], 'db', '10000')
-    redis.call('EXPIRE', KEYS[1], KEYS[5])
-    return {0, '0', '1', '1', '0'}
-  end
-
-  last = tonumber(last)
-  local burst = tonumber(redis.call('HGET', KEYS[1], 'b'))
-  -- Perform leak
-  if burst > 0 then
-   if last < tonumber(KEYS[2]) then
-    local rate = tonumber(KEYS[3])
-    dynr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000.0
-    if dynr == 0 then dynr = 0.0001 end
-    rate = rate * dynr
-    leaked = ((now - last) * rate)
-    if leaked > burst then leaked = burst end
-    burst = burst - leaked
-    redis.call('HINCRBYFLOAT', KEYS[1], 'b', -(leaked))
-    redis.call('HSET', KEYS[1], 'l', KEYS[2])
-   end
-
-   dynb = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000.0
-   if dynb == 0 then dynb = 0.0001 end
-
-   if burst > 0 and (burst + 1) > tonumber(KEYS[4]) * dynb then
-     return {1, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)}
-   end
-  else
-   burst = 0
-   redis.call('HSET', KEYS[1], 'b', '0')
-  end
-
-  return {0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)}
-]]
-local bucket_check_id
-
-
--- Updates a bucket
--- KEYS[1] - prefix to update, e.g. RL_<triplet>_<seconds>
--- KEYS[2] - current time in milliseconds
--- KEYS[3] - dynamic rate multiplier
--- KEYS[4] - dynamic burst multiplier
--- KEYS[5] - max dyn rate (min: 1/x)
--- KEYS[6] - max burst rate (min: 1/x)
--- KEYS[7] - expire for a bucket
--- Redis keys used:
---   l - last hit
---   b - current burst
---   dr - current dynamic rate multiplier
---   db - current dynamic burst multiplier
-local bucket_update_script = [[
-  local last = redis.call('HGET', KEYS[1], 'l')
-  local now = tonumber(KEYS[2])
-  if not last then
-    -- New bucket
-    redis.call('HSET', KEYS[1], 'l', KEYS[2])
-    redis.call('HSET', KEYS[1], 'b', '1')
-    redis.call('HSET', KEYS[1], 'dr', '10000')
-    redis.call('HSET', KEYS[1], 'db', '10000')
-    redis.call('EXPIRE', KEYS[1], KEYS[7])
-    return {1, 1, 1}
-  end
-
-  local dr, db = 1.0, 1.0
-
-  if tonumber(KEYS[5]) > 1 then
-    local rate_mult = tonumber(KEYS[3])
-    local rate_limit = tonumber(KEYS[5])
-    dr = tonumber(redis.call('HGET', KEYS[1], 'dr')) / 10000
-
-    if rate_mult > 1.0 and dr < rate_limit then
-      dr = dr * rate_mult
-      if dr > 0.0001 then
-        redis.call('HSET', KEYS[1], 'dr', tostring(math.floor(dr * 10000)))
-      else
-        redis.call('HSET', KEYS[1], 'dr', '1')
-      end
-    elseif rate_mult < 1.0 and dr > (1.0 / rate_limit) then
-      dr = dr * rate_mult
-      if dr > 0.0001 then
-        redis.call('HSET', KEYS[1], 'dr', tostring(math.floor(dr * 10000)))
-      else
-        redis.call('HSET', KEYS[1], 'dr', '1')
-      end
-    end
-  end
-
-  if tonumber(KEYS[6]) > 1 then
-    local rate_mult = tonumber(KEYS[4])
-    local rate_limit = tonumber(KEYS[6])
-    db = tonumber(redis.call('HGET', KEYS[1], 'db')) / 10000
-
-    if rate_mult > 1.0 and db < rate_limit then
-      db = db * rate_mult
-      if db > 0.0001 then
-        redis.call('HSET', KEYS[1], 'db', tostring(math.floor(db * 10000)))
-      else
-        redis.call('HSET', KEYS[1], 'db', '1')
-      end
-    elseif rate_mult < 1.0 and db > (1.0 / rate_limit) then
-      db = db * rate_mult
-      if db > 0.0001 then
-        redis.call('HSET', KEYS[1], 'db', tostring(math.floor(db * 10000)))
-      else
-        redis.call('HSET', KEYS[1], 'db', '1')
-      end
-    end
-  end
-
-  local burst = tonumber(redis.call('HGET', KEYS[1], 'b'))
-  if burst < 0 then burst = 0 end
-
-  redis.call('HINCRBYFLOAT', KEYS[1], 'b', 1)
-  redis.call('HSET', KEYS[1], 'l', KEYS[2])
-  redis.call('EXPIRE', KEYS[1], KEYS[7])
-
-  return {tostring(burst), tostring(dr), tostring(db)}
-]]
-local bucket_update_id
-
--- message_func(task, limit_type, prefix, bucket, limit_key)
-local message_func = function(_, limit_type, _, _, _)
-  return string.format('Ratelimit "%s" exceeded', limit_type)
-end
-
-
-local function load_scripts(cfg, ev_base)
-  bucket_check_id = lua_redis.add_redis_script(bucket_check_script, redis_params)
-  bucket_update_id = lua_redis.add_redis_script(bucket_update_script, redis_params)
-end
-
-local limit_parser
-local function parse_string_limit(lim, no_error)
-  local function parse_time_suffix(s)
-    if s == 's' then
-      return 1
-    elseif s == 'm' then
-      return 60
-    elseif s == 'h' then
-      return 3600
-    elseif s == 'd' then
-      return 86400
-    end
-  end
-  local function parse_num_suffix(s)
-    if s == '' then
-      return 1
-    elseif s == 'k' then
-      return 1000
-    elseif s == 'm' then
-      return 1000000
-    elseif s == 'g' then
-      return 1000000000
-    end
-  end
-  local lpeg = require "lpeg"
-
-  if not limit_parser then
-    local digit = lpeg.R("09")
-    limit_parser = {}
-    limit_parser.integer =
-    (lpeg.S("+-") ^ -1) *
-            (digit   ^  1)
-    limit_parser.fractional =
-    (lpeg.P(".")   ) *
-            (digit ^ 1)
-    limit_parser.number =
-    (limit_parser.integer *
-            (limit_parser.fractional ^ -1)) +
-            (lpeg.S("+-") * limit_parser.fractional)
-    limit_parser.time = lpeg.Cf(lpeg.Cc(1) *
-            (limit_parser.number / tonumber) *
-            ((lpeg.S("smhd") / parse_time_suffix) ^ -1),
-      function (acc, val) return acc * val end)
-    limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) *
-            (limit_parser.number / tonumber) *
-            ((lpeg.S("kmg") / parse_num_suffix) ^ -1),
-      function (acc, val) return acc * val end)
-    limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number *
-            (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) *
-            limit_parser.time)
-  end
-  local t = lpeg.match(limit_parser.limit, lim)
-
-  if t and t[1] and t[2] and t[2] ~= 0 then
-    return t[2], t[1]
-  end
-
-  if not no_error then
-    rspamd_logger.errx(rspamd_config, 'bad limit: %s', lim)
-  end
-
-  return nil
-end
-
-local function str_to_rate(str)
-  local divider,divisor = parse_string_limit(str, false)
-
-  if not divisor then
-    rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str)
-
-    return nil
-  end
-
-  return divisor / divider
-end
-
-local bucket_schema = ts.shape{
-  burst = ts.number + ts.string / lua_util.dehumanize_number,
-  rate = ts.number + ts.string / str_to_rate
-}
-
-local function parse_limit(name, data)
-  if type(data) == 'table' then
-    -- 2 cases here:
-    --  * old limit in format [burst, rate]
-    --  * vector of strings in Andrew's string format (removed from 1.8.2)
-    --  * proper bucket table
-    if #data == 2 and tonumber(data[1]) and tonumber(data[2]) then
-      -- Old style ratelimit
-      rspamd_logger.warnx(rspamd_config, 'old style ratelimit for %s', name)
-      if tonumber(data[1]) > 0 and tonumber(data[2]) > 0 then
-        return {
-          burst = data[1],
-          rate = data[2]
-        }
-      elseif data[1] ~= 0 then
-        rspamd_logger.warnx(rspamd_config, 'invalid numbers for %s', name)
-      else
-        rspamd_logger.infox(rspamd_config, 'disable limit %s, burst is zero', name)
-      end
-
-      return nil
-    else
-      local parsed_bucket,err = bucket_schema:transform(data)
-
-      if not parsed_bucket or err then
-        rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s',
-            name, err, data)
-      else
-        return parsed_bucket
-      end
-    end
-  elseif type(data) == 'string' then
-    local rep_rate, burst = parse_string_limit(data)
-    rspamd_logger.warnx(rspamd_config, 'old style rate bucket config detected for %s: %s',
-        name, data)
-    if rep_rate and burst then
-      return {
-        burst = burst,
-        rate = burst / rep_rate -- reciprocal
-      }
-    end
-  end
-
-  return nil
-end
-
---- Check whether this addr is bounce
-local function check_bounce(from)
-  return fun.any(function(b) return b == from end, settings.bounce_senders)
-end
-
-local keywords = {
-  ['ip'] = {
-    ['get_value'] = function(task)
-      local ip = task:get_ip()
-      if ip and ip:is_valid() then return tostring(ip) end
-      return nil
-    end,
-  },
-  ['rip'] = {
-    ['get_value'] = function(task)
-      local ip = task:get_ip()
-      if ip and ip:is_valid() and not ip:is_local() then return tostring(ip) end
-      return nil
-    end,
-  },
-  ['from'] = {
-    ['get_value'] = function(task)
-      local from = task:get_from(0)
-      if ((from or E)[1] or E).addr then
-        return string.lower(from[1]['addr'])
-      end
-      return nil
-    end,
-  },
-  ['bounce'] = {
-    ['get_value'] = function(task)
-      local from = task:get_from(0)
-      if not ((from or E)[1] or E).user then
-        return '_'
-      end
-      if check_bounce(from[1]['user']) then return '_' else return nil end
-    end,
-  },
-  ['asn'] = {
-    ['get_value'] = function(task)
-      local asn = task:get_mempool():get_variable('asn')
-      if not asn then
-        return nil
-      else
-        return asn
-      end
-    end,
-  },
-  ['user'] = {
-    ['get_value'] = function(task)
-      local auser = task:get_user()
-      if not auser then
-        return nil
-      else
-        return auser
-      end
-    end,
-  },
-  ['to'] = {
-    ['get_value'] = function(task)
-      return task:get_principal_recipient()
-    end,
-  },
-  ['digest'] = {
-    ['get_value'] = function(task)
-      return task:get_digest()
-    end,
-  },
-  ['attachments'] = {
-    ['get_value'] = function(task)
-      local parts = task:get_parts() or E
-      local digests = {}
-
-      for _,p in ipairs(parts) do
-        if p:get_filename() then
-          table.insert(digests, p:get_digest())
-        end
-      end
-
-      if #digests > 0 then
-        return table.concat(digests, '')
-      end
-
-      return nil
-    end,
-  },
-  ['files'] = {
-    ['get_value'] = function(task)
-      local parts = task:get_parts() or E
-      local files = {}
-
-      for _,p in ipairs(parts) do
-        local fname = p:get_filename()
-        if fname then
-          table.insert(files, fname)
-        end
-      end
-
-      if #files > 0 then
-        return table.concat(files, ':')
-      end
-
-      return nil
-    end,
-  },
-}
-
-local function gen_rate_key(task, rtype, bucket)
-  local key_t = {tostring(lua_util.round(100000.0 / bucket.burst))}
-  local key_keywords = lua_util.str_split(rtype, '_')
-  local have_user = false
-
-  for _, v in ipairs(key_keywords) do
-    local ret
-
-    if keywords[v] and type(keywords[v]['get_value']) == 'function' then
-      ret = keywords[v]['get_value'](task)
-    end
-    if not ret then return nil end
-    if v == 'user' then have_user = true end
-    if type(ret) ~= 'string' then ret = tostring(ret) end
-    table.insert(key_t, ret)
-  end
-
-  if have_user and not task:get_user() then
-    return nil
-  end
-
-  return table.concat(key_t, ":")
-end
-
-local function make_prefix(redis_key, name, bucket)
-  local hash_len = 24
-  if hash_len > #redis_key then hash_len = #redis_key end
-  local hash = settings.prefix ..
-      string.sub(rspamd_hash.create(redis_key):base32(), 1, hash_len)
-  -- Fill defaults
-  if not bucket.spam_factor_rate then
-    bucket.spam_factor_rate = settings.spam_factor_rate
-  end
-  if not bucket.ham_factor_rate then
-    bucket.ham_factor_rate = settings.ham_factor_rate
-  end
-  if not bucket.spam_factor_burst then
-    bucket.spam_factor_burst = settings.spam_factor_burst
-  end
-  if not bucket.ham_factor_burst then
-    bucket.ham_factor_burst = settings.ham_factor_burst
-  end
-
-  return {
-    bucket = bucket,
-    name = name,
-    hash = hash
-  }
-end
-
-local function limit_to_prefixes(task, k, v, prefixes)
-  local n = 0
-  for _,bucket in ipairs(v.buckets) do
-    if v.selector then
-      local selectors = lua_selectors.process_selectors(task, v.selector)
-      if selectors then
-        local combined = lua_selectors.combine_selectors(task, selectors, ':')
-        if type(combined) == 'string' then
-          prefixes[combined] = make_prefix(combined, k, bucket)
-          n = n + 1
-        else
-          fun.each(function(p)
-            prefixes[p] = make_prefix(p, k, bucket)
-            n = n + 1
-          end, combined)
-        end
-      end
-    else
-      local prefix = gen_rate_key(task, k, bucket)
-      if prefix then
-        if type(prefix) == 'string' then
-          prefixes[prefix] = make_prefix(prefix, k, bucket)
-          n = n + 1
-        else
-          fun.each(function(p)
-            prefixes[p] = make_prefix(p, k, bucket)
-            n = n + 1
-          end, prefix)
-        end
-      end
-    end
-  end
-
-  return n
-end
-
-local function ratelimit_cb(task)
-  if not settings.allow_local and
-          rspamd_lua_utils.is_rspamc_or_controller(task) then return end
-
-  -- Get initial task data
-  local ip = task:get_from_ip()
-  if ip and ip:is_valid() and settings.whitelisted_ip then
-    if settings.whitelisted_ip:get_key(ip) then
-      -- Do not check whitelisted ip
-      rspamd_logger.infox(task, 'skip ratelimit for whitelisted IP')
-      return
-    end
-  end
-  -- Parse all rcpts
-  local rcpts = task:get_recipients()
-  local rcpts_user = {}
-  if rcpts then
-    fun.each(function(r)
-      fun.each(function(type) table.insert(rcpts_user, r[type]) end, {'user', 'addr'})
-    end, rcpts)
-
-    if fun.any(function(r) return settings.whitelisted_rcpts:get_key(r) end, rcpts_user) then
-      rspamd_logger.infox(task, 'skip ratelimit for whitelisted recipient')
-      return
-    end
-  end
-  -- Get user (authuser)
-  if settings.whitelisted_user then
-    local auser = task:get_user()
-    if settings.whitelisted_user:get_key(auser) then
-      rspamd_logger.infox(task, 'skip ratelimit for whitelisted user')
-      return
-    end
-  end
-  -- Now create all ratelimit prefixes
-  local prefixes = {}
-  local nprefixes = 0
-
-  for k,v in pairs(settings.limits) do
-    nprefixes = nprefixes + limit_to_prefixes(task, k, v, prefixes)
-  end
-
-  for k, hdl in pairs(settings.custom_keywords or E) do
-    local ret, redis_key, bd = pcall(hdl, task)
-
-    if ret then
-      local bucket = parse_limit(k, bd)
-      if bucket then
-        prefixes[redis_key] = make_prefix(redis_key, k, bucket)
-      end
-      nprefixes = nprefixes + 1
-    else
-      rspamd_logger.errx(task, 'cannot call handler for %s: %s',
-          k, redis_key)
-    end
-  end
-
-  local function gen_check_cb(prefix, bucket, lim_name, lim_key)
-    return function(err, data)
-      if err then
-        rspamd_logger.errx('cannot check limit %s: %s %s', prefix, err, data)
-      elseif type(data) == 'table' and data[1] then
-        lua_util.debugm(N, task,
-            "got reply for limit %s (%s / %s); %s burst, %s:%s dyn, %s leaked",
-            prefix, bucket.burst, bucket.rate,
-            data[2], data[3], data[4], data[5])
-
-        if data[1] == 1 then
-          -- set symbol only and do NOT soft reject
-          if settings.symbol then
-            task:insert_result(settings.symbol, 0.0,
-                string.format('%s(%s)', lim_name, lim_key))
-            rspamd_logger.infox(task,
-                'set_symbol_only: ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn); redis key: %s',
-                lim_name, prefix,
-                bucket.burst, bucket.rate,
-                data[2], data[3], data[4], lim_key)
-            return
-            -- set INFO symbol and soft reject
-          elseif settings.info_symbol then
-            task:insert_result(settings.info_symbol, 1.0,
-                string.format('%s(%s)', lim_name, lim_key))
-          end
-          rspamd_logger.infox(task,
-              'ratelimit "%s(%s)" exceeded, (%s / %s): %s (%s:%s dyn); redis key: %s',
-              lim_name, prefix,
-              bucket.burst, bucket.rate,
-              data[2], data[3], data[4], lim_key)
-          task:set_pre_result('soft reject',
-              message_func(task, lim_name, prefix, bucket, lim_key), N)
-        end
-      end
-    end
-  end
-
-  -- Don't do anything if pre-result has been already set
-  if task:has_pre_result() then return end
-
-  if nprefixes > 0 then
-    -- Save prefixes to the cache to allow update
-    task:cache_set('ratelimit_prefixes', prefixes)
-    local now = rspamd_util.get_time()
-    now = lua_util.round(now * 1000.0) -- Get milliseconds
-    -- Now call check script for all defined prefixes
-
-    for pr,value in pairs(prefixes) do
-      local bucket = value.bucket
-      local rate = (bucket.rate) / 1000.0 -- Leak rate in messages/ms
-      lua_util.debugm(N, task, "check limit %s:%s -> %s (%s/%s)",
-          value.name, pr, value.hash, bucket.burst, bucket.rate)
-      lua_redis.exec_redis_script(bucket_check_id,
-              {key = value.hash, task = task, is_write = true},
-              gen_check_cb(pr, bucket, value.name, value.hash),
-              {value.hash, tostring(now), tostring(rate), tostring(bucket.burst),
-                  tostring(settings.expire)})
-    end
-  end
-end
-
-local function ratelimit_update_cb(task)
-  if task:has_flag('skip') then return end
-  if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then return end
-  local prefixes = task:cache_get('ratelimit_prefixes')
-
-  if prefixes then
-    if task:has_pre_result() then
-      -- Already rate limited/greylisted, do nothing
-      lua_util.debugm(N, task, 'pre-action has been set, do not update')
-      return
-    end
-
-    local verdict = lua_util.get_task_verdict(task)
-
-    -- Update each bucket
-    for k, v in pairs(prefixes) do
-      local bucket = v.bucket
-      local function update_bucket_cb(err, data)
-        if err then
-          rspamd_logger.errx(task, 'cannot update rate bucket %s: %s',
-                  k, err)
-        else
-          lua_util.debugm(N, task,
-              "updated limit %s:%s -> %s (%s/%s), burst: %s, dyn_rate: %s, dyn_burst: %s",
-              v.name, k, v.hash,
-              bucket.burst, bucket.rate,
-              data[1], data[2], data[3])
-        end
-      end
-      local now = rspamd_util.get_time()
-      now = lua_util.round(now * 1000.0) -- Get milliseconds
-      local mult_burst = 1.0
-      local mult_rate = 1.0
-
-      if verdict == 'spam' or verdict == 'junk' then
-        mult_burst = bucket.spam_factor_burst or 1.0
-        mult_rate = bucket.spam_factor_rate or 1.0
-      elseif verdict == 'ham' then
-        mult_burst = bucket.ham_factor_burst or 1.0
-        mult_rate = bucket.ham_factor_rate or 1.0
-      end
-
-      lua_redis.exec_redis_script(bucket_update_id,
-              {key = v.hash, task = task, is_write = true},
-              update_bucket_cb,
-              {v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst),
-               tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult),
-               tostring(settings.expire)})
-    end
-  end
-end
-
-local opts = rspamd_config:get_all_opt(N)
-if opts then
-
-  settings = lua_util.override_defaults(settings, opts)
-
-  if opts['limit'] then
-    rspamd_logger.errx(rspamd_config, 'Legacy ratelimit config format no longer supported')
-  end
-
-  if opts['rates'] and type(opts['rates']) == 'table' then
-    -- new way of setting limits
-    fun.each(function(t, lim)
-      local buckets = {}
-
-      if type(lim) == 'table' and lim.bucket then
-
-        if lim.bucket[1] then
-          for _,bucket in ipairs(lim.bucket) do
-            local b = parse_limit(t, bucket)
-
-            if not b then
-              rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"',
-                  t, b)
-              return
-            end
-
-            table.insert(buckets, b)
-          end
-        else
-          local bucket = parse_limit(t, lim.bucket)
-
-          if not bucket then
-            rspamd_logger.errx(rspamd_config, 'bad ratelimit bucket for %s: "%s"',
-                t, lim.bucket)
-            return
-          end
-
-          buckets = {bucket}
-        end
-
-        settings.limits[t] = {
-          buckets = buckets
-        }
-
-        if lim.selector then
-          local selector = lua_selectors.parse_selector(rspamd_config, lim.selector)
-          if not selector then
-            rspamd_logger.errx(rspamd_config, 'bad ratelimit selector for %s: "%s"',
-                t, lim.selector)
-            settings.limits[t] = nil
-            return
-          end
-
-          settings.limits[t].selector = selector
-        end
-      else
-        rspamd_logger.warnx(rspamd_config, 'old syntax for ratelimits: %s', lim)
-        buckets = parse_limit(t, lim)
-        if buckets then
-          settings.limits[t] = {
-            buckets = {buckets}
-          }
-        end
-      end
-    end, opts['rates'])
-  end
-
-  -- Display what's enabled
-  fun.each(function(s)
-    rspamd_logger.infox(rspamd_config, 'enabled ratelimit: %s', s)
-  end, fun.map(function(n,d)
-    return string.format('%s [%s]', n,
-        table.concat(fun.totable(fun.map(function(v)
-          return string.format('%s msgs burst, %s msgs/sec rate',
-              v.burst, v.rate)
-        end, d.buckets)), '; ')
-    )
-  end, settings.limits))
-
-  -- Ret, ret, ret: stupid legacy stuff:
-  -- If we have a string with commas then load it as as static map
-  -- otherwise, apply normal logic of Rspamd maps
-
-  local wrcpts = opts['whitelisted_rcpts']
-  if type(wrcpts) == 'string' then
-    if string.find(wrcpts, ',') then
-      settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(
-        lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts')
-    else
-      settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set',
-        'Ratelimit whitelisted rcpts')
-    end
-  elseif type(opts['whitelisted_rcpts']) == 'table' then
-    settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set',
-      'Ratelimit whitelisted rcpts')
-  else
-    -- Stupid default...
-    settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(
-        settings.whitelisted_rcpts, 'set', 'Ratelimit whitelisted rcpts')
-  end
-
-  if opts['whitelisted_ip'] then
-    settings.whitelisted_ip = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_ip', 'radix',
-      'Ratelimit whitelist ip map')
-  end
-
-  if opts['whitelisted_user'] then
-    settings.whitelisted_user = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_user', 'set',
-      'Ratelimit whitelist user map')
-  end
-
-  settings.custom_keywords = {}
-  if opts['custom_keywords'] then
-    local ret, res_or_err = pcall(loadfile(opts['custom_keywords']))
-
-    if ret then
-      opts['custom_keywords'] = {}
-      if type(res_or_err) == 'table' then
-        for k,hdl in pairs(res_or_err) do
-          settings['custom_keywords'][k] = hdl
-        end
-      elseif type(res_or_err) == 'function' then
-        settings['custom_keywords']['custom'] = res_or_err
-      end
-    else
-      rspamd_logger.errx(rspamd_config, 'cannot execute %s: %s',
-          opts['custom_keywords'], res_or_err)
-      settings['custom_keywords'] = {}
-    end
-  end
-
-  if opts['message_func'] then
-    message_func = assert(load(opts['message_func']))()
-  end
-
-  redis_params = lua_redis.parse_redis_server('ratelimit')
-
-  if not redis_params then
-    rspamd_logger.infox(rspamd_config, 'no servers are specified, disabling module')
-    lua_util.disable_module(N, "redis")
-  else
-    local s = {
-      type = 'prefilter,nostat',
-      name = 'RATELIMIT_CHECK',
-      priority = 7,
-      callback = ratelimit_cb,
-      flags = 'empty',
-    }
-
-    if settings.symbol then
-      s.name = settings.symbol
-    elseif settings.info_symbol then
-      s.name = settings.info_symbol
-    end
-
-    rspamd_config:register_symbol(s)
-    rspamd_config:register_symbol {
-      type = 'idempotent',
-      name = 'RATELIMIT_UPDATE',
-      callback = ratelimit_update_cb,
-    }
-  end
-end
-
-rspamd_config:add_on_load(function(cfg, ev_base, worker)
-  load_scripts(cfg, ev_base)
-end)

data/conf/rspamd/meta_exporter/pipe.php

@@ -51,7 +51,7 @@ $raw_data = mb_convert_encoding($raw_data_content, 'HTML-ENTITIES', "UTF-8");
 $headers = getallheaders();
 
 $qid      = $headers['X-Rspamd-Qid'];
-$subject      = $headers['X-Rspamd-Subject'];
+$subject  = $headers['X-Rspamd-Subject'];
 $score    = $headers['X-Rspamd-Score'];
 $rcpts    = $headers['X-Rspamd-Rcpt'];
 $user     = $headers['X-Rspamd-User'];
@@ -190,7 +190,7 @@ foreach ($rcpt_final_mailboxes as $rcpt) {
   error_log("QUARANTINE: quarantine pipe: processing quarantine message for rcpt " . $rcpt);
   try {
     $stmt = $pdo->prepare("INSERT INTO `quarantine` (`qid`, `subject`, `score`, `sender`, `rcpt`, `symbols`, `user`, `ip`, `msg`, `action`)
-      VALUES (:qid, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action)");
+      VALUES (:qid, :subject, :score, :sender, :rcpt, :symbols, :user, :ip, :msg, :action)");
     $stmt->execute(array(
       ':qid' => $qid,
       ':subject' => $subject,

data/web/inc/functions.quarantine.inc.php

@@ -393,7 +393,7 @@ function quarantine($_action, $_data = null) {
     break;
     case 'get':
       if ($_SESSION['mailcow_cc_role'] == "user") {
-        $stmt = $pdo->prepare('SELECT `id`, `qid`, `rcpt`, `sender`, UNIX_TIMESTAMP(`created`) AS `created` FROM `quarantine` WHERE `rcpt` = :mbox');
+        $stmt = $pdo->prepare('SELECT `id`, `qid`, `subject`, `rcpt`, `sender`, UNIX_TIMESTAMP(`created`) AS `created` FROM `quarantine` WHERE `rcpt` = :mbox');
         $stmt->execute(array(':mbox' => $_SESSION['mailcow_cc_username']));
         $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
         while($row = array_shift($rows)) {
@@ -401,7 +401,7 @@ function quarantine($_action, $_data = null) {
         }
       }
       elseif ($_SESSION['mailcow_cc_role'] == "admin") {
-        $stmt = $pdo->query('SELECT `id`, `qid`, `rcpt`, `sender`, UNIX_TIMESTAMP(`created`) AS `created` FROM `quarantine`');
+        $stmt = $pdo->query('SELECT `id`, `qid`, `subject`, `rcpt`, `sender`, UNIX_TIMESTAMP(`created`) AS `created` FROM `quarantine`');
         $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
         while($row = array_shift($rows)) {
           $q_meta[] = $row;
@@ -410,7 +410,7 @@ function quarantine($_action, $_data = null) {
       else {
         $domains = array_merge(mailbox('get', 'domains'), mailbox('get', 'alias_domains'));
         foreach ($domains as $domain) {
-          $stmt = $pdo->prepare('SELECT `id`, `qid`, `rcpt`, `sender`, UNIX_TIMESTAMP(`created`) AS `created` FROM `quarantine` WHERE `rcpt` REGEXP :domain');
+          $stmt = $pdo->prepare('SELECT `id`, `qid`, `subject`, `rcpt`, `sender`, UNIX_TIMESTAMP(`created`) AS `created` FROM `quarantine` WHERE `rcpt` REGEXP :domain');
           $stmt->execute(array(':domain' => '@' . $domain . '$'));
           $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
           while($row = array_shift($rows)) {

data/web/inc/init_db.inc.php

@@ -3,7 +3,7 @@ function init_db_schema() {
   try {
     global $pdo;
 
-    $db_version = "16012019_0717";
+    $db_version = "17012019_0717";
 
     $stmt = $pdo->query("SHOW TABLES LIKE 'versions'");
     $num_results = count($stmt->fetchAll(PDO::FETCH_ASSOC));
@@ -230,7 +230,7 @@ function init_db_schema() {
           "qid" => "VARCHAR(30) NOT NULL",
           "subject" => "VARCHAR(500)",
           "score" => "FLOAT(8,2)",
-          "ip" => "VARBINARY(16)",
+          "ip" => "VARCHAR(50)",
           "action" => "CHAR(20) NOT NULL DEFAULT 'unknown'",
           "symbols" => "JSON",
           "sender" => "VARCHAR(255) NOT NULL DEFAULT 'unknown'",

data/web/js/quarantine.js

@@ -14,6 +14,7 @@ jQuery(function($){
         {"name":"qid","type":"text","title":lang.qid,"style":{"width":"125px"}},
         {"name":"sender","style":{"word-break":"break-all"},"title":lang.sender,"breakpoints":"xs sm"},
         {"name":"rcpt","title":lang.rcpt, "type": "text"},
+        {"name":"subject","title":"Subject", "type": "text"},
         {"name":"created","formatter":function unix_time_format(tm) { var date = new Date(tm ? tm * 1000 : 0); return date.toLocaleString();},"title":lang.received,"style":{"width":"170px"}},
         {"name":"action","filterable": false,"sortable": false,"style":{"text-align":"right"},"style":{"width":"220px"},"type":"html","title":lang.action,"breakpoints":"xs sm"}
       ],
@@ -26,6 +27,12 @@ jQuery(function($){
         },
         success: function (data) {
           $.each(data, function (i, item) {
+            if (item.subject === null) {
+              item.subject = '<i>no preview</i>';
+            }
+            else {
+              item.subject = escapeHtml(item.subject);
+            }
             item.action = '<div class="btn-group">' +
               '<a href="#" data-item="' + encodeURI(item.id) + '" class="btn btn-xs btn-info show_qid_info"><span class="glyphicon glyphicon-modal-window"></span> ' + lang.show_item + '</a>' +
               '<a href="#" data-action="delete_selected" data-id="del-single-qitem" data-api-url="delete/qitem" data-item="' + encodeURI(item.id) + '" class="btn btn-xs btn-danger"><span class="glyphicon glyphicon-trash"></span> ' + lang.remove + '</a>' +

data/web/mailbox.php

@@ -193,7 +193,7 @@ $_SESSION['return_to'] = $_SERVER['REQUEST_URI'];
                   <li><a data-action="edit_selected" data-id="syncjob" data-api-url='edit/syncjob' data-api-attr='{"active":"1"}' href="#"><?=$lang['mailbox']['activate'];?></a></li>
                   <li><a data-action="edit_selected" data-id="syncjob" data-api-url='edit/syncjob' data-api-attr='{"active":"0"}' href="#"><?=$lang['mailbox']['deactivate'];?></a></li>
                   <li role="separator" class="divider"></li>
-                  <li><a data-action="delete_selected" data-text="<?=$lang['user']['eas_reset'];?>?" data-id="syncjob" data-api-url='delete/syncjob' href="#"><?=$lang['mailbox']['remove'];?></a></li>
+                  <li><a data-action="delete_selected" data-id="syncjob" data-api-url='delete/syncjob' href="#"><?=$lang['mailbox']['remove'];?></a></li>
                 </ul>
                 <a class="btn btn-sm btn-success" href="#" data-toggle="modal" data-target="#addSyncJobModalAdmin"><span class="glyphicon glyphicon-plus"></span> <?=$lang['user']['create_syncjob'];?></a>
               </div>

docker-compose.yml

@@ -71,7 +71,7 @@ services:
             - clamd
 
     rspamd-mailcow:
-      image: mailcow/rspamd:1.33
+      image: mailcow/rspamd:1.34
       build: ./data/Dockerfiles/rspamd
       stop_grace_period: 30s
       depends_on: