diff -u -r -N fail2ban-0.11.2/MANIFEST fail2ban/MANIFEST --- fail2ban-0.11.2/MANIFEST 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/MANIFEST 2021-10-29 13:50:59.498326039 +0200 @@ -3,10 +3,9 @@ bin/fail2ban-server bin/fail2ban-testcases ChangeLog +config/action.d/apprise.conf config/action.d/abuseipdb.conf config/action.d/apf.conf -config/action.d/badips.conf -config/action.d/badips.py config/action.d/blocklist_de.conf config/action.d/bsd-ipfw.conf config/action.d/cloudflare.conf @@ -220,7 +219,6 @@ fail2ban-testcases-all fail2ban-testcases-all-python3 fail2ban/tests/action_d/__init__.py -fail2ban/tests/action_d/test_badips.py fail2ban/tests/action_d/test_smtp.py fail2ban/tests/actionstestcase.py fail2ban/tests/actiontestcase.py diff -u -r -N fail2ban-0.11.2/config/action.d/apprise.conf fail2ban/config/action.d/apprise.conf --- fail2ban-0.11.2/config/action.d/apprise.conf 1970-01-01 01:00:00.000000000 +0100 +++ fail2ban/config/action.d/apprise.conf 2021-10-29 13:50:59.513326002 +0200 @@ -0,0 +1,49 @@ +# Fail2Ban configuration file +# +# Author: Chris Caron +# +# + +[Definition] + +# Option: actionstart +# Notes.: command executed once at the start of Fail2Ban. +# Values: CMD +# +actionstart = printf %%b "The jail as been started successfully." | -t "[Fail2Ban] : started on `uname -n`" + +# Option: actionstop +# Notes.: command executed once at the end of Fail2Ban +# Values: CMD +# +actionstop = printf %%b "The jail has been stopped." | -t "[Fail2Ban] : stopped on `uname -n`" + +# Option: actioncheck +# Notes.: command executed once before each actionban command +# Values: CMD +# +actioncheck = + +# Option: actionban +# Notes.: command executed when banning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionban = printf %%b "The IP has just been banned by Fail2Ban after attempts against " | -n "warning" -t "[Fail2Ban] : banned from `uname -n`" + +# Option: actionunban +# Notes.: command executed when unbanning an IP. Take care that the +# command is executed with Fail2Ban user rights. +# Tags: See jail.conf(5) man page +# Values: CMD +# +actionunban = + +[Init] + +# Define location of the default apprise configuration file to use +# +config = /etc/fail2ban/apprise.conf +# +apprise = apprise -c "" diff -u -r -N fail2ban-0.11.2/config/action.d/badips.conf fail2ban/config/action.d/badips.conf --- fail2ban-0.11.2/config/action.d/badips.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/badips.conf 1970-01-01 01:00:00.000000000 +0100 @@ -1,19 +0,0 @@ -# Fail2ban reporting to badips.com -# -# Note: This reports an IP only and does not actually ban traffic. Use -# another action in the same jail if you want bans to occur. -# -# Set the category to the appropriate value before use. -# -# To get see register and optional key to get personalised graphs see: -# http://www.badips.com/blog/personalized-statistics-track-the-attackers-of-all-your-servers-with-one-key - -[Definition] - -actionban = curl --fail --user-agent "" http://www.badips.com/add// - -[Init] - -# Option: category -# Notes.: Values are from the list here: http://www.badips.com/get/categories -category = diff -u -r -N fail2ban-0.11.2/config/action.d/badips.py fail2ban/config/action.d/badips.py --- fail2ban-0.11.2/config/action.d/badips.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/badips.py 1970-01-01 01:00:00.000000000 +0100 @@ -1,391 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import sys -if sys.version_info < (2, 7): # pragma: no cover - raise ImportError("badips.py action requires Python >= 2.7") -import json -import threading -import logging -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.request import Request, urlopen - from urllib.parse import urlencode - from urllib.error import HTTPError -else: # pragma: 3.x no cover - from urllib2 import Request, urlopen, HTTPError - from urllib import urlencode - -from fail2ban.server.actions import Actions, ActionBase, BanTicket -from fail2ban.helpers import splitwords, str2LogLevel - - - -class BadIPsAction(ActionBase): # pragma: no cover - may be unavailable - """Fail2Ban action which reports bans to badips.com, and also - blacklist bad IPs listed on badips.com by using another action's - ban method. - - Parameters - ---------- - jail : Jail - The jail which the action belongs to. - name : str - Name assigned to the action. - category : str - Valid badips.com category for reporting failures. - score : int, optional - Minimum score for bad IPs. Default 3. - age : str, optional - Age of last report for bad IPs, per badips.com syntax. - Default "24h" (24 hours) - banaction : str, optional - Name of banaction to use for blacklisting bad IPs. If `None`, - no blacklist of IPs will take place. - Default `None`. - bancategory : str, optional - Name of category to use for blacklisting, which can differ - from category used for reporting. e.g. may want to report - "postfix", but want to use whole "mail" category for blacklist. - Default `category`. - bankey : str, optional - Key issued by badips.com to retrieve personal list - of blacklist IPs. - updateperiod : int, optional - Time in seconds between updating bad IPs blacklist. - Default 900 (15 minutes) - loglevel : int/str, optional - Log level of the message when an IP is (un)banned. - Default `DEBUG`. - Can be also supplied as two-value list (comma- or space separated) to - provide level of the summary message when a group of IPs is (un)banned. - Example `DEBUG,INFO`. - agent : str, optional - User agent transmitted to server. - Default `Fail2Ban/ver.` - - Raises - ------ - ValueError - If invalid `category`, `score`, `banaction` or `updateperiod`. - """ - - TIMEOUT = 10 - _badips = "https://www.badips.com" - def _Request(self, url, **argv): - return Request(url, headers={'User-Agent': self.agent}, **argv) - - def __init__(self, jail, name, category, score=3, age="24h", - banaction=None, bancategory=None, bankey=None, updateperiod=900, - loglevel='DEBUG', agent="Fail2Ban", timeout=TIMEOUT): - super(BadIPsAction, self).__init__(jail, name) - - self.timeout = timeout - self.agent = agent - self.category = category - self.score = score - self.age = age - self.banaction = banaction - self.bancategory = bancategory or category - self.bankey = bankey - loglevel = splitwords(loglevel) - self.sumloglevel = str2LogLevel(loglevel[-1]) - self.loglevel = str2LogLevel(loglevel[0]) - self.updateperiod = updateperiod - - self._bannedips = set() - # Used later for threading.Timer for updating badips - self._timer = None - - @staticmethod - def isAvailable(timeout=1): - try: - response = urlopen(Request("/".join([BadIPsAction._badips]), - headers={'User-Agent': "Fail2Ban"}), timeout=timeout) - return True, '' - except Exception as e: # pragma: no cover - return False, e - - def logError(self, response, what=''): # pragma: no cover - sporadical (502: Bad Gateway, etc) - messages = {} - try: - messages = json.loads(response.read().decode('utf-8')) - except: - pass - self._logSys.error( - "%s. badips.com response: '%s'", what, - messages.get('err', 'Unknown')) - - def getCategories(self, incParents=False): - """Get badips.com categories. - - Returns - ------- - set - Set of categories. - - Raises - ------ - HTTPError - Any issues with badips.com request. - ValueError - If badips.com response didn't contain necessary information - """ - try: - response = urlopen( - self._Request("/".join([self._badips, "get", "categories"])), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch categories") - raise - else: - response_json = json.loads(response.read().decode('utf-8')) - if not 'categories' in response_json: - err = "badips.com response lacked categories specification. Response was: %s" \ - % (response_json,) - self._logSys.error(err) - raise ValueError(err) - categories = response_json['categories'] - categories_names = set( - value['Name'] for value in categories) - if incParents: - categories_names.update(set( - value['Parent'] for value in categories - if "Parent" in value)) - return categories_names - - def getList(self, category, score, age, key=None): - """Get badips.com list of bad IPs. - - Parameters - ---------- - category : str - Valid badips.com category. - score : int - Minimum score for bad IPs. - age : str - Age of last report for bad IPs, per badips.com syntax. - key : str, optional - Key issued by badips.com to fetch IPs reported with the - associated key. - - Returns - ------- - set - Set of bad IPs. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "?".join([ - "/".join([self._badips, "get", "list", category, str(score)]), - urlencode({'age': age})]) - if key: - url = "&".join([url, urlencode({'key': key})]) - self._logSys.debug('badips.com: get list, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to fetch bad IP list") - raise - else: - return set(response.read().decode('utf-8').split()) - - @property - def category(self): - """badips.com category for reporting IPs. - """ - return self._category - - @category.setter - def category(self, category): - if category not in self.getCategories(): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - category) - raise ValueError("Invalid category: %s" % category) - self._category = category - - @property - def bancategory(self): - """badips.com bancategory for fetching IPs. - """ - return self._bancategory - - @bancategory.setter - def bancategory(self, bancategory): - if bancategory != "any" and bancategory not in self.getCategories(incParents=True): - self._logSys.error("Category name '%s' not valid. " - "see badips.com for list of valid categories", - bancategory) - raise ValueError("Invalid bancategory: %s" % bancategory) - self._bancategory = bancategory - - @property - def score(self): - """badips.com minimum score for fetching IPs. - """ - return self._score - - @score.setter - def score(self, score): - score = int(score) - if 0 <= score <= 5: - self._score = score - else: - raise ValueError("Score must be 0-5") - - @property - def banaction(self): - """Jail action to use for banning/unbanning. - """ - return self._banaction - - @banaction.setter - def banaction(self, banaction): - if banaction is not None and banaction not in self._jail.actions: - self._logSys.error("Action name '%s' not in jail '%s'", - banaction, self._jail.name) - raise ValueError("Invalid banaction") - self._banaction = banaction - - @property - def updateperiod(self): - """Period in seconds between banned bad IPs will be updated. - """ - return self._updateperiod - - @updateperiod.setter - def updateperiod(self, updateperiod): - updateperiod = int(updateperiod) - if updateperiod > 0: - self._updateperiod = updateperiod - else: - raise ValueError("Update period must be integer greater than 0") - - def _banIPs(self, ips): - for ip in ips: - try: - ai = Actions.ActionInfo(BanTicket(ip), self._jail) - self._jail.actions[self.banaction].ban(ai) - except Exception as e: - self._logSys.error( - "Error banning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._bannedips.add(ip) - self._logSys.log(self.loglevel, - "Banned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - - def _unbanIPs(self, ips): - for ip in ips: - try: - ai = Actions.ActionInfo(BanTicket(ip), self._jail) - self._jail.actions[self.banaction].unban(ai) - except Exception as e: - self._logSys.error( - "Error unbanning IP %s for jail '%s' with action '%s': %s", - ip, self._jail.name, self.banaction, e, - exc_info=self._logSys.getEffectiveLevel()<=logging.DEBUG) - else: - self._logSys.log(self.loglevel, - "Unbanned IP %s for jail '%s' with action '%s'", - ip, self._jail.name, self.banaction) - finally: - self._bannedips.remove(ip) - - def start(self): - """If `banaction` set, blacklists bad IPs. - """ - if self.banaction is not None: - self.update() - - def update(self): - """If `banaction` set, updates blacklisted IPs. - - Queries badips.com for list of bad IPs, removing IPs from the - blacklist if no longer present, and adds new bad IPs to the - blacklist. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - - try: - ips = self.getList( - self.bancategory, self.score, self.age, self.bankey) - # Remove old IPs no longer listed - s = self._bannedips - ips - m = len(s) - self._unbanIPs(s) - # Add new IPs which are now listed - s = ips - self._bannedips - p = len(s) - self._banIPs(s) - if m != 0 or p != 0: - self._logSys.log(self.sumloglevel, - "Updated IPs for jail '%s' (-%d/+%d)", - self._jail.name, m, p) - self._logSys.debug( - "Next update for jail '%' in %i seconds", - self._jail.name, self.updateperiod) - finally: - self._timer = threading.Timer(self.updateperiod, self.update) - self._timer.start() - - def stop(self): - """If `banaction` set, clears blacklisted IPs. - """ - if self.banaction is not None: - if self._timer: - self._timer.cancel() - self._timer = None - self._unbanIPs(self._bannedips.copy()) - - def ban(self, aInfo): - """Reports banned IP to badips.com. - - Parameters - ---------- - aInfo : dict - Dictionary which includes information in relation to - the ban. - - Raises - ------ - HTTPError - Any issues with badips.com request. - """ - try: - url = "/".join([self._badips, "add", self.category, str(aInfo['ip'])]) - self._logSys.debug('badips.com: ban, url: %r', url) - response = urlopen(self._Request(url), timeout=self.timeout) - except HTTPError as response: # pragma: no cover - self.logError(response, "Failed to ban") - raise - else: - messages = json.loads(response.read().decode('utf-8')) - self._logSys.debug( - "Response from badips.com report: '%s'", - messages['suc']) - -Action = BadIPsAction diff -u -r -N fail2ban-0.11.2/config/action.d/cloudflare.conf fail2ban/config/action.d/cloudflare.conf --- fail2ban-0.11.2/config/action.d/cloudflare.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/cloudflare.conf 2021-10-29 13:50:59.516325994 +0200 @@ -44,7 +44,7 @@ #actionban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=ban' -d 'tkn=' -d 'email=' -d 'key=' # API v4 actionban = curl -s -o /dev/null -X POST <_cf_api_prms> \ - -d '{"mode":"block","configuration":{"target":"ip","value":""},"notes":"Fail2Ban "}' \ + -d '{"mode":"block","configuration":{"target":"","value":""},"notes":"Fail2Ban "}' \ <_cf_api_url> # Option: actionunban @@ -59,7 +59,7 @@ #actionunban = curl -s -o /dev/null https://www.cloudflare.com/api_json.html -d 'a=nul' -d 'tkn=' -d 'email=' -d 'key=' # API v4 actionunban = id=$(curl -s -X GET <_cf_api_prms> \ - "<_cf_api_url>?mode=block&configuration_target=ip&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ + "<_cf_api_url>?mode=block&configuration_target=&configuration_value=&page=1&per_page=1¬es=Fail2Ban%%20" \ | { jq -r '.result[0].id' 2>/dev/null || tr -d '\n' | sed -nE 's/^.*"result"\s*:\s*\[\s*\{\s*"id"\s*:\s*"([^"]+)".*$/\1/p'; }) if [ -z "$id" ]; then echo ": id for cannot be found"; exit 0; fi; curl -s -o /dev/null -X DELETE <_cf_api_prms> "<_cf_api_url>/$id" @@ -81,3 +81,8 @@ cftoken = cfuser = + +cftarget = ip + +[Init?family=inet6] +cftarget = ip6 diff -u -r -N fail2ban-0.11.2/config/action.d/complain.conf fail2ban/config/action.d/complain.conf --- fail2ban-0.11.2/config/action.d/complain.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/complain.conf 2021-10-29 13:50:59.517325992 +0200 @@ -102,7 +102,7 @@ # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Option: mailargs # Notes.: Additional arguments to mail command. e.g. for standard Unix mail: diff -u -r -N fail2ban-0.11.2/config/action.d/dshield.conf fail2ban/config/action.d/dshield.conf --- fail2ban-0.11.2/config/action.d/dshield.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/dshield.conf 2021-10-29 13:50:59.518325989 +0200 @@ -179,7 +179,7 @@ # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Option: mailargs # Notes.: Additional arguments to mail command. e.g. for standard Unix mail: diff -u -r -N fail2ban-0.11.2/config/action.d/mail-buffered.conf fail2ban/config/action.d/mail-buffered.conf --- fail2ban-0.11.2/config/action.d/mail-buffered.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/mail-buffered.conf 2021-10-29 13:50:59.538325939 +0200 @@ -17,7 +17,7 @@ The jail has been started successfully.\n Output will be buffered until lines are available.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -28,13 +28,13 @@ These hosts have been banned by Fail2Ban.\n `cat ` Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : Summary from " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : Summary from " rm fi printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command @@ -55,7 +55,7 @@ These hosts have been banned by Fail2Ban.\n `cat ` \nRegards,\n - Fail2Ban"|mail -s "[Fail2Ban] : Summary" + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : Summary" rm fi diff -u -r -N fail2ban-0.11.2/config/action.d/mail-whois-lines.conf fail2ban/config/action.d/mail-whois-lines.conf --- fail2ban-0.11.2/config/action.d/mail-whois-lines.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/mail-whois-lines.conf 2021-10-29 13:50:59.539325937 +0200 @@ -72,7 +72,7 @@ # Notes.: Your system mail command. Is passed 2 args: subject and recipient # Values: CMD # -mailcmd = mail -s +mailcmd = mail -E 'set escape' -s # Default name of the chain # diff -u -r -N fail2ban-0.11.2/config/action.d/mail-whois.conf fail2ban/config/action.d/mail-whois.conf --- fail2ban-0.11.2/config/action.d/mail-whois.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/mail-whois.conf 2021-10-29 13:50:59.540325934 +0200 @@ -20,7 +20,7 @@ actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -29,7 +29,7 @@ actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command @@ -49,7 +49,7 @@ Here is more information about :\n `%(_whois_command)s`\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : banned from " # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff -u -r -N fail2ban-0.11.2/config/action.d/mail.conf fail2ban/config/action.d/mail.conf --- fail2ban-0.11.2/config/action.d/mail.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/mail.conf 2021-10-29 13:50:59.541325932 +0200 @@ -16,7 +16,7 @@ actionstart = printf %%b "Hi,\n The jail has been started successfully.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : started on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : started on " # Option: actionstop # Notes.: command executed at the stop of jail (or at the end of Fail2Ban) @@ -25,7 +25,7 @@ actionstop = printf %%b "Hi,\n The jail has been stopped.\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : stopped on " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : stopped on " # Option: actioncheck # Notes.: command executed once before each actionban command @@ -43,7 +43,7 @@ The IP has just been banned by Fail2Ban after attempts against .\n Regards,\n - Fail2Ban"|mail -s "[Fail2Ban] : banned from " + Fail2Ban"|mail -E 'set escape' -s "[Fail2Ban] : banned from " # Option: actionunban # Notes.: command executed when unbanning an IP. Take care that the diff -u -r -N fail2ban-0.11.2/config/action.d/nginx-block-map.conf fail2ban/config/action.d/nginx-block-map.conf --- fail2ban-0.11.2/config/action.d/nginx-block-map.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/action.d/nginx-block-map.conf 2021-10-29 13:50:59.545325922 +0200 @@ -84,8 +84,15 @@ #srv_cmd = nginx -c %(srv_cfg_path)s/nginx.conf srv_cmd = nginx -# first test configuration is correct, hereafter send reload signal: -blck_lst_reload = %(srv_cmd)s -qt; if [ $? -eq 0 ]; then +# pid file (used to check nginx is running): +srv_pid = /run/nginx.pid + +# command used to check whether nginx is running and configuration is valid: +srv_is_running = [ -f "%(srv_pid)s" ] +srv_check_cmd = %(srv_is_running)s && %(srv_cmd)s -qt + +# first test nginx is running and configuration is correct, hereafter send reload signal: +blck_lst_reload = %(srv_check_cmd)s; if [ $? -eq 0 ]; then %(srv_cmd)s -s reload; if [ $? -ne 0 ]; then echo 'reload failed.'; fi; fi; diff -u -r -N fail2ban-0.11.2/config/fail2ban.conf fail2ban/config/fail2ban.conf --- fail2ban-0.11.2/config/fail2ban.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/fail2ban.conf 2021-10-29 13:50:59.562325880 +0200 @@ -55,6 +55,12 @@ # pidfile = /var/run/fail2ban/fail2ban.pid +# Option: allowipv6 +# Notes.: Allows IPv6 interface: +# Default: auto +# Values: [ auto yes (on, true, 1) no (off, false, 0) ] Default: auto +#allowipv6 = auto + # Options: dbfile # Notes.: Set the file for the fail2ban persistent data to be stored. # A value of ":memory:" means database is only stored in memory diff -u -r -N fail2ban-0.11.2/config/filter.d/apache-overflows.conf fail2ban/config/filter.d/apache-overflows.conf --- fail2ban-0.11.2/config/filter.d/apache-overflows.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/filter.d/apache-overflows.conf 2021-10-29 13:50:59.570325860 +0200 @@ -8,7 +8,7 @@ [Definition] -failregex = ^%(_apache_error_client)s (?:(?:AH0013[456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b) +failregex = ^%(_apache_error_client)s (?:(?:AH001[23][456]: )?Invalid (method|URI) in request\b|(?:AH00565: )?request failed: URI too long \(longer than \d+\)|request failed: erroneous characters after protocol string:|(?:AH00566: )?request failed: invalid characters in URI\b) ignoreregex = diff -u -r -N fail2ban-0.11.2/config/filter.d/dovecot.conf fail2ban/config/filter.d/dovecot.conf --- fail2ban-0.11.2/config/filter.d/dovecot.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/filter.d/dovecot.conf 2021-10-29 13:50:59.580325835 +0200 @@ -8,14 +8,15 @@ [Definition] _auth_worker = (?:dovecot: )?auth(?:-worker)? +_auth_worker_info = (?:conn \w+:auth(?:-worker)? \(uid=\w+\): auth(?:-worker)?<\d+>: )? _daemon = (?:dovecot(?:-auth)?|auth) -prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap|managesieve|submission)-login: )?(?:Info: )?.+$ +prefregex = ^%(__prefix_line)s(?:%(_auth_worker)s(?:\([^\)]+\))?: )?(?:%(__pam_auth)s(?:\(dovecot:auth\))?: |(?:pop3|imap|managesieve|submission)-login: )?(?:Info: )?%(_auth_worker_info)s.+$ failregex = ^authentication failure; logname=\S* uid=\S* euid=\S* tty=dovecot ruser=\S* rhost=(?:\s+user=\S*)?\s*$ ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:auth failed, \d+ attempts(?: in \d+ secs)?|tried to use (?:disabled|disallowed) \S+ auth|proxy dest auth failed)\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=(?:[^>]*(?:, session=<\S+>)?)\s*$ - ^pam\(\S+,(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \(password mismatch\?\)|Permission denied)\s*$ - ^[a-z\-]{3,15}\(\S*,(?:,\S*)?\): (?:unknown user|invalid credentials|Password mismatch) + ^pam\(\S+,(?:,\S*)?\): pam_authenticate\(\) failed: (?:User not known to the underlying authentication module: \d+ Time\(s\)|Authentication failure \([Pp]assword mismatch\?\)|Permission denied)\s*$ + ^[a-z\-]{3,15}\(\S*,(?:,\S*)?\): (?:[Uu]nknown user|[Ii]nvalid credentials|[Pp]assword mismatch) > mdre-aggressive = ^(?:Aborted login|Disconnected|Remote closed connection|Client has quit the connection)(?::(?: [^ \(]+)+)? \((?:no auth attempts|disconnected before auth was ready,|client didn't finish \S+ auth,)(?: (?:in|waited) \d+ secs)?\):(?: user=<[^>]*>,)?(?: method=\S+,)? rip=(?:[^>]*(?:, session=<\S+>)?)\s*$ diff -u -r -N fail2ban-0.11.2/config/filter.d/exim-common.conf fail2ban/config/filter.d/exim-common.conf --- fail2ban-0.11.2/config/filter.d/exim-common.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/filter.d/exim-common.conf 2021-10-29 13:50:59.583325827 +0200 @@ -12,7 +12,7 @@ host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)? host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s host_info = %(host_info_pre)s\[\]%(host_info_suf)s -pid = (?: \[\d+\])? +pid = (?: \[\d+\]| \w+ exim\[\d+\]:)? # DEV Notes: # From exim source code: ./src/receive.c:add_host_info_for_log diff -u -r -N fail2ban-0.11.2/config/filter.d/ignorecommands/apache-fakegooglebot fail2ban/config/filter.d/ignorecommands/apache-fakegooglebot --- fail2ban-0.11.2/config/filter.d/ignorecommands/apache-fakegooglebot 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/filter.d/ignorecommands/apache-fakegooglebot 2021-10-29 13:50:59.594325800 +0200 @@ -6,24 +6,35 @@ # import sys from fail2ban.server.ipdns import DNSUtils, IPAddr +from threading import Thread def process_args(argv): - if len(argv) != 2: - raise ValueError("Please provide a single IP as an argument. Got: %s\n" - % (argv[1:])) + if len(argv) - 1 not in (1, 2): + raise ValueError("Usage %s ip ?timeout?. Got: %s\n" + % (argv[0], argv[1:])) ip = argv[1] if not IPAddr(ip).isValid: raise ValueError("Argument must be a single valid IP. Got: %s\n" % ip) - return ip + return argv[1:] google_ips = None -def is_googlebot(ip): +def is_googlebot(ip, timeout=55): import re - host = DNSUtils.ipToName(ip) + timeout = float(timeout or 0) + if timeout: + def ipToNameTO(host, ip, timeout): + host[0] = DNSUtils.ipToName(ip) + host = [None] + th = Thread(target=ipToNameTO, args=(host, ip, timeout)); th.daemon=True; th.start() + th.join(timeout) + host = host[0] + else: + host = DNSUtils.ipToName(ip) + if not host or not re.match(r'.*\.google(bot)?\.com$', host): return False host_ips = DNSUtils.dnsToIp(host) @@ -31,7 +42,7 @@ if __name__ == '__main__': # pragma: no cover try: - ret = is_googlebot(process_args(sys.argv)) + ret = is_googlebot(*process_args(sys.argv)) except ValueError as e: sys.stderr.write(str(e)) sys.exit(2) diff -u -r -N fail2ban-0.11.2/config/filter.d/named-refused.conf fail2ban/config/filter.d/named-refused.conf --- fail2ban-0.11.2/config/filter.d/named-refused.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/filter.d/named-refused.conf 2021-10-29 13:50:59.601325783 +0200 @@ -32,7 +32,7 @@ # hostname daemon_id spaces # this can be optional (for instance if we match named native log files) -__line_prefix=(?:\s\S+ %(__daemon_combs_re)s\s+)? +__line_prefix=(?:\s*\S+ %(__daemon_combs_re)s\s+)? prefregex = ^%(__line_prefix)s(?: error:)?\s*client(?: @\S*)? #\S+(?: \([\S.]+\))?: .+\s(?:denied|\(NOTAUTH\))\s*$ diff -u -r -N fail2ban-0.11.2/config/filter.d/postfix.conf fail2ban/config/filter.d/postfix.conf --- fail2ban-0.11.2/config/filter.d/postfix.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/filter.d/postfix.conf 2021-10-29 13:50:59.610325760 +0200 @@ -12,16 +12,12 @@ _daemon = postfix(-\w+)?/\w+(?:/smtp[ds])? _port = (?::\d+)? +_pref = [A-Z]{4} prefregex = ^%(__prefix_line)s> .+$ -mdpr-normal = (?:\w+: reject:|(?:improper command pipelining|too many errors) after \S+) -mdre-normal=^RCPT from [^[]*\[\]%(_port)s: 55[04] 5\.7\.1\s - ^RCPT from [^[]*\[\]%(_port)s: 45[04] 4\.7\.\d+ (?:Service unavailable\b|Client host rejected: cannot find your (reverse )?hostname\b) - ^RCPT from [^[]*\[\]%(_port)s: 450 4\.7\.\d+ (<[^>]*>)?: Helo command rejected: Host not found\b - ^EHLO from [^[]*\[\]%(_port)s: 504 5\.5\.\d+ (<[^>]*>)?: Helo command rejected: need fully-qualified hostname\b - ^(RCPT|VRFY) from [^[]*\[\]%(_port)s: 550 5\.1\.1\s - ^RCPT from [^[]*\[\]%(_port)s: 450 4\.1\.\d+ (<[^>]*>)?: Sender address rejected: Domain not found\b +mdpr-normal = (?:\w+: (?:milter-)?reject:|(?:improper command pipelining|too many errors) after \S+) +mdre-normal=^%(_pref)s from [^[]*\[\]%(_port)s: [45][50][04] [45]\.\d\.\d+ (?:(?:<[^>]*>)?: )?(?:(?:Helo command|(?:Sender|Recipient) address) rejected: )?(?:Service unavailable|User unknown|(?:Client host|Command|Data command) rejected|Relay access denied|(?:Host|Domain) not found|need fully-qualified hostname|match)\b ^from [^[]*\[\]%(_port)s:? mdpr-auth = warning: @@ -31,7 +27,7 @@ # Mode "rbl" currently included in mode "normal", but if needed for jail "postfix-rbl" only: mdpr-rbl = %(mdpr-normal)s -mdre-rbl = ^RCPT from [^[]*\[\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b +mdre-rbl = ^%(_pref)s from [^[]*\[\]%(_port)s: [45]54 [45]\.7\.1 Service unavailable; Client host \[\S+\] blocked\b # Mode "rbl" currently included in mode "normal" (within 1st rule) mdpr-more = %(mdpr-normal)s diff -u -r -N fail2ban-0.11.2/config/filter.d/sendmail-auth.conf fail2ban/config/filter.d/sendmail-auth.conf --- fail2ban-0.11.2/config/filter.d/sendmail-auth.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/filter.d/sendmail-auth.conf 2021-10-29 13:50:59.617325743 +0200 @@ -15,7 +15,7 @@ prefregex = ^%(__prefix_line)s.+$ failregex = ^(\S+ )?\[%(addr)s\]( \(may be forged\))?: possible SMTP attack: command=AUTH, count=\d+$ - ^AUTH failure \(LOGIN\):(?: [^:]+:)? authentication failure: checkpass failed, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ + ^AUTH failure \([^\)]+\):(?: [^:]+:)? (?:authentication failure|user not found): [^,]*, user=(?:\S+|.*?), relay=(?:\S+ )?\[%(addr)s\](?: \(may be forged\))?$ ignoreregex = journalmatch = _SYSTEMD_UNIT=sendmail.service diff -u -r -N fail2ban-0.11.2/config/filter.d/sendmail-reject.conf fail2ban/config/filter.d/sendmail-reject.conf --- fail2ban-0.11.2/config/filter.d/sendmail-reject.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/filter.d/sendmail-reject.conf 2021-10-29 13:50:59.618325740 +0200 @@ -21,12 +21,12 @@ _daemon = (?:(sm-(mta|acceptingconnections)|sendmail)) __prefix_line = %(known/__prefix_line)s(?:\w{14,20}: )? -addr = (?:IPv6:|) +addr = (?:(?:IPv6:)?|) prefregex = ^%(__prefix_line)s.+$ -cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(550 5\.7\.1 (?P=email)\.\.\. Relaying denied\. (IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\])|553 5\.1\.8 (?P=email)\.\.\. Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ - ^ruleset=check_relay, arg1=(?P\S+), arg2=%(addr)s, relay=((?P=dom) )?\[(\d+\.){3}\d+\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ +cmnfailre = ^ruleset=check_rcpt, arg1=(?P<\S+@\S+>), relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=(?:550 5\.7\.1(?: (?P=email)\.\.\.)?(?: Relaying denied\.)? (?:IP name possibly forged \[(\d+\.){3}\d+\]|Proper authentication required\.|IP name lookup failed \[(\d+\.){3}\d+\]|Fix reverse DNS for \S+)|553 5\.1\.8(?: (?P=email)\.\.\.)? Domain of sender address \S+ does not exist|550 5\.[71]\.1 (?P=email)\.\.\. (Rejected: .*|User unknown))$ + ^ruleset=check_relay(?:, arg\d+=\S*)*, relay=(\S+ )?\[%(addr)s\](?: \(may be forged\))?, reject=421 4\.3\.2 (Connection rate limit exceeded\.|Too many open connections\.)$ ^rejecting commands from (\S* )?\[%(addr)s\] due to pre-greeting traffic after \d+ seconds$ ^(?:\S+ )?\[%(addr)s\]: (?:(?i)expn|vrfy) \S+ \[rejected\]$ ^<[^@]+@[^>]+>\.\.\. No such user here$ diff -u -r -N fail2ban-0.11.2/config/jail.conf fail2ban/config/jail.conf --- fail2ban-0.11.2/config/jail.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/jail.conf 2021-10-29 13:50:59.633325703 +0200 @@ -227,6 +227,15 @@ action_xarf = %(action_)s xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] +# ban & send a notification to one or more of the 50+ services supported by Apprise. +# See https://github.com/caronc/apprise/wiki for details on what is supported. +# +# You may optionally over-ride the default configuration line (containing the Apprise URLs) +# by using 'apprise[config="/alternate/path/to/apprise.cfg"]' otherwise +# /etc/fail2ban/apprise.conf is sourced for your supported notification configuration. +# action = %(action_)s +# apprise + # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] @@ -242,20 +251,6 @@ # action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] -# Report ban via badips.com, and use as blacklist -# -# See BadIPsAction docstring in config/action.d/badips.py for -# documentation for this action. -# -# NOTE: This action relies on banaction being present on start and therefore -# should be last action defined for a jail. -# -action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] -# -# Report ban via badips.com (uses action.d/badips.conf for reporting only) -# -action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] - # Report ban via abuseipdb.com. # # See action.d/abuseipdb.conf for usage example and details. diff -u -r -N fail2ban-0.11.2/config/paths-debian.conf fail2ban/config/paths-debian.conf --- fail2ban-0.11.2/config/paths-debian.conf 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/config/paths-debian.conf 2021-10-29 13:50:59.636325696 +0200 @@ -26,3 +26,5 @@ # was in debian squeezy but not in wheezy # /etc/proftpd/proftpd.conf (SystemLog) proftpd_log = /var/log/proftpd/proftpd.log + +roundcube_errors_log = /var/log/roundcube/errors.log diff -u -r -N fail2ban-0.11.2/fail2ban/client/fail2bancmdline.py fail2ban/fail2ban/client/fail2bancmdline.py --- fail2ban-0.11.2/fail2ban/client/fail2bancmdline.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/client/fail2bancmdline.py 2021-10-29 13:50:59.688325566 +0200 @@ -192,7 +192,7 @@ cmdOpts = 'hc:s:p:xfbdtviqV' cmdLongOpts = ['loglevel=', 'logtarget=', 'syslogsocket=', 'test', 'async', 'conf=', 'pidfile=', 'pname=', 'socket=', - 'timeout=', 'str2sec=', 'help', 'version', 'dp', '--dump-pretty'] + 'timeout=', 'str2sec=', 'help', 'version', 'dp', 'dump-pretty'] optList, self._args = getopt.getopt(self._argv[1:], cmdOpts, cmdLongOpts) except getopt.GetoptError: self.dispUsage() diff -u -r -N fail2ban-0.11.2/fail2ban/client/fail2banreader.py fail2ban/fail2ban/client/fail2banreader.py --- fail2ban-0.11.2/fail2ban/client/fail2banreader.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/client/fail2banreader.py 2021-10-29 13:50:59.689325564 +0200 @@ -53,6 +53,7 @@ opts = [["string", "loglevel", "INFO" ], ["string", "logtarget", "STDERR"], ["string", "syslogsocket", "auto"], + ["string", "allowipv6", "auto"], ["string", "dbfile", "/var/lib/fail2ban/fail2ban.sqlite3"], ["int", "dbmaxmatches", None], ["string", "dbpurgeage", "1d"]] @@ -74,6 +75,7 @@ # Also dbfile should be set before all other database options. # So adding order indices into items, to be stripped after sorting, upon return order = {"thread":0, "syslogsocket":11, "loglevel":12, "logtarget":13, + "allowipv6": 14, "dbfile":50, "dbmaxmatches":51, "dbpurgeage":51} stream = list() for opt in self.__opts: diff -u -r -N fail2ban-0.11.2/fail2ban/client/fail2banregex.py fail2ban/fail2ban/client/fail2banregex.py --- fail2ban-0.11.2/fail2ban/client/fail2banregex.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/client/fail2banregex.py 2021-10-29 13:50:59.690325561 +0200 @@ -35,11 +35,11 @@ import getopt import logging +import re import os import shlex import sys import time -import time import urllib from optparse import OptionParser, Option @@ -52,7 +52,7 @@ from ..version import version, normVersion from .filterreader import FilterReader -from ..server.filter import Filter, FileContainer +from ..server.filter import Filter, FileContainer, MyTime from ..server.failregex import Regex, RegexException from ..helpers import str2LogLevel, getVerbosityFormat, FormatterWithTraceBack, getLogger, \ @@ -269,15 +269,19 @@ self.setJournalMatch(shlex.split(opts.journalmatch)) if opts.timezone: self._filter.setLogTimeZone(opts.timezone) + self._filter.checkFindTime = False + if True: # not opts.out: + MyTime.setAlternateNow(0); # accept every date (years from 19xx up to end of current century, '%ExY' and 'Exy' patterns) + from ..server.strptime import _updateTimeRE + _updateTimeRE() if opts.datepattern: self.setDatePattern(opts.datepattern) if opts.usedns: self._filter.setUseDns(opts.usedns) self._filter.returnRawHost = opts.raw - self._filter.checkFindTime = False self._filter.checkAllRegex = opts.checkAllRegex and not opts.out # ignore pending (without ID/IP), added to matches if it hits later (if ID/IP can be retreved) - self._filter.ignorePending = opts.out + self._filter.ignorePending = bool(opts.out) # callback to increment ignored RE's by index (during process): self._filter.onIgnoreRegex = self._onIgnoreRegex self._backend = 'auto' @@ -326,26 +330,33 @@ regex = regextype + 'regex' # try to check - we've case filter?[options...]?: basedir = self._opts.config + fltName = value fltFile = None fltOpt = {} if regextype == 'fail': - fltName, fltOpt = extractOptions(value) - if fltName is not None: - if "." in fltName[~5:]: - tryNames = (fltName,) - else: - tryNames = (fltName, fltName + '.conf', fltName + '.local') - for fltFile in tryNames: - if not "/" in fltFile: - if os.path.basename(basedir) == 'filter.d': - fltFile = os.path.join(basedir, fltFile) - else: - fltFile = os.path.join(basedir, 'filter.d', fltFile) + if re.search(r'^/{0,3}[\w/_\-.]+(?:\[.*\])?$', value): + try: + fltName, fltOpt = extractOptions(value) + if "." in fltName[~5:]: + tryNames = (fltName,) else: - basedir = os.path.dirname(fltFile) - if os.path.isfile(fltFile): - break - fltFile = None + tryNames = (fltName, fltName + '.conf', fltName + '.local') + for fltFile in tryNames: + if not "/" in fltFile: + if os.path.basename(basedir) == 'filter.d': + fltFile = os.path.join(basedir, fltFile) + else: + fltFile = os.path.join(basedir, 'filter.d', fltFile) + else: + basedir = os.path.dirname(fltFile) + if os.path.isfile(fltFile): + break + fltFile = None + except Exception as e: + output("ERROR: Wrong filter name or options: %s" % (str(e),)) + output(" while parsing: %s" % (value,)) + if self._verbose: raise(e) + return False # if it is filter file: if fltFile is not None: if (basedir == self._opts.config diff -u -r -N fail2ban-0.11.2/fail2ban/client/jailreader.py fail2ban/fail2ban/client/jailreader.py --- fail2ban-0.11.2/fail2ban/client/jailreader.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/client/jailreader.py 2021-10-29 13:50:59.693325554 +0200 @@ -140,9 +140,10 @@ # Read filter flt = self.__opts["filter"] if flt: - filterName, filterOpt = extractOptions(flt) - if not filterName: - raise JailDefError("Invalid filter definition %r" % flt) + try: + filterName, filterOpt = extractOptions(flt) + except ValueError as e: + raise JailDefError("Invalid filter definition %r: %s" % (flt, e)) self.__filter = FilterReader( filterName, self.__name, filterOpt, share_config=self.share_config, basedir=self.getBaseDir()) @@ -174,10 +175,10 @@ if not act: # skip empty actions continue # join with previous line if needed (consider possible new-line): - actName, actOpt = extractOptions(act) - prevln = '' - if not actName: - raise JailDefError("Invalid action definition %r" % act) + try: + actName, actOpt = extractOptions(act) + except ValueError as e: + raise JailDefError("Invalid action definition %r: %s" % (act, e)) if actName.endswith(".py"): self.__actions.append([ "set", diff -u -r -N fail2ban-0.11.2/fail2ban/helpers.py fail2ban/fail2ban/helpers.py --- fail2ban-0.11.2/fail2ban/helpers.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/helpers.py 2021-10-29 13:50:59.695325549 +0200 @@ -371,7 +371,7 @@ # since v0.10 separator extended with `]\s*[` for support of multiple option groups, syntax # `action = act[p1=...][p2=...]` OPTION_EXTRACT_CRE = re.compile( - r'([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$)', re.DOTALL) + r'\s*([\w\-_\.]+)=(?:"([^"]*)"|\'([^\']*)\'|([^,\]]*))(?:,|\]\s*\[|$|(?P.+))|,?\s*$|(?P.+)', re.DOTALL) # split by new-line considering possible new-lines within options [...]: OPTION_SPLIT_CRE = re.compile( r'(?:[^\[\s]+(?:\s*\[\s*(?:[\w\-_\.]+=(?:"[^"]*"|\'[^\']*\'|[^,\]]*)\s*(?:,|\]\s*\[)?\s*)*\])?\s*|\S+)(?=\n\s*|\s+|$)', re.DOTALL) @@ -379,13 +379,19 @@ def extractOptions(option): match = OPTION_CRE.match(option) if not match: - # TODO proper error handling - return None, None + raise ValueError("unexpected option syntax") option_name, optstr = match.groups() option_opts = dict() if optstr: for optmatch in OPTION_EXTRACT_CRE.finditer(optstr): + if optmatch.group("wrngA"): + raise ValueError("unexpected syntax at %d after option %r: %s" % ( + optmatch.start("wrngA"), optmatch.group(1), optmatch.group("wrngA")[0:25])) + if optmatch.group("wrngB"): + raise ValueError("expected option, wrong syntax at %d: %s" % ( + optmatch.start("wrngB"), optmatch.group("wrngB")[0:25])) opt = optmatch.group(1) + if not opt: continue value = [ val for val in optmatch.group(2,3,4) if val is not None][0] option_opts[opt.strip()] = value.strip() diff -u -r -N fail2ban-0.11.2/fail2ban/server/action.py fail2ban/fail2ban/server/action.py --- fail2ban-0.11.2/fail2ban/server/action.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/action.py 2021-10-29 13:50:59.699325539 +0200 @@ -30,7 +30,10 @@ import threading import time from abc import ABCMeta -from collections import MutableMapping +try: + from collections.abc import MutableMapping +except ImportError: + from collections import MutableMapping from .failregex import mapTag2Opt from .ipdns import DNSUtils diff -u -r -N fail2ban-0.11.2/fail2ban/server/actions.py fail2ban/fail2ban/server/actions.py --- fail2ban-0.11.2/fail2ban/server/actions.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/actions.py 2021-10-29 13:50:59.700325537 +0200 @@ -28,7 +28,10 @@ import os import sys import time -from collections import Mapping +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping try: from collections import OrderedDict except ImportError: diff -u -r -N fail2ban-0.11.2/fail2ban/server/datedetector.py fail2ban/fail2ban/server/datedetector.py --- fail2ban-0.11.2/fail2ban/server/datedetector.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/datedetector.py 2021-10-29 13:50:59.723325479 +0200 @@ -35,7 +35,7 @@ # Gets the instance of the logger. logSys = getLogger(__name__) -logLevel = 6 +logLevel = 5 RE_DATE_PREMATCH = re.compile(r"(? time) + if item.getTime() > time) self.__bgSvc.service() def delFailure(self, fid): diff -u -r -N fail2ban-0.11.2/fail2ban/server/filter.py fail2ban/fail2ban/server/filter.py --- fail2ban-0.11.2/fail2ban/server/filter.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/filter.py 2021-10-29 13:50:59.734325452 +0200 @@ -94,6 +94,8 @@ ## Store last time stamp, applicable for multi-line self.__lastTimeText = "" self.__lastDate = None + ## Next service (cleanup) time + self.__nextSvcTime = -(1<<63) ## if set, treat log lines without explicit time zone to be in this time zone self.__logtimezone = None ## Default or preferred encoding (to decode bytes from file or journal): @@ -115,10 +117,10 @@ self.checkFindTime = True ## shows that filter is in operation mode (processing new messages): self.inOperation = True - ## if true prevents against retarded banning in case of RC by too many failures (disabled only for test purposes): - self.banASAP = True ## Ticks counter self.ticks = 0 + ## Processed lines counter + self.procLines = 0 ## Thread name: self.name="f2b/f."+self.jailName @@ -442,12 +444,23 @@ def performBan(self, ip=None): """Performs a ban for IPs (or given ip) that are reached maxretry of the jail.""" - try: # pragma: no branch - exception is the only way out - while True: + while True: + try: ticket = self.failManager.toBan(ip) - self.jail.putFailTicket(ticket) - except FailManagerEmpty: - self.failManager.cleanup(MyTime.time()) + except FailManagerEmpty: + break + self.jail.putFailTicket(ticket) + if ip: break + self.performSvc() + + def performSvc(self, force=False): + """Performs a service tasks (clean failure list).""" + tm = MyTime.time() + # avoid too early clean up: + if force or tm >= self.__nextSvcTime: + self.__nextSvcTime = tm + 5 + # clean up failure list: + self.failManager.cleanup(tm) def addAttempt(self, ip, *matches): """Generate a failed attempt for ip""" @@ -695,11 +708,15 @@ attempts = self.failManager.addFailure(tick) # avoid RC on busy filter (too many failures) - if attempts for IP/ID reached maxretry, # we can speedup ban, so do it as soon as possible: - if self.banASAP and attempts >= self.failManager.getMaxRetry(): + if attempts >= self.failManager.getMaxRetry(): self.performBan(ip) # report to observer - failure was found, for possibly increasing of it retry counter (asynchronous) if Observers.Main is not None: Observers.Main.add('failureFound', self.failManager, self.jail, tick) + self.procLines += 1 + # every 100 lines check need to perform service tasks: + if self.procLines % 100 == 0: + self.performSvc() # reset (halve) error counter (successfully processed line): if self._errors: self._errors //= 2 @@ -709,7 +726,7 @@ # incr common error counter: self.commonError() - def commonError(self): + def commonError(self, reason="common", exc=None): # incr error counter, stop processing (going idle) after 100th error : self._errors += 1 # sleep a little bit (to get around time-related errors): @@ -1068,6 +1085,7 @@ # is created and is added to the FailManager. def getFailures(self, filename, inOperation=None): + if self.idle: return False log = self.getLog(filename) if log is None: logSys.error("Unable to get failures in %s", filename) diff -u -r -N fail2ban-0.11.2/fail2ban/server/filtergamin.py fail2ban/fail2ban/server/filtergamin.py --- fail2ban-0.11.2/fail2ban/server/filtergamin.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/filtergamin.py 2021-10-29 13:50:59.735325449 +0200 @@ -55,7 +55,6 @@ def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Gamin monitor self.monitor = gamin.WatchMonitor() fd = self.monitor.get_fd() @@ -67,21 +66,9 @@ logSys.log(4, "Got event: " + repr(event) + " for " + path) if event in (gamin.GAMCreated, gamin.GAMChanged, gamin.GAMExists): logSys.debug("File changed: " + path) - self.__modified = True self.ticks += 1 - self._process_file(path) - - def _process_file(self, path): - """Process a given file - - TODO -- RF: - this is a common logic and must be shared/provided by FileFilter - """ self.getFailures(path) - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False ## # Add a log file path @@ -128,6 +115,9 @@ Utils.wait_for(lambda: not self.active or self._handleEvents(), self.sleeptime) self.ticks += 1 + if self.ticks % 10 == 0: + self.performSvc() + logSys.debug("[%s] filter terminated", self.jailName) return True diff -u -r -N fail2ban-0.11.2/fail2ban/server/filterpoll.py fail2ban/fail2ban/server/filterpoll.py --- fail2ban-0.11.2/fail2ban/server/filterpoll.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/filterpoll.py 2021-10-29 13:50:59.736325447 +0200 @@ -27,9 +27,7 @@ import os import time -from .failmanager import FailManagerEmpty from .filter import FileFilter -from .mytime import MyTime from .utils import Utils from ..helpers import getLogger, logging @@ -55,7 +53,6 @@ def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False ## The time of the last modification of the file. self.__prevStats = dict() self.__file404Cnt = dict() @@ -115,20 +112,17 @@ break for filename in modlst: self.getFailures(filename) - self.__modified = True self.ticks += 1 - if self.__modified: - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break logSys.error("Caught unhandled exception in main cycle: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # incr common error counter: - self.commonError() + self.commonError("unhandled", e) logSys.debug("[%s] filter terminated", self.jailName) return True diff -u -r -N fail2ban-0.11.2/fail2ban/server/filterpyinotify.py fail2ban/fail2ban/server/filterpyinotify.py --- fail2ban-0.11.2/fail2ban/server/filterpyinotify.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/filterpyinotify.py 2021-10-29 13:50:59.738325442 +0200 @@ -75,7 +75,6 @@ def __init__(self, jail): FileFilter.__init__(self, jail) - self.__modified = False # Pyinotify watch manager self.__monitor = pyinotify.WatchManager() self.__notifier = None @@ -140,9 +139,6 @@ """ if not self.idle: self.getFailures(path) - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = False def _addPending(self, path, reason, isDir=False): if path not in self.__pending: @@ -169,7 +165,7 @@ return found = {} minTime = 60 - for path, (retardTM, isDir) in self.__pending.iteritems(): + for path, (retardTM, isDir) in list(self.__pending.items()): if ntm - self.__pendingChkTime < retardTM: if minTime > retardTM: minTime = retardTM continue @@ -192,7 +188,7 @@ self._refreshWatcher(path, isDir=isDir) if isDir: # check all files belong to this dir: - for logpath in self.__watchFiles: + for logpath in list(self.__watchFiles): if logpath.startswith(path + pathsep): # if still no file - add to pending, otherwise refresh and process: if not os.path.isfile(logpath): @@ -272,15 +268,13 @@ def _addLogPath(self, path): self._addFileWatcher(path) - # initial scan: + # notify (wake up if in waiting): if self.active: - # we can execute it right now: - self._process_file(path) - else: - # retard until filter gets started, isDir=None signals special case: process file only (don't need to refresh monitor): - self._addPending(path, ('INITIAL', path), isDir=None) + self.__pendingMinTime = 0 + # retard until filter gets started, isDir=None signals special case: process file only (don't need to refresh monitor): + self._addPending(path, ('INITIAL', path), isDir=None) - ## + ## # Delete a log path # # @param path the log file to delete @@ -291,7 +285,7 @@ logSys.error("Failed to remove watch on path: %s", path) path_dir = dirname(path) - for k in self.__watchFiles: + for k in list(self.__watchFiles): if k.startswith(path_dir + pathsep): path_dir = None break @@ -345,16 +339,26 @@ self.__notifier.process_events() # wait for events / timeout: - notify_maxtout = self.__notify_maxtout def __check_events(): - return not self.active or self.__notifier.check_events(timeout=notify_maxtout) - if Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)): + return ( + not self.active + or bool(self.__notifier.check_events(timeout=self.__notify_maxtout)) + or (self.__pendingMinTime and self.__pending) + ) + wres = Utils.wait_for(__check_events, min(self.sleeptime, self.__pendingMinTime)) + if wres: if not self.active: break - self.__notifier.read_events() + if not isinstance(wres, dict): + self.__notifier.read_events() + + self.ticks += 1 # check pending files/dirs (logrotate ready): - if not self.idle: - self._checkPending() + if self.idle: + continue + self._checkPending() + if self.ticks % 10 == 0: + self.performSvc() except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... @@ -362,10 +366,8 @@ logSys.error("Caught unhandled exception in main cycle: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # incr common error counter: - self.commonError() + self.commonError("unhandled", e) - self.ticks += 1 - logSys.debug("[%s] filter exited (pyinotifier)", self.jailName) self.__notifier = None diff -u -r -N fail2ban-0.11.2/fail2ban/server/filtersystemd.py fail2ban/fail2ban/server/filtersystemd.py --- fail2ban-0.11.2/fail2ban/server/filtersystemd.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/filtersystemd.py 2021-10-29 13:50:59.739325439 +0200 @@ -61,6 +61,7 @@ # Initialise systemd-journal connection self.__journal = journal.Reader(**jrnlargs) self.__matches = [] + self.__nextUpdateTM = 0 self.setDatePattern(None) logSys.debug("Created FilterSystemd") @@ -94,6 +95,11 @@ # be sure all journal types will be opened if files specified (don't set flags): if 'files' not in args or not len(args['files']): args['flags'] = 4 + + try: + args['namespace'] = kwargs.pop('namespace') + except KeyError: + pass return args @@ -280,6 +286,7 @@ except OSError: pass # Reading failure, so safe to ignore + line = None while self.active: # wait for records (or for timeout in sleeptime seconds): try: @@ -317,20 +324,26 @@ break else: break - if self.__modified: - if not self.banASAP: # pragma: no cover - self.performBan() - self.__modified = 0 - # update position in log (time and iso string): - if self.jail.database is not None: - self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) + self.__modified = 0 + if self.ticks % 10 == 0: + self.performSvc() + # update position in log (time and iso string): + if (line and self.jail.database and ( + self.ticks % 10 == 0 + or MyTime.time() >= self.__nextUpdateTM + or not self.active + ) + ): + self.jail.database.updateJournal(self.jail, 'systemd-journal', line[1], line[0][1]) + self.__nextUpdateTM = MyTime.time() + Utils.DEFAULT_SLEEP_TIME * 5 + line = None except Exception as e: # pragma: no cover if not self.active: # if not active - error by stop... break logSys.error("Caught unhandled exception in main cycle: %r", e, exc_info=logSys.getEffectiveLevel()<=logging.DEBUG) # incr common error counter: - self.commonError() + self.commonError("unhandled", e) logSys.debug("[%s] filter terminated", self.jailName) diff -u -r -N fail2ban-0.11.2/fail2ban/server/ipdns.py fail2ban/fail2ban/server/ipdns.py --- fail2ban-0.11.2/fail2ban/server/ipdns.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/ipdns.py 2021-10-29 13:50:59.740325437 +0200 @@ -169,27 +169,31 @@ DNSUtils.CACHE_ipToName.set(key, name) return name + # key find cached own hostnames (this tuple-key cannot be used elsewhere): + _getSelfNames_key = ('self','dns') + @staticmethod def getSelfNames(): """Get own host names of self""" - # try find cached own hostnames (this tuple-key cannot be used elsewhere): - key = ('self','dns') - names = DNSUtils.CACHE_ipToName.get(key) + # try find cached own hostnames: + names = DNSUtils.CACHE_ipToName.get(DNSUtils._getSelfNames_key) # get it using different ways (a set with names of localhost, hostname, fully qualified): if names is None: names = set([ 'localhost', DNSUtils.getHostname(False), DNSUtils.getHostname(True) ]) - set(['']) # getHostname can return '' # cache and return : - DNSUtils.CACHE_ipToName.set(key, names) + DNSUtils.CACHE_ipToName.set(DNSUtils._getSelfNames_key, names) return names + # key to find cached own IPs (this tuple-key cannot be used elsewhere): + _getSelfIPs_key = ('self','ips') + @staticmethod def getSelfIPs(): """Get own IP addresses of self""" - # try find cached own IPs (this tuple-key cannot be used elsewhere): - key = ('self','ips') - ips = DNSUtils.CACHE_nameToIp.get(key) + # to find cached own IPs: + ips = DNSUtils.CACHE_nameToIp.get(DNSUtils._getSelfIPs_key) # get it using different ways (a set with IPs of localhost, hostname, fully qualified): if ips is None: ips = set() @@ -199,13 +203,30 @@ except Exception as e: # pragma: no cover logSys.warning("Retrieving own IPs of %s failed: %s", hostname, e) # cache and return : - DNSUtils.CACHE_nameToIp.set(key, ips) + DNSUtils.CACHE_nameToIp.set(DNSUtils._getSelfIPs_key, ips) return ips + _IPv6IsAllowed = None + + @staticmethod + def setIPv6IsAllowed(value): + DNSUtils._IPv6IsAllowed = value + logSys.debug("IPv6 is %s", ('on' if value else 'off') if value is not None else 'auto') + return value + + # key to find cached value of IPv6 allowance (this tuple-key cannot be used elsewhere): + _IPv6IsAllowed_key = ('self','ipv6-allowed') + @staticmethod def IPv6IsAllowed(): - # return os.path.exists("/proc/net/if_inet6") || any((':' in ip) for ip in DNSUtils.getSelfIPs()) - return any((':' in ip.ntoa) for ip in DNSUtils.getSelfIPs()) + if DNSUtils._IPv6IsAllowed is not None: + return DNSUtils._IPv6IsAllowed + v = DNSUtils.CACHE_nameToIp.get(DNSUtils._IPv6IsAllowed_key) + if v is not None: + return v + v = any((':' in ip.ntoa) for ip in DNSUtils.getSelfIPs()) + DNSUtils.CACHE_nameToIp.set(DNSUtils._IPv6IsAllowed_key, v) + return v ## diff -u -r -N fail2ban-0.11.2/fail2ban/server/jails.py fail2ban/fail2ban/server/jails.py --- fail2ban-0.11.2/fail2ban/server/jails.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/jails.py 2021-10-29 13:50:59.742325432 +0200 @@ -22,7 +22,10 @@ __license__ = "GPL" from threading import Lock -from collections import Mapping +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping from ..exceptions import DuplicateJailException, UnknownJailException from .jail import Jail diff -u -r -N fail2ban-0.11.2/fail2ban/server/server.py fail2ban/fail2ban/server/server.py --- fail2ban-0.11.2/fail2ban/server/server.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/server.py 2021-10-29 13:50:59.746325422 +0200 @@ -34,7 +34,7 @@ from .observer import Observers, ObserverThread from .jails import Jails -from .filter import FileFilter, JournalFilter +from .filter import DNSUtils, FileFilter, JournalFilter from .transmitter import Transmitter from .asyncserver import AsyncServer, AsyncServerException from .. import version @@ -293,6 +293,11 @@ for name in self.__jails.keys(): self.delJail(name, stop=False, join=True) + def clearCaches(self): + # we need to clear caches, to be able to recognize new IPs/families etc: + DNSUtils.CACHE_nameToIp.clear() + DNSUtils.CACHE_ipToName.clear() + def reloadJails(self, name, opts, begin): if begin: # begin reload: @@ -314,6 +319,8 @@ if "--restart" in opts: self.stopJail(name) else: + # invalidate caches by reload + self.clearCaches() # first unban all ips (will be not restored after (re)start): if "--unban" in opts: self.setUnbanIP() @@ -803,6 +810,11 @@ logSys.info("flush performed on %s" % self.__logTarget) return "flushed" + @staticmethod + def setIPv6IsAllowed(value): + value = _as_bool(value) if value != 'auto' else None + return DNSUtils.setIPv6IsAllowed(value) + def setThreadOptions(self, value): for o, v in value.iteritems(): if o == 'stacksize': diff -u -r -N fail2ban-0.11.2/fail2ban/server/strptime.py fail2ban/fail2ban/server/strptime.py --- fail2ban-0.11.2/fail2ban/server/strptime.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/strptime.py 2021-10-29 13:50:59.748325417 +0200 @@ -30,17 +30,6 @@ TZ_ABBR_RE = r"[A-Z](?:[A-Z]{2,4})?" FIXED_OFFSET_TZ_RE = re.compile(r"(%s)?([+-][01]\d(?::?\d{2})?)?$" % (TZ_ABBR_RE,)) -def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): - """ Build century regex for last year and the next years (distance). - - Thereby respect possible run in the test-cases (alternate date used there) - """ - cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t] - exprset = set( cent(now[0].year + i) for i in (-1, distance) ) - if len(now) and now[1]: - exprset |= set( cent(now[1].year + i) for i in (-1, distance) ) - return "(?:%s)" % "|".join(exprset) if len(exprset) > 1 else "".join(exprset) - timeRE = TimeRE() # %k - one- or two-digit number giving the hour of the day (0-23) on a 24-hour clock, @@ -63,20 +52,68 @@ timeRE['ExZ'] = r"(?P%s)" % (TZ_ABBR_RE,) timeRE['Exz'] = r"(?P(?:%s)?[+-][01]\d(?::?\d{2})?|%s)" % (TZ_ABBR_RE, TZ_ABBR_RE) +# overwrite default patterns, since they can be non-optimal: +timeRE['d'] = r"(?P[1-2]\d|[0 ]?[1-9]|3[0-1])" +timeRE['m'] = r"(?P0?[1-9]|1[0-2])" +timeRE['Y'] = r"(?P\d{4})" +timeRE['H'] = r"(?P[0-1]?\d|2[0-3])" +timeRE['M'] = r"(?P[0-5]?\d)" +timeRE['S'] = r"(?P[0-5]?\d|6[0-1])" + # Extend build-in TimeRE with some exact patterns # exact two-digit patterns: -timeRE['Exd'] = r"(?P3[0-1]|[1-2]\d|0[1-9])" -timeRE['Exm'] = r"(?P1[0-2]|0[1-9])" -timeRE['ExH'] = r"(?P2[0-3]|[0-1]\d)" -timeRE['Exk'] = r" ?(?P2[0-3]|[0-1]\d|\d)" +timeRE['Exd'] = r"(?P[1-2]\d|0[1-9]|3[0-1])" +timeRE['Exm'] = r"(?P0[1-9]|1[0-2])" +timeRE['ExH'] = r"(?P[0-1]\d|2[0-3])" +timeRE['Exk'] = r" ?(?P[0-1]?\d|2[0-3])" timeRE['Exl'] = r" ?(?P1[0-2]|\d)" timeRE['ExM'] = r"(?P[0-5]\d)" -timeRE['ExS'] = r"(?P6[0-1]|[0-5]\d)" -# more precise year patterns, within same century of last year and -# the next 3 years (for possible long uptime of fail2ban); thereby -# respect possible run in the test-cases (alternate date used there): -timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3) -timeRE['Exy'] = r"(?P%s\d)" % _getYearCentRE(cent=(2,3), distance=3) +timeRE['ExS'] = r"(?P[0-5]\d|6[0-1])" + +def _updateTimeRE(): + def _getYearCentRE(cent=(0,3), distance=3, now=(MyTime.now(), MyTime.alternateNow)): + """ Build century regex for last year and the next years (distance). + + Thereby respect possible run in the test-cases (alternate date used there) + """ + cent = lambda year, f=cent[0], t=cent[1]: str(year)[f:t] + def grp(exprset): + c = None + if len(exprset) > 1: + for i in exprset: + if c is None or i[0:-1] == c: + c = i[0:-1] + else: + c = None + break + if not c: + for i in exprset: + if c is None or i[0] == c: + c = i[0] + else: + c = None + break + if c: + return "%s%s" % (c, grp([i[len(c):] for i in exprset])) + return ("(?:%s)" % "|".join(exprset) if len(exprset[0]) > 1 else "[%s]" % "".join(exprset)) \ + if len(exprset) > 1 else "".join(exprset) + exprset = set( cent(now[0].year + i) for i in (-1, distance) ) + if len(now) > 1 and now[1]: + exprset |= set( cent(now[1].year + i) for i in xrange(-1, now[0].year-now[1].year+1, distance) ) + return grp(sorted(list(exprset))) + + # more precise year patterns, within same century of last year and + # the next 3 years (for possible long uptime of fail2ban); thereby + # consider possible run in the test-cases (alternate date used there), + # so accept years: 20xx (from test-date or 2001 up to current century) + timeRE['ExY'] = r"(?P%s\d)" % _getYearCentRE(cent=(0,3), distance=3, + now=(datetime.datetime.now(), datetime.datetime.fromtimestamp( + min(MyTime.alternateNowTime or 978393600, 978393600)) + ) + ) + timeRE['Exy'] = r"(?P\d{2})" + +_updateTimeRE() def getTimePatternRE(): keys = timeRE.keys() @@ -168,9 +205,9 @@ """ now = \ - year = month = day = hour = minute = tzoffset = \ + year = month = day = tzoffset = \ weekday = julian = week_of_year = None - second = fraction = 0 + hour = minute = second = fraction = 0 for key, val in found_dict.iteritems(): if val is None: continue # Directives not explicitly handled below: diff -u -r -N fail2ban-0.11.2/fail2ban/server/transmitter.py fail2ban/fail2ban/server/transmitter.py --- fail2ban-0.11.2/fail2ban/server/transmitter.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/server/transmitter.py 2021-10-29 13:50:59.750325412 +0200 @@ -173,6 +173,11 @@ return self.__server.getSyslogSocket() else: raise Exception("Failed to change syslog socket") + elif name == "allowipv6": + value = command[1] + self.__server.setIPv6IsAllowed(value) + if self.__quiet: return + return value #Thread elif name == "thread": value = command[1] diff -u -r -N fail2ban-0.11.2/fail2ban/tests/action_d/test_badips.py fail2ban/fail2ban/tests/action_d/test_badips.py --- fail2ban-0.11.2/fail2ban/tests/action_d/test_badips.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/action_d/test_badips.py 1970-01-01 01:00:00.000000000 +0100 @@ -1,157 +0,0 @@ -# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- -# vi: set ft=python sts=4 ts=4 sw=4 noet : - -# This file is part of Fail2Ban. -# -# Fail2Ban is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# Fail2Ban is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Fail2Ban; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -import os -import unittest -import sys -from functools import wraps -from socket import timeout -from ssl import SSLError - -from ..actiontestcase import CallingMap -from ..dummyjail import DummyJail -from ..servertestcase import IPAddr -from ..utils import LogCaptureTestCase, CONFIG_DIR - -if sys.version_info >= (3, ): # pragma: 2.x no cover - from urllib.error import HTTPError, URLError -else: # pragma: 3.x no cover - from urllib2 import HTTPError, URLError - -def skip_if_not_available(f): - """Helper to decorate tests to skip in case of timeout/http-errors like "502 bad gateway". - """ - @wraps(f) - def wrapper(self, *args): - try: - return f(self, *args) - except (SSLError, HTTPError, URLError, timeout) as e: # pragma: no cover - timeout/availability issues - if not isinstance(e, timeout) and 'timed out' not in str(e): - if not hasattr(e, 'code') or e.code > 200 and e.code <= 404: - raise - raise unittest.SkipTest('Skip test because of %s' % e) - return wrapper - -if sys.version_info >= (2,7): # pragma: no cover - may be unavailable - class BadIPsActionTest(LogCaptureTestCase): - - available = True, None - pythonModule = None - modAction = None - - @skip_if_not_available - def setUp(self): - """Call before every test case.""" - super(BadIPsActionTest, self).setUp() - unittest.F2B.SkipIfNoNetwork() - - self.jail = DummyJail() - - self.jail.actions.add("test") - - pythonModuleName = os.path.join(CONFIG_DIR, "action.d", "badips.py") - - # check availability (once if not alive, used shorter timeout as in test cases): - if BadIPsActionTest.available[0]: - if not BadIPsActionTest.modAction: - if not BadIPsActionTest.pythonModule: - BadIPsActionTest.pythonModule = self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.modAction = BadIPsActionTest.pythonModule.Action - self.jail.actions._load_python_module(pythonModuleName) - BadIPsActionTest.available = BadIPsActionTest.modAction.isAvailable(timeout=2 if unittest.F2B.fast else 30) - if not BadIPsActionTest.available[0]: - raise unittest.SkipTest('Skip test because service is not available: %s' % BadIPsActionTest.available[1]) - - self.jail.actions.add("badips", pythonModuleName, initOpts={ - 'category': "ssh", - 'banaction': "test", - 'age': "2w", - 'score': 5, - #'key': "fail2ban-test-suite", - #'bankey': "fail2ban-test-suite", - 'timeout': (3 if unittest.F2B.fast else 60), - }) - self.action = self.jail.actions["badips"] - - def tearDown(self): - """Call after every test case.""" - # Must cancel timer! - if self.action._timer: - self.action._timer.cancel() - super(BadIPsActionTest, self).tearDown() - - @skip_if_not_available - def testCategory(self): - categories = self.action.getCategories() - self.assertIn("ssh", categories) - self.assertTrue(len(categories) >= 10) - - self.assertRaises( - ValueError, setattr, self.action, "category", - "invalid-category") - - # Not valid for reporting category... - self.assertRaises( - ValueError, setattr, self.action, "category", "mail") - # but valid for blacklisting. - self.action.bancategory = "mail" - - @skip_if_not_available - def testScore(self): - self.assertRaises(ValueError, setattr, self.action, "score", -5) - self.action.score = 3 - self.action.score = "3" - - @skip_if_not_available - def testBanaction(self): - self.assertRaises( - ValueError, setattr, self.action, "banaction", - "invalid-action") - self.action.banaction = "test" - - @skip_if_not_available - def testUpdateperiod(self): - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", -50) - self.assertRaises( - ValueError, setattr, self.action, "updateperiod", 0) - self.action.updateperiod = 900 - self.action.updateperiod = "900" - - @skip_if_not_available - def testStartStop(self): - self.action.start() - self.assertTrue(len(self.action._bannedips) > 10, - "%s is fewer as 10: %r" % (len(self.action._bannedips), self.action._bannedips)) - self.action.stop() - self.assertTrue(len(self.action._bannedips) == 0) - - @skip_if_not_available - def testBanIP(self): - aInfo = CallingMap({ - 'ip': IPAddr('192.0.2.1') - }) - self.action.ban(aInfo) - self.assertLogged('badips.com: ban', wait=True) - self.pruneLog() - # produce an error using wrong category/IP: - self.action._category = 'f2b-this-category-dont-available-test-suite-only' - aInfo['ip'] = '' - self.assertRaises(BadIPsActionTest.pythonModule.HTTPError, self.action.ban, aInfo) - self.assertLogged('IP is invalid', 'invalid category', wait=True, all=False) diff -u -r -N fail2ban-0.11.2/fail2ban/tests/banmanagertestcase.py fail2ban/fail2ban/tests/banmanagertestcase.py --- fail2ban-0.11.2/fail2ban/tests/banmanagertestcase.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/banmanagertestcase.py 2021-10-29 13:50:59.760325387 +0200 @@ -29,6 +29,7 @@ from .utils import setUpMyTime, tearDownMyTime from ..server.banmanager import BanManager +from ..server.ipdns import DNSUtils from ..server.ticket import BanTicket class AddFailure(unittest.TestCase): @@ -176,10 +177,10 @@ super(StatusExtendedCymruInfo, self).setUp() unittest.F2B.SkipIfNoNetwork() setUpMyTime() - self.__ban_ip = "93.184.216.34" - self.__asn = "15133" - self.__country = "EU" - self.__rir = "ripencc" + self.__ban_ip = iter(DNSUtils.dnsToIp("resolver1.opendns.com")).next() + self.__asn = "36692" + self.__country = "US" + self.__rir = "arin" ticket = BanTicket(self.__ban_ip, 1167605999.0) self.__banManager = BanManager() self.assertTrue(self.__banManager.addBanTicket(ticket)) diff -u -r -N fail2ban-0.11.2/fail2ban/tests/clientreadertestcase.py fail2ban/fail2ban/tests/clientreadertestcase.py --- fail2ban-0.11.2/fail2ban/tests/clientreadertestcase.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/clientreadertestcase.py 2021-10-29 13:50:59.763325380 +0200 @@ -381,13 +381,16 @@ self.assertEqual(('mail.who_is', {'a':'cat', 'b':'dog'}), extractOptions("mail.who_is[a=cat,b=dog]")) self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is")) - self.assertEqual(('mail--ho_is', {}), extractOptions("mail--ho_is['s']")) - #print(self.getLog()) - #self.assertLogged("Invalid argument ['s'] in ''s''") - self.assertEqual(('mail', {'a': ','}), extractOptions("mail[a=',']")) + self.assertEqual(('mail', {'a': 'b'}), extractOptions("mail[a=b, ]")) + + self.assertRaises(ValueError, extractOptions ,'mail-how[') + + self.assertRaises(ValueError, extractOptions, """mail[a="test with interim (wrong) "" quotes"]""") + self.assertRaises(ValueError, extractOptions, """mail[a='test with interim (wrong) '' quotes']""") + self.assertRaises(ValueError, extractOptions, """mail[a='x, y, z', b=x, y, z]""") - #self.assertRaises(ValueError, extractOptions ,'mail-how[') + self.assertRaises(ValueError, extractOptions, """mail['s']""") # Empty option option = "abc[]" @@ -455,8 +458,6 @@ ('sender', 'f2b-test@example.com'), ('blocklist_de_apikey', 'test-key'), ('action', '%(action_blocklist_de)s\n' - '%(action_badips_report)s\n' - '%(action_badips)s\n' 'mynetwatchman[port=1234,protocol=udp,agent="%(fail2ban_agent)s"]' ), )) @@ -470,16 +471,14 @@ if len(cmd) <= 4: continue # differentiate between set and multi-set (wrop it here to single set): - if cmd[0] == 'set' and (cmd[4] == 'agent' or cmd[4].endswith('badips.py')): + if cmd[0] == 'set' and cmd[4] == 'agent': act.append(cmd) elif cmd[0] == 'multi-set': act.extend([['set'] + cmd[1:4] + o for o in cmd[4] if o[0] == 'agent']) useragent = 'Fail2Ban/%s' % version - self.assertEqual(len(act), 4) + self.assertEqual(len(act), 2) self.assertEqual(act[0], ['set', 'blocklisttest', 'action', 'blocklist_de', 'agent', useragent]) - self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'badips', 'agent', useragent]) - self.assertEqual(eval(act[2][5]).get('agent', ''), useragent) - self.assertEqual(act[3], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) + self.assertEqual(act[1], ['set', 'blocklisttest', 'action', 'mynetwatchman', 'agent', useragent]) @with_tmpdir def testGlob(self, d): @@ -752,9 +751,9 @@ ['add', 'tz_correct', 'auto'], ['start', 'tz_correct'], ['config-error', - "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo'"], + "Jail 'brokenactiondef' skipped, because of wrong configuration: Invalid action definition 'joho[foo': unexpected option syntax"], ['config-error', - "Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test'"], + "Jail 'brokenfilterdef' skipped, because of wrong configuration: Invalid filter definition 'flt[test': unexpected option syntax"], ['config-error', "Jail 'missingaction' skipped, because of wrong configuration: Unable to read action 'noactionfileforthisaction'"], ['config-error', @@ -975,6 +974,7 @@ ['set', 'syslogsocket', 'auto'], ['set', 'loglevel', "INFO"], ['set', 'logtarget', '/var/log/fail2ban.log'], + ['set', 'allowipv6', 'auto'], ['set', 'dbfile', '/var/lib/fail2ban/fail2ban.sqlite3'], ['set', 'dbmaxmatches', 10], ['set', 'dbpurgeage', '1d'], diff -u -r -N fail2ban-0.11.2/fail2ban/tests/datedetectortestcase.py fail2ban/fail2ban/tests/datedetectortestcase.py --- fail2ban-0.11.2/fail2ban/tests/datedetectortestcase.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/datedetectortestcase.py 2021-10-29 13:50:59.777325345 +0200 @@ -551,6 +551,9 @@ (1123970401.0, "^%ExH:%ExM:%ExS**", '00:00:01'), # cover date with current year, in test cases now == Aug 2005 -> back to last year (Sep 2004): (1094068799.0, "^%m/%d %ExH:%ExM:%ExS**", '09/01 21:59:59'), + # no time (only date) in pattern, assume local 00:00:00 for H:M:S : + (1093989600.0, "^%Y-%m-%d**", '2004-09-01'), + (1093996800.0, "^%Y-%m-%d%z**", '2004-09-01Z'), ): logSys.debug('== test: %r', (matched, dp, line)) dd = DateDetector() diff -u -r -N fail2ban-0.11.2/fail2ban/tests/fail2banclienttestcase.py fail2ban/fail2ban/tests/fail2banclienttestcase.py --- fail2ban-0.11.2/fail2ban/tests/fail2banclienttestcase.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/fail2banclienttestcase.py 2021-10-29 13:50:59.779325340 +0200 @@ -1326,7 +1326,7 @@ 'backend = polling', 'usedns = no', 'logpath = %(tmp)s/blck-failures.log', - 'action = nginx-block-map[blck_lst_reload="", blck_lst_file="%(tmp)s/blck-lst.map"]', + 'action = nginx-block-map[srv_cmd="echo nginx", srv_pid="%(tmp)s/f2b.pid", blck_lst_file="%(tmp)s/blck-lst.map"]', ' blocklist_de[actionban=\'curl() { echo "*** curl" "$*";}; \', email="Fail2Ban ", ' 'apikey="TEST-API-KEY", agent="fail2ban-test-agent", service=]', 'filter =', @@ -1366,6 +1366,8 @@ self.assertIn('\\125-000-004 1;\n', mp) self.assertIn('\\125-000-005 1;\n', mp) + # check nginx reload is logged (pid of fail2ban is used to simulate success check nginx is running): + self.assertLogged("stdout: 'nginx -qt'", "stdout: 'nginx -s reload'", all=True) # check blocklist_de substitution (e. g. new-line after ): self.assertLogged( "stdout: '*** curl --fail --data-urlencode server=Fail2Ban " diff -u -r -N fail2ban-0.11.2/fail2ban/tests/fail2banregextestcase.py fail2ban/fail2ban/tests/fail2banregextestcase.py --- fail2ban-0.11.2/fail2ban/tests/fail2banregextestcase.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/fail2banregextestcase.py 2021-10-29 13:50:59.781325335 +0200 @@ -141,6 +141,12 @@ )) self.assertLogged("Unable to compile regular expression") + def testWrongFilterOptions(self): + self.assertFalse(_test_exec( + "test", "flt[a='x,y,z',b=z,y,x]" + )) + self.assertLogged("Wrong filter name or options", "wrong syntax at 14: y,x", all=True) + def testDirectFound(self): self.assertTrue(_test_exec( "--datepattern", r"^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?", @@ -395,7 +401,17 @@ "Found a match but no valid date/time found", "Match without a timestamp:", all=True) - self.pruneLog() + def testIncompleteDateTime(self): + # datepattern in followed lines doesn't match previously known pattern + line is too short + # (logging break-off, no flush, etc): + self.assertTrue(_test_exec( + '-o', 'Found-ADDR:', + '192.0.2.1 - - [02/May/2021:18:40:55 +0100] "GET / HTTP/1.1" 302 328 "-" "Mozilla/5.0" "-"\n' + '192.0.2.2 - - [02/May/2021:18:40:55 +0100\n' + '192.0.2.3 - - [02/May/2021:18:40:55', + '^')) + self.assertLogged( + "Found-ADDR:192.0.2.1", "Found-ADDR:192.0.2.2", "Found-ADDR:192.0.2.3", all=True) def testFrmtOutputWrapML(self): unittest.F2B.SkipIfCfgMissing(stock=True) diff -u -r -N fail2ban-0.11.2/fail2ban/tests/files/logs/apache-overflows fail2ban/fail2ban/tests/files/logs/apache-overflows --- fail2ban-0.11.2/fail2ban/tests/files/logs/apache-overflows 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/files/logs/apache-overflows 2021-10-29 13:50:59.850325163 +0200 @@ -3,6 +3,8 @@ [Tue Mar 16 15:39:29 2010] [error] [client 58.179.109.179] Invalid URI in request \xf9h\xa9\xf3\x88\x8cXKj \xbf-l*4\x87n\xe4\xfe\xd4\x1d\x06\x8c\xf8m\\rS\xf6n\xeb\x8 # failJSON: { "time": "2010-03-15T15:44:47", "match": true , "host": "121.222.2.133" } [Mon Mar 15 15:44:47 2010] [error] [client 121.222.2.133] Invalid URI in request n\xed*\xbe*\xab\xefd\x80\xb5\xae\xf6\x01\x10M?\xf2\xce\x13\x9c\xd7\xa0N\xa7\xdb%0\xde\xe0\xfc\xd2\xa0\xfe\xe9w\xee\xc4`v\x9b[{\x0c:\xcb\x93\xc6\xa0\x93\x9c`l\\\x8d\xc9 +# failJSON: { "time": "2010-03-15T16:04:06", "match": true , "host": "192.0.2.1", "desc": "AH00126 failure, gh-2908" } +[Sat Mar 15 16:04:06.105212 2010] [core:error] [pid 17408] [client 192.0.2.1:55280] AH00126: Invalid URI in request GET /static/../../../a/../../../../etc/passwd HTTP/1.1 # http://forum.nconf.org/viewtopic.php?f=14&t=427&p=1488 # failJSON: { "time": "2010-07-30T11:23:54", "match": true , "host": "10.85.6.69" } diff -u -r -N fail2ban-0.11.2/fail2ban/tests/files/logs/dovecot fail2ban/fail2ban/tests/files/logs/dovecot --- fail2ban-0.11.2/fail2ban/tests/files/logs/dovecot 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/files/logs/dovecot 2021-10-29 13:50:59.863325131 +0200 @@ -34,6 +34,9 @@ # failJSON: { "time": "2005-01-29T05:32:50", "match": true , "host": "1.2.3.4" } Jan 29 05:32:50 mail dovecot: auth-worker(304): pam(username,1.2.3.4): pam_authenticate() failed: Authentication failure (password mismatch?) +# failJSON: { "time": "2005-01-29T18:55:55", "match": true , "host": "192.0.2.4", "desc": "Password mismatch (title case, gh-2880)" } +Jan 29 18:55:55 mail dovecot: auth-worker(12182): pam(user,192.0.2.4): pam_authenticate() failed: Authentication failure (Password mismatch?) + # failJSON: { "time": "2005-01-29T05:13:40", "match": true , "host": "1.2.3.4" } Jan 29 05:13:40 mail dovecot: auth-worker(31326): pam(username,1.2.3.4): unknown user @@ -52,6 +55,11 @@ #failJSON: { "time": "2005-06-11T13:57:17", "match": true, "host": "192.168.178.25", "desc": "allow more verbose logging, gh-2573" } Jun 11 13:57:17 main dovecot: auth: ldap(user@server.org,192.168.178.25,): Password mismatch (for LDAP bind) (SHA1 of given password: f638ff) +# failJSON: { "time": "2005-06-12T11:48:12", "match": true , "host": "192.0.2.6" } +Jun 12 11:48:12 auth-worker(80180): Info: conn unix:auth-worker (uid=143): auth-worker<13247>: sql(support,192.0.2.6): unknown user +# failJSON: { "time": "2005-06-12T23:06:05", "match": true , "host": "192.0.2.7" } +Jun 12 23:06:05 auth-worker(57065): Info: conn unix:auth-worker (uid=143): auth-worker<225622>: sql(user@domain.com,192.0.2.7,): Password mismatch + # failJSON: { "time": "2005-01-29T14:38:51", "match": true , "host": "192.0.2.6", "desc": "PAM Permission denied (gh-1897)" } Jan 29 14:38:51 example.com dovecot[24941]: auth-worker(30165): pam(user@example.com,192.0.2.6,): pam_authenticate() failed: Permission denied diff -u -r -N fail2ban-0.11.2/fail2ban/tests/files/logs/exim fail2ban/fail2ban/tests/files/logs/exim --- fail2ban-0.11.2/fail2ban/tests/files/logs/exim 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/files/logs/exim 2021-10-29 13:50:59.866325124 +0200 @@ -43,6 +43,9 @@ # failJSON: { "time": "2014-01-12T02:07:48", "match": true , "host": "85.214.85.40" } 2014-01-12 02:07:48 dovecot_login authenticator failed for h1832461.stratoserver.net (User) [85.214.85.40]: 535 Incorrect authentication data (set_id=scanner) +# failJSON: { "time": "2019-10-22T03:39:17", "match": true , "host": "192.0.2.37", "desc": "pid-prefix in form of 'mx1 exim[...]:', gh-2553" } +2019-10-22 03:39:17 mx1 exim[29786]: dovecot_login authenticator failed for (User) [192.0.2.37]: 535 Incorrect authentication data (set_id=test@domain.com) + # failJSON: { "time": "2014-12-02T03:00:23", "match": true , "host": "193.254.202.35" } 2014-12-02 03:00:23 auth_plain authenticator failed for (rom182) [193.254.202.35]:41556 I=[10.0.0.1]:25: 535 Incorrect authentication data (set_id=webmaster) diff -u -r -N fail2ban-0.11.2/fail2ban/tests/files/logs/named-refused fail2ban/fail2ban/tests/files/logs/named-refused --- fail2ban-0.11.2/fail2ban/tests/files/logs/named-refused 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/files/logs/named-refused 2021-10-29 13:50:59.879325091 +0200 @@ -26,3 +26,8 @@ # failJSON: { "time": "2004-08-27T16:59:00", "match": true , "host": "192.0.2.1", "desc": "new log format, 9.11.0 (#2406)" } Aug 27 16:59:00 host named[28098]: client @0x7f6450002ef0 192.0.2.1#23332 (example.com): bad zone transfer request: 'test.com/IN': non-authoritative zone (NOTAUTH) + +# filterOptions: {"logtype": "journal"} + +# failJSON: { "match": true , "host": "192.0.2.1", "desc": "systemd-journal entry" } +atom named[1806]: client @0x7fb13400eec0 192.0.2.1#61977 (.): query (cache) './ANY/IN' denied diff -u -r -N fail2ban-0.11.2/fail2ban/tests/files/logs/postfix fail2ban/fail2ban/tests/files/logs/postfix --- fail2ban-0.11.2/fail2ban/tests/files/logs/postfix 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/files/logs/postfix 2021-10-29 13:50:59.887325071 +0200 @@ -35,6 +35,16 @@ # failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } Jan 31 13:55:24 xxx postfix/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= +# failJSON: { "time": "2005-03-7T02:09:33", "match": true , "host": "192.0.2.151", "desc": "reject: DATA from, gh-2927" } +Mar 7 02:09:33 server postfix/smtpd[27246]: 1D8CC1CA0A7F: milter-reject: DATA from 66-220-155-151.mail-mail.facebook.com[192.0.2.151]: 550 5.7.1 Command rejected; from= to= proto=ESMTP helo=<192-0-2-151.mail-mail.example.com> +# failJSON: { "time": "2005-03-11T23:27:54", "match": true , "host": "192.0.2.109", "desc": "reject: BDAT from, gh-2927" } +Mar 11 23:27:54 server postfix-smo/submission/smtpd[22427]: 44JCRG5tYPzCqt2: reject: BDAT from signing-milter.example.com[192.0.2.109]: 550 5.5.3 : Data command rejected: Multi-recipient bounce; from=<> to= proto=ESMTP helo= + +# failJSON: { "time": "2005-04-06T13:05:01", "match": true , "host": "192.0.2.116", "desc": "RCPT from unknown, gh-2995" } +Apr 6 13:05:01 server postfix/smtpd[20589]: NOQUEUE: reject: RCPT from unknown[192.0.2.116]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; from= to= proto=ESMTP helo= +# failJSON: { "time": "2005-04-07T03:10:56", "match": true , "host": "192.0.2.246", "desc": "550 5.7.25 Client host rejected, gh-2996" } +Apr 7 03:10:56 server postfix/smtpd[7754]: NOQUEUE: reject: RCPT from unknown[192.0.2.246]: 550 5.7.25 Client host rejected: cannot find your hostname, [192.0.2.246]; from= to= proto=ESMTP helo=<[192.0.2.246]> + # failJSON: { "time": "2005-01-31T13:55:24", "match": true , "host": "78.107.251.238" } Jan 31 13:55:24 xxx postfix-incoming/smtpd[3462]: NOQUEUE: reject: EHLO from s271272.static.corbina.ru[78.107.251.238]: 504 5.5.2 : Helo command rejected: need fully-qualified hostname; proto=SMTP helo= diff -u -r -N fail2ban-0.11.2/fail2ban/tests/files/logs/sendmail-auth fail2ban/fail2ban/tests/files/logs/sendmail-auth --- fail2ban-0.11.2/fail2ban/tests/files/logs/sendmail-auth 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/files/logs/sendmail-auth 2021-10-29 13:50:59.893325056 +0200 @@ -22,3 +22,13 @@ Feb 25 04:02:27 relay1 sendmail[16664]: 06I02CNi016764: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=user@example.com, relay=example.com [192.0.2.3] (may be forged) # failJSON: { "time": "2005-02-25T04:02:28", "match": true , "host": "192.0.2.4", "desc": "injection attempt on user name" } Feb 25 04:02:28 relay1 sendmail[16665]: 06I02CNi016765: AUTH failure (LOGIN): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=criminal, relay=[192.0.2.100], relay=[192.0.2.4] (may be forged) + +# failJSON: { "time": "2005-05-24T01:58:40", "match": true , "host": "192.0.2.5", "desc": "user not found (gh-3030)" } +May 24 01:58:40 server sm-mta[65696]: 14NNwaRl065696: AUTH failure (DIGEST-MD5): user not found (-20) SASL(-13): user not found: unable to canonify user and get auxprops, user=scanner, relay=[192.0.2.5] +# failJSON: { "time": "2005-05-24T01:59:07", "match": true , "host": "192.0.2.6", "desc": "user not found (gh-3030)" } +May 24 01:59:07 server sm-mta[65815]: 14NNx65Q065815: AUTH failure (CRAM-MD5): user not found (-20) SASL(-13): user not found: user: scan@server.example.com property: userPassword not found in sasldb /usr/local/etc/sasldb2, user=scan, relay=[192.0.2.6] + +# failJSON: { "time": "2005-05-29T23:14:04", "match": true , "host": "192.0.2.7", "desc": "authentication failure, sendmail 8.16.1 (gh-2757)" } +May 29 23:14:04 mail sendmail[5976]: 09DJDgOM005976: AUTH failure (login): authentication failure (-13) SASL(-13): authentication failure: checkpass failed, user=test, relay=host.example.com [192.0.2.7] (may be forged) +# failJSON: { "time": "2005-05-29T23:14:04", "match": true , "host": "192.0.2.8", "desc": "authentication failure, sendmail 8.16.1 (gh-2757)" } +May 29 23:14:04 mail sendmail[5976]: 09DJDgOM005976: AUTH failure (PLAIN): authentication failure (-13) SASL(-13): authentication failure: Password verification failed, user=test, relay=host.example.com [192.0.2.8] diff -u -r -N fail2ban-0.11.2/fail2ban/tests/files/logs/sendmail-reject fail2ban/fail2ban/tests/files/logs/sendmail-reject --- fail2ban-0.11.2/fail2ban/tests/files/logs/sendmail-reject 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/files/logs/sendmail-reject 2021-10-29 13:50:59.894325054 +0200 @@ -40,6 +40,9 @@ # failJSON: { "time": "2005-02-19T18:01:50", "match": true , "host": "196.213.73.146" } Feb 19 18:01:50 batman sm-mta[78152]: ruleset=check_relay, arg1=[196.213.73.146], arg2=196.213.73.146, relay=[196.213.73.146], reject=421 4.3.2 Connection rate limit exceeded. +# failJSON: { "time": "2005-02-19T20:17:12", "match": true , "host": "192.0.2.123" } +Feb 19 20:17:12 server sm-mta[201892]: ruleset=check_relay, arg1=[192.0.2.123], arg2=192.0.2.123, relay=host.example.com [192.0.2.123] (may be forged), reject=421 4.3.2 Connection rate limit exceeded. + # failJSON: { "time": "2005-02-27T10:53:06", "match": true , "host": "209.15.212.253" } Feb 27 10:53:06 batman sm-mta[44307]: s1R9r60D044307: rejecting commands from [209.15.212.253] due to pre-greeting traffic after 0 seconds # failJSON: { "time": "2005-02-27T10:53:07", "match": true , "host": "1.2.3.4" } @@ -69,6 +72,8 @@ # failJSON: { "time": "2005-02-22T14:02:44", "match": true , "host": "24.73.201.194" } Feb 22 14:02:44 batman sm-mta[4030]: s1MD2hsd004030: rrcs-24-73-201-194.se.biz.rr.com [24.73.201.194]: VRFY root [rejected] +# failJSON: { "time": "2005-02-22T15:20:27", "match": true , "host": "192.0.2.5", "desc": "Fix reverse DNS for ... (gh-3012)" } +Feb 22 15:20:27 localhost sm-mta[275631]: 13O9Ixhq275631: ruleset=check_rcpt, arg1=, relay=[192.0.2.5], reject=550 5.7.1 ... Fix reverse DNS for 192.0.2.5 # failJSON: { "match": false } Nov 3 11:35:30 Microsoft sendmail[26254]: rA37ZTSC026250: ... No such user here @@ -106,4 +111,4 @@ # failJSON: { "time": "2005-03-29T22:51:45", "match": true , "host": "192.0.2.3", "desc": "sendmail 8.15.2 default names IPv4/6 (gh-2787)" } Mar 29 22:51:45 server sm-mta[50437]: 06QDQnNf050437: example.com [192.0.2.3] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv4 # failJSON: { "time": "2005-03-29T22:51:46", "match": true , "host": "2001:DB8::1", "desc": "IPv6" } -Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6 \ No newline at end of file +Mar 29 22:51:46 server sm-mta[50438]: 06QDQnNf050438: example.com [IPv6:2001:DB8::1] did not issue MAIL/EXPN/VRFY/ETRN during connection to IPv6 diff -u -r -N fail2ban-0.11.2/fail2ban/tests/filtertestcase.py fail2ban/fail2ban/tests/filtertestcase.py --- fail2ban-0.11.2/fail2ban/tests/filtertestcase.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/filtertestcase.py 2021-10-29 13:50:59.920324989 +0200 @@ -164,23 +164,31 @@ # get fail ticket from jail found.append(_ticket_tuple(filter_.getFailTicket())) else: - # when we are testing without jails - # wait for failures (up to max time) - Utils.wait_for( - lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), - _maxWaitTime(10)) - # get fail ticket(s) from filter - while tickcount: - try: - found.append(_ticket_tuple(filter_.failManager.toBan())) - except FailManagerEmpty: - break - tickcount -= 1 + # when we are testing without jails wait for failures (up to max time) + if filter_.jail: + while True: + t = filter_.jail.getFailTicket() + if not t: break + found.append(_ticket_tuple(t)) + if found: + tickcount -= len(found) + if tickcount > 0: + Utils.wait_for( + lambda: filter_.failManager.getFailCount() >= (tickcount, failcount), + _maxWaitTime(10)) + # get fail ticket(s) from filter + while tickcount: + try: + found.append(_ticket_tuple(filter_.failManager.toBan())) + except FailManagerEmpty: + break + tickcount -= 1 if not isinstance(output[0], (tuple,list)): utest.assertEqual(len(found), 1) _assert_equal_entries(utest, found[0], output, count) else: + utest.assertEqual(len(found), len(output)) # sort by string representation of ip (multiple failures with different ips): found = sorted(found, key=lambda x: str(x)) output = sorted(output, key=lambda x: str(x)) @@ -599,13 +607,14 @@ cmd = os.path.join(STOCK_CONF_DIR, "filter.d/ignorecommands/apache-fakegooglebot") ## below test direct as python module: mod = Utils.load_python_module(cmd) - self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "128.178.222.69"]))) - self.assertFalse(mod.is_googlebot(mod.process_args([cmd, "192.0.2.1"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "128.178.222.69"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "192.0.2.1"]))) + self.assertFalse(mod.is_googlebot(*mod.process_args([cmd, "192.0.2.1", 0.1]))) bot_ips = ['66.249.66.1'] for ip in bot_ips: - self.assertTrue(mod.is_googlebot(mod.process_args([cmd, str(ip)])), "test of googlebot ip %s failed" % ip) - self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd]))) - self.assertRaises(ValueError, lambda: mod.is_googlebot(mod.process_args([cmd, "192.0"]))) + self.assertTrue(mod.is_googlebot(*mod.process_args([cmd, str(ip)])), "test of googlebot ip %s failed" % ip) + self.assertRaises(ValueError, lambda: mod.is_googlebot(*mod.process_args([cmd]))) + self.assertRaises(ValueError, lambda: mod.is_googlebot(*mod.process_args([cmd, "192.0"]))) ## via command: self.filter.ignoreCommand = cmd + " " for ip in bot_ips: @@ -617,7 +626,7 @@ self.pruneLog() self.filter.ignoreCommand = cmd + " bad arguments " self.assertFalse(self.filter.inIgnoreIPList("192.0")) - self.assertLogged('Please provide a single IP as an argument.') + self.assertLogged('Usage') @@ -800,7 +809,6 @@ _, self.name = tempfile.mkstemp('fail2ban', 'monitorfailures') self.file = open(self.name, 'a') self.filter = FilterPoll(DummyJail()) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.addLogPath(self.name, autoSeek=False) self.filter.active = True self.filter.addFailRegex(r"(?:(?:Authentication failure|Failed [-/\w+]+) for(?: [iI](?:llegal|nvalid) user)?|[Ii](?:llegal|nvalid) user|ROOT LOGIN REFUSED) .*(?: from|FROM) ") @@ -960,7 +968,7 @@ os.rename(self.name, self.name + '.bak') _copy_lines_between_files(GetFailures.FILENAME_01, self.name, skip=14, n=1).close() self.filter.getFailures(self.name) - _assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) + #_assert_correct_last_attempt(self, self.filter, GetFailures.FAILURES_01) self.assertEqual(self.filter.failManager.getFailTotal(), 3) @@ -971,6 +979,10 @@ super(CommonMonitorTestCase, self).setUp() self._failTotal = 0 + def tearDown(self): + super(CommonMonitorTestCase, self).tearDown() + self.assertFalse(hasattr(self, "_unexpectedError")) + def waitFailTotal(self, count, delay=1): """Wait up to `delay` sec to assure that expected failure `count` reached """ @@ -996,6 +1008,16 @@ last_ticks = self.filter.ticks return Utils.wait_for(lambda: self.filter.ticks >= last_ticks + ticks, _maxWaitTime(delay)) + def commonFltError(self, reason="common", exc=None): + """ Mock-up for default common error handler to find catched unhandled exceptions + could occur in filters + """ + self._commonFltError(reason, exc) + if reason == "unhandled": + DefLogSys.critical("Caught unhandled exception in main cycle of %r : %r", self.filter, exc, exc_info=True) + self._unexpectedError = True + # self.assertNotEqual(reason, "unhandled") + def get_monitor_failures_testcase(Filter_): """Generator of TestCase's for different filters/backends @@ -1018,7 +1040,8 @@ self.file = open(self.name, 'a') self.jail = DummyJail() self.filter = Filter_(self.jail) - self.filter.banASAP = False # avoid immediate ban in this tests + # mock-up common error to find catched unhandled exceptions: + self._commonFltError, self.filter.commonError = self.filter.commonError, self.commonFltError self.filter.addLogPath(self.name, autoSeek=False) # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1111,12 +1134,13 @@ skip=12, n=3, mode='w') self.assert_correct_last_attempt(GetFailures.FAILURES_01) - def _wait4failures(self, count=2): + def _wait4failures(self, count=2, waitEmpty=True): # Poll might need more time - self.assertTrue(self.isEmpty(_maxWaitTime(5)), - "Queue must be empty but it is not: %s." - % (', '.join([str(x) for x in self.jail.queue]))) - self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) + if waitEmpty: + self.assertTrue(self.isEmpty(_maxWaitTime(5)), + "Queue must be empty but it is not: %s." + % (', '.join([str(x) for x in self.jail.queue]))) + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) Utils.wait_for(lambda: self.filter.failManager.getFailTotal() >= count, _maxWaitTime(10)) self.assertEqual(self.filter.failManager.getFailTotal(), count) @@ -1277,14 +1301,14 @@ # tail written before, so let's not copy anything yet #_copy_lines_between_files(GetFailures.FILENAME_01, self.name, n=100) # we should detect the failures - self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=6) # was needed if we write twice above + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # was needed if we write twice above # now copy and get even more _copy_lines_between_files(GetFailures.FILENAME_01, self.file, skip=12, n=3) # check for 3 failures (not 9), because 6 already get above... - self.assert_correct_last_attempt(GetFailures.FAILURES_01) + self.assert_correct_last_attempt(GetFailures.FAILURES_01, count=3) # total count in this test: - self.assertEqual(self.filter.failManager.getFailTotal(), 12) + self._wait4failures(12, False) cls = MonitorFailures cls.__qualname__ = cls.__name__ = "MonitorFailures<%s>(%s)" \ @@ -1316,7 +1340,8 @@ def _initFilter(self, **kwargs): self._getRuntimeJournal() # check journal available self.filter = Filter_(self.jail, **kwargs) - self.filter.banASAP = False # avoid immediate ban in this tests + # mock-up common error to find catched unhandled exceptions: + self._commonFltError, self.filter.commonError = self.filter.commonError, self.commonFltError self.filter.addJournalMatch([ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", @@ -1512,7 +1537,7 @@ "SYSLOG_IDENTIFIER=fail2ban-testcases", "TEST_FIELD=1", "TEST_UUID=%s" % self.test_uuid]) - self.assert_correct_ban("193.168.0.128", 4) + self.assert_correct_ban("193.168.0.128", 3) _copy_lines_to_journal( self.test_file, self.journal_fields, n=6, skip=10) # we should detect the failures @@ -1526,7 +1551,7 @@ self.test_file, self.journal_fields, skip=15, n=4) self.waitForTicks(1) self.assertTrue(self.isFilled(10)) - self.assert_correct_ban("87.142.124.10", 4) + self.assert_correct_ban("87.142.124.10", 3) # Add direct utf, unicode, blob: for l in ( "error: PAM: Authentication failure for \xe4\xf6\xfc\xdf from 192.0.2.1", @@ -1570,7 +1595,6 @@ setUpMyTime() self.jail = DummyJail() self.filter = FileFilter(self.jail) - self.filter.banASAP = False # avoid immediate ban in this tests self.filter.active = True # speedup search using exact date pattern: self.filter.setDatePattern(r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?') @@ -1641,6 +1665,7 @@ [u'Aug 14 11:%d:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:141.3.81.106 port 51332 ssh2' % m for m in 53, 54, 57, 58]) + self.filter.setMaxRetry(4) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=0) self.filter.addFailRegex(r"Failed .* from ") self.filter.getFailures(GetFailures.FILENAME_02) @@ -1649,6 +1674,7 @@ def testGetFailures03(self): output = ('203.162.223.135', 6, 1124013600.0) + self.filter.setMaxRetry(6) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03) @@ -1657,6 +1683,7 @@ def testGetFailures03_InOperation(self): output = ('203.162.223.135', 9, 1124013600.0) + self.filter.setMaxRetry(9) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=0) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") self.filter.getFailures(GetFailures.FILENAME_03, inOperation=True) @@ -1674,7 +1701,7 @@ def testGetFailures03_Seek2(self): # same test as above but with seek to 'Aug 14 11:59:04' - so other output ... output = ('203.162.223.135', 2, 1124013600.0) - self.filter.setMaxRetry(1) + self.filter.setMaxRetry(2) self.filter.addLogPath(GetFailures.FILENAME_03, autoSeek=output[2]) self.filter.addFailRegex(r"error,relay=,.*550 User unknown") @@ -1684,10 +1711,12 @@ def testGetFailures04(self): # because of not exact time in testcase04.log (no year), we should always use our test time: self.assertEqual(MyTime.time(), 1124013600) - # should find exact 4 failures for *.186 and 2 failures for *.185 - output = (('212.41.96.186', 4, 1124013600.0), - ('212.41.96.185', 2, 1124013598.0)) - + # should find exact 4 failures for *.186 and 2 failures for *.185, but maxretry is 2, so 3 tickets: + output = ( + ('212.41.96.186', 2, 1124013480.0), + ('212.41.96.186', 2, 1124013600.0), + ('212.41.96.185', 2, 1124013598.0) + ) # speedup search using exact date pattern: self.filter.setDatePattern((r'^%ExY(?P<_sep>[-/.])%m(?P=_sep)%d[T ]%H:%M:%S(?:[.,]%f)?(?:\s*%z)?', r'^(?:%a )?%b %d %H:%M:%S(?:\.%f)?(?: %ExY)?', @@ -1744,9 +1773,11 @@ unittest.F2B.SkipIfNoNetwork() # We should still catch failures with usedns = no ;-) output_yes = ( - ('93.184.216.34', 2, 1124013539.0, - [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2', - u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] + ('93.184.216.34', 1, 1124013299.0, + [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] + ), + ('93.184.216.34', 1, 1124013539.0, + [u'Aug 14 11:58:59 i60p295 sshd[12365]: Failed publickey for roehl from ::ffff:93.184.216.34 port 51332 ssh2'] ), ('2606:2800:220:1:248:1893:25c8:1946', 1, 1124013299.0, [u'Aug 14 11:54:59 i60p295 sshd[12365]: Failed publickey for roehl from example.com port 51332 ssh2'] @@ -1771,7 +1802,6 @@ self.pruneLog("[test-phase useDns=%s]" % useDns) jail = DummyJail() filter_ = FileFilter(jail, useDns=useDns) - filter_.banASAP = False # avoid immediate ban in this tests filter_.active = True filter_.failManager.setMaxRetry(1) # we might have just few failures @@ -1781,8 +1811,11 @@ _assert_correct_last_attempt(self, filter_, output) def testGetFailuresMultiRegex(self): - output = ('141.3.81.106', 8, 1124013541.0) + output = [ + ('141.3.81.106', 8, 1124013541.0) + ] + self.filter.setMaxRetry(8) self.filter.addLogPath(GetFailures.FILENAME_02, autoSeek=False) self.filter.addFailRegex(r"Failed .* from ") self.filter.addFailRegex(r"Accepted .* from ") @@ -1800,26 +1833,25 @@ self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLine(self): - output = [("192.0.43.10", 2, 1124013599.0), - ("192.0.43.11", 1, 1124013598.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), + ("192.0.43.11", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") self.filter.setMaxRetry(1) self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + + _assert_correct_last_attempt(self, self.filter, output) def testGetFailuresMultiLineIgnoreRegex(self): - output = [("192.0.43.10", 2, 1124013599.0)] + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -1828,14 +1860,17 @@ self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - _assert_correct_last_attempt(self, self.filter, output.pop()) + _assert_correct_last_attempt(self, self.filter, output) self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) def testGetFailuresMultiLineMultiRegex(self): - output = [("192.0.43.10", 2, 1124013599.0), + output = [ + ("192.0.43.10", 1, 1124013598.0), + ("192.0.43.10", 1, 1124013599.0), ("192.0.43.11", 1, 1124013598.0), - ("192.0.43.15", 1, 1124013598.0)] + ("192.0.43.15", 1, 1124013598.0) + ] self.filter.addLogPath(GetFailures.FILENAME_MULTILINE, autoSeek=False) self.filter.setMaxLines(100) self.filter.addFailRegex(r"^.*rsyncd\[(?P\d+)\]: connect from .+ \(\)$^.+ rsyncd\[(?P=pid)\]: rsync error: .*$") @@ -1844,14 +1879,9 @@ self.filter.getFailures(GetFailures.FILENAME_MULTILINE) - foundList = [] - while True: - try: - foundList.append( - _ticket_tuple(self.filter.failManager.toBan())[0:3]) - except FailManagerEmpty: - break - self.assertSortedEqual(foundList, output) + _assert_correct_last_attempt(self, self.filter, output) + + self.assertRaises(FailManagerEmpty, self.filter.failManager.toBan) class DNSUtilsTests(unittest.TestCase): diff -u -r -N fail2ban-0.11.2/fail2ban/tests/misctestcase.py fail2ban/fail2ban/tests/misctestcase.py --- fail2ban-0.11.2/fail2ban/tests/misctestcase.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/misctestcase.py 2021-10-29 13:50:59.921324987 +0200 @@ -111,7 +111,7 @@ supdbgout = ' >/dev/null 2>&1' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: # try dry-run: - os.system("%s %s --dry-run install --disable-2to3 --root=%s%s" + os.system("%s %s --dry-run install --root=%s%s" % (sys.executable, self.setup , tmp, supdbgout)) # check nothing was created: self.assertTrue(not os.listdir(tmp)) @@ -127,7 +127,7 @@ # suppress stdout (and stderr) if not heavydebug supdbgout = ' >/dev/null' if unittest.F2B.log_level >= logging.DEBUG else '' # HEAVYDEBUG try: - self.assertEqual(os.system("%s %s install --disable-2to3 --root=%s%s" + self.assertEqual(os.system("%s %s install --root=%s%s" % (sys.executable, self.setup, tmp, supdbgout)), 0) def strippath(l): diff -u -r -N fail2ban-0.11.2/fail2ban/tests/servertestcase.py fail2ban/fail2ban/tests/servertestcase.py --- fail2ban-0.11.2/fail2ban/tests/servertestcase.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/servertestcase.py 2021-10-29 13:50:59.925324977 +0200 @@ -35,7 +35,7 @@ from ..server.failregex import Regex, FailRegex, RegexException from ..server import actions as _actions from ..server.server import Server -from ..server.ipdns import IPAddr +from ..server.ipdns import DNSUtils, IPAddr from ..server.jail import Jail from ..server.jailthread import JailThread from ..server.ticket import BanTicket @@ -66,9 +66,12 @@ class TransmitterBase(LogCaptureTestCase): + TEST_SRV_CLASS = TestServer + def setUp(self): """Call before every test case.""" super(TransmitterBase, self).setUp() + self.server = self.TEST_SRV_CLASS() self.transm = self.server._Server__transm # To test thransmitter we don't need to start server... #self.server.start('/dev/null', '/dev/null', force=False) @@ -157,10 +160,6 @@ class Transmitter(TransmitterBase): - def setUp(self): - self.server = TestServer() - super(Transmitter, self).setUp() - def testServerIsNotStarted(self): # so far isStarted only tested but not used otherwise # and here we don't really .start server @@ -175,6 +174,19 @@ def testVersion(self): self.assertEqual(self.transm.proceed(["version"]), (0, version.version)) + def testSetIPv6(self): + try: + self.assertEqual(self.transm.proceed(["set", "allowipv6", 'yes']), (0, 'yes')) + self.assertTrue(DNSUtils.IPv6IsAllowed()) + self.assertLogged("IPv6 is on"); self.pruneLog() + self.assertEqual(self.transm.proceed(["set", "allowipv6", 'no']), (0, 'no')) + self.assertFalse(DNSUtils.IPv6IsAllowed()) + self.assertLogged("IPv6 is off"); self.pruneLog() + finally: + # restore back to auto: + self.assertEqual(self.transm.proceed(["set", "allowipv6", "auto"]), (0, "auto")) + self.assertLogged("IPv6 is auto"); self.pruneLog() + def testSleep(self): if not unittest.F2B.fast: t0 = time.time() @@ -924,8 +936,9 @@ class TransmitterLogging(TransmitterBase): + TEST_SRV_CLASS = Server + def setUp(self): - self.server = Server() super(TransmitterLogging, self).setUp() self.server.setLogTarget("/dev/null") self.server.setLogLevel("CRITICAL") diff -u -r -N fail2ban-0.11.2/fail2ban/tests/utils.py fail2ban/fail2ban/tests/utils.py --- fail2ban-0.11.2/fail2ban/tests/utils.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/fail2ban/tests/utils.py 2021-10-29 13:50:59.927324972 +0200 @@ -47,7 +47,7 @@ from ..version import version -logSys = getLogger(__name__) +logSys = getLogger("fail2ban") TEST_NOW = 1124013600 @@ -126,9 +126,6 @@ def initProcess(opts): # Logger: - global logSys - logSys = getLogger("fail2ban") - llev = None if opts.log_level is not None: # pragma: no cover # so we had explicit settings @@ -320,6 +317,7 @@ # precache all invalid ip's (TEST-NET-1, ..., TEST-NET-3 according to RFC 5737): c = DNSUtils.CACHE_ipToName + c.clear = lambda: logSys.warn('clear CACHE_ipToName is disabled in test suite') # increase max count and max time (too many entries, long time testing): c.setOptions(maxCount=10000, maxTime=5*60) for i in xrange(256): @@ -337,6 +335,7 @@ c.set('8.8.4.4', 'dns.google') # precache all dns to ip's used in test cases: c = DNSUtils.CACHE_nameToIp + c.clear = lambda: logSys.warn('clear CACHE_nameToIp is disabled in test suite') for i in ( ('999.999.999.999', set()), ('abcdef.abcdef', set()), @@ -780,8 +779,9 @@ """Call after every test case.""" # print "O: >>%s<<" % self._log.getvalue() self.pruneLog() + self._log.close() logSys.handlers = self._old_handlers - logSys.level = self._old_level + logSys.setLevel(self._old_level) super(LogCaptureTestCase, self).tearDown() def _is_logged(self, *s, **kwargs): diff -u -r -N fail2ban-0.11.2/man/jail.conf.5 fail2ban/man/jail.conf.5 --- fail2ban-0.11.2/man/jail.conf.5 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/man/jail.conf.5 2021-10-29 13:50:59.957324897 +0200 @@ -151,6 +151,11 @@ .br This is used to store the process ID of the fail2ban server. .TP +.B allowipv6 +option to allow IPv6 interface - auto, yes (on, true, 1) or no (off, false, 0). Default: auto +.br +This value can be used to declare fail2ban whether IPv6 is allowed or not. +.TP .B dbfile Database filename. Default: /var/lib/fail2ban/fail2ban.sqlite3 .br @@ -476,13 +481,29 @@ .IP \fI\fR - regex for IPv4 addresses. .IP -\fI\fR - regex for IPv6 addresses (also IP enclosed in brackets). +\fI\fR - regex for IPv6 addresses. .IP \fI\fR - regex to match hostnames. .IP \fI\fR - helper regex to match CIDR (simple integer form of net-mask). .IP \fI\fR - regex to match sub-net adresses (in form of IP/CIDR, also single IP is matched, so part /CIDR is optional). +.IP +\fI...\fR - free regex capturing group targeting identifier used for ban (instead of IP address or hostname). +.IP +\fI...\fR - free regex capturing named group stored in ticket, which can be used in action. +.nf +For example \fI[^@]+\fR matches and stores a user name, that can be used in action with interpolation tag \fI\fR. +.IP +\fI...\fR - free regex capturing alternative named group stored in ticket. +.nf +For example first found matched value defined in regex as \fI\fR, \fI\fR or \fI\fR would be stored as (if direct match is not found or empty). +.PP +Every of abovementioned tags can be specified in \fBprefregex\fR and in \fBfailregex\fR, thereby if specified in both, the value matched in \fBfailregex\fR overwrites a value matched in \fBprefregex\fR. +.TQ +All standard tags like IP4 or IP6 can be also specified with custom regex using \fI...\fR syntax, for example \fI(?:ip4:\\S+|ip6:\\S+)\fR. +.TQ +Tags \fI\fR, \fI\fR and \fI\fR would also match the IP address enclosed in square brackets. .PP \fBNOTE:\fR the \fBfailregex\fR will be applied to the remaining part of message after \fBprefregex\fR processing (if specified), which in turn takes place after \fBdatepattern\fR processing (whereby the string of timestamp matching the best pattern, cut out from the message). .PP diff -u -r -N fail2ban-0.11.2/setup.py fail2ban/setup.py --- fail2ban-0.11.2/setup.py 2020-11-23 21:43:03.000000000 +0100 +++ fail2ban/setup.py 2021-10-29 13:50:59.958324895 +0200 @@ -39,14 +39,6 @@ if setuptools is None: from distutils.command.install import install from distutils.command.install_scripts import install_scripts -try: - # python 3.x - from distutils.command.build_py import build_py_2to3 - from distutils.command.build_scripts import build_scripts_2to3 - _2to3 = True -except ImportError: - # python 2.x - _2to3 = False import os from os.path import isfile, join, isdir, realpath @@ -56,7 +48,7 @@ from glob import glob from fail2ban.setup import updatePyExec - +from fail2ban.version import version source_dir = os.path.realpath(os.path.dirname( # __file__ seems to be overwritten sometimes on some python versions (e.g. bug of 2.6 by running under cProfile, etc.): @@ -120,22 +112,12 @@ # Wrapper to specify fail2ban own options: class install_command_f2b(install): user_options = install.user_options + [ - ('disable-2to3', None, 'Specify to deactivate 2to3, e.g. if the install runs from fail2ban test-cases.'), ('without-tests', None, 'without tests files installation'), ] def initialize_options(self): - self.disable_2to3 = None self.without_tests = not with_tests install.initialize_options(self) def finalize_options(self): - global _2to3 - ## in the test cases 2to3 should be already done (fail2ban-2to3): - if self.disable_2to3: - _2to3 = False - if _2to3: - cmdclass = self.distribution.cmdclass - cmdclass['build_py'] = build_py_2to3 - cmdclass['build_scripts'] = build_scripts_2to3 if self.without_tests: self.distribution.scripts.remove('bin/fail2ban-testcases') @@ -186,7 +168,6 @@ if setuptools: setup_extra = { 'test_suite': "fail2ban.tests.utils.gatherTests", - 'use_2to3': True, } else: setup_extra = {} @@ -210,9 +191,6 @@ ('/usr/share/doc/fail2ban', doc_files) ) -# Get version number, avoiding importing fail2ban. -# This is due to tests not functioning for python3 as 2to3 takes place later -exec(open(join("fail2ban", "version.py")).read()) setup( name = "fail2ban",