#!/bin/bash
VERSION="3.9.12 [01 Jan 2026]"
THIS=$(basename "$0")
# by Dominic Raferd dominic@timedicer.co.uk
# for information, run with -h option
TMPBASE=$(dirname "$(mktemp -u)")/$THIS
TMPF=$TMPBASE-1
TMPG=$TMPBASE-2


# the match texts are in extended regex form (as interpreted by GNU sed -r)
ACTION_ON="(sent| said:|said: ) .*(unsolicited mail|security issue|suspicious due to|rate limited|DMARC policy|authentication checks|RFC 5322 compliant|at a rate that|not a valid RFC-5321|fails to pass SPF|from your IP Netblock).*gsmtp"
# DMARC_FAIL, LACK_AUTH must be subsets of ACTION_ON:
#   anything caught by ACTION_ON but not by DMARC_FAIL or LACK_AUTH, and then not matching SENDER_D, is deemed 'BAN_BY_F2B'
DMARC_FAIL="DMARC policy"
LACK_AUTH="authentication checks|not a valid RFC-5321|fails to pass SPF"
SENDER_D=" sending .*domain"
DKIM_RATE="DKIM domain"
DO_NOTHING="PTR record"
# SENDER_D_BANFILE is the BerkeleyDB-type source file to which sender bans
# are added; it should be set in a postfix restriction list e.g:
# check_sender_access hash:/etc/postfix/sender_access
SENDER_D_BANFILE="/etc/postfix/sender_access"
DKIM_RATE_BANFILE="/etc/postfix/check_header_wild.pcre"
# to trigger a message/email about an unrecognised response from gsmtp, the message must include
# $UNRECOGNISED_REQUIRE, but not if it includes $UNRECOGNISED_SKIP
UNRECOGNISED_REQUIRE="gsmtp"
UNRECOGNISED_SKIP="(temporarily rejected|Messages with multiple addresses in From: header|The email account that you tried to reach (is disabled|does not exist))"
MUAS="mail s-nail"	# list possible MUAs which must support options -t --attach -r
OPTIONS="$*"
while [ -n "$*" ]; do
  if [ "$1" = "-a" ]; then
  	FORCE_AMAVIS="y"
  elif [ "$1" = "-d" ]; then
    DEBUG=y; VERBOSE=y
  elif [ "$1" = "-f" ]; then
  	NEXTPARAM_CHAR1=$(echo "$2"|cut -c1)
  	if [ -n "$NEXTPARAM_CHAR1" ] && [ "$NEXTPARAM_CHAR1" != "-" ]; then
	  	shift
  		LOCALBACKUP="$1"
  	fi
	elif [ "$1" = "-h" ]; then
		HELP="y"
	elif [ "$1" = "-l" ]; then
		CHANGELOG="y"
  elif [ "$1" = "-m" ]; then
  	NEXTPARAM_CHAR1=$(echo "$2"|cut -c1)
  	if [ -n "$NEXTPARAM_CHAR1" ] && [ "$NEXTPARAM_CHAR1" != "-" ]; then
	  	shift
  		ADMIN_MAILTO="$1"
  	else
  		ADMIN_MAILTO="root"
  	fi
  elif [ "$1" = "-n" ]; then
  	FORCE_AMAVIS="n"
  elif [ "$1" = "-q" ]; then
    QUIET=y
  elif [ "$1" = "-r" ]; then
    RESTART=y
  elif [ "$1" = "-s" ]; then
    STOP=y
  elif [ "$1" = "-t" ]; then
    TEST=y; VERBOSE=y
  elif [ "$1" = "-v" ]; then
    VERBOSE=y
  elif [ "$1" = "-w" ]; then
  	COLUMNS=30000
  else
  	echo "Unknown option '$1', can't proceed" >&2; exit 0
  fi
  shift
done
if [ -n "$HELP$CHANGELOG" ]; then
	[ -z "$COLUMNS" ] && COLUMNS=$( (stty size 2>/dev/null||printf 80)|cut -d' ' -f2 )
 	echo -e "\n$THIS v$VERSION by Dominic (-h for help)\n${THIS//?/=}\n"
	#printf "%b" "\n$THIS v$VERSION by Dominic\n\n\n"
	if [ -n "$HELP" ]; then
		printf "%b" "$THIS enables a postfix-based mail server, which is relaying \
incoming emails on to Gmail ('gsmtp'), to act on reports which it has \
received back from Gmail about blocked emails. Depending on the reason given \
by Gmail (in the text of its report) $THIS will either block the \
offending incoming client's domain_name/ip_address, or forward \
the email, encapsulated as an attachment, to its intended recipient.

The intended use case is where Gmail is acting - without Google Workspace \
- as the backend for a domain's emails \
(including sending of emails from Gmail so that they appear to come \
from that domain). Setting up such a scenario, requiring your postfix mail \
server to be specially configured for relaying to Gmail, is not covered here. \
But once it is set up then, as your server relays emails into Gmail, some of \
these will be blocked by Gmail as being in some way 'bad'; worse, if \
there are too many of these then Gmail may block your mail server \
entirely, so that emails you relay into Gmail can no longer reach \
recipients' mailboxes. $THIS provides workarounds/mitigations \
for these blockages so that emails blocked mistakenly by Gmail can \
still be delivered, whilst - for emails blocked correctly by Gmail \
- $THIS will block the sender domain_name or ip_address on \
your own server, preventing a flood of such emails from being relayed \
to Gmail and damaging your server's reputation.

Note that $THIS is not a substitute for strong anti-spam and anti-virus \
defences on your mail server; indeed these are pre-requisites, so that as \
far as possible neither 'real' spam nor virus-laden emails are ever \
forwarded by your server to Gmail. $THIS is intended to catch the inevitable \
'edge case' emails that, although 'bad', have been missed by your own \
defences, as well as 'edge case' emails that are 'good' but are mistakenly \
identified as 'bad' by Gmail.

Usage: $THIS is intended to be run at startup e.g. by adding it as a root cron job thus:
    @reboot nohup /usr/local/bin/relay-enforcer -v </dev/null >>/var/log/relay-enforcer.log 2>&1 &
If required, if can be (re)started from command line (with superuser permissions) thus:
    /usr/local/bin/relay-enforcer -r -v </dev/null >>/var/log/relay-enforcer.log 2>&1 &

By default $THIS is silent until a matching message is found; use verbose, test or debug options to see \
more. Monitor ${THIS}'s output to see what it is doing. If run with '-v' it outputs a dot for each \
line of the mail log that it has inspected, and starts a new line with a datetime \
hourly; this allows you to be confident that it is running and processing the mail log. \
Actions generate output unless it is run with '-q'.

$THIS uses tail -f to inspect lines as they are added to the mail log. \
If it takes action this is reported on standard output (unless quiet option is chosen) and, \
if -m option is used, by email as well.

If a block (rather than a forward) action is required, $THIS normally \
(but see 'Sender_d actions' below) bans the IP of the offending sender. It \
does this by adding an entry in the system log which ends with the ip of the \
incoming client whose message gave rise to the bad report; \
this line is to be read by the matching fail2ban jail which then effects \
the ip ban from incoming mail ports for some period (see 'Ban actions' below).

$THIS runs under bash, and has been tested on Ubuntu 22.04, 20.04, 18.04, 16.04.

Amavis compatibility: $THIS is compatible with (but does not require) re-injection \
by amavis acting as a content filter (where the queue-id of the email \
rejected by the onward server is not the same as the \
queue-id of the mail from the incoming client). $THIS assesses whether \
to run with this compatibility automatically, however command line options \
can be used to force the desired behaviour - for instance if amavis is run \
as a milter. Warning: usage of $THIS without amavis re-injection is untested.

Note that $THIS is not currently compatible with re-injection (content filtering or \
post-queue filtering) by utilities other than amavis.

Welcomelist: A file named $THIS.welcomelist in the same location as $THIS is used to ensure that certain ips \
or hostnames cannot generate a report by $THIS and thus a potential ban; instead of a blocking or banning action, \
emails from a welcomelisted IP or hostname will be encapsulated and sent on (see DMARC forward actions below). \
If missing, $THIS.welcomelist \
is created at startup containing local ips and all possible local-network ips. $THIS.welcomelist uses \
plain, range or CIDR formats as understood by grepcidr; each pattern should start on a newline. \
You can also add hostname patterns using egrep (extended regex) syntax, so dots etc must be escaped. \
Comments (starting with #) and any space immediately preceding them are ignored, as are \
blank lines and any whitespace at the start of a non-blank line (so that indenting of valid lines is possible). \
$THIS.welcomelist can be updated while $THIS is running.

Sender_d actions: Where the rejection message from Gmail specifies that the problem was with the sending domain, \
then instead of adding a log entry which might trigger a source IP ban by fail2ban, $THIS will \
add this domain to \$SENDER_D_BANFILE (normally: '$SENDER_D_BANFILE') and hash it; provided this \
file is specified with a postfix restriction list for check_sender_access then this ensures that \
no more emails from this domain will (ever) be accepted.

Ban actions: When an IP is identified for banning, the actual banning action is carried out by \
fail2ban jail '$THIS' when it finds a 'bannable' log entry added by $THIS. This jail needs to be \
created, for instance:
(a) create /etc/fail2ban/filter.d/relay-enforcer.local:
    [INCLUDES]
    before = common.conf
    [Definition]
    _daemon = relay-enforcer
    failregex = ^%(__prefix_line)sbannable.* <HOST>$
    ignoreregex =
(b) add entries in, say, /etc/fail2ban/jail.local like this:
    [$THIS]
    enabled  = true
    maxretry = 1
    bantime  = 14700
    findtime = 14700
    port     = smtp,465,submission
    logpath  = %(syslog_ftp)s

DMARC forward actions: If the local mailserver is enforcing DMARC (e.g. with opendmarc), a rejection from an \
onward server should only occur if the offending email's 'From:' sender domain publishes a DMARC p=reject policy \
but the email lacks valid DKIM and relies for successful delivery on SPF (which is inevitably \
broken by forwarding). In such a case (which is rare) \
$THIS will extract the failed email from a local backup (e.g. /var/mail/backup - postfix settings \
'always_bcc = backup@localhost' and 'mail_spool_directory = /var/mail' \
can be used to save all mails here), and send it on as an attachment to the \
original recipient, with a covering message. To enable this DMARC forwarding action, first \
ensure that you have an effective DMARC enforcement policy on your local mailserver, and \
have all sent emails saved (via always_bcc as shown above) in a single mbox/mailbox-type file, then \
use $THIS's -f option to define the local backup file when starting $THIS.

Authentication checks failures: Gmail rejects some emails on the grounds of \
'$LACK_AUTH', these are usually round-robin emails from a Microsoft server, \
or from a sender with a 'hard fail' SPF policy. Experience \
has shown that they are always or nearly always good emails, so $THIS will \
resend them encapsulated (same as for DMARC forward actions, and with the same \
requirement that recent emails are saved in a local mbox/mailbox-type file).

Do nothing: If there is no remediation possible then $THIS takes no action, \
this is for messages from Gmail containing '$DO_NOTHING'.

Log rotation: To prevent $THIS from hanging upon log rotation of the mail log file, add 'copytruncate' to \
the relevant section of the relevant logrotate file (e.g. /etc/logrotate.d/rsyslog).

DSN code rewriting: $THIS does not rewrite the code responses that are \
received from Gmail and then relayed by the local mailserver to the original \
sender; this must be done separately as part of the postfix configuration, \
but only for messages to Gmail. 550 responses upon failure because of DMARC \
or authentication checks need to be re-written \
to 250 so that the original sender considers the message was delivered ok (as, after forwarding by $THIS, it \
should be), and temporary failure codes should be rewritten as permanent (i.e. 4xx to 5xx).

Example file extracts for postfix configuration are shown here -
/etc/postfix/master.cf -
    ...
    smtp_dsn_rw unix -       -       n       -       -       smtp -o smtp_reply_filter=pcre:/etc/postfix/smtp_dsn_rw_filter.pcre
    ...
/etc/postfix/main.cf -
    ...
    transport_maps = hash:/etc/postfix/transport
    ...
/etc/postfix/transport -
    ...
    gmail.com smtp_dsn_rw
    ...
/etc/postfix/smtp_dsn_rw_filter.pcre -"'
    # modify response codes to original sender
    # no change if there is a temporary problems with Gmail servers
    /^(421 4\.[0-9]\.[0-9] Temporary )/ ${1}
    # DMARC rejection by Gmail - change to 250 OK; we will forward it encapsulated
    /^550 5\.7\.26( DMARC.+gsmtp)/ 250 2.0.0${1}-fmod_dmarc
    # Authentication checks rejection by Gmail - change to 250 OK; we forward it encapsulated
    /^421 4\.7\.0( information.+gsmtp)/ 250 2.0.0${1}-fmod_lackauth
    /^550 5\.7\.26( information.+gsmtp)/ 250 2.0.0${1}-fmod_lackauth
    # Other temporary rejection by Gmail - make rejection permanent
    /^421 4\.7\.28( .+gmstp)/ 554 5.7.28${1}-fmod_unsol
    /^421 4\.([0-9]*\.[0-9]* review our Bulk.+gsmtp)/ 554 5.${1}-fmod_bulk
    /^421 4\.([0-9]*\.[0-9]* .+gsmtp)/ 554 5.${1}-fmod_generic
'"
Options: -a - force compatibility with amavis as content-filter (see -n)
         -d - debug mode - lots of extra text output, implies verbose
         -f /var/local/backup - specify a local backup mbox/mailbox-style file which will contain copies \
of recently sent mails (e.g. by postfix's always_bcc), so that it can be used to retrieve blocked mails for encapsulating and forwarding
         -h - show this help
         -l - show changelog
         -m mail@address - send report on any action to mail@address (default: root)
         -n - force non-compatibility with amavis as content-filter (see -a)
         -q - be quiet even if reportable actions are found
         -r - kill another instance of $THIS (if any), then continue running
         -s - stop another instance of $THIS (if any)
         -t - test mode (don't submit reports to system log, don't update count of lines in mail log)
         -v - be verbose
         -w columns - set terminal width e.g. 80 (for help/changelog word-wrapping only, normally determined automagically)

         Options must be specified individually not combined i.e. -v -r not -vr

Philosophy: $THIS does not follow RFC5321 which says of the \
reply code+text sent by an onward smtp receiver (such as Gmail), that 'an \
SMTP client MUST determine its actions only by the reply code, not by the \
text' (https://tools.ietf.org/html/rfc5321#section-4.2). \
If this troubles you, don't use $THIS.

Dependencies: bash
              fail2ban with $THIS jail - ($THIS cannot ban without this)
              grepcidr
              a mail user agent (MUA) which supports options -t [use message headers] --attach= [add attachment] -r [envelope_from = From: header] - currently only confirmed for s-nail (tested up to v14.9.23) and GNU Mailutils mail(x) (tested up to v3.17) (note that GNU Mailutils mail -t picks up all, or at least many, headers, not only - per documentation - the 'To:' header)
              opendkim
              opendmarc
              postfix
              python3

License: Copyright © 2026 Dominic Raferd. Licensed under the Apache License, \
Version 2.0 (the \"License\"); you may not use this file except in compliance \
with the License. You may obtain a copy of the License at \
https://www.apache.org/licenses/LICENSE-2.0. Unless required by applicable \
law or agreed to in writing, software distributed under the License is \
distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY \
KIND, either express or implied. See the License for the specific language \
governing permissions and limitations under the License.\n\n" | fold -sw "$COLUMNS"
	fi
	if [ -n "$CHANGELOG" ]; then
	  [ -n "$HELP" ] && echo "Changelog:"
		printf "%b" "\
Note - small changes and bugfixes may occur between releases listed here, these \
may be indicated by a sub-minor version change (3.4.0->3.4.1).
           3.9 [26 Jan 2024] - improved postsrsd compatibility (v2.0+)
           3.8 [17 Apr 2022] - bugfixes, make bash-dependent (compatibility break)
           3.7 [21 Mar 2021] - many small changes to achieve close compliance with shellcheck, use tail -f instead of inotifywait
           3.6 [06 Jan 2021] - opendkim and opendmarc are now dependencies (compatibility break)
           3.5 [10 Jul 2020] - all references to whitelist (except in this changelog) changed to welcomelist (compatibility break)
           3.4 [26 Jun 2020] - only one fail2ban ban type ($THIS, not $THIS-long and $THIS-short; compatibility break)
           3.3 [09 Jun 2020] - alter whitelist ip format (compatibility break)
           3.2 [17 Apr 2020] - add preceding comment when adding entry to $SENDER_D_BANFILE
           3.1 [01 Apr 2020] - work with logrotate copytruncate
           3.0 [13 Dec 2019] - bugfix paths for programs run by $THIS, tidy up some old code
           2.9 [07 Dec 2019] - bugfix for postfix QueueIDs of >10 characters
           2.8 [02 Dec 2019] - don't copy encapsulated emails to administrator
           2.7 [22 Jul 2019] - add an extra check to each candidate line from log to ensure that we didn't already process it (otherwise duplication occasionally happens)
           2.6 [27 Jun 2019] - forward (by encapsulating) on any '$LACK_AUTH' Gmail response (previous behaviour was to do this only when the From header was matched in \$LACK_AUTH_WHITELIST)
           2.5 [02 Jan 2019] - tweak output in non-verbose non-quiet mode
           2.4 [18 Oct 2018] - first published version
           2.3 [05 Apr 2018] - bugfix '$SENDER_D' blocking
           2.2 [29 Mar 2018] - add a 5s [now 10s] delay before triggering a fail2ban ban, to allow time for any \
outgoing email to original sender (also helps ensure that any subsequent recorded blocks by fail2ban \
represent *new* connection attempts by the now-banned host at that ip), also bugfix code for \
unexpected gsmtp responses
           2.1 [12 Feb 2018] - introduce '$SENDER_D' blocking via postfix if cause of Gmail unhappiness is stated to be sender domain - permanent block by sender domain name
           2.0 [04 Nov 2017] - introduce '$LACK_AUTH' forwarding if email 'From:' is in \$LACK_AUTH_WHITELIST
           1.9 [16 Jul 2017] - help info updated/bugfixed, log entry upon DMARC forwarding no longer says 'bannable'
           1.8 [13 Jul 2017] - email any unrecognised rejection messages to mail address (if defined by option -m)
           1.7 [08 Jul 2017] - output any unrecognised rejection messages
           1.6 [01 Jul 2017] - wait up to 20m (not 60s) for mail log to appear when starting
           1.5 [01 Jun 2017] - fix when restarting after mail log rotation
           1.4 [12 May 2017] - bugfix (grep -a option added throughout)
           1.3 [01 May 2017] - added instructions to avoid hang upon rotation of monitored log file
           1.2 [18 Mar 2017] - set Reply-To for forwarded emails
           1.1 [11 Mar 2017] - many bugfixes
           1.0 [22 Feb 2017] - add special DMARC treatment, option -f
           0.9 [18 Jan 2017] - various bugfixes
           0.8 [09 Jan 2017] - don't delete existing queued message
           0.7 [01 Jan 2017] - delete existing queued message, bugfixes
           0.6 [30 Dec 2016] - bugfixes
           0.5 [22 Dec 2016] - skip attempt to ban if ip is already banned
           0.4 [14 Dec 2016] - use inotifywait, run continuously
           0.3 [14 Dec 2016] - work without amavis (untested)
           0.2 [09 Dec 2016] - allow 'short' and 'long' ban specifications
           0.1 [06 Dec 2016] - first version\n\n" | fold -sw "$COLUMNS"
	fi
	exit 0
fi
# various bits of preparation
# get the parent PID of this process - we dont ever try to kill that
MYPPID=$(ps h -o ppid $$|awk '{print $1}')
TIMEMARK="%H"
[[ -z $QUIET ]] && 	echo -e "\n$THIS v$VERSION by Dominic Raferd (-h for help)\n${THIS//?/=}\n\nStarting at $(date +"%F %T"), PID $$, PPID $MYPPID"
[ -n "$DEBUG" ] && echo "Debug mode" && TIMEMARK="%H:%M"

# get locations of programs and check for critical program availability
POSTMAP=$(whereis postmap 2>/dev/null|cut -d" " -f2) || { echo "Cannot locate postmap command, unable to continue" >&2; exit 1; }
LOGGER=$(whereis logger 2>/dev/null|cut -d" " -f2) || { echo "Cannot locate logger command, unable to continue" >&2; exit 1; }
# do not rely on built-in kill which may not support all features
KILL=$(whereis kill 2>/dev/null|cut -d" " -f2) || { echo "Cannot locate kill command, unable to continue" >&2; exit 1; }
GREPCIDR=$(whereis grepcidr 2>/dev/null|cut -d" " -f2) || { GREPCIDR=$(whereis grep2cidr 2>/dev/null|cut -d" " -f2) || { echo "Cannot locate grepcidr command, unable to continue" >&2; exit 1; } ; }
MYDOMAIN=$($(whereis postconf 2>/dev/null|cut -d" " -f2) -h mydomain)
#hash inotifywait 2>/dev/null || { echo "Missing dependency: inotifywait - please install inotify-tools and retry" >&2; exit 1; }
if [[ -n $LOCALBACKUP || -n $ADMIN_MAILTO ]]; then
	# select the first of $MUAS which is installed
	for MUA_NAME in $MUAS nonesuch; do
		MUA=$(command -v "$MUA_NAME" 2>/dev/null)
		if [[ -n $MUA ]]; then
			# mailx is not usable because it does not support -t
			[[ $(readlink -f "$MUA") =~ mailx ]] || break
		fi
	done
	# note:
	[[ "$MUA_NAME" != "nonesuch" ]] || { echo "Can't locate Mail User Agent, requires one of $MUAS - please install and retry" >&2; exit 1; }
fi
SENDMAIL=$(whereis sendmail 2>/dev/null|cut -d" " -f2) # try to find where sendmail is located
[[ -z $SENDMAIL ]] && SENDMAIL=$MUA # if sendmail was not found, try to use the MUA
[[ -n $VERBOSE ]] && echo "Verbose mode"
# wait up to 20m for mail log file to be found (might not exist after rotation)
i=0; LOOPMAX=60; LOOPWAIT=20
while [ "$i" -lt $LOOPMAX ]; do
	if [ -z "$QUIET" ]; then
		if [ $i -eq 0 ]; then
			printf "%b" "Locating non-null mail log (waiting up to $(( (LOOPMAX * LOOPWAIT ) / 60 )) minutes, each dot (if any) is $LOOPWAIT seconds)"
		elif [ $i -eq 1 ]; then
			# this prints the first dot
			echo; printf "  ."
		else
			printf "."
		fi
	fi
	for MAILLOG in /var/log/mail.log /var/log/maillog; do
		[ -s "$MAILLOG" ] && FOUNDMAILLOG=y && break
	done
	[[ -s $MAILLOG && -z $QUIET ]] && printf "%b" "\n  found: $(ls -il "$MAILLOG")\n" && break
	sleep ${LOOPWAIT}s
	i=$((i + 1))
done
[ -n "$FOUNDMAILLOG" ] || { printf "%b" "\nCouldn't locate any mail log, aborting\n" >&2; exit 1; }

# try to work out where syslog is, obvs this could be wrong - but it is only used for reporting here
[ -s /var/log/messages ] && SYSLOG=/var/log/messages || SYSLOG=/var/log/syslog
# try to work out fail2ban log location
[ -d /etc/fail2ban ] && F2BLOG=$(grep -ahr "^logtarget" /etc/fail2ban/fail2ban.conf /etc/fail2ban/fail2ban.local|tail -n1|awk -F'[ =]' '{print $NF}')
[ -z "$F2BLOG" ] && echo "Warning: is fail2ban installed and running?" >&2
[ -z "$F2BLOG" ] || [ "$F2BLOG" = "SYSLOG" ] && F2BLOG=$SYSLOG

# check for other running instance of $THIS
PREVPIDS="$(pgrep -af "$THIS"|grep -Ev "nano|^$$"|cut -d" " -f1|tr "\n" " ")"
[ -n "$DEBUG" ] && printf "%b" "\$PREVPIDS: '$PREVPIDS'\n"
if [ -n "$PREVPIDS$STOP$RESTART" ]; then
	if [ "$(echo "$PREVPIDS"|wc -w)" -gt 3 ]; then
		printf "%b" "Unexpectedly large number of previous instances of $THIS found:\n $PREVPIDS\n" >&2
	fi
	if [ -n "$PREVPIDS" ]; then
		printf "%b" "Other instance(s) of $THIS already running with PIDs $PREVPIDS"
		[ -z "$STOP$RESTART" ] && printf "\n - use -r option to force restart or -s to stop\n - aborting\n" >&2 && exit 1
		# sometimes a PID has disappeared since we obtained it above, so ignore errors, note PREVPIDS may have >1 value (so should not be quoted)
		# shellcheck disable=SC2086
		$KILL -9 -- $PREVPIDS 2>/dev/null; echo " - killed"; echo "Killed instance(s) $PREVPIDS"|$LOGGER -et "$THIS"
	fi
	[ -n "$STOP" ] && { echo "Quitting"; exit 0; }
else
	[ -n "$DEBUG" ] && echo "No other running instances of $THIS were found"
fi

WELCOMELIST=$(dirname "$0")/$THIS.welcomelist

# AUTHSERVID is the identity used by opendmarc and (must also be) by opendkim locally
PROGRAMS="opendmarc opendkim"
for PROGRAM in $PROGRAMS; do
	if [ -s "/etc/${PROGRAM}.conf" ]; then
		AUTHSERVID1=$(sed -n '/^AuthservID /{s/#,*//;p}' "/etc/${PROGRAM}.conf"|awk '{print $2}')
		[ -z "$AUTHSERVID1" ] && AUTHSERVID1=$(hostname)
		if [ -z "$AUTHSERVID" ]; then
			AUTHSERVID=$AUTHSERVID1
		elif [ "$AUTHSERVID1" != "$AUTHSERVID" ]; then
			echo "Mismatched AuthservID settings in $PROGRAMS, aborting" >&2; exit 1
		fi
	else
		echo "Unable to locate /etc/${PROGRAM}.conf, is $PROGRAM installed? Aborting" >&2; exit 1
	fi
done

# create a default welcomelist containing local and lan ips if none already exists
if [ ! -f "$WELCOMELIST" ]; then
	[ -n "$VERBOSE" ] && echo "Creating $WELCOMELIST"
	printf "%b" "# $THIS welcomelist file\n\n# Local machine\n127.0.0.0/8\n# Local network\n192.168.0.0/16\n10.0.0.0/8\n172.16.0.0/12\n# add ips, ranges or CIDR per grepcidr syntax - each on a newline\n" >"$WELCOMELIST" || { echo "Unable to create $WELCOMELIST, aborting" >&2; exit 1; }
fi
# the default position is not using amavis as content_filter
MATCH_PREFIX[1]=""; MATCH_SUFFIX[1]=": client="; USING_AMAVIS="no"
# unless forcing, check if amavis is installed and decide whether to run amavis-compatible on this basis
[[ -z $FORCE_AMAVIS ]] && postconf -h content_filter 2>/dev/null|grep -q 10024 && FORCE_AMAVIS="y"
# if amavis-compatible, use different match text (used later for searching)
[[ $FORCE_AMAVIS == y ]] && { USING_AMAVIS="yes"; MATCH_PREFIX[2]="queued_as: "; MATCH_SUFFIX[2]=""; }
if [[ $USING_AMAVIS == yes ]]; then
	USEID=2
	# TBH could go wrong if amavis is not the only content_filter, or if a unix socket is used to pass to amavis
	TO_AMAVIS_PORT=$(postconf -h content_filter|awk -F: '{print $NF}')
else
	USEID=1
fi
LOGMSG="Amavis-as-content-filter-compatible: $USING_AMAVIS; options: $OPTIONS"
if [ -z "$QUIET" ]; then
	[[ -n $LOCALBACKUP || -n $ADMIN_MAILTO ]] && echo "Mail program: $MUA"
	if [ -n "$ADMIN_MAILTO" ]; then
		printf "Mailing on "
		[ -n "$VERBOSE" ] && printf "any" || printf "unexpected"
		echo " events to: $ADMIN_MAILTO"
	fi
	printf "%b" "$LOGMSG\nMonitoring $MAILLOG for a new line with text matching:\n  '$ACTION_ON'\nOnce such a match is found, action depends on text in the line matching the first of:\n"
	[[ -n $DO_NOTHING ]] && echo "  nothing happens upon: text matching '$DO_NOTHING'"
	[[ -n $SENDER_D_BANFILE ]] && echo "  sender domain will be blocked (by adding to $SENDER_D_BANFILE) upon: text matching '$SENDER_D'"
	[[ -n $DKIM_RATE ]] && echo "  DKIM domain will be blocked (by adding to $DKIM_RATE_BANFILE) upon: text matching '$DKIM_RATE'"
	if [ -n "$LOCALBACKUP" ]; then
		echo "  forwarding    will be triggered upon: text matching '$DMARC_FAIL'"
		echo "  forwarding    will be triggered upon: text matching '$LACK_AUTH'"
	fi
	echo "  fail2ban ban  will be requested otherwise"
fi
# if we don't have a SENDER_D_BANFILE/DKIM_RATE_BANFILE then set SENDER_D/DKIM_RATE to something that will never be found
[[ -f $SENDER_D_BANFILE ]] || SENDER_D="asassdflkajsdfvadfkjaself"
[[ -f $DKIM_RATE_BANFILE ]] || DKIM_RATE="asassdflkajsdfvadfkjaself"
echo "Starting instance $$; $LOGMSG"|$LOGGER -et "$THIS"
if [ -n "$DEBUG" ]; then
	printf "%b" "Key variables:\n  MYDOMAIN: $MYDOMAIN\n  PID: $$\n  TMPF: $TMPF\n  TMPG: $TMPG\n  SYSLOG: '$SYSLOG'\n  F2BLOG: '$F2BLOG'\n  DEBUG: '$DEBUG'\n  QUIET: '$QUIET'\n  VERBOSE: '$VERBOSE'\n  LOCALBACKUP: '$LOCALBACKUP'\n  ADMIN_MAILTO: '$ADMIN_MAILTO'\n  MUA: '$MUA'\n$WELCOMELIST:\n" && <"$WELCOMELIST" sed 's/^/  /'
fi

while :; do
	tail -fn0 --pid $$ "$MAILLOG" | while read -r MAILLOG_LINE; do
		# main loop - in practice we should never leave this loop
		echo "$MAILLOG_LINE"|sed -rn "s/[1-9][0-9]{2}-[1-9]\.[1-9][0-9]*.[1-9][0-9]* //g;/$ACTION_ON/"'{s/.*\]: ([A-F0-9]{10,}).*/\1/p}' >"$TMPF-f2b"
		# normally $TMPF-f2b will be empty and we jump to the end of the loop
		if [[ ! -s $TMPF-f2b ]]; then
			# see if the message is an unrecognised rejection message of a type that we aren't ignoring - if so
			# send an email and output text, but continue processing
			MESSAGETEXT=$(echo "$MAILLOG_LINE"|sed -rn "/$UNRECOGNISED_REQUIRE"'/{/ said: /{/[45][0-9][0-9] /{/'"$UNRECOGNISED_SKIP"'/d;p}}}')
			if [[ -n $MESSAGETEXT ]]; then
				if [[ -n $ADMIN_MAILTO ]]; then
					printf "%b" "From: $THIS <postmaster@$MYDOMAIN>\nTo: $ADMIN_MAILTO\nSubject: Unrecognised rejection message from Gmail\n\nThis is an automated advisory message from $THIS running on $(hostname -f) mail server. The following rejection message was received from Gmail when relaying an email to it, but the text does not match ${THIS}'s parameters so $THIS has taken no further action (however other daemons may react e.g. by sending a mail notification to the purported originator):\n\n$MESSAGETEXT\n" | $MUA -r postmaster@"$MYDOMAIN" -t
				fi
				echo "$MESSAGETEXT"|sed '1s/^/Unrecognised rejection message from onward relay server:\n/'
			fi
			if [[ -z $QUIET ]]; then
				# print the time on a new line if the hour has changed
				HOUR=$(date +"$TIMEMARK")
				if [ "$HOUR" != "$PREVHOUR" ]; then
					printf "%b" "\n$(date +"%F %T")"
					PREVHOUR="$HOUR"
				fi
				printf "%b" "."
			fi
			continue
		fi
		[[ -n $DEBUG ]] && { printf "%b" "\n$(date +'%F %T') Candidate(s) found: "; <"$TMPF-f2b" wc -l; cat "$TMPF-f2b"|sed 's/^/  /'; }
		if [[ -n $(echo "$MAILLOG_LINE"|sed -rn "/$DO_NOTHING/p") ]]; then
			echo "Found '$DO_NOTHING', ignoring"; continue
		fi
		# split out ban candidates by type into files $TMPF-$BANTYPE
		for BANTYPE in dmarc lack_auth sender_d dkim_rate; do
			MATCHTYPE=$USEID #i.e. 2 for amavis twice-queued, 1 for once-queued
			[[ -s $TMPF-$BANTYPE ]] && truncate -s0 "$TMPF-$BANTYPE"
			if [[ $BANTYPE == dmarc ]]; then
				[[ -n $LOCALBACKUP ]] || continue
				MATCHTEXT="$DMARC_FAIL"
			elif [[ $BANTYPE == sender_d ]]; then
				MATCHTEXT="$SENDER_D"
			elif [[ $BANTYPE == dkim_rate ]]; then
				MATCHTEXT="$DKIM_RATE"
			elif [[ $BANTYPE == lack_auth ]]; then
				[[ -n $LOCALBACKUP ]] || continue
				MATCHTEXT="$LACK_AUTH"
			fi
			if [[ -n $MATCHTEXT ]]; then
				[[ -n $DEBUG ]] && printf "%b" "Separating any '$BANTYPE' candidates i.e. matching '$MATCHTEXT'\n"
				# create list of $BANTYPE candidates
				# if using amavis post-queue filtering, check pre-queue instance for match if we cannot find a post-queue one -
				# sometimes mails may not have been queued twice (e.g. if originated locally or released from quarantine)
				while [[ $MATCHTYPE -ge 1 ]]; do
					echo "$MAILLOG_LINE"|sed -rn "s/[1-9][0-9]{2}-[1-9]\.[1-9][0-9]*.[1-9][0-9]* //g;/$ACTION_ON"'/{/'"$MATCHTEXT"'/s/.*\]: ([A-F0-9]{10,}).*/'"${MATCH_PREFIX[$MATCHTYPE]}"'\1'"${MATCH_SUFFIX[$MATCHTYPE]}"'/p}'|sort -u >"$TMPF-$BANTYPE"
					if [[ -s $TMPF-$BANTYPE ]]; then
						if tac "$MAILLOG"|grep -qm1 -aFf "$TMPF-$BANTYPE"; then
							[[ -n $VERBOSE ]] && echo -e "\n$(date +%T): found bantype $BANTYPE matchtype $MATCHTYPE"
							break
						fi
						[[ -n $VERBOSE ]] && echo -e "\n$(date +%T): could not locate bantype $BANTYPE matchtype $MATCHTYPE"
					else
						[[ -n $DEBUG ]] && echo -e "\n$(date +%T): Identified bantype $BANTYPE matchtype $MATCHTYPE was not found i.e.\n  echo \"$MAILLOG_LINE\"|\n  sed -rn \"/$ACTION_ON\"'/{/'\"$MATCHTEXT\"'/s/.*\]: ([A-F0-9]{10,}).*/'\"${MATCH_PREFIX[$MATCHTYPE]}\"'\1'\"${MATCH_SUFFIX[$MATCHTYPE]}\"'/p}'" >>"/var/tmp/$THIS-debug.log" # [31 May 2022]
					fi
					MATCHTYPE=$((MATCHTYPE-1))
				done
				# if we found a match then stop looking for other matches and remove the 'f2b' match
				[[ -s $TMPF-$BANTYPE ]] && { rm -- "$TMPF-f2b"; break; }
				# create list of 'f2b ban' candidates - ( but this is never used? - removed [19 Apr 2022]
				#cp "$TMPF-f2b" "$TMPF-f2b-tmp"; diff --suppress-common-lines "$TMPF-$BANTYPE" "$TMPF-f2b-tmp"|sed -n 's/^> //p' >"$TMPF-f2b"
				#rm -- "$TMPF-f2b-tmp"
			fi
		done
		if [[ -n $DEBUG ]]; then
			echo "After separation:"
			for BANTYPE in f2b dmarc lack_auth sender_d dkim_rate; do
				[[ -s "$TMPF-$BANTYPE" ]] && COUNT_BANTYPE=$(<"$TMPF-$BANTYPE" wc -l) || COUNT_BANTYPE=0
				if [[ $COUNT_BANTYPE -gt 0 ]]; then
					printf "%b" "$COUNT_BANTYPE '$BANTYPE' candidate found"
					if [[ -s $TMPF-$BANTYPE ]]; then
						echo ":"; sed 's/^/    /' "$TMPF-$BANTYPE" && rm -f -- "$TMPG-$BANTYPE"
					else
						echo
					fi
					break
				fi
			done
		fi

		SYSLOG_LINES_PRE=$(wc -l "$SYSLOG"|cut -d' ' -f1)
		[[ $SYSLOG == "$F2BLOG" ]] && F2BLOG_LINES_PRE=$SYSLOG_LINES_PRE || F2BLOG_LINES_PRE=$(wc -l "$F2BLOG"|cut -d' ' -f1)
		# if not already up to date, create the actual files we will use for welcomelisting: $TMPBASE-welcome(cidr|host)
		if [[ ! -f "$TMPBASE-welcomecidr" || "$WELCOMELIST" -nt "$TMPBASE-welcomecidr" ]]; then
			sed 's/^\s*//;s/\s*#.*//;/^$/d' "$WELCOMELIST"|grep -E "^[-/.0-9]+$" >"$TMPBASE-welcomecidr"
			sed 's/^\s*//;s/\s*#.*//;/^$/d' "$WELCOMELIST"|grep -Ev "^[-/.0-9]+$" >"$TMPBASE-welcomehost"
		fi

		for BANTYPE in dkim_rate f2b dmarc lack_auth sender_d; do
			[[ -s $TMPF-$BANTYPE ]] || { continue; }
			[[ -n $DEBUG ]] && echo "Identified bantype '$BANTYPE':"
			[[ $BANTYPE == f2b ]] && MATCHTYPE=$USEID # for f2b matches, restore the standard USEID [06 Jun 2022]
			for i in 1 2 3 4 5; do
				[[ $i -gt 1 ]] && sleep 3s
				if [[ $BANTYPE == dkim_rate ]]; then # enforcement handled here
					# get the DKIM domain from the matched message
					DKIM_RATE_DOMAIN=$(echo "$MAILLOG_LINE"|grep  -oE "domain \[\S+"|cut -c9-|sed 's/\./\\\\\\./g')
					if grep -x "/d=${DKIM_RATE_DOMAIN};/ REJECT" "$DKIM_RATE_BANFILE"; then
						echo -en "\n$(date +'%F %T'): Identified bantype $BANTYPE but $DKIM_RATE_DOMAIN already listed in $DKIM_RATE_BANFILE for DKIM-Signature rejection, skipping" >&2
					else
						# find the line number for the 1st 'endif' after 'if /^DKIM-Signature:/''
						DKIM_RATE_BANFILE_LINE_LOC=$(grep -aEn "^(end)?if" "$DKIM_RATE_BANFILE"|grep -FA1 'if /^DKIM-Signature:/'|tail -n1|cut -d: -f1)
						if [[ ! $DKIM_RATE_BANFILE_LINE_LOC =~ ^[1-9][0-9]*$ ]]; then
							echo -en "\n$(date +'%F %T'): Identified bantype $BANTYPE for $DKIM_RATE_DOMAIN but could not find line number in $DKIM_RATE_BANFILE, skipping" >&2
						else # we have a valid line number
							echo -en "\n$(date +'%F %T'): Identified bantype $BANTYPE, added $DKIM_RATE_DOMAIN for DKIM-Signature rejection to $DKIM_RATE_BANFILE (line $DKIM_RATE_BANFILE_LINE_LOC), "
							sed -i "${DKIM_RATE_BANFILE_LINE_LOC}s~^~# added by $THIS $(date +"%F %T")\n/d=$DKIM_RATE_DOMAIN;/ REJECT\n~" "$DKIM_RATE_BANFILE"
							postfix reload
						fi
					fi
					break # out of this for loop
				elif [[ $BANTYPE == sender_d ]]; then
					# awk1: convert whatever the search term is into: 'qmgr\\[.*0123456789: from=<'
					# grep: use the converted search term to return the relevant qmgr log entry
					# awk2: from the log entry extract the sender domain to $TMPG-$BANTYPE, one per line
					while read -r ENVADDR; do
						if [[ ${ENVADDR:0:3} == "SRS" ]]; then
							# it appears that we are using postsrsd, extract the original source domain
							# example: from SRS0=IsFY=5V=aprem-kg.com=bronze-sculptures@ourdomain.tld extract aprem-kg.com
							echo "$ENVADDR"|cut -d= -f4
						else # otherwise
							echo "$ENVADDR"|awk -F@ '{print $NF}'
						fi
					done < <(awk -F"[ :]" '{for (f=1; f<=NF; f++) {if ($f ~ /[0-9A-F]{10,}/) print "qmgr\\[.*" $f ": from=<"}}' "$TMPF-$BANTYPE"|grep -af- "$MAILLOG"|awk -F"[<>]" '{print $2}') >"$TMPG-$BANTYPE"
				elif [[ $MATCHTYPE == 2 ]]; then
					# grep: lookup the amavis message containing the 2nd Queue-ID we extracted earlier - and the original Queue-ID
					# sed: extract the original Queue-ID and build a search term to extract the connection log messages
					SEARCH_FOR_CONNECTION_LOG="$(tac "$MAILLOG"|grep -am1 "CLEAN.*$(cat "$TMPF-$BANTYPE")"|sed -n 's/.*Queue-ID: \([A-F0-9]\{10,\}\).*/\1: client=/p')"
					# if amavis logmsg does not contain original Queue-ID, look for/at to-amavis/smtp message [12 Oct 2022]
					[[ -z $SEARCH_FOR_CONNECTION_LOG && -n $TO_AMAVIS_PORT ]] && SEARCH_FOR_CONNECTION_LOG="$(tac "$MAILLOG"|grep -am1 ":$TO_AMAVIS_PORT,.*queued as $(cat "$TMPF-$BANTYPE"))"|sed 's/.*smtp\[[0-9]\+\]: //;s/:.*//')"
					if [[ -z $SEARCH_FOR_CONNECTION_LOG ]]; then
						[[ -n $VERBOSE ]] && echo -en "\n$(date +'%F %T'): Message $(cat "$TMPF-$BANTYPE") - "
						if tac "$MAILLOG"|grep -qaFm1 "$(cat "$TMPF-$BANTYPE"): resent-message-id="; then
							[[ -n $VERBOSE ]] && echo "resent from quarantine, ignoring [loop $i]"
						else
							# this is suggestive of a bug [11 Jun 2022]
							[[ -n $VERBOSE ]] && echo "could not find original Queue-ID, ignoring [loop $i]"
						fi
					else
						# grep: extract the original connection log message
						# sed: extract just the original Queue-ID, the hostname and the host_ip
						# result should be like this: 55B523ED2D versan.livingwealth.net 212.83.143.12
						tac "$MAILLOG"|grep -aFm1 "$SEARCH_FOR_CONNECTION_LOG"|sed 's/^[^]]*\]: //;s/client=//;s/\[/ /;s/[]:]//g' >"$TMPG-$BANTYPE.tmp"
						[[ -n $DEBUG ]] && echo -e "  \$SEARCH_FOR_CONNECTION_LOG: '$SEARCH_FOR_CONNECTION_LOG'\n  $TMPG-$BANTYPE.tmp: '$(cat "$TMPG-$BANTYPE.tmp")'"
						if [[ -s $TMPG-$BANTYPE.tmp ]]; then
							# prefix to each line the 2nd queue-ID (i.e. from the outgoing mail) - this is needed by postsuper
							# awk: check we have 4 fields in the result
							# result should be like this: 83D6F3E95F 55B523ED2D versan.livingwealth.net 212.83.143.12
							sed 's/.* \([A-F0-9]\{10,\}\).*/\1/' "$TMPF-$BANTYPE"|paste -d" " - "$TMPG-$BANTYPE.tmp"|awk '{if (NF==4) {print $0} else {print "\"" $0 "\" has " NF " fields instead of 4, skipping" > "/dev/stderr"}}' >"$TMPG-$BANTYPE"
							if [[ -n $DEBUG ]]; then # for debugging [17 Apr 2022]
								DEBUG_FILE_SUFFIX=$(shuf -i 100000-999999 -n1)
								cp "$TMPG-$BANTYPE.tmp" "$TMPG-$BANTYPE-$DEBUG_FILE_SUFFIX.tmp"
								cp "$TMPF-$BANTYPE" "$TMPF-$BANTYPE-$DEBUG_FILE_SUFFIX"
								cp "$TMPG-$BANTYPE" "$TMPG-$BANTYPE-$DEBUG_FILE_SUFFIX"
							fi
						elif [[ -n $VERBOSE ]]; then
							echo -e "\n$(date +'%F %T'): Unable to locate '$(cat "$TMPF-$BANTYPE")' in $MAILLOG [loop $i]"
						fi
						rm -- "$TMPG-$BANTYPE.tmp"
					fi
				else
					# grep: extract the original connection log message (searching for: '[Queue-ID]: client=')
					# sed: extract just the Queue-ID, the hostname and the host_ip
					# sort: remove any duplicates
					# awk: check we have 3 fields in the result
					# result should be like this: 55B523ED2D versan.livingwealth.net 212.83.143.12
					tac "$MAILLOG"|grep -am1 -Ff "$TMPF-$BANTYPE"|sed 's/^[^]]*\]: //;s/client=//;s/\[/ /;s/[]:]//g'|sort -u|awk '{if (NF==3) print $0}' >"$TMPG-$BANTYPE"
					if [[ -n $DEBUG ]]; then # for debugging [17 Apr 2022]
						DEBUG_FILE_SUFFIX=$(shuf -i 100000-999999 -n1)
						cp "$TMPF-$BANTYPE" "$TMPF-$BANTYPE-$DEBUG_FILE_SUFFIX"
						cp "$TMPG-$BANTYPE" "$TMPG-$BANTYPE-$DEBUG_FILE_SUFFIX"
					fi
				fi
				# there should be a match, if not, wait a bit and try again (up to 4 more times) [5 Jul 2020]
				[[ -s $TMPG-$BANTYPE ]] && { break; }
			done

			if [[ $BANTYPE = dkim_rate ]]; then
				break
			# apply welcomelisting (for which we use encapsulation&forwarding, not enforcement)
			elif [[ $BANTYPE = dmarc || $BANTYPE = lack_auth ]]; then # we always encapsulate&forward for these types
				cat "$TMPG-$BANTYPE" >>"$TMPG-$BANTYPE-welcome"
				truncate -s0 "$TMPG-$BANTYPE"
			elif [[ $BANTYPE != sender_d ]]; then # we never encapsulate&forward for sender_d, the ip is irrelevant
				# grepcidr: extract entries with welcomelisted ips into (empty) $TMPG-$BANTYPE-welcome
				$GREPCIDR -f "$TMPBASE-welcomecidr" "$TMPG-$BANTYPE" >>"$TMPG-$BANTYPE-welcome"
				grep -Ef "$TMPBASE-welcomehost" "$TMPG-$BANTYPE" >>"$TMPG-$BANTYPE-welcome"
				# grepcidr: remove entries with welcomelisted ips from $TMPG-$BANTYPE
				$GREPCIDR -vf "$TMPBASE-welcomecidr" "$TMPG-$BANTYPE"|grep -vEf "$TMPBASE-welcomehost" >"$TMPG-$BANTYPE.tmp"; mv "$TMPG-$BANTYPE.tmp" "$TMPG-$BANTYPE"
			fi

			# deal with encapsulation&forwarding
			if [[ -s $TMPG-$BANTYPE-welcome ]]; then
				if [[ ! -s $LOCALBACKUP ]]; then
					[[ -n $VERBOSE ]] && printf "%b" "\noops, can't find $LOCALBACKUP, skipping\n"
					continue
				fi
				# we read through $LOCALBACKUP 3x - twice by grep, once by formail :-(
				#
				# get a list of the line numbers that are the start of each email in $LOCALBACKUP:
				#   a rather horrible workaround ensures that the '^From ' is really start of an email and not in the body text:
				#   real 'From ' (starting an email) is always followed by a 'Return-Path' line
				grep -anA1 '^From ' "$LOCALBACKUP"|grep -aB1 -- '^[1-9][0-9]*-Return-Path: '|grep -aE '^[1-9][0-9]*:From '|cut -d: -f1 >"$TMPF-$BANTYPE-starts.txt"
				[ -n "$DEBUG" ] && echo "  there are $(<"$TMPF-$BANTYPE-starts.txt" wc -l) emails in $LOCALBACKUP"
				# get the number of emails preceding the one(s) we want (>dmarc-mailnums.txt for use by formail):
				#   awk:  get the queue-id(s) and precede by /
				#   grep: get number:line in $LOCALBACKUP to match '/queue-id' (openDMARC Authentication header)
				#         or ' queue-id(;|$)' (Received header) in outgoing email
				#         (opendmarc.conf must have AuthservIDWithJobID set to true or it cannot be found)
				#   cut:  get just the line number (of the openDMARC Authentication header line)
				LINENUM=$(awk -v USEID="$MATCHTYPE" '{print "(/"$USEID"|[ ]"$USEID";|[ ]"$USEID"$)"}' "$TMPG-$BANTYPE-welcome"|grep -Eanm1 -f- "$LOCALBACKUP"|cut -d: -f1)
				# if no match found, search via resent-message-id, in case blocked message was resent from amavis quarantine [27 Apr 2022]
				if [[ -z $LINENUM ]]; then
					LINENUM=$(awk -v USEID="$MATCHTYPE" '{print " "$USEID": resent-message-id=<"}' "$TMPG-$BANTYPE-welcome"|grep -Fm1 -f- "$MAILLOG"|awk -F'[<>]' '{print $2}'|grep -anFm1 -f- "$LOCALBACKUP"|cut -d: -f1)
					if [[ -n $DEBUG ]]; then
						[[ -n $LINENUM ]] && echo "  email located in $LOCALBACKUP via it's resent-message-id" || echo "Mail could not be located via resent-message-id"
					fi
				fi
				#   awk:  now find the line number that marks the start of the next email after the identified line,
				#         output the number count of that email in the file less 2 (or if none found - because the relevant
				#         email is the last - the total number of emails less 1)
				awk -v LINENUM="$LINENUM" '{if ($0>LINENUM) {FOUND=1; exit}} END {print (NR-1-FOUND)}' "$TMPF-$BANTYPE-starts.txt" >"$TMPF-$BANTYPE-mailnums.txt"
				[ -n "$DEBUG" ] && { printf "%b" "  we need email no(s) (0 up): "; sed 'N;s/\n/ /' "$TMPF-$BANTYPE-mailnums.txt"; }
				while read -r NUM; do
					[ -n "$DEBUG" ] && printf "%b" "    extracting to $NUM.eml: "
					# -f prevents addition of new mbox header line, and grep deletes existing mbox header line
					<"$LOCALBACKUP" formail +"$NUM" -1fY -I Status -I Content-Length -I Lines -I X-Original-To -I Delivered-To -I Message-ID -s|grep -Eav "^From \S*  ([[:upper:]][[:lower:]]{2} ){2}[ 123][0-9] ([0-9]{2}:){2}[0-9]{2} 2[0-9]{3}$" >/tmp/"$NUM.eml"
					[ -n "$DEBUG" ] && echo "done"
					if [ ! -s /tmp/"$NUM.eml" ]; then
						echo "    - extracted file is null, could not extract mail no.$((NUM+1)), skipping" >&2
					else
						[ -n "$DEBUG" ] && echo "    mail extraction succeeded"
						# use tr in hope that it works for multiple recipients i.e. multiple lines with same queue id (untested)
						for SEARCH in orig_to to; do
							RECIPIENTS=$(echo "$MAILLOG_LINE"|sed -rn "s/[1-9][0-9]{2}-[1-9]\.[1-9][0-9]*.[1-9][0-9]* //g;/$ACTION_ON"'/s/.* '"$SEARCH"'=<([^>]*).*/\1/p'|tr '\n' ','|sed 's/,$//')
							[ -n "$RECIPIENTS" ] && { break; }
						done
						[ -n "$RECIPIENTS" ] || { echo "Could not extract recipient data from '$MAILLOG_LINE', skipping" >&2; continue; }
						[ -n "$DEBUG" ] && echo "    recipient data extracted from $NUM.eml"
						# extract original From info, convert to readable format
						FROM=$(formail -Ycx "From:" </tmp/"$NUM.eml"|xargs -I {} python3 -c 'from email.header import Header, decode_header, make_header; print(make_header(decode_header("{}")))')
						REPLYTO=$(formail -Ycx "Reply-To:" </tmp/"$NUM.eml"|xargs -I {} python3 -c 'from email.header import Header, decode_header, make_header; print(make_header(decode_header("{}")))')
						[ -z "$REPLYTO" ] && REPLYTO=$FROM
						[ -n "$DEBUG" ] && printf "%b" "      Recipients: '$RECIPIENTS'\n      From: '$FROM'\n      ReplyTo: '$REPLYTO'\n"
						SUBJECT=$(formail -Ycx "Subject:" </tmp/"$NUM.eml"|xargs -I {} python3 -c 'from email.header import Header, decode_header, make_header; print(make_header(decode_header("{}")))')
						# note we don't send on to any cc recipients
						if [ -z "$RECIPIENTS" ]; then
							echo "Could not extract recipient name from /tmp/$NUM.eml, skipping" >&2
						else
							[ -n "$VERBOSE" ] && printf "%b" "\n$(date "+%F %T") Sending email to $(echo "$RECIPIENTS"|tr '\n' ' ')with /tmp/$NUM.eml attachment (ban type '$BANTYPE' triggered by log line: '$MAILLOG_LINE')\n"
							# AUTHSERVID does not have escaped periods but they are complicated to do shell-agnostically so we dont bother
							DMARCLINE=$(grep -m1 "^Authentication-Results: $AUTHSERVID.*dmarc=pass" /tmp/"$NUM.eml")
							if [ -n "$DMARCLINE" ] && [ "$BANTYPE" = "dmarc" ]; then
								MESSAGETEXT=", but is genuine - although it could still be spam. The reason for blocking by Gmail is"
								if echo "$DMARCLINE"|awk -F= '{printf "^DKIM-Signature: .*d=" gensub(/[.]/,"\\\\.","g",$NF) ";"}'|grep -qf- /tmp/"$NUM.eml"; then
									# there was a dkim line matching the domain in our dmarc pass test results
									MESSAGETEXT="$MESSAGETEXT uncertain (message had a valid DKIM signature when it passed through here)"
								else
									# the most recent dmarc test (ours) was a pass
									MESSAGETEXT="$MESSAGETEXT that it lacks a DKIM signature"
								fi
							elif [ $BANTYPE = "lack_auth" ]; then
								MESSAGETEXT=", but is probably genuine and safe to open"
							else
								# we can't verify the dmarc test, or it is lack_auth
								MESSAGETEXT=" - use your judgement as to whether it is genuine"
							fi
							# split up any very long text lines so they don't cause DKIM failure [21 Sep 2020]
							sed -i "/^Content-Type: text\/plain/I,/^Content-Type:/I{s/^\(.\{990\}\)\(.\)/\1\n\2/}" /tmp/"$NUM.eml"
							printf "%b" "From: Postmaster <postmaster@$MYDOMAIN>\nTo: $RECIPIENTS\nSubject: Fw: $SUBJECT\n\nThe attached email from $REPLYTO (sent via $(echo "$FROM"|sed 's/.*@//;s/>.*//')) was blocked by Gmail ($BANTYPE failure)$MESSAGETEXT.\n\nKind regards from $(hostname -f) mail server\n($THIS v$VERSION)\n" | $MUA -r postmaster@"$MYDOMAIN" -t --attach=/tmp/"$NUM.eml" >"$TMPG-mua.out"; MUA_ERR=$? # s-nail 14.x produces warning messages re mbox format
							sed -i '/mbox/Id' "$TMPG-mua.out"; rm -- "$TMPG-mua.out"
							if [ $MUA_ERR -eq 0 ]; then
								[ -n "$VERBOSE" ] && printf "%b" "$(date "+%F %T") Accepted by $MUA for onward delivery\n"; SENT="$SENT /tmp/$NUM.eml"
								[ -z "$DEBUG" ] && rm -- /tmp/"$NUM.eml"
								# in this case there should be no fail2ban action, but put a record in syslog anyway
								sed 's/^/forwarded /;s/queued_as: //' "$TMPG-$BANTYPE-welcome"|$LOGGER -et "$THIS-$BANTYPE"
							else
								echo "There was a problem sending email with /tmp/$NUM.eml attachment ($MUA error $MUA_ERR)" >&2
							fi
						fi
					fi
				done <"$TMPF-$BANTYPE-mailnums.txt"
			fi
			if [[ ! -s $TMPG-$BANTYPE ]]; then
				if [[ ! -s $TMPG-$BANTYPE-welcome ]]; then
					printf "%b" "\n$(date "+%F %T") Skipping bantype $BANTYPE as could not find '$(cat "$TMPF-$BANTYPE")' in $MAILLOG\n" >&2
				else
					truncate -s0 "$TMPG-$BANTYPE-welcome"
				fi
				continue
			else
				[[ -s $TMPG-$BANTYPE-welcome ]] && truncate -s0 "$TMPG-$BANTYPE-welcome"
			fi

			# deal with enforcement
			[[ -n $DEBUG ]] && { echo "  seeking:"; sed 's/^/    /' "$TMPG-$BANTYPE"; }
			if [[ $BANTYPE = sender_d ]]; then
				# exclude any that are already listed REJECT in $SENDER_D_BANFILE
				sed 's/$/ REJECT/' "$TMPG-$BANTYPE"|xargs -I {} /bin/sh -c "grep -qFx \"{}\" $SENDER_D_BANFILE || echo \"{}\"" >>"$TMPG-$BANTYPE.tmp"
				if [[ -s $TMPG-$BANTYPE.tmp ]]; then
					sort -uo "$TMPG-$BANTYPE.tmp" "$TMPG-$BANTYPE.tmp" # remove any duplicates
					# this is really a debug message [04 May 2018]
					printf "%b" "\nAdding line to $SENDER_D_BANFILE:'$(cat "$TMPG-$BANTYPE.tmp")'\n"
					# add the REJECT line (with a preceding comment)
					echo "# added by $THIS on $(hostname) $(date +"%F %T")" >>"$SENDER_D_BANFILE"
					cat "$TMPG-$BANTYPE.tmp" >>"$SENDER_D_BANFILE"
					# update the db file, then it will become live after a short period anyway
					printf "%b" "Updating hash file ${SENDER_D_BANFILE}.db: "
					$POSTMAP $SENDER_D_BANFILE && echo "OK" || echo "FAIL"
					# add a line to logger
					sed "s~^~block_sender ~;s~ REJECT~ via ${SENDER_D_BANFILE}~" "$TMPG-$BANTYPE.tmp"|$LOGGER -et "$THIS-$BANTYPE"
					# text output
					sed "s/^/\n$(date "+%F %T") Block_sender /;s~ REJECT~ via $SENDER_D_BANFILE\n~" "$TMPG-$BANTYPE.tmp"
					if [ -n "$ADMIN_MAILTO" ]; then
						# email to admin
						printf "%b" "From: Postmaster <postmaster@$MYDOMAIN>\nTo: $ADMIN_MAILTO\nSubject: $THIS - $BANTYPE action report\n\n$(hostname)'s log of report received from Gmail:\n$MAILLOG_LINE\nReason for blockage by gsmtp was identified as '$BANTYPE' and so we added '$(cut -d" " -f1 "$TMPG-$BANTYPE.tmp"|tr "\n" " ")REJECT' in $SENDER_D_BANFILE.\n" | $MUA -r postmaster@"$MYDOMAIN" -t
					fi
				else
					printf "%b" '\n'"$(date "+%F %T") Block_sender "
					cat "$TMPG-$BANTYPE.tmp"|tr '\n' ' '
					echo "already listed in $SENDER_D_BANFILE, so skipped"
				fi
				rm -- "$TMPG-$BANTYPE.tmp"
			else # f2b bantype
				# Remove from our list if already banned - not essential but keeps logs tidier
				# iptables: show instructions for this chain
				# e.g. -N f2b-relay-enforcer
				#      -A f2b-relay-enforcer -s 174.47.100.204/32 -j REJECT --reject-with icmp-port-unreachable
				#      -A f2b-relay-enforcer -j RETURN
				# awk: extract just banned ip(s), built into a set of sed delete instructions, with escaped full stops
				# sed: runs the generated instruction, deleting lines from our list where the ip matches what is already banned
				if [ -n "$DEBUG" ]; then
					printf "%b" "Debug mode dummy run actions:\n  Count bannable ip(s) listed in $TMPG-$BANTYPE: " && <"$TMPG-$BANTYPE" wc -l && echo "  Contents of $TMPG-$BANTYPE:" && sed 's/^/    /' "$TMPG-$BANTYPE"
					printf "%b" "  Checking if already banned\n    Extract any ips banned under iptables/f2b-$THIS-$BANTYPE and build sed script (>$TMPF-test1):\n"
					iptables -S "f2b-$THIS" 2>/dev/null|awk -F"[/ ]" '{if ($7=="REJECT") {gsub(/\./,"\\.",$4); printf "/"$4"/d;"}}'|tee "$TMPF-test1"|sed "s/^/      '/;s/$/'/"
					printf "%b" "\n    sed operation on $TMPG-$BANTYPE results in (>$TMPF-test2):\n"
					sed -f "$TMPF-test1" "$TMPG-$BANTYPE"|tee "$TMPF-test2"|sed 's/^/      /'
					printf "  Debug mode dummy run actions completed, now do it for real"
				fi
				sed -i "$(iptables -S "f2b-$THIS" 2>/dev/null|awk -F'[/ ]' '{if ($7=="REJECT") {gsub(/\./,"\\.",$4);printf "/" $4 "/d;"}}')" "$TMPG-$BANTYPE"
				[ -n "$VERBOSE" ] && printf "%b" "\n$(date +"%F %T") Identified a candidate ip for $BANTYPE ban"
				# there may be nothing left now...
				if [ ! -s "$TMPG-$BANTYPE" ]; then
					[ -n "$VERBOSE" ] && echo " but it is already banned, skipping"
				elif [ -z "$TEST" ]; then
					[ -n "$DEBUG" ] && printf "%b" "\nWait a few seconds to allow any ongoing mail transaction to complete (esp outgoing notification to original sender)\n"
					sleep 10s
					[ -n "$DEBUG" ] && printf "%b" "\nAdding message to system log with tag $THIS-$BANTYPE - can be picked up by fail2ban jail $THIS-$BANTYPE\n"
					# example line in $TYPG-$BANTYPE if w/o amavis: 0CF4C3E92D decla.mycashtube.com 146.0.229.84
					#                           ... or with amavis: A95F3B465C 0CF4C3E92D decla.mycashtube.com 146.0.229.84
					# example log line: 2016-12-06 09:13:24 vps344433 relay-enforcer: bannable 0CF4C3E92D decla.mycashtube.com 146.0.229.84
					sed 's/^/bannable /;s/queued_as: //' "$TMPG-$BANTYPE"|$LOGGER -et "$THIS"
					# show what we found...
					echo "$(hostname)'s log of report received from Gmail:" >"$TMPF-mail"
					# extract qmgr line in mail log to show envelope sender
					sed -n "/qmgr\[.*$(cut -d" " -f1 "$TMPG-$BANTYPE"): from/p" "$MAILLOG" >>"$TMPF-mail"
					echo "$MAILLOG_LINE"|sed -rn "s/[1-9][0-9]{2}-[1-9]\.[1-9][0-9]*.[1-9][0-9]* //g;/$ACTION_ON/p" >>"$TMPF-mail"
					if [ -z "$TEST" ]; then # (it's pointless to try this when testing)
						sleep 2s # give fail2ban time to act on the information
						# ... and what happened
						printf "%b" "\n$(hostname)'s log of its intervention:\n" >>"$TMPF-mail"
						# example: 2016-12-06 09:44:30 vps344433 fail2ban.actions[11699]: NOTICE [relay-enforcer] Ban 146.0.229.82
						sed -n "$((1+SYSLOG_LINES_PRE))"',${/'"$THIS"'/p}' "$SYSLOG" >>"$TMPF-mail"
						[ "$SYSLOG" != "$F2BLOG" ] && sed -n "$F2BLOG_LINES_PRE"',${/'"$THIS"'/p}' "$F2BLOG" >>"$TMPF-mail"
						IP=$(awk '{printf $NF" "}' "$TMPG-$BANTYPE")
						#BAN=$(grep -aEom1 " $THIS:" "$TMPF-mail"|tr -d ': ')
						printf "%b" "\nTo unban on $(hostname) before ban expiry ($(fail2ban-client get "$THIS" bantime)s): fail2ban-client unban $IP\n" >>"$TMPF-mail"
					fi
					[ -n "$VERBOSE" ] && echo
					[ -z "$QUIET" ] && cat "$TMPF-mail"
					# send email
					[ -n "$ADMIN_MAILTO" ] && [ -n "$VERBOSE" ] && printf "%b" "To: $ADMIN_MAILTO\nSubject: $THIS - $BANTYPE action report\n\n"|cat - "$TMPF-mail"|$SENDMAIL -r postmaster@"$MYDOMAIN" -t
					rm -- "$TMPF-mail"
				elif [ -n "$VERBOSE" ]; then
					printf "\nTest mode: skipped adding to log, fail2ban event will not be triggered\n"
				fi
			fi
		done
		if [ -n "$DEBUG" ]; then
			printf "\nRetained:\n"
			[ -n "$TMPF" ] && [ -n "$TMPG" ] && find /tmp \( -regex "${TMPF}.*" -or -regex "${TMPG}.*" -or -regex "/tmp/[0-9]*.eml" \) -mmin -3 -ls
		elif [ -n "$TMPF" ] && [ -n "$TMPG" ];then
			for BANTYPE in f2b dmarc lack_auth full; do
				rm -f -- "$TMPF-$BANTYPE" "$TMPG-$BANTYPE" "$TMPF-$BANTYPE.tmp" "$TMPG-$BANTYPE.tmp" "$TMPF-$BANTYPE-starts.txt" "$TMPF-$BANTYPE-mailnums.txt"
			done
		fi
	done # end of main loop
	echo -e "\n$(date +"%F %T") exited from main loop (was 'tail' operation killed?), relooping"
done
# we should never get here
echo -e "\n$(date +"%F %T") exited from overriding loop, this should NEVER happen, quitting"
exit 1