#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# SPDX-License-Identifier: GPL-2.0-or-later
# Copyright (C) 2020 by the Linux Foundation
#
__author__ = 'Konstantin Ryabitsev <konstantin@linuxfoundation.org>'

import argparse
import email
import email.parser
import email.utils
import fnmatch
import io
import json
import mailbox
import os
import pathlib
import re
import shlex
import shutil
import sys
import time
from email.message import EmailMessage
from string import Template
from typing import Any, Dict, List, Optional, Set, Tuple, Union

import b4

logger = b4.logger

DEFAULT_MERGE_TEMPLATE = """Merge ${patch_or_series} "${seriestitle}"

${authorname} <${authoremail}> says:

${covermessage}

Link: ${midurl}
"""


def save_msgs_as_mbox(
    dest: str, msgs: List[EmailMessage], filterdupes: bool = False
) -> int:
    if dest == '-':
        b4.save_mboxrd_mbox(msgs, sys.stdout.buffer, mangle_from=False)
        return len(msgs)

    if b4.is_maildir(dest):
        mdr = mailbox.Maildir(dest)
        have_msgids = set()
        added = 0
        if filterdupes:
            for emsg in mdr:
                have_msgids.add(b4.LoreMessage.get_clean_msgid(emsg))  # type: ignore[arg-type] # ty:ignore[invalid-argument-type] # this will go away when we update liblore
        for msg in msgs:
            if b4.LoreMessage.get_clean_msgid(msg) not in have_msgids:
                added += 1
                mdr.add(msg)
        return added

    with open(dest, 'wb') as fh:
        b4.save_mboxrd_mbox(msgs, fh, mangle_from=True)

    return len(msgs)


def get_base_commit(
    topdir: Optional[str], body: str, lser: b4.LoreSeries, cmdargs: argparse.Namespace
) -> str:
    base_commit = 'HEAD'

    if lser.prereq_base_commit:
        logger.debug(
            'Setting base-commit to prereq-base-commit: %s', lser.prereq_base_commit
        )
        base_commit = lser.prereq_base_commit
    else:
        matches = re.search(r'base-commit: .*?([\da-f]+)', body, re.MULTILINE)
        if matches:
            base_commit = matches.groups()[0]
        else:
            # Try a more relaxed search
            matches = re.search(r'based on .*?([\da-f]{40})', body, re.MULTILINE)
            if matches:
                base_commit = matches.groups()[0]

    if base_commit and topdir:
        # Does it actually exist in this tree?
        if not b4.git_commit_exists(topdir, base_commit):
            logger.warning(' Base: base-commit %s not known, ignoring', base_commit)
            base_commit = 'HEAD'
        elif not cmdargs.mergebase:
            logger.debug(' Base: using specified base-commit %s', base_commit)

    if base_commit == 'HEAD' and topdir and cmdargs.guessbase:
        logger.info(' Base: attempting to guess base-commit...')
        try:
            base_commit, nblobs, mismatches = lser.find_base(
                topdir, branches=cmdargs.guessbranch, maxdays=cmdargs.guessdays
            )
            if mismatches == 0:
                logger.critical(' Base: %s (exact match)', base_commit)
            elif nblobs == mismatches:
                logger.critical(' Base: failed to guess base')
            else:
                logger.critical(
                    ' Base: %s (best guess, %s/%s blobs matched)',
                    base_commit,
                    nblobs - mismatches,
                    nblobs,
                )
        except IndexError as ex:
            logger.critical(' Base: failed to guess base (%s)', ex)

    if cmdargs.mergebase:
        if base_commit:
            logger.debug(
                ' Base: overriding submitter provided base-commit %s with %s',
                base_commit,
                cmdargs.mergebase,
            )
        base_commit = cmdargs.mergebase

    return base_commit


def make_am(msgs: List[EmailMessage], cmdargs: argparse.Namespace, msgid: str) -> None:
    config = b4.get_main_config()
    outdir = cmdargs.outdir
    if outdir == '-':
        cmdargs.nocover = True
    if 'addmsgid' in cmdargs and cmdargs.addmsgid:
        logger.debug('Setting linktrailermask to Message-ID:')
        config['linktrailermask'] = 'Message-ID: <%s>'
        cmdargs.addlink = True
    wantver = cmdargs.wantver
    count = len(msgs)
    logger.info('Analyzing %s messages in the thread', count)
    lmbx = b4.LoreMailbox()
    # Go through the mbox once to populate base series
    load_codereview = True
    for msg in msgs:
        # Is it a collection of patches attached to the same message?
        # We only trigger this mode with --single-msg
        if msg.is_multipart() and ('singlemsg' in cmdargs and cmdargs.singlemsg):
            xpatches: List[EmailMessage] = list()
            xmsgid = b4.LoreMessage.get_clean_msgid(msg)
            for part in msg.walk():
                cte = part.get_content_type()
                if cte.find('/x-patch') < 0:
                    continue
                bpayload = part.get_payload(decode=True)
                if not isinstance(bpayload, bytes):
                    continue
                pcharset = part.get_content_charset()
                if not pcharset:
                    pcharset = 'utf-8'
                try:
                    payload = bpayload.decode(pcharset, errors='replace')
                except LookupError:
                    # what kind of encoding is that?
                    # Whatever, we'll use utf-8 and hope for the best
                    payload = bpayload.decode('utf-8', errors='replace')
                    part.set_param('charset', 'utf-8')
                if payload and b4.DIFF_RE.search(payload):
                    xmsg = email.parser.Parser(
                        policy=b4.emlpolicy, _class=EmailMessage
                    ).parsestr(payload)
                    # Needs to have Subject, From, Date for us to consider it
                    if xmsg.get('Subject') and xmsg.get('From') and xmsg.get('Date'):
                        logger.debug('Found attached patch: %s', xmsg.get('Subject'))
                        xmsg['Message-ID'] = f'<att{len(xpatches)}-{xmsgid}>'
                        xpatches.append(xmsg)
            if len(xpatches):
                logger.info(
                    'Warning: Found %s patches attached to the requested message',
                    len(xpatches),
                )
                logger.info(
                    '         This mode ignores any follow-up trailers, use with caution'
                )
                # Throw out lmbx and only use these
                lmbx = b4.LoreMailbox()
                load_codereview = False
                for xmsg in xpatches:
                    lmbx.add_message(xmsg)
                # Make a cover letter out of the original message
                cmsg = EmailMessage()
                cbody, ccharset = b4.LoreMessage.get_payload(msg, use_patch=False)
                cmsg['From'] = msg.get('From')
                cmsg['Date'] = msg.get('Date')
                cmsg['Message-ID'] = msg.get('Message-ID')
                cmsg['Subject'] = '[PATCH 0/0] ' + msg.get('Subject', '(no subject)')
                cmsg.set_payload(cbody, ccharset)
                lmbx.add_message(cmsg)
                break
        else:
            lmbx.add_message(msg)

    reroll = True
    if cmdargs.nopartialreroll:
        reroll = False

    lser = lmbx.get_series(
        revision=wantver,
        sloppytrailers=cmdargs.sloppytrailers,
        reroll=reroll,
        codereview_trailers=load_codereview,
    )
    if lser is None and cmdargs.cherrypick != '_':
        if wantver is None:
            logger.critical('No patches found.')
        else:
            logger.critical('Unable to find revision %s', wantver)
        return
    if lser is None:
        logger.critical('No patches found.')
        return

    if len(lmbx.series) > 1 and not wantver:
        logger.info('Will use the latest revision: v%s', lser.revision)
        logger.info('You can pick other revisions using the -vN flag')

    if cmdargs.cherrypick:
        cherrypick = list()
        if cmdargs.cherrypick == '_':
            # We might want to pick a patch sent as a followup, so create a fake series
            # and add followups with diffs
            for followup in lmbx.followups:
                if followup.has_diff:
                    lser.add_patch(followup)
            # Only grab the exact msgid provided
            for at, lmsg in enumerate(lser.patches[1:], 1):
                if lmsg and lmsg.msgid == msgid:
                    cherrypick = [at]
                    cmdargs.cherrypick = f'<{msgid}>'
                    break
            if not len(cherrypick):
                logger.critical(
                    'Specified msgid is not present in the series, cannot cherrypick'
                )
                sys.exit(1)
        elif cmdargs.cherrypick.find('*') >= 0:
            # Globbing on subject
            at = 0
            for lmsg in lser.patches[1:]:
                at += 1
                if lmsg is None:
                    continue
                if fnmatch.fnmatch(lmsg.subject, cmdargs.cherrypick):
                    cherrypick.append(at)
            if not len(cherrypick):
                logger.critical(
                    'Could not match "%s" to any subjects in the series',
                    cmdargs.cherrypick,
                )
                sys.exit(1)
        else:
            cherrypick = list(
                b4.parse_int_range(cmdargs.cherrypick, upper=len(lser.patches) - 1)
            )
    else:
        cherrypick = None

    am_msgs = lser.get_am_ready(
        noaddtrailers=cmdargs.noaddtrailers,
        addmysob=cmdargs.addmysob,
        addlink=cmdargs.addlink,
        cherrypick=cherrypick,
        copyccs=cmdargs.copyccs,
        allowbadchars=cmdargs.allowbadchars,
        showchecks=cmdargs.check,
    )
    logger.info('---')

    if cherrypick is None:
        logger.critical('Total patches: %s', len(am_msgs))
    else:
        logger.info(
            'Total patches: %s (cherrypicked: %s)', len(am_msgs), cmdargs.cherrypick
        )

    if len(lser.trailer_mismatches):
        logger.critical('---')
        logger.critical('NOTE: some trailers ignored due to from/email mismatches:')
        for tname, tvalue, fname, femail in lser.trailer_mismatches:
            logger.critical('    ! Trailer: %s: %s', tname, tvalue)
            logger.critical('     Msg From: %s <%s>', fname, femail)
        logger.critical('NOTE: Rerun with -S to apply them anyway')

    top_msgid = None
    first_body = None
    for lmsg in lser.patches:
        if lmsg is not None:
            first_body = lmsg.body
            top_msgid = lmsg.msgid
            break
    if top_msgid is None or first_body is None:
        logger.critical('Could not find any patches in the series.')
        return

    topdir = b4.git_get_toplevel()

    if cmdargs.threeway:
        if not topdir:
            logger.critical('WARNING: cannot prepare 3-way (not in a git dir)')
        elif not lser.complete:
            logger.critical('WARNING: cannot prepare 3-way (series incomplete)')
        else:
            _checked, mismatches = lser.check_applies_clean(gitdir=topdir)
            if mismatches:
                rstart, rend = lser.make_fake_am_range(gitdir=None)
                if rstart and rend:
                    logger.info(
                        'Prepared fake commit range for 3-way merge (%.12s..%.12s)',
                        rstart,
                        rend,
                    )

    logger.critical('---')
    if lser.partial_reroll:
        logger.critical(
            'WARNING: v%s is a partial reroll from previous revisions', lser.revision
        )
        logger.critical(
            '         Please carefully review the resulting series to ensure correctness'
        )
        logger.critical('         Pass --no-partial-reroll to disable')
        logger.critical('---')
    if not lser.complete and not cmdargs.cherrypick:
        logger.critical('WARNING: Thread incomplete!')

    am_filename = None

    linkmask = str(config.get('linkmask', ''))
    if '%s' not in linkmask:
        logger.critical('ERROR: linkmask must contain %s for the message-id')
        sys.exit(1)

    if cmdargs.subcmd == 'am':
        wantname = cmdargs.wantname
        if cmdargs.maildir or config.get('save-maildirs', 'no') == 'yes':
            save_maildir = True
            dftext = 'maildir'
        else:
            save_maildir = False
            dftext = 'mbx'

        if wantname:
            slug = wantname
            if wantname.find('.') > -1:
                slug = '.'.join(wantname.split('.')[:-1])
        else:
            slug = lser.get_slug(extended=True)

        if outdir != '-':
            pathlib.Path(outdir).mkdir(parents=True, exist_ok=True)

            am_filename = os.path.join(outdir, f'{slug}.{dftext}')
            am_cover = os.path.join(outdir, f'{slug}.cover')

            if os.path.exists(am_filename):
                if os.path.isdir(am_filename):
                    shutil.rmtree(am_filename)
                else:
                    os.unlink(am_filename)
            if save_maildir:
                b4.save_maildir(am_msgs, am_filename)
            else:
                with open(am_filename, 'wb') as fh:
                    b4.save_git_am_mbox(am_msgs, fh)
        else:
            am_cover = None
            b4.save_git_am_mbox(am_msgs, sys.stdout.buffer)

        if am_cover and lser.has_cover and not cmdargs.nocover:
            lser.save_cover(am_cover)

        linkurl = linkmask % top_msgid
        if cmdargs.quiltready:
            q_dirname = os.path.join(outdir, f'{slug}.patches')
            save_as_quilt(am_msgs, q_dirname)
            logger.critical('Quilt: %s', q_dirname)

        logger.critical(' Link: %s', linkurl)

    if cmdargs.subcmd == 'shazam':
        if not topdir:
            logger.critical(
                'Could not figure out where your git dir is, cannot shazam.'
            )
            sys.exit(1)

        ifh = io.BytesIO()
        if lser.prereq_patch_ids:
            logger.info(
                ' Deps: looking for dependencies matching %s patch-ids',
                len(lser.prereq_patch_ids),
            )
            query = ' OR '.join([f'patchid:{x}' for x in lser.prereq_patch_ids])
            logger.debug('query=%s', query)
            dmsgs = b4.get_pi_search_results(query)
            pmap = dict()
            if dmsgs:
                for dmsg in dmsgs:
                    dbody, _ = b4.LoreMessage.get_payload(dmsg)
                    if not b4.DIFF_RE.search(dbody):
                        continue
                    dlmsg = b4.LoreMessage(dmsg)
                    if dlmsg.git_patch_id in lser.prereq_patch_ids:
                        logger.debug('%s => %s', dlmsg.git_patch_id, dlmsg.subject)
                        pmap[dlmsg.git_patch_id] = dlmsg
            for ppid in lser.prereq_patch_ids:
                if ppid in pmap:
                    logger.info(
                        ' Deps: Applying prerequisite patch: %s',
                        pmap[ppid].full_subject,
                    )
                    pam_msg = pmap[ppid].get_am_message(add_trailers=False)
                    b4.save_mboxrd_mbox([pam_msg], ifh)

        b4.save_git_am_mbox(am_msgs, ifh)
        ambytes = ifh.getvalue()
        if not cmdargs.makefetchhead:
            amflags = str(config.get('shazam-am-flags', ''))
            sp = shlex.shlex(amflags, posix=True)
            sp.whitespace_split = True
            amargs = list(sp) + ['--patch-format=mboxrd']
            ecode, out = b4.git_run_command(
                topdir, ['am'] + amargs, stdin=ambytes, logstderr=True, rundir=topdir
            )
            logger.info(out.strip())
            if ecode == 0:
                thanks_record_am(lser, cherrypick=cherrypick)
            sys.exit(ecode)

        base_commit = get_base_commit(topdir, first_body, lser, cmdargs)
        linkurl = linkmask % top_msgid

        merge_template = DEFAULT_MERGE_TEMPLATE
        if config.get('shazam-merge-template'):
            # Try to load this template instead
            try:
                merge_template = b4.read_template(str(config['shazam-merge-template']))
            except FileNotFoundError:
                logger.critical(
                    'ERROR: shazam-merge-template says to use %s, but it does not exist',
                    config['shazam-merge-template'],
                )
                sys.exit(2)

        if lser.has_cover and lser.patches[0] is not None:
            clmsg: b4.LoreMessage = lser.patches[0]
            parts = b4.LoreMessage.get_body_parts(clmsg.body)
            covermessage = parts[1]
        else:
            if lser.patches[1] is None:
                logger.critical(
                    'No cover letter provided by the author and no first patch, cannot shazam'
                )
                sys.exit(1)

            clmsg = lser.patches[1]
            covermessage = (
                'NOTE: No cover letter provided by the author.\n'
                '      Add merge commit message here.'
            )

        tptvals = {
            'seriestitle': clmsg.subject,
            'authorname': clmsg.fromname,
            'authoremail': clmsg.fromemail,
            'covermessage': covermessage,
            'mid': top_msgid,
            'midurl': linkurl,
        }
        if len(am_msgs) > 1:
            tptvals['patch_or_series'] = 'patch series'
        else:
            tptvals['patch_or_series'] = 'patch'

        mergeflags = str(config.get('shazam-merge-flags', '--signoff'))

        am_flags = ['-3']
        amflags_cfg = str(config.get('shazam-am-flags', ''))
        if amflags_cfg:
            sp = shlex.shlex(amflags_cfg, posix=True)
            sp.whitespace_split = True
            am_flags.extend(list(sp))

        try:
            if cmdargs.mergebase:
                logger.info(' Base: %s', base_commit)
            else:
                logger.info(' Base: %s (use --merge-base to override)', base_commit)
            b4.git_fetch_am_into_repo(
                topdir,
                ambytes=ambytes,
                at_base=base_commit,
                origin=linkurl,
                am_flags=am_flags,
                resolve=cmdargs.shazam_resolve,
            )
        except b4.AmConflictError as cex:
            gwt = cex.worktree_path
            if not getattr(cmdargs, 'shazam_resolve', False):
                b4.git_run_command(topdir, ['worktree', 'remove', '--force', gwt])
                logger.critical('Unable to cleanly apply series, see failure log below')
                logger.critical('---')
                logger.critical(cex.output)
                logger.critical('---')
                logger.critical('Not fetching into FETCH_HEAD')
                logger.critical('Use --resolve to enable conflict resolution')
                sys.exit(1)

            # --resolve into a dirty tree can't finish: the user would resolve the
            # whole series in the subshell only for the final git-merge to refuse.
            # Fail fast before any of that resolution work is done.
            status_lines = b4.git_get_repo_status(topdir)
            if status_lines:
                logger.critical('You have uncommitted changes in your working tree.')
                logger.critical('Please commit or stash them before resolving.')
                b4.git_run_command(topdir, ['worktree', 'remove', '--force', gwt])
                sys.exit(1)

            # --resolve: drop into a subshell in the worktree and finish the
            # git-am inline. On success FETCH_HEAD holds the fully-applied series
            # and the worktree is gone, so we fall through to the same merge the
            # clean path runs. b4 stays the parent until then -- no exec/exit
            # here -- so _run_shazam_merge can only execvp git-merge at the very
            # end, once the worktree has already been dropped.
            if not b4.resolve_am_conflict_in_shell(topdir, cex, origin=linkurl):
                sys.exit(1)
        except RuntimeError:
            sys.exit(1)

        thanks_record_am(lser, cherrypick=cherrypick)
        _run_shazam_merge(
            topdir,
            merge_template=merge_template,
            tptvals=tptvals,
            merge_flags=mergeflags,
            no_interactive=cmdargs.no_interactive,
            do_merge=cmdargs.merge,
        )

    if topdir:
        base_commit = get_base_commit(topdir, first_body, lser, cmdargs)
        checked, mismatched = lser.check_applies_clean(topdir, at=cmdargs.guessbranch)
        if checked and len(mismatched) == 0 and checked != len(mismatched):
            logger.critical(' Base: applies clean to current tree')
        elif base_commit != 'HEAD':
            gitbranch = lser.get_slug(extended=False)
            logger.critical('       git checkout -b %s %s', gitbranch, base_commit)

    if cmdargs.outdir != '-':
        logger.critical(
            '       git am %s%s', '-3 ' if cmdargs.threeway else '', am_filename
        )

    thanks_record_am(lser, cherrypick=cherrypick)


def thanks_record_am(lser: b4.LoreSeries, cherrypick: Optional[List[int]]) -> None:
    # Are we tracking this already?
    datadir = b4.get_data_dir()
    slug = lser.get_slug(extended=True)
    filename = '%s.am' % slug

    patches = list()
    msgids = list()
    at = 0
    padlen = len(str(lser.expected))
    lmsg = None

    for pmsg in lser.patches:
        if pmsg is None:
            at += 1
            continue

        if lmsg is None:
            lmsg = pmsg

        if not pmsg.has_diff:
            # Don't care about the cover letter
            at += 1
            continue

        if cherrypick is not None and at not in cherrypick:
            logger.debug('Skipped non-cherrypicked: %s', at)
            at += 1
            continue

        # Add it for patchwork state tracking
        msgids.append(pmsg.msgid)

        if pmsg.pwhash is None:
            logger.debug(
                'Unable to get hashes for all patches, not tracking for thanks'
            )
            return

        prefix = '%s/%s' % (str(pmsg.counter).zfill(padlen), pmsg.expected)
        patches.append((pmsg.subject, pmsg.pwhash, pmsg.msgid, prefix))
        at += 1

    if lmsg is None:
        logger.debug('All patches missing, not tracking for thanks')
        return

    try:
        allto = email.utils.getaddresses([str(x) for x in lmsg.msg.get_all('to', [])])
    except Exception as ex:
        allto = []
        logger.debug('Unable to parse the To: header in %s: %s', lmsg.msgid, str(ex))
    try:
        allcc = email.utils.getaddresses([str(x) for x in lmsg.msg.get_all('cc', [])])
    except Exception as ex:
        allcc = []
        logger.debug('Unable to parse the Cc: header in %s: %s', lmsg.msgid, str(ex))

    # TODO: check for reply-to and x-original-from
    out = {
        'msgid': lmsg.msgid,
        'subject': lmsg.full_subject,
        'fromname': lmsg.fromname,
        'fromemail': lmsg.fromemail,
        'to': b4.format_addrs(allto, clean=False),
        'cc': b4.format_addrs(allcc, clean=False),
        'references': b4.LoreMessage.clean_header(lmsg.msg['References']),
        'sentdate': b4.LoreMessage.clean_header(lmsg.msg['Date']),
        'quote': b4.make_quote(lmsg.body, maxlines=5),
        'cherrypick': cherrypick is not None,
        'patches': patches,
    }
    fullpath = os.path.join(datadir, filename)
    with open(fullpath, 'w', encoding='utf-8') as fh:
        json.dump(out, fh, ensure_ascii=False, indent=4)
        logger.debug('Wrote %s for thanks tracking', filename)

    config = b4.get_main_config()
    pwstate = str(config.get('pw-review-state', ''))
    if pwstate:
        b4.patchwork_set_state(msgids, pwstate)


def save_as_quilt(am_msgs: List[EmailMessage], q_dirname: str) -> None:
    if os.path.exists(q_dirname):
        logger.critical(
            'ERROR: Directory %s exists, not saving quilt patches', q_dirname
        )
        return
    pathlib.Path(q_dirname).mkdir(parents=True)
    patch_filenames = list()
    for msg in am_msgs:
        lsubj = b4.LoreSubject(msg.get('subject', ''))
        slug = '%04d_%s' % (
            lsubj.counter,
            re.sub(r'\W+', '_', lsubj.subject).strip('_').lower(),
        )
        patch_filename = f'{slug}.patch'
        patch_filenames.append(patch_filename)
        quilt_out = os.path.join(q_dirname, patch_filename)
        i, m, p = b4.get_mailinfo(msg.as_bytes(policy=b4.emlpolicy), scissors=True)
        with open(quilt_out, 'wb') as fh:
            if i.get('Author'):
                fh.write(
                    b'From: %s <%s>\n'
                    % (i.get('Author', '').encode(), i.get('Email', '').encode())
                )
            else:
                fh.write(b'From: %s\n' % i.get('Email', '').encode())
            fh.write(b'Subject: %s\n' % i.get('Subject', '').encode())
            fh.write(b'Date: %s\n' % i.get('Date', '').encode())
            fh.write(b'\n')
            fh.write(m)
            fh.write(p)
        logger.debug('  Wrote: %s', patch_filename)
    # Write the series file
    with open(os.path.join(q_dirname, 'series'), 'w') as sfh:
        for patch_filename in patch_filenames:
            sfh.write('%s\n' % patch_filename)


def get_extra_series(
    msgs: List[EmailMessage],
    direction: int = 1,
    wantvers: Optional[List[int]] = None,
    nocache: bool = False,
) -> List[EmailMessage]:
    base_msg: Optional[EmailMessage] = None
    latest_revision: Optional[int] = None
    seen_msgids: Set[str] = set()
    seen_covers: Set[int] = set()
    queries: Set[str] = set()
    change_ids: Set[str] = set()
    by_changeid: bool = False
    for msg in msgs:
        msgid = b4.LoreMessage.get_clean_msgid(msg)
        if msgid is None:
            continue
        seen_msgids.add(msgid)
        lsub = b4.LoreSubject(msg['Subject'])

        # Ignore patches above 1
        if lsub.counter > 1:
            continue

        if base_msg is not None:
            logger.debug('Current base_msg: %s', base_msg['Subject'])

        if lsub.reply:
            logger.debug('Ignoring reply: %s', lsub.full_subject)
            continue

        payload, _ = b4.LoreMessage.get_payload(msg)
        if payload:
            matches = re.search(r'^change-id:\s+(\S+)', payload, flags=re.I | re.M)
            if matches:
                cid = matches.groups()[0]
                logger.debug('Found change-id %s', cid)
                change_ids.add(cid)
                by_changeid = True
                q = 'nq:"change-id: %s"' % cid
                queries.add(q)

        logger.debug('Checking the subject on %s', lsub.full_subject)
        if latest_revision is None or lsub.revision >= latest_revision:
            latest_revision = lsub.revision
            if lsub.counter == 0 and not lsub.counters_inferred:
                # And a cover letter, nice. This is the easy case
                base_msg = msg
                seen_covers.add(latest_revision)
            elif lsub.counter == 1 and latest_revision not in seen_covers:
                # A patch/series without a cover letter
                base_msg = msg

    if base_msg is None or latest_revision is None:
        return msgs

    # Get subject info from base_msg again
    lsub = b4.LoreSubject(base_msg['Subject'])
    if not len(lsub.prefixes):
        logger.debug('Not checking for new revisions: no prefixes on the cover letter.')
        return msgs
    if direction < 0 and latest_revision <= 1:
        logger.debug('This is the earliest version of the series')
        return msgs
    if direction < 0 and wantvers is None:
        wantvers = [latest_revision - 1]

    msgdate = email.utils.parsedate_tz(str(base_msg['Date']))
    if msgdate is None:
        logger.debug('Unable to parse the date, not checking for revisions')
        return msgs

    if not by_changeid:
        logger.debug('Using subject+from combo for query')
        fromeml = email.utils.getaddresses(base_msg.get_all('from', []))[0][1]
        q = '(s:"%s" AND f:"%s")' % (lsub.subject.replace('"', ''), fromeml)
        queries.add(q)

    startdate = time.strftime('%Y%m%d', msgdate[:9])
    if direction > 0:
        logger.info('Checking for newer revisions')
        datelim = 'd:%s..' % startdate
    else:
        logger.info('Checking for older revisions')
        # Cap backward search to 12 months to avoid matching years of
        # identically-named series (common with subject+from fallback).
        earliest = time.strftime(
            '%Y%m%d', time.gmtime(time.mktime(msgdate[:9]) - 365 * 86400)
        )
        datelim = 'd:%s..%s' % (earliest, startdate)

    q = '(%s) AND %s' % (' OR '.join(queries), datelim)
    logger.debug('Query: %s', q)
    q_msgs = b4.get_pi_search_results(q, nocache=nocache)
    if not q_msgs:
        return msgs

    seen_revisions: Dict[int, int] = dict()
    candidates: List[EmailMessage] = list()
    for q_msg in q_msgs:
        q_msgid = b4.LoreMessage.get_clean_msgid(q_msg)
        if q_msgid is None:
            continue
        _subject = q_msg.get('Subject', '(no subject)')
        lsub = b4.LoreSubject(_subject)
        if q_msgid in seen_msgids:
            logger.debug('Skipping %s: already have it', lsub.full_subject)
            continue
        if lsub.reply:
            # These will get sorted out later
            logger.debug('Adding reply: %s', lsub.full_subject)
            logger.debug('              msgid: %s', q_msgid)
            candidates.append(q_msg)
            seen_msgids.add(q_msgid)
            continue

        if direction > 0 and lsub.revision <= latest_revision:
            logger.debug('Ignoring result (not new revision): %s', lsub.full_subject)
            continue
        if direction < 0 and lsub.revision >= latest_revision:
            logger.debug('Ignoring result (not old revision): %s', lsub.full_subject)
            continue
        if direction < 0 and wantvers and lsub.revision not in wantvers:
            logger.debug(
                'Ignoring result (not revision we want): %s', lsub.full_subject
            )
            continue

        if lsub.revision == 1 and lsub.revision == latest_revision:
            # Someone sent a separate message with an identical title but no new vX in the subject line
            if direction > 0:
                # It's *probably* a new revision.
                logger.debug('Likely a new revision: %s', lsub.full_subject)
            else:
                # It's *probably* an older revision.
                logger.debug('Likely an older revision: %s', lsub.full_subject)
        elif direction > 0 and lsub.revision > latest_revision:
            logger.debug(
                'Definitely a new revision [v%s]: %s', lsub.revision, lsub.full_subject
            )
        elif direction < 0 and lsub.revision < latest_revision:
            logger.debug(
                'Definitely an older revision [v%s]: %s',
                lsub.revision,
                lsub.full_subject,
            )
        else:
            logger.debug('No idea what this is: %s', lsub.subject)
            continue
        if lsub.revision not in seen_revisions:
            seen_revisions[lsub.revision] = 0
        seen_revisions[lsub.revision] += 1
        logger.debug('Adding: %s', lsub.full_subject)
        candidates.append(q_msg)
        seen_msgids.add(q_msgid)

    if by_changeid and change_ids:
        # Verify that candidate revisions actually belong to the same
        # series.  The nq: search can match messages that merely
        # reference our change-id as a prerequisite-change-id, so we
        # check that at least one message per revision contains a
        # standalone "change-id: <id>" line (not prerequisite-change-id).
        valid_revisions: Set[int] = set()
        for q_msg in candidates:
            payload, _ = b4.LoreMessage.get_payload(q_msg)
            if not payload:
                continue
            for cid in change_ids:
                if re.search(
                    rf'^change-id:\s*{re.escape(cid)}\s*$', payload, flags=re.I | re.M
                ):
                    lsub = b4.LoreSubject(q_msg.get('Subject', ''))
                    valid_revisions.add(lsub.revision)
                    break
        rejected: List[int] = list()
        validated: List[EmailMessage] = list()
        for q_msg in candidates:
            lsub = b4.LoreSubject(q_msg.get('Subject', ''))
            if lsub.reply or lsub.revision in valid_revisions:
                validated.append(q_msg)
            else:
                if lsub.revision not in rejected:
                    rejected.append(lsub.revision)
                if lsub.revision in seen_revisions:
                    seen_revisions[lsub.revision] -= 1
        candidates = validated
        for rev in rejected:
            logger.debug('Rejected v%s: change-id mismatch', rev)
        # Clean up empty revision counts
        seen_revisions = {r: c for r, c in seen_revisions.items() if c > 0}

    msgs.extend(candidates)

    for rev, count in seen_revisions.items():
        logger.info('  Added from v%s: %s patches', rev, count)

    return msgs


def refetch(dest: str) -> None:
    mbox: Union[mailbox.Maildir, mailbox.mbox]
    if b4.is_maildir(dest):
        mbox = mailbox.Maildir(dest)
    else:
        mbox = mailbox.mbox(dest)

    by_msgid: Dict[str, EmailMessage] = dict()
    for key, msg in mbox.items():
        msgid = b4.LoreMessage.get_clean_msgid(msg)  # type: ignore[arg-type] # ty:ignore[invalid-argument-type] # this will go away when we update liblore
        if not msgid:
            continue
        if msgid not in by_msgid:
            amsgs = b4.get_pi_thread_by_msgid(msgid, nocache=True)
            if amsgs:
                for amsg in amsgs:
                    amsgid = b4.LoreMessage.get_clean_msgid(amsg)
                    if not amsgid:
                        continue
                    if amsgid not in by_msgid:
                        by_msgid[amsgid] = amsg
        if msgid in by_msgid:
            mbox.update(((key, by_msgid[msgid]),))
            logger.info('Refetched: %s', msg.get('Subject'))
        else:
            logger.warning('WARNING: Message-id not known: %s', msgid)
    mbox.close()


def minimize_thread(msgs: List[EmailMessage]) -> List[EmailMessage]:
    # We go through each message and minimize headers and body content
    wanthdrs = {
        'From',
        'Subject',
        'Date',
        'Message-ID',
        'Reply-To',
        'In-Reply-To',
    }
    mmsgs = list()
    for msg in msgs:
        mmsg = EmailMessage()
        for wanthdr in wanthdrs:
            cleanhdr = b4.LoreMessage.clean_header(msg[wanthdr])
            if cleanhdr:
                mmsg[wanthdr] = cleanhdr

        body, _charset = b4.LoreMessage.get_payload(msg)
        if not (b4.DIFF_RE.search(body) or b4.DIFFSTAT_RE.search(body)):
            _htrs, cmsg, _mtrs, _basement, _sig = b4.LoreMessage.get_body_parts(body)
            # split the message into quoted and unquoted chunks
            chunks: List[Tuple[bool, List[str]]] = list()
            chunk: List[str] = list()
            current = None
            for line in cmsg.rstrip().splitlines():
                quoted = line.startswith('>') and True or False
                if current is None:
                    current = quoted
                if current == quoted:
                    if quoted and re.search(r'^>\s*>', line):
                        # trim multiple levels of quoting
                        continue
                    if quoted and not chunk and line.strip() == '>':
                        # Trim empty lines with just > in them
                        continue
                    chunk.append(line)
                    continue

                if current:
                    while len(chunk) and chunk[-1].strip() == '>':
                        chunk.pop(-1)
                if chunk:
                    chunks.append((quoted, chunk))
                chunk = list()
                if not (quoted and line.strip() == '>'):
                    chunk.append(line)
                current = quoted

            if current is None:
                current = False

            # Don't append bottom quotes
            if chunk and not current:
                chunks.append((current, chunk))

            body = ''
            for _quoted, chunk in chunks:
                # Should we offer a way to trim the quote in some fashion?
                body += '\n'.join(chunk).strip() + '\n\n'
            if not body.strip():
                continue

        mmsg.set_payload(body, charset='utf-8')
        # mmsg.set_charset('utf-8')
        mmsgs.append(mmsg)

    return mmsgs


def _run_shazam_merge(
    topdir: str,
    *,
    merge_template: str,
    tptvals: Dict[str, Any],
    merge_flags: str,
    no_interactive: bool,
    do_merge: bool,
) -> None:
    """Merge the series sitting in FETCH_HEAD into the current branch.

    Shared by the clean ``b4 shazam`` path and the ``--resolve`` conflict path:
    render the cover letter as the merge message and either run ``git merge`` (handing
    the terminal to git so it can open the editor and resolve any conflicts
    natively) or just point the user at FETCH_HEAD.
    """
    gitargs = ['rev-parse', '--git-dir']
    ecode, out = b4.git_run_command(topdir, gitargs, logstderr=True)
    if ecode > 0:
        logger.critical('Unable to find git directory')
        logger.critical(out.strip())
        sys.exit(ecode)
    mmf = os.path.join(out.rstrip(), 'b4-cover')

    # Write out a sample merge message using the cover letter
    if os.path.exists(mmf):
        # Make sure any old cover letters don't confuse anyone
        os.unlink(mmf)

    body = Template(merge_template).safe_substitute(tptvals)
    with open(mmf, 'w') as mmh:
        mmh.write(body)

    if not no_interactive and not sys.stdin.isatty():
        # E.g. the mbox arrived via "-m -", so there is nobody to answer the
        # confirmation prompt and no terminal for the editor
        logger.debug('stdin is not a tty, not asking for confirmation')
        no_interactive = True

    sp = shlex.shlex(merge_flags, posix=True)
    sp.whitespace_split = True
    if no_interactive:
        edit = '--no-edit'
    else:
        edit = '--edit'
    mergeargs = ['merge', '--no-ff', '-F', mmf, edit, 'FETCH_HEAD'] + list(sp)
    mergecmd = ['git'] + mergeargs

    if do_merge:
        if not no_interactive:
            logger.info('Will exec: %s', ' '.join(mergecmd))
            try:
                input('Press Enter to continue or Ctrl-C to abort')
            except (KeyboardInterrupt, EOFError):
                logger.info('')
                sys.exit(130)
        else:
            logger.info('Invoking: %s', ' '.join(mergecmd))
        if hasattr(sys, '_running_in_pytest'):
            # Don't execvp, as this kills our tests
            _out = b4.git_run_command(None, mergeargs)
            sys.exit(_out[0])

        # We exec git-merge and let it take over
        os.execvp(mergecmd[0], mergecmd)

    logger.info('You can now merge or checkout FETCH_HEAD')
    logger.info('  e.g.: %s', ' '.join(mergecmd))
    sys.exit(0)


def main(cmdargs: argparse.Namespace) -> None:
    # We force some settings
    if cmdargs.subcmd == 'shazam':
        cmdargs.checknewer = True
        cmdargs.threeway = False
        cmdargs.nopartialreroll = False
        cmdargs.outdir = '-'
        cmdargs.guessbranch = None
        if cmdargs.merge:
            cmdargs.makefetchhead = True
        if cmdargs.makefetchhead:
            cmdargs.guessbase = True
        else:
            cmdargs.guessbase = False
    else:
        cmdargs.mergebase = False

    if cmdargs.checknewer:
        # Force nocache mode
        cmdargs.nocache = True

    if cmdargs.subcmd == 'mbox' and cmdargs.refetch:
        return refetch(cmdargs.refetch)

    try:
        msgid, msgs = b4.retrieve_messages(cmdargs)
    except LookupError as ex:
        logger.critical('CRITICAL: %s', ex)
        sys.exit(1)

    if not msgs or not msgid:
        sys.exit(1)

    if len(msgs) and cmdargs.checknewer and b4.can_network:
        msgs = get_extra_series(msgs, direction=1, nocache=cmdargs.nocache)

    if cmdargs.subcmd in ('am', 'shazam'):
        make_am(msgs, cmdargs, msgid)
        return

    logger.info('%s messages in the thread', len(msgs))
    if cmdargs.subcmd == 'mbox' and cmdargs.minimize:
        msgs = minimize_thread(msgs)

    if cmdargs.outdir == '-':
        logger.info('---')
        save_msgs_as_mbox('-', msgs)
        return

    # Check if outdir is an existing maildir
    if b4.is_maildir(cmdargs.outdir):
        added = save_msgs_as_mbox(cmdargs.outdir, msgs, filterdupes=cmdargs.filterdupes)
        logger.info('Added %s messages to maildir %s', added, cmdargs.outdir)
        return

    config = b4.get_main_config()
    if cmdargs.maildir or config.get('save-maildirs', 'no') == 'yes':
        save_maildir = True
        dftext = 'maildir'
    else:
        save_maildir = False
        dftext = 'mbx'

    if cmdargs.wantname:
        savename = os.path.join(cmdargs.outdir, cmdargs.wantname)
    else:
        safe_msgid = re.sub(r'[^\w@.+%-]+', '_', msgid).strip('_')
        savename = os.path.join(cmdargs.outdir, f'{safe_msgid}.{dftext}')

    if save_maildir:
        if os.path.isdir(savename):
            shutil.rmtree(savename)
        mailbox.Maildir(savename, create=True)
        save_msgs_as_mbox(savename, msgs)
        logger.info('Saved maildir %s', savename)
        return

    save_msgs_as_mbox(savename, msgs)
    logger.info('Saved %s', savename)
