ez: initial implementation of prep --check

Building upon the work by Frank Li, implement "b4 prep --check" to run
checkpatch.pl against each patch in the series.

Link: https://msgid.link/20240319045332.2304950-1-Frank.Li@nxp.com
Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>

3 files changed, 123 insertions, 11 deletions

diff --git a/plan.otl b/plan.otl
index ce48078..3358189 100644
--- a/plan.otl
+++ b/plan.otl
@@ -2,15 +2,21 @@ v0.14
 -----
 [X] Switch to using pyproject.toml
 [X] Use hashed requirements for added security and reproducible installs
-[_] Automatic interdependency resolution
+[X] Automatic dependency resolution
 	[X] Retrieve dependencies using the standard prerequisite-patch-id
 	[X] Define the prerequisite-change-id trailer
 	[X] Expand prerequisite-change-id into prerequisite-patch-id for locally sent series
 	[X] Add b4 prep --edit-deps to open an editor with dependencies
 	[X] Add b4 prep --check-deps to report if there are problems or updates available
 	[X] Expand non-local change-id and message-id deps into prerequisite-patch-id
-	[_] --check-deps should check if everything can be cleanly applied
-	[_] Dependency chaining (within reason)
+	[X] --check-deps should check if everything can be cleanly applied
+[_] Checkpatch and other pre-submit checks
+	[X] Configurable checks to run on each patch
+	[X] When checks are not defined, use Linux kernel defaults
+	[X] Display checkpatch checks using output similar to CI checks
+	[X] Cache checks for commits that haven't changed if the check command is the same
+	[_] Run b4-specific checks automatically (needs-editing, needs-auto-to-cc)
+	[_] Refuse to send if checks haven't been run
 
 v0.15
 -----
diff --git a/src/b4/command.py b/src/b4/command.py
index 5dc0184..d06e194 100644
--- a/src/b4/command.py
+++ b/src/b4/command.py
@@ -306,6 +306,8 @@ def setup_parser() -> argparse.ArgumentParser:
                        help='Edit the series dependencies in your defined $EDITOR (or core.editor)')
     spp_g.add_argument('--check-deps', action='store_true', default=False,
                        help='Run checks for any defined series dependencies')
+    spp_g.add_argument('--check', action='store_true', default=False,
+                       help='Run checks on the series')
     spp_g.add_argument('--show-revision', action='store_true', default=False,
                        help='Show current series revision number')
     spp_g.add_argument('--compare-to', metavar='vN',
diff --git a/src/b4/ez.py b/src/b4/ez.py
index 7055cdd..2d9b553 100644
--- a/src/b4/ez.py
+++ b/src/b4/ez.py
@@ -811,6 +811,10 @@ def edit_deps() -> None:
 def check_deps(cmdargs: argparse.Namespace) -> None:
     cover, tracking = load_cover()
     prereqs = tracking['series'].get('prerequisites', list())
+    if not prereqs:
+        logger.info('This series has no defined dependencies.')
+        logger.info('To add dependencies, use --edit-deps.')
+        return
     res = dict()
     prereq_patches = list()
     known_patches = dict()
@@ -1172,6 +1176,18 @@ def get_addresses_from_cmd(cmdargs: List[str], msgbytes: bytes) -> List[Tuple[st
     return utils.getaddresses(addrs.split('\n'))
 
 
+def get_check_results_from_cmd(cmdargs: List[str], msgbytes: bytes) -> List[str]:
+    if not cmdargs:
+        return list()
+    # Run this command from git toplevel
+    topdir = b4.git_get_toplevel()
+    ecode, out, err = b4._run_command(cmdargs, stdin=msgbytes, rundir=topdir)  # noqa
+    check_report = out.strip().decode(errors='ignore')
+    if not check_report:
+        return list()
+    return check_report.split('\n')
+
+
 def get_series_details(start_commit: Optional[str] = None, usebranch: Optional[str] = None
                        ) -> Tuple[str, str, str, List[str], str, str]:
     if usebranch:
@@ -1369,7 +1385,8 @@ def get_mailfrom() -> Tuple[str, str]:
 
 
 def get_prep_branch_as_patches(movefrom: bool = True, thread: bool = True, addtracking: bool = True,
-                               prefixes: Optional[List[str]] = None, usebranch: Optional[str] = None
+                               prefixes: Optional[List[str]] = None, usebranch: Optional[str] = None,
+                               expandprereqs: bool = True,
                                ) -> Tuple[List, List, str, List[Tuple[str, email.message.Message]]]:
     cover, tracking = load_cover(strip_comments=True, usebranch=usebranch)
 
@@ -1438,12 +1455,13 @@ def get_prep_branch_as_patches(movefrom: bool = True, thread: bool = True, addtr
         spatches = list()
         if prereq.startswith('message-id'):
             prerequisites += f'prerequisite-{prereq}\n'
-            msgid = chunks[1].strip('<>')
-            spatches = b4.get_pi_thread_by_msgid(msgid)
-            if not spatches:
-                logger.info('Nothing known about message-id: %s', msgid)
-                logger.info('Consider running --check-deps')
-                continue
+            if expandprereqs:
+                msgid = chunks[1].strip('<>')
+                spatches = b4.get_pi_thread_by_msgid(msgid)
+                if not spatches:
+                    logger.info('Nothing known about message-id: %s', msgid)
+                    logger.info('Consider running --check-deps')
+                    continue
 
         if prereq.startswith('change-id:'):
             prerequisites += f'prerequisite-{prereq}\n'
@@ -1451,7 +1469,7 @@ def get_prep_branch_as_patches(movefrom: bool = True, thread: bool = True, addtr
             if len(chunks) > 1:
                 pcid = chunks[1]
             pver = chunks[-1]
-            if pcid and pver:
+            if expandprereqs and pcid and pver:
                 tagname, revision = get_sent_tagname(pcid, SENT_TAG_PREFIX, pver)
                 logger.debug('Checking if we have a sent version')
                 try:
@@ -1590,6 +1608,89 @@ def format_patch(output_dir: str) -> None:
             logger.info('  %s', filen)
 
 
+def check(cmdargs: argparse.Namespace) -> None:
+    # Use recommended checkpatch defaults if we find checkpatch
+    topdir = b4.git_get_toplevel()
+    checkpatch = os.path.join(topdir, 'scripts', 'checkpatch.pl')
+    config = b4.get_main_config()
+    ppcmdstr = None
+    if config.get('prep-perpatch-check-cmd'):
+        ppcmdstr = config.get('prep-perpatch-check-cmd')
+    elif os.access(checkpatch, os.X_OK):
+        ppcmdstr = f'{checkpatch} -q --terse --no-summary --mailback --showfile'
+    ppcmd = list()
+    if ppcmdstr:
+        sp = shlex.shlex(ppcmdstr, posix=True)
+        sp.whitespace_split = True
+        ppcmd = list(sp)
+        logger.info('Will run per-patch checks using %s', os.path.basename(ppcmd[0]))
+    # TODO: support for a whole-series check command, (pytest, etc)
+
+    try:
+        todests, ccdests, tag_msg, patches = get_prep_branch_as_patches(expandprereqs=False)
+    except RuntimeError as ex:
+        logger.critical('CRITICAL: Failed to convert range to patches: %s', ex)
+        sys.exit(1)
+
+    cover, tracking = load_cover()
+    if cmdargs.nocache:
+        checks_cache = dict()
+    else:
+        cached = b4.get_cache(tracking['series']['change-id'], suffix='checks')
+        checks_cache = json.loads(cached) if cached else dict()
+        if checks_cache.get('prep-perpatch-check-cmd', '') != ppcmdstr:
+            logger.debug('Ignoring cached checks info because the check command has changed')
+            b4.clear_cache(tracking['series']['change-id'], suffix='checks')
+            checks_cache = dict()
+    checks_cache['prep-perpatch-check-cmd'] = ppcmdstr
+
+    logger.info('---')
+    seen = set()
+    summary = {
+        'warning': 0,
+        'fail': 0,
+        'success': 0
+    }
+    for commit, msg in patches:
+        if not msg or not commit:
+            continue
+        seen.add(commit)
+        lsubject = b4.LoreSubject(msg.get('Subject', ''))
+        csubject = f'{commit[:12]}: {lsubject.subject}'
+        if 'commits' not in checks_cache:
+            checks_cache['commits'] = dict()
+        if commit not in checks_cache['commits']:
+            logger.debug('Checking: %s', lsubject.subject)
+            bdata = b4.LoreMessage.get_msg_as_bytes(msg)
+            report = get_check_results_from_cmd(ppcmd, bdata)
+            checks_cache['commits'][commit] = report
+        else:
+            logger.debug('Using cached checks for %s', commit)
+            report = checks_cache['commits'][commit]
+        if not report:
+            logger.info('%s %s', b4.CI_FLAGS_FANCY['success'], csubject)
+            summary['success'] += 1
+            continue
+        worst = 'warning'
+        for line in report:
+            if 'ERROR:' in line:
+                worst = 'fail'
+                break
+        logger.info('%s %s', b4.CI_FLAGS_FANCY[worst], csubject)
+        for line in report:
+            flag = 'fail' if 'ERROR:' in line else 'warning'
+            summary[flag] += 1
+            logger.info('  %s %s', b4.CI_FLAGS_FANCY[flag], line)
+    # Throw out any stale checks
+    for commit in list(checks_cache['commits'].keys()):
+        if commit not in seen:
+            logger.debug('Removing stale check cache for non-existent commit %s', commit)
+            del(checks_cache['commits'][commit])
+    b4.save_cache(json.dumps(checks_cache), tracking['series']['change-id'], suffix='checks')
+    logger.info('---')
+    logger.info('Success: %s, Warning: %s, Error: %s', summary['success'], summary['warning'], summary['fail'])
+
+
 def cmd_send(cmdargs: argparse.Namespace) -> None:
     if cmdargs.auth_new:
         auth_new()
@@ -2560,6 +2661,9 @@ def cmd_prep(cmdargs: argparse.Namespace) -> None:
     if cmdargs.check_deps:
         return check_deps(cmdargs)
 
+    if cmdargs.check:
+        return check(cmdargs)
+
 
 def cmd_trailers(cmdargs: argparse.Namespace) -> None:
     check_can_gfr()