git2bz: initial implementation for trackig git commits

We should now be able to track commits for mentions of our bugs. This is
a rough initial implementation and will probably be enhanced.

Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>

4 files changed, 189 insertions, 1 deletions

diff --git a/default.config.toml b/default.config.toml
index f432095..8828990 100644
--- a/default.config.toml
+++ b/default.config.toml
@@ -39,6 +39,12 @@ pi_assign_regexes = ['^bugbot assign to (\S+)']
 bz_new_bugs_quicksearch = 'OPEN flag:bugbot+'
 bz_privacy_mode = true
 alwayscc = ['bugs@lists.linux.dev']
+# watched git repositories
+git_repos = [['/path/to/repo', 'master', 'https://web.url/foo/id=%.12s']]
+git_log_queries = ['bugzilla.kernel.org']
+git_log_mentions_regexes = ['https://bugzilla.kernel.org/.*id=(\d+)']
+git_log_closes_regexes = ['^Closes:\s+https://bugzilla.kernel.org/.*id=(\d+)']
+git_closes_with = ['RESOLVED', 'CODE_FIX']
 
 [templates]
 parse_bug_intro = '${author} writes:'
@@ -77,6 +83,14 @@ ${summary}
 You can reply to this message to join the discussion.
 '''
 
+new_commit_notify = '''
+${commit_author} writes in commit ${commit_id}:
+
+${commit_text}
+
+(via ${commit_url})
+'''
+
 botsig = '''
 -- 
 Deet-doot-dot, I am a bot.
diff --git a/peebz/__init__.py b/peebz/__init__.py
index e3ac371..9e2da76 100644
--- a/peebz/__init__.py
+++ b/peebz/__init__.py
@@ -140,6 +140,15 @@ def bz_add_atts_to_bug(bid: int, atts: List[Dict]) -> List[int]:
     return aids
 
 
+def bz_set_bug_status_resolution(bid: int, status: str, resolution: str) -> None:
+    payload = {
+        'status': status,
+        'resolution': resolution,
+    }
+    logger.debug('Setting bug_id=%s status=%s resolution=%s', bid, status, resolution)
+    bz_rest(f'bug/{bid}', payload, method='PUT')
+
+
 def bz_add_new_bug(payload: Dict) -> Tuple[int, int]:
     if 'version' not in payload:
         payload['version'] = 'unspecified'
@@ -325,6 +334,29 @@ def db_get_recipients(bid: int) -> Set[str]:
     return set(x[0] for x in fa)
 
 
+def db_get_bugs_for_commit(csha: str) -> Set[int]:
+    engine, dbconn = db_get_sa()
+    md = sa.MetaData()
+    t_cmap = sa.Table('commit_bug_mapping', md, autoload=True, autoload_with=engine)
+    logger.debug('Querying for commit_id=%s', csha)
+    q = sa.select([t_cmap.c.bug_id]).where(t_cmap.c.commit_id == csha)
+    rp = dbconn.execute(q)
+    fa = rp.fetchall()
+    if not len(fa):
+        return set()
+
+    return set(x[0] for x in fa)
+
+
+def db_store_bug_for_commit(csha: str, bid: int) -> None:
+    engine, dbconn = db_get_sa()
+    md = sa.MetaData()
+    t_cmap = sa.Table('commit_bug_mapping', md, autoload=True, autoload_with=engine)
+    logger.debug('Storing bug_id=%s for commit_id=%s', bid, csha)
+    q = sa.insert(t_cmap).values(bug_id=bid, commit_id=csha)
+    dbconn.execute(q)
+
+
 def db_store_recipients(bid: int, recipients: Set[str]) -> None:
     # TODO: add ability to unsubscribe?
     try:
@@ -344,7 +376,6 @@ def db_store_recipients(bid: int, recipients: Set[str]) -> None:
         q = sa.insert(t_recip).values(bug_id=bid, email=addr)
         dbconn.execute(q)
         logger.debug(' Added %s', addr)
-    return
 
 
 def db_get_meta_value(key: str) -> str:
@@ -922,6 +953,13 @@ def db_init_sa_sqlite_db(engine: sa.engine.Engine, dbconn: sa.engine.Connection)
     sa.Index('idx_msgid_commentid', bmap.c.message_id, bmap.c.comment_id, unique=True)
     sa.Index('idx_msgid_bugid', bmap.c.message_id, bmap.c.bug_id, unique=True)
 
+    cmap = sa.Table('commit_bug_mapping', md,
+                    sa.Column('row_id', sa.Integer(), primary_key=True),
+                    sa.Column('bug_id', sa.Integer(), nullable=False),
+                    sa.Column('commit_id', sa.Text(), nullable=False),
+                    )
+    # a commit can close multiple bugs
+    sa.Index('idx_commitid_bugid', cmap.c.commit_id, cmap.c.bug_id, unique=False)
     recip = sa.Table('recipients', md,
                      sa.Column('row_id', sa.Integer(), primary_key=True),
                      sa.Column('bug_id', sa.Integer(), nullable=False),
diff --git a/peebz/command.py b/peebz/command.py
index a2ebc7d..93abb8a 100644
--- a/peebz/command.py
+++ b/peebz/command.py
@@ -29,6 +29,11 @@ def cmd_pi2bz(cmdargs):
     peebz.pi2bz.main(cmdargs)
 
 
+def cmd_git2bz(cmdargs):
+    import peebz.git2bz
+    peebz.git2bz.main(cmdargs)
+
+
 def cmd_bzdump(cmdargs):
     import json
     from pygments import highlight, lexers, formatters
@@ -78,6 +83,12 @@ def setup_parser() -> argparse.ArgumentParser:
     sp_pi2bz.add_argument('--component', help='Only run queries for this component')
     sp_pi2bz.set_defaults(func=cmd_pi2bz)
 
+    # git2bz : query git repositories to find commits that interest us
+    sp_git2bz = subparsers.add_parser('git2bz', help='Query git repos for bug mentioning commits')
+    sp_git2bz.add_argument('--product', help='Only run queries for this product')
+    sp_git2bz.add_argument('--component', help='Only run queries for this component')
+    sp_git2bz.set_defaults(func=cmd_git2bz)
+
     # bz2pi: query bugzilla and sends out any mail updates
     sp_bz2pi = subparsers.add_parser('bz2pi', help='Send emails about bugzilla-originated changes')
     sp_bz2pi.set_defaults(func=cmd_bz2pi)
diff --git a/peebz/git2bz.py b/peebz/git2bz.py
new file mode 100644
index 0000000..935a19b
--- /dev/null
+++ b/peebz/git2bz.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (C) 2023 by the Linux Foundation
+
+import argparse
+import peebz
+import peebz.parse
+import b4
+import email.message
+import re
+
+logger = peebz.logger
+b4.logger = logger
+# force b4 to use EmailMessage factory
+b4.emlpolicy = peebz.emlpolicy
+
+
+def update_component(product: str, component: str, dry_run: bool = False):
+    logger.info('Running git2bz for %s/%s, dry_run=%s', product, component, dry_run)
+    cconf = peebz.get_component_config(product, component)
+    cres = cconf.get('git_log_closes_regexes', list())
+    mres = cconf.get('git_log_mentions_regexes', list())
+    since = cconf.get('git_log_since', '1.week')
+    tpt = peebz.get_template_by_product_component('new_commit_notify', product, component)
+    for entry in cconf.get('git_repos', list()):
+        repo = entry[0]
+        try:
+            branch = entry[1]
+        except IndexError:
+            branch = 'master'
+        try:
+            commit_mask = entry[2]
+        except IndexError:
+            commit_mask = None
+
+        logger.debug('Looking at %s:%s', repo, branch)
+        for query in cconf.get('git_log_queries', list()):
+            logger.debug(' query=%s', query)
+            gitargs = ['log', '--grep', query, '-F', '--pretty=oneline', f'--since={since}', branch]
+            lines = b4.git_get_command_lines(repo, gitargs)
+            if not lines:
+                logger.debug('No matches for %s', query)
+                continue
+            logger.debug(' query returned %s commits', len(lines))
+            for line in lines:
+                csha = line.split(maxsplit=1)[0]
+                known_bids = peebz.db_get_bugs_for_commit(csha)
+                if len(known_bids):
+                    logger.debug('This commit already processed, bugs: %s', known_bids)
+                    continue
+
+                bid = None
+                gitargs = ['show', '-s', '--format=%an---%B', csha]
+                ecode, cshow = b4.git_run_command(repo, gitargs)
+                if ecode > 0:
+                    logger.debug('Could not get commit body from %s', csha)
+                    continue
+                cauthor, cmsg = cshow.split('---', maxsplit=1)
+
+                # Check closes regexes
+                closes = False
+                for cre in cres:
+                    matches = re.search(cre, cmsg, flags=re.I | re.M)
+                    if matches:
+                        bid = int(matches.groups()[0])
+                        logger.debug(' closes bug %s', bid)
+                        closes = True
+                        break
+
+                if not closes:
+                    found = False
+                    # Check mentions regexes
+                    for mre in mres:
+                        matches = re.search(mre, cmsg, flags=re.I | re.M)
+                        if matches:
+                            bid = int(matches.groups()[0])
+                            logger.debug(' mentions bug %s', bid)
+                            found = True
+                            break
+                    if not found:
+                        logger.debug('No regexes matched')
+                        continue
+
+                if not bid:
+                    logger.debug('Could not get bug_id from comment %s', csha)
+                    continue
+
+                vals = {
+                    'commit_author': cauthor.strip(),
+                    'commit_id': csha,
+                    'commit_text': cmsg.strip(),
+                    'commit_url': '',
+                }
+                if commit_mask:
+                    vals['commit_url'] = commit_mask % csha
+
+                desc = tpt.safe_substitute(vals)
+                msg = email.message.EmailMessage()
+                body = peebz.add_bot_signature(desc)
+                msg.set_payload(body, charset='utf-8')
+                if not dry_run:
+                    cid = peebz.bz_add_new_comment(bid, desc)
+                    msgid = peebz.notify_bug(bid, cid, msg, dry_run=dry_run)
+                    if msgid:
+                        peebz.db_store_msgid_bid_cid(msgid, bid, cid)
+                    peebz.db_store_bug_for_commit(csha, bid)
+                    if closes:
+                        status, resolution = cconf.get('git_closes_with', ['RESOLVED', 'FIXED'])
+                        logger.info('Commit %s closes bug %s with %s/%s', csha, bid, status, resolution)
+                        peebz.bz_set_bug_status_resolution(bid, status, resolution)
+                    else:
+                        logger.info('Commit %s mentions bug %s', csha, bid)
+                else:
+                    peebz.notify_bug(bid, None, msg, dry_run=dry_run)
+
+
+def main(cmdargs: argparse.Namespace):
+    config = peebz.get_config()
+    # Iterate all components
+    for bz_product, bz_components in config['components'].items():
+        for bz_component in bz_components.keys():
+            if config['components'][bz_product][bz_component].get('git_repos') is None:
+                continue
+            update_component(bz_product, bz_component, dry_run=cmdargs.dry_run)