diff options
author | Thomas Gleixner <tglx@linutronix.de> | 2020-11-17 22:37:44 +0100 |
---|---|---|
committer | Thomas Gleixner <tglx@linutronix.de> | 2020-11-17 22:37:44 +0100 |
commit | 6dd4692940317698e49ce0e2622212670ea76b49 (patch) | |
tree | 969bace680fb754b896edc8488edd25c4cfebf20 | |
download | tip-bot-master.tar.gz |
The tip-bot machinery which spams^Winforms about patches which have been
merged into the tip tree.
Lacks documentation, but you know how to find me.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
-rw-r--r-- | __init__.py | 0 | ||||
-rw-r--r-- | setup.py | 19 | ||||
-rwxr-xr-x | tip-bot2-daemon | 61 | ||||
-rw-r--r-- | tip-bot2.service | 11 | ||||
-rw-r--r-- | tipbot.yaml | 34 | ||||
-rw-r--r-- | tipbot/__init__.py | 0 | ||||
-rw-r--r-- | tipbot/daemon.py | 172 | ||||
-rw-r--r-- | tipbot/git.py | 92 | ||||
-rw-r--r-- | tipbot/mail.py | 266 | ||||
-rw-r--r-- | tipbot/util.py | 50 | ||||
-rw-r--r-- | tipbot/version.py | 5 |
11 files changed, 710 insertions, 0 deletions
diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/__init__.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d625b73 --- /dev/null +++ b/setup.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner <tglx@linutronix.de> + +from glob import glob +from distutils.core import setup + +from tipbot.version import __version__ + +setup(name='tipbot', + version=__version__, + description='tipbot', + author='Thomas Gleixner', + author_email='tglx@linutronix.de', + packages=['tipbot'], + scripts=['tipbot_daemon'] + ) +data_files = [ + ('/lib/systemd/system', glob("tipbot.service"))], diff --git a/tip-bot2-daemon b/tip-bot2-daemon new file mode 100755 index 0000000..d40b650 --- /dev/null +++ b/tip-bot2-daemon @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL2.0 +# Copyright Thomas Gleixner <tglx@linutronix.de> + +from argparse import ArgumentParser +from tipbot.daemon import tipbot +from tipbot.util import logger +import yaml +import sys +import os + +if __name__ == '__main__': + + parser = ArgumentParser(description='TIP commit mail bot') + parser.add_argument('-l', '--linusdir', metavar='linusdir', + default='../linus', + help='linus tree directory') + parser.add_argument('-t', '--tipdir', metavar='tipdir', + default='../tip', + help='tip tree directory') + parser.add_argument('-T', '--test', dest='test', + action='store_true', help='Send mail to self') + parser.add_argument('-m', '--mbox', dest='mbox', type=str, + default=None, help='output to mbox') + parser.add_argument('-S', '--smtp', dest='smtp', action='store_true', + help='output to smtp (localhost)') + parser.add_argument('-f', '--forcelinus', dest='forcelinus', action='store_true', + help='force update of linus tree') + parser.add_argument('-L', '--limit', dest='limit', type=int, + default=100, + help='Limit the amount of mail to send in one go') + parser.add_argument('-p', '--pause', dest='pause', type=int, + default=5, + help='Pause between checks in minutes') + parser.add_argument('-k', '--known_commits', dest='known_commits', + default='known_commits', + help='Directory to store known commits files') + parser.add_argument('-s', '--syslog', dest='syslog', action='store_true', + help='Use syslog for logging') + parser.add_argument('-c', '--config', dest='config', + default='/home/tipbot/tipbot.yaml', help='Config file') + parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', + help='Verbose logging') + args = parser.parse_args() + + logger = logger(use_syslog=args.syslog, verbose = True) + + try: + cfg = yaml.load(open(args.config)) + for k, val in cfg.items(): + vars(args)[k] = val + logger.verbose = args.verbose + logger.use_syslog = args.syslog + os.chdir(args.workdir) + bot = tipbot(args, logger) + res = bot.run() + except Exception as ex: + logger.log_exception(ex, 'Unhandled exception in main') + res = 1 + + sys.exit(res) diff --git a/tip-bot2.service b/tip-bot2.service new file mode 100644 index 0000000..3aaa5f4 --- /dev/null +++ b/tip-bot2.service @@ -0,0 +1,11 @@ +[Unit] +Description=tip-commit-bot +After=syslog.target network.target +ConditionPathExists=/home/tipbot/tipbot.yaml + +[Service] +Type=simple +User=tipbot +ExecStart=/home/tipbot/tipbot/code/tip-bot2-daemon +[Install] +WantedBy=default.target diff --git a/tipbot.yaml b/tipbot.yaml new file mode 100644 index 0000000..63c0d8a --- /dev/null +++ b/tipbot.yaml @@ -0,0 +1,34 @@ +# tipbot configuration + +workdir: /home/tipbot/tipbot/data + +linusdir: /home/tipbot/tipbot/linus + +tipdir: /home/tipbot/tipbot/tip + +pause: 5 + +limit: 100 + +smtp: True + +syslog: True + +verbose: True + +mbox: sentmbox + +known_commits: known_commits + +test: True + +forcelinus: True + +forceccs: + -x86@kernel.org + -linux-kernel@vger.kernel.org + +optccs: + maz@kernel.org: + - irq/core + - irq/urgent diff --git a/tipbot/__init__.py b/tipbot/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tipbot/__init__.py diff --git a/tipbot/daemon.py b/tipbot/daemon.py new file mode 100644 index 0000000..a8906b6 --- /dev/null +++ b/tipbot/daemon.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL2.0 +# Copyright Thomas Gleixner <tglx@linutronix.de> + +from tipbot.git import git_repo +from tipbot.util import FatalException, FetchException +from tipbot.mail import mailer +import subprocess +import tempfile +import signal +import time +import os + +class tipbot(object): + def __init__(self, args, logger): + self.args = args + self.log = logger + self._should_stop = False + self._should_reload = False + self.siginstall() + + self.linus = git_repo(args.linusdir, logger) + self.tip = git_repo(args.tipdir, logger) + + self.mailer = mailer(args, logger) + + def term_handler(self, signum, frame): + self.log.log_debug('term_handler received SIG %d\n' % signum) + self._should_stop = True + + def reload_handler(self, signum, frame): + self.log.log_debug('reload_handler received SIG %d\n' % signum) + self._should_reload = True + + def siginstall(self): + signal.signal(signal.SIGINT, self.term_handler) + signal.signal(signal.SIGTERM, self.term_handler) + signal.signal(signal.SIGHUP, self.reload_handler) + + def should_stop(self): + return self._should_stop + + def known_commit(self, sha): + fname = sha[:2] + '-known_commits' + fname = os.path.join(self.args.known_commits, fname) + if not os.path.isfile(fname): + return False + for l in open(fname).readlines(): + if l.find(sha) == 0: + return True + return False + + def write_known_commit(self, sha): + fname = sha[:2] + '-known_commits' + fname = os.path.join(self.args.known_commits, fname) + with open(fname, 'a') as fd: + fd.write('%s\n' %sha) + + def write_known_commits(self, shas): + for sha in shas: + if not self.known_commit(sha): + self.write_known_commit(sha) + + def update_known_commits(self, shas, repomsg, repo=None): + msg = 'Update known commits from %s\n\n' %repomsg + + if repo: + for sha in shas: + msg += '%s\n' %repo.shortlog(sha) + + tf = tempfile.NamedTemporaryFile(delete=False) + tf.write(msg.encode('UTF-8')) + tf.close() + + args = [ 'git', 'commit', '-a', '-q', '-F', '%s' %tf.name ] + + res = subprocess.run(args) + try: + res.check_returncode() + os.unlink(tf.name) + except Exception as ex: + self.log.log_exception(ex, 'Commit known_commits failed\n') + os.unlink(tf.name) + raise FatalException + + args = [ 'git', 'push', '-q' ] + res = subprocess.run(args) + try: + res.check_returncode() + except Exception as ex: + self.log.log_exception(ex, 'Commit push failed\n') + raise FetchException + + def update_linus(self): + if self.args.forcelinus: + old_head = '0ecfebd2b52404ae0c54a878c872bb93363ada36' + self.args.forcelinus = False + else: + old_head = self.linus.get_branch_head() + self.linus.fetch() + new_head = self.linus.get_branch_head() + + self.linus_head = new_head + if old_head == new_head: + self.log.log_debug("Linus head unchanged %s\n" %old_head) + return + + self.log.log_debug("Linus head updated %s\n" %new_head) + shas = [] + for sha in self.linus.log_revs_from(old_head): + if not self.known_commit(sha): + shas.append(sha) + if len(shas): + self.write_known_commits(shas) + self.update_known_commits(shas, 'Linus tree') + + def update_tip(self): + self.tip.fetch() + self.tip.set_base_ref('refs/heads/linus-base', self.linus_head) + path = '.tip/auto-branches/auto-latest' + shas = [] + for ref in self.tip.get_autobranch_refs('tip', path): + for entry in self.tip.log_revs_ref_from('linus-base', ref): + if not self.known_commit(entry) and not entry in shas: + if len(shas) >= self.args.limit: + break + branch = ref.replace('/refs/heads/', '') + branch = ref.replace('refs/heads/', '') + self.mailer.send_mail(self.tip, branch, entry) + shas.append(entry) + self.write_known_commits([entry]) + + if len(shas): + self.update_known_commits(shas, 'tip', self.tip) + self.log.log_debug("Tip updated %d notifications\n" %len(shas)) + else: + self.log.log_debug("Tip unchanged\n") + + if len(shas) >= self.args.limit: + raise Exception('Mail limit %d reached' %self.args.limit) + + def run(self): + res = 0 + while not self.should_stop(): + try: + self.update_linus() + self.update_tip() + + except FetchException as ex: + # Don't try to be smart for now except for temporary + # network and name resolution failures + if (str(ex).find('Network is unreachable') < 0 and + str(ex).find('early EOF') < 0 and + str(ex).find('Temporary failure in name resolution') < 0): + res = 1 + break + + except FatalException as ex: + res = 1 + break + + except Exception as ex: + self.log.log_exception(ex) + res = 1 + break + + i = 0 + while i < (self.args.pause * 60) and not self.should_stop(): + i += 1 + time.sleep(1) + + return res diff --git a/tipbot/git.py b/tipbot/git.py new file mode 100644 index 0000000..0bb0d6b --- /dev/null +++ b/tipbot/git.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL2.0 +# Copyright Thomas Gleixner <tglx@linutronix.de> + +from tipbot.util import FatalException, FetchException +import subprocess +import pygit2 +import os + +class git_repo(object): + def __init__(self, repodir, logger): + self.repodir = os.path.abspath(repodir) + self.log = logger + self.repo = pygit2.Repository(self.repodir) + + def fetch(self, remote='origin'): + try: + self.repo.remotes[remote].fetch(prune=pygit2.GIT_FETCH_PRUNE) + except Exception as ex: + self.log.log_exception(ex, 'Fetch failed\n') + raise FetchException(ex) + + def set_base_ref(self, ref, sha): + self.repo.references.create(ref, sha) + + def get_blob(self, ref, path): + commitid = self.repo.lookup_reference(ref).target + bentry = self.repo[commitid].tree[path] + assert(bentry.type == 'blob') + return self.repo[bentry.id].data.decode() + + def get_list_from_blob(self, ref, path): + res = [] + for b in self.get_blob(ref, path).split('\n'): + b = b.strip() + if len(b) and not b.startswith('#'): + res.append(b) + return res + + def get_autobranch_refs(self, branch, basepath): + bref = 'refs/heads/%s' %branch + refs = [] + + for fname in self.get_list_from_blob(bref, basepath): + fname = os.path.join(os.path.dirname(basepath), fname) + for br in self.get_list_from_blob(bref, fname): + if br in list(self.repo.branches): + refs.append('refs/heads/%s' %br) + + return refs + + def log_revs(self, args): + ''' + Use git directly as pygit2 log is horribly slow + + Throws CalledProcessError if the return code is not 0 + ''' + oldpath = os.getcwd() + try: + os.chdir(self.repodir) + res = subprocess.run(args, capture_output=True) + res.check_returncode() + os.chdir(oldpath) + return res.stdout.decode().split() + + except Exception as ex: + self.log.log_exception(ex, 'Log revisions failed\n') + os.chdir(oldpath) + raise FatalException + + def log_revs_from(self, base): + ''' + Retrieve git log SHA1s from base to HEAD + ''' + args = [ 'git', 'log', '--pretty=%H', '%s..' %base ] + return self.log_revs(args) + + def log_revs_ref_from(self, base, ref): + ''' + Retrieve git log SHA1s from base to head of branch + ''' + args = [ 'git', 'log', '--no-merges', '--pretty=%H', + '%s..%s' %(base, ref) ] + return self.log_revs(args) + + def get_branch_head(self, branch='master'): + ref = 'refs/heads/%s' %branch + return self.repo.lookup_reference(ref).target + + def shortlog(self, sha): + subj = self.repo[sha].message.split('\n')[0] + return '%s ("%s")' %(sha[:12], subj.strip()) diff --git a/tipbot/mail.py b/tipbot/mail.py new file mode 100644 index 0000000..f8f56aa --- /dev/null +++ b/tipbot/mail.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL2.0 +# Copyright Thomas Gleixner <tglx@linutronix.de> +# + +from datetime import timedelta, datetime, timezone +from email.message import EmailMessage +from email.utils import make_msgid +from email.utils import formatdate +from email.header import Header, decode_header +from email.mime.text import MIMEText +from email import message_from_string +import email.policy +import mailbox +import smtplib +import pygit2 +import email +import time +import re +import os + +def build_raw(addr): + addr = addr.split('>')[0] + try: + return addr.split('<')[1] + except: + return addr + +re_fromchars = re.compile('[^a-zA-Z0-9 ]') +re_compress_space = re.compile('\s+') + +def clean_header(default, fallback): + default = re_compress_space.sub(' ', default).strip() + try: + return default.encode('ascii').decode() + except: + try: + return default.encode('UTF-8').decode() + except: + if fallback: + return re_compress_space.sub(' ', fallback).strip() + else: + return default + +def quote_name(name): + name = name.strip() + if re_fromchars.search(name): + name = '"%s"' %name.replace('"', '') + return name + +def clean_from(tip, default, fallback): + default = tip + ' ' + default + if fallback: + fallback = tip + ' ' + fallback + res = clean_header(default, fallback) + return quote_name(res) + +def clean_cc(default, fallback, utf8=False): + if default.find('>') > 0: + default = default.split('>')[0] + '>' + try: + name, addr = default.split('<', 1) + name = quote_name(name.strip()) + try: + name = name.encode('ascii').decode() + except: + if not utf8: + return fallback + name = name.encode('UTF-8').decode() + return name + ' <' + addr + except: + return fallback + +class mailer(object): + + cctags = [ + "Reported-and-tested-by", + "Reported-by", + "Suggested-by", + "Originally-from", + "Originally-by", + "Signed-off-by", + "Tested-by", + "Reviewed-by", + "Acked-by", + "Cc", + ] + + def __init__(self, args, logger): + self.args = args + self.log = logger + if args.mbox: + self.mbox = os.path.abspath(args.mbox) + + if args.test: + self.forceccs = [] + self.optccs = {} + else: + self.forceccs = vars(args).get('forceccs', []) + self.optccs = vars(args).get('optccs', {}) + + def send_mail(self, repo, branch, sha1): + + commit = repo.repo[sha1] + subj = commit.message.split('\n')[0].strip() + + ccs = {} + refid = None + for l in commit.message.split('\n'): + try: + tag, rest = l.strip().split(':', 1) + + tag = tag.strip() + rest = rest.strip() + + if tag in self.cctags and not self.args.test: + if rest.find('@') < 0: + continue + if rest.find('>') >= 0 and rest.find('<') >= 0: + mail = rest.rsplit('>', 1)[0] + '>' + else: + mail = rest + + raw = build_raw(mail) + # Don't try the UTF8 header mess + # google mail is unhappy with that + addr = clean_cc(mail, raw) + + if raw not in ccs: + ccs[raw] = addr + + elif tag == 'Link': + try: + mid = rest.rsplit('/', 1)[1] + if mid.find('@') > 0 and not refid: + refid = mid + except: + pass + except: + pass + + for cc in self.forceccs: + raw = build_raw(cc) + if raw not in ccs: + ccs[raw] = cc + + for cc, branches in self.optccs.items(): + if branch in branches: + raw = build_raw(cc) + if raw not in ccs: + ccs[raw] = cc + + body = '' + if self.args.test: + body += '\n-------------------- TEST ---------------------\n\n' + + body += 'The following commit has been merged into the %s branch of tip:\n\n' %branch + + body += 'Commit-ID: %s\n' %sha1 + body += 'Gitweb: https://git.kernel.org/tip/%s\n' %sha1 + body += 'Author: %s <%s>\n' %(commit.author.name, commit.author.email) + + td = timedelta(minutes = commit.author.offset) + tz = timezone(td) + dt = datetime.fromtimestamp(commit.author.time, tz) + tf = dt.strftime('%a, %d %b %Y %H:%M:%S %Z').replace('UTC', '') + + body += 'AuthorDate: %s\n' %tf + body += 'Committer: %s <%s>\n' %(commit.committer.name, commit.committer.email) + + td = timedelta(minutes = commit.committer.offset) + tz = timezone(td) + dt = datetime.fromtimestamp(commit.committer.time, tz) + tf = dt.strftime('%a, %d %b %Y %H:%M:%S %Z').replace('UTC', '') + + body += 'CommitterDate: %s\n' %tf + + body += '\n' + body += commit.message + body += '---\n' + + tree = commit.tree + ptre = commit.parents[0].tree + + diff = ptre.diff_to_tree(tree) + body += diff.stats.format(format=pygit2.GIT_DIFF_STATS_FULL | + pygit2.GIT_DIFF_STATS_INCLUDE_SUMMARY, + width=70) + body += '\n' + body += diff.patch + + body = body.encode('UTF-8').decode() + + msg = EmailMessage() + + msg['Return-path'] ='tip-bot2@linutronix.de' + msg['Date'] = '%s' %formatdate() + + name = clean_from('tip-bot2 for', commit.author.name, commit.author.email.split('@')[0]) + mfrom = '%s <tip-bot2@linutronix.de>' %name + msg['From'] = mfrom + + msg['Sender'] = 'tip-bot2@linutronix.de' + msg.set_unixfrom('From tip-bot2 ' + time.ctime(time.time())) + + if not self.args.test: + msg['Reply-to'] = 'linux-kernel@vger.kernel.org' + msg['To'] = 'linux-tip-commits@vger.kernel.org' + else: + msg['Reply-to'] = 'tglx@linutronix.de' + msg['To'] = 'Thomas Gleixner <tglx@linutronix.de>' + + subj = clean_header(subj, None) + subj = '[tip: %s] %s' %(branch, subj) + msg['Subject'] = subj + + if len(ccs) > 0: + rcpt = '' + for k, addr in ccs.items(): + rcpt += '%s, ' %addr + msg['Cc'] = rcpt.rstrip(', ') + + if refid: + msg['In-Reply-To'] = '<%s>' %refid + msg['References'] ='<%s>' %refid + + if not msg.get('MIME-Version'): + msg['MIME-Version'] = '1.0' + msg['Message-ID'] = '%s' %make_msgid('tip-bot2') + + msg['X-Mailer'] = 'tip-git-log-daemon' + msg['Robot-ID'] = '<tip-bot2.linutronix.de>' + msg['Robot-Unsubscribe'] = 'Contact <mailto:tglx@linutronix.de> to get blacklisted from these emails' + + msg['Content-Type'] = 'text/plain' + msg.set_param('charset', 'utf-8', header='Content-Type') + msg['Content-Transfer-Encoding'] = '8bit' + msg['Content-Disposition'] = 'inline' + + msg['Precedence'] = 'bulk' + + msg.set_content(body) + + if self.args.mbox: + mbox = mailbox.mbox(self.mbox, create=True) + try: + mbox.add(msg) + except: + pol = email.policy.EmailPolicy(utf8=True) + mbmsg = EmailMessage(pol) + for k in msg: + if k not in mbmsg: + mbmsg[k] = msg[k] + mbmsg.set_content(msg.get_content()) + mbox.add(mbmsg) + mbox.close() + elif not self.args.smtp: + print(msg.as_string()) + + if self.args.smtp: + to = msg['To'] + + server = smtplib.SMTP('localhost') + server.ehlo() + server.send_message(msg) + server.quit() diff --git a/tipbot/util.py b/tipbot/util.py new file mode 100644 index 0000000..68ba9a4 --- /dev/null +++ b/tipbot/util.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL2.0 +# Copyright Thomas Gleixner <tglx@linutronix.de> + +import traceback +import syslog +import sys + +class FetchException(Exception): + pass + +class FatalException(Exception): + pass + +class logger(object): + def __init__(self, use_syslog=False, verbose=False): + self.use_syslog = use_syslog + self.verbose = verbose + self.warnings = '' + self.exceptions = '' + self.syslog_warn = syslog.LOG_MAIL | syslog.LOG_WARNING + self.syslog_info = syslog.LOG_MAIL | syslog.LOG_INFO + self.syslog_debug = syslog.LOG_MAIL | syslog.LOG_DEBUG + + def log_debug(self, txt): + if self.verbose: + if self.use_syslog: + syslog.syslog(self.syslog_debug, txt) + else: + sys.stderr.write(txt) + + def log(self, txt): + if self.use_syslog: + syslog.syslog(self.syslog_info, txt) + else: + sys.stderr.write(txt) + + def log_warn(self, txt): + self.warnings += txt + if self.use_syslog: + syslog.syslog(self.syslog_warn, txt) + else: + sys.stderr.write(txt) + + def log_exception(self, ex, msg=''): + txt = 'tip-bot2: %s%s' %(msg, ex) + if self.verbose: + txt += '%s\n' % (traceback.format_exc()) + self.exceptions += txt + self.log_warn(txt) diff --git a/tipbot/version.py b/tipbot/version.py new file mode 100644 index 0000000..0334def --- /dev/null +++ b/tipbot/version.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0-only +# Copyright Thomas Gleixner <tglx@linutronix.de> + +__version__ = '0.1' |