#!/usr/bin/env python3 # -*- coding: utf-8 -*- __author__ = 'Konstantin Ryabitsev ' import os import sys import argparse import logging import subprocess from fcntl import lockf, LOCK_EX, LOCK_UN from typing import Optional, Tuple, Dict from email.utils import formatdate from collections import OrderedDict PI_HEAD = 'refs/meta/origins' logger = logging.getLogger(__name__) DEFAULT_NAME = 'PI Origin Maker' DEFAULT_ADDR = 'devnull@kernel.org' DEFAULT_SUBJ = 'Origin commit' def git_run_command(gitdir: str, args: list, stdin: Optional[bytes] = None, env: Optional[Dict] = None) -> Tuple[int, bytes, bytes]: if not env: env = dict() if gitdir: env['GIT_DIR'] = gitdir args = ['git', '--no-pager'] + args logger.debug('Running %s', ' '.join(args)) pp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) (output, error) = pp.communicate(input=stdin) return pp.returncode, output, error def check_valid_repo(repo: str) -> None: # check that it exists and has 'objects' and 'refs' if not os.path.isdir(repo): raise FileNotFoundError(f'Path does not exist: {repo}') musts = {'objects', 'refs'} for must in musts: if not os.path.exists(os.path.join(repo, must)): raise FileNotFoundError(f'Path is not a valid bare git repository: {repo}') def git_write_commit(repo: str, env: dict, c_msg: str, body: bytes, dest: str = 'i') -> None: # We use git porcelain commands here. We could use pygit2, but this would pull in a fairly # large external lib for what is effectively 4 commands that we need to run. # Lock the repository try: # The lock shouldn't be held open for very long, so try without a timeout lockfh = open(os.path.join(repo, 'ezpi.lock'), 'w') lockf(lockfh, LOCK_EX) except IOError: raise RuntimeError('Could not obtain an exclusive lock') # Create a blob first ee, out, err = git_run_command(repo, ['hash-object', '-w', '--stdin'], stdin=body) if ee > 0: raise RuntimeError(f'Could not create a blob in {repo}: {err.decode()}') blob = out.strip(b'\n') # Create a tree object now treeline = b'100644 blob ' + blob + b'\t' + dest.encode() # Now mktree ee, out, err = git_run_command(repo, ['mktree'], stdin=treeline) if ee > 0: raise RuntimeError(f'Could not mktree in {repo}: {err.decode()}') tree = out.decode().strip() # Find out if we are the first commit or not ee, out, err = git_run_command(repo, ['rev-parse', f'{PI_HEAD}^0']) if ee > 0: args = ['commit-tree', '-m', c_msg, tree] else: args = ['commit-tree', '-p', PI_HEAD, '-m', c_msg, tree] # Commit the tree ee, out, err = git_run_command(repo, args, env=env) if ee > 0: raise RuntimeError(f'Could not commit-tree in {repo}: {err.decode()}') # Finally, update the ref commit = out.decode().strip() ee, out, err = git_run_command(repo, ['update-ref', PI_HEAD, commit]) if ee > 0: raise RuntimeError(f'Could not update-ref in {repo}: {err.decode()}') lockf(lockfh, LOCK_UN) def read_config(cfgfile): from configparser import ConfigParser if not os.path.exists(cfgfile): sys.stderr.write('ERROR: config file %s does not exist' % cfgfile) sys.exit(1) # We don't support duplicates right now piconfig = ConfigParser(strict=False) piconfig.read(cfgfile) return piconfig def get_pi_repos(mainrepo: str) -> Dict: res = dict() at = 0 latest_origins = None latest_repo = None while True: repo = os.path.join(mainrepo, 'git', '%d.git' % at) try: check_valid_repo(repo) logger.debug('Checking current origins in %s', repo) ec, out, err = git_run_command(repo, ['show', f'{PI_HEAD}:i']) # If it's blank, then we force it to be written if not len(out): res[repo] = '' else: latest_repo = repo latest_origins = out at += 1 except FileNotFoundError: break if latest_origins is None or latest_repo is None: logger.debug('Did not find any valid pi repos in %s', mainrepo) return res res[latest_repo] = latest_origins.decode() return res def make_origins(config, cmdargs): for section in config.sections(): # We want named sections if section.find(' ') < 0: continue origin = OrderedDict() origin['infourl'] = cmdargs.infourl origin['contact'] = cmdargs.contact mainrepo = config[section].get('mainrepo') if not mainrepo: mainrepo = config[section].get('inboxdir') if not mainrepo or not os.path.isdir(mainrepo): logger.info('%s: mainrepo=%s does not exist', section, mainrepo) continue mainrepo = mainrepo.rstrip('/') if cmdargs.repotop and os.path.dirname(mainrepo) != cmdargs.repotop: logger.info('Skipped %s: not directly in %s', mainrepo, cmdargs.repotop) continue pirepos = get_pi_repos(mainrepo) if not len(pirepos): logger.info('%s contains no public-inbox repos', mainrepo) continue origin['address'] = config[section].get('address') origin['listid'] = config[section].get('listid') if not origin['listid']: origin['listid'] = origin['address'].replace('@', '.') origin['newsgroup'] = config[section].get('newsgroup') if not origin['newsgroup']: origin['newsgroup'] = '.'.join(reversed(origin['listid'].split('.'))) odata_new = '[publicinbox]\n' for opt, val in origin.items(): odata_new += f'{opt}={val}\n' for pirepo, odata_old in pirepos.items(): if odata_new != odata_old: logger.debug('Setting new origins for %s', pirepo) logger.debug(odata_new) env = { 'GIT_AUTHOR_NAME': DEFAULT_NAME, 'GIT_AUTHOR_EMAIL': DEFAULT_ADDR, 'GIT_AUTHOR_DATE': formatdate(), 'GIT_COMMITTER_NAME': DEFAULT_NAME, 'GIT_COMMITTER_EMAIL': DEFAULT_ADDR, 'GIT_COMMITTER_DATE': formatdate(), } try: git_write_commit(pirepo, env, DEFAULT_SUBJ, odata_new.encode()) logger.info('Updated origins for %s', pirepo) except RuntimeError as ex: logger.info('Could not update origins in %s: %s', pirepo, ex) if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-c', '--pi-config-file', dest='cfgfile', default='/etc/public-inbox/config', help='Public-Inbox config file to use') parser.add_argument('-i', '--infourl', dest='infourl', default='https://www.kernel.org/lore.html', help='infourl value') parser.add_argument('-e', '--contact-email', dest='contact', default='postmaster ', help='contact value') parser.add_argument('-t', '--repo-top', dest='repotop', help='Only work on repos in this topdir') parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', default=False, help='Quiet operation (cron mode)') parser.add_argument('-d', '--debug', dest='debug', action='store_true', default=False, help='Output debug information') parser.add_argument('-l', '--logfile', dest='logfile', help='Record activity in this log file') _cmdargs = parser.parse_args() _config = read_config(_cmdargs.cfgfile) logger.setLevel(logging.DEBUG) if _cmdargs.logfile: ch = logging.FileHandler(_cmdargs.logfile) formatter = logging.Formatter(f'[%(asctime)s] %(message)s') ch.setFormatter(formatter) ch.setLevel(logging.INFO) logger.addHandler(ch) ch = logging.StreamHandler() formatter = logging.Formatter('%(message)s') ch.setFormatter(formatter) if _cmdargs.quiet: ch.setLevel(logging.CRITICAL) elif _cmdargs.debug: ch.setLevel(logging.DEBUG) else: ch.setLevel(logging.INFO) logger.addHandler(ch) make_origins(_config, _cmdargs)