#!/usr/bin/env python3 # This is a very quick-and-dirty script to bring up a hardware builder in the # Equinix Metal platform for the members of the LF Stable Kernel group. # # SPDX-License-Identifier: GPL-2.0-or-later # # -*- coding: utf-8 -*- # __author__ = 'Konstantin Ryabitsev ' import packet import sys import os import logging import argparse import json import datetime from string import Template logger = logging.getLogger('builder-ci') def get_manager(config): eq_auth_token = config['core'].get('auth_token') if not eq_auth_token: logger.critical('core.auth_token not set') sys.exit(1) return packet.Manager(auth_token=eq_auth_token) def print_ips(device, user): sshflags = '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' for ipaddr in device.ip_addresses: if not ipaddr['public']: continue logger.info(' ssh %s %s@%s', sshflags, user, ipaddr['address']) def get_builder(manager, project_id, hostname): for device in manager.list_all_devices(project_id=project_id): if device.hostname == hostname: return device return None def create_builder(config, user): manager = get_manager(config) eq_project_id = config['core'].get('project_id') hostname = '%s-builder.ci.kernel.org' % user logger.info('Checking for existing %s', hostname) device = get_builder(manager, eq_project_id, hostname) if device: logger.critical(' device exists (%s)', device.id) print_ips(device, user) sys.exit(1) usec = 'user_%s' % user pkf = config[usec].get('pubkey') if not pkf: logger.critical('Need pubkey entry for %s', user) sys.exit(1) with open(pkf) as pkfh: pubkey = pkfh.read().strip() cif = config[usec].get('cloud_init') if cif: with open(cif) as cifh: citpt = cifh.read() tptdata = { 'username': user, 'pubkey': pubkey, 'msmtp_user': config['msmtp'].get('user'), 'msmtp_password': config['msmtp'].get('password'), } userdata = Template(citpt).safe_substitute(tptdata) eq_plan = config[usec].get('plan') eq_facility = config[usec].get('facility') eq_os = config[usec].get('os') if not eq_plan or not eq_facility or not eq_os: logger.critical('Need to set plan, facility, os') sys.exit(1) rip_after_hrs = config[usec].getint('rip_after_hrs', 2) rip_ts = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=rip_after_hrs) rip_after = rip_ts.isoformat(timespec='seconds') customdata = json.dumps({'rip_after': rip_after}) logger.info('Creating %s (facility=%s, os=%s)', hostname, eq_facility, eq_os) manager.create_device(project_id=eq_project_id, hostname=hostname, plan=eq_plan, facility=[eq_facility, 'any'], operating_system=eq_os, customdata=customdata, userdata=userdata) logger.info('You will get an email to %s@kernel.org when it is ready.', user) logger.info('It will be auto-ripped in %sH (at %s).', rip_after_hrs, rip_after) def destroy_builder(config, user): manager = get_manager(config) eq_project_id = config['core'].get('project_id') hostname = '%s-builder.ci.kernel.org' % user logger.info('Checking for existing %s', hostname) device = get_builder(manager, eq_project_id, hostname) if device: logger.info('Destroying %s', hostname) device.delete() else: logger.info('%s does not appear to be running', hostname) def get_rip_after(device): now = datetime.datetime.now(datetime.timezone.utc) if 'rip_after' in device.customdata: # Trick for python 3.6 compatibility isodate = device.customdata['rip_after'].replace('+00:00', '+0000') rip_after = datetime.datetime.strptime(isodate, '%Y-%m-%dT%H:%M:%S%z') else: rip_after = None return rip_after, now def check_builder(config, user): manager = get_manager(config) eq_project_id = config['core'].get('project_id') hostname = '%s-builder.ci.kernel.org' % user logger.info('Checking for existing %s', hostname) device = get_builder(manager, eq_project_id, hostname) if device: logger.info('Found active host:') print_ips(device, user) rip_after, now = get_rip_after(device) if rip_after is not None: if rip_after < now: logger.info('Will be ripped very soon (ish)') else: rip_when = rip_after - now mins = int(rip_when.seconds / 60) logger.info('Will be ripped in %s min (ish)', mins) else: logger.info('Not found: %s', hostname) def rip_builder(config, user): manager = get_manager(config) eq_project_id = config['core'].get('project_id') hostname = '%s-builder.ci.kernel.org' % user device = get_builder(manager, eq_project_id, hostname) if device and 'rip_after' in device.customdata: rip_after, now = get_rip_after(device) if rip_after and rip_after < now: logger.info('Auto-ripping %s (rip_after: %s)', hostname, device.customdata['rip_after']) device.delete() else: logger.info('Not ripping %s (rip_after: %s)', hostname, device.customdata['rip_after']) def extend_builder(config, user): manager = get_manager(config) eq_project_id = config['core'].get('project_id') hostname = '%s-builder.ci.kernel.org' % user device = get_builder(manager, eq_project_id, hostname) if device and 'rip_after' in device.customdata: rip_after, now = get_rip_after(device) if not rip_after: logger.info('Could not get auto-rip time for %s', hostname) return rip_when = rip_after - now if rip_after > now and rip_when.seconds > 3600: logger.info('%s already has more than 1H left', hostname) return rip_ts = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(hours=1) new_rip_after = rip_ts.isoformat(timespec='seconds') customdata = json.dumps({'rip_after': new_rip_after}) device.customdata = customdata device.update() logger.info('%s will be ripped in 1H (at %s).', hostname, new_rip_after) else: logger.info('Sorry, no such device: %s', hostname) def print_help(): logger.info('Command summary:') logger.info('---------------+------------------------------------------------') logger.info('create | create a builder system') logger.info(' | email will be sent once the system is started') logger.info('---------------+------------------------------------------------') logger.info('destroy | shut down a running builder system') logger.info('---------------+------------------------------------------------') logger.info('check | print out information about any running systems') logger.info('---------------+------------------------------------------------') logger.info('extend | add 1 hour to the current build system lifetime') logger.info('---------------+------------------------------------------------') def read_config(cfgfile): from configparser import ConfigParser, ExtendedInterpolation if not os.path.exists(cfgfile): sys.stderr.write('ERROR: config file %s does not exist' % cfgfile) sys.exit(1) fconfig = ConfigParser(interpolation=ExtendedInterpolation()) fconfig.read(cfgfile) if 'core' not in fconfig: sys.stderr.write('ERROR: missing [core] section in %s' % cfgfile) sys.exit(1) return fconfig if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('-c', '--config-file', dest='cfgfile', required=True, help='Config file to use') parser.add_argument('-u', '--user', dest='user', required=True, help='User section to parse') parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', default=False, help='Quiet operation (cron mode)') parser.add_argument('-d', '--debug', dest='debug', action='store_true', default=False, help='Output debug information') parser.add_argument('action', help='Action to perform') cmdargs = parser.parse_args() _config = read_config(cmdargs.cfgfile) logger.setLevel(logging.DEBUG) logfile = _config['core'].get('logfile', '') if logfile: ch = logging.FileHandler(logfile) formatter = logging.Formatter(f'[%(asctime)s] {cmdargs.user}: %(message)s') ch.setFormatter(formatter) ch.setLevel(logging.INFO) logger.addHandler(ch) ch = logging.StreamHandler() formatter = logging.Formatter('%(message)s') ch.setFormatter(formatter) if cmdargs.quiet: ch.setLevel(logging.CRITICAL) elif cmdargs.debug: ch.setLevel(logging.DEBUG) else: ch.setLevel(logging.INFO) logger.addHandler(ch) if 'user_%s' % cmdargs.user not in _config: logger.critical('Section [user_%s] not in %s', cmdargs.user, cmdargs.cfgfile) sys.exit(1) if not _config['core'].get('project_id'): logger.critical('core.project_id not set') sys.exit(1) if cmdargs.action == 'create': create_builder(_config, cmdargs.user) elif cmdargs.action == 'destroy': destroy_builder(_config, cmdargs.user) elif cmdargs.action == 'check': check_builder(_config, cmdargs.user) elif cmdargs.action == 'rip': rip_builder(_config, cmdargs.user) elif cmdargs.action == 'extend': extend_builder(_config, cmdargs.user) elif cmdargs.action == 'help': print_help() else: logger.critical('Unknown action: %s', cmdargs.action) sys.exit(1)