Anton Shestakov <av6@dwimlabs.net>, Thu, 14 Sep 2017 17:59:05 +0800
pipelines: retain ~/.cache/pip between runs
worker-queue.py
Permissions: -rwxr-xr-x
from argparse import ArgumentParser, FileType, SUPPRESS from os.path import exists, isfile from subprocess import check_output, STDOUT from candolint.queue import loop from candolint.utils import lookup_option, rel, timestamp def handle(isolation, rconn, change): payload = json.loads(change) flags = ['--commit', payload['change']] path = 'projects/{source}/{repo}.yml'.format(**payload) if exists(path) and isfile(path): logging.debug("Using config: %r", path) flags += ['--config', path] logging.debug("File %r doesn't exist, trying to bootstrap", path) flags += ['--data', json.dumps(payload)] logging.info('Checking %s (payload: %s)', payload['repo'], change) cmd = [sys.executable, '-u', rel('checker.py')] output = check_output(cmd + flags, stderr=STDOUT) elif isolation == 'docker': cmd = [rel('check-in-docker.sh')] output = check_output(cmd + flags) logging.info('Done checking %s', payload['repo']) rconn.rpush('candolint:queue:incoming', json.dumps(payload)) parser = ArgumentParser(argument_default=SUPPRESS) '-c', '--config', type=FileType('r'), help='configuration file (YAML)') '-i', '--isolation', choices=('none', 'docker'), default='docker', help='select isolation level for the checker') '-d', '--debug', action='store_true', default=False, help='enable debugging output') group = parser.add_argument_group( 'these options that can also be specified in the configuration file') group.add_argument('--redis-host', help='(default: 127.0.0.1)') group.add_argument('--redis-port', type=int, help='(default: 6379)') group.add_argument('--redis-password') args = parser.parse_args() config = yaml.safe_load(args.config) if hasattr(args, 'config') else {} rhost = lookup_option('redis-host', args, config, default='127.0.0.1') rport = lookup_option('redis-port', args, config, default=6379) rpass = lookup_option('redis-password', args, config) level=logging.DEBUG if args.debug else logging.INFO, format='%(asctime)s %(levelname)-8s %(message)s') def wrapper(rconn, change): return handle(args.isolation, rconn, change) logging.debug('Connecting to Redis server %s:%d', rhost, rport) rconn = redis.StrictRedis(host=rhost, port=rport, password=rpass, db=0) logging.info('Connected to Redis server') loop(rconn, 'candolint:queue:changes', wrapper) except redis.exceptions.ConnectionError: logging.warn('Connection to Redis lost, reconnecting in 1 minute') if __name__ == '__main__':