203:5fb816a7a437
Anton Shestakov <av6@dwimlabs.net>, Thu, 11 Aug 2016 20:49:37 +0800
pipelines: install dependencies, run tests and report coverage

next change 220:5379500bba11
previous change 200:51b38a0bdcea

worker-queue.py

Permissions: -rwxr-xr-x

Other formats: Feeds:
#!/usr/bin/env python
import json
import logging
import sys
from argparse import ArgumentParser, FileType, SUPPRESS
from os.path import exists, isfile
from subprocess import check_output, STDOUT
from time import sleep
import redis
import yaml
from candolint.queue import loop
from candolint.utils import lookup_option, rel, timestamp
def handle(isolation, rconn, change):
payload = json.loads(change)
flags = ['--commit', payload['change']]
path = 'projects/{source}/{repo}.yml'.format(**payload)
if exists(path) and isfile(path):
logging.debug("Using config: %r", path)
flags += ['--config', path]
else:
logging.debug("File %r doesn't exist, trying to bootstrap", path)
flags += ['--data', json.dumps(payload)]
logging.info('Checking %s (payload: %r)', payload['repo'], payload)
if isolation == 'none':
cmd = [sys.executable, '-u', rel('checker.py')]
output = check_output(cmd + flags, stderr=STDOUT)
elif isolation == 'docker':
cmd = [rel('check-in-docker.sh')]
output = check_output(cmd + flags)
logging.info('Done checking %s', payload['repo'])
payload.update({
'output': output,
'timestamp': timestamp()
})
rconn.rpush('candolint:queue:incoming', json.dumps(payload))
def main():
parser = ArgumentParser(argument_default=SUPPRESS)
parser.add_argument(
'-c', '--config', type=FileType('r'),
help='configuration file (YAML)')
parser.add_argument(
'-i', '--isolation', choices=('none', 'docker'), default='docker',
help='select isolation level for the checker')
parser.add_argument(
'-d', '--debug', action='store_true', default=False,
help='enable debugging output')
group = parser.add_argument_group(
'queue configuration',
'these options that can also be specified in the configuration file')
group.add_argument('--redis-host', help='(default: 127.0.0.1)')
group.add_argument('--redis-port', type=int, help='(default: 6379)')
group.add_argument('--redis-password')
args = parser.parse_args()
config = yaml.safe_load(args.config) if hasattr(args, 'config') else {}
rhost = lookup_option('redis-host', args, config, default='127.0.0.1')
rport = lookup_option('redis-port', args, config, default=6379)
rpass = lookup_option('redis-password', args, config)
logging.basicConfig(
level=logging.DEBUG if args.debug else logging.INFO,
format='%(asctime)s %(levelname)-8s %(message)s')
def wrapper(rconn, change):
return handle(args.isolation, rconn, change)
while True:
logging.debug('Connecting to Redis server %s:%d', rhost, rport)
rconn = redis.StrictRedis(host=rhost, port=rport, password=rpass, db=0)
logging.info('Connected to Redis server')
try:
loop(rconn, 'candolint:queue:changes', wrapper)
except redis.exceptions.ConnectionError:
logging.warn('Connection to Redis lost, reconnecting in 1 minute')
sleep(60)
if __name__ == '__main__':
main()