Anton Shestakov <av6@dwimlabs.net>, Sun, 24 Sep 2017 12:25:05 +0800
viewer: use subqueries to get data on index page
Before, there were bare columns in the aggregate query, their values were
undefined (but it somehow worked), as SQLite docs say. Good news is that now
this bigger query uses (project_id, ordinal) index and is really fast.
worker-queue.py
Permissions: -rwxr-xr-x
from argparse import ArgumentParser, FileType, SUPPRESS from os.path import exists, isfile from subprocess import check_output, STDOUT from candolint.queue import loop from candolint.utils import lookup_option, rel, timestamp def handle(isolation, rconn, change): payload = json.loads(change) flags = ['--commit', payload['change']] path = 'projects/{source}/{repo}.yml'.format(**payload) if exists(path) and isfile(path): logging.debug("Using config: %r", path) flags += ['--config', path] logging.debug("File %r doesn't exist, trying to bootstrap", path) flags += ['--data', json.dumps(payload)] logging.info('Checking %s (payload: %s)', payload['repo'], change) cmd = [sys.executable, '-u', rel('checker.py')] output = check_output(cmd + flags, stderr=STDOUT) elif isolation == 'docker': cmd = [rel('check-in-docker.sh')] output = check_output(cmd + flags) logging.info('Done checking %s', payload['repo']) rconn.rpush('candolint:queue:incoming', json.dumps(payload)) parser = ArgumentParser(argument_default=SUPPRESS) '-c', '--config', type=FileType('r'), help='configuration file (YAML)') '-i', '--isolation', choices=('none', 'docker'), default='docker', help='select isolation level for the checker') '-d', '--debug', action='store_true', default=False, help='enable debugging output') group = parser.add_argument_group( 'these options that can also be specified in the configuration file') group.add_argument('--redis-host', help='(default: 127.0.0.1)') group.add_argument('--redis-port', type=int, help='(default: 6379)') group.add_argument('--redis-password') args = parser.parse_args() config = yaml.safe_load(args.config) if hasattr(args, 'config') else {} rhost = lookup_option('redis-host', args, config, default='127.0.0.1') rport = lookup_option('redis-port', args, config, default=6379) rpass = lookup_option('redis-password', args, config) level=logging.DEBUG if args.debug else logging.INFO, format='%(asctime)s %(levelname)-8s %(message)s') def wrapper(rconn, change): return handle(args.isolation, rconn, change) logging.debug('Connecting to Redis server %s:%d', rhost, rport) rconn = redis.StrictRedis(host=rhost, port=rport, password=rpass, db=0) logging.info('Connected to Redis server') loop(rconn, 'candolint:queue:changes', wrapper) except redis.exceptions.ConnectionError: logging.warn('Connection to Redis lost, reconnecting in 1 minute') if __name__ == '__main__':