306:edefddec933d
Anton Shestakov <av6@dwimlabs.net>, Sun, 24 Sep 2017 12:25:05 +0800
viewer: use subqueries to get data on index page Before, there were bare columns in the aggregate query, their values were undefined (but it somehow worked), as SQLite docs say. Good news is that now this bigger query uses (project_id, ordinal) index and is really fast.

previous change 220:5379500bba11

worker-queue.py

Permissions: -rwxr-xr-x

Other formats: Feeds:
#!/usr/bin/env python
import json
import logging
import sys
from argparse import ArgumentParser, FileType, SUPPRESS
from os.path import exists, isfile
from subprocess import check_output, STDOUT
from time import sleep
import redis
import yaml
from candolint.queue import loop
from candolint.utils import lookup_option, rel, timestamp
def handle(isolation, rconn, change):
payload = json.loads(change)
flags = ['--commit', payload['change']]
path = 'projects/{source}/{repo}.yml'.format(**payload)
if exists(path) and isfile(path):
logging.debug("Using config: %r", path)
flags += ['--config', path]
else:
logging.debug("File %r doesn't exist, trying to bootstrap", path)
flags += ['--data', json.dumps(payload)]
logging.info('Checking %s (payload: %s)', payload['repo'], change)
if isolation == 'none':
cmd = [sys.executable, '-u', rel('checker.py')]
output = check_output(cmd + flags, stderr=STDOUT)
elif isolation == 'docker':
cmd = [rel('check-in-docker.sh')]
output = check_output(cmd + flags)
logging.info('Done checking %s', payload['repo'])
payload.update({
'output': output,
'timestamp': timestamp()
})
rconn.rpush('candolint:queue:incoming', json.dumps(payload))
def main():
parser = ArgumentParser(argument_default=SUPPRESS)
parser.add_argument(
'-c', '--config', type=FileType('r'),
help='configuration file (YAML)')
parser.add_argument(
'-i', '--isolation', choices=('none', 'docker'), default='docker',
help='select isolation level for the checker')
parser.add_argument(
'-d', '--debug', action='store_true', default=False,
help='enable debugging output')
group = parser.add_argument_group(
'queue configuration',
'these options that can also be specified in the configuration file')
group.add_argument('--redis-host', help='(default: 127.0.0.1)')
group.add_argument('--redis-port', type=int, help='(default: 6379)')
group.add_argument('--redis-password')
args = parser.parse_args()
config = yaml.safe_load(args.config) if hasattr(args, 'config') else {}
rhost = lookup_option('redis-host', args, config, default='127.0.0.1')
rport = lookup_option('redis-port', args, config, default=6379)
rpass = lookup_option('redis-password', args, config)
logging.basicConfig(
level=logging.DEBUG if args.debug else logging.INFO,
format='%(asctime)s %(levelname)-8s %(message)s')
def wrapper(rconn, change):
return handle(args.isolation, rconn, change)
while True:
logging.debug('Connecting to Redis server %s:%d', rhost, rport)
rconn = redis.StrictRedis(host=rhost, port=rport, password=rpass, db=0)
logging.info('Connected to Redis server')
try:
loop(rconn, 'candolint:queue:changes', wrapper)
except redis.exceptions.ConnectionError:
logging.warn('Connection to Redis lost, reconnecting in 1 minute')
sleep(60)
if __name__ == '__main__':
main()