Anton Shestakov <av6@dwimlabs.net>, Sun, 24 Sep 2017 12:25:05 +0800
viewer: use subqueries to get data on index page
Before, there were bare columns in the aggregate query, their values were
undefined (but it somehow worked), as SQLite docs say. Good news is that now
this bigger query uses (project_id, ordinal) index and is really fast.
hooks-queue.py
Permissions: -rwxr-xr-x
from argparse import ArgumentParser, FileType, SUPPRESS from tornado.ioloop import IOLoop from tornado.options import define, options from tornado.web import Application, HTTPError, RequestHandler, URLSpec from candolint.utils import lookup_option, timestamp class BaseHookHandler(RequestHandler): return self.application.rconn def write_error(self, status_code, **kwargs): result = getattr(kwargs['exc_info'][1], 'log_message', result) self.finish({'result': result}) def parse_json_payload(self): return json.loads(self.request.body) raise HTTPError(400, 'malformed JSON') def push(self, base, changes): logging.debug('Pushing %s', data) self.rconn.rpush('candolint:queue:changes', data) logging.info('Pushed a change for %s', item['repo']) self.finish({'result': 'OK', 'queued': len(changes)}) class BitbucketHookHandler(BaseHookHandler): event = self.request.headers.get('X-Event-Key') logging.debug('%s received event %r', self.__class__.__name__, event) payload = self.parse_json_payload() 'url': payload['repository']['links']['html']['href'], 'scm': payload['repository']['scm'], 'repo': payload['repository']['name'], 'timestamp': timestamp(), 'source': 'bitbucket.org' if payload['repository']['scm'] == 'git': for change in payload['push']['changes']: if change['new']['type'] == 'branch': changes.add(change['new']['name']) for change in payload['push']['changes']: for head in change['new'].get('heads', []): changes.add(head['hash']) changes.add(change['new']['target']['hash']) class GithubHookHandler(BaseHookHandler): event = self.request.headers.get('X-GitHub-Event', 'nonexistent') logging.debug('%s received event %r', self.__class__.__name__, event) getattr(self, 'handle_' + event, self.unknown_event)() self.finish({'result': 'pong'}) payload = self.parse_json_payload() 'url': payload['repository']['html_url'], 'repo': payload['repository']['name'], 'timestamp': timestamp(), ref = payload['ref'].rpartition('/')[-1] self.finish({'result': 'unknown event'}) class CandolintHooks(Application): def __init__(self, rconn, debug=False): URLSpec(r'/bitbucket', BitbucketHookHandler), URLSpec(r'/github', GithubHookHandler), super(CandolintHooks, self).__init__(handlers, debug=debug) def listen(self, port, address='', **kwargs): name = self.__class__.__name__ logging.info('%s is serving on http://%s:%d/', name, address, port) super(CandolintHooks, self).listen(port, address, **kwargs) define('listen', metavar='IP', default='127.0.0.1') define('port', metavar='PORT', default=8034, type=int) define('xheaders', metavar='True|False', default=False, type=bool) parser = ArgumentParser(argument_default=SUPPRESS) '-c', '--config', type=FileType('r'), help='configuration file (YAML)') '-d', '--debug', action='store_true', default=False, help='enable debugging output') group = parser.add_argument_group( 'these options that can also be specified in the configuration file') group.add_argument('--redis-host', help='(default: 127.0.0.1)') group.add_argument('--redis-port', type=int, help='(default: 6379)') group.add_argument('--redis-password') args, extra = parser.parse_known_args() options.parse_command_line([None] + extra) # 1st argument is ignored config = yaml.safe_load(args.config) if hasattr(args, 'config') else {} rhost = lookup_option('redis-host', args, config, default='127.0.0.1') rport = lookup_option('redis-port', args, config, default=6379) rpass = lookup_option('redis-password', args, config) level=logging.DEBUG if args.debug else logging.INFO, format='%(asctime)s %(levelname)-8s %(message)s') logging.debug('Connecting to Redis server') rconn = redis.StrictRedis(host=rhost, port=rport, password=rpass, db=0) logging.info('Connected to Redis server') application = CandolintHooks(rconn, args.debug) application.listen(options.port, options.listen, xheaders=options.xheaders) IOLoop.instance().start() if __name__ == '__main__':