66:db31e73bde12
Anton Shestakov <av6@dwimlabs.net>, Sat, 25 Jun 2016 18:16:39 +0800
models: drop Check.raw field (not worth storing in the same table) Replaying checks can be done (and is easier) when storing checker output in plain files, so let's lighten up the load on the database.

next change 87:ddcd7536f41d
previous change 59:07e5bd22bf0d

checker.py

Permissions: -rwxr-xr-x

Other formats: Feeds:
#!/usr/bin/env python -u
from __future__ import print_function
import os
from argparse import ArgumentParser, FileType
from datetime import datetime
from shutil import rmtree
from subprocess import check_call, check_output, CalledProcessError
from tempfile import mkdtemp
import yaml
rel = lambda *x: os.path.abspath(os.path.join(os.path.dirname(__file__), *x))
def run_ignore_codes(fn, args, codes):
try:
return fn(args)
except CalledProcessError as e:
if e.returncode not in codes:
raise
return e.output
def run(args, silent=False, get_output=False, ignore_codes=None):
if get_output:
fn = check_output
else:
fn = check_call
try:
if not silent:
print('$ ' + ' '.join(args))
if ignore_codes:
result = run_ignore_codes(fn, args, ignore_codes)
else:
result = fn(args)
if get_output:
return result
else:
return True
except Exception as e:
print('# C&O error: {}'.format(e))
print('# C&O job failed')
return False
def now():
return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S+00:00')
def read_linter_config(name, path=rel('linters')):
try:
fd = open(os.path.join(path, '{}.yml'.format(name)))
except Exception as e:
print("# C&O couldn't load linter config: {}".format(e))
return None
with fd:
try:
return yaml.safe_load(fd)
except Exception as e:
print("# C&O couldn't parse linter config: {}".format(e))
return None
def execute(config):
ok = True
print('# C&O job started: {}'.format(now()))
tmp = mkdtemp(prefix='candolint.')
os.chdir(tmp)
print('# C&O task: clone')
print('# C&O project URL: {}'.format(config['url']))
source = './source'
if ok and not run(['hg', 'clone', config['url'], source]):
ok = False
if ok:
print('$ cd {}'.format(source))
os.chdir(source)
if ok and not run(['hg', 'sum']):
ok = False
if ok:
template = (r'# C&O commit: {rev}:{node} {branch}\n'
r'# C&O commit date: {date|isodatesec}\n'
r'# C&O commit author: {author|person}\n'
r'# C&O commit message: {desc|firstline}\n')
if not run(['hg', 'log', '-r', '.', '-T', template], silent=True):
ok = False
if ok:
print('# C&O task: setup')
linter_config = {}
for linter in config['linters']:
if 'name' not in linter:
continue
name = linter['name']
if name in linter_config:
continue
lc = read_linter_config(name)
if lc is None or 'exec' not in lc:
continue
linter_config[name] = lc
for item in lc.get('setup', []):
if not run(item):
ok = False
break
if 'version' in lc and not run(lc['exec'] + lc['version']):
ok = False
break
if ok:
print('# C&O task: checks')
for linter in config['linters']:
if 'name' not in linter or 'include' not in linter:
continue
if linter['name'] not in linter_config:
continue
cmd = ['hg', 'files']
for pat in linter['include']:
cmd.extend(('-I', pat))
for pat in linter.get('exclude', []):
cmd.extend(('-X', pat))
files = run(cmd, silent=True, get_output=True, ignore_codes=(1,))
if files is False:
ok = False
break
lc = linter_config[linter['name']]
for f in files.splitlines():
cmd = lc['exec']
flags = lc.get('flags', []) + linter.get('flags', [])
pf = lc.get('post_flags', []) + linter.get('post_flags', [])
codes = lc.get('codes', (1,))
if not run(cmd + flags + [f] + pf, ignore_codes=codes):
ok = False
break
print('# C&O task: cleanup')
rmtree(tmp)
print('# C&O job finished: {}'.format(now()))
def bad_exit():
print('# C&O job failed (exception not caught)')
print('# C&O job finished: {}'.format(now()))
def main():
parser = ArgumentParser()
parser.add_argument('config', help='configuration file (YAML)', type=FileType('r'))
args = parser.parse_args()
config = yaml.safe_load(args.config)
try:
execute(config)
except:
bad_exit()
if __name__ == '__main__':
main()