354:23e5b4d4f272 default tip
Anton Shestakov <av6@dwimlabs.net>, Mon, 13 Apr 2020 19:01:24 +0800
tests: update a link

previous change 348:80539f2c61fc

checker.py

Permissions: -rwxr-xr-x

Other formats: Feeds:
#!/usr/bin/env -S python -u
from __future__ import absolute_import, print_function
import os
from argparse import ArgumentParser, FileType
from fnmatch import fnmatchcase
from shutil import rmtree
from subprocess import check_call, check_output, CalledProcessError
from tempfile import mkdtemp
import yaml
from candolint.utils import rel, timestamp
def run_ignore_codes(fn, args, codes, **kwargs):
try:
return fn(args, **kwargs)
except CalledProcessError as e:
if e.returncode not in codes:
raise
return e.output
def run(args, silent=False, get_output=False, ignore_codes=None, env=None):
if get_output:
fn = check_output
else:
fn = check_call
if env:
full_env = os.environ.copy()
full_env.update(env)
else:
full_env = None
try:
if not silent:
print('$ ' + ' '.join(args))
if ignore_codes:
result = run_ignore_codes(fn, args, ignore_codes, env=full_env)
else:
result = fn(args, env=full_env)
if get_output:
return result
else:
return True
except Exception as e:
print('# C&O error: {}'.format(e))
return False
def run_git(args, **kwargs):
env = kwargs.pop('env', {})
env.update({'LC_ALL': 'C.UTF-8'})
return run(['git'] + args, env=env, **kwargs)
def run_hg(args, **kwargs):
env = kwargs.pop('env', {})
env.update({'HGPLAIN': '1'})
env.update({'HGENCODING': 'UTF-8'})
return run(['hg'] + args, env=env, **kwargs)
def read_linter_config(name, path=rel('linters')):
try:
fd = open(os.path.join(path, '{}.yml'.format(name)))
except Exception as e:
print("# C&O couldn't load linter config: {}".format(e))
return None
with fd:
try:
return yaml.safe_load(fd)
except Exception as e:
print("# C&O couldn't parse linter config: {}".format(e))
return None
def read_intree_config():
try:
fd = open('candolint.yml')
except Exception as e:
print("# C&O couldn't load in-tree config: {}".format(e))
return None
with fd:
try:
return yaml.safe_load(fd)
except Exception as e:
print("# C&O couldn't parse in-tree config: {}".format(e))
return None
def prevalidate(config):
ok = True
if 'url' not in config:
print("# C&O config doesn't have 'url' defined.")
ok = False
if 'scm' not in config:
print("# C&O config doesn't have 'scm' defined.")
ok = False
elif config['scm'] not in ('git', 'hg'):
print("# C&O checker doesn't support {} yet.".format(config['scm']))
ok = False
return ok
def combined(a, b):
if a and b:
result = a.copy()
result.update(b)
else:
result = a or b
return result
def process_config(lc, pc):
c = {
'env': combined(lc.get('env'), pc.get('env')),
'vars': combined(lc.get('vars', {}), pc.get('vars', {})),
'codes': lc.get('codes', (1,))
}
def render(items):
return [item % c['vars'] for item in items]
for key in ('flags', 'post_flags'):
c[key] = render(lc.get(key, []) + pc.get(key, []))
c['setup'] = [render(steps) for steps in lc.get('setup', [])]
c['exec'] = render(lc['exec'])
c['version'] = render(lc['version'])
return c
def git_clone(url, dest):
return run_git(['clone', url, dest])
def hg_clone(url, dest):
return run_hg(['clone', url, dest])
def git_checkout(commit):
return run_git(['checkout', commit])
def hg_update(commit):
return run_hg(['update', commit])
def git_status():
return run_git(['status'])
def hg_summary():
return run_hg(['summary'])
def git_show():
template = (r'# C&O commit: %H%n'
r'# C&O commit ref names: %D%n'
r'# C&O commit date: %ai%n'
r'# C&O commit author: %aN%n'
r'# C&O commit message: %s%n')
return run_git(['show', '-q', '--format=format:' + template], silent=True)
def hg_log():
template = (r'# C&O commit: {rev}:{node}\n'
r'# C&O commit branch: {branch}\n'
r'# C&O commit date: {date|isodatesec}\n'
r'# C&O commit author: {author|person}\n'
r'# C&O commit message: {desc|firstline}\n')
return run_hg(['log', '-r', '.', '-T', template], silent=True)
def git_files(include, exclude):
cmd = ['ls-files']
cmd.extend(include)
files = run_git(cmd, silent=True, get_output=True)
if files is not False:
files = files.splitlines()
for e in exclude:
files = [f for f in files if not fnmatchcase(f, e)]
return files
def hg_files(include, exclude):
cmd = ['files']
for pat in include:
cmd.extend(('-I', pat))
for pat in exclude:
cmd.extend(('-X', pat))
files = run_hg(cmd, silent=True, get_output=True, ignore_codes=(1,))
if files is not False:
files = files.splitlines()
return files
def execute(tmp, config, args):
if not prevalidate(config):
return False
os.chdir(tmp)
source = './source'
print('# C&O task: clone')
print('# C&O project URL: {}'.format(config['url']))
if config['scm'] == 'git':
if not git_clone(config['url'], source):
return False
elif config['scm'] == 'hg':
if not hg_clone(config['url'], source):
return False
print('$ cd {}'.format(source))
os.chdir(source)
if args.commit is not None:
if config['scm'] == 'git':
if not git_checkout(args.commit):
return False
elif config['scm'] == 'hg':
if not hg_update(args.commit):
return False
if config['scm'] == 'git':
if not git_status():
return False
elif config['scm'] == 'hg':
if not hg_summary():
return False
if config['scm'] == 'git':
if not git_show():
return False
elif config['scm'] == 'hg':
if not hg_log():
return False
print('# C&O task: setup')
if args.data is not None:
intree = read_intree_config()
if intree is not None:
config.update(intree)
print('# C&O using in-tree config')
else:
print('# C&O in-tree config is required while bootstrapping')
return False
else:
print('# C&O not using in-tree config')
linters = []
distinct = []
for linter in config.get('linters', []):
if 'name' not in linter:
print("# C&O skipping linter without 'name': {}".format(linter))
continue
name = linter['name']
if 'include' not in linter:
print("# C&O skipping '{}': no 'include' in config".format(name))
continue
linters.append(linter)
if name not in distinct:
distinct.append(name)
linter_config = {}
for name in distinct:
if name in linter_config:
continue
lc = read_linter_config(name)
if lc is None:
print("# C&O skipping '{}': no config".format(name))
continue
if 'exec' not in lc:
print("# C&O skipping '{}': no 'exec' in config".format(name))
continue
linter_config[name] = lc
installing = [name for name in distinct if name in linter_config]
print('# C&O installing linters: {}'.format(' '.join(installing)))
installed = []
setup_cache = set()
for linter in linters:
name = linter['name']
if name not in linter_config:
continue
lc = process_config(linter_config[name], linter)
key = (name, str(lc.get('setup', [])))
if key in setup_cache:
continue
for item in lc.get('setup', []):
if not run(item):
return False
if 'version' in lc:
if not run(lc['exec'] + lc['version'], env=lc['env']):
return False
setup_cache.add(key)
if name not in installed:
installed.append(name)
print('# C&O linters installed: {}'.format(' '.join(installed)))
print('# C&O task: checks')
for linter in linters:
name = linter['name']
if name not in linter_config:
continue
if config['scm'] == 'git':
files = git_files(linter['include'], linter.get('exclude', []))
elif config['scm'] == 'hg':
files = hg_files(linter['include'], linter.get('exclude', []))
if files is False:
return False
lc = process_config(linter_config[name], linter)
for f in files:
cmd = lc['exec'] + lc['flags'] + [f] + lc['post_flags']
if not run(cmd, ignore_codes=lc['codes'], env=lc['env']):
return False
return True
def wrapper(config, args):
print('# C&O job started: {}'.format(timestamp()))
tmp = mkdtemp(prefix='candolint.')
if not execute(tmp, config, args):
print('# C&O job failed')
print('# C&O task: cleanup')
rmtree(tmp)
print('# C&O job finished: {}'.format(timestamp()))
def bad_exit():
print('# C&O job failed (exception not caught)')
print('# C&O job finished: {}'.format(timestamp()))
def main():
parser = ArgumentParser()
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'--config', metavar='FILE', type=FileType('r'),
help='project configuration file (YAML)')
group.add_argument(
'--data', metavar='YAML',
help='bootstrapping data as a YAML string')
parser.add_argument('--commit', help='a commit to update to')
args = parser.parse_args()
if args.data is not None:
config = yaml.safe_load(args.data)
else:
config = yaml.safe_load(args.config)
try:
wrapper(config, args)
except:
bad_exit()
raise
if __name__ == '__main__':
main()