# HG changeset patch # User Anton Shestakov <av6@dwimlabs.net> # Date 1528528484 -28800 # Node ID 0e84a4a22acd282e221b933f07dd9e415625301d # Parent 0dca68b49bc56bfce31524ea8cde07c3cd404428 robots: disallow some non-essential pages diff -r 0dca68b49bc5 -r 0e84a4a22acd roles/hgweb/files/robots.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/roles/hgweb/files/robots.txt Sat Jun 09 15:14:44 2018 +0800 @@ -0,0 +1,25 @@ +User-agent: * + +# Expensive pages +Disallow: /*/annotate/ +Disallow: /*/comparison/ +Disallow: /*/diff/ +Disallow: /*/archive/ + +# File log is too much +Disallow: /*/log/*/ + +# API is off-limits +Disallow: /*/json-* +Disallow: /*/raw-* + +# Paginate instead +Disallow: /*?revcount= + +# Graph page is a more expensive log +Disallow: /*/graph$ +Disallow: /*/graph/ + +# It's for humans +Disallow: /*/help$ +Disallow: /*/help/ diff -r 0dca68b49bc5 -r 0e84a4a22acd roles/hgweb/tasks/appinstall.yml --- a/roles/hgweb/tasks/appinstall.yml Tue Jan 02 19:27:00 2018 +0800 +++ b/roles/hgweb/tasks/appinstall.yml Sat Jun 09 15:14:44 2018 +0800 @@ -37,6 +37,13 @@ notify: - restart hgweb +- name: Copy robots.txt + copy: + src: robots.txt + dest: /home/{{ hgweb_user }}/hgwebfiles/robots.txt + mode: 0644 + tags: [robots] + - name: Copy requirements.txt copy: src: requirements.txt diff -r 0dca68b49bc5 -r 0e84a4a22acd roles/hgweb/templates/etc/nginx/sites-available/hgweb --- a/roles/hgweb/templates/etc/nginx/sites-available/hgweb Tue Jan 02 19:27:00 2018 +0800 +++ b/roles/hgweb/templates/etc/nginx/sites-available/hgweb Sat Jun 09 15:14:44 2018 +0800 @@ -32,6 +32,10 @@ expires 30d; } + location = /robots.txt { + alias /home/{{ hgweb_user }}/hgwebfiles/robots.txt; + } + location = /favicon.ico { return 301 /static/hgicon.png; }