--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/roles/hgweb/files/robots.txt	Sat Jun 09 15:14:44 2018 +0800
@@ -0,0 +1,25 @@
+User-agent: *
+
+# Expensive pages
+Disallow: /*/annotate/
+Disallow: /*/comparison/
+Disallow: /*/diff/
+Disallow: /*/archive/
+
+# File log is too much
+Disallow: /*/log/*/
+
+# API is off-limits
+Disallow: /*/json-*
+Disallow: /*/raw-*
+
+# Paginate instead
+Disallow: /*?revcount=
+
+# Graph page is a more expensive log
+Disallow: /*/graph$
+Disallow: /*/graph/
+
+# It's for humans
+Disallow: /*/help$
+Disallow: /*/help/

--- a/roles/hgweb/tasks/appinstall.yml	Tue Jan 02 19:27:00 2018 +0800
+++ b/roles/hgweb/tasks/appinstall.yml	Sat Jun 09 15:14:44 2018 +0800
@@ -37,6 +37,13 @@
   notify:
     - restart hgweb
 
+- name: Copy robots.txt
+  copy:
+    src: robots.txt
+    dest: /home/{{ hgweb_user }}/hgwebfiles/robots.txt
+    mode: 0644
+  tags: [robots]
+
 - name: Copy requirements.txt
   copy:
     src: requirements.txt

--- a/roles/hgweb/templates/etc/nginx/sites-available/hgweb	Tue Jan 02 19:27:00 2018 +0800
+++ b/roles/hgweb/templates/etc/nginx/sites-available/hgweb	Sat Jun 09 15:14:44 2018 +0800
@@ -32,6 +32,10 @@
         expires 30d;
     }
 
+    location = /robots.txt {
+        alias /home/{{ hgweb_user }}/hgwebfiles/robots.txt;
+    }
+
     location = /favicon.ico {
         return 301 /static/hgicon.png;
     }