Download:
child 31:65daf0ee01ec
parent 29:0dca68b49bc5
30:0e84a4a22acd
Anton Shestakov <av6@dwimlabs.net>, Sat, 09 Jun 2018 15:14:44 +0800
robots: disallow some non-essential pages

3 файлов изменено, 36 вставок(+), 0 удалений(-) [+]
roles/hgweb/files/robots.txt file | annotate | diff | comparison | revisions
roles/hgweb/tasks/appinstall.yml file | annotate | diff | comparison | revisions
roles/hgweb/templates/etc/nginx/sites-available/hgweb file | annotate | diff | comparison | revisions
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/roles/hgweb/files/robots.txt Sat Jun 09 15:14:44 2018 +0800
@@ -0,0 +1,25 @@
+User-agent: *
+
+# Expensive pages
+Disallow: /*/annotate/
+Disallow: /*/comparison/
+Disallow: /*/diff/
+Disallow: /*/archive/
+
+# File log is too much
+Disallow: /*/log/*/
+
+# API is off-limits
+Disallow: /*/json-*
+Disallow: /*/raw-*
+
+# Paginate instead
+Disallow: /*?revcount=
+
+# Graph page is a more expensive log
+Disallow: /*/graph$
+Disallow: /*/graph/
+
+# It's for humans
+Disallow: /*/help$
+Disallow: /*/help/
--- a/roles/hgweb/tasks/appinstall.yml Tue Jan 02 19:27:00 2018 +0800
+++ b/roles/hgweb/tasks/appinstall.yml Sat Jun 09 15:14:44 2018 +0800
@@ -37,6 +37,13 @@
notify:
- restart hgweb
+- name: Copy robots.txt
+ copy:
+ src: robots.txt
+ dest: /home/{{ hgweb_user }}/hgwebfiles/robots.txt
+ mode: 0644
+ tags: [robots]
+
- name: Copy requirements.txt
copy:
src: requirements.txt
--- a/roles/hgweb/templates/etc/nginx/sites-available/hgweb Tue Jan 02 19:27:00 2018 +0800
+++ b/roles/hgweb/templates/etc/nginx/sites-available/hgweb Sat Jun 09 15:14:44 2018 +0800
@@ -32,6 +32,10 @@
expires 30d;
}
+ location = /robots.txt {
+ alias /home/{{ hgweb_user }}/hgwebfiles/robots.txt;
+ }
+
location = /favicon.ico {
return 301 /static/hgicon.png;
}