From 8b283f17c78b6c24eea50eb003ca39e69f6ac88f Mon Sep 17 00:00:00 2001 From: Nikita Shulga Date: Wed, 3 Nov 2021 10:26:01 -0700 Subject: [PATCH] Add older docs to exclusion list on robots.txt All HTML files in those folders are already covered by noindex, but not .txt and others --- robots.txt | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/robots.txt b/robots.txt index 6af339057a77..e27e679df092 100644 --- a/robots.txt +++ b/robots.txt @@ -1,9 +1,16 @@ User-agent: * Disallow: /docs/0.1.12/ Disallow: /docs/0.2.0/ -Disallow: /docs/0.3.0/ -Disallow: /docs/0.3.1/ -Disallow: /docs/0.4.0/ -Disallow: /docs/0.4.1/ -Disallow: /docs/1.0.0/ -Disallow: /docs/1.0.1/ +Disallow: /docs/0.3.* +Disallow: /docs/0.4.* +Disallow: /docs/1.0.* +Disallow: /docs/1.1.0/ +Disallow: /docs/1.2.0/ +Disallow: /docs/1.3.* +Disallow: /docs/1.4.0/ +Disallow: /docs/1.5.* +Disallow: /docs/1.6.0/ +Disallow: /docs/1.7.* +Disallow: /docs/1.8.* +Disallow: /docs/1.9.* +Disallow: /docs/master/