From 6300906b8141a840ff04cef6192721e8df876381 Mon Sep 17 00:00:00 2001 From: Helen Chong <119173961+helenclx@users.noreply.github.com> Date: Tue, 20 Aug 2024 08:38:24 +0800 Subject: [PATCH] Remove default crawler block --- src/robots.txt.njk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/robots.txt.njk b/src/robots.txt.njk index e6f49675..267012fd 100644 --- a/src/robots.txt.njk +++ b/src/robots.txt.njk @@ -5,7 +5,7 @@ eleventyExcludeFromCollections: true Sitemap: {{ sitemeta.siteUrl }}/sitemap/ User-agent: * -Disallow: / +Disallow: User-agent: Amazonbot User-agent: Applebot