From b08bcd7a89ad29cdb7521926fdb506abac3d74b3 Mon Sep 17 00:00:00 2001
From: Helen Chong <119173961+helenclx@users.noreply.github.com>
Date: Fri, 30 Aug 2024 13:52:46 +0800
Subject: [PATCH] Add AI2Bot and Ai2Bot-Dolma

---
 .htaccess  | 2 ++
 robots.txt | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/.htaccess b/.htaccess
index 0896be4..bdccf72 100644
--- a/.htaccess
+++ b/.htaccess
@@ -21,6 +21,8 @@ RewriteRule ^.+$ index.php [L]
 
 # Block bad bots
 RewriteEngine On
+RewriteCond %{HTTP_USER_AGENT} (AI2Bot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Ai2Bot-Dolma) [NC,OR]
 RewriteCond %{HTTP_USER_AGENT} (Amazonbot) [NC,OR]
 RewriteCond %{HTTP_USER_AGENT} (Applebot-Extended) [NC,OR]
 RewriteCond %{HTTP_USER_AGENT} (Bytespider) [NC,OR]
diff --git a/robots.txt b/robots.txt
index 90e789d..0b76314 100644
--- a/robots.txt
+++ b/robots.txt
@@ -1,6 +1,8 @@
 User-agent: *
 Disallow:
 
+User-agent: AI2Bot
+User-agent: Ai2Bot-Dolma
 User-agent: Amazonbot
 User-agent: Applebot-Extended
 User-agent: Bytespider
@@ -26,6 +28,7 @@ User-agent: PetalBot
 User-agent: Scrapy
 User-agent: Timpibot
 User-agent: VelenPublicWebCrawler
+User-agent: Webzio-Extended
 User-agent: YouBot
 User-agent: anthropic-ai
 User-agent: cohere-ai
@@ -33,5 +36,4 @@ User-agent: facebookexternalhit
 User-agent: img2dataset
 User-agent: omgili
 User-agent: omgilibot
-User-agent: Webzio-Extended
 Disallow: /
\ No newline at end of file