diff --git a/.htaccess b/.htaccess index 5cf35de..845c687 100644 --- a/.htaccess +++ b/.htaccess @@ -38,23 +38,25 @@ RewriteCond %{HTTP_USER_AGENT} (Google-Extended) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (GoogleOther) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (GoogleOther-Image) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (GoogleOther-Video) [NC,OR] -RewriteCond %{HTTP_USER_AGENT} (iaskspider/2.0) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (ICC-Crawler) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (ImagesiftBot) [NC,OR] +RewriteCond %{HTTP_USER_AGENT} (Kangaroo Bot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (Meta-ExternalAgent) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (Meta-ExternalFetcher) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (OAI-SearchBot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (PerplexityBot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (PetalBot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (Scrapy) [NC,OR] +RewriteCond %{HTTP_USER_AGENT} (Sidetrade indexer bot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (Timpibot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (VelenPublicWebCrawler) [NC,OR] +RewriteCond %{HTTP_USER_AGENT} (Webzio-Extended) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (YouBot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (anthropic-ai) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (cohere-ai) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (facebookexternalhit) [NC,OR] +RewriteCond %{HTTP_USER_AGENT} (iaskspider/2.0) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (img2dataset) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (omgili) [NC,OR] -RewriteCond %{HTTP_USER_AGENT} (omgilibot) [NC,OR] -RewriteCond %{HTTP_USER_AGENT} (Webzio-Extended) [NC] +RewriteCond %{HTTP_USER_AGENT} (omgilibot) [NC] RewriteRule .* https://nocommercialuse.org/ [L] \ No newline at end of file diff --git a/robots.txt b/robots.txt index fecdebe..21abf82 100644 --- a/robots.txt +++ b/robots.txt @@ -18,15 +18,16 @@ User-agent: Google-Extended User-agent: GoogleOther User-agent: GoogleOther-Image User-agent: GoogleOther-Video -User-agent: iaskspider/2.0 User-agent: ICC-Crawler User-agent: ImagesiftBot +User-agent: Kangaroo Bot User-agent: Meta-ExternalAgent User-agent: Meta-ExternalFetcher User-agent: OAI-SearchBot User-agent: PerplexityBot User-agent: PetalBot User-agent: Scrapy +User-agent: Sidetrade indexer bot User-agent: Timpibot User-agent: VelenPublicWebCrawler User-agent: Webzio-Extended @@ -34,6 +35,7 @@ User-agent: YouBot User-agent: anthropic-ai User-agent: cohere-ai User-agent: facebookexternalhit +User-agent: iaskspider/2.0 User-agent: img2dataset User-agent: omgili User-agent: omgilibot