Leilukin
/
leilukin-tumbleblog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update list of bad bots to block

This commit is contained in:
Helen Chong 2024-09-29 09:55:21 +08:00
parent a5fc743bee
commit a404ac8093
2 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.htaccess
View File

@ -38,23 +38,25 @@ RewriteCond %{HTTP_USER_AGENT} (Google-Extended) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (GoogleOther) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (GoogleOther) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (GoogleOther-Image) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (GoogleOther-Image) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (GoogleOther-Video) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (GoogleOther-Video) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (iaskspider/2.0) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (ICC-Crawler) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (ICC-Crawler) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (ImagesiftBot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (ImagesiftBot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Kangaroo Bot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Meta-ExternalAgent) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (Meta-ExternalAgent) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Meta-ExternalFetcher) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (Meta-ExternalFetcher) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (OAI-SearchBot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (OAI-SearchBot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (PerplexityBot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (PerplexityBot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (PetalBot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (PetalBot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Scrapy) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (Scrapy) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Sidetrade indexer bot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Timpibot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (Timpibot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (VelenPublicWebCrawler) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (VelenPublicWebCrawler) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (Webzio-Extended) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (YouBot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (YouBot) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (anthropic-ai) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (anthropic-ai) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (cohere-ai) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (cohere-ai) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (facebookexternalhit) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (facebookexternalhit) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (iaskspider/2.0) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (img2dataset) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (img2dataset) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (omgili) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (omgili) [NC,OR]
RewriteCond %{HTTP_USER_AGENT} (omgilibot) [NC,OR] RewriteCond %{HTTP_USER_AGENT} (omgilibot) [NC]
RewriteCond %{HTTP_USER_AGENT} (Webzio-Extended) [NC]
RewriteRule .* https://nocommercialuse.org/ [L] RewriteRule .* https://nocommercialuse.org/ [L]

4
robots.txt
View File

@ -18,15 +18,16 @@ User-agent: Google-Extended
User-agent: GoogleOther User-agent: GoogleOther
User-agent: GoogleOther-Image User-agent: GoogleOther-Image
User-agent: GoogleOther-Video User-agent: GoogleOther-Video
User-agent: iaskspider/2.0
User-agent: ICC-Crawler User-agent: ICC-Crawler
User-agent: ImagesiftBot User-agent: ImagesiftBot
User-agent: Kangaroo Bot
User-agent: Meta-ExternalAgent User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher User-agent: Meta-ExternalFetcher
User-agent: OAI-SearchBot User-agent: OAI-SearchBot
User-agent: PerplexityBot User-agent: PerplexityBot
User-agent: PetalBot User-agent: PetalBot
User-agent: Scrapy User-agent: Scrapy
User-agent: Sidetrade indexer bot
User-agent: Timpibot User-agent: Timpibot
User-agent: VelenPublicWebCrawler User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended User-agent: Webzio-Extended
@ -34,6 +35,7 @@ User-agent: YouBot
User-agent: anthropic-ai User-agent: anthropic-ai
User-agent: cohere-ai User-agent: cohere-ai
User-agent: facebookexternalhit User-agent: facebookexternalhit
User-agent: iaskspider/2.0
User-agent: img2dataset User-agent: img2dataset
User-agent: omgili User-agent: omgili
User-agent: omgilibot User-agent: omgilibot