From 9a52eca1a5790a63d0e25cba5410553d99f4140c Mon Sep 17 00:00:00 2001 From: BinaryDigit Date: Mon, 29 Jan 2024 09:56:23 -0500 Subject: [PATCH] update robots txt --- static/robots.txt | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/static/robots.txt b/static/robots.txt index e8806f2..e9e32c3 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -6,6 +6,12 @@ User-agent: anthropic-ai Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/bytespider + +User-agent: Bytespider +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/ccbot @@ -31,18 +37,9 @@ User-agent: GPTBot Disallow: / # AI Data Scraper -# https://darkvisitors.com/agents/omgilibot - -User-agent: omgilibot -Disallow: / +# https://darkvisitors.com/agents/omgili User-agent: omgili Disallow: / -User-agent: ChatGPT-User -Disallow: / - -User-agent: AdsBot-Google -Disallow: / - Sitemap: https://kilobytekeith.com/sitemap.xml