Add CCBot to bot exceptions

src/_data/robots.js

@@ -1,28 +1,28 @@
-// by Robb Knight: https://rknight.me/blog/blocking-bots-with-nginx/
+/*
+	Modified from Robb Knight's script:
+	https://rknight.me/blog/blocking-bots-with-nginx/
+*/
 
 import EleventyFetch from "@11ty/eleventy-fetch";
 
 export default async function () {
-	const url =
+	const url = "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt";
-		"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt";
 	let txt = await EleventyFetch(url, {
 		duration: "1w",
 		type: "text",
 	});
 
+	const botExceptions = ["Applebot", "CCBot"];
+	const botExceptionsFullStr = botExceptions.map(bot => "User-agent: " + bot)
+
 	txt = txt
 		.split("\n")
-		.filter((line) => line !== "User-agent: Applebot")
+		.filter((line) => !botExceptionsFullStr.includes(line))
 		.join("\n");
 
 	const bots = txt
 		.split("\n")
-		.filter((line) => {
+		.filter((line) => line.startsWith("User-agent:"))
-			return (
-				line.startsWith("User-agent:") &&
-				line !== "User-agent: Applebot"
-			);
-		})
 		.map((line) => line.split(":")[1].trim().replace(/\s/gi, ".*"));
 
 	return {