package.json

@@ -10,7 +10,6 @@
   },
   "dependencies": {
     "@11ty/eleventy": "^3.0.0-beta.1",
-    "@11ty/eleventy-fetch": "^4.0.1",
     "@11ty/eleventy-navigation": "^0.3.5",
     "@11ty/eleventy-plugin-rss": "^2.0.2",
     "@11ty/eleventy-plugin-syntaxhighlight": "^5.0.0",

src/.htaccess.njk

@@ -14,5 +14,37 @@ Header set Cache-Control "no-cache, public"
 RewriteEngine on
 
 # Block bad bots
-RewriteCond %{HTTP_USER_AGENT} {{ robots.htaccess }} [NC]
+RewriteCond %{HTTP_USER_AGENT} (Amazonbot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Applebot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Applebot-Extended) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Bytespider) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (CCBot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (ChatGPT-User) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Claude-Web) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (ClaudeBot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Diffbot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (FacebookBot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (FriendlyCrawler) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (GPTBot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Google-Extended) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (GoogleOther) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (GoogleOther-Image) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (GoogleOther-Video) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (ICC-Crawler) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (ImagesiftBot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Meta-ExternalAgent) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Meta-ExternalFetcher) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (OAI-SearchBot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (PerplexityBot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (PetalBot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Scrapy) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (Timpibot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (VelenPublicWebCrawler) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (YouBot) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (anthropic-ai) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (cohere-ai) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (facebookexternalhit) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (img2dataset) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (omgili) [NC,OR]
+RewriteCond %{HTTP_USER_AGENT} (omgilibot) [NC]
 RewriteRule .* https://nocommercialuse.org/ [L]

src/_data/robots.js

@@ -1,32 +0,0 @@
-// by Robb Knight: https://rknight.me/blog/blocking-bots-with-nginx/
-
-import EleventyFetch from "@11ty/eleventy-fetch";
-
-export default async function () {
-	const url =
-		"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt";
-	let txt = await EleventyFetch(url, {
-		duration: "1w",
-		type: "text",
-	});
-
-	txt = txt
-		.split("\n")
-		.filter((line) => line !== "User-agent: Applebot")
-		.join("\n");
-
-	const bots = txt
-		.split("\n")
-		.filter((line) => {
-			return (
-				line.startsWith("User-agent:") &&
-				line !== "User-agent: Applebot"
-			);
-		})
-		.map((line) => line.split(":")[1].trim());
-
-	return {
-		txt: txt,
-		htaccess: bots.join("|"),
-	};
-}

src/robots.txt.njk

@@ -1,10 +1,43 @@
 ---
-permalink: robots.txt
+permalink: /robots.txt
 eleventyExcludeFromCollections: true
 ---
+Sitemap: {{ sitemeta.siteUrl }}/sitemap/
+
 User-agent: *
 Disallow:
 
-{{ robots.txt }}
+User-agent: Amazonbot
-
+User-agent: Applebot
-Sitemap: {{ sitemeta.siteUrl }}/sitemap/
+User-agent: Applebot-Extended
+User-agent: Bytespider
+User-agent: CCBot
+User-agent: ChatGPT-User
+User-agent: Claude-Web
+User-agent: ClaudeBot
+User-agent: Diffbot
+User-agent: FacebookBot
+User-agent: FriendlyCrawler
+User-agent: GPTBot
+User-agent: Google-Extended
+User-agent: GoogleOther
+User-agent: GoogleOther-Image
+User-agent: GoogleOther-Video
+User-agent: ICC-Crawler
+User-agent: ImagesiftBot
+User-agent: Meta-ExternalAgent
+User-agent: Meta-ExternalFetcher
+User-agent: OAI-SearchBot
+User-agent: PerplexityBot
+User-agent: PetalBot
+User-agent: Scrapy
+User-agent: Timpibot
+User-agent: VelenPublicWebCrawler
+User-agent: YouBot
+User-agent: anthropic-ai
+User-agent: cohere-ai
+User-agent: facebookexternalhit
+User-agent: img2dataset
+User-agent: omgili
+User-agent: omgilibot
+Disallow: /

src/statements/colophon.md

@@ -1,7 +1,7 @@
 ---
 title: Colophon
 keyword: colophon page
-updated: 2024-08-20T09:55:39+0800
+updated: 2024-08-02
 toc: true
 eleventyNavigation:
     order: 14
@@ -14,7 +14,6 @@ Information about how this website is built.
 * Templating languages: Nunjucks and Markdown
 * Code written with Visual Studio Code
 * JavaScript runtime: [Bun](https://bun.sh)
-* Tested browsers: Mozilla Firefox and Lynx
 * Source code available on [GitHub](https://github.com/helenclx/leilukin-site) and its [Gitea mirror](https://git.32bit.cafe/Leilukin/leilukin-site)
 
 ## Host and Domain
@@ -27,7 +26,6 @@ Previously, this website was hosted on [Neocities](https://neocities.org/) until
 * Eleventy's official [RSS](https://www.11ty.dev/docs/plugins/rss/) plugin
 * Eleventy's official [Navigation](https://www.11ty.dev/docs/plugins/navigation/) plugin
 * Eleventy's official [Syntax Highlighting](https://www.11ty.dev/docs/plugins/syntaxhighlight/) plugin
-* Eleventy's official [Fetch](https://www.11ty.dev/docs/plugins/fetch/) plugin
 * [@zachleat/details-utils](https://www.npmjs.com/package/@zachleat/details-utils)
 
 ### Eleventy Community Plugins