exclude new ai bots in robot.txt, removed xing, reworked imprint with tdm, remove yt, threads & instragram from dp, added mastodon & pixelfed to dp, added license text, changed LICENSE, new copyright footer

docs/robots.txt

@@ -1,35 +1,38 @@
-Sitemap: https://stephan.hadan.de/sitemap.xml
-User-agent: GPTBot
-Disallow: /
-User-agent: ChatGPT-User
-Disallow: /
-User-agent: Google-Extended
-Disallow: /
-User-agent: PerplexityBot
-Disallow: /
-User-agent: Amazonbot
-Disallow: /
-User-agent: ClaudeBot
-Disallow: /
-User-agent: Omgilibot
-Disallow: /
-User-Agent: FacebookBot
-Disallow: /
-User-Agent: Applebot
-Disallow: /
+# Humans and normal search engines are allowed to read/index
+User-agent: *
+Disallow: /cv/
+
+# But we don't feed into the AI/ML hype here. Stop wasting the planet's resources.
+User-agent: CCbot
 User-agent: anthropic-ai
-Disallow: /
-User-agent: Bytespider
-Disallow: /
 User-agent: Claude-Web
-Disallow: /
-User-agent: Diffbot
-Disallow: /
-User-agent: ImagesiftBot
-Disallow: /
+User-agent: ClaudeBot
+User-agent: FacebookBot
+User-agent: Meta-ExternalFetcher
+User-agent: Meta-ExternalAgent
+User-agent: Google-Extended
+User-agent: GPTBot
+User-agent: ChatGPT-User
+User-agent: PiplBot
+User-agent: PerplexityBot
+User-agent: Omgilibot
+User-Agent: Applebot
+User-agent: Applebot-Extended
+User-agent: Amazonbot
+User-agent: Bytespider
+User-agent: Diffbot
+User-agent: ImagesiftBot
 User-agent: Omgilibot
-Disallow: /
 User-agent: Omgili
-Disallow: /
 User-agent: YouBot
+User-agent: Ai2Bot
+User-agent: Ai2Bot-Dolma
+User-agent: FriendlyCrawler
+User-agent: Scrapy
+User-agent: Timpibot
+User-agent: PetalBot
+User-agent: img2dataset
+User-agent: AhrefsBot
 Disallow: /
+
+Sitemap: https://stephan.hadan.de/sitemap.xml