diff --git a/internal/web/robots.go b/internal/web/robots.go
index 4945aa9fc..1e084d6a2 100644
--- a/internal/web/robots.go
+++ b/internal/web/robots.go
@@ -33,9 +33,9 @@
 # https://github.com/ai-robots-txt/ai.robots.txt/
 User-agent: AI2Bot
 User-agent: Ai2Bot-Dolma
-User-agent: AdsBot-Google
 User-agent: Amazonbot
 User-agent: anthropic-ai
+User-agent: Applebot
 User-agent: Applebot-Extended
 User-agent: Bytespider
 User-agent: CCBot
@@ -43,9 +43,10 @@
 User-agent: ClaudeBot
 User-agent: Claude-Web
 User-agent: cohere-ai
+User-agent: cohere-training-data-crawler
 User-agent: Diffbot
+User-agent: DuckAssistBot
 User-agent: FacebookBot
-User-agent: facebookexternalhit
 User-agent: FriendlyCrawler
 User-agent: Google-Extended
 User-agent: GoogleOther
@@ -56,14 +57,18 @@
 User-agent: ICC-Crawler
 User-agent: ImagesiftBot
 User-agent: img2dataset
+User-agent: ISSCyberRiskCrawler
+User-agent: Kangaroo Bot
 User-agent: Meta-ExternalAgent
 User-agent: Meta-ExternalFetcher
 User-agent: OAI-SearchBot
 User-agent: omgili
 User-agent: omgilibot
+User-agent: PanguBot
 User-agent: PerplexityBot
 User-agent: PetalBot
 User-agent: Scrapy
+User-agent: Sidetrade indexer bot
 User-agent: Timpibot
 User-agent: VelenPublicWebCrawler
 User-agent: Webzio-Extended