# ==========================================================

# ============================================================ # LinkdSys robots.txt # Updated: 2026-03-16 # ============================================================ # Standard crawlers User-agent: * Allow: / Allow: /assets/ Allow: /sitemap.xml Disallow: /admin/ Disallow: /dashboard/ Disallow: /api/ Disallow: /oauth/ Disallow: /config/ Disallow: /cron/ Disallow: /install/ Disallow: /scripts/ Disallow: /includes/ Disallow: /logs/ Disallow: /vendor/ Disallow: /*.json$ # Google User-agent: Googlebot Crawl-delay: 0 Allow: / Disallow: /dashboard/ Disallow: /admin/ # Disallow Google AI training (allow discovery, block training corpus) User-agent: Google-Extended Disallow: / # Bing User-agent: bingbot Crawl-delay: 1 Allow: / Disallow: /dashboard/ Disallow: /admin/ # AI answer engines (allow crawling for answers & citations) User-agent: GPTBot Allow: / Disallow: /dashboard/ Disallow: /admin/ User-agent: ChatGPT-User Allow: / Disallow: /dashboard/ Disallow: /admin/ User-agent: ClaudeBot Allow: / Disallow: /dashboard/ Disallow: /admin/ User-agent: PerplexityBot Allow: / Disallow: /dashboard/ Disallow: /admin/ User-agent: YouBot Allow: / Disallow: /dashboard/ Disallow: /admin/ User-agent: cohere-ai Allow: / Disallow: /dashboard/ Disallow: /admin/ # Apple AI training User-agent: Applebot-Extended Disallow: / # Common Crawl / bulk training datasets User-agent: CCBot Disallow: / # Rate limiting for SEO crawlers User-agent: AhrefsBot Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 10 User-agent: MJ12bot Disallow: / User-agent: Baiduspider Crawl-delay: 2 User-agent: AdsBot-Google Crawl-delay: 1 # Sitemaps Sitemap: https://linkdsys.com/sitemap.xml Sitemap: https://linkdsys.com/sitemap-pages.xml