Sitemap: https://reclaimdesign.org/sitemap_index.xml # ============================================================================= # LEGITIMATE SEARCH ENGINES - ALLOWED # ============================================================================= # Google crawlers User-agent: GoogleBot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-News Allow: / User-agent: Googlebot-Video Allow: / User-agent: Google-InspectionTool Allow: / User-agent: Storebot-Google Allow: / User-agent: AdsBot-Google Allow: / User-agent: Feedfetcher-Google Allow: / User-agent: Google-Read-Aloud Allow: / User-agent: Google Favicon Allow: / # Bing / Microsoft User-agent: Bingbot Allow: / # Other legitimate search engines User-agent: DuckDuckBot Allow: / User-agent: YandexBot Allow: / User-agent: Slurp Allow: / # Social media crawlers for link previews User-agent: facebookexternalhit Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / # ============================================================================= # AI SEARCH BOTS - ALLOWED (for discovery and citation) # ============================================================================= User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / User-agent: Claude-Web Allow: / User-agent: PerplexityBot Allow: / User-agent: YouBot Allow: / User-agent: PhindBot Allow: / User-agent: ExaBot Allow: / User-agent: Gemini-Deep-Research Allow: / User-agent: AppleBot Allow: / User-agent: DuckAssistant Allow: / # ============================================================================= # AI TRAINING BOTS - BLOCKED (no benefit, just scrape content) # ============================================================================= User-agent: GPTBot Disallow: / User-agent: CCBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: FacebookBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Applebot-Extended Disallow: / User-agent: Google-Extended Disallow: / User-agent: anthropic-ai Disallow: / User-agent: cohere-ai Disallow: / User-agent: Diffbot Disallow: / User-agent: omgilibot Disallow: / User-agent: Omgili Disallow: / User-agent: Timpibot Disallow: / User-agent: ImagesiftBot Disallow: / User-agent: Image2dataset Disallow: / User-agent: img2dataset Disallow: / User-agent: FriendlyCrawler Disallow: / User-agent: DeepseekBot Disallow: / User-agent: PanguBot Disallow: / User-agent: ErnieBot Disallow: / User-agent: QwenBot Disallow: / # ============================================================================= # SEO TOOL BOTS - BLOCKED (data harvesting, no benefit) # ============================================================================= User-agent: SemrushBot Disallow: / User-agent: SEMrushBot Disallow: / User-agent: SemrushBot-SA Disallow: / User-agent: SemrushBot-BA Disallow: / User-agent: SemrushBot-SI Disallow: / User-agent: SemrushBot-SWA Disallow: / User-agent: SemrushBot-OCOB Disallow: / User-agent: SplitSignalBot Disallow: / User-agent: SiteAuditBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: ahrefsbot Disallow: / User-agent: AhrefsSiteAudit Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: dotbot Disallow: / User-agent: Rogerbot Disallow: / User-agent: rogerbot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: Barkrowler Disallow: / User-agent: barkrowler Disallow: / User-agent: serpstatbot Disallow: / User-agent: SEOkicks Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: linkdexbot Disallow: / User-agent: SISTRIX Disallow: / User-agent: SearchmetricsBot Disallow: / # ============================================================================= # CONTENT SCRAPERS & DOWNLOADERS - BLOCKED # ============================================================================= User-agent: HTTrack Disallow: / User-agent: Wget Disallow: / User-agent: wget Disallow: / User-agent: Scrapy Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: WebCopier Disallow: / User-agent: WebReaper Disallow: / User-agent: WebSauger Disallow: / User-agent: WebStripper Disallow: / User-agent: CherryPicker Disallow: / User-agent: python-requests Disallow: / User-agent: Go-http-client Disallow: / User-agent: news-please Disallow: / User-agent: magpie-crawler Disallow: / User-agent: MicroMessenger Disallow: / # ============================================================================= # FOREIGN SEARCH ENGINES - BLOCKED (not targeting these markets) # ============================================================================= User-agent: Baiduspider Disallow: / User-agent: Baidu Disallow: / User-agent: Sogou Disallow: / User-agent: PetalBot Disallow: / User-agent: 360Spider Disallow: / User-agent: ChatGLM-Spider Disallow: / User-agent: HaosouSpider Disallow: / User-agent: YisouSpider Disallow: / # ============================================================================= # BUSINESS INTELLIGENCE / MONITORING BOTS - BLOCKED # ============================================================================= User-agent: AwarioBot Disallow: / User-agent: AwarioSmartBot/1.0 Disallow: / User-agent: Zoominfobot Disallow: / User-agent: ZoominfoBot Disallow: / User-agent: peer39_crawler Disallow: / User-agent: Dataprovider Disallow: / User-agent: dataprovider Disallow: / User-agent: BW/1.1 Disallow: / # ============================================================================= # AGGRESSIVE / MALICIOUS BOTS - BLOCKED # ============================================================================= User-agent: grub Disallow: / User-agent: Zeus Disallow: / User-agent: Orbbot Disallow: / User-agent: VelenPublicWebCrawler Disallow: / User-agent: NPBot Disallow: / User-agent: NPbot Disallow: / User-agent: 008 Disallow: / User-agent: Amazonbot Disallow: / # ============================================================================= # DEFAULT RULES FOR ALL OTHER BOTS # ============================================================================= User-agent: * # Allow important files Allow: /sitemap_index.xml Allow: /sitemap Allow: /llms.txt Allow: /ai.txt # Allow uploads (images, media) Allow: /wp-content/uploads/ # Allow admin-ajax for functionality Allow: /wp-admin/admin-ajax.php # Block WordPress admin and system directories Disallow: /wp-admin/ Disallow: /wp-includes/ Disallow: /wp-content/uploads/wpallimport/ Disallow: /wp-content/uploads/sucuri/ Disallow: /wp-content/uploads/ao_ccss/ # Block system files Disallow: /license.txt Disallow: /xmlrpc.php Disallow: /readme.html # Block plugin-specific paths Disallow: /*blackhole Disallow: /?blackhole Disallow: /*?reviews-page=* Disallow: /*?*reviews-page=* # Block e-commerce checkout and private pages Disallow: /home-decor-shop/checkout/ Disallow: /home-decor-shop#!/p/ Disallow: /home-decor-shop/signIn Disallow: /payment-completed # Block search and archive pages Disallow: /?s= Disallow: /search/ Disallow: /sustainable-blog/previous/ Disallow: /sustainable-blog/archives/ Disallow: /sustainable-blog/category/ # Block campaign and tracking URLs Disallow: /refer/ Disallow: /*?utm_content=buffer* Disallow: /*?ao_noptimize=1