# store.procure.ae robots.txt: bot access policy (29 Oct 2026). # Search engines, AI search bots and the allowed AI training bots (GPTBot, ClaudeBot, Applebot-Extended) # follow the "*" group. # Automated bulk copying, scraping or dataset building is not permitted without written permission. # Terms: https://store.procure.ae/page/terms-of-use (Automated Access, Copying and AI Use) User-agent: * Content-Signal: search=yes, ai-input=yes, ai-train=yes Disallow: /api/ Disallow: /catalog-export-feed Allow: / # AdsBot ignores the "*" group, so it must be named explicitly. # Collection product listings and the menu stay open so ad landing # pages still render their products for quality checks. User-agent: AdsBot-Google User-agent: AdsBot-Google-Mobile Disallow: /api/ Disallow: /catalog-export-feed Allow: /api/products?collection_id= Allow: /api/storefront/menu Allow: / # AI assistants fetching pages for a person: welcome, but no bulk reading (about 20 pages an hour). User-agent: Claude-User User-agent: ChatGPT-User User-agent: Perplexity-User Crawl-delay: 180 Disallow: /api/ Disallow: /catalog-export-feed Allow: / # SEO-research crawlers, dataset builders and scraping tools: not allowed. User-agent: AhrefsBot User-agent: AhrefsSiteAudit User-agent: SemrushBot User-agent: SemrushBot-SA User-agent: SiteAuditBot User-agent: MJ12bot User-agent: DotBot User-agent: BLEXBot User-agent: serpstatbot User-agent: barkrowler User-agent: SeznamBot User-agent: CCBot User-agent: Bytespider User-agent: Meta-ExternalAgent User-agent: Amazonbot User-agent: Diffbot User-agent: ImagesiftBot User-agent: Omgilibot User-agent: omgili User-agent: img2dataset User-agent: cohere-training-data-crawler User-agent: Timpibot User-agent: ia_archiver User-agent: archive.org_bot User-agent: Scrapy User-agent: Wget User-agent: WebCopy User-agent: HTTrack User-agent: Nutch Disallow: / Sitemap: https://store.procure.ae/sitemap.xml