User-agent: * Disallow: /Search Disallow: /ml # Throttle this bozo search engine User-agent: Slurp Crawl-delay: 10 # ...and another User-agent: ScoutJet Crawl-delay: 10 # Robota non grata User-Agent: CCBot User-agent: MJ12bot User-agent: Mail.RU_Bot User-agent: Mail.RU_Bot/2.0 User-agent: MegaIndex User-agent: MegaIndex.ru User-agent: trendkite-akashic-crawler User-agent: Jooblebot User-agent: HTTrack User-agent: yacybot User-agent: PetalBot User-agent: GPTBot User-agent: SemrushBot User-agent: AhrefsBot User-agent: Qwantbot User-agent: meta-externalagent User-agent: ChatGLM-Spider Disallow: /