# The Common Crawl dataset. Original source for GPT and others.
User-agent: CCBot
Disallow: /

# The example for img2dataset, although the default is *None*
User-agent: img2dataset
Disallow: /

# GPTBot is OpenAI's web crawler
User-agent: GPTBot
Disallow: /

# Google's Bard and Vertex AI generative APIs
User-agent: Google-Extended
Disallow: /