Sitemap: https://kbland.kr/sitemap_index.xml

# Block everything by default
User-agent: *
Disallow: /

# Allow all Google crawlers
User-agent: Googlebot
Disallow:

# Allow all Bing crawlers
User-agent: Bingbot
Disallow:
Crawl-delay: 30

User-agent: MSNBot
Disallow:
Crawl-delay: 30

# Allow all Naver crawlers
User-agent: Yeti
Disallow:
Allow: /

# Allow OpenAI crawlers (ChatGPT search/training)
User-agent: GPTBot
Disallow:

User-agent: ChatGPT-User
Disallow:

User-agent: OAI-SearchBot
Disallow:

# Allow Google AI training/citation crawler (separate from Googlebot)
User-agent: Google-Extended
Disallow:

# Allow Perplexity crawlers
User-agent: PerplexityBot
Disallow:

User-agent: Perplexity-User
Disallow:

# Allow Anthropic crawlers
User-agent: ClaudeBot
Disallow:

User-agent: Claude-User
Disallow:

# Allow Apple Intelligence crawler (separate from Applebot)
User-agent: Applebot-Extended
Disallow:

# Allow Amazon crawler
User-agent: Amazonbot
Disallow:

# Allow Meta AI crawler
User-agent: meta-externalagent
Disallow: