# Two prefix-match traps are handled below, both by the same technique: the
# longer, more specific rule wins, so a narrow Allow overrides a broad Disallow.
#
# 1. "Disallow: /record" also matches the /recorder marketing page for the iOS
#    app, so each group carries an explicit "Allow: /recorder".
# 2. "Disallow: /api/" also matches /api/og, which generates every page's
#    social share card. Crawlers that honour robots.txt (Googlebot-Image,
#    facebookexternalhit, Twitterbot, LinkedInBot, Slackbot) would refuse to
#    fetch the card and render a bare text link, so each group carries an
#    explicit "Allow: /api/og". Every other /api/ route stays blocked.

# Default rules for all crawlers
User-agent: *
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Allow: /recorder
Disallow: /record
Allow: /api/og
Disallow: /api/
Disallow: /auth/
Disallow: /forgot-password
Disallow: /_next/
Disallow: /*.json$
Crawl-delay: 1

# Specific rules for Google - more permissive
User-agent: Googlebot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Allow: /recorder
Disallow: /record
Allow: /api/og
Disallow: /api/
Disallow: /auth/
Disallow: /forgot-password

# Specific rules for Bing
User-agent: Bingbot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Allow: /recorder
Disallow: /record
Allow: /api/og
Disallow: /api/
Disallow: /auth/
Disallow: /forgot-password
Crawl-delay: 1

# AhrefsBot allowed (with same protected-path rules as other crawlers):
# we run Ahrefs analytics, so blocking AhrefsBot locked our own Ahrefs
# site audit out of the site.
User-agent: AhrefsBot
Allow: /
Disallow: /dashboard
Disallow: /dashboard/
Allow: /recorder
Disallow: /record
Allow: /api/og
Disallow: /api/
Disallow: /auth/
Disallow: /forgot-password
Crawl-delay: 2

# Block aggressive SEO scrapers that consume bandwidth without sending traffic
User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: YandexBot
Disallow: /

# Allow Applebot (powers Siri, Spotlight, Safari suggestions)
User-agent: Applebot
Allow: /

# Sitemap location
Sitemap: https://www.takenote.ai/sitemap.xml