# ==============================================================================
# robots.txt — ean-shopping.de
# Zuletzt aktualisiert: 2026-08-27
# SEO- + AEO-/GEO-optimiert · vollstaendige KI-Bot-Whitelist · Sicherheitsbereiche gesperrt
# ==============================================================================


# ------------------------------------------------------------------------------
# 1. STANDARD-SUCHMASCHINEN — Vollzugriff mit gezielten Ausnahmen
# ------------------------------------------------------------------------------

User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Bingbot
User-agent: Slurp
User-agent: DuckDuckBot
User-agent: Baiduspider
User-agent: YandexBot
User-agent: facebot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Applebot
User-agent: ia_archiver
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /suche/
Allow: /produkt/
Allow: /ids/
Allow: /hersteller/
Allow: /sitemaps/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/

# Interne Systembereiche sperren
Disallow: /admin/
Disallow: /admin
Disallow: /logs/
Disallow: /logs
Disallow: /config/
Disallow: /config
Disallow: /scripte/
Disallow: /scripte

# Suchergebnisseiten — noindex per Meta-Tag, aber hier zusaetzlich
# alle parametrisierten Suchanfragen aus dem Index fernhalten
Disallow: /suche/?q=
Disallow: /suche/?*q=
# Weitere Filter-Parameter (redundant, aber explizit)
Disallow: /suche/?*sort=
Disallow: /suche/?*seite=
Disallow: /suche/?*min_preis=
Disallow: /suche/?*max_preis=
Disallow: /suche/?*verfuegbar=
Disallow: /api/
Disallow: /api
Disallow: /*.php$
Disallow: /wp-
Disallow: /login
Disallow: /logout
Disallow: /status/
Disallow: /queue/

Crawl-delay: 1


# ------------------------------------------------------------------------------
# 2. KI-CRAWLER / GENERATIVE ENGINES — vollstaendige Whitelist (Stand 2026-08)
#    Sammelt alle bekannten Bots, die fuer die grossen KI-Suchen holen.
#    Fuer Landingpages (/css-partner/, /api-zugang/, /geo-shops/, /agenturen/)
#    ausdruecklich mit Allow: /*.php - IIS-Rewrites zeigen dorthin.
#    Crawl-delay einheitlich 2s - ausreichend hoeflich fuer >13M-EAN-Katalog,
#    aber schnell genug fuer taegliche Aktualisierung.
# ------------------------------------------------------------------------------

# OpenAI / ChatGPT — Training + Live-Search
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# Anthropic / Claude — Training + Live-Search
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Claude-SearchBot
User-agent: anthropic-ai
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# Perplexity — Training + Live-Search
User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# Google KI (Gemini / SGE / AI Overviews / AI Mode)
User-agent: Google-Extended
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# Meta AI (Llama-Suche, Facebook/WhatsApp-AI)
User-agent: FacebookBot
User-agent: meta-externalagent
User-agent: meta-externalfetcher
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# DuckDuckGo AI (DuckAssist)
User-agent: DuckAssistBot
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# Apple Intelligence (Apple-AI-Training / Siri-Wissen)
User-agent: Applebot-Extended
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# Common Crawl (Grundlage fuer viele Open-Source-Modelle)
User-agent: CCBot
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# You.com AI
User-agent: YouBot
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /admin/
Disallow: /logs/
Disallow: /config/
Disallow: /scripte/
Disallow: /suche/
Disallow: /api/
Crawl-delay: 2

# Diffbot / Apify (Datenextraktions-Dienste, teilweise KI-genutzt)
User-agent: Diffbot
User-agent: apify
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /ids/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Disallow: /
Crawl-delay: 5


# ------------------------------------------------------------------------------
# 3. AGGRESSIV BLOCKIERTE BOTS — Security-Scanner, Spam, parasitaeres Crawling
#    Anmerkung: Amazonbot bleibt gesperrt (Rufus/Alexa liest fuer Konkurrenz-Shop).
#    Bytespider gesperrt (TikTok Search, im dt. Markt fuer B2B irrelevant).
# ------------------------------------------------------------------------------

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: PetalBot
User-agent: AspiegelBot
User-agent: serpstatbot
User-agent: SEOkicks
User-agent: seoscanners.net
User-agent: DataForSeoBot
User-agent: Bytespider
User-agent: Amazonbot
User-agent: ImagesiftBot
User-agent: Seekr
User-agent: magpie-crawler
User-agent: Scrapy
User-agent: python-requests
User-agent: curl
User-agent: wget
Disallow: /


# ------------------------------------------------------------------------------
# 4. FALLBACK — Alle nicht explizit genannten Bots
# ------------------------------------------------------------------------------

User-agent: *
Allow: /
Allow: /ean/
Allow: /kategorie/
Allow: /hersteller/
Allow: /suche/
Allow: /ids/
Allow: /sitemaps/
Allow: /css-partner/
Allow: /api-zugang/
Allow: /geo-shops/
Allow: /agenturen/
Allow: /shopanmeldung/
Disallow: /suche/?q=
Disallow: /suche/?*q=
Disallow: /admin/
Disallow: /logs/
Disallow: /logs
Disallow: /config/
Disallow: /config
Disallow: /scripte/
Disallow: /scripte
Disallow: /api/
Disallow: /*.php$
Disallow: /login
Disallow: /logout
Crawl-delay: 2


# ------------------------------------------------------------------------------
# 5. SITEMAP-VERZEICHNIS
# ------------------------------------------------------------------------------

Sitemap: https://ean-shopping.de/sitemaps/sitemap-index.xml