# robots.txt для сайта rbt.su
# Оптимизированная версия от 23.03.2026

# === ГЛОБАЛЬНЫЕ НАСТРОЙКИ ===
Sitemap: https://rbt.su/sitemap.xml
Host: https://rbt.su

# === БЛОКИРОВКА СПАМ-БОТОВ (размещено ДО общего правила) ===
User-agent: SemrushBot*
Disallow: /

User-agent: AhrefsBot*
Disallow: /

User-agent: MJ12bot*
Disallow: /

User-agent: DotBot*
Disallow: /

User-agent: MauiBot*
Disallow: /

User-agent: Screaming Frog*
Disallow: /

User-agent: BLEXBot*
Disallow: /

User-agent: Cliqzbot*
Disallow: /

User-agent: CCBot*
Disallow: /

User-agent: GrapeshotCrawler*
Disallow: /

# === YANDEX (приоритет для РФ) ===
User-agent: Yandex
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/
Disallow: /author/
Disallow: /users/
Disallow: /code/
Disallow: /empty/
Disallow: /tag/
Disallow: /*/page/
Disallow: /*?s=
Disallow: /*?filter_
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: */trackback/
Disallow: */attachment/
Disallow: */print/
Disallow: */embed/
Disallow: */wp-json/

Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php
Allow: /feed/
Allow: /feed/zen/
Allow: /feed/turbo/
Allow: /feed/wxturbo/
Allow: /*.css$
Allow: /*.js$

Clean-param: utm_source&utm_medium&utm_campaign
Clean-param: utm_term&utm_content&openstat
Clean-param: from&filter_&s&ref

Crawl-delay: 2

# === GOOGLE ===
User-agent: Googlebot
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/
Disallow: /author/
Disallow: /users/
Disallow: /code/
Disallow: /empty/
Disallow: /tag/
Disallow: /*/page/
Disallow: /*?s=
Disallow: /*?filter_
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: */trackback/
Disallow: */attachment/
Disallow: */print/
Disallow: */embed/
Disallow: */wp-json/

Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php
Allow: /feed/
Allow: /*.css$
Allow: /*.js$

# === GOOGLE IMAGES ===
User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /tag/
Disallow: /*/page/

# === MAIL.RU ===
User-agent: Mail.Ru
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/
Disallow: /author/
Disallow: /users/
Disallow: /code/
Disallow: /empty/
Disallow: /tag/
Disallow: /*/page/
Disallow: /*?s=
Disallow: /*?filter_
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: */trackback/
Disallow: */attachment/
Disallow: */print/
Disallow: */embed/
Disallow: */wp-json/

Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php
Allow: /feed/
Allow: /feed/mailrss*
Allow: /*.css$
Allow: /*.js$

# === ВСЕ ОСТАЛЬНЫЕ БОТЫ ===
User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /cgi-bin/
Disallow: /author/
Disallow: /users/
Disallow: /code/
Disallow: /empty/
Disallow: /tag/
Disallow: /*/page/
Disallow: /*?s=
Disallow: /*?filter_
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: */trackback/
Disallow: */attachment/
Disallow: */print/
Disallow: */embed/
Disallow: */wp-json/

Allow: /wp-content/uploads/
Allow: /wp-admin/admin-ajax.php
Allow: /feed/
Allow: /*.css$
Allow: /*.js$

Crawl-delay: 3