User-agent: *
Disallow: /u/
Disallow: /api/
Disallow: /rest/
Disallow: /plugins/
Disallow: /app/
Disallow: /vendor/
Disallow: /cache_test/
Disallow: /music/_framed_ad
Disallow: /music/_goitunes
Disallow: /appconfig.js
Disallow: /search/
# crawl-delay only works on yahoo, bing, and yandex
Crawl-delay: 10

#all others
User-agent: AdsBot
User-agent: AdsBot-Naver
User-agent: AhrefsBot
User-agent: AlphaBot
User-agent: BLEXBot
User-agent: BUbiNG
User-agent: Baiduspider
User-agent: Barkrowler experimental crawler
User-agent: Blackboard Safeassign
User-agent: BotPants
User-agent: Buck
User-agent: Bytespider
User-agent: CCBot
User-agent: CheckMarkNetwork
User-agent: CheckMarkNetwork/1.0 (+https://www.checkmarknetwork.com/spider.html)
User-agent: Clickagy Intelligence Bot v2
User-agent: Cliqzbot
User-agent: Companybook Crawler
User-agent: Companybook-Crawler
User-agent: DataForSeoBot
User-agent: Daum
User-agent: DeuSu
User-agent: DnyzBot
User-agent: DomainCrawler
User-agent: DomainStatsBot
User-agent: DotBot
User-agent: Embedly
User-agent: ExtLinksBot
User-agent: Ezooms
User-agent: FemtosearchBot
User-agent: Findxbot
User-agent: FriendlyCrawler
User-agent: GarlikCrawler
User-agent: GeedoBot
User-agent: GetIntent Crawler
User-agent: Gluten Free Crawler
User-agent: GrapeshotCrawler
User-agent: HyScore
User-agent: ICC-Crawler
User-agent: JobboerseBot
User-agent: Laserlikebot
User-agent: Leikibot
User-agent: Linespider
User-agent: Linguee
User-agent: MJ12bot
User-agent: MTRobot
User-agent: Mail.RU_Bot
User-agent: Mappy
User-agent: MarketwireBot
User-agent: MauiBot
User-agent: MaxPointCrawler/Nutch-1.10
User-agent: MegaIndex
User-agent: MegaIndex.ru
User-agent: MojeekBot
User-agent: Neevabot
User-agent: NetSeer crawler
User-agent: PaperLiBot
User-agent: Pinterestbot
User-agent: PiplBot
User-agent: Qwantify
User-agent: RukiCrawler
User-agent: SEOkicks-Robot
User-agent: SMTBot
User-agent: SabsimBot
User-agent: ScooperBot
User-agent: ScoutJet
User-agent: Seekport Crawler
User-agent: SeekportBot
User-agent: SemanticScholarBot
User-agent: SemrushBot
User-agent: SemrushBot-BA
User-agent: SemrushBot-BM
User-agent: SemrushBot-CT
User-agent: SemrushBot-SA
User-agent: SemrushBot-SI
User-agent: SemrushBot-SWA
User-agent: SeznamBot
User-agent: Sidetrade
User-agent: SiteExplorer
User-agent: SocialRankIOBot
User-agent: Sogou web spider
User-agent: Sosospider+
User-agent: SurdotlyBot
User-agent: SurveyBot
User-agent: TurnitinBot
User-agent: Tweetmeme
User-agent: VelenPublicWebCrawler
User-agent: WireReaderBot
User-agent: Xenu Link Sleuth
User-agent: Y!J-ASR
User-agent: Y!J-WSC
User-agent: YaK
User-agent: Yahoo! Slurp
User-agent: Yandex
User-agent: YandexAccessibilityBot
User-agent: YandexBot
User-agent: YandexMobileBot
User-agent: Yeti
User-agent: YouBot
User-agent: ZoomBot
User-agent: adbeat_bot
User-agent: admantx-adform
User-agent: admantx-ussy02
User-agent: adsbot
User-agent: bingbot
User-agent: cXensebot
User-agent: coccocbot-web
User-agent: contxbot
User-agent: demandbase-bot
User-agent: eContext
User-agent: eright
User-agent: fidget-spinner-bot
User-agent: foobar2000
User-agent: gocrawl
User-agent: hyscore
User-agent: ia_archiver
User-agent: ias_crawler
User-agent: keys-so-bot
User-agent: linkdexbot
User-agent: ltx71
User-agent: megaIndex.ru
User-agent: msnbot
User-agent: netEstate NE Crawler
User-agent: oBot
User-agent: proximic
User-agent: serpstatbot
User-agent: sistrix
User-agent: special_archiver
User-agent: startmebot
User-agent: trendictionbot
User-agent: vebidoobot
User-agent: www.ru
Disallow: /