# Block webservices, block main noindex pages, allow all others.
User-agent: *
Disallow: /cart
Disallow: /create-account
Disallow: /login
Disallow: /login-new
Disallow: /logout
Disallow: /my-garage
Disallow: /my-lists
Disallow: /my-machines
Disallow: /my-orders
Disallow: /search
Disallow: /svc/

# Block known spiders causing issues on the website
User-agent: AhrefsBot
User-agent: Fatbot
User-agent: Halebot
User-Agent: Halebot (Mozilla/5.0 compatible; Halebot/2.1) 
User-agent: MotorcycleGold
User-agent: NetcraftSurveyAgent
User-agent: Pimonster
User-agent: Rogerbot
User-agent: ScoutJet
Disallow: / 

# Issue a 10 "second" crawl delay for MSN Bing and Slurp
User-agent: msnbot
User-agent: bingbot
User-agent: Slurp
Crawl-delay: 10
Disallow: /cart
Disallow: /create-account
Disallow: /login
Disallow: /login-new
Disallow: /logout
Disallow: /my-garage
Disallow: /my-lists
Disallow: /my-machines
Disallow: /my-orders
Disallow: /search
Disallow: /svc/

# Issue a 5 second crawl delay on possibly friendly bots
User-agent: Yandex
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Mail.RU_Bot
User-agent: GrapeshotCrawler
User-agent: CCBot
Crawl-delay: 5
Disallow: /cart
Disallow: /create-account
Disallow: /login
Disallow: /login-new
Disallow: /logout
Disallow: /my-garage
Disallow: /my-lists
Disallow: /my-machines
Disallow: /my-orders
Disallow: /search
Disallow: /svc/

Sitemap: http://www.bikebandit.com/sitemap_index.xml