# robots.txt for http://www.math.columbia.edu/

# Unwanted Crawlers Setup
User-agent: MegaIndex.ru
User-agent: MegaIndex.ru/
User-agent: MegaIndex.ru/2.0
User-agent: Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +https://www.megaindex.ru/?tab=linkAnalyze)
User-agent: MJ12
User-agent: Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)
User-agent: wotbox
User-agent: ltx71
User-agent: ScoutJet
User-agent: Mozilla/5.0 (compatible; Blekkobot; ScoutJet; +http://blekko.com/about/blekkobot)
User-Agent: Springbot
User-Agent: ShopSpring SpringBot
User-Agent: Arachni
User-agent: SemrushBot
User-agent: SemrushBot-SA
User-agent: Barkrowler
User-agent: PiplBot
User-agent: Discordbot
User-agent: Exabot
User-agent: AhrefsBot
User-agent: ZoomBot
User-agent: NewsAnglrBot
User-agent: BublupBot
Disallow: /

User-agent: *
Disallow: /cgi-bin/ # we don't want robots crawling our scripts
Disallow: /events/
Disallow: /event/
Disallow: /horde/
Disallow: /department/horde/
Disallow: /~belmans/
Disallow: /people/
Disallow: /~bayer/symmetry/wallpaper/