# robots.txt for University of Southern Indiana (usi.edu)
# https://www.usi.edu/robots.txt

# -----------------------------------------------
# Default: allow crawling but protect admin/search areas
# -----------------------------------------------
User-agent: *
Disallow: /search
Disallow: /search/
Disallow: /search?q=
Disallow: /search/?q=
Disallow: /umbraco/
Disallow: /umbraco-preview/
Disallow: /install/
Disallow: /mini-profiler-resources/
Disallow: /admin/

# -----------------------------------------------
# Block known bad/aggressive bots
# -----------------------------------------------

# Vulnerability scanners & attack tools
User-agent: Nikto
Disallow: /

User-agent: sqlmap
Disallow: /

User-agent: Havij
Disallow: /

User-agent: masscan
Disallow: /

User-agent: zgrab
Disallow: /

# Email harvesters
User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: EmailWolf
Disallow: /

# Aggressive/disreputable scrapers
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: 008
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: Buck
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: SiteExplorer
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: Omgilibot
Disallow: /

# -----------------------------------------------
# Sitemap
# -----------------------------------------------
Sitemap: https://www.usi.edu/sitemap.xml