# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
 User-agent: *
 Disallow: /web
 Disallow: /settings
 Disallow: /login
 Disallow: /user-search
 Disallow: /registration
 Disallow: /cgi-bin
 Disallow: /pages/DMCA.html
 Disallow: /oauth
 Disallow: /search