# robots.txt for http://www.searchengineguide.com/ User-agent: yahoo-newscrawler Disallow: User-agent: Yahoo! Slurp Disallow: User-agent: Mediapartners-Google* Disallow: User-agent: * Disallow: /wire/archive/ Disallow: /cgi-bin/mt/ Disallow: /cgi-bin/news/recommend_it.cgi Disallow: /links/page.cgi