# # robots.txt for http://www.citizen.org/ # # v 2.0 2010/10/22 9:03:16 AM # # exclude some access-controlled areas User-agent: * Crawl-delay: 60 Disallow: /admin # exclude some areas and pages that would be duplicative Disallow: /pcapps/stealthpacs Disallow: /trade/taa/taa_info.cfm Disallow: /trade/taa/searchResults.cfm Disallow: /slideshow/ Disallow: /dontindex/ Disallow: /hrg/articles.cfm Disallow: /trade/articles.cfm Disallow: /litigation/articles.cfm Disallow: /texas/articles.cfm Disallow: /cmep/articles.cfm Disallow: /congress/articles.cfm Disallow: /autosafety/articles.cfm Disallow: /jobs/articles.cfm Disallow: /publication/release.cfm Disallow: /pressroom/release.cfm Disallow: /hot_issues/issue.cfm Disallow: /pressroom/getlink.cfm Disallow: /litigation/forms/cases/caseDetails.cfm User-agent: ShopWiki Disallow: /