User-agent: *
Allow: /
Disallow: /robots.txt
Disallow: /cgi-bin/
Disallow: /cgi-bin/sal/
Disallow: /cgi-bin/sam/
Disallow: /common/scripts/
Disallow: /lgo/companies/
Disallow: /lgo/warehouse-companies/companies/
Disallow: /lgo/food-grade-storage-warehouses/companies/
Disallow: /lgo/cold-storage-warehouses/companies/
Disallow: /lgo/hazmat-storage-warehouses/companies/
Disallow: /lgo/order-fulfillment-warehouses/companies/
Disallow: /ncc/sample*
Disallow: /nwdd/sample*
Disallow: /ntpld/sample*
Disallow: /iacd/sample*
Disallow: /rsd/sample*

User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: psbot
Disallow: /
User-agent: turnitinbot
Disallow: /
User-agent: TwengaBot-Discover
Disallow: /
User-agent: TwengaBot
Disallow: /
User-agent: dotbot 
Disallow: /
User-agent: exabot 
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: spbot
Disallow: /