User-agent: * Allow: / # Allow all search engines to crawl the site User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / # Block access to admin and private areas Disallow: /admin/ Disallow: /member/ Disallow: /emember/ Disallow: /companydistributor/ Disallow: /companystockist/ # Cart Disallow: /avail.sys?Op=300382 # Allow important files for rendering Allow: /user_upload/ # Set crawl delay for less aggressive crawling Crawl-delay: 1