# General Crawler Rules for website
User-agent: *
Allow: /

# Block sensitive directories and payment processing paths
Disallow: /ccbill/
Disallow: /members/

# Block specific search query strings to prevent duplicate content
Disallow: /*?s=
Disallow: /*?search=

# Explicitly allow Google and Bing to crawl your media thumbnails
User-agent: Googlebot-Image
Allow: /images/

User-agent: bingspider
Allow: /images/