# Crawlers Setup
User-agent: *           #Allow crawling over the following paths
Allow: /js/
Allow: /media/js/
Allow: /media/css_secure/
Allow: /skin/
Allow: /*.js
Allow: /*.css
Allow: /skin/frontend/
# Directories
Disallow: /404/
Disallow: /app/
Disallow: /cgi-bin/
Disallow: /downloader/
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /media/
Disallow: */instantcart/*
Disallow: */f/*
Disallow: */id/*
Disallow: */page_id/*
# we allow product /catalog and wyiwyg media images to be indexed for seo benefits
Allow: /media/catalog/product/
Allow: /media/catalog/category/
Allow: /media/wysiwyg/
Disallow: /pkginfo/
Disallow: /report/
Disallow: /scripts/
Disallow: /shell/
Disallow: /stats/
Disallow: /var/

# Paths (clean URLs)
Disallow: /premio/index*
Disallow: /premio/index/index*
Disallow: /index.php/
Disallow: /catalog/product_compare/
Disallow: */catalog/category*
Disallow: */catalog/product*
Disallow: /catalogsearch/
Disallow: *flavour_cup_size=*
Disallow: *coffee_intensity=*
Disallow: *capsules_number=*
Disallow: *flavour_caffeine2=*
Disallow: *hot_or_cold=*
Disallow: *price=*
Disallow: *machine_capacity_water_tank=*
Disallow: *machine_search_color=*
Disallow: *machine_type=*
Disallow: /control/
Disallow: /contacts/
Disallow: */customer/*
Disallow: /customize/
Disallow: /newsletter/
Disallow: /poll/
Disallow: /review/
Disallow: *show=review-form*
Disallow: /sendfriend/
Disallow: /tag/
Disallow: /wishlist/
Disallow: /catalog/product/gallery/
Disallow: /test-refund-offer

## Do not crawl checkout and user account pages
Disallow: */checkout/*
Disallow: /onestepcheckout/
Disallow: */customer/*


# Files
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /STATUS.txt

# Paths (no clean URLs)
Disallow: /*.php$
Disallow: /*?SID=

## Do not crawl sub category pages that are sorted or filtered.
Disallow: /*?dir*
Disallow: /*?dir=desc
Disallow: /*?dir=asc
Disallow: /*?limit=all
Disallow: /*?mode*
Disallow: /*?gclid=

User-agent: Googlebot-Image
Allow: /media/catalog/product/
Allow: /media/catalog/category/
Allow: /media/wysiwyg/

# Do not crawl add to cart for mobile
Disallow: /m/instantcart/



Disallow: */reward/*
Disallow: *p=*
Disallow: *rating=*
Disallow: *gQT=*
Disallow: *review*
Disallow: *cat=*
Disallow: *am_on_sale=*
Disallow: *subscription=*
Disallow: *utm_source*
Disallow: /*/index
Disallow: *limit=*
Disallow: *gDelivery=*
Disallow: *fbclid=*
Disallow: *price*
Disallow: */m/*




Sitemap: https://www.dolce-gusto.co.kr/sitemap.xml