User-agent: *

## Do not crawl common server technical folders and files
Disallow: /cgi-bin/
Disallow: /magento-cleanup.php
Disallow: /phpinfo.php

## Do not crawl checkout and user account pages
Disallow: /checkout/
Disallow: /checkout/cart/
Disallow: /onestepcheckout/
Disallow: /customer/
Disallow: /customer/account/
Disallow: /customer/account/login/

## Do not crawl common Magento files
Disallow: /api.php
Disallow: /cron.php
Disallow: /cron.sh
Disallow: /error_log
Disallow: /get.php
Disallow: /install.php
Disallow: /LICENSE.html
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /README.txt
Disallow: /RELEASE_NOTES.txt

## Do not crawl common Magento technical folders
Disallow: /errors/
Disallow: /includes/
Disallow: /lib/
Disallow: /shell/
Disallow: /var/

## Do not crawl development files and folders: CVS, svn directories and dump files
Disallow: /CVS
Disallow: /*.svn$
Disallow: /*.idea$
Disallow: /*.sql$
Disallow: /*.tgz$
Disallow: /*?SID=*