# Enables robots.txt rules for all crawlers User-agent: * Sitemap: https://www.werder.de/shop/media/sitemap.xml Sitemap: https://www.werder.de/shop/media/sitemap_en.xml # Disallow forum paths Disallow: /forum/?* Disallow: /forum/attachments/ Disallow: /forum/forums/*/?direction Disallow: /forum/forums/*/?order Disallow: /forum/help/ Disallow: /forum/login/ Disallow: /forum/lost-password/ Disallow: /forum/members/ Disallow: /forum/misc/ Disallow: /forum/online/ Disallow: /forum/posts/ Disallow: /forum/register/ Disallow: /forum/search/ Disallow: /forum/goto/ # # Do not allow indexing files and folders that are required during development: CVS, SVN directory and dump files Disallow: /*.git$ Disallow: /*.idea$ Disallow: /*.sql$ Disallow: /*.tgz$ Disallow: /*.tar.gz$ Disallow: /*.zip$ Disallow: /session.php # # GENERAL SETTINGS For MAGENTO Disallow: /shop/admin/ # # Do not index the general technical Magento directory Disallow: /shop/app/ Disallow: /shop/downloader/ Disallow: /shop/errors/ Disallow: /shop/includes/ Disallow: /shop/lib/ Disallow: /shop/pkginfo/ Disallow: /shop/shell/ Disallow: /shop/skin/ Disallow: /shop/var/ Disallow: /shop/zu-hause/sky.html Disallow: /shop/sizes-table # # Do not index the shared files Magento Disallow: /shop/api.php Disallow: /shop/cron.php Disallow: /shop/cron.sh Disallow: /shop/error_log Disallow: /shop/get.php Disallow: /shop/install.php Disallow: /shop/LICENSE.html Disallow: /shop/LICENSE.txt Disallow: /shop/LICENSE_AFL.txt Disallow: /shop/README.txt Disallow: /shop/RELEASE_NOTES.txt Disallow: /shop/STATUS.txt # # MAGENTO SEA IMPROVEMENT # # Do not index the page subcategories that are sorted or filtered. Disallow: /shop/*?Dir* Disallow: /shop/*?Dir=desc Disallow: /shop/*?Dir=asc Disallow: /shop/*?Limit=all Disallow: /shop/*?Mode* # # Do not index the second copy of the home page (example.com / index.php /). Un-comment only if you have activated Magento SEO URLs. Disallow: /shop/index.php/ # # Do not index the link from the session ID Disallow: /shop/*?SID= # disallow all esi parameters Disallow: /*?*esi=1* Disallow: /*/esi/* # # Do not index the page checkout and user account Disallow: /shop/catalog/product_compare/ Disallow: /shop/checkout/ Disallow: /shop/onestepcheckout/ Disallow: /shop/newsletter/ Disallow: /shop/poll/ Disallow: /shop/review/ Disallow: /shop/sendfriend/ Disallow: /shop/en/catalog/product_compare/ Disallow: /shop/en/checkout/ Disallow: /shop/en/onestepcheckout/ Disallow: /shop/en/newsletter/ Disallow: /shop/en/poll/ Disallow: /shop/en/review/ Disallow: /shop/en/sendfriend/ # # Do not index the search page and CEO, non-optimized link categories Disallow: /shop/catalog/category/view/ Disallow: /shop/catalog/product/view/ Disallow: /shop/catalog/product/gallery/ Disallow: /shop/en/catalog/category/view/ Disallow: /shop/en/catalog/product/view/ Disallow: /shop/en/catalog/product/gallery/ # # Exclude Various Robots User-agent: MJ12bot Disallow: / User-agent: YandexBot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: FatBot Disallow: / User-agent: SputnikBot Disallow: / User-agent: trendictionbot0.5.0 Disallow: / User-agent: Owlin Disallow: / User-agent: SeznamBot Disallow: / User-agent: SEOkicks Disallow: / User-agent: SEOkicks-Robot Disallow: / User-agent: Stashbot Disallow: / User-agent: MetaJobBot Disallow: / User-agent: Exabot Disallow: / User-agent: DotBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: linkdexbot Disallow: / User-agent: Screaming Frog SEO Spider Disallow: / User-agent: naver Disallow: / User-agent: Yeti Disallow: / User-agent: MegaIndex.ru Disallow: / User-agent: megaindex.com Disallow: / User-agent: PetalBot Disallow: / User-Agent: proximic Disallow: / User-agent: AspiegelBot Disallow: / User-Agent: Squidbot Disallow: / User-Agent: coccocbot-web Disallow: / User-Agent: Veooz Disallow: / User-Agent: Seekport Crawler Disallow: / User-agent: BLEXBot Disallow: /