User-Agent: * Disallow: /secured/ # # Disallow for WebResource.axd caching issues. Several instances below to cover all search engines. # # To specify matching the end of a URL, use $ # Disallow: /*.axd$ # # However, WebResource.axd and ScriptResource.axd always include a query string parameter the URL does # not end with .axd thus, the correct robots.txt record for Google would be: # Disallow: /*.axd # # Not all crawlers recognize the wildcard '*' syntax. To comply with the robots.txt draft RFC # Note that the records are case sensitive, and error page is showing the requests to be in lower case # so let's include both cases below: # Disallow: /*.axd* Disallow: /ScriptResource.axd Disallow: /WebResource.axd Disallow: /page-not-found.aspx Disallow: /error.aspx