diff options
Diffstat (limited to 'robots.txt')
-rw-r--r-- | robots.txt | 57 |
1 files changed, 57 insertions, 0 deletions
diff --git a/robots.txt b/robots.txt new file mode 100644 index 000000000..c41c1a424 --- /dev/null +++ b/robots.txt @@ -0,0 +1,57 @@ +# robots.txt +# +# This file aims to prevent the crawling and idexing of certain parts +# of your site by web crawlers and spiders run by sites like Yahoo! +# and Google. By telling these "robots" where not to go on your site, +# you save bandwidth and server resources. +# +# For more information about the robots.txt standard, see: +# http://www.robotstxt.org/wc/robots.html +# +# See this site for syntax checking: +# http://www.sxw.org.uk/computing/robots/check.html + +User-agent: * +Crawl-delay: 10 +# Directories +Disallow: /files/ +Disallow: /database/ +Disallow: /includes/ +Disallow: /misc/ +Disallow: /modules/ +Disallow: /sites/ +Disallow: /themes/ +Disallow: /scripts/ +Disallow: /updates/ +Disallow: /profiles/ +# Files +Disallow: /xmlrpc.php +Disallow: /cron.php +Disallow: /update.php +Disallow: /install.php +Disallow: /INSTALL.mysql.txt +Disallow: /INSTALL.pgsql.txt +Disallow: /CHANGELOG.txt +Disallow: /MAINTAINERS.txt +Disallow: /LICENSE.txt +Disallow: /UPGRADE.txt +# Paths (clean URLs) +Disallow: /admin/ +Disallow: /aggregator/ +Disallow: /comment/reply/ +Disallow: /contact/ +Disallow: /logout/ +Disallow: /node/add/ +Disallow: /search/ +Disallow: /user/register/ +Disallow: /user/password/ +# Paths (no clean URLs) +Disallow: /?q=admin/ +Disallow: /?q=aggregator/ +Disallow: /?q=comment/reply/ +Disallow: /?q=contact/ +Disallow: /?q=logout/ +Disallow: /?q=node/add/ +Disallow: /?q=search/ +Disallow: /?q=user/password/ +Disallow: /?q=user/register/ |