Default training installation for wizzlern.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
This repo is archived. You can view files and clone it, but cannot push or open issues/pull-requests.

robots.txt 1.6KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061
  1. # $Id: robots.txt,v 1.9.2.1 2008/12/10 20:12:19 goba Exp $
  2. #
  3. # robots.txt
  4. #
  5. # This file is to prevent the crawling and indexing of certain parts
  6. # of your site by web crawlers and spiders run by sites like Yahoo!
  7. # and Google. By telling these "robots" where not to go on your site,
  8. # you save bandwidth and server resources.
  9. #
  10. # This file will be ignored unless it is at the root of your host:
  11. # Used: http://example.com/robots.txt
  12. # Ignored: http://example.com/site/robots.txt
  13. #
  14. # For more information about the robots.txt standard, see:
  15. # http://www.robotstxt.org/wc/robots.html
  16. #
  17. # For syntax checking, see:
  18. # http://www.sxw.org.uk/computing/robots/check.html
  19. User-agent: *
  20. Crawl-delay: 10
  21. # Directories
  22. Disallow: /includes/
  23. Disallow: /misc/
  24. Disallow: /modules/
  25. Disallow: /profiles/
  26. Disallow: /scripts/
  27. Disallow: /sites/
  28. Disallow: /themes/
  29. # Files
  30. Disallow: /CHANGELOG.txt
  31. Disallow: /cron.php
  32. Disallow: /INSTALL.mysql.txt
  33. Disallow: /INSTALL.pgsql.txt
  34. Disallow: /install.php
  35. Disallow: /INSTALL.txt
  36. Disallow: /LICENSE.txt
  37. Disallow: /MAINTAINERS.txt
  38. Disallow: /update.php
  39. Disallow: /UPGRADE.txt
  40. Disallow: /xmlrpc.php
  41. # Paths (clean URLs)
  42. Disallow: /admin/
  43. Disallow: /comment/reply/
  44. Disallow: /contact/
  45. Disallow: /logout/
  46. Disallow: /node/add/
  47. Disallow: /search/
  48. Disallow: /user/register/
  49. Disallow: /user/password/
  50. Disallow: /user/login/
  51. # Paths (no clean URLs)
  52. Disallow: /?q=admin/
  53. Disallow: /?q=comment/reply/
  54. Disallow: /?q=contact/
  55. Disallow: /?q=logout/
  56. Disallow: /?q=node/add/
  57. Disallow: /?q=search/
  58. Disallow: /?q=user/password/
  59. Disallow: /?q=user/register/
  60. Disallow: /?q=user/login/