# Tells Scanning Robots Where They Are And Are Not Welcome
# User-agent:can also specify by name; "*" is for everyone
# Disallow:if this matches first part of requested path,

User-agent: *    # applies to all robots
Allow: /pages/
Allow: /common/
Allow: /forum/
Disallow: /images/      # disallow indexing of images folder
Disallow: /test/      # disallow indexing of testing pages
Disallow: /tsep.0942/      # disallow indexing of testing pages
Disallow: /phpcrawl/      # disallow indexing of testing pages