# exclude subdirs User-agent: * Disallow: /1008/ Disallow: /includes/ Disallow: /loggedin_neu/ Disallow: /download/ Disallow: /images/ Disallow: /logfiles/ # but allow htdig to index our doc-tree User-agent: susedig Disallow: # disallow stress test user-agent: stress-agent Disallow: /