User-agent: Falconsbot
User-agent: hclsreport
User-agent: msnbot
User-agent: heritrix
User-agent: Baiduspider
User-agent: Googlebot
User-agent: sindice
User-agent: Sogou
User-agent: Twiceler
Disallow: /


User-agent: *

Disallow: /label/
Disallow: /links/
# Allow robots to go through linksns as it is relatively efficient
#Disallow: /linksns/
Disallow: /linkstonamespace/
Disallow: /proteinlinks/
Disallow: /search/
Disallow: /searchns/
Disallow: /related/
Disallow: /suppliers/
Disallow: /index/
Disallow: /html/
Disallow: /image/
Disallow: /license/
Disallow: /ihop/
Disallow: /xml/
Disallow: /data/
Disallow: /countlinks/
Disallow: /countlinksns/
Disallow: /counttriples/
Disallow: /namespacecount/
Disallow: /namespaceuricount/
Disallow: /namespacetypeinternallinks/
Disallow: /cogcategories/
Disallow: /pubmedrefs/
Disallow: /pipes/
Disallow: /typens/
Disallow: /tripleresultref/
Disallow: /linkresultref/
Disallow: /asn1/
Disallow: /fasta/
Disallow: /genbankpartial/
Disallow: /genbankfull/
Disallow: /fulltext/
Disallow: /urirank/
Disallow: /commonuris/
Disallow: /queryplan/
Disallow: /medline/
Disallow: /hubmed/
Disallow: /pubmedcentral/

# /page/ versions of the normalised set above

Disallow: /page/label/
Disallow: /page/links/
#Disallow: /page/linksns/
Disallow: /page/linkstonamespace/
Disallow: /page/proteinlinks/
Disallow: /page/search/
Disallow: /page/searchns/
Disallow: /page/related/
Disallow: /page/suppliers/
Disallow: /page/index/
Disallow: /page/html/
Disallow: /page/image/
Disallow: /page/license/
Disallow: /page/ihop/
Disallow: /page/xml/
Disallow: /page/data/
Disallow: /page/countlinks/
Disallow: /page/countlinksns/
Disallow: /page/counttriples/
Disallow: /page/namespacecount/
Disallow: /page/namespaceuricount/
Disallow: /page/namespacetypeinternallinks/
Disallow: /page/cogcategories/
Disallow: /page/pubmedrefs/
Disallow: /page/pipes/
Disallow: /page/typens/
Disallow: /page/tripleresultref/
Disallow: /page/linkresultref/
Disallow: /page/asn1/
Disallow: /page/fasta/
Disallow: /page/genbankpartial/
Disallow: /page/genbankfull/
Disallow: /page/fulltext/
Disallow: /page/urirank/
Disallow: /page/commonuris/
Disallow: /page/queryplan/
Disallow: /page/medline/
Disallow: /page/hubmed/
Disallow: /page/pubmedcentral/

# /rdfxml/ versions of the normalised set above

Disallow: /rdfxml/label/
Disallow: /rdfxml/links/
#Disallow: /rdfxml/linksns/
Disallow: /rdfxml/linkstonamespace/
Disallow: /rdfxml/proteinlinks/
Disallow: /rdfxml/search/
Disallow: /rdfxml/searchns/
Disallow: /rdfxml/related/
Disallow: /rdfxml/suppliers/
Disallow: /rdfxml/index/
Disallow: /rdfxml/html/
Disallow: /rdfxml/image/
Disallow: /rdfxml/license/
Disallow: /rdfxml/ihop/
Disallow: /rdfxml/xml/
Disallow: /rdfxml/data/
Disallow: /rdfxml/countlinks/
Disallow: /rdfxml/countlinksns/
Disallow: /rdfxml/counttriples/
Disallow: /rdfxml/namespacecount/
Disallow: /rdfxml/namespaceuricount/
Disallow: /rdfxml/namespacetypeinternallinks/
Disallow: /rdfxml/cogcategories/
Disallow: /rdfxml/pubmedrefs/
Disallow: /rdfxml/pipes/
Disallow: /rdfxml/typens/
Disallow: /rdfxml/tripleresultref/
Disallow: /rdfxml/linkresultref/
Disallow: /rdfxml/asn1/
Disallow: /rdfxml/fasta/
Disallow: /rdfxml/genbankpartial/
Disallow: /rdfxml/genbankfull/
Disallow: /rdfxml/fulltext/
Disallow: /rdfxml/urirank/
Disallow: /rdfxml/commonuris/
Disallow: /rdfxml/queryplan/
Disallow: /rdfxml/medline/
Disallow: /rdfxml/hubmed/
Disallow: /rdfxml/pubmedcentral/

# /n3/ versions of the normalised set above

Disallow: /n3/label/
Disallow: /n3/links/
#Disallow: /n3/linksns/
Disallow: /n3/linkstonamespace/
Disallow: /n3/proteinlinks/
Disallow: /n3/search/
Disallow: /n3/searchns/
Disallow: /n3/related/
Disallow: /n3/suppliers/
Disallow: /n3/index/
Disallow: /n3/html/
Disallow: /n3/image/
Disallow: /n3/license/
Disallow: /n3/ihop/
Disallow: /n3/xml/
Disallow: /n3/data/
Disallow: /n3/countlinks/
Disallow: /n3/countlinksns/
Disallow: /n3/counttriples/
Disallow: /n3/namespacecount/
Disallow: /n3/namespaceuricount/
Disallow: /n3/namespacetypeinternallinks/
Disallow: /n3/cogcategories/
Disallow: /n3/pubmedrefs/
Disallow: /n3/pipes/
Disallow: /n3/typens/
Disallow: /n3/tripleresultref/
Disallow: /n3/linkresultref/
Disallow: /n3/asn1/
Disallow: /n3/fasta/
Disallow: /n3/genbankpartial/
Disallow: /n3/genbankfull/
Disallow: /n3/fulltext/
Disallow: /n3/urirank/
Disallow: /n3/commonuris/
Disallow: /n3/queryplan/
Disallow: /n3/medline/
Disallow: /n3/hubmed/
Disallow: /n3/pubmedcentral/

# More generalised items below
Disallow: /fasta:
Disallow: /page/fasta:
Disallow: /rdfxml/fasta:
Disallow: /n3/fasta:

Disallow: /pageoffset
Disallow: /page/pageoffset
Disallow: /rdfxml/pageoffset
Disallow: /n3/pageoffset

Disallow: /multiple
Disallow: /page/multiple
Disallow: /rdfxml/multiple
Disallow: /n3/multiple

Disallow: /statistics
Disallow: /page/statistics
Disallow: /rdfxml/statistics
Disallow: /n3/statistics

# The local wiki doesn't exist anymore, so preempt this by telling bots not to access it
Disallow: /wiki/
Disallow: /JSPWiki/

Disallow: /admin/
Disallow: /error/
Disallow: /download/

Crawl-delay: 10			  # All users should wait 10 seconds between requests. If not, automatic blocker may activate