# robots.txt for http://www.broad.mit.edu/ # # Please keep away from endless cgis User-agent: * Disallow: /ftp/ Disallow: /*SelectMouseStsInfo Disallow: /*ext_view_chrom.cgi Disallow: /*ext_view_clone.cgi Disallow: /*ext_view_contig.cgi Disallow: /*ext_view_marker.cgi Disallow: /*findfeatures.cgi Disallow: /*find_motifs.cgi Disallow: /*find_orfs.cgi Disallow: /*locus.pl Disallow: /*lookup_contig Disallow: /*sequence-ruler.pl Disallow: /*sequence_info Disallow: /*snp_info Disallow: /*snp_sequence_info Disallow: /*sts_by_chrom Disallow: /*sts_info Disallow: /*yac_info Disallow: /ftp # Annotation pages with endless links # but let crawl Home, Info, FAQ Disallow: /annotation/genome/*/GeneDetails.html Disallow: /annotation/genome/*/GeneIndex.html Disallow: /annotation/genome/*/AssemblyStructure.html Disallow: /annotation/genome/*/Blast.html Disallow: /annotation/genome/*/FeatureDetails.html Disallow: /annotation/genome/*/FeatureSearch.html Disallow: /annotation/genome/*/MultiDownloads.html Disallow: /annotation/genome/*/Regions.html Disallow: /annotation/genome/*/OrthologSearch.html Disallow: /annotation/genome/*/Rfam.html # Since some robots just dont get it Disallow: /cgi-bin/annotation/aspergillus/findfeatures.cgi Disallow: /cgi-bin/annotation/aspergillus_new/findfeatures.cgi Disallow: /cgi-bin/annotation/ciona/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/aspergillus_nidulans/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/botrytis_cinerea/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/candida_guilliermondii/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/candida_lusitaniae/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/chaetomium_globosum/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/coccidioides_immitis/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/comp_yeasts/find_motifs.cgi Disallow: /cgi-bin/annotation/fungi/comp_yeasts/find_orfs.cgi Disallow: /cgi-bin/annotation/fungi/coprinus_cinereus/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/cryptococcus_neoformans/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/magnaporthe_4/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/magnaporthe_grisea/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/neurospora_crassa_3/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/neurospora_crassa_7/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/rhizopus_oryzae/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/sclerotinia_sclerotiorum/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/stagonospora_nodorum/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/ustilago_maydis/findfeatures.cgi Disallow: /cgi-bin/annotation/fungi/ustilago_maydis_new/findfeatures.cgi Disallow: /cgi-bin/annotation/fusarium/findfeatures.cgi Disallow: /cgi-bin/annotation/fusarium_new/findfeatures.cgi Disallow: /cgi-bin/annotation/hawk2/findfeatures.cgi Disallow: /cgi-bin/annotation/human_chr_x/findfeatures.cgi Disallow: /cgi-bin/annotation/internal/pipeline/findfeatures.cgi Disallow: /cgi-bin/annotation/magnaporthe/blast_page.cgi Disallow: /cgi-bin/annotation/magnaporthe/findfeatures.cgi Disallow: /cgi-bin/annotation/magnaporthe_copy/findfeatures.cgi Disallow: /cgi-bin/annotation/magnaporthe_new/findfeatures.cgi Disallow: /cgi-bin/annotation/methanosarcina/findfeatures.cgi Disallow: /cgi-bin/annotation/microbes/mesoplasma_florum/findfeatures.cgi Disallow: /cgi-bin/annotation/microbes/mesoplasma_florum_new/findfeatures.cgi Disallow: /cgi-bin/annotation/microbes/mycobacterium_tuberculosis_f11/findfeatures.cgi Disallow: /cgi-bin/annotation/microbes/mycoplasma_mobile/findfeatures.cgi Disallow: /cgi-bin/annotation/mycoplasma/findfeatures.cgi Disallow: /cgi-bin/annotation/neurospora/findfeatures.cgi Disallow: /cgi-bin/annotation/neurospora_new/findfeatures.cgi Disallow: /cgi-bin/annotation/oldstuff/internal/pipeline/findfeatures.cgi Disallow: /cgi-bin/annotation/sequence-ruler.pl Disallow: /cgi-bin/contig/lookup_contig Disallow: /cgi-bin/contig/sequence_info Disallow: /cgi-bin/contig/sts_by_chrom Disallow: /cgi-bin/contig/sts_info Disallow: /cgi-bin/contig/yac_info Disallow: /cgi-bin/mouse/lookup_contig Disallow: /cgi-bin/mouse/lookup_contig_cleanmouse Disallow: /cgi-bin/mouse/sequence_info Disallow: /cgi-bin/mouse/snp_info.cgi Disallow: /cgi-bin/mouse/sts_by_chrom Disallow: /cgi-bin/mouse/sts_by_chrom_double Disallow: /cgi-bin/mouse/sts_info Disallow: /cgi-bin/mouse/yac_info Disallow: /cgi-bin/rat/sequence_info Disallow: /cgi-bin/rat/sts_by_chrom Disallow: /cgi-bin/rat/sts_info Disallow: /cgi-bin/seq/bac/ext_view_chrom.cgi Disallow: /cgi-bin/seq/bac/ext_view_clone.cgi Disallow: /cgi-bin/seq/bac/ext_view_contig.cgi Disallow: /cgi-bin/seq/bac/ext_view_marker.cgi Disallow: /cgi-bin/seq/contig/lookup_contig Disallow: /cgi-bin/seq/contig/sequence_info Disallow: /cgi-bin/seq/contig/sts_by_chrom Disallow: /cgi-bin/seq/contig/sts_info Disallow: /cgi-bin/seq/contig/sts_info.mod Disallow: /cgi-bin/seq/contig/yac_info Disallow: /cgi-bin/seq/primer/sts_by_chrom Disallow: /cgi-bin/snp/human/snp_info Disallow: /cgi-bin/snp/human/snp_sequence_info Disallow: /cgi-bin/snp/human/sts_info.lstein Disallow: /cgi-bin/snp/mouse/SelectMouseStsInfo Disallow: /annotation/genome/aedes_aegypti/AssemblyStructure.html Disallow: /annotation/genome/aedes_aegypti/Regions.html Disallow: /annotation/genome/aspergillus_group/AssemblyStructure.html Disallow: /annotation/genome/aspergillus_group/Blast.html Disallow: /annotation/genome/aspergillus_group/Downloads.html Disallow: /annotation/genome/aspergillus_group/DownloadResults.html Disallow: /annotation/genome/aspergillus_group/FeatureDetails.html Disallow: /annotation/genome/aspergillus_group/FeatureSearch.html Disallow: /annotation/genome/aspergillus_group/GeneDetails.html Disallow: /annotation/genome/aspergillus_group/Home.html Disallow: /annotation/genome/aspergillus_group/MultiDownloads.html Disallow: /annotation/genome/aspergillus_group/MultiHome.html Disallow: /annotation/genome/aspergillus_group/Regions.html Disallow: /annotation/genome/aspergillus_group/OrthologSearch.html Disallow: /annotation/genome/aspergillus_group/Rfam.html Disallow: /annotation/genome/aspergillus_nidulans/AssemblyStructure.html Disallow: /annotation/genome/aspergillus_nidulans/Blast.html Disallow: /annotation/genome/aspergillus_nidulans/FeatureMap.html Disallow: /annotation/genome/aspergillus_nidulans/FeatureSearch.html Disallow: /annotation/genome/aspergillus_nidulans/GeneDetails.html Disallow: /annotation/genome/aspergillus_nidulans/Home.html Disallow: /annotation/genome/aspergillus_nidulans/MultiHome.html Disallow: /annotation/genome/aspergillus_group/AssemblyStructure.html Disallow: /annotation/genome/aspergillus_group/FeatureSearch.html Disallow: /annotation/genome/aspergillus_group/GeneIndex.html Disallow: /annotation/genome/aspergillus_group/OrthologSearch.html Disallow: /annotation/genome/aspergillus_nidulans/Regions.html Disallow: /annotation/genome/aspergillus_terreus/Blast.html Disallow: /annotation/genome/aspergillus_terreus/DownloadResults.html Disallow: /annotation/genome/aspergillus_terreus/FeatureDetails.html Disallow: /annotation/genome/aspergillus_terreus/Pfam.html Disallow: /annotation/genome/batrachochytrium_dendrobatidis/AssemblyStructure.html Disallow: /annotation/genome/batrachochytrium_dendrobatidis/FeatureSearch.html Disallow: /annotation/genome/botrytis_cinerea/AssemblyStructure.html Disallow: /annotation/genome/botrytis_cinerea/Blast.html Disallow: /annotation/genome/botrytis_cinerea/GeneFinding.html Disallow: /annotation/genome/botrytis_cinerea/FeatureSearch.html Disallow: /annotation/genome/botrytis_cinerea/GeneDetails.html Disallow: /annotation/genome/botrytis_cinerea/Home.html Disallow: /annotation/genome/botrytis_cinerea/TranscriptDetails.html Disallow: /annotation/genome/burkholderia_cenocepacia/AssemblyStructure.html Disallow: /annotation/genome/burkholderia_cenocepacia/Blast.html Disallow: /annotation/genome/burkholderia_cenocepacia/DownloadResults.html Disallow: /annotation/genome/burkholderia_dolosa/AssemblyStructure.html Disallow: /annotation/genome/burkholderia_dolosa/Comments.html Disallow: /annotation/genome/burkholderia_dolosa/FeatureSearch.html Disallow: /annotation/genome/burkholderia_dolosa/Rfam.html Disallow: /annotation/genome/candida_albicans/AssemblyStructure.html Disallow: /annotation/genome/candida_albicans/Blast.html Disallow: /annotation/genome/candida_albicans/FeatureSearchResults.html Disallow: /annotation/genome/candida_group/AssemblyStructure.html Disallow: /annotation/genome/candida_group/DownloadResults.html Disallow: /annotation/genome/candida_group/FeatureSearch.html Disallow: /annotation/genome/candida_group/GeneLocus.html Disallow: /annotation/genome/candida_group/OrthologSearch.html Disallow: /annotation/genome/candida_group/Rfam.html Disallow: /annotation/genome/candida_lusitaniae/AssemblyStructure.html Disallow: /annotation/genome/candida_lusitaniae/FeatureSearchResults.html Disallow: /annotation/genome/candida_lusitaniae/TextSearchHelp.html Disallow: /annotation/genome/candida_tropicalis/Comments.html Disallow: /annotation/genome/candida_tropicalis/DownloadResults.html Disallow: /annotation/genome/candida_tropicalis/FeatureSearch.html Disallow: /annotation/genome/candida_tropicalis/FeatureSearchResults.html Disallow: /annotation/genome/candida_tropicalis/Home.html Disallow: /annotation/genome/candida_tropicalis/Info.html Disallow: /annotation/genome/candida_tropicalis/GeneDetails.html Disallow: /annotation/genome/candida_tropicalis/Pfam.html Disallow: /annotation/genome/chaetomium_globosum/AssemblyStructure.html Disallow: /annotation/genome/chaetomium_globosum/Blast.html Disallow: /annotation/genome/chaetomium_globosum/Comments.html Disallow: /annotation/genome/chaetomium_globosum/Home.html Disallow: /annotation/genome/chaetomium_globosum/Info.html Disallow: /annotation/genome/chaetomium_globosum/Pfam.html Disallow: /annotation/genome/coccidioides_group/AssemblyStructure.html Disallow: /annotation/genome/coccidioides_group/DownloadResults.html Disallow: /annotation/genome/coccidioides_group/GeneDetails.html Disallow: /annotation/genome/coccidioides_group/Pfam.html Disallow: /annotation/genome/coccidioides_group/TranscriptDetails.htm Disallow: /annotation/genome/coccidioides_group/DownloadResults.html Disallow: /annotation/genome/coccidioides_immitis/AssemblyStructure.html Disallow: /annotation/genome/coccidioides_immitis/DownloadResults.html Disallow: /annotation/genome/coccidioides_immitis/FeatureDetails.html Disallow: /annotation/genome/coccidioides_immitis/FeatureSearchResults.html Disallow: /annotation/genome/coccidioides_immitis/GeneDetails.html Disallow: /annotation/genome/coccidioides_immitis/Rfam.html Disallow: /annotation/genome/coprinus_cinereus/AssemblyStructure.html Disallow: /annotation/genome/coprinus_cinereus/GeneDetails.html Disallow: /annotation/genome/coprinus_cinereus/Home.html Disallow: /annotation/genome/coprinus_cinereus/Pfam.html Disallow: /annotation/genome/coprinus_cinereus.2/GeneIndex.html Disallow: /annotation/genome/cryptococcus_neoformans/AssemblyStructure.html Disallow: /annotation/genome/cryptococcus_neoformans/Blast.html Disallow: /annotation/genome/cryptococcus_neoformans/DownloadResults.html Disallow: /annotation/genome/cryptococcus_neoformans/FeatureSearchResults.html Disallow: /annotation/genome/cryptococcus_neoformans/GeneIndex.html Disallow: /annotation/genome/cryptococcus_neoformans.2/Info.html Disallow: /annotation/genome/cryptococcus_neoformans/Home.html Disallow: /annotation/genome/cryptococcus_neoformans_b/AssemblyStructure.html Disallow: /annotation/genome/cryptococcus_neoformans_b/FeatureSearch.html Disallow: /annotation/genome/cryptococcus_neoformans_b/Home.html Disallow: /annotation/genome/culex_pipiens/AssemblyStructure.html Disallow: /annotation/genome/culex_pipiens/FeatureRedirect.html Disallow: /annotation/genome/culex_pipiens/FeatureSearch.html Disallow: /annotation/genome/francisella_tularensis_group/AssemblyStructure.html Disallow: /annotation/genome/francisella_tularensis_group/DownloadResults.html Disallow: /annotation/genome/francisella_tularensis_group/FeatureSearch.html Disallow: /annotation/genome/francisella_tularensis_group/FeatureSearchResults.html Disallow: /annotation/genome/francisella_tularensis_group/Home.html Disallow: /annotation/genome/francisella_tularensis_group/Pfam.html Disallow: /annotation/genome/fusarium_graminearum/AssemblyStructure.html Disallow: /annotation/genome/fusarium_graminearum/Blast.html Disallow: /annotation/genome/fusarium_graminearum/FeatureSearchResults.html Disallow: /annotation/genome/fusarium_graminearum/GeneDetails.html Disallow: /annotation/genome/fusarium_graminearum/Home.html Disallow: /annotation/genome/fusarium_group/AssemblyStructure.html Disallow: /annotation/genome/fusarium_group/Comments.html Disallow: /annotation/genome/fusarium_group/Contact.html Disallow: /annotation/genome/fusarium_group/FeatureSearch.html Disallow: /annotation/genome/fusarium_group/GeneDetails.html Disallow: /annotation/genome/fusarium_group/MultiHome.html Disallow: /annotation/genome/fusarium_group/TranscriptDetails.html Disallow: /annotation/genome/fusarium_verticillioides/AssemblyStructure.html Disallow: /annotation/genome/fusarium_verticillioides/Blast.html Disallow: /annotation/genome/fusarium_verticillioides/GeneDetails.html Disallow: /annotation/genome/histoplasma_capsulatum/AssemblyStructure.html Disallow: /annotation/genome/histoplasma_capsulatum/Blast.html Disallow: /annotation/genome/histoplasma_capsulatum/FeatureDetails.html Disallow: /annotation/genome/histoplasma_capsulatum/GenomesIndex.html Disallow: /annotation/genome/histoplasma_capsulatum/MultiHome.html Disallow: /annotation/genome/listeria_group/AssemblyStructure.html Disallow: /annotation/genome/listeria_group/Comments.html Disallow: /annotation/genome/listeria_group/Home.html Disallow: /annotation/genome/lodderomyces_elongisporus/AssemblyStructure.html Disallow: /annotation/genome/lodderomyces_elongisporus/FeatureDetails Disallow: /annotation/genome/lodderomyces_elongisporus/GeneDetails.html Disallow: /annotation/genome/magnaporthe_grisea/AssemblyStructure.html Disallow: /annotation/genome/magnaporthe_grisea/Blast.html Disallow: /annotation/genome/magnaporthe_grisea/FeatureDetails.html Disallow: /annotation/genome/magnaporthe_grisea/FeatureSearch.html Disallow: /annotation/genome/magnaporthe_grisea/FeatureSearchResults.html Disallow: /annotation/genome/magnaporthe_grisea/GenomeStats.html Disallow: /annotation/genome/magnaporthe_grisea/GeneDetails.html Disallow: /annotation/genome/magnaporthe_grisea/GeneLocus.html Disallow: /annotation/genome/magnaporthe_grisea/Home.html Disallow: /annotation/genome/magnaporthe_grisea/Pfam.html Disallow: /annotation/genome/magnaporthe_grisea/Regions.html Disallow: /annotation/genome/mesoplasma_florum/AssemblyStructure.html Disallow: /annotation/genome/mesoplasma_florum/Home.html Disallow: /annotation/genome/mycobacterium_tuberculosis_c/AssemblyStructure.html Disallow: /annotation/genome/mycobacterium_tuberculosis_c/FeatureSearch.html Disallow: /annotation/genome/mycobacterium_tuberculosis_c/FeatureSearchResults.html Disallow: /annotation/genome/mycobacterium_tuberculosis_spp/GeneDetails.html Disallow: /annotation/genome/mycobacterium_tuberculosis_spp/GenomesIndex.html Disallow: /annotation/genome/mycobacterium_tuberculosis_spp/Pfam.html Disallow: /annotation/genome/mycoplasma_mobile/AssemblyStructure.html Disallow: /annotation/genome/mycoplasma_mobile/Pfam.html Disallow: /annotation/genome/neurospora/AlleleDetails.html Disallow: /annotation/genome/neurospora/Alleles.html Disallow: /annotation/genome/neurospora/AssemblyStructure.html Disallow: /annotation/genome/neurospora/Blast.html Disallow: /annotation/genome/neurospora/Comments.html Disallow: /annotation/genome/neurospora/FeatureDetails.html Disallow: /annotation/genome/neurospora/FeatureRedirect.html Disallow: /annotation/genome/neurospora/FeatureSearch.html Disallow: /annotation/genome/neurospora/FeatureSearchResults.html Disallow: /annotation/genome/neurospora/GeneDetails.html Disallow: /annotation/genome/neurospora/GeneLocus.html Disallow: /annotation/genome/neurospora/Home.html Disallow: /annotation/genome/neurospora/Pfam.html Disallow: /annotation/genome/neurospora/PhenoEntry.html Disallow: /annotation/genome/neurospora/PhenotypeImage.html Disallow: /annotation/genome/neurospora/RetrieveGenes.html Disallow: /annotation/genome/neurospora/Rfam.html Disallow: /annotation/genome/neurospora/TranscriptDetails.html Disallow: /annotation/genome/phytophthora_infestans/AssemblyStructure.html Disallow: /annotation/genome/phytophthora_infestans/DownloadResults.html Disallow: /annotation/genome/phytophthora_infestans/FeatureSearchResults.html Disallow: /annotation/genome/phytophthora_infestans/Home.html Disallow: /annotation/genome/phytophthora_infestans/RetrieveGenes.htm Disallow: /annotation/genome/plasmodium_falciparum_spp/AssemblyStructure.html Disallow: /annotation/genome/plasmodium_falciparum_spp/FeatureSearch.html Disallow: /annotation/genome/plasmodium_falciparum_spp/MultiDownloads.html Disallow: /annotation/genome/plasmodium_falciparum_spp/News.html Disallow: /annotation/genome/plasmodium_falciparum_spp/Pfam.html Disallow: /annotation/genome/plasmodium_falciparum_spp/Rfam.html Disallow: /annotation/genome/puccinia_graminis/AssemblyStructure.html Disallow: /annotation/genome/puccinia_graminis/Blast.html Disallow: /annotation/genome/puccinia_graminis/Comments.html Disallow: /annotation/genome/puccinia_graminis/DownloadResults.html Disallow: /annotation/genome/puccinia_graminis/FeatureSearchResults.html Disallow: /annotation/genome/puccinia_graminis/Rfam.html Disallow: /annotation/genome/puccinia_graminis.3/AssemblyStructure.html Disallow: /annotation/genome/puccinia_graminis.3/Blast.html Disallow: /annotation/genome/puccinia_graminis.3/Comments.html Disallow: /annotation/genome/puccinia_graminis.3/DownloadResults.html Disallow: /annotation/genome/puccinia_graminis.3/FeatureSearchResults.html Disallow: /annotation/genome/puccinia_graminis.3/Rfam.html Disallow: /annotation/genome/pyrenophora_tritici_repentis/AssemblyStructure.html Disallow: /annotation/genome/pyrenophora_tritici_repentis/Downloads.html Disallow: /annotation/genome/pyrenophora_tritici_repentis/Regions.html Disallow: /annotation/genome/pyrenophora_tritici_repentis.3/AssemblyStructure.html Disallow: /annotation/genome/pyrenophora_tritici_repentis.3/Downloads.html Disallow: /annotation/genome/pyrenophora_tritici_repentis.3/Regions.html Disallow: /annotation/genome/rhizopus_oryzae/AssemblyStructure.html Disallow: /annotation/genome/rhizopus_oryzae/Blast.html Disallow: /annotation/genome/rhizopus_oryzae/FeatureDetails.html Disallow: /annotation/genome/rhizopus_oryzae/FeatureSearchResults.html Disallow: /annotation/genome/rhizopus_oryzae/GeneDetails.html Disallow: /annotation/genome/rhizopus_oryzae/Home.html Disallow: /annotation/genome/rhizopus_oryzae/TranscriptDetails.html Disallow: /annotation/genome/saccharomyces_cerevisiae/AssemblyStructure.html Disallow: /annotation/genome/saccharomyces_cerevisiae/Home.html Disallow: /annotation/genome/schizosaccharomyces_group.1/Blast.html Disallow: /annotation/genome/schizosaccharomyces_japonicus/Downloads.html Disallow: /annotation/genome/schizosaccharomyces_japonicus/MultiHome.html Disallow: /annotation/genome/sclerotinia_sclerotiorum/AssemblyStructure.html Disallow: /annotation/genome/sclerotinia_sclerotiorum/Blast.html Disallow: /annotation/genome/sclerotinia_sclerotiorum/DownloadResults.html Disallow: /annotation/genome/sclerotinia_sclerotiorum/FeatureSearch.html Disallow: /annotation/genome/sclerotinia_sclerotiorum/GeneDetails.htm Disallow: /annotation/genome/sclerotinia_sclerotiorum/Pfam.htm Disallow: /annotation/genome/sclerotinia_sclerotiorum/TranscriptDetails.html Disallow: /annotation/genome/stagonospora_nodorum/AssemblyStructure.html Disallow: /annotation/genome/stagonospora_nodorum/Blast.html Disallow: /annotation/genome/stagonospora_nodorum/Comments.html Disallow: /annotation/genome/stagonospora_nodorum/DownloadResults.htm Disallow: /annotation/genome/stagonospora_nodorum/FeatureSearch.html Disallow: /annotation/genome/stagonospora_nodorum/FeatureSearchResult.html Disallow: /annotation/genome/stagonospora_nodorum/GeneDetails.html Disallow: /annotation/genome/stagonospora_nodorum/GeneIndex.html Disallow: /annotation/genome/stagonospora_nodorum/Pfam.html Disallow: /annotation/genome/stagonospora_nodorum.2/Pfam.html Disallow: /annotation/genome/tbdb/AssemblyStructure.html Disallow: /annotation/genome/tbdb/Blast.html Disallow: /annotation/genome/tbdb/GeneIndex.html Disallow: /annotation/genome/tbdb/GeneLocus.html Disallow: /annotation/genome/tbdb/GenomeDescriptions.html Disallow: /annotation/genome/tbdb/GenomeStats.html Disallow: /annotation/genome/tbdb/MultiDownloads.html Disallow: /annotation/genome/tbdb/Pfam.html Disallow: /annotation/genome/tbdb/SearchOntologyFlat.html Disallow: /annotation/genome/uncinocarpus_reesii/AssemblyStructure.html Disallow: /annotation/genome/uncinocarpus_reesii/DownloadResults.html Disallow: /annotation/genome/uncinocarpus_reesii/FeatureDetails.html Disallow: /annotation/genome/uncinocarpus_reesii/FeatureSearchResults.html Disallow: /annotation/genome/uncinocarpus_reesii/GeneDetails.html Disallow: /annotation/genome/ustilago_maydis/AssemblyStructure.html Disallow: /annotation/genome/ustilago_maydis/Blast.html Disallow: /annotation/genome/ustilago_maydis/FeatureSearch.html Disallow: /annotation/genome/ustilago_maydis/Home.html Disallow: /annotation/genome/ustilago_maydis/Info.html Disallow: /annotation/genome/verticillium_dahliae.1/MultiHome.html # trying to save carbon from falling over completely from chembank crawling # never mind, it only reads it when it starts crawling :( # leaving it here in case gageorge wants to know what was being crawled # Disallow: /assays # Disallow: /assays/view-assay.htm # Disallow: /assays/view-plate.htm # Disallow: /assays/search/execute.htm # Disallow: /chemistry # Disallow: /chemistry/viewMolecule.htm # Disallow: /chemistry/search/execute.htm # ticket 97853 User-agent: BecomeBot Disallow: /annotation/genome/ #disallow Slurp spider from crawling annotation/genome 11/28/06 jcr #User-agent: Slurp #Disallow: /annotation/genome/ User-agent: msnbot Crawl-delay: 120 # # Allow us to index our own ftp # Broad SearchBot (Swish-e v2.4.0) User-agent: SwishSpider Allow: /ftp #Disallow: /cgi-bin/contig/sts #Disallow: /cgi-bin/seq/bac/ext_view #Disallow: /cgi-bin/annotation/neurospora/findfeatures.cgi #Disallow: /cgi-bin/annotation/methanosarcina/findfeatures.cgi #Disallow: /cgi-bin/annotation/magnaporthe/findfeatures.cgi