Sitemap: http://www.unix.com/sitemap_index.xml.gz 

User-agent: *
Disallow: /bot-trap

User-agent: spbot
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: CometBird
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: ScoutJet
Disallow:  /

User-agent: BuzzBot
Disallow:  /

User-agent: Speedy Spider
Disallow:  /

User-agent: Twiceler
Disallow:  /

User-agent: GurujiBot
Disallow:  /

User-agent: VoilaBot
Disallow:  /

User-agent: Python-urllib
Disallow:  /

User-agent: Yanga WorldSearch Bot
Disallow:  /

User-agent: Mediapartners-Google
Crawl-delay: 0
Disallow: /attachment.php
Disallow: /avatar.php
Disallow: /d
Disallow: /calendar.php
Disallow: /editpost.php
Disallow: /login.php
Disallow: /misc.php
Disallow: /moderator.php
Disallow: /newreply.php
Disallow: /newthread.php
Disallow: /online.php
Disallow: /poll.php
Disallow: /postings.php
Disallow: /printthread.php
Disallow: /private.php
Disallow: /private2.php
Disallow: /project.php
Disallow: /profile.php
Disallow: /report.php
Disallow: /register.php
Disallow: /search.php
Disallow: /sendmessage.php
Disallow: /subscription.php
Disallow: /sendtofriend.php
Disallow: /threadrate.php
Disallow: /usercp.php
Disallow: /admincp/
Disallow: /modcp/
Disallow: /members/
Disallow: /moderators-forum/
Disallow: /main-cellblock/
Disallow: /underground/
Disallow: /jail-house/ 
Disallow: /vip-member-account-services/
Disallow: /members-only/ 


User-agent: Technoratibot
Crawl-delay: 3
Disallow: /attachment.php
Disallow: /avatar.php
Disallow: /d
Disallow: /calendar.php
Disallow: /editpost.php
Disallow: /login.php
Disallow: /misc.php
Disallow: /moderator.php
Disallow: /newreply.php
Disallow: /newthread.php
Disallow: /online.php
Disallow: /poll.php
Disallow: /postings.php
Disallow: /printthread.php
Disallow: /private.php
Disallow: /private2.php
Disallow: /project.php
Disallow: /profile.php
Disallow: /report.php
Disallow: /register.php
Disallow: /search.php
Disallow: /sendmessage.php
Disallow: /subscription.php
Disallow: /sendtofriend.php
Disallow: /threadrate.php
Disallow: /usercp.php
Disallow: /admincp/
Disallow: /modcp/
Disallow: /members/
Disallow: /moderators-forum/
Disallow: /main-cellblock/
Disallow: /underground/
Disallow: /jail-house/ 
Disallow: /vip-member-account-services/
Disallow: /members-only/ 

User-agent: Yeti
Disallow: /

User-agent: YebolBot
Disallow: /

User-agent: WebAlta Crawler
Disallow: /

User-agent: WebAlta
Disallow: /

User-agent: 3GSE
Disallow: /

User-agent: SnapPreviewBot
Disallow: /

User-agent: Spinn3r
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: 133.9.238.77
Disallow: /

User-agent: psbot
Disallow: /

User-agent: dotbot 
Disallow: /

User-agent: wget
Disallow: /

User-agent: WISEbot
Disallow: /

User-agent: ia_archiver
Disallow: /attachment.php
Disallow: /avatar.php
Disallow: /d
Disallow: /calendar.php
Disallow: /editpost.php
Disallow: /login.php
Disallow: /misc.php
Disallow: /moderator.php
Disallow: /newreply.php
Disallow: /newthread.php
Disallow: /online.php
Disallow: /poll.php
Disallow: /postings.php
Disallow: /printthread.php
Disallow: /private.php
Disallow: /private2.php
Disallow: /project.php
Disallow: /profile.php
Disallow: /report.php
Disallow: /register.php
Disallow: /search.php
Disallow: /sendmessage.php
Disallow: /subscription.php
Disallow: /sendtofriend.php
Disallow: /threadrate.php
Disallow: /usercp.php
Disallow: /admincp
Disallow: /modcp
Disallow: /members/
Disallow: /moderators-forum/
Disallow: /main-cellblock/
Disallow: /underground/
Disallow: /jail-house/ 
Disallow: /vip-member-account-services/
Disallow: /members-only/ 
Disallow: /unix-linux-job-board/
Disallow: /administrators-forum/ 

User-agent: Googlebot
Disallow: /attachment.php
Disallow: /avatar.php
Disallow: /calendar.php
Disallow: /*/calendar.php
Disallow: /casino.php
Disallow: /editpost.php
Disallow: /login.php
Disallow: /misc.php
Disallow: /moderator.php
Disallow: /newreply.php
Disallow: /newthread.php
Disallow: /nominate_topic.php
Disallow: /*/nominate_topic.php
Disallow: /online.php
Disallow: /poll.php
Disallow: /postings.php
Disallow: /printthread.php
Disallow: /private.php
Disallow: /private2.php
Disallow: /project.php
Disallow: /profile.php
Disallow: /report.php
Disallow: /register.php
Disallow: /search.php
Disallow: /sendmessage.php
Disallow: /subscription.php
Disallow: /sendtofriend.php
Disallow: /threadrate.php
Disallow: /usercp.php
Disallow: /admincp/
Disallow: /modcp/
Disallow: /members/
Disallow: /moderators-forum/
Disallow: /main-cellblock/
Disallow: /underground/
Disallow: /jail-house/ 
Disallow: /vip-member-account-services/
Disallow: /members-only/ 
Disallow: /administrators-forum/ 
Disallow: /openx/
Disallow: /man-page/All/
Disallow: /clientscript/
Disallow: /blog/wp-login.php


User-agent: Slurp
Crawl-delay: 0
Disallow: /nominate_topic.php
Disallow: /*/nominate_topic.php
Disallow: /attachment.php
Disallow: /avatar.php
Disallow: /calendar.php
Disallow: /editpost.php
Disallow: /login.php
Disallow: /members/
Disallow: /memberlist.php
Disallow: /member2.php
Disallow: /misc.php
Disallow: /moderator.php
Disallow: /newreply.php
Disallow: /newthread.php
Disallow: /online.php
Disallow: /poll.php
Disallow: /postings.php
Disallow: /printthread.php
Disallow: /private.php
Disallow: /private2.php
Disallow: /project.php
Disallow: /profile.php
Disallow: /report.php
Disallow: /register.php
Disallow: /search.php
Disallow: /sendmessage.php
Disallow: /subscription.php
Disallow: /sendtofriend.php
Disallow: /threadrate.php
Disallow: /usercp.php
Disallow: /admincp/
Disallow: /modcp/
Disallow: /members/
Disallow: /moderators-forum/
Disallow: /main-cellblock/
Disallow: /underground/
Disallow: /jail-house/ 
Disallow: /vip-member-account-services/
Disallow: /members-only/ 
Disallow: /unix-linux-job-board/
Disallow: /administrators-forum/ 
Disallow: /openx/
Disallow: /clientscript/

User-agent: * 
Crawl-delay: 4
Disallow: /attachment.php
Disallow: /avatar.php
Disallow: /calendar.php
Disallow: /editpost.php
Disallow: /login.php
Disallow: /memberlist.php
Disallow: /misc.php
Disallow: /moderator.php
Disallow: /newreply.php
Disallow: /newthread.php
Disallow: /online.php
Disallow: /poll.php
Disallow: /postings.php
Disallow: /printthread.php
Disallow: /private.php
Disallow: /private2.php
Disallow: /project.php
Disallow: /profile.php
Disallow: /report.php
Disallow: /register.php
Disallow: /search.php
Disallow: /sendmessage.php
Disallow: /subscription.php
Disallow: /sendtofriend.php
Disallow: /threadrate.php
Disallow: /usercp.php
Disallow: /admincp/
Disallow: /modcp/
Disallow: /members/
Disallow: /moderators-forum/
Disallow: /main-cellblock/
Disallow: /underground/
Disallow: /jail-house/ 
Disallow: /vip-member-account-services/
Disallow: /members-only/ 
Disallow: /administrators-forum/ 
Disallow: /nominate_topic.php
Disallow: /*/nominate_topic.php
Disallow: /openx/
Disallow: /clientscript/