User-Agent: *
Disallow: /bitrix/
Disallow: /admin/
Disallow: /full/
Disallow: /_book/
Disallow: /software/download/
Disallow: /tools/_services/download/
Disallow: /bitrix/tools/
Disallow: /search/
Disallow: /forum/view_profile.php
Disallow: /forum/user/
Disallow: /forum/search/
Disallow: /forum/topic/new/
Disallow: /forum/topic/add/
Disallow: /forum/subscr_list.php
Disallow: /forum/send_message.php 
Disallow: /forum/search.php
Disallow: /forum/new_topic.php 
Disallow: /forum/move.php
Disallow: /forum/active.php
Disallow: /forum/forum_auth.php
Disallow: /forum/forum_posts.asp
Disallow: /auth.php
Disallow: /blog/blog_auth.php
Disallow: /user.php
Disallow: /blog/video/

User-agent: Slurp
Crawl-delay: 120

User-agent: Mozilla/4.0 (compatible; Netcraft Web Server Survey)  
Disallow: /

User-agent: NetCraft
Disallow: /

User-agent: msnbot
Crawl-delay: 120

User-agent: OmniExplorer_Bot
Disallow: /

User-agent: FreeFind
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: Nutch
Disallow: /

User-agent: Jetbot/1.0
Disallow: /

User-agent: Jetbot
Disallow: /

User-agent: WebVac
Disallow: /

User-agent: Stanford
Disallow: /

User-agent: naver
Disallow: /

User-agent: dumbot
Disallow: /

User-agent: Hatena Antenna
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: grub
Disallow: /

User-agent: looksmart
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: larbin
Disallow: /

User-agent: b2w/0.1
Disallow: /

User-agent: Copernic
Disallow: /

User-agent: psbot
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: NetMechanic
Disallow: /

User-agent: URL_Spider_Pro
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: CopyRightCheck
Disallow: /

User-agent: Crescent
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: ProWebWalker
Disallow: /

User-agent: CheeseBot
Disallow: /

User-agent: LNSpiderguy
Disallow: /

User-agent: Mozilla
Disallow: /

User-agent: mozilla
Disallow: /

User-agent: mozilla/3
Disallow: /

User-agent: Twiceler
Disallow: /

User-agent: Webalta
Disallow: /

User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT)
Disallow: /

User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)
Disallow: /

User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98)
Disallow: /

User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP)
Disallow: /

User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000)
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ia_archiver/1.6
Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: twiceler
Disallow: /

User-agent: Googlebot
Disallow: /cgi/
Disallow: /glossary/
  
User-agent: Indy Library
Disallow: /

User-agent: WebCopier	
Disallow: /

User-agent: Netcraft
User-agent: Netcraft SSL Server Survey - contact abuse@netcraft.com
Disallow: /