# Disallow : untuk melarang robot untuk masuk ke jelajahi daerah2 yang dilarang ex: /admin/
# User-agent : * --> berarti semua robot search engine 
# User-agent : xxxx --> berarti robot search engine bernama xxxx
# keterangan lebih jelas : http://www.searchtools.com/robots/robots-txt.html
# taruh filenya : di "folder utama/robots.txt"
# Terus di index tambahin tag ini sebagai meta, pilih salah satu 
#	<META NAME="ROBOTS" CONTENT="INDEX,FOLLOW">
#	<META NAME="ROBOTS" CONTENT="NOINDEX,FOLLOW">
#	<META NAME="ROBOTS" CONTENT="INDEX,NOFOLLOW">
#	<META NAME="ROBOTS" CONTENT="NOINDEX,NOFOLLOW">
# fungsi cari digoogle
# http://www.yourhtmlsource.com/promotion/metatags.html

User-agent: *

Disallow: /background/

Disallow: /blog/

Disallow: /cgi-bin/

Disallow: /info/articles/

Disallow: /info/conferences-past.html

Disallow: /info/meetings/examples/

Disallow: /info/meetings/thunderlizard/examples/

Disallow: /info/robots/

Disallow: /info/slides/

Disallow: /lists/

Disallow: /related/

Disallow: /reviews/

Disallow: /test/relativelinks/2ndlevel/http://

Disallow: /test/relativelinks/rtestprob/http://searchtools/about/

Disallow: /test/relativelinks/rtestprob/http://searchtools/analysis/

Disallow: /test/relativelinks/rtestprob/http://searchtools/guide/

Disallow: /test/relativelinks/rtestprob/http://searchtools/info/

Disallow: /test/relativelinks/rtestprob/http://searchtools/pub/

Disallow: /test/relativelinks/rtestprob/http://searchtools/robots/

Disallow: /test/relativelinks/rtestprob/http://searchtools/search/

Disallow: /test/relativelinks/rtestprob/http://searchtools/site/

Disallow: /test/relativelinks/rtestprob/http://searchtools/slides/

Disallow: /test/relativelinks/rtestprob/http://searchtools/surveys/

Disallow: /test/relativelinks/rtestprob/http://searchtools/tools/

Disallow: /test/robots/disal/

Disallow: /test/robots/noindex/

Disallow: /test/robots/partial

Disallow: /searchtools/

Disallow: /slides/examples/

Disallow: /ST/

Disallow: /st/

Disallow: /St/

Disallow: /wr/



# don't let search engines see the RSS feed, it's just confusing.



User-agent: Googlebot

User-agent: InfoNaviRobot

User-agent: TV33_Mercator

User-agent: AVSearch

User-agent: Mercator

User-agent: Scooter

User-agent: Slurp

User-agent: SearchengineLicenceSheep

User-agent: shadow

User-agent: MultiText

User-agent: FAST-WebCrawler

User-agent: Lycos_Spider

User-agent: Atomz

User-agent: htdig

User-agent: spider00.logika.net

User-agent: NetMechanic

User-agent: libwww-perl

User-agent: Teleport Pro

Disallow: /searchtools-rss.xml



# updated 2002-03-22 (disallow rtestprob links)

# updated 2002-06-25 (disallow info/slides links, info/robots/)

# updated 2002-07-25 (disallow /searchtools/ which is an alias)