robots.txt 351 Bytes
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

########## robots.txt ############################
##################################################

#  Allowing asqatasun crawler access to all content
User-Agent: asqatasun
Allow: /
Disallow:

# Blocking all web crawlers from all content
User-agent: *
Disallow: /

# Allowing all web crawlers access to all content
# User-agent: *
# Disallow: