# 
# robots.txt
# 
# This file is to prevent the crawling and indexing of certain parts 
# of your site by web crawlers and spiders run by sites like Yahoo! 
# and Google. By telling these "robots" where not to go on your site, 
# you save bandwidth and server resources
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt 
# Ignored: http://example.com/site/robots.txt 
# 
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html  

User-agent: * 
Disallow: /*/404
Sitemap: https://www.tatasteeluk.com/sitemap.xml
# CSS, JS, Images 
Allow: /core/*.css$ 
Allow: /core/*.css?
Allow: /core/*.js$ 
Allow: /core/*.js? 
Allow: /core/*.gif 
Allow: /core/*.jpg 
Allow: /core/*.jpeg
Allow: /core/*.png 
Allow: /core/*.svg 
Allow: /profiles/*.css$ 
Allow: /profiles/*.css?
Allow: /profiles/*.js$ 
Allow: /profiles/*.js? 
Allow: /profiles/*.gif 
Allow: /profiles/*.jpg 
Allow: /profiles/*.jpeg
Allow: /profiles/*.png 
Allow: /profiles/*.svg 
# Directories 
Disallow: /core/ 
Disallow: /profiles/ 
# Files 
Disallow: /README.txt 
Disallow: /web.config 

# Paths (clean URLs) 

Disallow: /admin/ 

Disallow: /comment/reply/ 

Disallow: /filter/tips 

Disallow: /node/add/ 

Disallow: /search/ 

Disallow: /search* 

Disallow: /nl/search* 

Disallow: /de/search* 

Disallow: /en-US/search* 

Disallow: /it/search* 

Disallow: /da/search* 

Disallow: /ru* 

Disallow: /fi* 

Disallow: /no/search* 

Disallow: /sv/search* 

Disallow: /es/search* 

Disallow: /fr/search* 

Disallow: /pl/search* 

Disallow: /?q=search/ 

Disallow: /user/register/ 

Disallow: /user/password/ 

Disallow: /user/login/ 

Disallow: /user/logout/ 

Disallow: /node* 

Disallow: /nl/node* 

Disallow: /de/node* 

Disallow: /en-US/node* 

Disallow: /it/node* 

Disallow: /da/node* 

Disallow: /no/node* 

Disallow: /sv/node* 

Disallow: /es/node* 

Disallow: /fr/node* 

Disallow: /pl/node* 

Disallow: *node 

Disallow: *taxonomy 

Disallow: *search 

Disallow: *download 

Disallow: *.dwg 

Disallow: /taxonomy* 

Disallow: /nl/taxonomy* 

Disallow: /de/taxonomy* 

Disallow: /en-US/taxonomy* 

Disallow: /it/taxonomy* 

Disallow: /da/taxonomy* 

Disallow: /ru/taxonomy* 

Disallow: /fi/taxonomy* 

Disallow: /no/taxonomy* 

Disallow: /sv/taxonomy* 

Disallow: /es/taxonomy* 

Disallow: /fr/taxonomy* 

Disallow: /pl/taxonomy* 

Disallow: /products/cold-rolled/forming/dc01 

Disallow: /products/cold-rolled/structural-steel/ca200 

Disallow: /products/hot-rolled/durbar-floor-plate/s275jr+ar 

Disallow: /products/hot-rolled/durbar-floor-plate/s235jr+ar 

Disallow: /products/hot-rolled/forming/dd11 

Disallow: /products/hot-rolled/hss/s315mc 

Disallow: /products/hot-rolled/hss/s355mc 

Disallow: /products/hot-rolled/hss/s460mc 

Disallow: /products/hot-rolled/hss/s500mc 

Disallow: /products/hot-rolled/hss/s550mc 

Disallow: /products/hot-rolled/hss/s700mc 

Disallow: /products/hot-rolled/structural-steel/s275j2+ar 

Disallow: /products/hot-rolled/structural-steel/s275jr+ar 

Disallow: /products/hot-rolled/structural-steel/s355j0+ar 

Disallow: /products/hot-rolled/structural-steel/s355j2+ar 

Disallow: /products/hot-rolled/structural-steel/s355jr+ar 

Disallow: /products/metallic-coated/steel-for-forming/dx51d+z 

Disallow: /download* 

Disallow: /nl/download* 

Disallow: /de/download* 

Disallow: /en-US/download* 

Disallow: /it/download* 

Disallow: /da/download* 

Disallow: /ru/download* 

Disallow: /fi/download* 

Disallow: /no/download* 

Disallow: /sv/download* 

Disallow: /es/download* 

Disallow: /fr/download* 

Disallow: /pl/download* 


 

# Paths (no clean URLs) 

Disallow: /index.php/admin/ 

Disallow: /index.php/comment/reply/ 

Disallow: /index.php/filter/tips 

Disallow: /index.php/node/add/ 

Disallow: /index.php/search/ 

Disallow: /index.php/user/password/ 

Disallow: /index.php/user/register/ 

Disallow: /index.php/user/login/ 

Disallow: /index.php/user/logout/ 

Disallow: /index.php/node/ 

Disallow: /index.php/products/cold-rolled/forming/dc01 

Disallow: /index.php/products/cold-rolled/structural-steel/ca200 

Disallow: /index.php/products/hot-rolled/durbar-floor-plate/s275jr+ar 

Disallow: /index.php/products/hot-rolled/durbar-floor-plate/s235jr+ar 

Disallow: /index.php/products/hot-rolled/forming/dd11 

Disallow: /index.php/products/hot-rolled/hss/s315mc 

Disallow: /index.php/products/hot-rolled/hss/s355mc 

Disallow: /index.php/products/hot-rolled/hss/s460mc 

Disallow: /index.php/products/hot-rolled/hss/s500mc 

Disallow: /index.php/products/hot-rolled/hss/s550mc 

Disallow: /index.php/products/hot-rolled/hss/s700mc 

Disallow: /index.php/products/hot-rolled/structural-steel/s275j2+ar 

Disallow: /index.php/products/hot-rolled/structural-steel/s275jr+ar 

Disallow: /index.php/products/hot-rolled/structural-steel/s355j0+ar 

Disallow: /index.php/products/hot-rolled/structural-steel/s355j2+ar 

Disallow: /index.php/products/hot-rolled/structural-steel/s355jr+ar 

Disallow: /index.php/products/metallic-coated/steel-for-forming/dx51d+z 

# block Yandexbot
User-agent: Yandex
Disallow: /*f%5B0%5D=

User-agent: Barkrowler
Disallow: /*f%5B0%5D=

User-agent: GPTBot
Disallow: /*f%5B0%5D=

User-agent: Bytespider
Disallow: /*f%5B0%5D=

User-agent: meta-externalagent
Disallow: /search