# Robots.txt file for dadoftheroads.com
# Controls crawler access to specified areas of the site

User-agent: *  # Applies to all web crawlers
Allow: /  # Allow crawling of all public pages

# Disallow access to specific directories to protect sensitive areas
Disallow: /cgi-bin/       # Prevent crawling of CGI scripts
Disallow: /admin/         # Prevent crawling of the admin panel
Disallow: /login/         # Prevent crawling of the login page
Disallow: /private/       # Prevent crawling of private or sensitive data

# Disallow access to staging and test environments
Disallow: /staging/       # Prevent crawling of the staging environment
Disallow: /test/          # Prevent crawling of test areas

# Disallow access to uploads if files should not be indexed
Disallow: /uploads/       # Prevent crawling of uploaded files

# Block specific file types from being crawled
Disallow: /*.pdf$         # Block crawling of PDF files
Disallow: /*.zip$         # Block crawling of ZIP files
Disallow: /*.tmp$         # Block crawling of temporary files

# Set a crawl delay to manage server load
Crawl-delay: 5            # Suggest a 5-second delay between requests

# Provide the sitemap location for efficient indexing
Sitemap: https://dadoftheroads.com/sitemap.xml

# Optional: Inform search engines about alternate sitemap types
https://dadoftheroads.com/us-location/sitemap.xml
https://dadoftheroads.com/automotive/sitemap.xml
https://dadoftheroads.com/blog/sitemap.xml
https://dadoftheroads.com/blog/Tire-Shop/sitemap.xml
https://tireathome.com/sitemap.xml
https://dadoftheroads.com/blog/tires/sitemap.xml
https://dadoftheroads.com/blog/tires/295-75R22.5/sitemap.xml