# robots.txt for PubTier
# Allow crawlers to index the application, but disallow crawling of raw data source files.

User-agent: *

# Disallow search engine crawlers from scraping the raw CORE CSV files (to save crawl budget and prevent duplicate content)
Disallow: /CORE/

# Allow indexing of all other paths
Allow: /