User-agent: Scrapy Allow: / # Default robots file version:2 User-agent: * Disallow: /wp-admin/ Allow: /*.css Allow: /*.js Disallow: /*? Sitemap: https://thephoenixlimited.com/sitemap.xml