robots.php

# Google Lighthouse erratically flags robots.txt as invalid

Below a list of only big-tech disallows, smaller bots just read the line with a * and bots.php to avoid being blocked

User-agent: *
Crawl-delay: 10

User-agent: Googlebot
Disallow: /

User-agent: Google-Read-Aloud
Disallow: /

User-agent: Storebot-Google
Disallow: /

User-agent: Google-InspectionTool
Disallow: /

User-agent: GoogleOther
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: APIs-Google
Disallow: /

User-agent: AdsBot-Google-Mobile
Disallow: /

User-agent: AdsBot-Google
Disallow: /

User-agent: Mediapartners-Google
Disallow: /

User-agent: Google-Web-Preview
Disallow: /

User-agent: Bingbot
Disallow: /

User-agent: AdIdxBot
Disallow: /

User-agent: BingPreview
Disallow: /

User-agent: DuckDuckBot
Disallow: /

User-agent: MicrosoftPreview
Disallow: /

User-agent: msnbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: facebookexternalhit
Disallow: /

User-agent: facebookcatalog
Disallow: /

User-agent: InstagramBot
Disallow: /

User-agent: Instagram-Embed
Disallow: /

User-agent: meta-webindexer
Disallow: /

User-agent: meta-externalads
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: meta-externalfetcher
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: AliyunSecBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Baiduspider-image
Disallow: /

User-agent: Baiduspider-video
Disallow: /

User-agent: Baiduspider-news
Disallow: /

User-agent: Baiduspider-favo
Disallow: /

User-agent: Baiduspider-ads
Disallow: /

User-agent: Baiduspider-cpro
Disallow: /

Sitemap: https://thewww.top/sitemapindex.xml
Sitemap: https://thewww.top/sitemap.txt
Sitemap: https://thewww.top/sitemap.php
Sitemap: https://thewww.top/sitemap.xml