{
 "dataset": "How the Tranco top 200 address AI in robots.txt and llms.txt",
 "snapshot": "2026-10-24",
 "fetch_started": "2026-10-24",
 "list_date": "2026-10-23",
 "upload_date": "2026-10-24",
 "list_id": "262N9",
 "source": "https://tranco-list.eu/download/262N9/1000000",
 "list_sha256": "9029ed7ae14aa46a825a41fbf3412b9d391be1a90ecc6525f7096f29e4894126",
 "license": "CC BY 4.0",
 "summary": {
  "sites": 200,
  "robots_parseable": 95,
  "robots_states": {
   "parseable": 95,
   "no_website_at_ranked_domain": 50,
   "absent_or_unparseable": 42,
   "not_checked": 13
  },
  "content_signal_sites": 2,
  "llms_states": {
   "absent": 65,
   "present": 17,
   "not_checked": 68,
   "no_website_at_ranked_domain": 50
  },
  "failure_reasons": {
   "robots": {
    "dns_failure": 50,
    "robots_unreachable": 9,
    "robots_disallows": 0,
    "redirect_left_domain": 0,
    "timeout": 0,
    "other": 4
   },
   "llms": {
    "dns_failure": 50,
    "robots_unreachable": 10,
    "robots_disallows": 16,
    "redirect_left_domain": 26,
    "timeout": 0,
    "other": 16
   }
  },
  "crawlers": {
   "GPTBot": {
    "named": 28,
    "named_full_block": 12,
    "effective_full_block": 20
   },
   "ClaudeBot": {
    "named": 27,
    "named_full_block": 14,
    "effective_full_block": 22
   },
   "Google-Extended": {
    "named": 28,
    "named_full_block": 13,
    "effective_full_block": 19
   },
   "PerplexityBot": {
    "named": 29,
    "named_full_block": 14,
    "effective_full_block": 21
   },
   "CCBot": {
    "named": 21,
    "named_full_block": 12,
    "effective_full_block": 24
   },
   "ChatGPT-User": {
    "named": 18,
    "named_full_block": 7,
    "effective_full_block": 17
   },
   "OAI-SearchBot": {
    "named": 19,
    "named_full_block": 5,
    "effective_full_block": 15
   },
   "Claude-SearchBot": {
    "named": 15,
    "named_full_block": 6,
    "effective_full_block": 16
   },
   "Claude-User": {
    "named": 14,
    "named_full_block": 7,
    "effective_full_block": 17
   },
   "Perplexity-User": {
    "named": 10,
    "named_full_block": 6,
    "effective_full_block": 18
   },
   "Bytespider": {
    "named": 20,
    "named_full_block": 13,
    "effective_full_block": 25
   },
   "Amazonbot": {
    "named": 15,
    "named_full_block": 10,
    "effective_full_block": 19
   },
   "Applebot-Extended": {
    "named": 21,
    "named_full_block": 12,
    "effective_full_block": 21
   },
   "meta-externalagent": {
    "named": 15,
    "named_full_block": 8,
    "effective_full_block": 18
   },
   "meta-externalfetch": {
    "named": 0,
    "named_full_block": 0,
    "effective_full_block": 12
   },
   "cohere-ai": {
    "named": 11,
    "named_full_block": 7,
    "effective_full_block": 19
   },
   "cohere-training-ai-bot": {
    "named": 0,
    "named_full_block": 0,
    "effective_full_block": 12
   },
   "Diffbot": {
    "named": 10,
    "named_full_block": 8,
    "effective_full_block": 20
   },
   "AI2Bot": {
    "named": 6,
    "named_full_block": 6,
    "effective_full_block": 18
   },
   "anthropic-ai": {
    "named": 17,
    "named_full_block": 9,
    "effective_full_block": 21
   },
   "FacebookBot": {
    "named": 14,
    "named_full_block": 9,
    "effective_full_block": 18
   },
   "omgili": {
    "named": 12,
    "named_full_block": 10,
    "effective_full_block": 22
   },
   "omgilibot": {
    "named": 10,
    "named_full_block": 8,
    "effective_full_block": 20
   }
  }
 },
 "rows": [
  {
   "rank": 1,
   "domain": "google.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c",
    "error": null,
    "truncated": false,
    "url": "https://www.google.com/robots.txt",
    "requested_url": "https://google.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7",
    "error": null,
    "truncated": false,
    "url": "https://google.com/llms.txt",
    "requested_url": "https://google.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 2,
   "domain": "cloudflare.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98",
    "error": null,
    "truncated": false,
    "url": "https://www.cloudflare.com/robots.txt",
    "requested_url": "https://cloudflare.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [
     "Content-Signal: ai-train=yes, search=yes, ai-input=yes"
    ],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "ccbot",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "cohere-ai",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "anthropic-ai",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "81d53b543800b2a5fb3521421b0b522d09097c19f8e19ba092c25901d97db7ac",
    "error": null,
    "truncated": false,
    "url": "https://www.cloudflare.com/llms.txt",
    "requested_url": "https://cloudflare.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 3,
   "domain": "amazonaws.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain;charset=UTF-8",
    "sha256": "c1f42540a70f0dcd98d98565c46f486ed49a464f077ef1f1362c5470b638ae98",
    "error": null,
    "truncated": false,
    "url": "https://aws.amazon.com:443/robots.txt",
    "requested_url": "https://amazonaws.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://aws.amazon.com:443/llms.txt",
    "requested_url": "https://amazonaws.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 4,
   "domain": "facebook.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "ae34650d8199df7bcfbcd9da82b8fa911e522d3201af364ee6cae6e9872653d2",
    "error": null,
    "truncated": false,
    "url": "https://facebook.com/robots.txt",
    "requested_url": "https://facebook.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://facebook.com/llms.txt",
    "requested_url": "https://facebook.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 5,
   "domain": "gstatic.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "8c4889928ff7cb4f10356ba998e21e3bd8496a01bc03a0875349417f3c4dba27",
    "error": null,
    "truncated": false,
    "url": "https://www.gstatic.com/robots.txt",
    "requested_url": "https://gstatic.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://gstatic.com/llms.txt",
    "requested_url": "https://gstatic.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 6,
   "domain": "ezviz7.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://ezviz7.com/robots.txt",
    "requested_url": "https://ezviz7.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://ezviz7.com/llms.txt",
    "requested_url": "https://ezviz7.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 7,
   "domain": "googleapis.com",
   "robots": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2",
    "error": null,
    "truncated": false,
    "url": "https://googleapis.com/robots.txt",
    "requested_url": "https://googleapis.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7",
    "error": null,
    "truncated": false,
    "url": "https://googleapis.com/llms.txt",
    "requested_url": "https://googleapis.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 8,
   "domain": "akamai.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akamai.net/robots.txt",
    "requested_url": "https://akamai.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akamai.net/llms.txt",
    "requested_url": "https://akamai.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 9,
   "domain": "microsoft.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain;charset=iso-8859-1",
    "sha256": "79a2a24e3e2c00deaba2fc1922b9113f5508e74133911eadea4ebb00b5de9768",
    "error": null,
    "truncated": false,
    "url": "https://www.microsoft.com/robots.txt",
    "requested_url": "https://microsoft.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "6769df293e90f9bf5e31c25958f1ea6c3b7715aa49f3176afe7694e324e0e6b1",
    "error": null,
    "truncated": false,
    "url": "https://www.microsoft.com/llms.txt",
    "requested_url": "https://microsoft.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 10,
   "domain": "fbcdn.net",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "8155e929427b274a20a74eca43333ed5d5fe3ba4eea4035aea704f57a72f0542",
    "error": null,
    "truncated": false,
    "url": "https://fbcdn.net/robots.txt",
    "requested_url": "https://fbcdn.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://fbcdn.net/llms.txt",
    "requested_url": "https://fbcdn.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 11,
   "domain": "youtube.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "96f8640ca357626e7d9a894057fef8b5746eb838c29ed68fc176af58c9348c6f",
    "error": null,
    "truncated": false,
    "url": "https://www.youtube.com/robots.txt",
    "requested_url": "https://youtube.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "1a658204f72b2a399c49d1e5ba78595c7c1c19e29552697b4f1378bfb8d573b8",
    "error": null,
    "truncated": false,
    "url": "https://www.youtube.com/llms.txt",
    "requested_url": "https://youtube.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 12,
   "domain": "apple.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7",
    "error": null,
    "truncated": false,
    "url": "https://www.apple.com/robots.txt",
    "requested_url": "https://apple.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "2c33262e1cb35568064b86d703038e420bda53717cb88b78b7ff81e861ac5cde",
    "error": null,
    "truncated": false,
    "url": "https://www.apple.com/llms.txt",
    "requested_url": "https://apple.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 13,
   "domain": "instagram.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "fe84b3913976aa37e983fd44f2ace9cf62973a512ae694eb18f3cf02066602b1",
    "error": null,
    "truncated": false,
    "url": "https://www.instagram.com/robots.txt",
    "requested_url": "https://instagram.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://instagram.com/llms.txt",
    "requested_url": "https://instagram.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 14,
   "domain": "hicloudcam.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://hicloudcam.com/robots.txt",
    "requested_url": "https://hicloudcam.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://hicloudcam.com/llms.txt",
    "requested_url": "https://hicloudcam.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 15,
   "domain": "mail.ru",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "d3fd6768e203ab4008918459ce8d1ccc9e31d703cc01c9765902bcfc894822bf",
    "error": null,
    "truncated": false,
    "url": "https://mail.ru/robots.txt",
    "requested_url": "https://mail.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://mail.ru/llms.txt",
    "requested_url": "https://mail.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 16,
   "domain": "twitter.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain;charset=utf-8",
    "sha256": "14a72e33b3e9af07e1e8f5f645bfa00ed3fe9f9c87249d861d369d33d3f3ddc6",
    "error": null,
    "truncated": false,
    "url": "https://twitter.com/robots.txt",
    "requested_url": "https://twitter.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": false,
      "effective_group": "meta-externalagent",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "facebookbot",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://twitter.com/llms.txt",
    "requested_url": "https://twitter.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 17,
   "domain": "akamaiedge.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akamaiedge.net/robots.txt",
    "requested_url": "https://akamaiedge.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akamaiedge.net/llms.txt",
    "requested_url": "https://akamaiedge.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 18,
   "domain": "linkedin.com",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "ffc578fc5037662ca9e046e913abe9df7c31e978632a2dd649951237fcde2fdd",
    "error": null,
    "truncated": false,
    "url": "https://linkedin.com/robots.txt",
    "requested_url": "https://linkedin.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "63abaaa18e7cf3be08ab912f3a0a8e23c9d4fb9f5c5f75eedf7f267fc11acbfc",
    "error": null,
    "truncated": false,
    "url": "https://linkedin.com/llms.txt",
    "requested_url": "https://linkedin.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 19,
   "domain": "akadns.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akadns.net/robots.txt",
    "requested_url": "https://akadns.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akadns.net/llms.txt",
    "requested_url": "https://akadns.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 20,
   "domain": "dzen.ru",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "a6e7d16a66dcbbfe8d64ec966726922ac7736fd28f3a4b5bede7897992c92086",
    "error": null,
    "truncated": false,
    "url": "https://dzen.ru/robots.txt",
    "requested_url": "https://dzen.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://sso.passport.yandex.ru/push",
    "requested_url": "https://dzen.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 21,
   "domain": "googlevideo.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "URLError",
    "truncated": null,
    "url": "https://googlevideo.com/robots.txt",
    "requested_url": "https://googlevideo.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "robots_unreachable"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://googlevideo.com/llms.txt",
    "requested_url": "https://googlevideo.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 22,
   "domain": "domaincontrol.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://domaincontrol.com/robots.txt",
    "requested_url": "https://domaincontrol.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "other"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://domaincontrol.com/llms.txt",
    "requested_url": "https://domaincontrol.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 23,
   "domain": "azure.com",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "5525de1d828d7703e06d04f11101f679d8e01322eccbeec39bfab596134d039b",
    "error": null,
    "truncated": false,
    "url": "https://azure.microsoft.com/en-us",
    "requested_url": "https://azure.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://azure.microsoft.com/llms.txt",
    "requested_url": "https://azure.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 24,
   "domain": "gtld-servers.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://gtld-servers.net/robots.txt",
    "requested_url": "https://gtld-servers.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://gtld-servers.net/llms.txt",
    "requested_url": "https://gtld-servers.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 25,
   "domain": "googletagmanager.com",
   "robots": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2",
    "error": null,
    "truncated": false,
    "url": "https://googletagmanager.com/robots.txt",
    "requested_url": "https://googletagmanager.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7",
    "error": null,
    "truncated": false,
    "url": "https://googletagmanager.com/llms.txt",
    "requested_url": "https://googletagmanager.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 26,
   "domain": "office.com",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "02688096e3d851a90ad674186f27fddeade41303797f3134ff1844497efccc00",
    "error": null,
    "truncated": false,
    "url": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
    "requested_url": "https://office.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
    "requested_url": "https://office.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 27,
   "domain": "live.com",
   "robots": {
    "status": 417,
    "content_type": "",
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": null,
    "truncated": false,
    "url": "https://outlook.live.com/mail/",
    "requested_url": "https://live.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 417,
    "content_type": "",
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": null,
    "truncated": false,
    "url": "https://outlook.live.com/mail/",
    "requested_url": "https://live.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 28,
   "domain": "shiabank.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "URLError",
    "truncated": null,
    "url": "https://shiabank.com/robots.txt",
    "requested_url": "https://shiabank.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "robots_unreachable"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://shiabank.com/llms.txt",
    "requested_url": "https://shiabank.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 29,
   "domain": "cloudfront.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://cloudfront.net/robots.txt",
    "requested_url": "https://cloudfront.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://cloudfront.net/llms.txt",
    "requested_url": "https://cloudfront.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 30,
   "domain": "amazon.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "74fa27e28cd9d80bdd155ffecf3a033ea2e195a80c83d6fa46acb20665c3cb4d",
    "error": null,
    "truncated": false,
    "url": "https://www.amazon.com/robots.txt",
    "requested_url": "https://amazon.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "chatgpt-user",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "oai-searchbot",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-searchbot",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-user",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexity-user",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": true,
      "effective_group": "meta-externalagent",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "cohere-ai",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "diffbot",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ai2bot",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html;charset=UTF-8",
    "sha256": "abeb2137f14f1973cb5949a7ea52daf3d9cb22e85cc8d8abd7e0bbff114f1048",
    "error": null,
    "truncated": false,
    "url": "https://www.amazon.com/llms.txt",
    "requested_url": "https://amazon.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 31,
   "domain": "appsflyersdk.com",
   "robots": {
    "status": 403,
    "content_type": "text/xml",
    "sha256": "95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83",
    "error": null,
    "truncated": false,
    "url": "https://appsflyersdk.com/robots.txt",
    "requested_url": "https://appsflyersdk.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 403,
    "content_type": "text/xml",
    "sha256": "95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83",
    "error": null,
    "truncated": false,
    "url": "https://appsflyersdk.com/llms.txt",
    "requested_url": "https://appsflyersdk.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 32,
   "domain": "github.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "b622e2dd9b60e8540ffd1f4b0974c9dbee73bce606f9660c6344d09357889956",
    "error": null,
    "truncated": false,
    "url": "https://github.com/robots.txt",
    "requested_url": "https://github.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "anthropic-ai",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "71d217ec3186152052ca1050932c0e6e48e766d3bc5a1359b50aa140436d297f",
    "error": null,
    "truncated": false,
    "url": "https://github.com/llms.txt",
    "requested_url": "https://github.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 33,
   "domain": "wikipedia.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "48b98d80c8444d37cdf50cc9a4d2257b66039477e581066d2356062a4cbd678f",
    "error": null,
    "truncated": false,
    "url": "https://en.wikipedia.org/robots.txt",
    "requested_url": "https://wikipedia.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "76563b4eeb07e461f97c018d6f0744fdaf2b0af79ebd90410e0452733d164bdf",
    "error": null,
    "truncated": false,
    "url": "https://en.wikipedia.org/llms.txt",
    "requested_url": "https://wikipedia.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 34,
   "domain": "aaplimg.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://aaplimg.com/robots.txt",
    "requested_url": "https://aaplimg.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://aaplimg.com/llms.txt",
    "requested_url": "https://aaplimg.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 35,
   "domain": "trafficmanager.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://trafficmanager.net/robots.txt",
    "requested_url": "https://trafficmanager.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://trafficmanager.net/llms.txt",
    "requested_url": "https://trafficmanager.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 36,
   "domain": "sharepoint.com",
   "robots": {
    "status": 302,
    "content_type": "text/html",
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "redirect_limit",
    "truncated": false,
    "url": "https://www.microsoft.com/microsoft-365/sharepoint/collaboration",
    "requested_url": "https://sharepoint.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "other"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://sharepoint.com/llms.txt",
    "requested_url": "https://sharepoint.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 37,
   "domain": "fastly.net",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "23afd0d2868ab27736e15dd91bfa2b061fff83da62d60d131f335d21311dc6f3",
    "error": null,
    "truncated": false,
    "url": "https://www.fastly.com/robots.txt",
    "requested_url": "https://fastly.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://www.fastly.com/llms.txt",
    "requested_url": "https://fastly.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 38,
   "domain": "cloud.microsoft",
   "robots": {
    "status": 200,
    "content_type": "text/html",
    "sha256": "3f2f5dc8fcb6d3aa1ee1fc853f68c0c8b9a2c2c9d350127120471681520628bb",
    "error": null,
    "truncated": false,
    "url": "https://m365.cloud.microsoft/",
    "requested_url": "https://cloud.microsoft/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/html",
    "sha256": "3f2f5dc8fcb6d3aa1ee1fc853f68c0c8b9a2c2c9d350127120471681520628bb",
    "error": null,
    "truncated": false,
    "url": "https://m365.cloud.microsoft/",
    "requested_url": "https://cloud.microsoft/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 39,
   "domain": "whatsapp.net",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "b0dc29a03f811635b16c416f0c890b4acad4112533b78feed9bdb13de7156cf9",
    "error": null,
    "truncated": false,
    "url": "https://www.whatsapp.com/robots.txt",
    "requested_url": "https://whatsapp.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://www.whatsapp.com/llms.txt",
    "requested_url": "https://whatsapp.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 40,
   "domain": "apple-dns.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://apple-dns.net/robots.txt",
    "requested_url": "https://apple-dns.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://apple-dns.net/llms.txt",
    "requested_url": "https://apple-dns.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 41,
   "domain": "office.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://office.net/robots.txt",
    "requested_url": "https://office.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://office.net/llms.txt",
    "requested_url": "https://office.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 42,
   "domain": "bing.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "40d2da70c3b443225d5b01cc7b05117944980347637cc95ab285c6e4f5548235",
    "error": null,
    "truncated": false,
    "url": "https://www.bing.com:443/robots.txt",
    "requested_url": "https://bing.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "",
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": null,
    "truncated": false,
    "url": "https://www.bing.com:443/llms.txt",
    "requested_url": "https://bing.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 43,
   "domain": "triplinkintl.com",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "879da012c83f73e6247a625d3831da6283a0649c768213c3db4ee028d8a93fdf",
    "error": null,
    "truncated": false,
    "url": "https://www.triplinkintl.com",
    "requested_url": "https://triplinkintl.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "7452e47592077e9930ff02b203371fbe3f942a6569c2465e6b3ecf7222b1d096",
    "error": null,
    "truncated": false,
    "url": "https://www.triplinkintl.com",
    "requested_url": "https://triplinkintl.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 44,
   "domain": "netflix.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "1e14fa0284e9757b52b48dde38b390effae68afdd72447088e72f18f49b1aa09",
    "error": null,
    "truncated": false,
    "url": "https://www.netflix.com/robots.txt",
    "requested_url": "https://netflix.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-searchbot",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-user",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://netflix.com/llms.txt",
    "requested_url": "https://netflix.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 45,
   "domain": "gwfb.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://gwfb.net/robots.txt",
    "requested_url": "https://gwfb.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://gwfb.net/llms.txt",
    "requested_url": "https://gwfb.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 46,
   "domain": "googleusercontent.com",
   "robots": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2",
    "error": null,
    "truncated": false,
    "url": "https://googleusercontent.com/robots.txt",
    "requested_url": "https://googleusercontent.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7",
    "error": null,
    "truncated": false,
    "url": "https://googleusercontent.com/llms.txt",
    "requested_url": "https://googleusercontent.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 47,
   "domain": "x.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain;charset=utf-8",
    "sha256": "14a72e33b3e9af07e1e8f5f645bfa00ed3fe9f9c87249d861d369d33d3f3ddc6",
    "error": null,
    "truncated": false,
    "url": "https://x.com/robots.txt",
    "requested_url": "https://x.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": false,
      "effective_group": "meta-externalagent",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "facebookbot",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://x.com/llms.txt",
    "requested_url": "https://x.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 48,
   "domain": "gandi.net",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "3fd451a6143f6a84977d65af0ae1e26a153466423e8d7e08e672f3d1466e1cdc",
    "error": null,
    "truncated": false,
    "url": "https://www.gandi.net/robots.txt",
    "requested_url": "https://gandi.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "faa3c94bfe16278cb8f380b5f0884f67ce86ac93c2cca2d440addef229e2f37c",
    "error": null,
    "truncated": false,
    "url": "https://www.gandi.net/llms.txt",
    "requested_url": "https://gandi.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 49,
   "domain": "youtu.be",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "c290485c8002c51c24c95765948dbcec9ec88c1f181638d245418a36194fd1a0",
    "error": null,
    "truncated": false,
    "url": "https://youtu.be/robots.txt",
    "requested_url": "https://youtu.be/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://www.youtube.com/watch",
    "requested_url": "https://youtu.be/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 50,
   "domain": "doubleclick.net",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c",
    "error": null,
    "truncated": false,
    "url": "https://doubleclick.net/robots.txt",
    "requested_url": "https://doubleclick.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7",
    "error": null,
    "truncated": false,
    "url": "https://doubleclick.net/llms.txt",
    "requested_url": "https://doubleclick.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 51,
   "domain": "microsoftonline.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://microsoftonline.com/robots.txt",
    "requested_url": "https://microsoftonline.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://microsoftonline.com/llms.txt",
    "requested_url": "https://microsoftonline.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 52,
   "domain": "pinterest.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "9304b88a1d231c58e0dd32aa74590048f58fbb8314c268041f101f546163cfab",
    "error": null,
    "truncated": false,
    "url": "https://www.pinterest.com/robots.txt",
    "requested_url": "https://pinterest.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "facebookbot",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://pinterest.com/llms.txt",
    "requested_url": "https://pinterest.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 53,
   "domain": "icloud.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "7c0da5b622a8703a69e156d3df581d1a2c5d4106a0da2e85c4db8f18c5a9e9c8",
    "error": null,
    "truncated": false,
    "url": "https://www.icloud.com/robots.txt",
    "requested_url": "https://icloud.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "60b391103f3f6b7ac9e37bd76d1ad2ac9c0b3fa775afb0ed276aa94908251b45",
    "error": null,
    "truncated": false,
    "url": "https://www.icloud.com/llms.txt",
    "requested_url": "https://icloud.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 54,
   "domain": "googlesyndication.com",
   "robots": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2",
    "error": null,
    "truncated": false,
    "url": "https://googlesyndication.com/robots.txt",
    "requested_url": "https://googlesyndication.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7",
    "error": null,
    "truncated": false,
    "url": "https://googlesyndication.com/llms.txt",
    "requested_url": "https://googlesyndication.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 55,
   "domain": "wordpress.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "65185a701db3ab195bcb2f86657300e13153f758af110ac0ffeeea5dc19ab5dc",
    "error": null,
    "truncated": false,
    "url": "https://wordpress.org/robots.txt",
    "requested_url": "https://wordpress.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "ccbot",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": false,
      "effective_group": "bytespider",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "applebot-extended",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "anthropic-ai",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "731a4c979c79b5a350375f8b089fc26ed70b20d395de39ad305b716de233b4af",
    "error": null,
    "truncated": false,
    "url": "https://wordpress.org/llms.txt",
    "requested_url": "https://wordpress.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 56,
   "domain": "cloudflare.net",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "de73d32142f114b8d18bbfd7fa5779ccee5f7a1753873402cfdab59444834ccd",
    "error": null,
    "truncated": false,
    "url": "https://cloudflare.net/robots.txt",
    "requested_url": "https://cloudflare.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285",
    "error": null,
    "truncated": false,
    "url": "https://cloudflare.net/llms.txt",
    "requested_url": "https://cloudflare.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 57,
   "domain": "skype.com",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "a67bf2ad0c54aee256365bba89e2197a7cbc964221a9162bcfc298f0834c6a72",
    "error": null,
    "truncated": false,
    "url": "https://teams.live.com/free",
    "requested_url": "https://skype.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://teams.live.com/free",
    "requested_url": "https://skype.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 58,
   "domain": "windows.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "URLError",
    "truncated": null,
    "url": "https://windows.net/robots.txt",
    "requested_url": "https://windows.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "robots_unreachable"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://windows.net/llms.txt",
    "requested_url": "https://windows.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 59,
   "domain": "goo.gl",
   "robots": {
    "status": 400,
    "content_type": "text/html; charset=utf-8",
    "sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11",
    "error": null,
    "truncated": false,
    "url": "https://goo.gl/robots.txt",
    "requested_url": "https://goo.gl/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 400,
    "content_type": "text/html; charset=utf-8",
    "sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11",
    "error": null,
    "truncated": false,
    "url": "https://goo.gl/llms.txt",
    "requested_url": "https://goo.gl/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 60,
   "domain": "adobe.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=UTF-8",
    "sha256": "41081faa0fd323774e225ab51e8a73262055b24ecc7732557229fb93938f9d84",
    "error": null,
    "truncated": false,
    "url": "https://www.adobe.com/robots.txt",
    "requested_url": "https://adobe.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=UTF-8",
    "sha256": "811353cb603ff604ecd257b7e24643f1c7e157837015221a951d9d2db0c494d8",
    "error": null,
    "truncated": false,
    "url": "https://www.adobe.com/llms.txt",
    "requested_url": "https://adobe.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 61,
   "domain": "windowsupdate.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://windowsupdate.com/robots.txt",
    "requested_url": "https://windowsupdate.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://windowsupdate.com/llms.txt",
    "requested_url": "https://windowsupdate.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 62,
   "domain": "pv-cdn.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://pv-cdn.net/robots.txt",
    "requested_url": "https://pv-cdn.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://pv-cdn.net/llms.txt",
    "requested_url": "https://pv-cdn.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 63,
   "domain": "tiktok.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "835705a2ecb862f49a813b3aa4061e0edfefbb8318316b371f8fa2dba4018802",
    "error": null,
    "truncated": false,
    "url": "https://www.tiktok.com/robots.txt",
    "requested_url": "https://tiktok.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "chatgpt-user",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "oai-searchbot",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-searchbot",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-user",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexity-user",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": true,
      "effective_group": "meta-externalagent",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ai2bot",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "anthropic-ai",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "18a9607871a2a3dd32d37c3a541367d3abf67e2a5350d1c4b7b7274dd2df6df1",
    "error": null,
    "truncated": false,
    "url": "https://www.tiktok.com/404",
    "requested_url": "https://tiktok.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 64,
   "domain": "whatsapp.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "b0dc29a03f811635b16c416f0c890b4acad4112533b78feed9bdb13de7156cf9",
    "error": null,
    "truncated": false,
    "url": "https://www.whatsapp.com/robots.txt",
    "requested_url": "https://whatsapp.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=\"utf-8\"",
    "sha256": "8ddcc1da653356a0eed50c6bb315ae27c264e4fb58cbf26f51101f8e7462f5ed",
    "error": null,
    "truncated": false,
    "url": "https://www.whatsapp.com/llms.txt",
    "requested_url": "https://whatsapp.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 65,
   "domain": "digicert.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=ISO-8859-1",
    "sha256": "a139a54387d10bfd57f5223202cb56558934677d20460b789d4445d2549537f7",
    "error": null,
    "truncated": false,
    "url": "https://www.digicert.com/robots.txt",
    "requested_url": "https://digicert.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-searchbot",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-user",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexity-user",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=ISO-8859-1",
    "sha256": "02169ec84a351a1d16d7d35ae9186f3edaa67dc827c2a7b55f9cea8f32abd439",
    "error": null,
    "truncated": false,
    "url": "https://www.digicert.com/llms.txt",
    "requested_url": "https://digicert.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 66,
   "domain": "gvt1.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://gvt1.com/robots.txt",
    "requested_url": "https://gvt1.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://gvt1.com/llms.txt",
    "requested_url": "https://gvt1.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 67,
   "domain": "googledomains.com",
   "robots": {
    "status": 200,
    "content_type": "text/html",
    "sha256": "cc1d1aa533262d560de5fa9ce308eae59d20c0fe846bd1d5f86e078c57f00854",
    "error": null,
    "truncated": false,
    "url": "https://domains.google/",
    "requested_url": "https://googledomains.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://domains.google/",
    "requested_url": "https://googledomains.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 68,
   "domain": "zoom.us",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "b3ed9f53f38c480b046b4d5a648aa7a879c6720957301506dc282b32fb2f5593",
    "error": null,
    "truncated": false,
    "url": "https://zoom.us/robots.txt",
    "requested_url": "https://zoom.us/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html;charset=utf-8",
    "sha256": "2dc58d17dbc6271fa1ab3dee459d898854a2ebecbd1e4022d7b660e076807775",
    "error": null,
    "truncated": true,
    "url": "https://zoom.us/llms.txt",
    "requested_url": "https://zoom.us/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 69,
   "domain": "yahoo.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "dfcd18f101783bde40c3bdca97bb90dbea1653321fd8b57f81cf5a6a6dc00b4b",
    "error": null,
    "truncated": false,
    "url": "https://www.yahoo.com/robots.txt",
    "requested_url": "https://yahoo.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "chatgpt-user",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-searchbot",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "cohere-ai",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "diffbot",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "anthropic-ai",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "facebookbot",
      "effective_full_block": true
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgilibot",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "747d07de457fb3fc9d842113ac9d244f5487cf2d1f0ba5d0101e492c91e082ae",
    "error": null,
    "truncated": false,
    "url": "https://www.yahoo.com/llms.txt",
    "requested_url": "https://yahoo.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 70,
   "domain": "myfritz.net",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "0267852a7136ff65cf570bc69e024a5198e7d31ef564f90165cd5333250a62ac",
    "error": null,
    "truncated": false,
    "url": "https://myfritz.net/robots.txt",
    "requested_url": "https://myfritz.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://myfritz.net/llms.txt",
    "requested_url": "https://myfritz.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 71,
   "domain": "endpoints.news",
   "robots": {
    "status": 403,
    "content_type": "text/html",
    "sha256": "d9155d3dd03031e2d6ef17f9201c49210c31ea3806a2b12d4fecf13cc6b6e431",
    "error": null,
    "truncated": false,
    "url": "https://endpoints.news/robots.txt",
    "requested_url": "https://endpoints.news/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 403,
    "content_type": "text/html",
    "sha256": "ff1de46879b4d5c6b98367bb7fcc1040f5351622a29db3edfce4056e69fd28c5",
    "error": null,
    "truncated": false,
    "url": "https://endpoints.news/llms.txt",
    "requested_url": "https://endpoints.news/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 72,
   "domain": "wa.me",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "6ceebd7d44206c973321a46e46b94102a8edcc37eea87394b2dd94ffe713df8d",
    "error": null,
    "truncated": false,
    "url": "https://wa.me/robots.txt",
    "requested_url": "https://wa.me/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://api.whatsapp.com/resolve/",
    "requested_url": "https://wa.me/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 73,
   "domain": "lsrelayaccess.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "URLError",
    "truncated": null,
    "url": "https://lsrelayaccess.com/robots.txt",
    "requested_url": "https://lsrelayaccess.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "robots_unreachable"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://lsrelayaccess.com/llms.txt",
    "requested_url": "https://lsrelayaccess.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 74,
   "domain": "edgekey.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://edgekey.net/robots.txt",
    "requested_url": "https://edgekey.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://edgekey.net/llms.txt",
    "requested_url": "https://edgekey.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 75,
   "domain": "tiktokcdn.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://tiktokcdn.com/robots.txt",
    "requested_url": "https://tiktokcdn.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://tiktokcdn.com/llms.txt",
    "requested_url": "https://tiktokcdn.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 76,
   "domain": "opera.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "0c6b878d90d9783f0a0972e38888f8dfef8393e317be80a266a82f3da548f20f",
    "error": null,
    "truncated": false,
    "url": "https://www.opera.com/robots.txt",
    "requested_url": "https://opera.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "abefbde9fb5c1ef40e063c0d21aeff38c71e1f9899b3b7cc8d924a89deffb013",
    "error": null,
    "truncated": false,
    "url": "https://www.opera.com/llms.txt",
    "requested_url": "https://opera.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 77,
   "domain": "roblox.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "a4a9450540cd8263a373527026c92df49b7289076e87a3bb5bb72a7c28bc5213",
    "error": null,
    "truncated": false,
    "url": "https://www.roblox.com/robots.txt",
    "requested_url": "https://roblox.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "51070764a8aa1a0a2f8072ae82d236abe9701c424bde3c685cc4939151ba1031",
    "error": null,
    "truncated": false,
    "url": "https://www.roblox.com/llms.txt",
    "requested_url": "https://roblox.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 78,
   "domain": "spotify.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=UTF-8",
    "sha256": "b68c89548f813d5c465c13051353b9452dba066f14146cafabb46dafd239b738",
    "error": null,
    "truncated": false,
    "url": "https://www.spotify.com/robots.txt",
    "requested_url": "https://spotify.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "f3d80b19dacce7d9edf1b4f5a4adfe80acd089230d5c202c7c802919d5ef9339",
    "error": null,
    "truncated": false,
    "url": "https://www.spotify.com/ph-en/llms.txt",
    "requested_url": "https://spotify.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 79,
   "domain": "workers.dev",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "28b75dddbb74d985f2d73f0f8cd9a2d12540712d6ce213653fc8c7a4dbbd9600",
    "error": null,
    "truncated": true,
    "url": "https://www.cloudflare.com/products/workers/",
    "requested_url": "https://workers.dev/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "other"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://workers.dev/llms.txt",
    "requested_url": "https://workers.dev/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "other"
   }
  },
  {
   "rank": 80,
   "domain": "baidu.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "c28ebeffb7baf9bd6407109e80dc95d87c75c0eaecc20b23884cb33111cc5df0",
    "error": null,
    "truncated": false,
    "url": "https://www.baidu.com/robots.txt",
    "requested_url": "https://baidu.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://baidu.com/llms.txt",
    "requested_url": "https://baidu.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 81,
   "domain": "tiktokv.com",
   "robots": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248",
    "error": null,
    "truncated": false,
    "url": "https://tiktokv.com/robots.txt",
    "requested_url": "https://tiktokv.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248",
    "error": null,
    "truncated": false,
    "url": "https://tiktokv.com/llms.txt",
    "requested_url": "https://tiktokv.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 82,
   "domain": "w2rt6byl5nf1opmc-a.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://w2rt6byl5nf1opmc-a.com/robots.txt",
    "requested_url": "https://w2rt6byl5nf1opmc-a.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://w2rt6byl5nf1opmc-a.com/llms.txt",
    "requested_url": "https://w2rt6byl5nf1opmc-a.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 83,
   "domain": "yandex.net",
   "robots": {
    "status": 404,
    "content_type": "",
    "sha256": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5",
    "error": null,
    "truncated": false,
    "url": "https://yandex.net/robots.txt",
    "requested_url": "https://yandex.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://ya.ru/llms.txt",
    "requested_url": "https://yandex.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 84,
   "domain": "akam.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akam.net/robots.txt",
    "requested_url": "https://akam.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akam.net/llms.txt",
    "requested_url": "https://akam.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 85,
   "domain": "ntp.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "5a5a4434f108ef034c256b9d3e169e516fb806aa6f6353e53eff9655732fbb79",
    "error": null,
    "truncated": false,
    "url": "https://www.ntp.org/robots.txt",
    "requested_url": "https://ntp.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394",
    "error": null,
    "truncated": false,
    "url": "https://www.ntp.org/llms.txt",
    "requested_url": "https://ntp.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 86,
   "domain": "nginx.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "8f5b42f5d23ca623fa42f917a5c90b8677ef8d7c9b5ffc0324bbea8e7067f26e",
    "error": null,
    "truncated": false,
    "url": "https://nginx.org/robots.txt",
    "requested_url": "https://nginx.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "c139e927eb2b8ee58310e535724a39059f1b2ac33f8bda130ac2c52087cc3a7b",
    "error": null,
    "truncated": false,
    "url": "https://nginx.org/llms.txt",
    "requested_url": "https://nginx.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 87,
   "domain": "msn.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "aefbffa0a0a083e30637972046b42afc1569f7813c5687739a5ecdee0a6be4dd",
    "error": null,
    "truncated": false,
    "url": "https://www.msn.com/robots.txt",
    "requested_url": "https://msn.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "chatgpt-user",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "oai-searchbot",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-searchbot",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-user",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexity-user",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": true,
      "effective_group": "meta-externalagent",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "cohere-ai",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "diffbot",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ai2bot",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "anthropic-ai",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "facebookbot",
      "effective_full_block": true
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgilibot",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff",
    "error": null,
    "truncated": false,
    "url": "https://www.msn.com/llms.txt",
    "requested_url": "https://msn.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 88,
   "domain": "a2z.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://a2z.com/robots.txt",
    "requested_url": "https://a2z.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://a2z.com/llms.txt",
    "requested_url": "https://a2z.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 89,
   "domain": "qq.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "1f5d4ddc6513b31998acadbb56fe02e8b9eb244a3eb64d36f446f146e0bc3d5f",
    "error": null,
    "truncated": false,
    "url": "https://www.qq.com/robots.txt",
    "requested_url": "https://qq.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/html",
    "sha256": "dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464",
    "error": null,
    "truncated": false,
    "url": "https://h5.news.qq.com/static/babyhome.htm",
    "requested_url": "https://qq.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 90,
   "domain": "vimeo.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain;charset=UTF-8",
    "sha256": "021ea0af6b29208b5ad678715c9e6671373ae3a937b8e879a366e5f497f88b10",
    "error": null,
    "truncated": false,
    "url": "https://vimeo.com/robots.txt",
    "requested_url": "https://vimeo.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "diffbot",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "anthropic-ai",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "facebookbot",
      "effective_full_block": true
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgilibot",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "f5e4eadf97e088aebb9395adca8b0d28ff5a61cbbac6ee3478c4fa4941086d23",
    "error": null,
    "truncated": false,
    "url": "https://vimeo.com/llms.txt",
    "requested_url": "https://vimeo.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 91,
   "domain": "amazonvideo.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "e924a08cd574df9664040688a18d722c6f08c1f8a7352ea8748dcda4795d04c8",
    "error": null,
    "truncated": false,
    "url": "https://www.primevideo.com/robots.txt",
    "requested_url": "https://amazonvideo.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "chatgpt-user",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-user",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexity-user",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": true,
      "effective_group": "meta-externalagent",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "cohere-ai",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "diffbot",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ai2bot",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://www.primevideo.com/llms.txt",
    "requested_url": "https://amazonvideo.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 92,
   "domain": "3gppnetwork.org",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://3gppnetwork.org/robots.txt",
    "requested_url": "https://3gppnetwork.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://3gppnetwork.org/llms.txt",
    "requested_url": "https://3gppnetwork.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 93,
   "domain": "b-cdn.net",
   "robots": {
    "status": 200,
    "content_type": "text/html",
    "sha256": "e4d0d8e9e204b8bfc33dbf21ed0cdb4246d55d823de1cae7feb68d4a31bc1033",
    "error": null,
    "truncated": false,
    "url": "https://bunny.net",
    "requested_url": "https://b-cdn.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://bunny.net",
    "requested_url": "https://b-cdn.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 94,
   "domain": "blogspot.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "2076a51a5ea6b870209770053655a5e2499bddd4ae6a6a40b5453b9898168a34",
    "error": null,
    "truncated": false,
    "url": "https://www.blogger.com/robots.txt",
    "requested_url": "https://blogspot.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://www.blogger.com/llms.txt",
    "requested_url": "https://blogspot.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 95,
   "domain": "mozilla.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "cf0fd2842659297b0d6fce407bed86bf424ae6666051b881dbd72e871d687fe3",
    "error": null,
    "truncated": false,
    "url": "https://www.mozilla.org/robots.txt",
    "requested_url": "https://mozilla.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "5a7ce4f28f278c632d3dd264885c354861c1166a955befc50849587dd002b1fb",
    "error": null,
    "truncated": false,
    "url": "https://www.mozilla.org/en-US/llms.txt",
    "requested_url": "https://mozilla.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 96,
   "domain": "f5.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "eca371645a1484bc5d7dfc8e150f188b7e70d7754e6c00c1770de364f38e1401",
    "error": null,
    "truncated": false,
    "url": "https://www.f5.com/robots.txt",
    "requested_url": "https://f5.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "6009d711f0d0fdc6def5d0dcdf570240d69d2e06bcd4bda64d5db23b2d37e4a8",
    "error": null,
    "truncated": false,
    "url": "https://www.f5.com/llms.txt",
    "requested_url": "https://f5.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 97,
   "domain": "root-servers.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://root-servers.net/robots.txt",
    "requested_url": "https://root-servers.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://root-servers.net/llms.txt",
    "requested_url": "https://root-servers.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 98,
   "domain": "epicgames.com",
   "robots": {
    "status": 403,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "7885d8b20076befaf5811ff04e717e0cdf3a6c29f69851ee2dea9061fd611d5c",
    "error": null,
    "truncated": false,
    "url": "https://www.epicgames.com/site/robots.txt",
    "requested_url": "https://epicgames.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 403,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "4f5b69cdf4c248c6b321e823c99b45ddfeb9b84e1746896f4b2cd33baa029b73",
    "error": null,
    "truncated": false,
    "url": "https://www.epicgames.com/site/llms.txt",
    "requested_url": "https://epicgames.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 99,
   "domain": "t.me",
   "robots": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "bd5e6537d1fa8b7e136d3589dd722410eb3836f3b4b8e21733fba881463f5272",
    "error": null,
    "truncated": false,
    "url": "https://t.me/robots.txt",
    "requested_url": "https://t.me/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://telegram.org/",
    "requested_url": "https://t.me/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 100,
   "domain": "nginx.com",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "a9ecaf663420d012c71efea83a8069a1c6c99e28777c27fd97644a12ba39bf6b",
    "error": null,
    "truncated": false,
    "url": "https://www.f5.com/products/nginx",
    "requested_url": "https://nginx.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://www.f5.com/products/nginx",
    "requested_url": "https://nginx.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 101,
   "domain": "wordpress.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "84c4c3060151ce89afc882d80efb25de85a58540cf16272a931b591beb8e7e3e",
    "error": null,
    "truncated": false,
    "url": "https://wordpress.com/robots.txt",
    "requested_url": "https://wordpress.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "aa47908f5a28b561e6ccd4976c6233a395d94ab0ded5a170f0b0709a29ceb712",
    "error": null,
    "truncated": false,
    "url": "https://wordpress.com/llms.txt",
    "requested_url": "https://wordpress.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 102,
   "domain": "reddit.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "41aea42110242175c7b4247f222fa95ac6caee62044af686b684785832781857",
    "error": null,
    "truncated": false,
    "url": "https://www.reddit.com/robots.txt",
    "requested_url": "https://reddit.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://reddit.com/llms.txt",
    "requested_url": "https://reddit.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 103,
   "domain": "yandex.ru",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "c51a208183d07eb761c7dea99d104021b60c0840f91d43194f016aa5a44a3f46",
    "error": null,
    "truncated": false,
    "url": "https://yandex.ru/robots.txt",
    "requested_url": "https://yandex.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "35c1da8e993cc1135f6181f46d68514cbb58f60e1229f79691005c195dd4e3c8",
    "error": null,
    "truncated": false,
    "url": "https://yandex.ru/llms.txt",
    "requested_url": "https://yandex.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 104,
   "domain": "aws.dev",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://aws.dev/robots.txt",
    "requested_url": "https://aws.dev/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://aws.dev/llms.txt",
    "requested_url": "https://aws.dev/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 105,
   "domain": "edgesuite.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://edgesuite.net/robots.txt",
    "requested_url": "https://edgesuite.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://edgesuite.net/llms.txt",
    "requested_url": "https://edgesuite.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 106,
   "domain": "sentry.io",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "c0f7d803d97db6c0bf75804b97eed4a533f7249c27994e253ad4fa403652da96",
    "error": null,
    "truncated": false,
    "url": "https://sentry.io/robots.txt",
    "requested_url": "https://sentry.io/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [
     "Content-Signal: search=yes, ai-input=yes, ai-train=yes"
    ],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "2a102f2d4fba74e64d9de527b121ea698399187eae0d8711a6243b65d4047e19",
    "error": null,
    "truncated": false,
    "url": "https://sentry.io/llms.txt",
    "requested_url": "https://sentry.io/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 107,
   "domain": "lencr.org",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "ce272a3973c82f315270e83afcb1f39770ffb454b0d0c3bc71ab54c7cadc8703",
    "error": null,
    "truncated": false,
    "url": "https://letsencrypt.org/docs/lencr.org/",
    "requested_url": "https://lencr.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://letsencrypt.org/docs/lencr.org",
    "requested_url": "https://lencr.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 108,
   "domain": "ui.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "3a07e69b7c6f2ba775e98171bf02b7c40e7a4624ef9a0819de63a02d903f360e",
    "error": null,
    "truncated": false,
    "url": "https://ui.com/robots.txt",
    "requested_url": "https://ui.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "ccbot",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "applebot-extended",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "297dacfa76cb5f76cb6a5d24f86726855771a7ce2b8b553833d39c777e3dfaeb",
    "error": null,
    "truncated": false,
    "url": "https://ui.com/llms.txt",
    "requested_url": "https://ui.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 109,
   "domain": "europa.eu",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=UTF-8",
    "sha256": "bede8dd1fdd4b10f8a4ba2f0087e13cce1d83a52c192323b7959fd8213aeeb31",
    "error": null,
    "truncated": false,
    "url": "https://europa.eu/robots.txt",
    "requested_url": "https://europa.eu/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "f4e967da9fbc255ba1b32a6154c3c4a25ca619ccdff02a49fe6249a58707ecc0",
    "error": null,
    "truncated": false,
    "url": "https://european-union.europa.eu/not-found_en",
    "requested_url": "https://europa.eu/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 110,
   "domain": "yccdn.ru",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://yccdn.ru/robots.txt",
    "requested_url": "https://yccdn.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://yccdn.ru/llms.txt",
    "requested_url": "https://yccdn.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 111,
   "domain": "gvt2.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://gvt2.com/robots.txt",
    "requested_url": "https://gvt2.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://gvt2.com/llms.txt",
    "requested_url": "https://gvt2.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 112,
   "domain": "2mdn.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://2mdn.net/robots.txt",
    "requested_url": "https://2mdn.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://2mdn.net/llms.txt",
    "requested_url": "https://2mdn.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 113,
   "domain": "chatgpt.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "0676a43da83b5eb1bd955755568418567669886c53729349582fbc9b548e0679",
    "error": null,
    "truncated": false,
    "url": "https://chatgpt.com/robots.txt",
    "requested_url": "https://chatgpt.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "anthropic-ai",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "facebookbot",
      "effective_full_block": true
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgilibot",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://chatgpt.com/llms.txt",
    "requested_url": "https://chatgpt.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 114,
   "domain": "ipify.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "e5c4b84484ee4216e9373be99380320c25dd94805f99f0a805846f087636553f",
    "error": null,
    "truncated": false,
    "url": "https://www.ipify.org/robots.txt",
    "requested_url": "https://ipify.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "56b9de14bbce84faf8e4e3c99d35037a15edc78aa09f1d7c1e1f24a196a482d9",
    "error": null,
    "truncated": false,
    "url": "https://www.ipify.org/llms.txt",
    "requested_url": "https://ipify.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 115,
   "domain": "okcdn.ru",
   "robots": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "9145e451d2330fa2b4c9f1accfc1d289eae481ab19f0f1030ca870946881b343",
    "error": null,
    "truncated": false,
    "url": "https://st.okcdn.ru/robots.txt",
    "requested_url": "https://okcdn.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "URLError",
    "truncated": null,
    "url": "https://st.okcdn.ru/llms.txt",
    "requested_url": "https://okcdn.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 116,
   "domain": "cdninstagram.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://cdninstagram.com/robots.txt",
    "requested_url": "https://cdninstagram.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://cdninstagram.com/llms.txt",
    "requested_url": "https://cdninstagram.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 117,
   "domain": "bit.ly",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "aeddaf37d71afe500676c334c63a84340d99c91651b9d41bc0c4b2f9437d6cd8",
    "error": null,
    "truncated": false,
    "url": "https://bit.ly/robots.txt",
    "requested_url": "https://bit.ly/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "http://zu.lk/llms.txt",
    "requested_url": "https://bit.ly/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 118,
   "domain": "cdn77.org",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://cdn77.org/robots.txt",
    "requested_url": "https://cdn77.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://cdn77.org/llms.txt",
    "requested_url": "https://cdn77.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 119,
   "domain": "intuit.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "99f9354d3dc7685b0a7c25eb9370fb367e240013a28fd6600bf705984c5c1444",
    "error": null,
    "truncated": false,
    "url": "https://www.intuit.com/robots.txt",
    "requested_url": "https://intuit.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "23db7511adac8d695410917aa5bdd7e2cad0fd29ef382968db19194a0a37d57c",
    "error": null,
    "truncated": false,
    "url": "https://www.intuit.com/llms.txt",
    "requested_url": "https://intuit.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 120,
   "domain": "nic.ru",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "d0db68a344d1ad23628a7baec8bdbdf58a763073717bd3cf01ebba3d3478e8ef",
    "error": null,
    "truncated": false,
    "url": "https://nic.ru/robots.txt",
    "requested_url": "https://nic.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "d0db68a344d1ad23628a7baec8bdbdf58a763073717bd3cf01ebba3d3478e8ef",
    "error": null,
    "truncated": false,
    "url": "https://nic.ru/llms.txt",
    "requested_url": "https://nic.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 121,
   "domain": "samsung.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "e07ce06f3220d0496ee9bec373ddb2e2fdc272391a4b0153c453bd39b08e39f8",
    "error": null,
    "truncated": false,
    "url": "https://www.samsung.com/robots.txt",
    "requested_url": "https://samsung.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "d248d7acb33e01eff77c0c2fe1b2f25196ce8d9265ce133583435a80b686589f",
    "error": null,
    "truncated": false,
    "url": "https://www.samsung.com/llms.txt",
    "requested_url": "https://samsung.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 122,
   "domain": "office365.com",
   "robots": {
    "status": 403,
    "content_type": "text/html",
    "sha256": "4686b7c61f163cc8b8c09e6f5645d6f30407f6033e314e62d1cdec1a745fa7b5",
    "error": null,
    "truncated": false,
    "url": "https://office365.com/robots.txt",
    "requested_url": "https://office365.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 403,
    "content_type": "text/html",
    "sha256": "4686b7c61f163cc8b8c09e6f5645d6f30407f6033e314e62d1cdec1a745fa7b5",
    "error": null,
    "truncated": false,
    "url": "https://office365.com/llms.txt",
    "requested_url": "https://office365.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 123,
   "domain": "aliyuncs.com",
   "robots": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "9ef27bcedb3684d735d27a353a31b13f972d65e77b86f0ca433143f9c9da0a7a",
    "error": null,
    "truncated": false,
    "url": "https://aliyuncs.com/robots.txt",
    "requested_url": "https://aliyuncs.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "e051bcdd0afc05f9a45422b8c34062c0c31b78f36655298ff9da190b0bc89358",
    "error": null,
    "truncated": false,
    "url": "https://aliyuncs.com/llms.txt",
    "requested_url": "https://aliyuncs.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 124,
   "domain": "fastly-edge.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://fastly-edge.com/robots.txt",
    "requested_url": "https://fastly-edge.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://fastly-edge.com/llms.txt",
    "requested_url": "https://fastly-edge.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 125,
   "domain": "skybridge.click",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "331ea9090db0c9f6f597bd9840fd5b171830f6e0b3ba1cb24dfa91f0c95aedc1",
    "error": null,
    "truncated": false,
    "url": "https://skybridge.click/robots.txt",
    "requested_url": "https://skybridge.click/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://skybridge.click/llms.txt",
    "requested_url": "https://skybridge.click/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 126,
   "domain": "outlook.com",
   "robots": {
    "status": 417,
    "content_type": "",
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": null,
    "truncated": false,
    "url": "https://outlook.com/mail/",
    "requested_url": "https://outlook.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 417,
    "content_type": "",
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": null,
    "truncated": false,
    "url": "https://outlook.com/mail/",
    "requested_url": "https://outlook.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 127,
   "domain": "h264.io",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://h264.io/robots.txt",
    "requested_url": "https://h264.io/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://h264.io/llms.txt",
    "requested_url": "https://h264.io/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 128,
   "domain": "ytimg.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://ytimg.com/robots.txt",
    "requested_url": "https://ytimg.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://ytimg.com/llms.txt",
    "requested_url": "https://ytimg.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 129,
   "domain": "registrar-servers.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "331ea9090db0c9f6f597bd9840fd5b171830f6e0b3ba1cb24dfa91f0c95aedc1",
    "error": null,
    "truncated": false,
    "url": "https://registrar-servers.com/robots.txt",
    "requested_url": "https://registrar-servers.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://registrar-servers.com/llms.txt",
    "requested_url": "https://registrar-servers.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 130,
   "domain": "openai.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498",
    "error": null,
    "truncated": false,
    "url": "https://openai.com/robots.txt",
    "requested_url": "https://openai.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 403,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "17693ac432dce629f0ec07ed35c5387496917abed9df0aadcbcfaf55d8c682e0",
    "error": null,
    "truncated": false,
    "url": "https://openai.com/llms.txt",
    "requested_url": "https://openai.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 131,
   "domain": "windows.com",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "8e5c4bd60c587b5e214e298174b62224347fa33fb14299a3a0823e97001e0954",
    "error": null,
    "truncated": false,
    "url": "https://www.microsoft.com/en-ph/windows/",
    "requested_url": "https://windows.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://www.microsoft.com/windows/llms.txt",
    "requested_url": "https://windows.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 132,
   "domain": "cloudflare-dns.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "8fa3036c68bfcbd32365f6225d24333264093b7cd38d306e106b4dbdc934fd5b",
    "error": null,
    "truncated": false,
    "url": "https://one.one.one.one/robots.txt",
    "requested_url": "https://cloudflare-dns.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://one.one.one.one/llms.txt",
    "requested_url": "https://cloudflare-dns.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 133,
   "domain": "github.io",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "c974a5f7c122118328ea0c3fa14e494ddf01fa6a77d0e70e6e3c6773125d81ad",
    "error": null,
    "truncated": false,
    "url": "https://pages.github.com/",
    "requested_url": "https://github.io/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://pages.github.com/",
    "requested_url": "https://github.io/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 134,
   "domain": "dropbox.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "8f93be1d520cf78404b62229519cb4edae55608144f36a41debd3b726494b527",
    "error": null,
    "truncated": false,
    "url": "https://www.dropbox.com/robots.txt",
    "requested_url": "https://dropbox.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "ad9ed13f7ca90a0a91d36799eac5aa2bf9bdc985e66ddf8b8583f83a30556171",
    "error": null,
    "truncated": false,
    "url": "https://www.dropbox.com/llms.txt",
    "requested_url": "https://dropbox.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 135,
   "domain": "discord.gg",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "16ceb5ee3e0dc13aa9adf31a3ebbe45a1d965b8c2b9f72eaf84e5911e140ed95",
    "error": null,
    "truncated": false,
    "url": "https://discord.gg/robots.txt",
    "requested_url": "https://discord.gg/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://discord.com",
    "requested_url": "https://discord.gg/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 136,
   "domain": "amazon.dev",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://amazon.dev/robots.txt",
    "requested_url": "https://amazon.dev/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://amazon.dev/llms.txt",
    "requested_url": "https://amazon.dev/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 137,
   "domain": "adobe.io",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "7b9b15178930b581386131eff19e6e51fdf72a1c7cc8dc9c545cdb1f6fc53786",
    "error": null,
    "truncated": false,
    "url": "https://developer.adobe.com/robots.txt",
    "requested_url": "https://adobe.io/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://developer.adobe.com/llms.txt",
    "requested_url": "https://adobe.io/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 138,
   "domain": "android.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "0a7b28459b8a6ea07fec6fa8a85e04521a516387762531c8b3ded73ecc10ecf9",
    "error": null,
    "truncated": false,
    "url": "https://www.android.com/robots.txt",
    "requested_url": "https://android.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "ccbot",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-searchbot",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "applebot-extended",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "cohere-ai",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "diffbot",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "anthropic-ai",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=ISO-8859-1",
    "sha256": "15d19a9b3413572683a32e4f713a89cb9b622e48af108cd282b8d410e046076e",
    "error": null,
    "truncated": false,
    "url": "https://www.android.com/llms.txt",
    "requested_url": "https://android.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 139,
   "domain": "static.microsoft",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://static.microsoft/robots.txt",
    "requested_url": "https://static.microsoft/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://static.microsoft/llms.txt",
    "requested_url": "https://static.microsoft/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 140,
   "domain": "apache.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "e969244acbbadc64dd66254a7d2bb20017fd9ed3115290694c9a1f562a070c53",
    "error": null,
    "truncated": false,
    "url": "https://apache.org/robots.txt",
    "requested_url": "https://apache.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "f041a1bf51904498945c3b024a6a7976d52d384eaf8ec5c6224240fdbef158ac",
    "error": null,
    "truncated": false,
    "url": "https://apache.org/llms.txt",
    "requested_url": "https://apache.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 141,
   "domain": "nih.gov",
   "robots": {
    "status": 403,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "d5460f8aab3af377d7e1d1de421987bdbaba68012344c3b367f11393e3a202ae",
    "error": null,
    "truncated": false,
    "url": "https://www.nih.gov/robots.txt",
    "requested_url": "https://nih.gov/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 403,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "1115ce52ab9f71129dc04e486557ed5928c7145ad355b85ba311faf33dd79a20",
    "error": null,
    "truncated": false,
    "url": "https://www.nih.gov/llms.txt",
    "requested_url": "https://nih.gov/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 142,
   "domain": "akamaized.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akamaized.net/robots.txt",
    "requested_url": "https://akamaized.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://akamaized.net/llms.txt",
    "requested_url": "https://akamaized.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 143,
   "domain": "vk.ru",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "573219d395c1aa085175955ea564938f81e5c31b8cd52ba15d49a2171ffb06c4",
    "error": null,
    "truncated": false,
    "url": "https://vk.ru/robots.txt",
    "requested_url": "https://vk.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "865e3f506773019127e20b4632f53bf841fe62c758f86756aa166e5dea714345",
    "error": null,
    "truncated": false,
    "url": "https://vk.ru/llms.txt",
    "requested_url": "https://vk.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 144,
   "domain": "azurefd.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://azurefd.net/robots.txt",
    "requested_url": "https://azurefd.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://azurefd.net/llms.txt",
    "requested_url": "https://azurefd.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 145,
   "domain": "trbcdn.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "URLError",
    "truncated": null,
    "url": "https://trbcdn.net/robots.txt",
    "requested_url": "https://trbcdn.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "robots_unreachable"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://trbcdn.net/llms.txt",
    "requested_url": "https://trbcdn.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 146,
   "domain": "t-online.de",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "aefcf62006193c765ed22c8c6e46711397b52d962c0875c6c43e1cdae0b4153c",
    "error": null,
    "truncated": false,
    "url": "https://www.t-online.de/robots.txt",
    "requested_url": "https://t-online.de/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "ccbot",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": false,
      "effective_group": "bytespider",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "applebot-extended",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": false,
      "effective_group": "meta-externalagent",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "f5779fc3e16db06405216fae8af6b7a5f7e81c3c979fe46142fb50d7cb895dbc",
    "error": null,
    "truncated": false,
    "url": "https://www.t-online.de/llms.txt",
    "requested_url": "https://t-online.de/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 147,
   "domain": "vk.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "573219d395c1aa085175955ea564938f81e5c31b8cd52ba15d49a2171ffb06c4",
    "error": null,
    "truncated": false,
    "url": "https://vk.com/robots.txt",
    "requested_url": "https://vk.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "865e3f506773019127e20b4632f53bf841fe62c758f86756aa166e5dea714345",
    "error": null,
    "truncated": false,
    "url": "https://vk.com/llms.txt",
    "requested_url": "https://vk.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 148,
   "domain": "dnsowl.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://dnsowl.com/robots.txt",
    "requested_url": "https://dnsowl.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://dnsowl.com/llms.txt",
    "requested_url": "https://dnsowl.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 149,
   "domain": "dns-parking.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://dns-parking.com/robots.txt",
    "requested_url": "https://dns-parking.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://dns-parking.com/llms.txt",
    "requested_url": "https://dns-parking.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 150,
   "domain": "reg.ru",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "d0db68a344d1ad23628a7baec8bdbdf58a763073717bd3cf01ebba3d3478e8ef",
    "error": null,
    "truncated": false,
    "url": "https://reg.ru/robots.txt",
    "requested_url": "https://reg.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "d0db68a344d1ad23628a7baec8bdbdf58a763073717bd3cf01ebba3d3478e8ef",
    "error": null,
    "truncated": false,
    "url": "https://reg.ru/llms.txt",
    "requested_url": "https://reg.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 151,
   "domain": "archive.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "83e00f46c71e9fa872694af73fa9c15a615b9052b2f10820a3d146bc506c6161",
    "error": null,
    "truncated": false,
    "url": "https://archive.org/robots.txt",
    "requested_url": "https://archive.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "91ee43d65d9cfb290e66bfaab0ca535c4015a3408bc77b6569bdb835ab4ae44c",
    "error": null,
    "truncated": false,
    "url": "https://archive.org/llms.txt",
    "requested_url": "https://archive.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 152,
   "domain": "forms.gle",
   "robots": {
    "status": 400,
    "content_type": "text/html; charset=utf-8",
    "sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11",
    "error": null,
    "truncated": false,
    "url": "https://forms.gle/robots.txt",
    "requested_url": "https://forms.gle/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 400,
    "content_type": "text/html; charset=utf-8",
    "sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11",
    "error": null,
    "truncated": false,
    "url": "https://forms.gle/llms.txt",
    "requested_url": "https://forms.gle/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 153,
   "domain": "webex.com",
   "robots": {
    "status": 403,
    "content_type": "text/html",
    "sha256": "0a5951bdf14f5e29181ab340aabaca3fa17063dc8641ab121a1830f17b5ca2ee",
    "error": null,
    "truncated": false,
    "url": "https://www.webex.com/robots.txt",
    "requested_url": "https://webex.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=ISO-8859-1",
    "sha256": "9c6cfc41bb1fe1e65c3c7f7a2939efd0e223391a4c9f117adf9dbb391f23dec0",
    "error": null,
    "truncated": false,
    "url": "https://www.webex.com/llms.txt",
    "requested_url": "https://webex.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 154,
   "domain": "nflxso.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://nflxso.net/robots.txt",
    "requested_url": "https://nflxso.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://nflxso.net/llms.txt",
    "requested_url": "https://nflxso.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 155,
   "domain": "snapchat.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "ee1ba42e1ce0f9bfcbd1a7620dc7c31c50917675fd01fae255b50ded35c18f8b",
    "error": null,
    "truncated": false,
    "url": "https://www.snapchat.com:443/robots.txt",
    "requested_url": "https://snapchat.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-searchbot",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-user",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexity-user",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": true,
      "effective_group": "meta-externalagent",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "cohere-ai",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "diffbot",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ai2bot",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "anthropic-ai",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "facebookbot",
      "effective_full_block": true
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgilibot",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "ea23de9f052d5fe5c45941756c359ee76d290a09256f1f4e2ebf76e8ec10046e",
    "error": null,
    "truncated": false,
    "url": "https://www.snapchat.com:443/llms.txt",
    "requested_url": "https://snapchat.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 156,
   "domain": "godaddy.com",
   "robots": {
    "status": 403,
    "content_type": "text/html",
    "sha256": "38b4f57aae6eaeebc22427fb6fc61058c48757947983d8941a1bf825b6dd6827",
    "error": null,
    "truncated": false,
    "url": "https://www.godaddy.com/robots.txt",
    "requested_url": "https://godaddy.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 403,
    "content_type": "text/html",
    "sha256": "fc60db183414476fdb90b46582f2d9c950d7d246a5717092aabde5795e6567f5",
    "error": null,
    "truncated": false,
    "url": "https://www.godaddy.com/llms.txt",
    "requested_url": "https://godaddy.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 157,
   "domain": "crashlytics.com",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "79a260f5046815b83becf5023888dfae142d662c0bf17b84087ad5abe56e2426",
    "error": null,
    "truncated": false,
    "url": "https://firebase.google.com/products/crashlytics/",
    "requested_url": "https://crashlytics.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://firebase.google.com/products/crashlytics/",
    "requested_url": "https://crashlytics.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 158,
   "domain": "tm-azurefd.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://tm-azurefd.net/robots.txt",
    "requested_url": "https://tm-azurefd.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://tm-azurefd.net/llms.txt",
    "requested_url": "https://tm-azurefd.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 159,
   "domain": "kaspersky.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=UTF-8",
    "sha256": "9efe9b0f74014e7bd3c10828a1af580cd30907b83f13b9aa52f001e3ab5ea3b7",
    "error": null,
    "truncated": false,
    "url": "https://www.kaspersky.com/robots.txt",
    "requested_url": "https://kaspersky.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "ccbot",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-user",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexity-user",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "applebot-extended",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "diffbot",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=UTF-8",
    "sha256": "95f6695565e741729ec2b77206288669f5da7f17028b6cae8f62d563cce22168",
    "error": null,
    "truncated": false,
    "url": "https://www.kaspersky.com/llms.txt",
    "requested_url": "https://kaspersky.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 160,
   "domain": "tumblr.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain;charset=utf-8",
    "sha256": "2892c51fb7e06e7629ccb662ecf47dec3b0bfa6a679b4e112f25f7565ac1fcb0",
    "error": null,
    "truncated": false,
    "url": "https://www.tumblr.com/robots.txt",
    "requested_url": "https://tumblr.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": true,
      "effective_group": "meta-externalagent",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "anthropic-ai",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "facebookbot",
      "effective_full_block": true
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgilibot",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "f6e42682c0b61c38bb66ba03641c2f37ce7f3119e961938dc53eb821da45a1c1",
    "error": null,
    "truncated": false,
    "url": "https://www.tumblr.com/llms.txt",
    "requested_url": "https://tumblr.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 161,
   "domain": "nytimes.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=UTF-8",
    "sha256": "04dee26f145ee77839c65ff45a4e1b887ae06be06b3b91fbc5043dc37277fcea",
    "error": null,
    "truncated": false,
    "url": "https://www.nytimes.com/robots.txt",
    "requested_url": "https://nytimes.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "google-extended",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "chatgpt-user",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "oai-searchbot",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-searchbot",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-user",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexity-user",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "amazonbot",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": true,
      "effective_group": "meta-externalagent",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "cohere-ai",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "diffbot",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "anthropic-ai",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "facebookbot",
      "effective_full_block": true
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgilibot",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "e9e5717f2113fdd93ab6545310cac13501e00127141790c5d718686c46d166fe",
    "error": null,
    "truncated": false,
    "url": "https://www.nytimes.com/llms.txt",
    "requested_url": "https://nytimes.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 162,
   "domain": "gravatar.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "75e9d90f44e5644b58fb6ec2361a61d48214863bd10d37b7fd6595b9b2f1bf48",
    "error": null,
    "truncated": false,
    "url": "https://gravatar.com/robots.txt",
    "requested_url": "https://gravatar.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "ccbot",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": false,
      "effective_group": "bytespider",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "amazonbot",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "facebookbot",
      "effective_full_block": false
     },
     "omgili": {
      "named": true,
      "named_full_block": false,
      "effective_group": "omgili",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "omgilibot",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "c07c986048f9e393b5bbb07332446daf593a4102091a5863e53ab92bb47806e7",
    "error": null,
    "truncated": false,
    "url": "https://gravatar.com/llms.txt",
    "requested_url": "https://gravatar.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 163,
   "domain": "jomodns.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://jomodns.com/robots.txt",
    "requested_url": "https://jomodns.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://jomodns.com/llms.txt",
    "requested_url": "https://jomodns.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 164,
   "domain": "dotaplabs.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://dotaplabs.net/robots.txt",
    "requested_url": "https://dotaplabs.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://dotaplabs.net/llms.txt",
    "requested_url": "https://dotaplabs.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 165,
   "domain": "azurewebsites.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://azurewebsites.net/robots.txt",
    "requested_url": "https://azurewebsites.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://azurewebsites.net/llms.txt",
    "requested_url": "https://azurewebsites.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 166,
   "domain": "amazon-adsystem.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "RemoteDisconnected",
    "truncated": null,
    "url": "https://amazon-adsystem.com/robots.txt",
    "requested_url": "https://amazon-adsystem.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "robots_unreachable"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://amazon-adsystem.com/llms.txt",
    "requested_url": "https://amazon-adsystem.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 167,
   "domain": "duckdns.org",
   "robots": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "1b00afd81186b6186c7da8f7f78b6cb3569baf28d168fd597fbd5efdd8fd3901",
    "error": null,
    "truncated": false,
    "url": "https://www.duckdns.org/robots.txt",
    "requested_url": "https://duckdns.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "1b00afd81186b6186c7da8f7f78b6cb3569baf28d168fd597fbd5efdd8fd3901",
    "error": null,
    "truncated": false,
    "url": "https://www.duckdns.org/llms.txt",
    "requested_url": "https://duckdns.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 168,
   "domain": "google-analytics.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "43b87e6af8d60fc13ffb0a175234363f7edbe90f5dfe39200215feb4c30a3de9",
    "error": null,
    "truncated": false,
    "url": "https://google-analytics.com/robots.txt",
    "requested_url": "https://google-analytics.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7",
    "error": null,
    "truncated": false,
    "url": "https://google-analytics.com/llms.txt",
    "requested_url": "https://google-analytics.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 169,
   "domain": "ax-msedge.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://ax-msedge.net/robots.txt",
    "requested_url": "https://ax-msedge.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://ax-msedge.net/llms.txt",
    "requested_url": "https://ax-msedge.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 170,
   "domain": "ubuntu.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "aab936ae074e1cf70e1e9cd3ff99efa91cac964d37b8e6ae5b695580c9adb5ae",
    "error": null,
    "truncated": false,
    "url": "https://ubuntu.com/robots.txt",
    "requested_url": "https://ubuntu.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-searchbot",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexity-user",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": false,
      "effective_group": "bytespider",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "applebot-extended",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": false,
      "effective_group": "meta-externalagent",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "cohere-ai",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "anthropic-ai",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "73127cc2f34665b4661f5405ac319b54b0556db4ecfce6a773d8acbe71e79274",
    "error": null,
    "truncated": false,
    "url": "https://ubuntu.com/llms.txt",
    "requested_url": "https://ubuntu.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 171,
   "domain": "shopify.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "a55c031afe721803d5e722c76910b8b569511cce2fc45459933c60d8e0764aa7",
    "error": null,
    "truncated": false,
    "url": "https://shopify.com/robots.txt",
    "requested_url": "https://shopify.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "fd5d372a21847ff2333682645367699f10c44ded4a1dbf08d8e93d512beb765a",
    "error": null,
    "truncated": false,
    "url": "https://www.shopify.com/llms.txt",
    "requested_url": "https://shopify.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 172,
   "domain": "spo-msedge.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://spo-msedge.net/robots.txt",
    "requested_url": "https://spo-msedge.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://spo-msedge.net/llms.txt",
    "requested_url": "https://spo-msedge.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 173,
   "domain": "paypal.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "9c2f2f57e870e9b8406b1b738f0afbae6434f9991f24812890de2a6e317fc083",
    "error": null,
    "truncated": false,
    "url": "https://www.paypal.com/robots.txt",
    "requested_url": "https://paypal.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "1e1be425396507ad6c9bdecdd342c9ada72019ec9a0572ec40c5804122932fa8",
    "error": null,
    "truncated": false,
    "url": "https://www.paypal.com/llms.txt",
    "requested_url": "https://paypal.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "present"
   }
  },
  {
   "rank": 174,
   "domain": "example.com",
   "robots": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "25ddf2c883e0d1958ea971d279a7e4f0fd446724ee3db7db19dadabd4a62e484",
    "error": null,
    "truncated": false,
    "url": "https://example.com/robots.txt",
    "requested_url": "https://example.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "25ddf2c883e0d1958ea971d279a7e4f0fd446724ee3db7db19dadabd4a62e484",
    "error": null,
    "truncated": false,
    "url": "https://example.com/llms.txt",
    "requested_url": "https://example.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 175,
   "domain": "virginm.net",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://virginm.net/robots.txt",
    "requested_url": "https://virginm.net/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://virginm.net/llms.txt",
    "requested_url": "https://virginm.net/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 176,
   "domain": "telegram.org",
   "robots": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "bd5e6537d1fa8b7e136d3589dd722410eb3836f3b4b8e21733fba881463f5272",
    "error": null,
    "truncated": false,
    "url": "https://telegram.org/robots.txt",
    "requested_url": "https://telegram.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "4c736e782cc282848cd0411c01df4d1b409db26128c538561f99c444d650c372",
    "error": null,
    "truncated": false,
    "url": "https://telegram.org/llms.txt",
    "requested_url": "https://telegram.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 177,
   "domain": "scorecardresearch.com",
   "robots": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "dc1d54dab6ec8c00f70137927504e4f222c8395f10760b6beecfcfa94e08249f",
    "error": null,
    "truncated": false,
    "url": "https://scorecardresearch.com/robots.txt",
    "requested_url": "https://scorecardresearch.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "dc1d54dab6ec8c00f70137927504e4f222c8395f10760b6beecfcfa94e08249f",
    "error": null,
    "truncated": false,
    "url": "https://scorecardresearch.com/llms.txt",
    "requested_url": "https://scorecardresearch.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 178,
   "domain": "vkuserphoto.ru",
   "robots": {
    "status": 418,
    "content_type": "",
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": null,
    "truncated": false,
    "url": "https://ps.userapi.com/robots.txt",
    "requested_url": "https://vkuserphoto.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://ps.userapi.com/llms.txt",
    "requested_url": "https://vkuserphoto.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 179,
   "domain": "pki.goog",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "18bec4c62e19bead2c2e6e19a36c1ba6be280c92ca6a3ad4230c3f1c9c7b2bb4",
    "error": null,
    "truncated": false,
    "url": "https://pki.goog/robots.txt",
    "requested_url": "https://pki.goog/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7",
    "error": null,
    "truncated": false,
    "url": "https://pki.goog/llms.txt",
    "requested_url": "https://pki.goog/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 180,
   "domain": "mts.ru",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "46a6e3c31f4f92c111a22e76b16f18b540446d1aa9817c68130ab3c88e72193f",
    "error": null,
    "truncated": false,
    "url": "https://moskva.mts.ru/robotstxt",
    "requested_url": "https://mts.ru/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "ea953254e7fcf9068091aff044f8af67ca96fccf1caf1b08176f8b7fa0d1666c",
    "error": null,
    "truncated": false,
    "url": "https://moskva.mts.ru/llmstxt",
    "requested_url": "https://mts.ru/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 181,
   "domain": "samsungcloud.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "c46cb537f640e0b9194aef1e038dac520c739d6b0aba40a5f2aeac76995d652c",
    "error": null,
    "truncated": false,
    "url": "https://www.samsungcloud.com/robots.txt",
    "requested_url": "https://samsungcloud.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "5984786022817806b3ec371afc3c7c82683f97a27d3d0e6d8039a0c5385fdcfa",
    "error": null,
    "truncated": false,
    "url": "https://www.samsungcloud.com/llms.txt",
    "requested_url": "https://samsungcloud.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 182,
   "domain": "iiko.it",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://iiko.it/robots.txt",
    "requested_url": "https://iiko.it/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://iiko.it/llms.txt",
    "requested_url": "https://iiko.it/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 183,
   "domain": "flickr.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain;charset=UTF-8",
    "sha256": "93b2b4f0def4c7f2ba6075d6197551381b20997ec9812d1de87e9a18b9b1a068",
    "error": null,
    "truncated": false,
    "url": "https://flickr.com/robots.txt",
    "requested_url": "https://flickr.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-searchbot",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-user",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://flickr.com/llms.txt",
    "requested_url": "https://flickr.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_rules",
    "failure_reason": "robots_disallows"
   }
  },
  {
   "rank": 184,
   "domain": "afternic.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=UTF-8",
    "sha256": "e0a36ac62f8496ae1152654bc23729c1b005c5ca366ff2be745c928d84702129",
    "error": null,
    "truncated": false,
    "url": "https://www.afternic.com/robots.txt",
    "requested_url": "https://afternic.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "49f0313126a624c1ab720858382b51fb3ad7e6be8c5bb84ce756a209951c170f",
    "error": null,
    "truncated": false,
    "url": "https://www.afternic.com/llms.txt",
    "requested_url": "https://afternic.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 185,
   "domain": "w3.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "444079f3a1dade59d593ff5968013596b910f004fcfb0d33eef897f322f914ab",
    "error": null,
    "truncated": false,
    "url": "https://www.w3.org/robots.txt",
    "requested_url": "https://w3.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=utf-8",
    "sha256": "b06dad1348e4416456b7abe6476d35d122d9f6027a90765297bdd8ae381c1255",
    "error": null,
    "truncated": false,
    "url": "https://www.w3.org/llms.txt",
    "requested_url": "https://w3.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 186,
   "domain": "discord.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "fd74e7f8abd01d90ce80462af73d0896ed098dd0a61b20f24bf1137dd37979a5",
    "error": null,
    "truncated": false,
    "url": "https://discord.com/robots.txt",
    "requested_url": "https://discord.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "3dfac8363c487ffae361b92fd77643638ac1e1e252eb74e561e86ceb0b0b39be",
    "error": null,
    "truncated": false,
    "url": "https://discord.com/llms.txt",
    "requested_url": "https://discord.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 187,
   "domain": "unity3d.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "1a90eb94d80776f80da593d687c6d1e61cd6c479568fc00b8bbb3b941eaa0c80",
    "error": null,
    "truncated": false,
    "url": "https://unity.com/robots.txt",
    "requested_url": "https://unity3d.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://unity.com/llms.txt",
    "requested_url": "https://unity3d.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "redirect_left_domain"
   }
  },
  {
   "rank": 188,
   "domain": "macromedia.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "URLError",
    "truncated": null,
    "url": "https://macromedia.com/robots.txt",
    "requested_url": "https://macromedia.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "robots_unreachable"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://macromedia.com/llms.txt",
    "requested_url": "https://macromedia.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 189,
   "domain": "creativecommons.org",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "d7a6362bab284aaa0b08d19dcbd455a2054914002c9859e14061e15e071f4705",
    "error": null,
    "truncated": false,
    "url": "https://creativecommons.org/robots.txt",
    "requested_url": "https://creativecommons.org/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "f5d7d5a29435423e763516275ad2eaa3642b454c4cf322eb0633abb961aed0a5",
    "error": null,
    "truncated": false,
    "url": "https://creativecommons.org/llms.txt",
    "requested_url": "https://creativecommons.org/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 190,
   "domain": "soundcloud.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=UTF-8",
    "sha256": "5b1dabbd7efb0756df654e23cb2ab1b0e09f5cdb77e8688c978946d0908e492f",
    "error": null,
    "truncated": false,
    "url": "https://soundcloud.com/robots.txt",
    "requested_url": "https://soundcloud.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "google-extended",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "ccbot",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-searchbot",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-user",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": false,
      "effective_group": "bytespider",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "amazonbot",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "applebot-extended",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": false,
      "effective_group": "meta-externalagent",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "cohere-ai",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "anthropic-ai",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": true,
      "named_full_block": false,
      "effective_group": "omgili",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "omgilibot",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "",
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": null,
    "truncated": false,
    "url": "https://soundcloud.com/llms.txt",
    "requested_url": "https://soundcloud.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 191,
   "domain": "nflximg.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "URLError",
    "truncated": null,
    "url": "https://nflximg.com/robots.txt",
    "requested_url": "https://nflximg.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "robots_unreachable"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://nflximg.com/llms.txt",
    "requested_url": "https://nflximg.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 192,
   "domain": "pages.dev",
   "robots": {
    "status": 200,
    "content_type": "text/html; charset=utf-8",
    "sha256": "386cdab4b4f20c8b7491bd2bdd26411b9f18aec85ed8aabc3d1950000d699492",
    "error": null,
    "truncated": true,
    "url": "https://www.cloudflare.com/products/pages/",
    "requested_url": "https://pages.dev/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "other"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://pages.dev/llms.txt",
    "requested_url": "https://pages.dev/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "other"
   }
  },
  {
   "rank": 193,
   "domain": "theguardian.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain; charset=utf-8",
    "sha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2",
    "error": null,
    "truncated": false,
    "url": "https://www.theguardian.com/robots.txt",
    "requested_url": "https://theguardian.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-searchbot",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-user",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": true,
      "effective_group": "meta-externalagent",
      "effective_full_block": true
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "anthropic-ai",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "facebookbot",
      "effective_full_block": true
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "5a504aff032fe8faab92965fc40e0c4bcaa7709d90347f0d14415572e6a0586a",
    "error": null,
    "truncated": false,
    "url": "https://www.theguardian.com/llms.txt",
    "requested_url": "https://theguardian.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 194,
   "domain": "ebay.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "221797cd7a314aeba19bb83d7868af3ea04325e4f2f8fd9b8ba465649e091970",
    "error": null,
    "truncated": false,
    "url": "https://www.ebay.com/robots.txt",
    "requested_url": "https://ebay.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "perplexitybot",
      "effective_full_block": false
     },
     "CCBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "ccbot",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "chatgpt-user",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "oai-searchbot",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-searchbot",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claude-user",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": false,
      "effective_group": "bytespider",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "amazonbot",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "applebot-extended",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": false,
      "effective_group": "meta-externalagent",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": false,
      "effective_group": "anthropic-ai",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html;charset=UTF-8",
    "sha256": "7afc152d641259ebee63c2a610021601dab99b18dc6dbc7424b6bf146c7abf78",
    "error": null,
    "truncated": false,
    "url": "https://pages.ebay.com/llms.txt",
    "requested_url": "https://ebay.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 195,
   "domain": "hichina.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://hichina.com/robots.txt",
    "requested_url": "https://hichina.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://hichina.com/llms.txt",
    "requested_url": "https://hichina.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 196,
   "domain": "cnn.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "b6cf25ecc82c7881ee486c77f58af3c085f7aa7169e962c14d0cbaec2580bd85",
    "error": null,
    "truncated": false,
    "url": "https://www.cnn.com/robots.txt",
    "requested_url": "https://cnn.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "gptbot",
      "effective_full_block": true
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claudebot",
      "effective_full_block": true
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexitybot",
      "effective_full_block": true
     },
     "CCBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ccbot",
      "effective_full_block": true
     },
     "ChatGPT-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "chatgpt-user",
      "effective_full_block": true
     },
     "OAI-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "oai-searchbot",
      "effective_full_block": true
     },
     "Claude-SearchBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-searchbot",
      "effective_full_block": true
     },
     "Claude-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "claude-user",
      "effective_full_block": true
     },
     "Perplexity-User": {
      "named": true,
      "named_full_block": true,
      "effective_group": "perplexity-user",
      "effective_full_block": true
     },
     "Bytespider": {
      "named": true,
      "named_full_block": true,
      "effective_group": "bytespider",
      "effective_full_block": true
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "amazonbot",
      "effective_full_block": true
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": true,
      "effective_group": "applebot-extended",
      "effective_full_block": true
     },
     "meta-externalagent": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "cohere-ai",
      "effective_full_block": true
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "diffbot",
      "effective_full_block": true
     },
     "AI2Bot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "ai2bot",
      "effective_full_block": true
     },
     "anthropic-ai": {
      "named": true,
      "named_full_block": true,
      "effective_group": "anthropic-ai",
      "effective_full_block": true
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "facebookbot",
      "effective_full_block": true
     },
     "omgili": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgili",
      "effective_full_block": true
     },
     "omgilibot": {
      "named": true,
      "named_full_block": true,
      "effective_group": "omgilibot",
      "effective_full_block": true
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html",
    "sha256": "f299694b5cd369ce4d6683decf80f3433461b0a811cb36bedee2eb40497c4369",
    "error": null,
    "truncated": true,
    "url": "https://www.cnn.com/llms.txt",
    "requested_url": "https://cnn.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  },
  {
   "rank": 197,
   "domain": "medium.com",
   "robots": {
    "status": 200,
    "content_type": "text/plain",
    "sha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74",
    "error": null,
    "truncated": false,
    "url": "https://medium.com/robots.txt",
    "requested_url": "https://medium.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": true,
    "state": "parseable",
    "fetch_state": "fetched",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "gptbot",
      "effective_full_block": false
     },
     "ClaudeBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "claudebot",
      "effective_full_block": false
     },
     "Google-Extended": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "PerplexityBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "CCBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "ChatGPT-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "OAI-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-SearchBot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Claude-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Perplexity-User": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Bytespider": {
      "named": true,
      "named_full_block": false,
      "effective_group": "bytespider",
      "effective_full_block": false
     },
     "Amazonbot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "amazonbot",
      "effective_full_block": false
     },
     "Applebot-Extended": {
      "named": true,
      "named_full_block": false,
      "effective_group": "applebot-extended",
      "effective_full_block": false
     },
     "meta-externalagent": {
      "named": true,
      "named_full_block": false,
      "effective_group": "meta-externalagent",
      "effective_full_block": false
     },
     "meta-externalfetch": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "cohere-training-ai-bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "Diffbot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "AI2Bot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "anthropic-ai": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "FacebookBot": {
      "named": true,
      "named_full_block": false,
      "effective_group": "facebookbot",
      "effective_full_block": false
     },
     "omgili": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     },
     "omgilibot": {
      "named": false,
      "named_full_block": false,
      "effective_group": "*",
      "effective_full_block": false
     }
    }
   },
   "llms": {
    "status": 403,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "4621d2b8d221bb4d651caaea391b1777efa4ba121b301e948bb317d761ffb7fa",
    "error": null,
    "truncated": false,
    "url": "https://medium.com/llms.txt",
    "requested_url": "https://medium.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "failure_reason": "other"
   }
  },
  {
   "rank": 198,
   "domain": "youtube-nocookie.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "URLError",
    "truncated": null,
    "url": "https://youtube-nocookie.com/robots.txt",
    "requested_url": "https://youtube-nocookie.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "not_checked",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "robots_unreachable"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "ValueError",
    "truncated": null,
    "url": "https://youtube-nocookie.com/llms.txt",
    "requested_url": "https://youtube-nocookie.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "not_checked",
    "reason": "robots_unverified",
    "failure_reason": "robots_unreachable"
   }
  },
  {
   "rank": 199,
   "domain": "ksyuncdn.com",
   "robots": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://ksyuncdn.com/robots.txt",
    "requested_url": "https://ksyuncdn.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "no_website_at_ranked_domain",
    "fetch_state": "unreachable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    },
    "failure_reason": "dns_failure"
   },
   "llms": {
    "status": null,
    "content_type": null,
    "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
    "error": "gaierror",
    "truncated": null,
    "url": "https://ksyuncdn.com/llms.txt",
    "requested_url": "https://ksyuncdn.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "no_website_at_ranked_domain",
    "failure_reason": "dns_failure"
   }
  },
  {
   "rank": 200,
   "domain": "app-measurement.com",
   "robots": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2",
    "error": null,
    "truncated": false,
    "url": "https://app-measurement.com/robots.txt",
    "requested_url": "https://app-measurement.com/robots.txt",
    "fetch_date": "2026-10-24",
    "parseable": false,
    "state": "absent_or_unparseable",
    "fetch_state": "unavailable",
    "content_signals": [],
    "crawlers": {
     "GPTBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ClaudeBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Google-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "PerplexityBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "CCBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "ChatGPT-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "OAI-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-SearchBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Claude-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Perplexity-User": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Bytespider": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Amazonbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Applebot-Extended": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalagent": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "meta-externalfetch": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "cohere-training-ai-bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "Diffbot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "AI2Bot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "anthropic-ai": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "FacebookBot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgili": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     },
     "omgilibot": {
      "named": null,
      "named_full_block": null,
      "effective_group": null,
      "effective_full_block": null
     }
    }
   },
   "llms": {
    "status": 404,
    "content_type": "text/html; charset=UTF-8",
    "sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7",
    "error": null,
    "truncated": false,
    "url": "https://app-measurement.com/llms.txt",
    "requested_url": "https://app-measurement.com/llms.txt",
    "fetch_date": "2026-10-24",
    "state": "absent"
   }
  }
 ]
}