section,metric,count,denominator,percent,denominator_means
robots_txt_response,ok,5539,10000,55.4,domains scanned
robots_txt_response,http_403,1208,10000,12.1,domains scanned
robots_txt_response,http_404_410,667,10000,6.7,domains scanned
robots_txt_response,other_4xx,61,10000,0.6,domains scanned
robots_txt_response,http_503,920,10000,9.2,domains scanned
robots_txt_response,other_5xx_429,70,10000,0.7,domains scanned
robots_txt_response,soft404_html,480,10000,4.8,domains scanned
robots_txt_response,unreachable,1054,10000,10.5,domains scanned
robots_txt_response,other,1,10000,0.0,domains scanned
crawler_blocked_sitewide,OAI-SearchBot,441,5539,8.0,parseable robots.txt files
crawler_named,OAI-SearchBot,891,5539,16.1,parseable robots.txt files
crawler_blocked_sitewide_top_1000,OAI-SearchBot,64,495,12.9,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,OAI-SearchBot,377,5044,7.5,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,ChatGPT-User,528,5539,9.5,parseable robots.txt files
crawler_named,ChatGPT-User,967,5539,17.5,parseable robots.txt files
crawler_blocked_sitewide_top_1000,ChatGPT-User,74,495,14.9,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,ChatGPT-User,454,5044,9.0,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,PerplexityBot,612,5539,11.0,parseable robots.txt files
crawler_named,PerplexityBot,1024,5539,18.5,parseable robots.txt files
crawler_blocked_sitewide_top_1000,PerplexityBot,91,495,18.4,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,PerplexityBot,521,5044,10.3,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Perplexity-User,446,5539,8.1,parseable robots.txt files
crawler_named,Perplexity-User,675,5539,12.2,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Perplexity-User,78,495,15.8,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Perplexity-User,368,5044,7.3,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Claude-SearchBot,421,5539,7.6,parseable robots.txt files
crawler_named,Claude-SearchBot,674,5539,12.2,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Claude-SearchBot,71,495,14.3,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Claude-SearchBot,350,5044,6.9,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Claude-User,430,5539,7.8,parseable robots.txt files
crawler_named,Claude-User,678,5539,12.2,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Claude-User,75,495,15.2,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Claude-User,355,5044,7.0,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,DuckAssistBot,458,5539,8.3,parseable robots.txt files
crawler_named,DuckAssistBot,627,5539,11.3,parseable robots.txt files
crawler_blocked_sitewide_top_1000,DuckAssistBot,72,495,14.5,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,DuckAssistBot,386,5044,7.7,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Amzn-SearchBot,316,5539,5.7,parseable robots.txt files
crawler_named,Amzn-SearchBot,435,5539,7.9,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Amzn-SearchBot,52,495,10.5,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Amzn-SearchBot,264,5044,5.2,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Amzn-User,310,5539,5.6,parseable robots.txt files
crawler_named,Amzn-User,424,5539,7.7,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Amzn-User,50,495,10.1,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Amzn-User,260,5044,5.2,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,GPTBot,820,5539,14.8,parseable robots.txt files
crawler_named,GPTBot,1268,5539,22.9,parseable robots.txt files
crawler_blocked_sitewide_top_1000,GPTBot,107,495,21.6,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,GPTBot,713,5044,14.1,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,ClaudeBot,763,5539,13.8,parseable robots.txt files
crawler_named,ClaudeBot,1123,5539,20.3,parseable robots.txt files
crawler_blocked_sitewide_top_1000,ClaudeBot,106,495,21.4,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,ClaudeBot,657,5044,13.0,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Google-Extended,668,5539,12.1,parseable robots.txt files
crawler_named,Google-Extended,1045,5539,18.9,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Google-Extended,94,495,19.0,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Google-Extended,574,5044,11.4,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Applebot-Extended,642,5539,11.6,parseable robots.txt files
crawler_named,Applebot-Extended,865,5539,15.6,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Applebot-Extended,90,495,18.2,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Applebot-Extended,552,5044,10.9,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,meta-externalagent,668,5539,12.1,parseable robots.txt files
crawler_named,meta-externalagent,880,5539,15.9,parseable robots.txt files
crawler_blocked_sitewide_top_1000,meta-externalagent,99,495,20.0,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,meta-externalagent,569,5044,11.3,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Amazonbot,634,5539,11.4,parseable robots.txt files
crawler_named,Amazonbot,870,5539,15.7,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Amazonbot,83,495,16.8,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Amazonbot,551,5044,10.9,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,CCBot,848,5539,15.3,parseable robots.txt files
crawler_named,CCBot,1090,5539,19.7,parseable robots.txt files
crawler_blocked_sitewide_top_1000,CCBot,118,495,23.8,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,CCBot,730,5044,14.5,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Bytespider,838,5539,15.1,parseable robots.txt files
crawler_named,Bytespider,1019,5539,18.4,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Bytespider,123,495,24.8,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Bytespider,715,5044,14.2,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,meta-externalfetcher,455,5539,8.2,parseable robots.txt files
crawler_named,meta-externalfetcher,603,5539,10.9,parseable robots.txt files
crawler_blocked_sitewide_top_1000,meta-externalfetcher,69,495,13.9,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,meta-externalfetcher,386,5044,7.7,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,Google-CloudVertexBot,414,5539,7.5,parseable robots.txt files
crawler_named,Google-CloudVertexBot,552,5539,10.0,parseable robots.txt files
crawler_blocked_sitewide_top_1000,Google-CloudVertexBot,75,495,15.2,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,Google-CloudVertexBot,339,5044,6.7,"parseable robots.txt files, ranks 1001-10000"
crawler_blocked_sitewide,OAI-AdsBot,245,5539,4.4,parseable robots.txt files
crawler_named,OAI-AdsBot,365,5539,6.6,parseable robots.txt files
crawler_blocked_sitewide_top_1000,OAI-AdsBot,45,495,9.1,"parseable robots.txt files, ranks 1-1000"
crawler_blocked_sitewide_1001_10000,OAI-AdsBot,200,5044,4.0,"parseable robots.txt files, ranks 1001-10000"
training_vs_answer,any_ai_named,1574,5539,28.4,parseable robots.txt files
training_vs_answer,any_training_blocked,1106,5539,20.0,parseable robots.txt files
training_vs_answer,any_answer_blocked,727,5539,13.1,parseable robots.txt files
training_vs_answer,training_only_block,392,5539,7.1,parseable robots.txt files
training_vs_answer,all_ai_blocked,194,5539,3.5,parseable robots.txt files
training_vs_answer,all_ai_blocked_by_name,1,5539,0.0,parseable robots.txt files
training_vs_answer,star_root_blocked,238,5539,4.3,parseable robots.txt files
inheritance_gap,files_with_gap,329,1105,29.8,files that name an AI crawler and disallow paths under User-agent: *
inheritance_gap,gap_groups_only_allow_all,182,329,55.3,files with a gap
inheritance_gap_path_type,search,155,329,47.1,files with a gap (a file can count in several types)
inheritance_gap_path_type,login_account,136,329,41.3,files with a gap (a file can count in several types)
inheritance_gap_path_type,api,92,329,28.0,files with a gap (a file can count in several types)
inheritance_gap_path_type,admin,79,329,24.0,files with a gap (a file can count in several types)
inheritance_gap_path_type,cart_checkout,70,329,21.3,files with a gap (a file can count in several types)
inheritance_gap_path_type,feed,44,329,13.4,files with a gap (a file can count in several types)
inheritance_gap_crawler_let_in,GPTBot,187,329,56.8,files with a gap
inheritance_gap_crawler_let_in,OAI-SearchBot,179,329,54.4,files with a gap
inheritance_gap_crawler_let_in,ChatGPT-User,178,329,54.1,files with a gap
inheritance_gap_crawler_let_in,PerplexityBot,150,329,45.6,files with a gap
inheritance_gap_crawler_let_in,ClaudeBot,139,329,42.2,files with a gap
inheritance_gap_crawler_let_in,Google-Extended,137,329,41.6,files with a gap
inheritance_gap_crawler_let_in,CCBot,84,329,25.5,files with a gap
inheritance_gap_crawler_let_in,Amazonbot,65,329,19.8,files with a gap
inheritance_gap_crawler_let_in,Claude-SearchBot,64,329,19.5,files with a gap
inheritance_gap_crawler_let_in,Claude-User,63,329,19.1,files with a gap
inheritance_gap_crawler_let_in,Perplexity-User,62,329,18.8,files with a gap
inheritance_gap_crawler_let_in,meta-externalagent,57,329,17.3,files with a gap
inheritance_gap_crawler_let_in,Applebot-Extended,55,329,16.7,files with a gap
inheritance_gap_crawler_let_in,Bytespider,49,329,14.9,files with a gap
inheritance_gap_crawler_let_in,DuckAssistBot,26,329,7.9,files with a gap
inheritance_gap_crawler_let_in,meta-externalfetcher,18,329,5.5,files with a gap
inheritance_gap_crawler_let_in,OAI-AdsBot,17,329,5.2,files with a gap
inheritance_gap_crawler_let_in,Google-CloudVertexBot,11,329,3.3,files with a gap
inheritance_gap_crawler_let_in,Amzn-SearchBot,5,329,1.5,files with a gap
inheritance_gap_crawler_let_in,Amzn-User,3,329,0.9,files with a gap
llms_txt,valid,878,6082,14.4,"sites that answered /llms.txt (excludes 403, 429, 5xx, network errors)"
llms_txt,valid_top_1000,98,529,18.5,"sites that answered, ranks 1-1000"
llms_txt,valid_1001_10000,780,5553,14.0,"sites that answered, ranks 1001-10000"
other_robots_lines,content_signal,150,5539,2.7,parseable robots.txt files
other_robots_lines,content_signal_train_no,90,5539,1.6,parseable robots.txt files
other_robots_lines,cf_managed,58,5539,1.0,parseable robots.txt files
other_robots_lines,crawl_delay,609,5539,11.0,parseable robots.txt files
other_robots_lines,noindex_lines,23,5539,0.4,parseable robots.txt files
other_robots_lines,sitemap_line,3726,5539,67.3,parseable robots.txt files
wordpress_sites,kept_wordpress_default_rules,149,361,41.3,parseable robots.txt files in group
wordpress_sites,names_an_ai_crawler,65,361,18.0,parseable robots.txt files in group
wordpress_sites,blocks_gptbot_sitewide,36,361,10.0,parseable robots.txt files in group
wordpress_sites,llms_txt_valid,64,386,16.6,sites in group that answered /llms.txt
other_sites,kept_wordpress_default_rules,0,3959,0.0,parseable robots.txt files in group
other_sites,names_an_ai_crawler,1115,3959,28.2,parseable robots.txt files in group
other_sites,blocks_gptbot_sitewide,414,3959,10.5,parseable robots.txt files in group
other_sites,llms_txt_valid,690,4701,14.7,sites in group that answered /llms.txt