{"rule_id":"AB-001","category":"discovery","icon":"🔍","title":"Robots.txt File","short_description":"Your site has a robots.txt file that tells crawlers and AI agents what they can access.","user_value":"Without this file, AI agents don't know which parts of your site they're allowed to read, and may skip your content entirely.","wrong_example":"No robots.txt file found. Agents assume nothing is allowed.","right_example":"A clear robots.txt welcomes AI agents to explore your content.","effort_hint":"quick","estimated_cost":"$0","severity":"low","pillar":"discovery","version":"1.0.0","counted_in_score":true,"check":{"type":"http_fetch","target":"/robots.txt"},"fix":{"eligible":true,"type":"deterministic","note":"Scaffold a permissive default robots.txt"},"checklist_anchor":"#AB-001","checklist_url":"/agent-readiness-checklist#AB-001","html_url":"/rules/AB-001","json_url":"/rules/AB-001.json","related_rules":[{"rule_id":"AB-002","title":"Sitemap File","category":"discovery"},{"rule_id":"AB-003","title":"Agent Guide","category":"discovery"},{"rule_id":"AB-014","title":"LLMs.txt File","category":"discovery"},{"rule_id":"AB-061","title":"Link Relationships","category":"discovery"},{"rule_id":"AB-062","title":"API Catalog","category":"discovery"},{"rule_id":"AB-066","title":"Content Signals","category":"discovery"},{"rule_id":"AB-068","title":"DNS Agent Discovery","category":"discovery"},{"rule_id":"AB-110","title":"Blog articles in AI sitemap","category":"discovery"},{"rule_id":"AB-111","title":"Crawl-delay directive in robots.txt","category":"discovery"},{"rule_id":"AB-113","title":"LLM policy file","category":"discovery"}]}