{"id":8097,"date":"2026-10-01T07:30:00","date_gmt":"2026-10-01T05:30:00","guid":{"rendered":"https:\/\/taismo.de\/unkategorisiert\/gptbot-claudebot-and-perplexitybot-how-to-control-ai-crawlers\/"},"modified":"2026-10-04T16:16:24","modified_gmt":"2026-10-04T14:16:24","slug":"gptbot","status":"publish","type":"post","link":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/","title":{"rendered":"GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers"},"content":{"rendered":"<style>\n.taismo-art{display:grid;grid-template-columns:320px minmax(0,1fr);gap:36px;align-items:start}\n.taismo-art .toc-sticky{position:sticky;top:80px}\n.taismo-art .toc-sticky ol li{margin-bottom:9px}\n.taismo-art .toc-sticky ol a:hover{color:var(--toc-hover) !important}\n.taismo-art .toc-sticky a[href*=\"preferences\"]:hover{background:#0d353c !important;color:#fff !important}\n.taismo-art figure{margin:26px 0}\n.taismo-art figcaption{font-size:0.9em;color:#5a6a67;margin-top:6px}\n.taismo-art h2{margin-top:38px}\n.taismo-art table{border-collapse:collapse;width:100%;font-size:0.95em}\n.taismo-art table th{background:#072328;color:#fff;text-align:left;padding:9px 12px}\n.taismo-art table td{border-bottom:1px solid #e6e6e6;padding:9px 12px;vertical-align:top}\n.taismo-art pre{background:#072328;color:#e8f0ef;border-radius:10px;padding:16px 18px;overflow-x:auto;font-size:0.86em;line-height:1.6}\n@media(max-width:900px){.taismo-art{grid-template-columns:1fr}.taismo-art .toc-sticky{position:static}}\n<\/style>\n<div class=\"taismo-art\">\n<aside class=\"toc-sticky\" style=\"--toc-hover:#072328;background:#ED8924;border-radius:14px;padding:22px 24px\">\n<div style=\"text-align:center;margin-bottom:14px\"><svg role=\"img\" aria-label=\"taismo SEO magazine\" fill=\"#ffffff\" style=\"width:180px;max-width:80%;height:auto;display:inline-block\" id=\"Ebene_1\" data-name=\"Ebene 1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 209.04 52.32\"><title>taismo SEO magazine<\/title><g id=\"Gruppe_796\" data-name=\"Gruppe 796\"><path id=\"Pfad_1293\" data-name=\"Pfad 1293\" class=\"cls-1\" d=\"M6.17,17.93h-3.28v-6.05h3.37v-6.3h7.41v6.3h5.24v6.05h-5.14v8.27c-.07.97.12,1.94.55,2.8.37.55,1.14.83,2.31.83.5,0,.93,0,1.28-.03.38-.02.76-.06,1.13-.12v5.85c-.69.18-1.4.29-2.12.33-.94.08-1.8.12-2.57.12-1.3.04-2.59-.16-3.83-.55-.99-.33-1.86-.93-2.54-1.71-.7-.88-1.19-1.9-1.42-3-.31-1.43-.45-2.89-.42-4.36v-8.42h.02Z\"\/><path id=\"Pfad_1294\" data-name=\"Pfad 1294\" class=\"cls-1\" d=\"M40.74,35.57h-6.51l-.41-1.82c-.63.66-1.36,1.24-2.17,1.66-.88.42-1.85.64-2.83.61-.91,0-1.83-.09-2.72-.3-.85-.19-1.64-.58-2.31-1.13-.71-.61-1.26-1.37-1.61-2.24-.44-1.18-.65-2.42-.61-3.68v-1.31c-.04-1.08.18-2.15.65-3.12.41-.8,1-1.5,1.74-2.02.76-.52,1.6-.88,2.5-1.09.98-.22,1.97-.33,2.98-.33.8,0,1.59.06,2.37.2.66.13,1.3.33,1.92.61v-.86c0-.5-.06-.99-.2-1.47-.15-.44-.42-.84-.8-1.11-.51-.35-1.09-.58-1.69-.71-.92-.18-1.85-.27-2.8-.26-.84,0-1.67.08-2.5.23-.89.15-1.66.29-2.3.42v-5.64c.44-.12.88-.2,1.34-.26.53-.06,1.07-.13,1.64-.2.57-.06,1.12-.12,1.64-.15.53-.04.96-.05,1.34-.05,1.94-.05,3.85.18,5.72.65,1.29.34,2.48,1,3.45,1.92.85.87,1.45,1.96,1.71,3.15.31,1.42.47,2.86.45,4.31v13.96h0,0ZM30.86,30.89c1.5.11,2.8-1.01,2.91-2.51v-2.38c-.71-.53-1.59-.8-2.47-.76-.78-.04-1.56.15-2.24.53-.55.35-.83,1.07-.83,2.15v.35c-.06.72.15,1.43.61,1.99.55.47,1.28.7,2.02.64\"\/><rect id=\"Rechteck_760\" data-name=\"Rechteck 760\" class=\"cls-1\" x=\"45.38\" y=\"11.88\" width=\"7.61\" height=\"23.7\"\/><path class=\"cls-1\" d=\"M120.87,23.5v-.39c-.06-1.47.11-2.96.48-4.39.32-1.01,1.13-1.51,2.44-1.51.54-.03,1.07.1,1.53.38.4.27.69.67.83,1.13.18.61.29,1.24.33,1.86.05.75.07,1.59.07,2.53v.39h7.72v-.93c.04-1.77-.21-3.53-.76-5.22-.44-1.32-1.17-2.52-2.15-3.5-.94-.91-2.09-1.59-3.36-1.96-1.41-.41-2.88-.62-4.34-.61-1.45,0-2.88.18-4.26.61-1.25.38-2.4,1.05-3.33,1.96-.98.99-1.71,2.18-2.17,3.5-.56,1.68-.82,3.45-.78,5.22v.93h7.72Z\"\/><path class=\"cls-1\" d=\"M85.51,23.5v-2.19c-.06-.81.15-1.62.58-2.31.36-.53.96-.82,1.59-.8.24,0,.47.03.71.07.24.06.47.18.65.35.23.22.4.5.48.8.14.47.19.95.18,1.44v2.65h7.56v-2.49c-.05-.74.15-1.47.55-2.09.37-.48.95-.76,1.56-.73.5,0,.99.16,1.39.45.41.3.64,1.04.64,2.22v2.64h7.56v-4.6c.03-1.22-.16-2.43-.55-3.59-.31-.88-.83-1.69-1.51-2.34-.64-.58-1.39-1.02-2.22-1.28-.85-.28-1.73-.41-2.62-.41-1.07-.02-2.12.19-3.1.64-1,.51-1.91,1.2-2.65,2.05-.53-.88-1.27-1.59-2.17-2.06-1.07-.45-2.22-.66-3.37-.61-1.09-.04-2.17.2-3.12.7-.91.53-1.73,1.21-2.42,2.02l-.5-2.12h-6.77v11.62h7.56v-.03Z\"\/><path class=\"cls-1\" d=\"M67.27,26.57c0-1.7,1.37-3.07,3.07-3.07h2.01c-.8-.78-1.79-1.35-2.88-1.66l-1.61-.55c-.67-.23-1.24-.42-1.71-.58-.4-.12-.78-.28-1.16-.45-.26-.12-.48-.3-.65-.53-.15-.26-.22-.55-.2-.86,0-.2.04-.41.12-.61.12-.23.31-.42.55-.53.4-.18.82-.31,1.25-.38.75-.12,1.49-.17,2.24-.15.85,0,1.71.06,2.54.2.76.1,1.5.26,2.24.45v-5.64c-.44-.12-.88-.2-1.34-.26-.52-.06-1.07-.13-1.64-.2-.56-.06-1.12-.12-1.64-.15s-.97-.05-1.34-.05c-3.43,0-6,.69-7.71,2.09-1.64,1.24-2.6,3.21-2.57,5.26-.03.99.1,1.96.38,2.9.22.7.59,1.35,1.09,1.89.48.52,1.06.93,1.69,1.24.73.35,1.47.66,2.24.93l1.16.41c.77.26,1.5.56,2.22.93.43.26.69.74.65,1.24.06.66-.32,1.29-.96,1.51-1.13.29-2.31.41-3.47.35-.88,0-1.75-.08-2.62-.23-.98-.15-1.78-.29-2.42-.42v5.64c.46.12.92.2,1.39.26.55.06,1.13.13,1.74.2.61.06,1.2.12,1.77.15.57.03,1.04.05,1.41.05,1.38.03,2.77-.09,4.14-.33v-9.06h.03Z\"\/><\/g><g><path class=\"cls-1\" d=\"M71.17,44.23v-16.63h4.21l4.42,6.56,4.44-6.56h4.21v16.63h-3.83v-10.69l-4.8,7.03-4.82-7.03v10.69h-3.83Z\"\/><path class=\"cls-1\" d=\"M90.25,44.23l6.46-16.63h4.09l6.46,16.63h-4.04l-1.21-3.16h-6.51l-1.21,3.16h-4.04ZM96.74,37.69h4.02l-2-5.32-2.02,5.32Z\"\/><path class=\"cls-1\" d=\"M115.84,44.51c-1.71,0-3.22-.37-4.53-1.1-1.31-.74-2.33-1.75-3.08-3.04-.74-1.29-1.12-2.78-1.12-4.46s.37-3.16,1.12-4.46c.75-1.29,1.77-2.3,3.08-3.04s2.82-1.11,4.53-1.11c1.06,0,2.02.16,2.89.46.86.31,1.62.73,2.27,1.27s1.19,1.16,1.62,1.85l-3.16,1.9c-.24-.38-.55-.72-.94-1.02-.39-.3-.81-.54-1.27-.7-.46-.17-.93-.25-1.4-.25-.93,0-1.76.21-2.48.64-.72.43-1.29,1.03-1.7,1.79-.41.77-.62,1.65-.62,2.65s.2,1.88.59,2.65c.4.77.97,1.38,1.71,1.82.75.44,1.61.67,2.59.67.68,0,1.3-.13,1.85-.38.55-.25,1-.6,1.33-1.04.33-.44.53-.97.59-1.57h-3.75v-2.95h7.44v2.26c-.03,1.55-.38,2.86-1.04,3.93-.67,1.07-1.56,1.87-2.67,2.41-1.12.54-2.4.81-3.84.81Z\"\/><path class=\"cls-1\" d=\"M123.3,44.23l6.46-16.63h4.09l6.46,16.63h-4.04l-1.21-3.16h-6.51l-1.21,3.16h-4.04ZM129.79,37.69h4.02l-2-5.32-2.02,5.32Z\"\/><path class=\"cls-1\" d=\"M141.31,44.23v-3.61l8.34-9.41h-8.27v-3.61h13.23v3.61l-8.34,9.41h8.27v3.61h-13.23Z\"\/><path class=\"cls-1\" d=\"M157.04,44.23v-16.63h3.83v16.63h-3.83Z\"\/><path class=\"cls-1\" d=\"M164.1,44.23v-16.63h4.02l7.29,10.15v-10.15h3.83v16.63h-3.83l-7.48-10.43v10.43h-3.83Z\"\/><\/g><path class=\"cls-1\" d=\"M205.77,18.4h-4.03c-.44-2.75-2.83-4.86-5.7-4.86-.5,0-.91.41-.91.91v3.95h-10.76c-.5,0-.91.41-.91.91v24.31c0,.5.41.91.91.91h21.4c.5,0,.91-.41.91-.91v-24.31c0-.5-.41-.91-.91-.91ZM189.24,23.26h5.9v1.82h-5.9c-.5,0-.91-.41-.91-.91s.41-.91.91-.91ZM199.89,39.66h-10.66c-.5,0-.91-.41-.91-.91v-9.73c0-.5.41-.91.91-.91h5.9v5.77c0,.5.41.91.91.91,2.18,0,3.95,1.77,3.95,3.96,0,.31-.04.62-.11.91ZM200,34.56c-.82-.78-1.88-1.32-3.05-1.5v-13.74s0,0,0,0,0,0,0,0v-3.84c1.74.41,3.05,1.98,3.05,3.85v15.25ZM204.86,42.71h-4.62c.97-1.03,1.57-2.43,1.57-3.95v-18.54h3.05v22.5Z\"\/><\/svg><\/div>\n<h3 style=\"text-align:center;color:#fff;margin:0 0 14px;font-size:1.1em\">Table of contents<\/h3>\n<ol style=\"margin:0;padding-left:1.2em;line-height:1.5;font-weight:600;color:#fff;font-size:0.9em\">\n<li><a href=\"#roles\" style=\"color:#fff;text-decoration:none\">Three jobs, one operator<\/a><\/li>\n<li><a href=\"#bot-list\" style=\"color:#fff;text-decoration:none\">The AI crawler list 2026<\/a><\/li>\n<li><a href=\"#two-switches\" style=\"color:#fff;text-decoration:none\">GPTBot vs. OAI-SearchBot<\/a><\/li>\n<li><a href=\"#robots-txt\" style=\"color:#fff;text-decoration:none\">How robots.txt is read<\/a><\/li>\n<li><a href=\"#limits\" style=\"color:#fff;text-decoration:none\">Where robots.txt ends<\/a><\/li>\n<li><a href=\"#decision\" style=\"color:#fff;text-decoration:none\">Block or allow<\/a><\/li>\n<li><a href=\"#our-setup\" style=\"color:#fff;text-decoration:none\">Our own setup<\/a><\/li>\n<li><a href=\"#verify\" style=\"color:#fff;text-decoration:none\">Verify your setup<\/a><\/li>\n<li><a href=\"#mistakes\" style=\"color:#fff;text-decoration:none\">Five common mistakes<\/a><\/li>\n<li><a href=\"#faq\" style=\"color:#fff;text-decoration:none\">FAQ<\/a><\/li>\n<li><a href=\"#sources\" style=\"color:#fff;text-decoration:none\">Sources<\/a><\/li>\n<\/ol>\n<p>    <a href=\"https:\/\/www.google.com\/preferences\/source?q=https:\/\/taismo.de\" title=\"Set taismo as a preferred source on Google\" target=\"_blank\" rel=\"noopener noreferrer\" style=\"display:block;text-align:center;margin-top:16px;background:#072328;color:#fff;font-size:0.82em;font-weight:700;padding:9px 12px;border-radius:8px;text-decoration:none\">\u2606 Prefer taismo on Google<\/a><br \/>\n  <\/aside>\n<div class=\"taismo-main\">\n<h1 class=\"brand-claim\" style=\"margin:0 0 14px;color:#072328\">GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers<\/h1>\n<figure style=\"margin:0 0 24px\"><img fetchpriority=\"high\" src=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/2026-09-18_ki-crawler-steuern_hero.webp\" alt=\"GPTBot, ClaudeBot and PerplexityBot: Controlling AI crawlers with robots.txt by allowing or blocking each user agent\" title=\"GPTBot, ClaudeBot and PerplexityBot: How to control AI crawlers\" width=\"1344\" height=\"768\" loading=\"eager\" decoding=\"async\" style=\"width:100%;height:auto;border-radius:14px;display:block\"><\/figure>\n<div class=\"taismo-author-top\" style=\"display:flex;flex-wrap:wrap;gap:16px;align-items:center;border:1px solid #e6e6e6;background:#e6e6e6;border-radius:12px;padding:16px 20px;margin:0 0 24px\">\n      <img decoding=\"async\" src=\"https:\/\/taismo.de\/wp-content\/uploads\/2025\/11\/Kostenloses-S-3.png\" alt=\"Dominik Breitbach\" width=\"64\" height=\"64\" loading=\"lazy\" style=\"width:64px;height:64px;border-radius:50%;object-fit:cover;border:2px solid #072328;flex:0 0 auto\"><\/p>\n<div style=\"flex:1 1 260px;min-width:220px\">\n<div style=\"font-weight:800;color:#072328\">Dominik Breitbach <span style=\"font-weight:500;color:#5a6a67\">\u00b7 Founder &amp; Lead SEO Strategist at taismo<\/span><\/div>\n<p style=\"margin:3px 0 0;font-size:0.92em;line-height:1.5;color:#3a4744\">Dominik is the founder and managing director of taismo, an SEO and GEO agency from Munich. He has worked in search marketing since 2010 and focuses on ongoing SEO support and visibility in AI answers, for companies in Germany and for international firms that want to be found in the German market.<\/p>\n<\/p><\/div>\n<div style=\"display:flex;flex-wrap:wrap;gap:8px;flex:1 1 100%\">\n        <span style=\"background:#072328;color:#fff;font-size:0.8em;font-weight:700;padding:5px 12px;border-radius:999px\">\u23f1 Reading time: 14 min<\/span><br \/>\n        <span style=\"background:#072328;color:#fff;font-size:0.8em;font-weight:700;padding:5px 12px;border-radius:999px\">\ud83d\udd04 Last updated: 1 October 2026<\/span>\n      <\/div>\n<\/p><\/div>\n<p class=\"lead-def\" style=\"font-size:1.12em;line-height:1.6;color:#072328;border-left:4px solid #ED8924;padding-left:16px\"><strong>GPTBot<\/strong> is the web crawler OpenAI uses to collect content for training its AI models, and you control it with a <code>User-agent: GPTBot<\/code> group in your robots.txt. Blocking GPTBot keeps your pages out of future OpenAI training data; it leaves your citations in ChatGPT search untouched, because those come from a second crawler called OAI-SearchBot. That split runs through the whole industry: AI crawlers do three different jobs, and each job has its own user agent. This guide lists the 17 user agents that matter in 2026, shows how robots.txt rules are evaluated, where they stop working, and how to decide what to block.<\/p>\n<p class=\"taismo-cta-text\" style=\"border-left:3px solid #ED8924;padding:4px 0 4px 14px;margin:6px 0 22px;color:#072328\">\ud83d\udc49 Before you change a single line, see how AI systems read your page today: <a href=\"https:\/\/taismo.de\/en\/seo-magazine\/seo-and-geo-check\/\" style=\"color:#ED8924;font-weight:700\">Free AI visibility checker<\/a><\/p>\n<h2 id=\"roles\">AI crawlers do three jobs: Training, search index and live fetch<\/h2>\n<p>The question &#8220;should we allow AI crawlers?&#8221; has no single answer, because it bundles three processes that work very differently. The general concept is covered in our glossary entry on the <a href=\"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/\">AI crawler<\/a>. This guide is about control: Which user agent to address, with which rule, and what the rule changes.<\/p>\n<p>AI crawlers fall into three roles:<\/p>\n<ol>\n<li><strong>Training.<\/strong> The bot collects text that a model later learns from. Whatever it takes ends up in the model weights and cannot be pulled back out. Training alone gives you no citation and no link.<\/li>\n<li><strong>Search index.<\/strong> The bot builds a separate index that the AI system cites from when a user asks a question. This is where your brand gets named together with a link to your page. <strong>The search index role is the one that drives your visibility in AI answers.<\/strong><\/li>\n<li><strong>Live fetch.<\/strong> A person just asked a question, and the system loads your page at that moment. Technically this is a user-triggered request and no crawl, which is why several of these agents openly ignore robots.txt.<\/li>\n<\/ol>\n<p>The detail that costs companies visibility: <strong>One operator runs several bots with different roles<\/strong>, and each bot has its own user agent. OpenAI runs four, Anthropic three, Perplexity two. Anyone who blocks only the best-known name, usually GPTBot, almost always targets the wrong job.<\/p>\n<figure><a href=\"https:\/\/taismo.de\/en\/what-is\/crawler\/\" title=\"Glossary: What a crawler is and how it works\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 900 430\" role=\"img\" aria-label=\"Diagram: The three jobs of AI crawlers, training, search index and live fetch, with their user agents and the consequence for visibility\" style=\"width:100%;height:auto;display:block;border-radius:12px\"><title>The three jobs of AI crawlers and their consequences<\/title><rect x=\"0\" y=\"0\" width=\"900\" height=\"430\" fill=\"#fdf5ec\" rx=\"12\"\/><text x=\"34\" y=\"46\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"21\" font-weight=\"700\" fill=\"#072328\">Three jobs, three consequences<\/text><text x=\"34\" y=\"70\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"14\" fill=\"#5a6866\">One operator often runs all three, each with its own user agent.<\/text><g><rect x=\"34\" y=\"94\" width=\"262\" height=\"196\" rx=\"10\" fill=\"#072328\"\/><text x=\"56\" y=\"128\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"16\" font-weight=\"700\" fill=\"#ED8924\">1 &#183; Training<\/text><text x=\"56\" y=\"156\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#e8f0ef\">Text flows into the model.<\/text><text x=\"56\" y=\"176\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#e8f0ef\">No source credit,<\/text><text x=\"56\" y=\"196\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#e8f0ef\">cannot be taken back.<\/text><rect x=\"56\" y=\"216\" width=\"218\" height=\"52\" rx=\"6\" fill=\"#0d353c\"\/><text x=\"68\" y=\"238\" font-family=\"Consolas,Menlo,monospace\" font-size=\"12.5\" fill=\"#ED8924\">GPTBot &#183; ClaudeBot<\/text><text x=\"68\" y=\"258\" font-family=\"Consolas,Menlo,monospace\" font-size=\"12.5\" fill=\"#ED8924\">Google-Extended &#183; CCBot<\/text><\/g><g><rect x=\"318\" y=\"94\" width=\"262\" height=\"196\" rx=\"10\" fill=\"#ED8924\"\/><text x=\"340\" y=\"128\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"16\" font-weight=\"700\" fill=\"#072328\">2 \u00b7 Search index<\/text><text x=\"340\" y=\"156\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#3a2508\">This is where the mention<\/text><text x=\"340\" y=\"176\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#3a2508\">and the link to your page<\/text><text x=\"340\" y=\"196\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#3a2508\">come from. Visibility.<\/text><rect x=\"340\" y=\"216\" width=\"218\" height=\"52\" rx=\"6\" fill=\"#fff2e2\"\/><text x=\"352\" y=\"238\" font-family=\"Consolas,Menlo,monospace\" font-size=\"12.5\" fill=\"#072328\">OAI-SearchBot<\/text><text x=\"352\" y=\"258\" font-family=\"Consolas,Menlo,monospace\" font-size=\"12.5\" fill=\"#072328\">PerplexityBot &#183; Claude-SearchBot<\/text><\/g><g><rect x=\"602\" y=\"94\" width=\"262\" height=\"196\" rx=\"10\" fill=\"#072328\"\/><text x=\"624\" y=\"128\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"16\" font-weight=\"700\" fill=\"#ED8924\">3 \u00b7 Live fetch<\/text><text x=\"624\" y=\"156\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#e8f0ef\">A person just asked.<\/text><text x=\"624\" y=\"176\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#e8f0ef\">robots.txt usually<\/text><text x=\"624\" y=\"196\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#e8f0ef\">does not apply here.<\/text><rect x=\"624\" y=\"216\" width=\"218\" height=\"52\" rx=\"6\" fill=\"#0d353c\"\/><text x=\"636\" y=\"238\" font-family=\"Consolas,Menlo,monospace\" font-size=\"12.5\" fill=\"#ED8924\">ChatGPT-User<\/text><text x=\"636\" y=\"258\" font-family=\"Consolas,Menlo,monospace\" font-size=\"12.5\" fill=\"#ED8924\">Perplexity-User &#183; Claude-User<\/text><\/g><rect x=\"34\" y=\"312\" width=\"830\" height=\"62\" rx=\"10\" fill=\"#fff\" stroke=\"#f0e0cf\"\/><text x=\"56\" y=\"340\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"14.5\" font-weight=\"700\" fill=\"#072328\">Blocking all three with one line also hits job 2.<\/text><text x=\"56\" y=\"362\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13.5\" fill=\"#5a6866\">Job 2 is where AI answers name you. Job 3 ignores that line anyway.<\/text><text x=\"34\" y=\"404\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"11.5\" fill=\"#9aa8a6\">Fig. 1 \u00b7 taismo<\/text><\/svg><\/a><figcaption><strong>Fig. 1: The three jobs of AI crawlers.<\/strong> A blanket block on all three also removes the job that creates mentions in AI answers.<\/figcaption><\/figure>\n<h2 id=\"bot-list\">The AI crawler list 2026: User agents, operators and purpose<\/h2>\n<p>The table below was checked against each operator&#8217;s own documentation, because second-hand lists go stale within months. The first column is exactly what goes into the <code>User-agent<\/code> line. Case does not matter there: Google&#8217;s robots.txt specification treats the user agent token as case-insensitive.<\/p>\n<div style=\"overflow-x:auto;margin:20px 0\">\n<table>\n<tr>\n<th>User agent<\/th>\n<th>Operator<\/th>\n<th>Role<\/th>\n<th>Honors robots.txt<\/th>\n<\/tr>\n<tr>\n<td><code>GPTBot<\/code><\/td>\n<td>OpenAI<\/td>\n<td>Training<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>OAI-SearchBot<\/code><\/td>\n<td>OpenAI<\/td>\n<td>Search index (ChatGPT search)<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>ChatGPT-User<\/code><\/td>\n<td>OpenAI<\/td>\n<td>Live fetch<\/td>\n<td><strong>may not apply<\/strong><\/td>\n<\/tr>\n<tr>\n<td><code>OAI-AdsBot<\/code><\/td>\n<td>OpenAI<\/td>\n<td>Checks landing pages of ChatGPT ads<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>ClaudeBot<\/code><\/td>\n<td>Anthropic<\/td>\n<td>Training<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>Claude-SearchBot<\/code><\/td>\n<td>Anthropic<\/td>\n<td>Search index<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>Claude-User<\/code><\/td>\n<td>Anthropic<\/td>\n<td>Live fetch<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>Google-Extended<\/code><\/td>\n<td>Google<\/td>\n<td>Gemini training and grounding<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>Googlebot<\/code><\/td>\n<td>Google<\/td>\n<td>Google Search and AI Overviews<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>PerplexityBot<\/code><\/td>\n<td>Perplexity<\/td>\n<td>Search index<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>Perplexity-User<\/code><\/td>\n<td>Perplexity<\/td>\n<td>Live fetch<\/td>\n<td><strong>generally ignored<\/strong><\/td>\n<\/tr>\n<tr>\n<td><code>meta-externalagent<\/code><\/td>\n<td>Meta<\/td>\n<td>Training<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>meta-webindexer<\/code><\/td>\n<td>Meta<\/td>\n<td>Search index<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>meta-externalfetcher<\/code><\/td>\n<td>Meta<\/td>\n<td>Live fetch and agents<\/td>\n<td><strong>may not apply<\/strong><\/td>\n<\/tr>\n<tr>\n<td><code>Applebot-Extended<\/code><\/td>\n<td>Apple<\/td>\n<td>Usage signal for training<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>Applebot<\/code><\/td>\n<td>Apple<\/td>\n<td>Siri, Spotlight, Safari<\/td>\n<td>yes<\/td>\n<\/tr>\n<tr>\n<td><code>CCBot<\/code><\/td>\n<td>Common Crawl<\/td>\n<td>Open web archive, widely used for training<\/td>\n<td>yes<\/td>\n<\/tr>\n<\/table>\n<\/div>\n<p>Three rows deserve a second look. <strong>OAI-AdsBot<\/strong> is new: According to OpenAI, it validates the safety of web pages submitted as ads on ChatGPT and only visits those submitted landing pages. <strong>Applebot-Extended does no crawling at all.<\/strong> The token only signals how content that Applebot already collected may be used for training. And <strong>CCBot<\/strong> belongs to the nonprofit Common Crawl Foundation, no AI company. Its open archive is still one of the most widely used training sources, so a block there indirectly reaches many models.<\/p>\n<p>OpenAI, Anthropic, Perplexity and Common Crawl publish the IP ranges of their bots as JSON files. Matching a request&#8217;s IP address against these files is the only reliable way to tell a real bot from a fake one; the file locations are listed in the sources below.<\/p>\n<h2 id=\"two-switches\">GPTBot vs. OAI-SearchBot: Two switches that get mixed up<\/h2>\n<p>The most expensive misconception in this field sounds like this: &#8220;We blocked GPTBot, so we are out of ChatGPT.&#8221; That statement is wrong in both directions.<\/p>\n<p><strong>GPTBot controls training, OAI-SearchBot controls the search index.<\/strong> OpenAI states that &#8220;each setting is independent of the others&#8221;. If you block GPTBot and allow OAI-SearchBot, ChatGPT search keeps citing you, and your texts stay out of model training. If you block only OAI-SearchBot, you disappear from ChatGPT&#8217;s search answers and still feed the training set as long as GPTBot gets through. The ChatGPT side of this is covered in depth in our guide to <a href=\"https:\/\/taismo.de\/en\/seo-magazine\/chatgpt-seo\/\">ChatGPT SEO<\/a>.<\/p>\n<p>Google draws the same line between <code>Googlebot<\/code> and <code>Google-Extended<\/code>. Google writes that Google-Extended &#8220;does not impact a site&#8217;s inclusion in Google Search nor is it used as a ranking signal in Google Search.&#8221; One detail is often missed: According to the same documentation, Google-Extended governs training of future Gemini models <em>and<\/em> grounding, which means feeding Search content to Gemini Apps and Vertex AI at answer time. A Google-Extended block therefore reaches further than training. Apple describes <code>Applebot-Extended<\/code> in the same spirit: Pages that block it can still appear in Apple&#8217;s search features.<\/p>\n<p>In practice this means: <strong>A training block is a copyright decision. A search index block is a visibility decision.<\/strong> Writing both into the same line merges two decisions without looking at either. How AI systems choose which sources to name is explained in our article on <a href=\"https:\/\/taismo.de\/en\/seo-magazine\/geo-ranking\/\">ranking in AI answers<\/a>.<\/p>\n<div class=\"taismo-pro-tipp\" style=\"display:flex;gap:12px;background:#fdf5ec;border:1px solid #f0e0cf;border-left:5px solid #ED8924;border-radius:10px;padding:16px 20px;margin:24px 0\">\n<div style=\"font-size:1.4em;line-height:1.1\">\ud83d\udca1<\/div>\n<div style=\"color:#3a4744;line-height:1.6\"><strong style=\"color:#ED8924\">Pro tip:<\/strong> Before you change a line, write down the role of every user agent you plan to block. If the column says &#8220;search index&#8221;, you are about to remove your own mentions from AI answers. That one column prevents the most common wrong decision in this field.<\/div>\n<\/div>\n<h2 id=\"robots-txt\">How crawlers read your robots.txt: Four rules<\/h2>\n<p>A <a href=\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/\">robots.txt<\/a> file looks simpler than it is. The format is standardized as the Robots Exclusion Protocol in RFC 9309 (2022), and four rules decide whether your file does what you think it does.<\/p>\n<p><strong>Rule 1: A crawler follows exactly one group.<\/strong> It picks the group with the most specific matching user agent and ignores all others. If <code>User-agent: *<\/code> contains a rule and a separate group for <code>GPTBot<\/code> follows further down, GPTBot obeys <em>only<\/em> its own group. The wildcard rules are not added on top. This is how a newly added bot group can silently cancel an existing block.<\/p>\n<p><strong>Rule 2: When rules conflict, the longer path wins.<\/strong> Google measures the length of the rule path; with equal length, the least restrictive rule applies. That lets you open a single directory while the rest of the site stays closed.<\/p>\n<p><strong>Rule 3: A server error on robots.txt halts crawling.<\/strong> A 404 means &#8220;no restrictions&#8221; to Google. A 5xx error makes Google stop crawling the site for the first 12 hours; after that, Google uses the last good copy for up to 30 days. A robots.txt that throws a 503 under load is more dangerous than no file at all.<\/p>\n<p><strong>Rule 4: <code>Disallow<\/code> does not mean <code>noindex<\/code>.<\/strong> Google says a blocked URL &#8220;might still be indexed without visiting the page&#8221; if other pages link to it, and then appears without a description. To keep a page out of the index, you need a meta robots tag or the <a href=\"https:\/\/taismo.de\/en\/what-is\/x-robots-tag\/\">X-Robots-Tag<\/a> HTTP header, and the page has to stay crawlable so the crawler can read that instruction. Setting both at once achieves the opposite: The bot may not load the page, so it never sees your noindex.<\/p>\n<figure><a href=\"https:\/\/taismo.de\/en\/what-is\/indexing\/\" title=\"Glossary: What happens when a page gets indexed\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 900 400\" role=\"img\" aria-label=\"Flow diagram: How a crawler evaluates robots.txt, from choosing a group to the longest rule, and why Disallow does not prevent indexing\" style=\"width:100%;height:auto;display:block;border-radius:12px\"><title>How a crawler evaluates robots.txt<\/title><rect x=\"0\" y=\"0\" width=\"900\" height=\"400\" fill=\"#fdf5ec\" rx=\"12\"\/><text x=\"34\" y=\"46\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"21\" font-weight=\"700\" fill=\"#072328\">Four steps before a rule applies<\/text><rect x=\"34\" y=\"72\" width=\"196\" height=\"100\" rx=\"10\" fill=\"#072328\"\/><text x=\"52\" y=\"102\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"14\" font-weight=\"700\" fill=\"#ED8924\">1. Pick a group<\/text><text x=\"52\" y=\"126\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">The most specific<\/text><text x=\"52\" y=\"144\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">user agent wins.<\/text><text x=\"52\" y=\"162\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#ED8924\">Only ONE group applies.<\/text><path d=\"M238,122 L262,122\" stroke=\"#ED8924\" stroke-width=\"3\"\/><path d=\"M262,116 l10,6 -10,6 z\" fill=\"#ED8924\"\/><rect x=\"278\" y=\"72\" width=\"196\" height=\"100\" rx=\"10\" fill=\"#072328\"\/><text x=\"296\" y=\"102\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"14\" font-weight=\"700\" fill=\"#ED8924\">2. Longest rule<\/text><text x=\"296\" y=\"126\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">More characters in the path<\/text><text x=\"296\" y=\"144\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">beat a shorter path.<\/text><text x=\"296\" y=\"162\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">Same length: Allow.<\/text><path d=\"M482,122 L506,122\" stroke=\"#ED8924\" stroke-width=\"3\"\/><path d=\"M506,116 l10,6 -10,6 z\" fill=\"#ED8924\"\/><rect x=\"522\" y=\"72\" width=\"196\" height=\"100\" rx=\"10\" fill=\"#072328\"\/><text x=\"540\" y=\"102\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"14\" font-weight=\"700\" fill=\"#ED8924\">3. File reachable?<\/text><text x=\"540\" y=\"126\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">404 = no rules.<\/text><text x=\"540\" y=\"144\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">5xx = crawling stops,<\/text><text x=\"540\" y=\"162\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">then up to 30 days cache.<\/text><path d=\"M726,122 L750,122\" stroke=\"#ED8924\" stroke-width=\"3\"\/><path d=\"M750,116 l10,6 -10,6 z\" fill=\"#ED8924\"\/><rect x=\"766\" y=\"72\" width=\"100\" height=\"100\" rx=\"10\" fill=\"#ED8924\"\/><text x=\"788\" y=\"116\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"14\" font-weight=\"700\" fill=\"#072328\">4. Rule<\/text><text x=\"788\" y=\"136\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"14\" font-weight=\"700\" fill=\"#072328\">applies<\/text><rect x=\"34\" y=\"200\" width=\"832\" height=\"130\" rx=\"10\" fill=\"#fff\" stroke=\"#f0e0cf\"\/><text x=\"56\" y=\"232\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"15.5\" font-weight=\"700\" fill=\"#072328\">And the most common mix-up: Disallow does not mean noindex<\/text><rect x=\"56\" y=\"248\" width=\"386\" height=\"64\" rx=\"7\" fill=\"#fdf5ec\"\/><text x=\"74\" y=\"272\" font-family=\"Consolas,Menlo,monospace\" font-size=\"13\" font-weight=\"700\" fill=\"#072328\">Disallow<\/text><text x=\"74\" y=\"292\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#5a6866\">The content is not read.<\/text><text x=\"74\" y=\"306\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#5a6866\">The URL can still end up in the index.<\/text><rect x=\"458\" y=\"248\" width=\"386\" height=\"64\" rx=\"7\" fill=\"#072328\"\/><text x=\"476\" y=\"272\" font-family=\"Consolas,Menlo,monospace\" font-size=\"13\" font-weight=\"700\" fill=\"#ED8924\">noindex<\/text><text x=\"476\" y=\"292\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">The URL stays out of the index.<\/text><text x=\"476\" y=\"306\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"12.5\" fill=\"#e8f0ef\">Only read if the page stays crawlable.<\/text><text x=\"34\" y=\"368\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"11.5\" fill=\"#9aa8a6\">Fig. 2 \u00b7 taismo<\/text><\/svg><\/a><figcaption><strong>Fig. 2: How a crawler evaluates robots.txt.<\/strong> Only one group applies, the longest rule wins, and Disallow alone does not prevent indexing.<\/figcaption><\/figure>\n<p>Here is an example that sets the two switches separately: Training crawlers blocked, search crawlers and live fetchers open.<\/p>\n<pre>User-agent: GPTBot\nDisallow: \/\n\nUser-agent: ClaudeBot\nDisallow: \/\n\nUser-agent: Google-Extended\nDisallow: \/\n\nUser-agent: Applebot-Extended\nDisallow: \/\n\nUser-agent: CCBot\nDisallow: \/\n\nUser-agent: OAI-SearchBot\nAllow: \/\n\nUser-agent: Claude-SearchBot\nAllow: \/\n\nUser-agent: PerplexityBot\nAllow: \/\n\nSitemap: https:\/\/www.example.com\/sitemap.xml<\/pre>\n<p>This file does not touch <code>Googlebot<\/code>, and that is deliberate. Googlebot feeds classic Google Search and AI Overviews. Blocking it costs you far more than a slice of AI visibility: It takes you out of Google Search entirely.<\/p>\n<h2 id=\"limits\">Where robots.txt reaches its limits<\/h2>\n<p>Three gaps remain, and they explain why a clean robots.txt is only half of the answer.<\/p>\n<p><strong>Live fetch agents follow their own rules, and their operators say so.<\/strong> OpenAI writes about <code>ChatGPT-User<\/code> that &#8220;robots.txt rules may not apply&#8221;, because a user initiated the request. Perplexity states that <code>Perplexity-User<\/code> &#8220;generally ignores robots.txt rules&#8221;. Meta says the same about <code>meta-externalfetcher<\/code>. When a person asks an AI system about your page, the page gets loaded, whatever your file says. Anthropic is the exception: According to its documentation, <code>Claude-User<\/code> lets site owners control which sites these user-initiated requests can access.<\/p>\n<p><strong>A user agent can be faked.<\/strong> The user agent is free text in the request header. A server rule that only checks this text blocks the bots that identify themselves honestly and lets every serious scraper through. Server-side blocking needs an IP check against the published ranges.<\/p>\n<p><strong>Third parties block for you without telling you.<\/strong> On July 1, 2025, Cloudflare announced it was changing its default to block AI crawlers. Add bot protection that answers fast request bursts with a 429, and hosting providers that filter at network level. The result is a contradiction nobody sees in the source code: The robots.txt file says <code>Allow<\/code>, the server door says 403.<\/p>\n<p>Cloudflare backed the change with a striking comparison: Getting traffic from OpenAI is 750 times harder than it used to be from Google, and from Anthropic 30,000 times harder. AI systems read a lot and send back little. Concluding that allowing them does not pay mixes up two metrics. <strong>The value of a mention in an AI answer lies in the shortlist it puts you on, long before any click.<\/strong> That is why <a href=\"https:\/\/taismo.de\/en\/geo\/\">answer engine optimization<\/a> (AEO, also called generative engine optimization or GEO) is measured in mentions and citations first and in sessions second.<\/p>\n<p>The legal side matters for anyone selling into Europe. Article 4 of the EU Directive on Copyright in the Digital Single Market lets rights holders reserve their content from text and data mining, and for content published online it names &#8220;machine-readable means&#8221; as the appropriate form. Germany implemented this in Section 44b of its Copyright Act, and the EU AI Act obliges providers of general-purpose AI models to identify and comply with such reservations. robots.txt is the most common machine-readable way to declare that reservation. Whether a specific block holds up in a dispute is a legal question for a lawyer. In the United States there is no comparable statutory opt-out; robots.txt there works as an industry convention that the large operators commit to in their documentation.<\/p>\n<div class=\"taismo-cta-context\" style=\"background:#fdf5ec;border:1px solid #f0e0cf;border-left:5px solid #ED8924;border-radius:10px;padding:20px 24px;margin:28px 0\">\n<div style=\"font-weight:800;color:#072328;font-size:1.1em;margin-bottom:6px\">Do AI systems actually reach your website?<\/div>\n<p style=\"margin:0 0 14px;color:#3a4744\">Our AEO audit checks exactly that: Which user agents arrive, which status codes they get, whether robots.txt and server agree, and where visibility in AI answers is lost. You get findings in priority order, ready for your team or for us to implement.<\/p>\n<p>      <a href=\"https:\/\/taismo.de\/en\/geo-audit\/\" style=\"display:inline-block;background:#ED8924;color:#fff;font-weight:700;padding:10px 18px;border-radius:8px;text-decoration:none\">See the audit<\/a>\n    <\/div>\n<h2 id=\"decision\">Block or allow GPTBot: One question decides<\/h2>\n<p>We sell visibility in AI answers, so here is our disclosure up front: We have an interest in companies allowing search crawlers. There are still cases in which a block is the right call, and one question separates them.<\/p>\n<h3 id=\"product-or-advertising\">Is your content the product or the advertising for the product?<\/h3>\n<p>If people pay for your texts because the texts themselves are the value, every reuse in an AI answer costs you revenue. That applies to publishers, course providers, databases, image archives and paywalled newsrooms. For them, a training block is the obvious choice, and often a search index block as well.<\/p>\n<p>If your texts explain what you sell, the math flips. A service page, a guide or a glossary has one job: To be found and understood. Blocking here removes you from exactly the questions in which a buyer looks for a provider, and protects texts you give away anyway. That describes most companies with a service that needs explaining, from B2B manufacturers to software vendors.<\/p>\n<figure><a href=\"https:\/\/taismo.de\/en\/geo\/\" title=\"Answer engine optimization by taismo: Getting your brand cited in AI answers\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 900 430\" role=\"img\" aria-label=\"Decision diagram: Whether a company should block or allow AI crawlers depends on whether its content is the product or advertising for the product\" style=\"width:100%;height:auto;display:block;border-radius:12px\"><title>Block or allow AI crawlers: The decision in one question<\/title><rect x=\"0\" y=\"0\" width=\"900\" height=\"430\" fill=\"#072328\" rx=\"12\"\/><g opacity=\"0.07\" transform=\"translate(600,150) scale(0.26)\"><path fill=\"#ED8924\" d=\"M1122.08,481.04l-132.61-85.2h-.19c-2.19-1.43-3.97-3.41-5.16-5.74-2.21-4.37-3.05-9.3-2.42-14.16.76-8.35,4.11-36.09,7.27-61.66,1.56-12.75,3.19-25.06,4.24-34.12s1.88-14.92,1.88-14.95c.16-1.49-.91-2.83-2.4-2.99-.82-.09-1.64.2-2.23.79l-125.88,131.4-8.04,8.39-19.55,17.03c-15.82,12.94-32.67,24.58-50.38,34.79-13.56,10.1-30.33,14.95-47.19,13.65-32.06-1.03-63.83-6.46-94.41-16.13-59.13-27.3-123.67-40.89-188.79-39.76l-154,13.68s-30.9,2.86-60.51,32.43C50.6,470.53,4.98,559.22,4.98,559.22c0,0-11.35,25.55,15.66,47.06,0,0,20.79,17.79,43.61,4.53l40.03-26.27s25.07-15.9,39.5-5.15c0,0,17.39,8.06,17.39,42.55v269.11s-2.6,25.87,23.09,25.87h86.86s24.39,3.34,24.39-27.14v-152.4s-1.96-26.91,30.22-19.79c0,0,66.98,18.51,158.47,18.51h130.68s25.04.86,25.04,25.44v130.29s-1.3,25.09,20.49,25.09h92.06s21.79,1.72,21.79-24.15v-266.53s-1.95-38.68,32.83-58.04c0,0,68.28-30.4,131.35-97.03,0,0,15.87-15.76,55.7-15.76h124.82s7.15-1.51,5.85-11.4c-.44-3.24-2.3-6.12-5.07-7.86l-97.98-59.76\"\/><\/g><text x=\"34\" y=\"48\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"21\" font-weight=\"700\" fill=\"#ED8924\">One question decides<\/text><rect x=\"34\" y=\"72\" width=\"832\" height=\"52\" rx=\"10\" fill=\"#ED8924\"\/><text x=\"450\" y=\"104\" text-anchor=\"middle\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"17\" font-weight=\"700\" fill=\"#072328\">Is your content the product, or the advertising for the product?<\/text><path d=\"M300,124 L220,154\" stroke=\"#ED8924\" stroke-width=\"3\"\/><path d=\"M220,148 l-12,8 12,4 z\" fill=\"#ED8924\"\/><path d=\"M600,124 L680,154\" stroke=\"#ED8924\" stroke-width=\"3\"\/><path d=\"M680,148 l12,8 -12,4 z\" fill=\"#ED8924\"\/><rect x=\"34\" y=\"166\" width=\"400\" height=\"182\" rx=\"10\" fill=\"#0d353c\" stroke=\"#1d4d55\"\/><text x=\"54\" y=\"196\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"15.5\" font-weight=\"700\" fill=\"#fff\">The content IS the product<\/text><text x=\"54\" y=\"222\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#b6c8c6\">Publisher, course provider, database,<\/text><text x=\"54\" y=\"240\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#b6c8c6\">image archive, paywalled newsroom<\/text><rect x=\"54\" y=\"256\" width=\"360\" height=\"72\" rx=\"7\" fill=\"#072328\"\/><text x=\"70\" y=\"280\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13.5\" font-weight=\"700\" fill=\"#ED8924\">Block training<\/text><text x=\"70\" y=\"300\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#e8f0ef\">Weigh the search index, often block it too.<\/text><text x=\"70\" y=\"318\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#e8f0ef\">Every reuse costs revenue here.<\/text><rect x=\"466\" y=\"166\" width=\"400\" height=\"182\" rx=\"10\" fill=\"#0d353c\" stroke=\"#1d4d55\"\/><text x=\"486\" y=\"196\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"15.5\" font-weight=\"700\" fill=\"#fff\">The content PROMOTES the product<\/text><text x=\"486\" y=\"222\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#b6c8c6\">Service page, guide, glossary,<\/text><text x=\"486\" y=\"240\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#b6c8c6\">a service that needs explaining<\/text><rect x=\"486\" y=\"256\" width=\"360\" height=\"72\" rx=\"7\" fill=\"#ED8924\"\/><text x=\"502\" y=\"280\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13.5\" font-weight=\"700\" fill=\"#072328\">Allow the search index<\/text><text x=\"502\" y=\"300\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#3a2508\">Decide on training separately.<\/text><text x=\"502\" y=\"318\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13\" fill=\"#3a2508\">Blocking costs you the mention.<\/text><text x=\"34\" y=\"378\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"13.5\" fill=\"#b6c8c6\">In both cases: Deciding per directory is cleaner than all or nothing.<\/text><text x=\"34\" y=\"406\" font-family=\"Helvetica,Arial,sans-serif\" font-size=\"11.5\" fill=\"#5f7472\">Fig. 3 \u00b7 taismo<\/text><\/svg><\/a><figcaption><strong>Fig. 3: Block or allow AI crawlers.<\/strong> The decision follows from one question about the role of your content.<\/figcaption><\/figure>\n<h3 id=\"middle-ground\">The middle ground we recommend in most cases<\/h3>\n<p>Block training crawlers, allow search crawlers. You keep the mention with a link and declare your text and data mining reservation at the same time. This option has a price, and it belongs on the table: If a model does not know your brand from training, it names you less often when it answers without a live web search. For brands with little awareness, allowing training on purpose can make sense. How the two disciplines interact is covered in our comparison of <a href=\"https:\/\/taismo.de\/en\/seo-magazine\/geo-vs-seo\/\">AEO and SEO<\/a>.<\/p>\n<p>A third option is rarely mentioned and often fits best: <strong>Decide per directory.<\/strong> The guide section stays open, the member area, download folder or paid archive is blocked. Because the longer rule wins, this fits cleanly into a single group:<\/p>\n<pre>User-agent: GPTBot\nDisallow: \/members\/\nDisallow: \/downloads\/\nAllow: \/<\/pre>\n<h3 id=\"weak-arguments\">Server load and other weak arguments<\/h3>\n<p>Server load is rarely the deciding factor. If a bot really fetches too much, the right answer is a <code>Crawl-delay<\/code> directive or rate limiting. Anthropic explicitly supports the non-standard <code>Crawl-delay<\/code> extension and shows it in its own documentation. Crawl volume becomes a <a href=\"https:\/\/taismo.de\/en\/what-is\/crawl-budget\/\">crawl budget<\/a> topic for very large sites; the typical company website with a few hundred URLs stays far below that line.<\/p>\n<p>&#8220;Everyone else blocks it&#8221; is just as weak. What is right for a newspaper is wrong for an engineering firm, and the other way round. The user agents are the same for everyone; the decision is individual.<\/p>\n<div class=\"taismo-pro-tipp\" style=\"display:flex;gap:12px;background:#fdf5ec;border:1px solid #f0e0cf;border-left:5px solid #ED8924;border-radius:10px;padding:16px 20px;margin:24px 0\">\n<div style=\"font-size:1.4em;line-height:1.1\">\ud83d\udca1<\/div>\n<div style=\"color:#3a4744;line-height:1.6\"><strong style=\"color:#ED8924\">Pro tip:<\/strong> Put a recurring reminder at the start of each quarter: Compare your robots.txt with the current operator documentation, then break down your access logs by user agent. Fifteen minutes catch new bots before they run unnoticed for a year.<\/div>\n<\/div>\n<h2 id=\"our-setup\">How we configure our own robots.txt<\/h2>\n<p>Anyone giving advice should show their own setup. Our robots.txt is short on purpose. It blocks one thing, the paginated archive pages under <code>\/page\/*<\/code>, and points to the sitemap. <strong>We block no AI crawler at all<\/strong>, neither for training nor for search. That is a decision: Our texts are the advertising for our service.<\/p>\n<p>The positive counterpart is our llms.txt, a briefing document for language models that we maintain by hand. It currently has <strong>19,136 characters with 94 annotated links<\/strong>, bilingual in one file. It tells a model in structured form who we are, which claims are backed by evidence and which page answers which question. What the format can and cannot do is explained in our glossary entry on the <a href=\"https:\/\/taismo.de\/en\/what-is\/llms-txt\/\">llms.txt file<\/a>; the way from draft to launch is described in our <a href=\"https:\/\/taismo.de\/en\/seo-magazine\/llms-txt-guide\/\">llms.txt guide<\/a>.<\/p>\n<p>Two lessons from running it that you will not find in the documentation:<\/p>\n<ol>\n<li><strong>A static root file deserves a deployment with backup, health check and rollback.<\/strong> Our script saves the live version, uploads the new one, checks that it loads and rolls back automatically on failure. When you compare your local copy with the live file, preserve the line endings. Otherwise the comparison reports a difference that does not exist, which cost us one wrong diagnosis.<\/li>\n<li><strong>A language folder can be virtual.<\/strong> We wanted a separate llms.txt for our English site under <code>\/en\/<\/code>. In our multilingual setup, <code>\/en\/<\/code> is a rewrite route, and the rewrite rule only fires while no real folder of that name exists. Uploading <code>en\/llms.txt<\/code> would have created the folder and taken the English homepage offline. The clean solution is a virtual route in code.<\/li>\n<\/ol>\n<p>And one classification that has to be honest: <strong>The llms.txt file is an offer to AI systems.<\/strong> Jeremy Howard proposed the format in September 2024. No operator has committed to reading it, and it controls nothing. Access is governed by robots.txt and the server, and llms.txt complements both.<\/p>\n<h2 id=\"verify\">How to verify that your AI crawler setup works<\/h2>\n<p>The most common state in practice is an unnoticed decision. robots.txt says one thing, the server does another. Four steps settle this in fifteen minutes:<\/p>\n<ol>\n<li><strong>Step 1: Fetch your robots.txt yourself<\/strong> and check the status code. You want a 200. A 5xx is an emergency, a 404 means no rules apply at all.<\/li>\n<li><strong>Step 2: Request your page as a bot.<\/strong> This separates what you configured from what actually happens. A request with the user agent set shows the real status code.\n<pre>curl -sI -A \"GPTBot\" https:\/\/www.example.com\/\ncurl -sI -A \"OAI-SearchBot\" https:\/\/www.example.com\/\ncurl -sI -A \"ClaudeBot\" https:\/\/www.example.com\/\ncurl -sI -A \"PerplexityBot\" https:\/\/www.example.com\/<\/pre>\n<p>A 403, a 429 or a redirect to a challenge page means a firewall, bot protection or your host is blocking. That block appears in no robots.txt and in no normal crawl, because nobody asks as a bot.<\/li>\n<li><strong>Step 3: Read your access logs.<\/strong> Set up a report by user agent and watch for four weeks which agents arrive, how often, and with which status code. A block you set has to show up as a 403. A bot that mostly sees 404s has a site structure problem and no access problem.<\/li>\n<li><strong>Step 4: Check suspicious traffic against the IP lists.<\/strong> If a user agent shows up unusually often, match its IP with the operator&#8217;s JSON file. An address missing from the file belongs to someone using the name. That tells you whether to block a user agent or a single address.<\/li>\n<\/ol>\n<h2 id=\"mistakes\">Five robots.txt mistakes we see in audits<\/h2>\n<p>These five come up again and again, across industries and content management systems:<\/p>\n<ul>\n<li><strong>Everything blocked because a template said so.<\/strong> Ready-made AI blocklists are mostly written by publishers for publishers. A service company that copies one loses exactly the visibility it pays for elsewhere.<\/li>\n<li><strong><code>Disallow<\/code> and <code>noindex<\/code> on the same page.<\/strong> The page stays in the index without a snippet, the opposite of the intent.<\/li>\n<li><strong>A new bot group cancels the wildcard rules.<\/strong> Since only one group applies, a bot with its own group loses all general blocks. Block <code>\/internal\/<\/code> under <code>User-agent: *<\/code>, add a GPTBot group later, and <code>\/internal\/<\/code> is open to GPTBot.<\/li>\n<li><strong>Server-side blocking without an IP check.<\/strong> A rule that only reads the user agent text hits the honest bots and misses everyone else.<\/li>\n<li><strong>The file dates from 2023 and nobody looks at it.<\/strong> OAI-SearchBot, Claude-SearchBot, meta-webindexer and OAI-AdsBot did not exist when the debate started. robots.txt belongs on the table once a quarter, together with the logs.<\/li>\n<\/ul>\n<p>None of these mistakes is expensive to fix. They are expensive because they stay unnoticed: Nobody reads the file after it was created. If you would rather not keep an eye on it yourself, the quarterly check fits into ongoing <a href=\"https:\/\/taismo.de\/en\/seo-services\/\">SEO support<\/a>, where it belongs anyway.<\/p>\n<div class=\"taismo-cta-final\" style=\"background:#072328;color:#fff;border-radius:12px;padding:28px;margin:32px 0\">\n<div style=\"font-size:1.3em;font-weight:800;margin-bottom:8px\">Let\u2019s look at your setup together<\/div>\n<p style=\"margin:0 0 18px;color:#c9d6d3;max-width:640px\">Not sure whether your robots.txt does what it should? We clarify that in a short call, in English or German. We tell you what we see, and you decide whether to implement it yourself or hand it over.<\/p>\n<p>      <a href=\"https:\/\/taismo.de\/en\/request\/\" style=\"display:inline-block;background:#ED8924;color:#fff;font-weight:800;padding:12px 22px;border-radius:8px;text-decoration:none\">Book a free first call<\/a>\n    <\/div>\n<h2 id=\"faq\">FAQ about GPTBot and AI crawlers<\/h2>\n<h3>What is GPTBot?<\/h3>\n<p>GPTBot is OpenAI&#8217;s web crawler that collects publicly available content for training its AI models. You can block it with a <code>User-agent: GPTBot<\/code> group in robots.txt, and OpenAI publishes its IP ranges for verification.<\/p>\n<h3>Should I block GPTBot?<\/h3>\n<p>Block GPTBot if your content is your product, for example paid articles, courses or databases. If your content promotes your services, allowing it is usually the better choice, and in both cases you should keep OAI-SearchBot open for ChatGPT search.<\/p>\n<h3>Does blocking GPTBot remove my website from ChatGPT?<\/h3>\n<p>Blocking GPTBot keeps your content out of OpenAI&#8217;s model training; your citations in ChatGPT search stay, because they depend on OAI-SearchBot. Only blocking OAI-SearchBot removes you from ChatGPT&#8217;s search answers.<\/p>\n<h3>Do AI crawlers respect robots.txt?<\/h3>\n<p>The training and search crawlers of the large operators state that they follow robots.txt. Live fetch agents such as ChatGPT-User and Perplexity-User may ignore it, because a user triggered the request.<\/p>\n<h3>How do I recognize a fake GPTBot?<\/h3>\n<p>Check the IP address against OpenAI&#8217;s published file at openai.com\/gptbot.json. A request with the GPTBot user agent from an address outside those ranges comes from someone else using the name.<\/p>\n<h3>Does a robots.txt Disallow keep a page out of Google?<\/h3>\n<p>A Disallow rule stops crawling of the content; the URL itself can still be indexed without a description if other pages link to it. To keep a page out of the index, use noindex and leave the page crawlable.<\/p>\n<h2 id=\"sources\">Sources<\/h2>\n<ul class=\"taismo-quellen\" style=\"font-size:0.92em;line-height:1.65;color:#3a4744\">\n<li>OpenAI: &#8220;<a href=\"https:\/\/developers.openai.com\/api\/docs\/bots\" target=\"_blank\" rel=\"noopener noreferrer\">Overview of OpenAI crawlers<\/a>&#8220;, OpenAI Developer Documentation, retrieved October 4, 2026.<\/li>\n<li>OpenAI: &#8220;<a href=\"https:\/\/openai.com\/gptbot.json\" target=\"_blank\" rel=\"noopener noreferrer\">GPTBot IP ranges (JSON)<\/a>&#8220;, OpenAI, retrieved September 18, 2026.<\/li>\n<li>Anthropic: &#8220;<a href=\"https:\/\/support.claude.com\/en\/articles\/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler\" target=\"_blank\" rel=\"noopener noreferrer\">Does Anthropic crawl data from the web, and how can site owners block the crawler?<\/a>&#8220;, Anthropic Support, retrieved October 4, 2026.<\/li>\n<li>Anthropic: &#8220;<a href=\"https:\/\/claude.com\/crawling\/bots.json\" target=\"_blank\" rel=\"noopener noreferrer\">IP ranges of Anthropic crawlers (JSON)<\/a>&#8220;, Anthropic, retrieved September 18, 2026.<\/li>\n<li>Google: &#8220;<a href=\"https:\/\/developers.google.com\/search\/docs\/crawling-indexing\/google-common-crawlers\" target=\"_blank\" rel=\"noopener noreferrer\">Overview of Google crawlers and fetchers<\/a>&#8220;, Google Search Central, retrieved October 4, 2026.<\/li>\n<li>Google: &#8220;<a href=\"https:\/\/developers.google.com\/search\/docs\/crawling-indexing\/robots\/robots_txt\" target=\"_blank\" rel=\"noopener noreferrer\">How Google interprets the robots.txt specification<\/a>&#8220;, Google Search Central, retrieved October 4, 2026.<\/li>\n<li>Perplexity: &#8220;<a href=\"https:\/\/docs.perplexity.ai\/guides\/bots\" target=\"_blank\" rel=\"noopener noreferrer\">Perplexity crawlers<\/a>&#8220;, Perplexity Documentation, retrieved October 4, 2026.<\/li>\n<li>Meta: &#8220;<a href=\"https:\/\/developers.facebook.com\/docs\/sharing\/webmasters\/web-crawlers\/\" target=\"_blank\" rel=\"noopener noreferrer\">Meta web crawlers<\/a>&#8220;, Meta for Developers, retrieved September 18, 2026.<\/li>\n<li>Apple: &#8220;<a href=\"https:\/\/support.apple.com\/en-us\/119829\" target=\"_blank\" rel=\"noopener noreferrer\">About Applebot<\/a>&#8220;, Apple Support, retrieved September 18, 2026.<\/li>\n<li>Common Crawl Foundation: &#8220;<a href=\"https:\/\/commoncrawl.org\/ccbot\" target=\"_blank\" rel=\"noopener noreferrer\">CCBot<\/a>&#8220;, Common Crawl, retrieved September 18, 2026.<\/li>\n<li>M. Koster, G. Illyes, H. Zeller, L. Sassman: &#8220;<a href=\"https:\/\/www.rfc-editor.org\/rfc\/rfc9309.html\" target=\"_blank\" rel=\"noopener noreferrer\">RFC 9309: Robots Exclusion Protocol<\/a>&#8220;, IETF, September 2022.<\/li>\n<li>European Union: &#8220;<a href=\"https:\/\/eur-lex.europa.eu\/eli\/dir\/2019\/790\/oj\" target=\"_blank\" rel=\"noopener noreferrer\">Directive (EU) 2019\/790 on copyright and related rights in the Digital Single Market<\/a>&#8220;, EUR-Lex, April 17, 2019.<\/li>\n<li>European Union: &#8220;<a href=\"https:\/\/eur-lex.europa.eu\/eli\/reg\/2024\/1689\/oj\" target=\"_blank\" rel=\"noopener noreferrer\">Regulation (EU) 2024\/1689 (AI Act)<\/a>&#8220;, EUR-Lex, June 13, 2024.<\/li>\n<li>Federal Office of Justice: &#8220;<a href=\"https:\/\/www.gesetze-im-internet.de\/urhg\/__44b.html\" target=\"_blank\" rel=\"noopener noreferrer\">Section 44b UrhG, text and data mining (German Copyright Act, German original)<\/a>&#8220;, Gesetze im Internet, retrieved September 18, 2026.<\/li>\n<li>Matthew Prince: &#8220;<a href=\"https:\/\/blog.cloudflare.com\/content-independence-day-no-ai-crawl-without-compensation\/\" target=\"_blank\" rel=\"noopener noreferrer\">Content Independence Day: no AI crawl without compensation<\/a>&#8220;, The Cloudflare Blog, July 1, 2025.<\/li>\n<li>Jeremy Howard: &#8220;<a href=\"https:\/\/llmstxt.org\/\" target=\"_blank\" rel=\"noopener noreferrer\">The \/llms.txt file<\/a>&#8220;, llmstxt.org, retrieved September 18, 2026.<\/li>\n<\/ul><\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>taismo SEO magazine Table of contents Three jobs, one operator The AI crawler list 2026 GPTBot vs. OAI-SearchBot How robots.txt is read Where robots.txt ends Block or allow Our own setup Verify your setup Five common mistakes FAQ Sources \u2606 Prefer taismo on Google GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers Dominik Breitbach [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":7980,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_taismo_jsonld":"","_taismo_h1":"","taismo_jsonld":"{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/\",\"url\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/\",\"name\":\"GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers\",\"description\":\"Which AI crawlers read your website in 2026, what GPTBot, OAI-SearchBot, ClaudeBot and PerplexityBot each control, how robots.txt is evaluated, and how to decide what to block.\",\"inLanguage\":\"en-US\",\"isPartOf\":{\"@id\":\"https:\/\/taismo.de\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#primaryimage\"},\"breadcrumb\":{\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#breadcrumb\"},\"author\":{\"@id\":\"https:\/\/taismo.de\/#person-dominik-breitbach\"},\"publisher\":{\"@id\":\"https:\/\/taismo.de\/#organization\"},\"datePublished\":\"2026-10-01T07:30:00+02:00\",\"dateModified\":\"2026-10-01T07:30:00+02:00\",\"mainEntity\":{\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#article\"},\"hasPart\":{\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#faqpage\"},\"speakable\":{\"@type\":\"SpeakableSpecification\",\"cssSelector\":[\".brand-claim\",\".lead-def\"]}},{\"@type\":\"BlogPosting\",\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#article\",\"headline\":\"GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers\",\"name\":\"GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers\",\"description\":\"A practical guide to GPTBot and the other AI crawlers: 17 user agents with operator and role, the difference between training and search crawlers, robots.txt evaluation rules, the limits of robots.txt, a decision framework and a four-step verification.\",\"url\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/\",\"mainEntityOfPage\":{\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/\"},\"isPartOf\":{\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/\"},\"inLanguage\":\"en-US\",\"datePublished\":\"2026-10-01T07:30:00+02:00\",\"dateModified\":\"2026-10-01T07:30:00+02:00\",\"author\":{\"@id\":\"https:\/\/taismo.de\/#person-dominik-breitbach\"},\"publisher\":{\"@id\":\"https:\/\/taismo.de\/#organization\"},\"image\":{\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#primaryimage\"},\"articleSection\":\"SEO Magazine\",\"wordCount\":3710,\"keywords\":[\"gptbot\",\"block gptbot\",\"gptbot robots.txt\",\"oai-searchbot\",\"claudebot\",\"perplexitybot\",\"ai crawlers\",\"robots.txt\",\"google-extended\",\"answer engine optimization\"],\"about\":[{\"@type\":\"Thing\",\"name\":\"AI crawler control with robots.txt\"},{\"@type\":\"Thing\",\"name\":\"GPTBot\",\"sameAs\":\"https:\/\/developers.openai.com\/api\/docs\/bots\"}],\"mentions\":[{\"@type\":\"Thing\",\"name\":\"OAI-SearchBot\",\"sameAs\":\"https:\/\/developers.openai.com\/api\/docs\/bots\"},{\"@type\":\"Thing\",\"name\":\"ClaudeBot\",\"sameAs\":\"https:\/\/support.claude.com\/en\/articles\/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler\"},{\"@type\":\"Thing\",\"name\":\"PerplexityBot\",\"sameAs\":\"https:\/\/docs.perplexity.ai\/guides\/bots\"},{\"@type\":\"Thing\",\"name\":\"Google-Extended\",\"sameAs\":\"https:\/\/developers.google.com\/search\/docs\/crawling-indexing\/google-common-crawlers\"},{\"@type\":\"Thing\",\"name\":\"CCBot\",\"sameAs\":\"https:\/\/commoncrawl.org\/ccbot\"},{\"@type\":\"Thing\",\"name\":\"robots.txt\",\"sameAs\":\"https:\/\/www.rfc-editor.org\/rfc\/rfc9309.html\"}],\"translationOfWork\":{\"@id\":\"https:\/\/taismo.de\/seo-magazin\/ki-crawler-steuern-gptbot\/#article\"}},{\"@type\":\"ImageObject\",\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#primaryimage\",\"url\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/2026-09-18_ki-crawler-steuern_hero.webp\",\"contentUrl\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/2026-09-18_ki-crawler-steuern_hero.webp\",\"width\":1344,\"height\":768,\"caption\":\"GPTBot, ClaudeBot and PerplexityBot: Controlling AI crawlers with robots.txt by allowing or blocking each user agent\",\"inLanguage\":\"en-US\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\/\/taismo.de\/en\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"SEO Magazine\",\"item\":\"https:\/\/taismo.de\/en\/seo-magazine\/\"},{\"@type\":\"ListItem\",\"position\":3,\"name\":\"GPTBot and AI Crawlers\",\"item\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/\"}]},{\"@type\":\"FAQPage\",\"@id\":\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#faqpage\",\"inLanguage\":\"en-US\",\"mainEntity\":[{\"@type\":\"Question\",\"name\":\"What is GPTBot?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"GPTBot is OpenAI's web crawler that collects publicly available content for training its AI models. You can block it with a User-agent: GPTBot group in robots.txt, and OpenAI publishes its IP ranges for verification.\"}},{\"@type\":\"Question\",\"name\":\"Should I block GPTBot?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"Block GPTBot if your content is your product, for example paid articles, courses or databases. If your content promotes your services, allowing it is usually the better choice, and in both cases you should keep OAI-SearchBot open for ChatGPT search.\"}},{\"@type\":\"Question\",\"name\":\"Does blocking GPTBot remove my website from ChatGPT?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"Blocking GPTBot keeps your content out of OpenAI's model training; your citations in ChatGPT search stay, because they depend on OAI-SearchBot. Only blocking OAI-SearchBot removes you from ChatGPT's search answers.\"}},{\"@type\":\"Question\",\"name\":\"Do AI crawlers respect robots.txt?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"The training and search crawlers of the large operators state that they follow robots.txt. Live fetch agents such as ChatGPT-User and Perplexity-User may ignore it, because a user triggered the request.\"}},{\"@type\":\"Question\",\"name\":\"How do I recognize a fake GPTBot?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"Check the IP address against OpenAI's published file at openai.com\/gptbot.json. A request with the GPTBot user agent from an address outside those ranges comes from someone else using the name.\"}},{\"@type\":\"Question\",\"name\":\"Does a robots.txt Disallow keep a page out of Google?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"A Disallow rule stops crawling of the content; the URL itself can still be indexed without a description if other pages link to it. To keep a page out of the index, use noindex and leave the page crawlable.\"}}]}]}","footnotes":""},"categories":[81],"tags":[],"class_list":["post-8097","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-seo-magazine"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v28.1 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>GPTBot: Block or Allow AI Crawlers in robots.txt | taismo<\/title>\n<meta name=\"description\" content=\"Should you block GPTBot? 17 AI crawler user agents, what each controls, and a robots.txt that keeps your AI citations. Get the list \ud83e\udd16\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"GPTBot: Block or Allow AI Crawlers in robots.txt | taismo\" \/>\n<meta property=\"og:description\" content=\"Should you block GPTBot? 17 AI crawler user agents, what each controls, and a robots.txt that keeps your AI citations. Get the list \ud83e\udd16\" \/>\n<meta property=\"og:url\" content=\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/\" \/>\n<meta property=\"og:site_name\" content=\"taismo| Agentur f\u00fcr SEO &amp; Webdesign\" \/>\n<meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/TaismoOnlineMarketing\" \/>\n<meta property=\"article:published_time\" content=\"2026-10-01T05:30:00+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2026-10-04T14:16:24+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/2026-09-18_ki-crawler-steuern_hero.webp\" \/>\n\t<meta property=\"og:image:width\" content=\"1344\" \/>\n\t<meta property=\"og:image:height\" content=\"768\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/webp\" \/>\n<meta name=\"author\" content=\"Dominik Breitbach\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/\"},\"author\":{\"name\":\"Dominik Breitbach\",\"@id\":\"https:\\\/\\\/taismo.de\\\/#person-dominik-breitbach\"},\"headline\":\"GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers\",\"datePublished\":\"2026-10-01T05:30:00+00:00\",\"dateModified\":\"2026-10-04T14:16:24+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/\"},\"wordCount\":3950,\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/09\\\/2026-09-18_ki-crawler-steuern_hero.webp\",\"articleSection\":[\"SEO Magazine\"],\"inLanguage\":\"en-US\"},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/\",\"name\":\"GPTBot: Block or Allow AI Crawlers in robots.txt | taismo\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/09\\\/2026-09-18_ki-crawler-steuern_hero.webp\",\"datePublished\":\"2026-10-01T05:30:00+00:00\",\"dateModified\":\"2026-10-04T14:16:24+00:00\",\"description\":\"Should you block GPTBot? 17 AI crawler user agents, what each controls, and a robots.txt that keeps your AI citations. Get the list \ud83e\udd16\",\"breadcrumb\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/#primaryimage\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/09\\\/2026-09-18_ki-crawler-steuern_hero.webp\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/09\\\/2026-09-18_ki-crawler-steuern_hero.webp\",\"width\":1344,\"height\":768,\"caption\":\"KI-Crawler steuern: GPTBot, ClaudeBot und PerplexityBot \u00fcber die robots.txt zulassen oder sperren\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/seo-magazine\\\/gptbot\\\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Startseite\",\"item\":\"https:\\\/\\\/taismo.de\\\/en\\\/search-marketing\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"name\":\"taismo| Agentur f\u00fcr SEO & Webdesign\",\"description\":\"\",\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"alternateName\":\"taismo\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/taismo.de\\\/en\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Organization\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\",\"name\":\"taismo SEO Agentur M\u00fcnchen\",\"alternateName\":\"taismo\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"logo\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"width\":512,\"height\":512,\"caption\":\"taismo SEO Agentur M\u00fcnchen\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\"},\"sameAs\":[\"https:\\\/\\\/www.facebook.com\\\/TaismoOnlineMarketing\",\"https:\\\/\\\/branchenbuch.portal.muenchen.de\\\/mhp\\\/1380360\\\/\",\"https:\\\/\\\/www.crunchbase.com\\\/organization\\\/taismo-gmbh\",\"https:\\\/\\\/de.trustpilot.com\\\/review\\\/taismo.de\",\"https:\\\/\\\/www.linkedin.com\\\/company\\\/taismo\\\/\",\"https:\\\/\\\/www.instagram.com\\\/taismo.de\",\"https:\\\/\\\/onlinemarketing.de\\\/dienstleister\\\/online-marketing-agentur\\\/kirchheim-bei-muenchen\\\/taismo\",\"https:\\\/\\\/www.agenturtipp.de\\\/agentur\\\/taismo-online-marketing\\\/\",\"https:\\\/\\\/startupvalley.news\\\/de\\\/taismo-online-marketing-agentur\\\/\",\"https:\\\/\\\/www.wlw.de\\\/de\\\/firma\\\/taismo-gmbh-1918135\",\"https:\\\/\\\/www.youtube.com\\\/@taismo8632\",\"https:\\\/\\\/techbehemoths.com\\\/company\\\/taismo-gmbh\",\"https:\\\/\\\/www.kununu.com\\\/de\\\/taismo1\",\"https:\\\/\\\/www.provenexpert.com\\\/taismo1\\\/\",\"https:\\\/\\\/www.provenemployer.com\\\/p\\\/taismo-employer\\\/\",\"https:\\\/\\\/www.trustedshops.de\\\/bewertung\\\/taismo-de\",\"https:\\\/\\\/www.kennstdueinen.de\\\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html\",\"https:\\\/\\\/www.ibusiness.de\\\/dienstleister\\\/jb\\\/5920834fb726.html\",\"https:\\\/\\\/www.seo-united.de\\\/seo-agenturen\\\/taismo-3345\\\/\"]}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"GPTBot: Block or Allow AI Crawlers in robots.txt | taismo","description":"Should you block GPTBot? 17 AI crawler user agents, what each controls, and a robots.txt that keeps your AI citations. Get the list \ud83e\udd16","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/","og_locale":"en_US","og_type":"article","og_title":"GPTBot: Block or Allow AI Crawlers in robots.txt | taismo","og_description":"Should you block GPTBot? 17 AI crawler user agents, what each controls, and a robots.txt that keeps your AI citations. Get the list \ud83e\udd16","og_url":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/","og_site_name":"taismo| Agentur f\u00fcr SEO &amp; Webdesign","article_publisher":"https:\/\/www.facebook.com\/TaismoOnlineMarketing","article_published_time":"2026-10-01T05:30:00+00:00","article_modified_time":"2026-10-04T14:16:24+00:00","og_image":[{"width":1344,"height":768,"url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/2026-09-18_ki-crawler-steuern_hero.webp","type":"image\/webp"}],"author":"Dominik Breitbach","twitter_card":"summary_large_image","schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#article","isPartOf":{"@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/"},"author":{"name":"Dominik Breitbach","@id":"https:\/\/taismo.de\/#person-dominik-breitbach"},"headline":"GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers","datePublished":"2026-10-01T05:30:00+00:00","dateModified":"2026-10-04T14:16:24+00:00","mainEntityOfPage":{"@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/"},"wordCount":3950,"publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"image":{"@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/2026-09-18_ki-crawler-steuern_hero.webp","articleSection":["SEO Magazine"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/","url":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/","name":"GPTBot: Block or Allow AI Crawlers in robots.txt | taismo","isPartOf":{"@id":"https:\/\/taismo.de\/en\/#website"},"primaryImageOfPage":{"@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#primaryimage"},"image":{"@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/2026-09-18_ki-crawler-steuern_hero.webp","datePublished":"2026-10-01T05:30:00+00:00","dateModified":"2026-10-04T14:16:24+00:00","description":"Should you block GPTBot? 17 AI crawler user agents, what each controls, and a robots.txt that keeps your AI citations. Get the list \ud83e\udd16","breadcrumb":{"@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#primaryimage","url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/2026-09-18_ki-crawler-steuern_hero.webp","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/2026-09-18_ki-crawler-steuern_hero.webp","width":1344,"height":768,"caption":"KI-Crawler steuern: GPTBot, ClaudeBot und PerplexityBot \u00fcber die robots.txt zulassen oder sperren"},{"@type":"BreadcrumbList","@id":"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Startseite","item":"https:\/\/taismo.de\/en\/search-marketing\/"},{"@type":"ListItem","position":2,"name":"GPTBot, ClaudeBot and PerplexityBot: How to Control AI Crawlers"}]},{"@type":"WebSite","@id":"https:\/\/taismo.de\/en\/#website","url":"https:\/\/taismo.de\/en\/","name":"taismo| Agentur f\u00fcr SEO & Webdesign","description":"","publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"alternateName":"taismo","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/taismo.de\/en\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/taismo.de\/en\/#organization","name":"taismo SEO Agentur M\u00fcnchen","alternateName":"taismo","url":"https:\/\/taismo.de\/en\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/","url":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","width":512,"height":512,"caption":"taismo SEO Agentur M\u00fcnchen"},"image":{"@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/TaismoOnlineMarketing","https:\/\/branchenbuch.portal.muenchen.de\/mhp\/1380360\/","https:\/\/www.crunchbase.com\/organization\/taismo-gmbh","https:\/\/de.trustpilot.com\/review\/taismo.de","https:\/\/www.linkedin.com\/company\/taismo\/","https:\/\/www.instagram.com\/taismo.de","https:\/\/onlinemarketing.de\/dienstleister\/online-marketing-agentur\/kirchheim-bei-muenchen\/taismo","https:\/\/www.agenturtipp.de\/agentur\/taismo-online-marketing\/","https:\/\/startupvalley.news\/de\/taismo-online-marketing-agentur\/","https:\/\/www.wlw.de\/de\/firma\/taismo-gmbh-1918135","https:\/\/www.youtube.com\/@taismo8632","https:\/\/techbehemoths.com\/company\/taismo-gmbh","https:\/\/www.kununu.com\/de\/taismo1","https:\/\/www.provenexpert.com\/taismo1\/","https:\/\/www.provenemployer.com\/p\/taismo-employer\/","https:\/\/www.trustedshops.de\/bewertung\/taismo-de","https:\/\/www.kennstdueinen.de\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html","https:\/\/www.ibusiness.de\/dienstleister\/jb\/5920834fb726.html","https:\/\/www.seo-united.de\/seo-agenturen\/taismo-3345\/"]}]}},"_links":{"self":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/8097","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/comments?post=8097"}],"version-history":[{"count":1,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/8097\/revisions"}],"predecessor-version":[{"id":8098,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/8097\/revisions\/8098"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media\/7980"}],"wp:attachment":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media?parent=8097"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/categories?post=8097"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/tags?post=8097"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}