{"id":7765,"date":"2026-08-21T07:30:00","date_gmt":"2026-08-21T05:30:00","guid":{"rendered":"https:\/\/taismo.de\/?p=7765"},"modified":"2026-08-11T13:20:26","modified_gmt":"2026-08-11T11:20:26","slug":"ai-crawler","status":"publish","type":"post","link":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/","title":{"rendered":"AI crawler"},"content":{"rendered":"<style>\n.taismo-art{display:grid;grid-template-columns:320px minmax(0,1fr);gap:36px;align-items:start}\n.taismo-art .toc-sticky{position:sticky;top:80px}\n.taismo-art .toc-sticky ol li{margin-bottom:9px}\n.taismo-art .toc-sticky ol a:hover{color:var(--toc-hover) !important}\n.taismo-art .taismo-main h2{scroll-margin-top:90px;margin-top:38px}\n.taismo-art .lead-def{font-size:1.15em;line-height:1.6;background:#f7f2ec;border-left:4px solid #ED8924;padding:16px 20px;border-radius:8px}\n.taismo-art table{border-collapse:collapse;width:100%;margin:18px 0}\n.taismo-art th,.taismo-art td{border:1px solid #d9e0df;padding:9px 12px;text-align:left;font-size:0.95em}\n.taismo-art th{background:#072328;color:#fff}\n.taismo-art figure{margin:26px 0}\n.taismo-art figcaption{font-size:0.9em;color:#55605f;margin-top:8px}\n.taismo-art pre{background:#072328;color:#f5f5f5;padding:16px 18px;border-radius:10px;overflow-x:auto;font-size:0.86em;line-height:1.5}\n@media(max-width:900px){.taismo-art{grid-template-columns:1fr}.taismo-art .toc-sticky{position:static}}\n<\/style>\n<div class=\"taismo-art\">\n<aside class=\"toc-sticky\" style=\"--toc-hover:#ED8924;background:#072328;border-radius:14px;padding:22px 24px\">\n<div style=\"text-align:center;margin-bottom:14px\"><svg fill=\"#ffffff\" role=\"img\" aria-label=\"taismo Wiki\" style=\"width:200px;max-width:80%;height:auto;display:inline-block\" id=\"Ebene_1\" data-name=\"Ebene 1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 209.04 52.32\"> <g id=\"Layer_2\" data-name=\"Layer 2\"> <g id=\"wikipedia\"> <path d=\"M97.08,27.77h-17.76c-.77,0-1.4.63-1.4,1.4v17.76c0,.77.63,1.4,1.4,1.4h17.76c.77,0,1.4-.63,1.4-1.4v-17.76c0-.77-.63-1.4-1.4-1.4ZM97.22,33.22c0,.25-.21.46-.46.46h-.67c-.07.03-.13.07-.19.11-.19.18-.33.41-.4.66,0,0,0,.01,0,.02-.03.07-2.9,7.29-3.87,9.67v.02c-.09.17-.19.33-.31.48-.15.2-.38.32-.63.33-.56,0-.84-.63-.93-.84-.28-.62-.79-1.83-1.51-3.57-.55,1.24-1.25,2.8-1.64,3.59h0c-.19.36-.49.83-.97.83-.56,0-.82-.63-.9-.84h0c-.49-1.25-1.23-2.99-1.95-4.68-.79-1.83-2.39-5.73-2.43-5.78h-.72c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h4.13c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.82s.03.09.05.14c.59,1.61,2.36,5.75,3.1,7.47l1.28-2.81c-.15-.36-2.12-4.68-2.2-4.8h-.4c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.98c.25,0,.46.21.46.46v.58c0,.25-.21.46-.46.46h-.1c.06.14.94,2.12.94,2.12,0,0,.93-2.06.95-2.12h-.39c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.81c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.43s-2.06,4.08-2.06,4.08l1.56,3.51s2.9-7.59,2.9-7.6h-.96c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h3.78c.25,0,.46.2.46.46v.58Z\"\/> <\/g> <\/g> <text font-family=\"Arial,Helvetica,sans-serif\" font-size=\"27.79\" font-weight=\"800\" transform=\"translate(99.44 47.64) scale(.99 1)\"><tspan x=\"0\" y=\"0\">IKI<\/tspan><\/text> <g id=\"Gruppe_796\" data-name=\"Gruppe 796\"> <path id=\"Pfad_1293\" data-name=\"Pfad 1293\" d=\"M6.92,16.77h-3.56v-6.56h3.66V3.37h8.04v6.84h5.69v6.56h-5.58v8.97c-.08,1.05.13,2.1.6,3.04.4.6,1.24.9,2.51.9.54,0,1.01,0,1.39-.03.41-.02.82-.07,1.23-.13v6.35c-.75.19-1.52.31-2.3.36-1.02.09-1.95.13-2.79.13-1.41.04-2.81-.17-4.15-.6-1.07-.36-2.02-1.01-2.76-1.86-.76-.95-1.29-2.06-1.54-3.25-.34-1.55-.49-3.14-.46-4.73v-9.14h0Z\"\/> <path id=\"Pfad_1294\" data-name=\"Pfad 1294\" d=\"M44.43,35.91h-7.06l-.44-1.97c-.68.72-1.48,1.34-2.35,1.8-.95.46-2.01.69-3.07.66-.99,0-1.98-.1-2.95-.33-.92-.21-1.78-.63-2.51-1.23-.77-.66-1.37-1.49-1.75-2.43-.48-1.28-.71-2.63-.66-3.99v-1.42c-.04-1.17.2-2.33.71-3.39.44-.87,1.09-1.63,1.89-2.19.82-.56,1.74-.96,2.71-1.18,1.06-.24,2.14-.36,3.23-.36.87,0,1.72.06,2.57.22.72.14,1.41.36,2.08.66v-.93c0-.54-.07-1.07-.22-1.59-.16-.48-.46-.91-.87-1.2-.55-.38-1.18-.63-1.83-.77-1-.2-2.01-.29-3.04-.28-.91,0-1.81.09-2.71.25-.97.16-1.8.32-2.49.46v-6.12c.48-.13.96-.22,1.45-.28.57-.07,1.16-.14,1.78-.22.62-.07,1.22-.13,1.78-.16.57-.04,1.04-.05,1.45-.05,2.1-.05,4.18.19,6.21.71,1.4.37,2.69,1.08,3.74,2.08.92.94,1.57,2.13,1.86,3.42.34,1.54.51,3.1.49,4.68v15.15h.01,0ZM33.71,30.83c1.63.12,3.04-1.1,3.16-2.72v-2.58c-.77-.58-1.72-.87-2.68-.82-.85-.04-1.69.16-2.43.57-.6.38-.9,1.16-.9,2.33v.38c-.07.78.16,1.55.66,2.16.6.51,1.39.76,2.19.69\"\/> <rect id=\"Rechteck_760\" data-name=\"Rechteck 760\" x=\"49.46\" y=\"10.21\" width=\"8.26\" height=\"25.71\"\/> <path d=\"M131.36,22.82v-.42c-.06-1.6.12-3.21.52-4.76.35-1.1,1.23-1.64,2.65-1.64.59-.03,1.16.11,1.66.41.43.29.75.73.9,1.23.2.66.32,1.34.36,2.02.05.81.08,1.72.08,2.74v.42h8.38v-1.01c.04-1.92-.23-3.83-.82-5.66-.48-1.43-1.27-2.73-2.33-3.8-1.02-.99-2.27-1.73-3.64-2.13-1.53-.45-3.12-.67-4.71-.66-1.57-.01-3.12.2-4.62.66-1.36.41-2.6,1.14-3.61,2.13-1.06,1.07-1.86,2.37-2.35,3.8-.61,1.82-.89,3.74-.85,5.66v1.01h8.38Z\"\/> <path d=\"M93,22.82v-2.38c-.06-.88.16-1.76.63-2.51.39-.57,1.04-.89,1.72-.87.26,0,.51.03.77.08.26.06.51.19.71.38.25.24.43.54.52.87.15.51.21,1.03.19,1.56v2.88h8.2v-2.7c-.05-.8.16-1.6.6-2.27.4-.52,1.03-.82,1.69-.79.54,0,1.07.17,1.51.49.45.33.69,1.13.69,2.41v2.86h8.2v-4.99c.03-1.32-.17-2.64-.6-3.89-.34-.96-.9-1.83-1.64-2.54-.69-.63-1.51-1.11-2.41-1.39-.92-.3-1.88-.44-2.84-.44-1.16-.02-2.3.21-3.36.69-1.09.55-2.07,1.3-2.87,2.22-.57-.95-1.38-1.72-2.35-2.24-1.16-.49-2.41-.72-3.66-.66-1.18-.04-2.35.22-3.39.76-.99.57-1.88,1.31-2.63,2.19l-.54-2.3h-7.34v12.61h8.2Z\"\/> <path d=\"M73.21,26.15c0-1.84,1.49-3.33,3.33-3.33h2.18c-.87-.85-1.94-1.47-3.12-1.8l-1.75-.6c-.73-.25-1.35-.46-1.86-.63-.43-.13-.85-.3-1.26-.49-.28-.13-.52-.33-.71-.57-.16-.28-.24-.6-.22-.93,0-.22.04-.45.13-.66.13-.25.34-.46.6-.57.43-.2.89-.34,1.36-.41.81-.13,1.62-.18,2.43-.16.92,0,1.85.07,2.76.22.82.11,1.63.28,2.43.49v-6.12c-.48-.13-.95-.22-1.45-.28-.56-.07-1.16-.14-1.78-.22-.61-.07-1.21-.13-1.78-.16s-1.05-.05-1.45-.05c-3.72,0-6.51.75-8.37,2.27-1.78,1.35-2.82,3.48-2.79,5.71-.03,1.07.11,2.13.41,3.15.24.76.64,1.46,1.18,2.05.52.56,1.15,1.01,1.83,1.34.79.38,1.6.72,2.43,1.01l1.26.44c.83.28,1.63.61,2.41,1.01.47.28.75.8.71,1.34.07.72-.35,1.4-1.04,1.64-1.23.31-2.51.45-3.77.38-.95,0-1.9-.09-2.84-.25-1.06-.16-1.93-.31-2.63-.46v6.12c.5.13,1,.22,1.51.28.6.07,1.23.14,1.89.22.66.07,1.3.13,1.92.16.62.03,1.13.05,1.53.05,1.5.03,3.01-.1,4.49-.36v-9.83Z\"\/> <\/g> <g id=\"e\"> <path d=\"M150.81,38.2c-.26-1.16-1.38-2.22-2.54-2.45-.71-.14-2.69-.32-3.99-.3-1.31-.01-3.28.16-3.99.3-1.16.23-2.29,1.29-2.54,2.45-.15.7-.29,1.66-.29,2.97,0,1.31.15,2.27.29,2.97.26,1.16,1.38,2.22,2.54,2.45.38.08,1.14.16,1.95.22.09.15.19.3.3.46.43.66.77,1.11,1.02,1.43.42.53,1.03.53,1.45,0,.25-.32.58-.78,1.02-1.43.11-.16.21-.32.3-.46.82-.06,1.57-.15,1.95-.22,1.16-.23,2.29-1.29,2.54-2.45.15-.7.29-1.66.29-2.97,0-1.31-.15-2.27-.29-2.97h0ZM145.03,45.13c-.03.16-.18.31-.34.34-.1.02-.23.03-.42.03-.18,0-.32-.02-.42-.03-.16-.03-.31-.18-.34-.34-.02-.1-.03-.23-.03-.42,0-.18.02-.32.03-.42.03-.16.18-.31.34-.34.1-.02.23-.03.42-.03.18,0,.32.02.42.03.16.03.31.18.34.34.02.1.03.23.03.42,0,.18-.02.32-.03.42ZM146.65,40.76c-.21.29-.51.56-.97.84-.58.36-.67.5-.67.79v.11c0,.41-.33.74-.74.74s-.74-.33-.74-.74v-.26c0-.34.08-.63.26-.88.17-.24.42-.46.8-.68.69-.41.82-.63.82-.94,0-.38-.32-.63-.83-.63-.55,0-.91.25-1.06.74-.1.32-.4.54-.75.54-.02,0-.04,0-.06,0-.24-.02-.46-.14-.6-.35-.14-.2-.17-.45-.1-.68.17-.52.49-.96.93-1.26.45-.3,1.01-.47,1.63-.47s1.24.21,1.69.58c.45.38.71.91.71,1.49,0,.4-.1.74-.32,1.04h0Z\"\/> <\/g> <g id=\"e-2\" data-name=\"e\"> <path d=\"M156.19,28.79c-.17-.75-.89-1.44-1.64-1.58-.46-.09-1.73-.2-2.58-.2-.84,0-2.12.1-2.58.2-.75.15-1.48.83-1.64,1.58-.09.45-.19,1.07-.19,1.92,0,.84.09,1.46.19,1.92.17.75.89,1.44,1.64,1.58.25.05.73.11,1.26.14.06.09.12.19.19.3.28.42.49.72.66.93.27.34.66.34.93,0,.16-.21.38-.5.66-.93.07-.11.13-.21.19-.3.53-.04,1.01-.09,1.26-.14.75-.15,1.48-.83,1.64-1.58.09-.45.19-1.07.19-1.92,0-.84-.09-1.46-.19-1.92h0ZM152.46,33.27c-.02.1-.11.2-.22.22-.06.01-.15.02-.27.02-.12,0-.2,0-.27-.02-.1-.02-.2-.11-.22-.22-.01-.06-.02-.15-.02-.27,0-.12,0-.2.02-.27.02-.1.11-.2.22-.22.06-.01.15-.02.27-.02.12,0,.2,0,.27.02.1.02.2.11.22.22.01.06.02.15.02.27,0,.12,0,.2-.02.27ZM153.51,30.45c-.13.19-.33.36-.63.54-.37.23-.43.32-.43.51v.07c0,.26-.21.48-.48.48s-.48-.21-.48-.48v-.17c0-.22.05-.4.17-.57.11-.16.27-.3.51-.44.45-.26.53-.41.53-.61,0-.25-.21-.4-.53-.4-.35,0-.59.16-.69.48-.07.21-.26.35-.48.35-.01,0-.02,0-.04,0-.16-.01-.3-.09-.39-.22-.09-.13-.11-.29-.06-.44.11-.34.32-.62.6-.81.29-.2.65-.3,1.05-.3s.8.13,1.09.37c.29.25.46.59.46.96,0,.26-.07.48-.21.67h0Z\"\/> <\/g> <g id=\"e-3\" data-name=\"e\"> <path d=\"M144.91,28.64c-.1-.44-.52-.84-.96-.92-.27-.05-1.01-.12-1.5-.11-.49,0-1.24.06-1.5.11-.44.08-.86.49-.96.92-.06.27-.11.63-.11,1.12,0,.49.05.85.11,1.12.1.44.52.84.96.92.14.03.43.06.74.08.03.05.07.11.11.17.16.25.29.42.38.54.16.2.39.2.54,0,.09-.12.22-.29.38-.54.04-.06.08-.12.11-.17.31-.02.59-.06.74-.08.44-.08.86-.49.96-.92.06-.27.11-.63.11-1.12,0-.49-.05-.85-.11-1.12h0ZM142.73,31.26c-.01.06-.07.12-.13.13-.04,0-.09.01-.16.01-.07,0-.12,0-.16-.01-.06-.01-.12-.07-.13-.13,0-.04-.01-.09-.01-.16,0-.07,0-.12.01-.16.01-.06.07-.12.13-.13.04,0,.09-.01.16-.01.07,0,.12,0,.16.01.06.01.12.07.13.13,0,.04.01.09.01.16,0,.07,0,.12-.01.16ZM143.34,29.61c-.08.11-.19.21-.37.32-.22.13-.25.19-.25.3v.04c0,.15-.12.28-.28.28s-.28-.12-.28-.28v-.1c0-.13.03-.24.1-.33.06-.09.16-.17.3-.26.26-.15.31-.24.31-.35,0-.14-.12-.24-.31-.24-.21,0-.34.09-.4.28-.04.12-.15.2-.28.2,0,0-.01,0-.02,0-.09,0-.17-.05-.23-.13-.05-.08-.06-.17-.04-.25.06-.2.18-.36.35-.47.17-.11.38-.18.61-.18s.47.08.63.22c.17.14.27.34.27.56,0,.15-.04.28-.12.39h0Z\"\/> <\/g> <\/svg><\/div>\n<h3 style=\"text-align:center;color:#fff;margin:0 0 14px;font-size:1.1em\">Contents<\/h3>\n<ol style=\"margin:0;padding-left:1.2em;line-height:1.5;font-weight:600;color:#fff;font-size:0.9em\">\n<li><a href=\"#which-ai-crawlers-exist\" style=\"color:#fff;text-decoration:none\">Which AI crawlers exist?<\/a><\/li>\n<li><a href=\"#control-via-robots-txt\" style=\"color:#fff;text-decoration:none\">How do you control AI crawlers via robots.txt?<\/a><\/li>\n<li><a href=\"#gptbot-and-oai-searchbot\" style=\"color:#fff;text-decoration:none\">Why does blocking GPTBot not remove you from ChatGPT search?<\/a><\/li>\n<li><a href=\"#block-or-allow\" style=\"color:#fff;text-decoration:none\">Should you block or allow AI crawlers?<\/a><\/li>\n<li><a href=\"#faq\" style=\"color:#fff;text-decoration:none\">Frequently asked questions about AI crawlers<\/a><\/li>\n<\/ol>\n<\/aside>\n<div class=\"taismo-main\">\n<h1 class=\"brand-claim\">What is an AI crawler?<\/h1>\n<p class=\"lead-def\"><strong>AI crawlers<\/strong> are programs that fetch web pages automatically to collect content for AI systems. They serve three purposes: training language models, building the search index of AI search systems, and fetching individual pages live during a user request. Well-known AI crawlers are <strong>GPTBot<\/strong> and <strong>OAI-SearchBot<\/strong> from OpenAI, ClaudeBot from Anthropic, and PerplexityBot.<\/p>\n<figure><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/08\/ki-crawler-gptbot-oai-searchbot-steuern.webp\" alt=\"AI crawlers explained: GPTBot, OAI-SearchBot, and ChatGPT-User collect web content for training, AI search, and live retrieval, taismo SEO glossary\" title=\"AI crawlers: who collects your content for AI systems\" width=\"1200\" height=\"675\" style=\"width:100%;height:auto;border-radius:12px\" \/><\/figure>\n<h2 id=\"which-ai-crawlers-exist\">Which AI crawlers exist?<\/h2>\n<p>Six identifiers appear most often in log files and robots.txt discussions:<\/p>\n<table>\n<tr>\n<th>Identifier<\/th>\n<th>Operator<\/th>\n<th>Purpose<\/th>\n<\/tr>\n<tr>\n<td>GPTBot<\/td>\n<td>OpenAI<\/td>\n<td>Collects content for training the GPT models<\/td>\n<\/tr>\n<tr>\n<td>OAI-SearchBot<\/td>\n<td>OpenAI<\/td>\n<td>Builds the index of ChatGPT search<\/td>\n<\/tr>\n<tr>\n<td>ChatGPT-User<\/td>\n<td>OpenAI<\/td>\n<td>Fetches pages live when a user asks for them in ChatGPT<\/td>\n<\/tr>\n<tr>\n<td>ClaudeBot<\/td>\n<td>Anthropic<\/td>\n<td>Collects content for training the Claude models<\/td>\n<\/tr>\n<tr>\n<td>PerplexityBot<\/td>\n<td>Perplexity AI<\/td>\n<td>Builds the search index of Perplexity<\/td>\n<\/tr>\n<tr>\n<td>Google-Extended<\/td>\n<td>Google<\/td>\n<td>Not a crawler of its own but a control token: governs whether your content is used for training the Gemini models<\/td>\n<\/tr>\n<\/table>\n<p>Unlike a classic <a href=\"https:\/\/taismo.de\/en\/what-is\/crawler\/\">crawler<\/a> such as Googlebot, many of these bots do not work for a search index but for a language model. That is exactly why it pays to check the purpose of each identifier before blocking it.<\/p>\n<h2 id=\"control-via-robots-txt\">How do you control AI crawlers via robots.txt?<\/h2>\n<p>You control AI crawlers like any bot through the <a href=\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/\">robots.txt<\/a> in the root directory of your domain. Every rule applies exactly to the named identifier:<\/p>\n<pre><code>User-agent: GPTBot\nDisallow: \/\n\nUser-agent: OAI-SearchBot\nAllow: \/<\/code><\/pre>\n<p>This example forbids OpenAI from collecting content for model training and at the same time allows inclusion in ChatGPT search. Important for your expectations: robots.txt is an instruction, not a barrier. Reputable providers comply, rogue scrapers do not.<\/p>\n<figure><a href=\"https:\/\/taismo.de\/en\/geo\/\" title=\"GEO: optimization for AI visibility by taismo\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 1000 320\" role=\"img\" aria-labelledby=\"abb5t abb5d\" style=\"width:100%;height:auto;max-width:820px;display:block;margin:0 auto\"><title id=\"abb5t\">Three OpenAI crawlers, three independent switches<\/title><desc id=\"abb5d\">GPTBot only controls model training, OAI-SearchBot controls visibility in ChatGPT search, ChatGPT-User controls live retrieval on user request.<\/desc><rect x=\"0\" y=\"0\" width=\"1000\" height=\"320\" fill=\"#ffffff\"\/><text x=\"500\" y=\"40\" fill=\"#072328\" font-family=\"Arial,sans-serif\" font-size=\"21\" font-weight=\"800\" text-anchor=\"middle\">Your robots.txt: three identifiers, three independent switches<\/text><rect x=\"40\" y=\"70\" width=\"290\" height=\"150\" rx=\"14\" fill=\"#fbf1e6\" stroke=\"#ED8924\" stroke-width=\"2.5\"\/><text x=\"185\" y=\"108\" fill=\"#072328\" font-family=\"Arial,sans-serif\" font-size=\"21\" font-weight=\"800\" text-anchor=\"middle\">GPTBot<\/text><rect x=\"105\" y=\"122\" width=\"160\" height=\"30\" rx=\"15\" fill=\"#ED8924\"\/><text x=\"185\" y=\"143\" fill=\"#ffffff\" font-family=\"Arial,sans-serif\" font-size=\"15\" font-weight=\"700\" text-anchor=\"middle\">model training<\/text><text x=\"185\" y=\"180\" fill=\"#5a6b6d\" font-family=\"Arial,sans-serif\" font-size=\"14\" text-anchor=\"middle\">block = no training,<\/text><text x=\"185\" y=\"200\" fill=\"#5a6b6d\" font-family=\"Arial,sans-serif\" font-size=\"14\" text-anchor=\"middle\">search stays untouched<\/text><rect x=\"355\" y=\"70\" width=\"290\" height=\"150\" rx=\"14\" fill=\"#fbf1e6\" stroke=\"#ED8924\" stroke-width=\"2.5\"\/><text x=\"500\" y=\"108\" fill=\"#072328\" font-family=\"Arial,sans-serif\" font-size=\"21\" font-weight=\"800\" text-anchor=\"middle\">OAI-SearchBot<\/text><rect x=\"415\" y=\"122\" width=\"170\" height=\"30\" rx=\"15\" fill=\"#072328\"\/><text x=\"500\" y=\"143\" fill=\"#ffffff\" font-family=\"Arial,sans-serif\" font-size=\"15\" font-weight=\"700\" text-anchor=\"middle\">ChatGPT search<\/text><text x=\"500\" y=\"180\" fill=\"#5a6b6d\" font-family=\"Arial,sans-serif\" font-size=\"14\" text-anchor=\"middle\">block = removed from<\/text><text x=\"500\" y=\"200\" fill=\"#5a6b6d\" font-family=\"Arial,sans-serif\" font-size=\"14\" text-anchor=\"middle\">ChatGPT search answers<\/text><rect x=\"670\" y=\"70\" width=\"290\" height=\"150\" rx=\"14\" fill=\"#fbf1e6\" stroke=\"#ED8924\" stroke-width=\"2.5\"\/><text x=\"815\" y=\"108\" fill=\"#072328\" font-family=\"Arial,sans-serif\" font-size=\"21\" font-weight=\"800\" text-anchor=\"middle\">ChatGPT-User<\/text><rect x=\"740\" y=\"122\" width=\"150\" height=\"30\" rx=\"15\" fill=\"#ED8924\"\/><text x=\"815\" y=\"143\" fill=\"#ffffff\" font-family=\"Arial,sans-serif\" font-size=\"15\" font-weight=\"700\" text-anchor=\"middle\">live retrieval<\/text><text x=\"815\" y=\"180\" fill=\"#5a6b6d\" font-family=\"Arial,sans-serif\" font-size=\"14\" text-anchor=\"middle\">user asks, bot fetches the page;<\/text><text x=\"815\" y=\"200\" fill=\"#5a6b6d\" font-family=\"Arial,sans-serif\" font-size=\"14\" text-anchor=\"middle\">robots.txt may not apply<\/text><text x=\"500\" y=\"265\" fill=\"#072328\" font-family=\"Arial,sans-serif\" font-size=\"17\" font-weight=\"700\" text-anchor=\"middle\">Blocking GPTBot does NOT remove your website from ChatGPT search.<\/text><text x=\"990\" y=\"308\" fill=\"#9aa9ab\" font-family=\"Arial,sans-serif\" font-size=\"13\" text-anchor=\"end\">Fig. 1 \u00b7 taismo<\/text><\/svg><\/a><figcaption><strong>Fig. 1: Three OpenAI crawlers, three independent switches: GPTBot only controls training, OAI-SearchBot controls ChatGPT search, ChatGPT-User controls live retrieval.<\/strong><\/figcaption><\/figure>\n<h2 id=\"gptbot-and-oai-searchbot\">Why does blocking GPTBot not remove you from ChatGPT search?<\/h2>\n<p><strong>Because GPTBot and OAI-SearchBot are independent switches.<\/strong> OpenAI documents the separation explicitly: each bot only controls its own purpose, and a site owner can allow OAI-SearchBot while disallowing GPTBot. In practice this means:<\/p>\n<ul>\n<li><strong>GPTBot blocked, OAI-SearchBot allowed:<\/strong> Your content does not flow into model training, but your website still appears in the answers of ChatGPT search.<\/li>\n<li><strong>OAI-SearchBot blocked:<\/strong> According to OpenAI, your website is no longer shown in ChatGPT search answers, regardless of what applies to GPTBot.<\/li>\n<li><strong>ChatGPT-User:<\/strong> Fetches pages on direct user request. Because the action is initiated by a human, robots.txt rules may not apply here according to OpenAI.<\/li>\n<\/ul>\n<p>The common shortcut \u201cblock GPTBot = out of ChatGPT\u201d is therefore wrong in both directions: the training block costs no visibility in AI search, and whoever wants to disappear has to hit the right switch.<\/p>\n<h2 id=\"block-or-allow\">Should you block or allow AI crawlers?<\/h2>\n<p>For most companies the answer is: <strong>Allow them, at least the search crawlers.<\/strong> Answers from ChatGPT, Perplexity, and AI search are a growing visibility channel; blocking OAI-SearchBot or PerplexityBot means giving up mentions and source links. A training block via GPTBot or Google-Extended is a separate decision, for example for exclusive content.<\/p>\n<p>If you allow AI crawlers, you can make their work easier: an <a href=\"https:\/\/taismo.de\/en\/what-is\/llms-txt\/\">llms.txt<\/a> summarizes the most important pages of a domain as a briefing, and citably structured content raises the chance of being mentioned. Both belong to <a href=\"https:\/\/taismo.de\/en\/what-is\/generative-engine-optimization\/\">Generative Engine Optimization (GEO)<\/a> and are the core of our <a href=\"https:\/\/taismo.de\/en\/geo\/\">optimization for AI visibility<\/a>.<\/p>\n<h2 id=\"faq\">Frequently asked questions about AI crawlers<\/h2>\n<p><strong>Is Googlebot an AI crawler?<\/strong><br \/>\nNo. Googlebot is Google&#8217;s classic search crawler; its data, however, also feeds AI Overviews and AI Mode. For training the Gemini models, Google offers a separate robots.txt switch called Google-Extended, which does not affect Google Search.<\/p>\n<p><strong>Do AI crawlers respect robots.txt?<\/strong><br \/>\nMajor providers such as OpenAI, Anthropic, and Google document their crawlers and state that they respect robots.txt. The file is not a technical barrier, though: rogue bots ignore it. If you want to protect content strictly, you need server-side blocks.<\/p>\n<p><strong>How do I recognize AI crawlers in my log files?<\/strong><br \/>\nBy the user agent: identifiers such as GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, or PerplexityBot appear in the user agent string of every request. The providers also publish IP lists that let you distinguish real crawlers from impostors.<\/p>\n<\/p><\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>IKI Contents Which AI crawlers exist? How do you control AI crawlers via robots.txt? Why does blocking GPTBot not remove you from ChatGPT search? Should you block or allow AI crawlers? Frequently asked questions about AI crawlers What is an AI crawler? AI crawlers are programs that fetch web pages automatically to collect content for [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":7703,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_taismo_jsonld":"","taismo_jsonld":"{\"@context\": \"https:\/\/schema.org\", \"@graph\": [{\"@type\": \"WebPage\", \"@id\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#webpage\", \"url\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/\", \"name\": \"What Is an AI Crawler? GPTBot and Co Explained\", \"isPartOf\": {\"@id\": \"https:\/\/taismo.de\/#website\"}, \"inLanguage\": \"en-US\", \"primaryImageOfPage\": {\"@id\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#primaryimage\"}, \"mainEntity\": {\"@id\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#definedterm\"}, \"breadcrumb\": {\"@id\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#breadcrumb\"}, \"datePublished\": \"2026-08-21T07:30:00+02:00\", \"dateModified\": \"2026-08-21T07:30:00+02:00\", \"speakable\": {\"@type\": \"SpeakableSpecification\", \"cssSelector\": [\".brand-claim\", \".lead-def\"]}, \"author\": {\"@id\": \"https:\/\/taismo.de\/#person-dominik-breitbach\"}, \"publisher\": {\"@id\": \"https:\/\/taismo.de\/#organization\"}}, {\"@type\": \"ImageObject\", \"@id\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#primaryimage\", \"contentUrl\": \"https:\/\/taismo.de\/wp-content\/uploads\/2026\/08\/ki-crawler-gptbot-oai-searchbot-steuern.webp\", \"url\": \"https:\/\/taismo.de\/wp-content\/uploads\/2026\/08\/ki-crawler-gptbot-oai-searchbot-steuern.webp\", \"width\": 1200, \"height\": 675, \"caption\": \"AI crawlers: who collects your content for AI systems\"}, {\"@type\": \"DefinedTerm\", \"@id\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#definedterm\", \"name\": \"AI crawler\", \"alternateName\": \"KI-Crawler\", \"termCode\": \"ai-crawler\", \"url\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/\", \"inLanguage\": \"en-US\", \"inDefinedTermSet\": {\"@id\": \"https:\/\/taismo.de\/#seo-wiki\"}, \"description\": \"AI crawlers are programs that fetch web pages automatically to collect content for AI systems. They serve three purposes: training language models, building the search index of AI search systems, and fetching individual pages live during a user request. Well-known AI crawlers are GPTBot and OAI-SearchBot from OpenAI, ClaudeBot from Anthropic, and PerplexityBot.\", \"disambiguatingDescription\": \"Crawlers working for AI systems such as ChatGPT, Claude, or Perplexity. Not to be confused with classic search engine crawlers like Googlebot, which builds the index of Google Search.\", \"author\": {\"@id\": \"https:\/\/taismo.de\/#person-dominik-breitbach\"}, \"publisher\": {\"@id\": \"https:\/\/taismo.de\/#organization\"}}, {\"@type\": \"BreadcrumbList\", \"@id\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#breadcrumb\", \"itemListElement\": [{\"@type\": \"ListItem\", \"position\": 1, \"name\": \"Home\", \"item\": \"https:\/\/taismo.de\/en\/\"}, {\"@type\": \"ListItem\", \"position\": 2, \"name\": \"SEO Glossary\", \"item\": \"https:\/\/taismo.de\/en\/what-is\/\"}, {\"@type\": \"ListItem\", \"position\": 3, \"name\": \"AI crawler\"}]}, {\"@type\": \"FAQPage\", \"@id\": \"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#faqpage\", \"mainEntity\": [{\"@type\": \"Question\", \"name\": \"Is Googlebot an AI crawler?\", \"acceptedAnswer\": {\"@type\": \"Answer\", \"text\": \"No. Googlebot is Google's classic search crawler; its data, however, also feeds AI Overviews and AI Mode. For training the Gemini models, Google offers a separate robots.txt switch called Google-Extended, which does not affect Google Search.\"}}, {\"@type\": \"Question\", \"name\": \"Do AI crawlers respect robots.txt?\", \"acceptedAnswer\": {\"@type\": \"Answer\", \"text\": \"Major providers such as OpenAI, Anthropic, and Google document their crawlers and state that they respect robots.txt. The file is not a technical barrier, though: rogue bots ignore it. If you want to protect content strictly, you need server-side blocks.\"}}, {\"@type\": \"Question\", \"name\": \"How do I recognize AI crawlers in my log files?\", \"acceptedAnswer\": {\"@type\": \"Answer\", \"text\": \"By the user agent: identifiers such as GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, or PerplexityBot appear in the user agent string of every request. The providers also publish IP lists that let you distinguish real crawlers from impostors.\"}}]}]}","footnotes":""},"categories":[80],"tags":[92],"class_list":["post-7765","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-what-is","tag-k-en"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v28.1 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>What Is an AI Crawler? GPTBot and Co Explained | taismo<\/title>\n<meta name=\"description\" content=\"AI crawlers explained: what GPTBot, OAI-SearchBot, and ClaudeBot do, how to control them via robots.txt, and what a wrong block costs you. \ud83e\udd16\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"What Is an AI Crawler? GPTBot and Co Explained | taismo\" \/>\n<meta property=\"og:description\" content=\"AI crawlers explained: what GPTBot, OAI-SearchBot, and ClaudeBot do, how to control them via robots.txt, and what a wrong block costs you. \ud83e\udd16\" \/>\n<meta property=\"og:url\" content=\"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/\" \/>\n<meta property=\"og:site_name\" content=\"taismo| Agentur f\u00fcr SEO &amp; Webdesign\" \/>\n<meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/TaismoOnlineMarketing\" \/>\n<meta property=\"article:published_time\" content=\"2026-08-21T05:30:00+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/08\/ki-crawler-gptbot-oai-searchbot-steuern.webp\" \/>\n\t<meta property=\"og:image:width\" content=\"1200\" \/>\n\t<meta property=\"og:image:height\" content=\"675\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/webp\" \/>\n<meta name=\"author\" content=\"Dominik Breitbach\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/\"},\"author\":{\"name\":\"Dominik Breitbach\",\"@id\":\"https:\\\/\\\/taismo.de\\\/#person-dominik-breitbach\"},\"headline\":\"AI crawler\",\"datePublished\":\"2026-08-21T05:30:00+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/\"},\"wordCount\":795,\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/08\\\/ki-crawler-gptbot-oai-searchbot-steuern.webp\",\"keywords\":[\"K\"],\"articleSection\":[\"SEO Glossary\"],\"inLanguage\":\"en-US\"},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/\",\"name\":\"What Is an AI Crawler? GPTBot and Co Explained | taismo\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/08\\\/ki-crawler-gptbot-oai-searchbot-steuern.webp\",\"datePublished\":\"2026-08-21T05:30:00+00:00\",\"description\":\"AI crawlers explained: what GPTBot, OAI-SearchBot, and ClaudeBot do, how to control them via robots.txt, and what a wrong block costs you. \ud83e\udd16\",\"breadcrumb\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/#primaryimage\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/08\\\/ki-crawler-gptbot-oai-searchbot-steuern.webp\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/08\\\/ki-crawler-gptbot-oai-searchbot-steuern.webp\",\"width\":1200,\"height\":675,\"caption\":\"KI-Crawler: Wer deine Inhalte f\u00fcr KI-Systeme einsammelt\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/ai-crawler\\\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Startseite\",\"item\":\"https:\\\/\\\/taismo.de\\\/en\\\/search-marketing\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"AI crawler\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"name\":\"taismo| Agentur f\u00fcr SEO & Webdesign\",\"description\":\"\",\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"alternateName\":\"taismo\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/taismo.de\\\/en\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Organization\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\",\"name\":\"taismo SEO Agentur M\u00fcnchen\",\"alternateName\":\"taismo\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"logo\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"width\":512,\"height\":512,\"caption\":\"taismo SEO Agentur M\u00fcnchen\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\"},\"sameAs\":[\"https:\\\/\\\/www.facebook.com\\\/TaismoOnlineMarketing\",\"https:\\\/\\\/branchenbuch.portal.muenchen.de\\\/mhp\\\/1380360\\\/\",\"https:\\\/\\\/www.crunchbase.com\\\/organization\\\/taismo-gmbh\",\"https:\\\/\\\/de.trustpilot.com\\\/review\\\/taismo.de\",\"https:\\\/\\\/www.linkedin.com\\\/company\\\/taismo\\\/\",\"https:\\\/\\\/www.instagram.com\\\/taismo.de\",\"https:\\\/\\\/onlinemarketing.de\\\/dienstleister\\\/online-marketing-agentur\\\/kirchheim-bei-muenchen\\\/taismo\",\"https:\\\/\\\/www.agenturtipp.de\\\/agentur\\\/taismo-online-marketing\\\/\",\"https:\\\/\\\/startupvalley.news\\\/de\\\/taismo-online-marketing-agentur\\\/\",\"https:\\\/\\\/www.wlw.de\\\/de\\\/firma\\\/taismo-gmbh-1918135\",\"https:\\\/\\\/www.youtube.com\\\/@taismo8632\",\"https:\\\/\\\/techbehemoths.com\\\/company\\\/taismo-gmbh\",\"https:\\\/\\\/www.kununu.com\\\/de\\\/taismo1\",\"https:\\\/\\\/www.provenexpert.com\\\/taismo1\\\/\",\"https:\\\/\\\/www.provenemployer.com\\\/p\\\/taismo-employer\\\/\",\"https:\\\/\\\/www.trustedshops.de\\\/bewertung\\\/taismo-de\",\"https:\\\/\\\/www.kennstdueinen.de\\\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html\",\"https:\\\/\\\/www.ibusiness.de\\\/dienstleister\\\/jb\\\/5920834fb726.html\",\"https:\\\/\\\/www.seo-united.de\\\/seo-agenturen\\\/taismo-3345\\\/\"]}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"What Is an AI Crawler? GPTBot and Co Explained | taismo","description":"AI crawlers explained: what GPTBot, OAI-SearchBot, and ClaudeBot do, how to control them via robots.txt, and what a wrong block costs you. \ud83e\udd16","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/","og_locale":"en_US","og_type":"article","og_title":"What Is an AI Crawler? GPTBot and Co Explained | taismo","og_description":"AI crawlers explained: what GPTBot, OAI-SearchBot, and ClaudeBot do, how to control them via robots.txt, and what a wrong block costs you. \ud83e\udd16","og_url":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/","og_site_name":"taismo| Agentur f\u00fcr SEO &amp; Webdesign","article_publisher":"https:\/\/www.facebook.com\/TaismoOnlineMarketing","article_published_time":"2026-08-21T05:30:00+00:00","og_image":[{"width":1200,"height":675,"url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/08\/ki-crawler-gptbot-oai-searchbot-steuern.webp","type":"image\/webp"}],"author":"Dominik Breitbach","twitter_card":"summary_large_image","schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#article","isPartOf":{"@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/"},"author":{"name":"Dominik Breitbach","@id":"https:\/\/taismo.de\/#person-dominik-breitbach"},"headline":"AI crawler","datePublished":"2026-08-21T05:30:00+00:00","mainEntityOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/"},"wordCount":795,"publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/08\/ki-crawler-gptbot-oai-searchbot-steuern.webp","keywords":["K"],"articleSection":["SEO Glossary"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/","url":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/","name":"What Is an AI Crawler? GPTBot and Co Explained | taismo","isPartOf":{"@id":"https:\/\/taismo.de\/en\/#website"},"primaryImageOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#primaryimage"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/08\/ki-crawler-gptbot-oai-searchbot-steuern.webp","datePublished":"2026-08-21T05:30:00+00:00","description":"AI crawlers explained: what GPTBot, OAI-SearchBot, and ClaudeBot do, how to control them via robots.txt, and what a wrong block costs you. \ud83e\udd16","breadcrumb":{"@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/taismo.de\/en\/what-is\/ai-crawler\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#primaryimage","url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/08\/ki-crawler-gptbot-oai-searchbot-steuern.webp","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/08\/ki-crawler-gptbot-oai-searchbot-steuern.webp","width":1200,"height":675,"caption":"KI-Crawler: Wer deine Inhalte f\u00fcr KI-Systeme einsammelt"},{"@type":"BreadcrumbList","@id":"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Startseite","item":"https:\/\/taismo.de\/en\/search-marketing\/"},{"@type":"ListItem","position":2,"name":"AI crawler"}]},{"@type":"WebSite","@id":"https:\/\/taismo.de\/en\/#website","url":"https:\/\/taismo.de\/en\/","name":"taismo| Agentur f\u00fcr SEO & Webdesign","description":"","publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"alternateName":"taismo","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/taismo.de\/en\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/taismo.de\/en\/#organization","name":"taismo SEO Agentur M\u00fcnchen","alternateName":"taismo","url":"https:\/\/taismo.de\/en\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/","url":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","width":512,"height":512,"caption":"taismo SEO Agentur M\u00fcnchen"},"image":{"@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/TaismoOnlineMarketing","https:\/\/branchenbuch.portal.muenchen.de\/mhp\/1380360\/","https:\/\/www.crunchbase.com\/organization\/taismo-gmbh","https:\/\/de.trustpilot.com\/review\/taismo.de","https:\/\/www.linkedin.com\/company\/taismo\/","https:\/\/www.instagram.com\/taismo.de","https:\/\/onlinemarketing.de\/dienstleister\/online-marketing-agentur\/kirchheim-bei-muenchen\/taismo","https:\/\/www.agenturtipp.de\/agentur\/taismo-online-marketing\/","https:\/\/startupvalley.news\/de\/taismo-online-marketing-agentur\/","https:\/\/www.wlw.de\/de\/firma\/taismo-gmbh-1918135","https:\/\/www.youtube.com\/@taismo8632","https:\/\/techbehemoths.com\/company\/taismo-gmbh","https:\/\/www.kununu.com\/de\/taismo1","https:\/\/www.provenexpert.com\/taismo1\/","https:\/\/www.provenemployer.com\/p\/taismo-employer\/","https:\/\/www.trustedshops.de\/bewertung\/taismo-de","https:\/\/www.kennstdueinen.de\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html","https:\/\/www.ibusiness.de\/dienstleister\/jb\/5920834fb726.html","https:\/\/www.seo-united.de\/seo-agenturen\/taismo-3345\/"]}]}},"_links":{"self":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/7765","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/comments?post=7765"}],"version-history":[{"count":1,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/7765\/revisions"}],"predecessor-version":[{"id":7766,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/7765\/revisions\/7766"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media\/7703"}],"wp:attachment":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media?parent=7765"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/categories?post=7765"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/tags?post=7765"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}