{"id":8210,"date":"2026-10-27T07:30:00","date_gmt":"2026-10-27T05:30:00","guid":{"rendered":"https:\/\/taismo.de\/?p=8210"},"modified":"2026-10-12T12:06:47","modified_gmt":"2026-10-12T10:06:47","slug":"user-agent","status":"publish","type":"post","link":"https:\/\/taismo.de\/en\/what-is\/user-agent\/","title":{"rendered":"user agent"},"content":{"rendered":"<style>\n.taismo-art{display:grid;grid-template-columns:320px minmax(0,1fr);gap:36px;align-items:start}\n.taismo-art .toc-sticky{position:sticky;top:80px}\n.taismo-art .toc-sticky ol li{margin-bottom:9px}\n.taismo-art .toc-sticky ol a:hover{color:var(--toc-hover) !important}\n.taismo-art .taismo-main h2{scroll-margin-top:90px;margin-top:38px}\n.taismo-art .lead-def{font-size:1.15em;line-height:1.6;background:#f7f2ec;border-left:4px solid #ED8924;padding:16px 20px;border-radius:8px}\n.taismo-art table{border-collapse:collapse;width:100%;margin:18px 0}\n.taismo-art th,.taismo-art td{border:1px solid #d9e0df;padding:9px 12px;text-align:left;font-size:0.95em}\n.taismo-art th{background:#072328;color:#fff}\n.taismo-art figure{margin:26px 0}\n.taismo-art figcaption{font-size:0.9em;color:#55605f;margin-top:8px}\n.taismo-art pre{background:#072328;color:#f5f5f5;padding:16px 18px;border-radius:10px;overflow-x:auto;font-size:0.86em;line-height:1.5}\n@media(max-width:900px){.taismo-art{grid-template-columns:1fr}.taismo-art .toc-sticky{position:static}}\n<\/style>\n<div class=\"taismo-art\">\n<aside class=\"toc-sticky\" style=\"--toc-hover:#ED8924;background:#072328;border-radius:14px;padding:22px 24px\">\n<div style=\"text-align:center;margin-bottom:14px\"><svg fill=\"#ffffff\" role=\"img\" aria-label=\"taismo Wiki\" style=\"width:200px;max-width:80%;height:auto;display:inline-block\" id=\"Ebene_1\" data-name=\"Ebene 1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 209.04 52.32\"> <g id=\"Layer_2\" data-name=\"Layer 2\"> <g id=\"wikipedia\"> <path d=\"M97.08,27.77h-17.76c-.77,0-1.4.63-1.4,1.4v17.76c0,.77.63,1.4,1.4,1.4h17.76c.77,0,1.4-.63,1.4-1.4v-17.76c0-.77-.63-1.4-1.4-1.4ZM97.22,33.22c0,.25-.21.46-.46.46h-.67c-.07.03-.13.07-.19.11-.19.18-.33.41-.4.66,0,0,0,.01,0,.02-.03.07-2.9,7.29-3.87,9.67v.02c-.09.17-.19.33-.31.48-.15.2-.38.32-.63.33-.56,0-.84-.63-.93-.84-.28-.62-.79-1.83-1.51-3.57-.55,1.24-1.25,2.8-1.64,3.59h0c-.19.36-.49.83-.97.83-.56,0-.82-.63-.9-.84h0c-.49-1.25-1.23-2.99-1.95-4.68-.79-1.83-2.39-5.73-2.43-5.78h-.72c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h4.13c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.82s.03.09.05.14c.59,1.61,2.36,5.75,3.1,7.47l1.28-2.81c-.15-.36-2.12-4.68-2.2-4.8h-.4c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.98c.25,0,.46.21.46.46v.58c0,.25-.21.46-.46.46h-.1c.06.14.94,2.12.94,2.12,0,0,.93-2.06.95-2.12h-.39c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.81c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.43s-2.06,4.08-2.06,4.08l1.56,3.51s2.9-7.59,2.9-7.6h-.96c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h3.78c.25,0,.46.2.46.46v.58Z\"\/> <\/g> <\/g> <text font-family=\"Arial,Helvetica,sans-serif\" font-size=\"27.79\" font-weight=\"800\" transform=\"translate(99.44 47.64) scale(.99 1)\"><tspan x=\"0\" y=\"0\">IKI<\/tspan><\/text> <g id=\"Gruppe_796\" data-name=\"Gruppe 796\"> <path id=\"Pfad_1293\" data-name=\"Pfad 1293\" d=\"M6.92,16.77h-3.56v-6.56h3.66V3.37h8.04v6.84h5.69v6.56h-5.58v8.97c-.08,1.05.13,2.1.6,3.04.4.6,1.24.9,2.51.9.54,0,1.01,0,1.39-.03.41-.02.82-.07,1.23-.13v6.35c-.75.19-1.52.31-2.3.36-1.02.09-1.95.13-2.79.13-1.41.04-2.81-.17-4.15-.6-1.07-.36-2.02-1.01-2.76-1.86-.76-.95-1.29-2.06-1.54-3.25-.34-1.55-.49-3.14-.46-4.73v-9.14h0Z\"\/> <path id=\"Pfad_1294\" data-name=\"Pfad 1294\" d=\"M44.43,35.91h-7.06l-.44-1.97c-.68.72-1.48,1.34-2.35,1.8-.95.46-2.01.69-3.07.66-.99,0-1.98-.1-2.95-.33-.92-.21-1.78-.63-2.51-1.23-.77-.66-1.37-1.49-1.75-2.43-.48-1.28-.71-2.63-.66-3.99v-1.42c-.04-1.17.2-2.33.71-3.39.44-.87,1.09-1.63,1.89-2.19.82-.56,1.74-.96,2.71-1.18,1.06-.24,2.14-.36,3.23-.36.87,0,1.72.06,2.57.22.72.14,1.41.36,2.08.66v-.93c0-.54-.07-1.07-.22-1.59-.16-.48-.46-.91-.87-1.2-.55-.38-1.18-.63-1.83-.77-1-.2-2.01-.29-3.04-.28-.91,0-1.81.09-2.71.25-.97.16-1.8.32-2.49.46v-6.12c.48-.13.96-.22,1.45-.28.57-.07,1.16-.14,1.78-.22.62-.07,1.22-.13,1.78-.16.57-.04,1.04-.05,1.45-.05,2.1-.05,4.18.19,6.21.71,1.4.37,2.69,1.08,3.74,2.08.92.94,1.57,2.13,1.86,3.42.34,1.54.51,3.1.49,4.68v15.15h.01,0ZM33.71,30.83c1.63.12,3.04-1.1,3.16-2.72v-2.58c-.77-.58-1.72-.87-2.68-.82-.85-.04-1.69.16-2.43.57-.6.38-.9,1.16-.9,2.33v.38c-.07.78.16,1.55.66,2.16.6.51,1.39.76,2.19.69\"\/> <rect id=\"Rechteck_760\" data-name=\"Rechteck 760\" x=\"49.46\" y=\"10.21\" width=\"8.26\" height=\"25.71\"\/> <path d=\"M131.36,22.82v-.42c-.06-1.6.12-3.21.52-4.76.35-1.1,1.23-1.64,2.65-1.64.59-.03,1.16.11,1.66.41.43.29.75.73.9,1.23.2.66.32,1.34.36,2.02.05.81.08,1.72.08,2.74v.42h8.38v-1.01c.04-1.92-.23-3.83-.82-5.66-.48-1.43-1.27-2.73-2.33-3.8-1.02-.99-2.27-1.73-3.64-2.13-1.53-.45-3.12-.67-4.71-.66-1.57-.01-3.12.2-4.62.66-1.36.41-2.6,1.14-3.61,2.13-1.06,1.07-1.86,2.37-2.35,3.8-.61,1.82-.89,3.74-.85,5.66v1.01h8.38Z\"\/> <path d=\"M93,22.82v-2.38c-.06-.88.16-1.76.63-2.51.39-.57,1.04-.89,1.72-.87.26,0,.51.03.77.08.26.06.51.19.71.38.25.24.43.54.52.87.15.51.21,1.03.19,1.56v2.88h8.2v-2.7c-.05-.8.16-1.6.6-2.27.4-.52,1.03-.82,1.69-.79.54,0,1.07.17,1.51.49.45.33.69,1.13.69,2.41v2.86h8.2v-4.99c.03-1.32-.17-2.64-.6-3.89-.34-.96-.9-1.83-1.64-2.54-.69-.63-1.51-1.11-2.41-1.39-.92-.3-1.88-.44-2.84-.44-1.16-.02-2.3.21-3.36.69-1.09.55-2.07,1.3-2.87,2.22-.57-.95-1.38-1.72-2.35-2.24-1.16-.49-2.41-.72-3.66-.66-1.18-.04-2.35.22-3.39.76-.99.57-1.88,1.31-2.63,2.19l-.54-2.3h-7.34v12.61h8.2Z\"\/> <path d=\"M73.21,26.15c0-1.84,1.49-3.33,3.33-3.33h2.18c-.87-.85-1.94-1.47-3.12-1.8l-1.75-.6c-.73-.25-1.35-.46-1.86-.63-.43-.13-.85-.3-1.26-.49-.28-.13-.52-.33-.71-.57-.16-.28-.24-.6-.22-.93,0-.22.04-.45.13-.66.13-.25.34-.46.6-.57.43-.2.89-.34,1.36-.41.81-.13,1.62-.18,2.43-.16.92,0,1.85.07,2.76.22.82.11,1.63.28,2.43.49v-6.12c-.48-.13-.95-.22-1.45-.28-.56-.07-1.16-.14-1.78-.22-.61-.07-1.21-.13-1.78-.16s-1.05-.05-1.45-.05c-3.72,0-6.51.75-8.37,2.27-1.78,1.35-2.82,3.48-2.79,5.71-.03,1.07.11,2.13.41,3.15.24.76.64,1.46,1.18,2.05.52.56,1.15,1.01,1.83,1.34.79.38,1.6.72,2.43,1.01l1.26.44c.83.28,1.63.61,2.41,1.01.47.28.75.8.71,1.34.07.72-.35,1.4-1.04,1.64-1.23.31-2.51.45-3.77.38-.95,0-1.9-.09-2.84-.25-1.06-.16-1.93-.31-2.63-.46v6.12c.5.13,1,.22,1.51.28.6.07,1.23.14,1.89.22.66.07,1.3.13,1.92.16.62.03,1.13.05,1.53.05,1.5.03,3.01-.1,4.49-.36v-9.83Z\"\/> <\/g> <g id=\"e\"> <path d=\"M150.81,38.2c-.26-1.16-1.38-2.22-2.54-2.45-.71-.14-2.69-.32-3.99-.3-1.31-.01-3.28.16-3.99.3-1.16.23-2.29,1.29-2.54,2.45-.15.7-.29,1.66-.29,2.97,0,1.31.15,2.27.29,2.97.26,1.16,1.38,2.22,2.54,2.45.38.08,1.14.16,1.95.22.09.15.19.3.3.46.43.66.77,1.11,1.02,1.43.42.53,1.03.53,1.45,0,.25-.32.58-.78,1.02-1.43.11-.16.21-.32.3-.46.82-.06,1.57-.15,1.95-.22,1.16-.23,2.29-1.29,2.54-2.45.15-.7.29-1.66.29-2.97,0-1.31-.15-2.27-.29-2.97h0ZM145.03,45.13c-.03.16-.18.31-.34.34-.1.02-.23.03-.42.03-.18,0-.32-.02-.42-.03-.16-.03-.31-.18-.34-.34-.02-.1-.03-.23-.03-.42,0-.18.02-.32.03-.42.03-.16.18-.31.34-.34.1-.02.23-.03.42-.03.18,0,.32.02.42.03.16.03.31.18.34.34.02.1.03.23.03.42,0,.18-.02.32-.03.42ZM146.65,40.76c-.21.29-.51.56-.97.84-.58.36-.67.5-.67.79v.11c0,.41-.33.74-.74.74s-.74-.33-.74-.74v-.26c0-.34.08-.63.26-.88.17-.24.42-.46.8-.68.69-.41.82-.63.82-.94,0-.38-.32-.63-.83-.63-.55,0-.91.25-1.06.74-.1.32-.4.54-.75.54-.02,0-.04,0-.06,0-.24-.02-.46-.14-.6-.35-.14-.2-.17-.45-.1-.68.17-.52.49-.96.93-1.26.45-.3,1.01-.47,1.63-.47s1.24.21,1.69.58c.45.38.71.91.71,1.49,0,.4-.1.74-.32,1.04h0Z\"\/> <\/g> <g id=\"e-2\" data-name=\"e\"> <path d=\"M156.19,28.79c-.17-.75-.89-1.44-1.64-1.58-.46-.09-1.73-.2-2.58-.2-.84,0-2.12.1-2.58.2-.75.15-1.48.83-1.64,1.58-.09.45-.19,1.07-.19,1.92,0,.84.09,1.46.19,1.92.17.75.89,1.44,1.64,1.58.25.05.73.11,1.26.14.06.09.12.19.19.3.28.42.49.72.66.93.27.34.66.34.93,0,.16-.21.38-.5.66-.93.07-.11.13-.21.19-.3.53-.04,1.01-.09,1.26-.14.75-.15,1.48-.83,1.64-1.58.09-.45.19-1.07.19-1.92,0-.84-.09-1.46-.19-1.92h0ZM152.46,33.27c-.02.1-.11.2-.22.22-.06.01-.15.02-.27.02-.12,0-.2,0-.27-.02-.1-.02-.2-.11-.22-.22-.01-.06-.02-.15-.02-.27,0-.12,0-.2.02-.27.02-.1.11-.2.22-.22.06-.01.15-.02.27-.02.12,0,.2,0,.27.02.1.02.2.11.22.22.01.06.02.15.02.27,0,.12,0,.2-.02.27ZM153.51,30.45c-.13.19-.33.36-.63.54-.37.23-.43.32-.43.51v.07c0,.26-.21.48-.48.48s-.48-.21-.48-.48v-.17c0-.22.05-.4.17-.57.11-.16.27-.3.51-.44.45-.26.53-.41.53-.61,0-.25-.21-.4-.53-.4-.35,0-.59.16-.69.48-.07.21-.26.35-.48.35-.01,0-.02,0-.04,0-.16-.01-.3-.09-.39-.22-.09-.13-.11-.29-.06-.44.11-.34.32-.62.6-.81.29-.2.65-.3,1.05-.3s.8.13,1.09.37c.29.25.46.59.46.96,0,.26-.07.48-.21.67h0Z\"\/> <\/g> <g id=\"e-3\" data-name=\"e\"> <path d=\"M144.91,28.64c-.1-.44-.52-.84-.96-.92-.27-.05-1.01-.12-1.5-.11-.49,0-1.24.06-1.5.11-.44.08-.86.49-.96.92-.06.27-.11.63-.11,1.12,0,.49.05.85.11,1.12.1.44.52.84.96.92.14.03.43.06.74.08.03.05.07.11.11.17.16.25.29.42.38.54.16.2.39.2.54,0,.09-.12.22-.29.38-.54.04-.06.08-.12.11-.17.31-.02.59-.06.74-.08.44-.08.86-.49.96-.92.06-.27.11-.63.11-1.12,0-.49-.05-.85-.11-1.12h0ZM142.73,31.26c-.01.06-.07.12-.13.13-.04,0-.09.01-.16.01-.07,0-.12,0-.16-.01-.06-.01-.12-.07-.13-.13,0-.04-.01-.09-.01-.16,0-.07,0-.12.01-.16.01-.06.07-.12.13-.13.04,0,.09-.01.16-.01.07,0,.12,0,.16.01.06.01.12.07.13.13,0,.04.01.09.01.16,0,.07,0,.12-.01.16ZM143.34,29.61c-.08.11-.19.21-.37.32-.22.13-.25.19-.25.3v.04c0,.15-.12.28-.28.28s-.28-.12-.28-.28v-.1c0-.13.03-.24.1-.33.06-.09.16-.17.3-.26.26-.15.31-.24.31-.35,0-.14-.12-.24-.31-.24-.21,0-.34.09-.4.28-.04.12-.15.2-.28.2,0,0-.01,0-.02,0-.09,0-.17-.05-.23-.13-.05-.08-.06-.17-.04-.25.06-.2.18-.36.35-.47.17-.11.38-.18.61-.18s.47.08.63.22c.17.14.27.34.27.56,0,.15-.04.28-.12.39h0Z\"\/> <\/g> <\/svg><\/div>\n<h3 style=\"text-align:center;color:#fff;margin:0 0 14px;font-size:1.1em\">Contents<\/h3>\n<ol style=\"margin:0;padding-left:1.2em;line-height:1.5;font-weight:600;color:#fff;font-size:0.9em\">\n<li><a href=\"#term-profile\" style=\"color:#fff;text-decoration:none\">Term profile<\/a><\/li>\n<li><a href=\"#structure\" style=\"color:#fff;text-decoration:none\">How is a user agent string structured?<\/a><\/li>\n<li><a href=\"#search-and-ai-crawlers\" style=\"color:#fff;text-decoration:none\">Which user agents do search engines and AI crawlers use?<\/a><\/li>\n<li><a href=\"#robots-txt\" style=\"color:#fff;text-decoration:none\">How do you control user agents in robots.txt?<\/a><\/li>\n<li><a href=\"#verify-crawlers\" style=\"color:#fff;text-decoration:none\">How do you check whether a crawler is genuine?<\/a><\/li>\n<li><a href=\"#seo\" style=\"color:#fff;text-decoration:none\">What role does the user agent play in SEO?<\/a><\/li>\n<li><a href=\"#firewalls\" style=\"color:#fff;text-decoration:none\">Why do firewalls block some user agents?<\/a><\/li>\n<li><a href=\"#faq\" style=\"color:#fff;text-decoration:none\">Frequently asked questions about user agents<\/a><\/li>\n<\/ol>\n<\/aside>\n<div class=\"taismo-main\">\n<h1 class=\"brand-claim\">What is a user agent?<\/h1>\n<p class=\"lead-def\"><strong>A user agent<\/strong> is the program that retrieves content from the web on behalf of a user, such as a browser, an app, or a crawler. In the narrower sense, user agent refers to the identifier this program sends to the web server with every request in the HTTP header <code>User-Agent<\/code>. Web servers use this identifier to recognize who is making the request, and the rules of a robots.txt for search engines and AI crawlers take effect through it.<\/p>\n<figure>\n      <img decoding=\"async\" src=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/10\/user-agent-kennung-browser-crawler.webp\" alt=\"User agent explained: the identifier in the HTTP header that lets web servers recognize browsers, Googlebot, and AI crawlers like GPTBot, taismo SEO glossary\" title=\"User agent: the identifier of browsers, search engines, and AI crawlers\" width=\"1200\" height=\"675\" loading=\"lazy\" style=\"width:100%;height:auto;border-radius:12px\" \/><br \/>\n    <\/figure>\n<h2 id=\"term-profile\">Term profile<\/h2>\n<table>\n<tr>\n<th>Attribute<\/th>\n<th>Details<\/th>\n<\/tr>\n<tr>\n<td>Spellings<\/td>\n<td>user agent, User-Agent (as an HTTP header), abbreviation UA<\/td>\n<\/tr>\n<tr>\n<td>Technical basis<\/td>\n<td>HTTP request header <code>User-Agent<\/code>, described in RFC 9110<\/td>\n<\/tr>\n<tr>\n<td>Typical user agents<\/td>\n<td>Browsers, search engine crawlers, AI crawlers, apps, scripts<\/td>\n<\/tr>\n<tr>\n<td>Reliability<\/td>\n<td>freely chosen and therefore easy to fake<\/td>\n<\/tr>\n<tr>\n<td>In robots.txt<\/td>\n<td>The line <code>User-agent:<\/code> with the short name (token) of a crawler<\/td>\n<\/tr>\n<tr>\n<td>Supplement in the browser<\/td>\n<td>User-Agent Client Hints, such as the header <code>Sec-CH-UA<\/code><\/td>\n<\/tr>\n<\/table>\n<h2 id=\"structure\">How is a user agent string structured?<\/h2>\n<p><strong>A user agent string consists of product names with version numbers, supplemented by details in parentheses about the operating system, device, or purpose.<\/strong> Two examples, a browser and Google&#8217;s crawler for smartphones:<\/p>\n<pre><code>Mozilla\/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit\/537.36\n  (KHTML, like Gecko) Chrome\/150.0.0.0 Safari\/537.36\n\nMozilla\/5.0 (Linux; Android 6.0.1; Nexus 5X Build\/MMB29P) AppleWebKit\/537.36\n  (KHTML, like Gecko) Chrome\/W.X.Y.Z Mobile Safari\/537.36\n  (compatible; Googlebot\/2.1; +http:\/\/www.google.com\/bot.html)<\/code><\/pre>\n<p>The <strong>Mozilla\/5.0<\/strong> at the beginning is a historical leftover. In the 1990s, browsers identified themselves as Mozilla so that web servers would serve them the modern versions of pages, and almost every user agent keeps that opening to this day. Reputable crawlers append their name and a URL with an explanation, in the example <code>Googlebot\/2.1<\/code> and the link to Google&#8217;s documentation.<\/p>\n<p>Since 2022, Chrome has been gradually shortening the user agent string and now provides details such as the exact version or the device model only on request via User-Agent Client Hints. The string is therefore less informative and offers fewer features for recognizing individual devices.<\/p>\n<h2 id=\"search-and-ai-crawlers\">Which user agents do search engines and AI crawlers use?<\/h2>\n<p><strong>Every reputable <a href=\"https:\/\/taismo.de\/en\/what-is\/crawler\/\">crawler<\/a> identifies itself with its own name, and the major providers operate several user agents depending on the purpose.<\/strong> The most important identifiers:<\/p>\n<table>\n<tr>\n<th>Provider<\/th>\n<th>Token<\/th>\n<th>Purpose<\/th>\n<\/tr>\n<tr>\n<td>Google<\/td>\n<td>Googlebot<\/td>\n<td>Crawling for Google Search, in a smartphone and a desktop version<\/td>\n<\/tr>\n<tr>\n<td>Google<\/td>\n<td>Google-Extended<\/td>\n<td>only a token in robots.txt for the use of content in Gemini models, without its own user agent string<\/td>\n<\/tr>\n<tr>\n<td>Microsoft<\/td>\n<td>bingbot<\/td>\n<td>Crawling for Bing and Copilot<\/td>\n<\/tr>\n<tr>\n<td>OpenAI<\/td>\n<td>GPTBot<\/td>\n<td>Collecting training data<\/td>\n<\/tr>\n<tr>\n<td>OpenAI<\/td>\n<td>OAI-SearchBot<\/td>\n<td>Index for search in ChatGPT<\/td>\n<\/tr>\n<tr>\n<td>OpenAI<\/td>\n<td>ChatGPT-User<\/td>\n<td>Fetching a page when a user asks for it in ChatGPT<\/td>\n<\/tr>\n<tr>\n<td>Anthropic<\/td>\n<td>ClaudeBot<\/td>\n<td>Collecting training data for Claude<\/td>\n<\/tr>\n<tr>\n<td>Perplexity<\/td>\n<td>PerplexityBot<\/td>\n<td>Index for Perplexity&#8217;s answers<\/td>\n<\/tr>\n<tr>\n<td>Common Crawl<\/td>\n<td>CCBot<\/td>\n<td>Open web archive, the basis of many training datasets<\/td>\n<\/tr>\n<\/table>\n<p>The table shows why the distinction matters: One and the same provider can collect training data with one user agent and supply answers with sources through another. If you block an <a href=\"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/\">AI crawler<\/a>, you therefore decide per token whether your website is missing from training, from search, or from both. The entry on <a href=\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/\">Common Crawl<\/a> explains what is behind the CCBot archive.<\/p>\n<figure>\n      <a href=\"https:\/\/taismo.de\/en\/geo-audit\/\" title=\"AEO and GEO audit by taismo: do AI crawlers reach your website?\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 780 340\" role=\"img\" aria-labelledby=\"abb1t abb1d\" style=\"width:100%;height:auto;max-width:780px;font-family:Arial,Helvetica,sans-serif\"><title id=\"abb1t\">Three checkpoints for a request with a user agent<\/title><desc id=\"abb1d\">On the left are three senders: a browser, Googlebot, and GPTBot. Each request carries a User-Agent header and passes three checkpoints: first the firewall or CDN, which can block suspicious identifiers, then the robots.txt, whose rules apply per token and which only reputable crawlers follow, and finally the check of the IP address, because any user agent can be faked. Only then does the web server deliver the page.<\/desc><rect x=\"0\" y=\"0\" width=\"780\" height=\"340\" fill=\"#ffffff\"\/><g opacity=\"0.08\" transform=\"translate(22,296) scale(0.04)\"><path fill=\"#072328\" d=\"M1122.08,481.04l-132.61-85.2h-.19c-2.19-1.43-3.97-3.41-5.16-5.74-2.21-4.37-3.05-9.3-2.42-14.16.76-8.35,4.11-36.09,7.27-61.66,1.56-12.75,3.19-25.06,4.24-34.12s1.88-14.92,1.88-14.95c.16-1.49-.91-2.83-2.4-2.99-.82-.09-1.64.2-2.23.79l-125.88,131.4-8.04,8.39-19.55,17.03c-15.82,12.94-32.67,24.58-50.38,34.79-13.56,10.1-30.33,14.95-47.19,13.65-32.06-1.03-63.83-6.46-94.41-16.13-59.13-27.3-123.67-40.89-188.79-39.76l-154,13.68s-38.52,7.46-47.22-19.9c-17.4-52.51-40.3-103.03-68.33-150.72-18.96-28.44-28.95-61.92-28.7-96.1,0,0,6.22-116.55,68.06-133.32,0,0,53.12-19.63,96.05,36.25,0,0,25.03,35.71,80.37,21.79,0,0,111.05-30.87,214.33,25.55,0,0,84.14,40.28,141.09,24.21,0,0,29.25-6.19,42.36-27.71,0,0,17.4-27.71,50.17-18.85,0,0,32.5,6.19,32.23,52.05,0,0,1.61,54.47-38.13,89.39,0,0-14.45,16.6-1.34,32.13l68.6,79.57s10.16,12.04,26.11,12.31l160.06.54s6.26-.27,6.62,3.05c.36,3.32-2.87,4.21-2.87,4.21l-52.7,18.4s-5.72,1.61-5.9,6.01c-.18,4.39,1.79,5.83,3.94,6.9l70.75,33.29s3.22,1.61,2.68,4.75c-.54,3.14-4.66,3.14-4.66,3.14l-116.29,14.83s-12.13,1.7-15.97,10.83c-3.85,9.13-1.7,15.61-1.7,15.61l63.5,142.42s1.79,4.75-1.34,6.19-5.19-.98-5.19-.98l-113.7-128.04s-6.44-8.5-16.06-6.63c-9.62,1.88-11.14,11.55-11.14,11.55l-38.94,168.6s-1.16,5.46-4.66,5.46-4.3-4.03-4.3-4.03l-30.71-165.02s-1.61-11.1-14.72-11.64c-13.11-.54-16.24,9.67-16.24,9.67l-53.06,132.61s-2.06,5.19-5.55,4.12c-3.5-1.07-2.15-6.72-2.15-6.72l16.6-166.36s1.7-12.22-10.34-14.99c-12.04-2.78-17.32,7.17-17.32,7.17l-84.5,126.25s-3.32,4.48-6.72,2.33.36-6.99.36-6.99l53.87-152.24s3.14-10.61-6.9-15.44c-10.03-4.84-16.6,3.14-16.6,3.14l-120.32,103.15s-4.66,3.5-6.63.36c-1.97-3.14,1.7-6.63,1.7-6.63l94.85-136.73s5.28-8.86-3.14-15.71c-8.41-6.85-15.62-1.16-15.62-1.16l-145.64,72.71s-5.1,2.15-6.45-.63c-1.34-2.78,2.51-5.46,2.51-5.46l129.03-107.63s7.35-6.63.72-14.9c-6.63-8.28-15.17-4.66-15.17-4.66l-160.6,44.29s-5.37.81-6.09-2.24,3.14-4.84,3.14-4.84l154.29-79.16s8.15-4.03,4.21-13.29c-3.94-9.27-14.19-8.19-14.19-8.19l-166.45,6.09s-5.19-.36-5.19-3.59,4.48-3.94,4.48-3.94l169.14-42.09s8.86-2.24,7.53-11.91c-1.34-9.67-11.55-11.28-11.55-11.28l-164.03-32.32s-4.84-1.34-4.21-4.48c.63-3.14,5.55-2.51,5.55-2.51l171.29,5.55s9.98.36,11.28-9.4c1.3-9.76-7.98-13.65-7.98-13.65\"\/><\/g><text x=\"390\" y=\"28\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"15\" font-weight=\"bold\">The path of a request: where the user agent counts<\/text><rect x=\"20\" y=\"56\" width=\"150\" height=\"50\" rx=\"10\" fill=\"#ffffff\" stroke=\"#072328\" stroke-width=\"2\"\/><text x=\"95\" y=\"78\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"13\" font-weight=\"bold\">Browser<\/text><text x=\"95\" y=\"95\" text-anchor=\"middle\" fill=\"#55605f\" font-size=\"10\">Chrome\/150.0.0.0<\/text><rect x=\"20\" y=\"126\" width=\"150\" height=\"50\" rx=\"10\" fill=\"#ffffff\" stroke=\"#072328\" stroke-width=\"2\"\/><text x=\"95\" y=\"148\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"13\" font-weight=\"bold\">Googlebot<\/text><text x=\"95\" y=\"165\" text-anchor=\"middle\" fill=\"#55605f\" font-size=\"10\">Googlebot\/2.1<\/text><rect x=\"20\" y=\"196\" width=\"150\" height=\"50\" rx=\"10\" fill=\"#ffffff\" stroke=\"#072328\" stroke-width=\"2\"\/><text x=\"95\" y=\"218\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"13\" font-weight=\"bold\">AI crawlers<\/text><text x=\"95\" y=\"235\" text-anchor=\"middle\" fill=\"#55605f\" font-size=\"10\">GPTBot, ClaudeBot<\/text><path d=\"M170 81 L196 151\" stroke=\"#ED8924\" stroke-width=\"2\"\/><path d=\"M170 151 L196 151\" stroke=\"#ED8924\" stroke-width=\"2\"\/><path d=\"M170 221 L196 151\" stroke=\"#ED8924\" stroke-width=\"2\"\/><path d=\"M196 151 L206 151\" stroke=\"#ED8924\" stroke-width=\"2.5\"\/><path d=\"M214 151 L203 145 L203 157 Z\" fill=\"#ED8924\"\/><rect x=\"216\" y=\"96\" width=\"150\" height=\"110\" rx=\"12\" fill=\"#072328\"\/><text x=\"291\" y=\"124\" text-anchor=\"middle\" fill=\"#ED8924\" font-size=\"12\" font-weight=\"bold\">1 \u00b7 Firewall \/ CDN<\/text><text x=\"291\" y=\"148\" text-anchor=\"middle\" fill=\"#ffffff\" font-size=\"11\">blocks suspicious<\/text><text x=\"291\" y=\"164\" text-anchor=\"middle\" fill=\"#ffffff\" font-size=\"11\">identifiers, often also<\/text><text x=\"291\" y=\"180\" text-anchor=\"middle\" fill=\"#ffffff\" font-size=\"11\">wanted crawlers<\/text><path d=\"M366 151 L380 151\" stroke=\"#ED8924\" stroke-width=\"2.5\"\/><path d=\"M388 151 L377 145 L377 157 Z\" fill=\"#ED8924\"\/><rect x=\"390\" y=\"96\" width=\"150\" height=\"110\" rx=\"12\" fill=\"#072328\"\/><text x=\"465\" y=\"124\" text-anchor=\"middle\" fill=\"#ED8924\" font-size=\"12\" font-weight=\"bold\">2 \u00b7 robots.txt<\/text><text x=\"465\" y=\"148\" text-anchor=\"middle\" fill=\"#ffffff\" font-size=\"11\">rules per token,<\/text><text x=\"465\" y=\"164\" text-anchor=\"middle\" fill=\"#ffffff\" font-size=\"11\">followed only by<\/text><text x=\"465\" y=\"180\" text-anchor=\"middle\" fill=\"#ffffff\" font-size=\"11\">reputable crawlers<\/text><path d=\"M540 151 L554 151\" stroke=\"#ED8924\" stroke-width=\"2.5\"\/><path d=\"M562 151 L551 145 L551 157 Z\" fill=\"#ED8924\"\/><rect x=\"564\" y=\"96\" width=\"196\" height=\"110\" rx=\"12\" fill=\"#ED8924\"\/><text x=\"662\" y=\"124\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"12\" font-weight=\"bold\">3 \u00b7 Origin check<\/text><text x=\"662\" y=\"148\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"11\">IP address against DNS and<\/text><text x=\"662\" y=\"164\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"11\">published lists, because<\/text><text x=\"662\" y=\"180\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"11\">the user agent can be faked<\/text><rect x=\"216\" y=\"236\" width=\"544\" height=\"44\" rx=\"10\" fill=\"#fdf1e3\"\/><text x=\"488\" y=\"263\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"13\" font-weight=\"bold\">The web server delivers the page, for Googlebot usually the smartphone version<\/text><path d=\"M662 206 L662 230\" stroke=\"#072328\" stroke-width=\"2\"\/><path d=\"M662 236 L656 226 L668 226 Z\" fill=\"#072328\"\/><text x=\"764\" y=\"332\" text-anchor=\"end\" fill=\"#9aa8a7\" font-size=\"11\">Fig. 1 \u00b7 taismo<\/text><\/svg><\/a><figcaption><strong>Fig. 1: The path of a request: Firewall, robots.txt, and the origin check decide whether a user agent receives the page.<\/strong><\/figcaption><\/figure>\n<h2 id=\"robots-txt\">How do you control user agents in robots.txt?<\/h2>\n<p><strong>In <a href=\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/\">robots.txt<\/a>, each group of rules sits under a <code>User-agent:<\/code> line with the token of the crawler it applies to.<\/strong> An example that locks out training crawlers and allows search crawlers:<\/p>\n<pre><code>User-agent: GPTBot\nDisallow: \/\n\nUser-agent: Google-Extended\nDisallow: \/\n\nUser-agent: OAI-SearchBot\nAllow: \/\n\nUser-agent: *\nAllow: \/<\/code><\/pre>\n<p>3 rules apply to the evaluation. A crawler follows the group whose token matches it most specifically and then ignores the group with the asterisk. Upper and lower case in the token do not matter. The robots.txt is a request that reputable crawlers follow. Technically, it stops no one. For individual pages or file types, the <a href=\"https:\/\/taismo.de\/en\/what-is\/x-robots-tag\/\">X-Robots-Tag<\/a> also controls indexing. How to specifically allow or block individual AI crawlers is shown in the magazine article <a href=\"https:\/\/taismo.de\/en\/seo-magazine\/gptbot\/\">Controlling AI crawlers: GPTBot, ClaudeBot, and the others<\/a>.<\/p>\n<h2 id=\"verify-crawlers\">How do you check whether a crawler is genuine?<\/h2>\n<p><strong>Whether a crawler is genuine is shown by the IP address of the request, because any program can set the user agent string freely.<\/strong> Google explicitly warns in its documentation that the user agent can be spoofed. The check works in 3 steps:<\/p>\n<ol>\n<li><strong>Read the log file:<\/strong> The <a href=\"https:\/\/taismo.de\/en\/what-is\/web-server\/\">web server<\/a> logs the user agent and IP address for every request.<\/li>\n<li><strong>Run a reverse lookup:<\/strong> For an IP address claiming to be Googlebot, the reverse DNS lookup returns a name under googlebot.com or google.com. The forward lookup of that name must return the same IP address again.<\/li>\n<li><strong>Compare with published lists:<\/strong> Google and OpenAI publish the IP ranges of their crawlers as JSON files. A request named GPTBot from an IP address outside these ranges comes from someone else.<\/li>\n<\/ol>\n<p>Fake Googlebots are often scrapers that copy content, or tools trying to get around blocks. Firewalls that only look at the user agent let them through.<\/p>\n<h2 id=\"seo\">What role does the user agent play in SEO?<\/h2>\n<p><strong>For SEO, the user agent determines which version of a page Google sees and evaluates.<\/strong> 3 connections matter:<\/p>\n<ul>\n<li><strong>Mobile-first indexing:<\/strong> Google crawls most websites with the smartphone Googlebot. With <a href=\"https:\/\/taismo.de\/en\/what-is\/mobile-first-indexing\/\">mobile-first indexing<\/a>, the version this user agent receives is the one that counts.<\/li>\n<li><strong>Cloaking:<\/strong> If a crawler deliberately receives different content than a human, that violates Google&#8217;s spam policies. <a href=\"https:\/\/taismo.de\/en\/what-is\/cloaking\/\">Cloaking<\/a> via the user agent is the classic form of it.<\/li>\n<li><strong>Rendering:<\/strong> Some websites deliver a pre-rendered HTML version to crawlers and a JavaScript application to humans. Google now calls this dynamic rendering a workaround and recommends server-side rendering. Background is in the entry on <a href=\"https:\/\/taismo.de\/en\/what-is\/javascript-seo\/\">JavaScript SEO<\/a>.<\/li>\n<\/ul>\n<h2 id=\"firewalls\">Why do firewalls block some user agents?<\/h2>\n<p><strong>Firewalls and content delivery networks block user agents they classify as automated or suspicious, and in doing so they also hit requests that are wanted.<\/strong> Many rules block requests without a browser identifier or with the default identifier of programming libraries. taismo measured this with its own data requests: A service behind Cloudflare answered requests without a browser identifier with the <a href=\"https:\/\/taismo.de\/en\/what-is\/http-statuscode\/\">HTTP status code<\/a> 403 and error 1010. The response looked like a missing access right, and with a browser identifier the same request went through immediately.<\/p>\n<p>The same mechanism affects AI crawlers. If a security rule at the host or <a href=\"https:\/\/taismo.de\/en\/what-is\/content-delivery-network-cdn\/\">CDN<\/a> blocks GPTBot, ClaudeBot, or PerplexityBot, the website is missing from their answers even though robots.txt allows everything. Whether AI crawlers actually reach a website is checked by an <a href=\"https:\/\/taismo.de\/en\/geo-audit\/\">AEO audit (also called GEO audit)<\/a> with requests under the real user agents and a look at the log files.<\/p>\n<h2 id=\"faq\">Frequently asked questions about user agents<\/h2>\n<p><strong>Can I change my user agent?<\/strong><br \/>\nYes. The developer tools in Chrome, Firefox, and Safari let you simulate a different user agent, for example to see a page as a smartphone or as Googlebot would.<\/p>\n<p><strong>Is the user agent personal data?<\/strong><br \/>\nOn its own, the user agent usually does not identify a person. Together with the IP address, screen size, and other features, it can become part of a fingerprint that allows devices to be recognized.<\/p>\n<p><strong>What does User-agent: * mean in robots.txt?<\/strong><br \/>\nThe asterisk stands for all crawlers that do not have their own group. If a crawler has its own group, only that group applies to it.<\/p>\n<p><strong>What is the difference between the user agent and the IP address?<\/strong><br \/>\nThe user agent is an identifier the program chooses itself. The IP address is the network address the request comes from, which is why it can be used to check where a request originates.<\/p>\n<\/p><\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>IKI Contents Term profile How is a user agent string structured? Which user agents do search engines and AI crawlers use? How do you control user agents in robots.txt? How do you check whether a crawler is genuine? What role does the user agent play in SEO? Why do firewalls block some user agents? Frequently [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":8206,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_taismo_jsonld":"","_taismo_h1":"","taismo_jsonld":"{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/\",\"url\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/\",\"name\":\"What Is a User Agent? Browsers and Crawlers\",\"isPartOf\":{\"@id\":\"https:\/\/taismo.de\/#website\"},\"inLanguage\":\"en-US\",\"primaryImageOfPage\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/#primaryimage\"},\"mainEntity\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/#definedterm\"},\"breadcrumb\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/#breadcrumb\"},\"datePublished\":\"2026-10-27T07:30:00+02:00\",\"dateModified\":\"2026-10-27T07:30:00+02:00\",\"speakable\":{\"@type\":\"SpeakableSpecification\",\"cssSelector\":[\".brand-claim\",\".lead-def\"]},\"author\":{\"@id\":\"https:\/\/taismo.de\/#person-dominik-breitbach\"},\"publisher\":{\"@id\":\"https:\/\/taismo.de\/#organization\"}},{\"@type\":\"ImageObject\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/#primaryimage\",\"contentUrl\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/10\/user-agent-kennung-browser-crawler.webp\",\"url\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/10\/user-agent-kennung-browser-crawler.webp\",\"width\":1200,\"height\":675,\"caption\":\"User agent: the identifier of browsers, search engines, and AI crawlers\"},{\"@type\":\"DefinedTerm\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/#definedterm\",\"name\":\"user agent\",\"termCode\":\"user-agent\",\"url\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/\",\"inDefinedTermSet\":{\"@id\":\"https:\/\/taismo.de\/#seo-wiki\"},\"description\":\"A user agent is the program that retrieves content from the web on behalf of a user, such as a browser, an app, or a crawler. In the narrower sense, user agent refers to the identifier this program sends to the web server with every request in the HTTP header User-Agent. Web servers use this identifier to recognize who is making the request, and the rules of a robots.txt for search engines and AI crawlers take effect through it.\",\"alternateName\":[\"User-Agent\",\"UA\",\"user agent string\"]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\/\/taismo.de\/en\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"SEO Glossary\",\"item\":\"https:\/\/taismo.de\/en\/what-is\/\"},{\"@type\":\"ListItem\",\"position\":3,\"name\":\"User agent\"}]},{\"@type\":\"FAQPage\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/user-agent\/#faqpage\",\"mainEntity\":[{\"@type\":\"Question\",\"name\":\"Can I change my user agent?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"Yes. The developer tools in Chrome, Firefox, and Safari let you simulate a different user agent, for example to see a page as a smartphone or as Googlebot would.\"}},{\"@type\":\"Question\",\"name\":\"Is the user agent personal data?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"On its own, the user agent usually does not identify a person. Together with the IP address, screen size, and other features, it can become part of a fingerprint that allows devices to be recognized.\"}},{\"@type\":\"Question\",\"name\":\"What does User-agent: * mean in robots.txt?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"The asterisk stands for all crawlers that do not have their own group. If a crawler has its own group, only that group applies to it.\"}},{\"@type\":\"Question\",\"name\":\"What is the difference between the user agent and the IP address?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"The user agent is an identifier the program chooses itself. The IP address is the network address the request comes from, which is why it can be used to check where a request originates.\"}}]}]}","footnotes":""},"categories":[80],"tags":[102],"class_list":["post-8210","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-what-is","tag-u-en"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v28.1 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>What Is a User Agent? Browsers and Crawlers | taismo<\/title>\n<meta name=\"description\" content=\"The user agent reveals who fetches a page: browser, Googlebot, or AI crawler. Structure, robots.txt rules, and how to spot fakes. Read on! \ud83e\udd16\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/taismo.de\/en\/what-is\/user-agent\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"What Is a User Agent? Browsers and Crawlers | taismo\" \/>\n<meta property=\"og:description\" content=\"The user agent reveals who fetches a page: browser, Googlebot, or AI crawler. Structure, robots.txt rules, and how to spot fakes. Read on! \ud83e\udd16\" \/>\n<meta property=\"og:url\" content=\"https:\/\/taismo.de\/en\/what-is\/user-agent\/\" \/>\n<meta property=\"og:site_name\" content=\"taismo| Agentur f\u00fcr SEO &amp; Webdesign\" \/>\n<meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/TaismoOnlineMarketing\" \/>\n<meta property=\"article:published_time\" content=\"2026-10-27T05:30:00+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/10\/user-agent-kennung-browser-crawler.webp\" \/>\n\t<meta property=\"og:image:width\" content=\"1200\" \/>\n\t<meta property=\"og:image:height\" content=\"675\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/webp\" \/>\n<meta name=\"author\" content=\"Dominik Breitbach\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/\"},\"author\":{\"name\":\"Dominik Breitbach\",\"@id\":\"https:\\\/\\\/taismo.de\\\/#person-dominik-breitbach\"},\"headline\":\"user agent\",\"datePublished\":\"2026-10-27T05:30:00+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/\"},\"wordCount\":1435,\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/10\\\/user-agent-kennung-browser-crawler.webp\",\"keywords\":[\"U\"],\"articleSection\":[\"SEO Glossary\"],\"inLanguage\":\"en-US\"},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/\",\"name\":\"What Is a User Agent? Browsers and Crawlers | taismo\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/10\\\/user-agent-kennung-browser-crawler.webp\",\"datePublished\":\"2026-10-27T05:30:00+00:00\",\"description\":\"The user agent reveals who fetches a page: browser, Googlebot, or AI crawler. Structure, robots.txt rules, and how to spot fakes. Read on! \ud83e\udd16\",\"breadcrumb\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/#primaryimage\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/10\\\/user-agent-kennung-browser-crawler.webp\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/10\\\/user-agent-kennung-browser-crawler.webp\",\"width\":1200,\"height\":675,\"caption\":\"User Agent erkl\u00e4rt: die Kennung im HTTP-Header, an der Webserver Browser, Googlebot und KI-Crawler wie GPTBot erkennen, taismo SEO-Wiki\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/user-agent\\\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Startseite\",\"item\":\"https:\\\/\\\/taismo.de\\\/en\\\/search-marketing\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"user agent\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"name\":\"taismo| Agentur f\u00fcr SEO & Webdesign\",\"description\":\"\",\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"alternateName\":\"taismo\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/taismo.de\\\/en\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Organization\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\",\"name\":\"taismo SEO- & GEO-Agentur M\u00fcnchen\",\"alternateName\":\"taismo\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"logo\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"width\":512,\"height\":512,\"caption\":\"taismo SEO- & GEO-Agentur M\u00fcnchen\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\"},\"sameAs\":[\"https:\\\/\\\/www.facebook.com\\\/TaismoOnlineMarketing\",\"https:\\\/\\\/branchenbuch.portal.muenchen.de\\\/mhp\\\/1380360\\\/\",\"https:\\\/\\\/www.crunchbase.com\\\/organization\\\/taismo-gmbh\",\"https:\\\/\\\/de.trustpilot.com\\\/review\\\/taismo.de\",\"https:\\\/\\\/www.linkedin.com\\\/company\\\/taismo\\\/\",\"https:\\\/\\\/www.instagram.com\\\/taismo.de\",\"https:\\\/\\\/onlinemarketing.de\\\/dienstleister\\\/online-marketing-agentur\\\/kirchheim-bei-muenchen\\\/taismo\",\"https:\\\/\\\/www.agenturtipp.de\\\/agentur\\\/taismo-online-marketing\\\/\",\"https:\\\/\\\/www.wlw.de\\\/de\\\/firma\\\/taismo-gmbh-1918135\",\"https:\\\/\\\/www.youtube.com\\\/@taismo8632\",\"https:\\\/\\\/techbehemoths.com\\\/company\\\/taismo-gmbh\",\"https:\\\/\\\/www.provenexpert.com\\\/taismo1\\\/\",\"https:\\\/\\\/www.provenemployer.com\\\/p\\\/taismo-employer\\\/\",\"https:\\\/\\\/www.kennstdueinen.de\\\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html\",\"https:\\\/\\\/www.ibusiness.de\\\/dienstleister\\\/jb\\\/5920834fb726.html\",\"https:\\\/\\\/www.seo-united.de\\\/seo-agenturen\\\/taismo-3345\\\/\",\"https:\\\/\\\/www.wikidata.org\\\/wiki\\\/Q141914543\"]}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"What Is a User Agent? Browsers and Crawlers | taismo","description":"The user agent reveals who fetches a page: browser, Googlebot, or AI crawler. Structure, robots.txt rules, and how to spot fakes. Read on! \ud83e\udd16","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/taismo.de\/en\/what-is\/user-agent\/","og_locale":"en_US","og_type":"article","og_title":"What Is a User Agent? Browsers and Crawlers | taismo","og_description":"The user agent reveals who fetches a page: browser, Googlebot, or AI crawler. Structure, robots.txt rules, and how to spot fakes. Read on! \ud83e\udd16","og_url":"https:\/\/taismo.de\/en\/what-is\/user-agent\/","og_site_name":"taismo| Agentur f\u00fcr SEO &amp; Webdesign","article_publisher":"https:\/\/www.facebook.com\/TaismoOnlineMarketing","article_published_time":"2026-10-27T05:30:00+00:00","og_image":[{"width":1200,"height":675,"url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/10\/user-agent-kennung-browser-crawler.webp","type":"image\/webp"}],"author":"Dominik Breitbach","twitter_card":"summary_large_image","schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/#article","isPartOf":{"@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/"},"author":{"name":"Dominik Breitbach","@id":"https:\/\/taismo.de\/#person-dominik-breitbach"},"headline":"user agent","datePublished":"2026-10-27T05:30:00+00:00","mainEntityOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/"},"wordCount":1435,"publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/10\/user-agent-kennung-browser-crawler.webp","keywords":["U"],"articleSection":["SEO Glossary"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/","url":"https:\/\/taismo.de\/en\/what-is\/user-agent\/","name":"What Is a User Agent? Browsers and Crawlers | taismo","isPartOf":{"@id":"https:\/\/taismo.de\/en\/#website"},"primaryImageOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/#primaryimage"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/10\/user-agent-kennung-browser-crawler.webp","datePublished":"2026-10-27T05:30:00+00:00","description":"The user agent reveals who fetches a page: browser, Googlebot, or AI crawler. Structure, robots.txt rules, and how to spot fakes. Read on! \ud83e\udd16","breadcrumb":{"@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/taismo.de\/en\/what-is\/user-agent\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/#primaryimage","url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/10\/user-agent-kennung-browser-crawler.webp","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/10\/user-agent-kennung-browser-crawler.webp","width":1200,"height":675,"caption":"User Agent erkl\u00e4rt: die Kennung im HTTP-Header, an der Webserver Browser, Googlebot und KI-Crawler wie GPTBot erkennen, taismo SEO-Wiki"},{"@type":"BreadcrumbList","@id":"https:\/\/taismo.de\/en\/what-is\/user-agent\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Startseite","item":"https:\/\/taismo.de\/en\/search-marketing\/"},{"@type":"ListItem","position":2,"name":"user agent"}]},{"@type":"WebSite","@id":"https:\/\/taismo.de\/en\/#website","url":"https:\/\/taismo.de\/en\/","name":"taismo| Agentur f\u00fcr SEO & Webdesign","description":"","publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"alternateName":"taismo","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/taismo.de\/en\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/taismo.de\/en\/#organization","name":"taismo SEO- & GEO-Agentur M\u00fcnchen","alternateName":"taismo","url":"https:\/\/taismo.de\/en\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/","url":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","width":512,"height":512,"caption":"taismo SEO- & GEO-Agentur M\u00fcnchen"},"image":{"@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/TaismoOnlineMarketing","https:\/\/branchenbuch.portal.muenchen.de\/mhp\/1380360\/","https:\/\/www.crunchbase.com\/organization\/taismo-gmbh","https:\/\/de.trustpilot.com\/review\/taismo.de","https:\/\/www.linkedin.com\/company\/taismo\/","https:\/\/www.instagram.com\/taismo.de","https:\/\/onlinemarketing.de\/dienstleister\/online-marketing-agentur\/kirchheim-bei-muenchen\/taismo","https:\/\/www.agenturtipp.de\/agentur\/taismo-online-marketing\/","https:\/\/www.wlw.de\/de\/firma\/taismo-gmbh-1918135","https:\/\/www.youtube.com\/@taismo8632","https:\/\/techbehemoths.com\/company\/taismo-gmbh","https:\/\/www.provenexpert.com\/taismo1\/","https:\/\/www.provenemployer.com\/p\/taismo-employer\/","https:\/\/www.kennstdueinen.de\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html","https:\/\/www.ibusiness.de\/dienstleister\/jb\/5920834fb726.html","https:\/\/www.seo-united.de\/seo-agenturen\/taismo-3345\/","https:\/\/www.wikidata.org\/wiki\/Q141914543"]}]}},"_links":{"self":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/8210","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/comments?post=8210"}],"version-history":[{"count":2,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/8210\/revisions"}],"predecessor-version":[{"id":8410,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/8210\/revisions\/8410"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media\/8206"}],"wp:attachment":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media?parent=8210"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/categories?post=8210"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/tags?post=8210"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}