{"id":7117,"date":"2025-10-08T15:20:23","date_gmt":"2025-10-08T13:20:23","guid":{"rendered":"https:\/\/taismo.de\/unkategorisiert\/robots-txt-2\/"},"modified":"2026-07-27T23:14:46","modified_gmt":"2026-07-27T21:14:46","slug":"robots-txt","status":"publish","type":"post","link":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/","title":{"rendered":"robots.txt"},"content":{"rendered":"<style>\n.taismo-art{display:grid;grid-template-columns:320px minmax(0,1fr);gap:36px;align-items:start}\n.taismo-art .toc-sticky{position:sticky;top:80px}\n.taismo-art .toc-sticky ol li{margin-bottom:9px}\n.taismo-art .toc-sticky ol a:hover{color:var(--toc-hover) !important}\n.taismo-art .taismo-main h2{scroll-margin-top:90px;margin-top:38px}\n.taismo-art .lead-def{font-size:1.15em;line-height:1.6;background:#f7f2ec;border-left:4px solid #ED8924;padding:16px 20px;border-radius:8px}\n.taismo-art table{border-collapse:collapse;width:100%;margin:18px 0}\n.taismo-art th,.taismo-art td{border:1px solid #d9e0df;padding:9px 12px;text-align:left;font-size:0.95em}\n.taismo-art th{background:#072328;color:#fff}\n.taismo-art figure{margin:26px 0}\n.taismo-art figcaption{font-size:0.9em;color:#55605f;margin-top:8px}\n.taismo-art pre{background:#072328;color:#f5f5f5;padding:16px 18px;border-radius:10px;overflow-x:auto;font-size:0.86em;line-height:1.5}\n@media(max-width:900px){.taismo-art{grid-template-columns:1fr}.taismo-art .toc-sticky{position:static}}\n<\/style>\n<div class=\"taismo-art\">\n<aside class=\"toc-sticky\" style=\"--toc-hover:#ED8924;background:#072328;border-radius:14px;padding:22px 24px\">\n<div style=\"text-align:center;margin-bottom:14px\"><svg fill=\"#ffffff\" role=\"img\" aria-label=\"taismo Wiki\" style=\"width:200px;max-width:80%;height:auto;display:inline-block\" id=\"Ebene_1\" data-name=\"Ebene 1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 209.04 52.32\"> <g id=\"Layer_2\" data-name=\"Layer 2\"> <g id=\"wikipedia\"> <path d=\"M97.08,27.77h-17.76c-.77,0-1.4.63-1.4,1.4v17.76c0,.77.63,1.4,1.4,1.4h17.76c.77,0,1.4-.63,1.4-1.4v-17.76c0-.77-.63-1.4-1.4-1.4ZM97.22,33.22c0,.25-.21.46-.46.46h-.67c-.07.03-.13.07-.19.11-.19.18-.33.41-.4.66,0,0,0,.01,0,.02-.03.07-2.9,7.29-3.87,9.67v.02c-.09.17-.19.33-.31.48-.15.2-.38.32-.63.33-.56,0-.84-.63-.93-.84-.28-.62-.79-1.83-1.51-3.57-.55,1.24-1.25,2.8-1.64,3.59h0c-.19.36-.49.83-.97.83-.56,0-.82-.63-.9-.84h0c-.49-1.25-1.23-2.99-1.95-4.68-.79-1.83-2.39-5.73-2.43-5.78h-.72c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h4.13c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.82s.03.09.05.14c.59,1.61,2.36,5.75,3.1,7.47l1.28-2.81c-.15-.36-2.12-4.68-2.2-4.8h-.4c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.98c.25,0,.46.21.46.46v.58c0,.25-.21.46-.46.46h-.1c.06.14.94,2.12.94,2.12,0,0,.93-2.06.95-2.12h-.39c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.81c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.43s-2.06,4.08-2.06,4.08l1.56,3.51s2.9-7.59,2.9-7.6h-.96c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h3.78c.25,0,.46.2.46.46v.58Z\"\/> <\/g> <\/g> <text font-family=\"Arial,Helvetica,sans-serif\" font-size=\"27.79\" font-weight=\"800\" transform=\"translate(99.44 47.64) scale(.99 1)\"><tspan x=\"0\" y=\"0\">IKI<\/tspan><\/text> <g id=\"Gruppe_796\" data-name=\"Gruppe 796\"> <path id=\"Pfad_1293\" data-name=\"Pfad 1293\" d=\"M6.92,16.77h-3.56v-6.56h3.66V3.37h8.04v6.84h5.69v6.56h-5.58v8.97c-.08,1.05.13,2.1.6,3.04.4.6,1.24.9,2.51.9.54,0,1.01,0,1.39-.03.41-.02.82-.07,1.23-.13v6.35c-.75.19-1.52.31-2.3.36-1.02.09-1.95.13-2.79.13-1.41.04-2.81-.17-4.15-.6-1.07-.36-2.02-1.01-2.76-1.86-.76-.95-1.29-2.06-1.54-3.25-.34-1.55-.49-3.14-.46-4.73v-9.14h0Z\"\/> <path id=\"Pfad_1294\" data-name=\"Pfad 1294\" d=\"M44.43,35.91h-7.06l-.44-1.97c-.68.72-1.48,1.34-2.35,1.8-.95.46-2.01.69-3.07.66-.99,0-1.98-.1-2.95-.33-.92-.21-1.78-.63-2.51-1.23-.77-.66-1.37-1.49-1.75-2.43-.48-1.28-.71-2.63-.66-3.99v-1.42c-.04-1.17.2-2.33.71-3.39.44-.87,1.09-1.63,1.89-2.19.82-.56,1.74-.96,2.71-1.18,1.06-.24,2.14-.36,3.23-.36.87,0,1.72.06,2.57.22.72.14,1.41.36,2.08.66v-.93c0-.54-.07-1.07-.22-1.59-.16-.48-.46-.91-.87-1.2-.55-.38-1.18-.63-1.83-.77-1-.2-2.01-.29-3.04-.28-.91,0-1.81.09-2.71.25-.97.16-1.8.32-2.49.46v-6.12c.48-.13.96-.22,1.45-.28.57-.07,1.16-.14,1.78-.22.62-.07,1.22-.13,1.78-.16.57-.04,1.04-.05,1.45-.05,2.1-.05,4.18.19,6.21.71,1.4.37,2.69,1.08,3.74,2.08.92.94,1.57,2.13,1.86,3.42.34,1.54.51,3.1.49,4.68v15.15h.01,0ZM33.71,30.83c1.63.12,3.04-1.1,3.16-2.72v-2.58c-.77-.58-1.72-.87-2.68-.82-.85-.04-1.69.16-2.43.57-.6.38-.9,1.16-.9,2.33v.38c-.07.78.16,1.55.66,2.16.6.51,1.39.76,2.19.69\"\/> <rect id=\"Rechteck_760\" data-name=\"Rechteck 760\" x=\"49.46\" y=\"10.21\" width=\"8.26\" height=\"25.71\"\/> <path d=\"M131.36,22.82v-.42c-.06-1.6.12-3.21.52-4.76.35-1.1,1.23-1.64,2.65-1.64.59-.03,1.16.11,1.66.41.43.29.75.73.9,1.23.2.66.32,1.34.36,2.02.05.81.08,1.72.08,2.74v.42h8.38v-1.01c.04-1.92-.23-3.83-.82-5.66-.48-1.43-1.27-2.73-2.33-3.8-1.02-.99-2.27-1.73-3.64-2.13-1.53-.45-3.12-.67-4.71-.66-1.57-.01-3.12.2-4.62.66-1.36.41-2.6,1.14-3.61,2.13-1.06,1.07-1.86,2.37-2.35,3.8-.61,1.82-.89,3.74-.85,5.66v1.01h8.38Z\"\/> <path d=\"M93,22.82v-2.38c-.06-.88.16-1.76.63-2.51.39-.57,1.04-.89,1.72-.87.26,0,.51.03.77.08.26.06.51.19.71.38.25.24.43.54.52.87.15.51.21,1.03.19,1.56v2.88h8.2v-2.7c-.05-.8.16-1.6.6-2.27.4-.52,1.03-.82,1.69-.79.54,0,1.07.17,1.51.49.45.33.69,1.13.69,2.41v2.86h8.2v-4.99c.03-1.32-.17-2.64-.6-3.89-.34-.96-.9-1.83-1.64-2.54-.69-.63-1.51-1.11-2.41-1.39-.92-.3-1.88-.44-2.84-.44-1.16-.02-2.3.21-3.36.69-1.09.55-2.07,1.3-2.87,2.22-.57-.95-1.38-1.72-2.35-2.24-1.16-.49-2.41-.72-3.66-.66-1.18-.04-2.35.22-3.39.76-.99.57-1.88,1.31-2.63,2.19l-.54-2.3h-7.34v12.61h8.2Z\"\/> <path d=\"M73.21,26.15c0-1.84,1.49-3.33,3.33-3.33h2.18c-.87-.85-1.94-1.47-3.12-1.8l-1.75-.6c-.73-.25-1.35-.46-1.86-.63-.43-.13-.85-.3-1.26-.49-.28-.13-.52-.33-.71-.57-.16-.28-.24-.6-.22-.93,0-.22.04-.45.13-.66.13-.25.34-.46.6-.57.43-.2.89-.34,1.36-.41.81-.13,1.62-.18,2.43-.16.92,0,1.85.07,2.76.22.82.11,1.63.28,2.43.49v-6.12c-.48-.13-.95-.22-1.45-.28-.56-.07-1.16-.14-1.78-.22-.61-.07-1.21-.13-1.78-.16s-1.05-.05-1.45-.05c-3.72,0-6.51.75-8.37,2.27-1.78,1.35-2.82,3.48-2.79,5.71-.03,1.07.11,2.13.41,3.15.24.76.64,1.46,1.18,2.05.52.56,1.15,1.01,1.83,1.34.79.38,1.6.72,2.43,1.01l1.26.44c.83.28,1.63.61,2.41,1.01.47.28.75.8.71,1.34.07.72-.35,1.4-1.04,1.64-1.23.31-2.51.45-3.77.38-.95,0-1.9-.09-2.84-.25-1.06-.16-1.93-.31-2.63-.46v6.12c.5.13,1,.22,1.51.28.6.07,1.23.14,1.89.22.66.07,1.3.13,1.92.16.62.03,1.13.05,1.53.05,1.5.03,3.01-.1,4.49-.36v-9.83Z\"\/> <\/g> <g id=\"e\"> <path d=\"M150.81,38.2c-.26-1.16-1.38-2.22-2.54-2.45-.71-.14-2.69-.32-3.99-.3-1.31-.01-3.28.16-3.99.3-1.16.23-2.29,1.29-2.54,2.45-.15.7-.29,1.66-.29,2.97,0,1.31.15,2.27.29,2.97.26,1.16,1.38,2.22,2.54,2.45.38.08,1.14.16,1.95.22.09.15.19.3.3.46.43.66.77,1.11,1.02,1.43.42.53,1.03.53,1.45,0,.25-.32.58-.78,1.02-1.43.11-.16.21-.32.3-.46.82-.06,1.57-.15,1.95-.22,1.16-.23,2.29-1.29,2.54-2.45.15-.7.29-1.66.29-2.97,0-1.31-.15-2.27-.29-2.97h0ZM145.03,45.13c-.03.16-.18.31-.34.34-.1.02-.23.03-.42.03-.18,0-.32-.02-.42-.03-.16-.03-.31-.18-.34-.34-.02-.1-.03-.23-.03-.42,0-.18.02-.32.03-.42.03-.16.18-.31.34-.34.1-.02.23-.03.42-.03.18,0,.32.02.42.03.16.03.31.18.34.34.02.1.03.23.03.42,0,.18-.02.32-.03.42ZM146.65,40.76c-.21.29-.51.56-.97.84-.58.36-.67.5-.67.79v.11c0,.41-.33.74-.74.74s-.74-.33-.74-.74v-.26c0-.34.08-.63.26-.88.17-.24.42-.46.8-.68.69-.41.82-.63.82-.94,0-.38-.32-.63-.83-.63-.55,0-.91.25-1.06.74-.1.32-.4.54-.75.54-.02,0-.04,0-.06,0-.24-.02-.46-.14-.6-.35-.14-.2-.17-.45-.1-.68.17-.52.49-.96.93-1.26.45-.3,1.01-.47,1.63-.47s1.24.21,1.69.58c.45.38.71.91.71,1.49,0,.4-.1.74-.32,1.04h0Z\"\/> <\/g> <g id=\"e-2\" data-name=\"e\"> <path d=\"M156.19,28.79c-.17-.75-.89-1.44-1.64-1.58-.46-.09-1.73-.2-2.58-.2-.84,0-2.12.1-2.58.2-.75.15-1.48.83-1.64,1.58-.09.45-.19,1.07-.19,1.92,0,.84.09,1.46.19,1.92.17.75.89,1.44,1.64,1.58.25.05.73.11,1.26.14.06.09.12.19.19.3.28.42.49.72.66.93.27.34.66.34.93,0,.16-.21.38-.5.66-.93.07-.11.13-.21.19-.3.53-.04,1.01-.09,1.26-.14.75-.15,1.48-.83,1.64-1.58.09-.45.19-1.07.19-1.92,0-.84-.09-1.46-.19-1.92h0ZM152.46,33.27c-.02.1-.11.2-.22.22-.06.01-.15.02-.27.02-.12,0-.2,0-.27-.02-.1-.02-.2-.11-.22-.22-.01-.06-.02-.15-.02-.27,0-.12,0-.2.02-.27.02-.1.11-.2.22-.22.06-.01.15-.02.27-.02.12,0,.2,0,.27.02.1.02.2.11.22.22.01.06.02.15.02.27,0,.12,0,.2-.02.27ZM153.51,30.45c-.13.19-.33.36-.63.54-.37.23-.43.32-.43.51v.07c0,.26-.21.48-.48.48s-.48-.21-.48-.48v-.17c0-.22.05-.4.17-.57.11-.16.27-.3.51-.44.45-.26.53-.41.53-.61,0-.25-.21-.4-.53-.4-.35,0-.59.16-.69.48-.07.21-.26.35-.48.35-.01,0-.02,0-.04,0-.16-.01-.3-.09-.39-.22-.09-.13-.11-.29-.06-.44.11-.34.32-.62.6-.81.29-.2.65-.3,1.05-.3s.8.13,1.09.37c.29.25.46.59.46.96,0,.26-.07.48-.21.67h0Z\"\/> <\/g> <g id=\"e-3\" data-name=\"e\"> <path d=\"M144.91,28.64c-.1-.44-.52-.84-.96-.92-.27-.05-1.01-.12-1.5-.11-.49,0-1.24.06-1.5.11-.44.08-.86.49-.96.92-.06.27-.11.63-.11,1.12,0,.49.05.85.11,1.12.1.44.52.84.96.92.14.03.43.06.74.08.03.05.07.11.11.17.16.25.29.42.38.54.16.2.39.2.54,0,.09-.12.22-.29.38-.54.04-.06.08-.12.11-.17.31-.02.59-.06.74-.08.44-.08.86-.49.96-.92.06-.27.11-.63.11-1.12,0-.49-.05-.85-.11-1.12h0ZM142.73,31.26c-.01.06-.07.12-.13.13-.04,0-.09.01-.16.01-.07,0-.12,0-.16-.01-.06-.01-.12-.07-.13-.13,0-.04-.01-.09-.01-.16,0-.07,0-.12.01-.16.01-.06.07-.12.13-.13.04,0,.09-.01.16-.01.07,0,.12,0,.16.01.06.01.12.07.13.13,0,.04.01.09.01.16,0,.07,0,.12-.01.16ZM143.34,29.61c-.08.11-.19.21-.37.32-.22.13-.25.19-.25.3v.04c0,.15-.12.28-.28.28s-.28-.12-.28-.28v-.1c0-.13.03-.24.1-.33.06-.09.16-.17.3-.26.26-.15.31-.24.31-.35,0-.14-.12-.24-.31-.24-.21,0-.34.09-.4.28-.04.12-.15.2-.28.2,0,0-.01,0-.02,0-.09,0-.17-.05-.23-.13-.05-.08-.06-.17-.04-.25.06-.2.18-.36.35-.47.17-.11.38-.18.61-.18s.47.08.63.22c.17.14.27.34.27.56,0,.15-.04.28-.12.39h0Z\"\/> <\/g> <\/svg><\/div>\n<h3 style=\"text-align:center;color:#fff;margin:0 0 14px;font-size:1.1em\">Contents<\/h3>\n<ol style=\"margin:0;padding-left:1.2em;line-height:1.5;font-weight:600;color:#fff;font-size:0.9em\">\n<li><a href=\"#do-search-engines-have-to-follow-robots-txt\" style=\"color:#fff;text-decoration:none\">Do search engines have to follow the instructions in the robots.txt?<\/a><\/li>\n<li><a href=\"#where-does-the-robots-txt-file-belong\" style=\"color:#fff;text-decoration:none\">Where do I put the robots.txt file on my web server?<\/a><\/li>\n<li><a href=\"#how-is-a-robots-txt-file-structured\" style=\"color:#fff;text-decoration:none\">How is a robots.txt file structured?<\/a><\/li>\n<li><a href=\"#how-does-robots-txt-affect-seo\" style=\"color:#fff;text-decoration:none\">What effect does the robots.txt have on SEO?<\/a><\/li>\n<\/ol>\n<\/aside>\n<div class=\"taismo-main\">\n<h1 class=\"brand-claim\">What is the robots.txt?<\/h1>\n<p class=\"lead-def\">The <strong>robots.txt<\/strong> is a text file on a web server that follows the Robots Exclusion Standard Protocol. It holds instructions for the web <a href=\"https:\/\/taismo.de\/en\/what-is\/crawler\/\">crawlers<\/a> that scan a website in order to index it for a search engine.<\/p>\n<figure><img decoding=\"async\" src=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/robots-txt.png\" width=\"1200\" height=\"675\" alt=\"robots.txt explained: a text file in the root directory of a web server that tells crawlers which areas they may read\" title=\"robots.txt: crawl instructions in the root directory of a web server\" loading=\"lazy\"><\/figure>\n<h2 id=\"do-search-engines-have-to-follow-robots-txt\">Do search engines have to follow the instructions in the robots.txt?<\/h2>\n<p>No. What stands in the robots.txt file is only a <strong>reading recommendation<\/strong> from the site owner to the search engine. Large search engines such as Google, Bing and Yahoo agreed in 2008 to follow the instructions in a robots.txt, but there is never a hundred percent guarantee.<\/p>\n<h2 id=\"where-does-the-robots-txt-file-belong\">Where do I put the robots.txt file on my web server?<\/h2>\n<p>The file has to sit in the root directory, meaning the topmost directory of the web server. It is therefore always reachable by adding <em>\/robots.txt<\/em> to the <a href=\"https:\/\/taismo.de\/en\/what-is\/url\/\">URL<\/a> after the top level domain.<\/p>\n<p>Example: https:\/\/taismo.de\/robots.txt<\/p>\n<h2 id=\"how-is-a-robots-txt-file-structured\">How is a robots.txt file structured?<\/h2>\n<p>The syntax is compact and rests on three basic elements:<\/p>\n<ul>\n<li><strong>User-agent:<\/strong>\n<ul>\n<li>This element sets which bot the following instructions apply to.\n<ul>\n<li>Example: <em>googlebot<\/em><\/li>\n<\/ul>\n<\/li>\n<li>If an instruction is meant for every bot, a * can be used.\n<ul>\n<li>Example: <em>User-agent: *<\/em><\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<li><strong>Allow:<\/strong>\n<ul>\n<li>This element names a directory or a specific page that should be crawled.<\/li>\n<\/ul>\n<\/li>\n<li><strong>Disallow:<\/strong>\n<ul>\n<li>This element names a directory or a specific page that should <strong>not<\/strong> be crawled.<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>Example:<\/p>\n<pre><code class=\"el-content\">User-agent: *\nDisallow: \/wp-admin\/\nAllow: \/wp-admin\/admin-ajax.php\n<\/code><\/pre>\n<p>Explanation: this snippet carries instructions for every crawler. They should leave the folder <em>wp-admin<\/em> alone, with the exception of the PHP file <em>admin-ajax.php<\/em>.<\/p>\n<h2 id=\"how-does-robots-txt-affect-seo\">What effect does the robots.txt have on SEO?<\/h2>\n<p>A robots.txt that is set up wrongly can keep parts of your website out of the search results, because they are shut out from crawling. If certain pages do not rank the way you want them to, the instructions in that file are worth a look.<\/p>\n<p>The robots.txt can also pass further <a href=\"https:\/\/taismo.de\/en\/what-is\/seo\/\">SEO<\/a> relevant information to the web crawler. Among other things, the path to the <a href=\"https:\/\/taismo.de\/en\/what-is\/sitemap\/\">sitemap<\/a> can be stored in it. How that works is covered by the sitemap entry of our <a href=\"https:\/\/taismo.de\/en\/what-is\/\">SEO glossary<\/a>.<\/p>\n<p>Whether your robots.txt blocks anything it should not, and whether it points crawlers to the right sitemap, is one of the points we check in an <a href=\"https:\/\/taismo.de\/en\/seo-audit\/\">SEO audit<\/a>.<\/p>\n<\/p><\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>IKI Contents Do search engines have to follow the instructions in the robots.txt? Where do I put the robots.txt file on my web server? How is a robots.txt file structured? What effect does the robots.txt have on SEO? What is the robots.txt? The robots.txt is a text file on a web server that follows the [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":7554,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_taismo_jsonld":"","taismo_jsonld":"{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#webpage\",\"url\":\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/\",\"name\":\"What is the robots.txt?\",\"isPartOf\":{\"@id\":\"https:\/\/taismo.de\/#website\"},\"inLanguage\":\"en\",\"primaryImageOfPage\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#primaryimage\"},\"mainEntity\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#definedterm\"},\"breadcrumb\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#breadcrumb\"},\"speakable\":{\"@type\":\"SpeakableSpecification\",\"cssSelector\":[\".brand-claim\",\".lead-def\"]},\"author\":{\"@id\":\"https:\/\/taismo.de\/#person-dominik-breitbach\"},\"publisher\":{\"@id\":\"https:\/\/taismo.de\/#organization\"},\"datePublished\":\"2025-10-08T15:20:23+02:00\",\"dateModified\":\"2026-07-14T12:00:00+02:00\"},{\"@type\":\"ImageObject\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#primaryimage\",\"contentUrl\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/robots-txt.png\",\"url\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/robots-txt.png\",\"width\":1200,\"height\":675,\"caption\":\"robots.txt explained: a text file in the root directory of a web server that tells crawlers which areas they may read\"},{\"@type\":\"DefinedTerm\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#definedterm\",\"name\":\"robots.txt\",\"termCode\":\"robots-txt\",\"url\":\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/\",\"inLanguage\":\"en\",\"inDefinedTermSet\":{\"@id\":\"https:\/\/taismo.de\/#seo-wiki\"},\"description\":\"The robots.txt is a text file on a web server that follows the Robots Exclusion Standard Protocol. It holds instructions for the web crawlers that scan a website in order to index it for a search engine.\",\"author\":{\"@id\":\"https:\/\/taismo.de\/#person-dominik-breitbach\"},\"publisher\":{\"@id\":\"https:\/\/taismo.de\/#organization\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\/\/taismo.de\/en\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"SEO Glossary\",\"item\":\"https:\/\/taismo.de\/en\/what-is\/\"},{\"@type\":\"ListItem\",\"position\":3,\"name\":\"robots.txt\"}]}]}","footnotes":""},"categories":[80],"tags":[99],"class_list":["post-7117","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-what-is","tag-r-en"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v28.1 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>What Is robots.txt? Crawl Rules Explained | taismo<\/title>\n<meta name=\"description\" content=\"The robots.txt tells crawlers which parts of your site they may read. See how it is built, where it belongs and what it means for your SEO \ud83e\udd16\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"What Is robots.txt? Crawl Rules Explained | taismo\" \/>\n<meta property=\"og:description\" content=\"The robots.txt tells crawlers which parts of your site they may read. See how it is built, where it belongs and what it means for your SEO \ud83e\udd16\" \/>\n<meta property=\"og:url\" content=\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/\" \/>\n<meta property=\"og:site_name\" content=\"taismo| Agentur f\u00fcr SEO &amp; Webdesign\" \/>\n<meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/TaismoOnlineMarketing\" \/>\n<meta property=\"article:published_time\" content=\"2025-10-08T13:20:23+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2026-07-27T21:14:46+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/robots-txt_featured.png\" \/>\n\t<meta property=\"og:image:width\" content=\"1200\" \/>\n\t<meta property=\"og:image:height\" content=\"675\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/png\" \/>\n<meta name=\"author\" content=\"Dominik Breitbach\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/\"},\"author\":{\"name\":\"Dominik Breitbach\",\"@id\":\"https:\\\/\\\/taismo.de\\\/#person-dominik-breitbach\"},\"headline\":\"robots.txt\",\"datePublished\":\"2025-10-08T13:20:23+00:00\",\"dateModified\":\"2026-07-27T21:14:46+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/\"},\"wordCount\":422,\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/07\\\/robots-txt_featured.png\",\"keywords\":[\"R\"],\"articleSection\":[\"SEO Glossary\"],\"inLanguage\":\"en-US\"},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/\",\"name\":\"What Is robots.txt? Crawl Rules Explained | taismo\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/07\\\/robots-txt_featured.png\",\"datePublished\":\"2025-10-08T13:20:23+00:00\",\"dateModified\":\"2026-07-27T21:14:46+00:00\",\"description\":\"The robots.txt tells crawlers which parts of your site they may read. See how it is built, where it belongs and what it means for your SEO \ud83e\udd16\",\"breadcrumb\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/#primaryimage\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/07\\\/robots-txt_featured.png\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/07\\\/robots-txt_featured.png\",\"width\":1200,\"height\":675,\"caption\":\"robots.txt: The robots.txt is a text file on a web server that follows the Robots Exclusion Standard Protocol.\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/robots-txt\\\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Startseite\",\"item\":\"https:\\\/\\\/taismo.de\\\/en\\\/search-marketing\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"robots.txt\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"name\":\"taismo| Agentur f\u00fcr SEO & Webdesign\",\"description\":\"\",\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"alternateName\":\"taismo\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/taismo.de\\\/en\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Organization\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\",\"name\":\"taismo SEO Agentur M\u00fcnchen\",\"alternateName\":\"taismo\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"logo\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"width\":512,\"height\":512,\"caption\":\"taismo SEO Agentur M\u00fcnchen\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\"},\"sameAs\":[\"https:\\\/\\\/www.facebook.com\\\/TaismoOnlineMarketing\",\"https:\\\/\\\/branchenbuch.portal.muenchen.de\\\/mhp\\\/1380360\\\/\",\"https:\\\/\\\/www.crunchbase.com\\\/organization\\\/taismo-gmbh\",\"https:\\\/\\\/de.trustpilot.com\\\/review\\\/taismo.de\",\"https:\\\/\\\/www.linkedin.com\\\/company\\\/taismo\\\/\",\"https:\\\/\\\/www.instagram.com\\\/taismo.de\",\"https:\\\/\\\/onlinemarketing.de\\\/dienstleister\\\/online-marketing-agentur\\\/kirchheim-bei-muenchen\\\/taismo\",\"https:\\\/\\\/www.agenturtipp.de\\\/agentur\\\/taismo-online-marketing\\\/\",\"https:\\\/\\\/startupvalley.news\\\/de\\\/taismo-online-marketing-agentur\\\/\",\"https:\\\/\\\/www.wlw.de\\\/de\\\/firma\\\/taismo-gmbh-1918135\",\"https:\\\/\\\/www.youtube.com\\\/@taismo8632\",\"https:\\\/\\\/techbehemoths.com\\\/company\\\/taismo-gmbh\",\"https:\\\/\\\/www.kununu.com\\\/de\\\/taismo1\",\"https:\\\/\\\/www.provenexpert.com\\\/taismo1\\\/\",\"https:\\\/\\\/www.provenemployer.com\\\/p\\\/taismo-employer\\\/\",\"https:\\\/\\\/www.trustedshops.de\\\/bewertung\\\/taismo-de\",\"https:\\\/\\\/www.kennstdueinen.de\\\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html\",\"https:\\\/\\\/www.ibusiness.de\\\/dienstleister\\\/jb\\\/5920834fb726.html\",\"https:\\\/\\\/www.seo-united.de\\\/seo-agenturen\\\/taismo-3345\\\/\"]}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"What Is robots.txt? Crawl Rules Explained | taismo","description":"The robots.txt tells crawlers which parts of your site they may read. See how it is built, where it belongs and what it means for your SEO \ud83e\udd16","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/","og_locale":"en_US","og_type":"article","og_title":"What Is robots.txt? Crawl Rules Explained | taismo","og_description":"The robots.txt tells crawlers which parts of your site they may read. See how it is built, where it belongs and what it means for your SEO \ud83e\udd16","og_url":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/","og_site_name":"taismo| Agentur f\u00fcr SEO &amp; Webdesign","article_publisher":"https:\/\/www.facebook.com\/TaismoOnlineMarketing","article_published_time":"2025-10-08T13:20:23+00:00","article_modified_time":"2026-07-27T21:14:46+00:00","og_image":[{"width":1200,"height":675,"url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/robots-txt_featured.png","type":"image\/png"}],"author":"Dominik Breitbach","twitter_card":"summary_large_image","schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#article","isPartOf":{"@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/"},"author":{"name":"Dominik Breitbach","@id":"https:\/\/taismo.de\/#person-dominik-breitbach"},"headline":"robots.txt","datePublished":"2025-10-08T13:20:23+00:00","dateModified":"2026-07-27T21:14:46+00:00","mainEntityOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/"},"wordCount":422,"publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/robots-txt_featured.png","keywords":["R"],"articleSection":["SEO Glossary"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/","url":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/","name":"What Is robots.txt? Crawl Rules Explained | taismo","isPartOf":{"@id":"https:\/\/taismo.de\/en\/#website"},"primaryImageOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#primaryimage"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/robots-txt_featured.png","datePublished":"2025-10-08T13:20:23+00:00","dateModified":"2026-07-27T21:14:46+00:00","description":"The robots.txt tells crawlers which parts of your site they may read. See how it is built, where it belongs and what it means for your SEO \ud83e\udd16","breadcrumb":{"@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/taismo.de\/en\/what-is\/robots-txt\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#primaryimage","url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/robots-txt_featured.png","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/robots-txt_featured.png","width":1200,"height":675,"caption":"robots.txt: The robots.txt is a text file on a web server that follows the Robots Exclusion Standard Protocol."},{"@type":"BreadcrumbList","@id":"https:\/\/taismo.de\/en\/what-is\/robots-txt\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Startseite","item":"https:\/\/taismo.de\/en\/search-marketing\/"},{"@type":"ListItem","position":2,"name":"robots.txt"}]},{"@type":"WebSite","@id":"https:\/\/taismo.de\/en\/#website","url":"https:\/\/taismo.de\/en\/","name":"taismo| Agentur f\u00fcr SEO & Webdesign","description":"","publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"alternateName":"taismo","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/taismo.de\/en\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/taismo.de\/en\/#organization","name":"taismo SEO Agentur M\u00fcnchen","alternateName":"taismo","url":"https:\/\/taismo.de\/en\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/","url":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","width":512,"height":512,"caption":"taismo SEO Agentur M\u00fcnchen"},"image":{"@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/TaismoOnlineMarketing","https:\/\/branchenbuch.portal.muenchen.de\/mhp\/1380360\/","https:\/\/www.crunchbase.com\/organization\/taismo-gmbh","https:\/\/de.trustpilot.com\/review\/taismo.de","https:\/\/www.linkedin.com\/company\/taismo\/","https:\/\/www.instagram.com\/taismo.de","https:\/\/onlinemarketing.de\/dienstleister\/online-marketing-agentur\/kirchheim-bei-muenchen\/taismo","https:\/\/www.agenturtipp.de\/agentur\/taismo-online-marketing\/","https:\/\/startupvalley.news\/de\/taismo-online-marketing-agentur\/","https:\/\/www.wlw.de\/de\/firma\/taismo-gmbh-1918135","https:\/\/www.youtube.com\/@taismo8632","https:\/\/techbehemoths.com\/company\/taismo-gmbh","https:\/\/www.kununu.com\/de\/taismo1","https:\/\/www.provenexpert.com\/taismo1\/","https:\/\/www.provenemployer.com\/p\/taismo-employer\/","https:\/\/www.trustedshops.de\/bewertung\/taismo-de","https:\/\/www.kennstdueinen.de\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html","https:\/\/www.ibusiness.de\/dienstleister\/jb\/5920834fb726.html","https:\/\/www.seo-united.de\/seo-agenturen\/taismo-3345\/"]}]}},"_links":{"self":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/7117","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/comments?post=7117"}],"version-history":[{"count":3,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/7117\/revisions"}],"predecessor-version":[{"id":7218,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/7117\/revisions\/7218"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media\/7554"}],"wp:attachment":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media?parent=7117"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/categories?post=7117"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/tags?post=7117"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}