{"id":7106,"date":"2025-10-07T15:18:01","date_gmt":"2025-10-07T13:18:01","guid":{"rendered":"https:\/\/taismo.de\/unkategorisiert\/crawler-2\/"},"modified":"2026-07-27T23:14:20","modified_gmt":"2026-07-27T21:14:20","slug":"crawler","status":"publish","type":"post","link":"https:\/\/taismo.de\/en\/what-is\/crawler\/","title":{"rendered":"crawler"},"content":{"rendered":"<style>\n.taismo-art{display:grid;grid-template-columns:320px minmax(0,1fr);gap:36px;align-items:start}\n.taismo-art .toc-sticky{position:sticky;top:80px}\n.taismo-art .toc-sticky ol li{margin-bottom:9px}\n.taismo-art .toc-sticky ol a:hover{color:var(--toc-hover) !important}\n.taismo-art .taismo-main h2{scroll-margin-top:90px;margin-top:38px}\n.taismo-art .lead-def{font-size:1.15em;line-height:1.6;background:#f7f2ec;border-left:4px solid #ED8924;padding:16px 20px;border-radius:8px}\n.taismo-art table{border-collapse:collapse;width:100%;margin:18px 0}\n.taismo-art th,.taismo-art td{border:1px solid #d9e0df;padding:9px 12px;text-align:left;font-size:0.95em}\n.taismo-art th{background:#072328;color:#fff}\n.taismo-art figure{margin:26px 0}\n.taismo-art figcaption{font-size:0.9em;color:#55605f;margin-top:8px}\n.taismo-art pre{background:#072328;color:#f5f5f5;padding:16px 18px;border-radius:10px;overflow-x:auto;font-size:0.86em;line-height:1.5}\n@media(max-width:900px){.taismo-art{grid-template-columns:1fr}.taismo-art .toc-sticky{position:static}}\n<\/style>\n<div class=\"taismo-art\">\n<aside class=\"toc-sticky\" style=\"--toc-hover:#ED8924;background:#072328;border-radius:14px;padding:22px 24px\">\n<div style=\"text-align:center;margin-bottom:14px\"><svg fill=\"#ffffff\" role=\"img\" aria-label=\"taismo Wiki\" style=\"width:200px;max-width:80%;height:auto;display:inline-block\" id=\"Ebene_1\" data-name=\"Ebene 1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 209.04 52.32\"> <g id=\"Layer_2\" data-name=\"Layer 2\"> <g id=\"wikipedia\"> <path d=\"M97.08,27.77h-17.76c-.77,0-1.4.63-1.4,1.4v17.76c0,.77.63,1.4,1.4,1.4h17.76c.77,0,1.4-.63,1.4-1.4v-17.76c0-.77-.63-1.4-1.4-1.4ZM97.22,33.22c0,.25-.21.46-.46.46h-.67c-.07.03-.13.07-.19.11-.19.18-.33.41-.4.66,0,0,0,.01,0,.02-.03.07-2.9,7.29-3.87,9.67v.02c-.09.17-.19.33-.31.48-.15.2-.38.32-.63.33-.56,0-.84-.63-.93-.84-.28-.62-.79-1.83-1.51-3.57-.55,1.24-1.25,2.8-1.64,3.59h0c-.19.36-.49.83-.97.83-.56,0-.82-.63-.9-.84h0c-.49-1.25-1.23-2.99-1.95-4.68-.79-1.83-2.39-5.73-2.43-5.78h-.72c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h4.13c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.82s.03.09.05.14c.59,1.61,2.36,5.75,3.1,7.47l1.28-2.81c-.15-.36-2.12-4.68-2.2-4.8h-.4c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.98c.25,0,.46.21.46.46v.58c0,.25-.21.46-.46.46h-.1c.06.14.94,2.12.94,2.12,0,0,.93-2.06.95-2.12h-.39c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.81c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.43s-2.06,4.08-2.06,4.08l1.56,3.51s2.9-7.59,2.9-7.6h-.96c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h3.78c.25,0,.46.2.46.46v.58Z\"\/> <\/g> <\/g> <text font-family=\"Arial,Helvetica,sans-serif\" font-size=\"27.79\" font-weight=\"800\" transform=\"translate(99.44 47.64) scale(.99 1)\"><tspan x=\"0\" y=\"0\">IKI<\/tspan><\/text> <g id=\"Gruppe_796\" data-name=\"Gruppe 796\"> <path id=\"Pfad_1293\" data-name=\"Pfad 1293\" d=\"M6.92,16.77h-3.56v-6.56h3.66V3.37h8.04v6.84h5.69v6.56h-5.58v8.97c-.08,1.05.13,2.1.6,3.04.4.6,1.24.9,2.51.9.54,0,1.01,0,1.39-.03.41-.02.82-.07,1.23-.13v6.35c-.75.19-1.52.31-2.3.36-1.02.09-1.95.13-2.79.13-1.41.04-2.81-.17-4.15-.6-1.07-.36-2.02-1.01-2.76-1.86-.76-.95-1.29-2.06-1.54-3.25-.34-1.55-.49-3.14-.46-4.73v-9.14h0Z\"\/> <path id=\"Pfad_1294\" data-name=\"Pfad 1294\" d=\"M44.43,35.91h-7.06l-.44-1.97c-.68.72-1.48,1.34-2.35,1.8-.95.46-2.01.69-3.07.66-.99,0-1.98-.1-2.95-.33-.92-.21-1.78-.63-2.51-1.23-.77-.66-1.37-1.49-1.75-2.43-.48-1.28-.71-2.63-.66-3.99v-1.42c-.04-1.17.2-2.33.71-3.39.44-.87,1.09-1.63,1.89-2.19.82-.56,1.74-.96,2.71-1.18,1.06-.24,2.14-.36,3.23-.36.87,0,1.72.06,2.57.22.72.14,1.41.36,2.08.66v-.93c0-.54-.07-1.07-.22-1.59-.16-.48-.46-.91-.87-1.2-.55-.38-1.18-.63-1.83-.77-1-.2-2.01-.29-3.04-.28-.91,0-1.81.09-2.71.25-.97.16-1.8.32-2.49.46v-6.12c.48-.13.96-.22,1.45-.28.57-.07,1.16-.14,1.78-.22.62-.07,1.22-.13,1.78-.16.57-.04,1.04-.05,1.45-.05,2.1-.05,4.18.19,6.21.71,1.4.37,2.69,1.08,3.74,2.08.92.94,1.57,2.13,1.86,3.42.34,1.54.51,3.1.49,4.68v15.15h.01,0ZM33.71,30.83c1.63.12,3.04-1.1,3.16-2.72v-2.58c-.77-.58-1.72-.87-2.68-.82-.85-.04-1.69.16-2.43.57-.6.38-.9,1.16-.9,2.33v.38c-.07.78.16,1.55.66,2.16.6.51,1.39.76,2.19.69\"\/> <rect id=\"Rechteck_760\" data-name=\"Rechteck 760\" x=\"49.46\" y=\"10.21\" width=\"8.26\" height=\"25.71\"\/> <path d=\"M131.36,22.82v-.42c-.06-1.6.12-3.21.52-4.76.35-1.1,1.23-1.64,2.65-1.64.59-.03,1.16.11,1.66.41.43.29.75.73.9,1.23.2.66.32,1.34.36,2.02.05.81.08,1.72.08,2.74v.42h8.38v-1.01c.04-1.92-.23-3.83-.82-5.66-.48-1.43-1.27-2.73-2.33-3.8-1.02-.99-2.27-1.73-3.64-2.13-1.53-.45-3.12-.67-4.71-.66-1.57-.01-3.12.2-4.62.66-1.36.41-2.6,1.14-3.61,2.13-1.06,1.07-1.86,2.37-2.35,3.8-.61,1.82-.89,3.74-.85,5.66v1.01h8.38Z\"\/> <path d=\"M93,22.82v-2.38c-.06-.88.16-1.76.63-2.51.39-.57,1.04-.89,1.72-.87.26,0,.51.03.77.08.26.06.51.19.71.38.25.24.43.54.52.87.15.51.21,1.03.19,1.56v2.88h8.2v-2.7c-.05-.8.16-1.6.6-2.27.4-.52,1.03-.82,1.69-.79.54,0,1.07.17,1.51.49.45.33.69,1.13.69,2.41v2.86h8.2v-4.99c.03-1.32-.17-2.64-.6-3.89-.34-.96-.9-1.83-1.64-2.54-.69-.63-1.51-1.11-2.41-1.39-.92-.3-1.88-.44-2.84-.44-1.16-.02-2.3.21-3.36.69-1.09.55-2.07,1.3-2.87,2.22-.57-.95-1.38-1.72-2.35-2.24-1.16-.49-2.41-.72-3.66-.66-1.18-.04-2.35.22-3.39.76-.99.57-1.88,1.31-2.63,2.19l-.54-2.3h-7.34v12.61h8.2Z\"\/> <path d=\"M73.21,26.15c0-1.84,1.49-3.33,3.33-3.33h2.18c-.87-.85-1.94-1.47-3.12-1.8l-1.75-.6c-.73-.25-1.35-.46-1.86-.63-.43-.13-.85-.3-1.26-.49-.28-.13-.52-.33-.71-.57-.16-.28-.24-.6-.22-.93,0-.22.04-.45.13-.66.13-.25.34-.46.6-.57.43-.2.89-.34,1.36-.41.81-.13,1.62-.18,2.43-.16.92,0,1.85.07,2.76.22.82.11,1.63.28,2.43.49v-6.12c-.48-.13-.95-.22-1.45-.28-.56-.07-1.16-.14-1.78-.22-.61-.07-1.21-.13-1.78-.16s-1.05-.05-1.45-.05c-3.72,0-6.51.75-8.37,2.27-1.78,1.35-2.82,3.48-2.79,5.71-.03,1.07.11,2.13.41,3.15.24.76.64,1.46,1.18,2.05.52.56,1.15,1.01,1.83,1.34.79.38,1.6.72,2.43,1.01l1.26.44c.83.28,1.63.61,2.41,1.01.47.28.75.8.71,1.34.07.72-.35,1.4-1.04,1.64-1.23.31-2.51.45-3.77.38-.95,0-1.9-.09-2.84-.25-1.06-.16-1.93-.31-2.63-.46v6.12c.5.13,1,.22,1.51.28.6.07,1.23.14,1.89.22.66.07,1.3.13,1.92.16.62.03,1.13.05,1.53.05,1.5.03,3.01-.1,4.49-.36v-9.83Z\"\/> <\/g> <g id=\"e\"> <path d=\"M150.81,38.2c-.26-1.16-1.38-2.22-2.54-2.45-.71-.14-2.69-.32-3.99-.3-1.31-.01-3.28.16-3.99.3-1.16.23-2.29,1.29-2.54,2.45-.15.7-.29,1.66-.29,2.97,0,1.31.15,2.27.29,2.97.26,1.16,1.38,2.22,2.54,2.45.38.08,1.14.16,1.95.22.09.15.19.3.3.46.43.66.77,1.11,1.02,1.43.42.53,1.03.53,1.45,0,.25-.32.58-.78,1.02-1.43.11-.16.21-.32.3-.46.82-.06,1.57-.15,1.95-.22,1.16-.23,2.29-1.29,2.54-2.45.15-.7.29-1.66.29-2.97,0-1.31-.15-2.27-.29-2.97h0ZM145.03,45.13c-.03.16-.18.31-.34.34-.1.02-.23.03-.42.03-.18,0-.32-.02-.42-.03-.16-.03-.31-.18-.34-.34-.02-.1-.03-.23-.03-.42,0-.18.02-.32.03-.42.03-.16.18-.31.34-.34.1-.02.23-.03.42-.03.18,0,.32.02.42.03.16.03.31.18.34.34.02.1.03.23.03.42,0,.18-.02.32-.03.42ZM146.65,40.76c-.21.29-.51.56-.97.84-.58.36-.67.5-.67.79v.11c0,.41-.33.74-.74.74s-.74-.33-.74-.74v-.26c0-.34.08-.63.26-.88.17-.24.42-.46.8-.68.69-.41.82-.63.82-.94,0-.38-.32-.63-.83-.63-.55,0-.91.25-1.06.74-.1.32-.4.54-.75.54-.02,0-.04,0-.06,0-.24-.02-.46-.14-.6-.35-.14-.2-.17-.45-.1-.68.17-.52.49-.96.93-1.26.45-.3,1.01-.47,1.63-.47s1.24.21,1.69.58c.45.38.71.91.71,1.49,0,.4-.1.74-.32,1.04h0Z\"\/> <\/g> <g id=\"e-2\" data-name=\"e\"> <path d=\"M156.19,28.79c-.17-.75-.89-1.44-1.64-1.58-.46-.09-1.73-.2-2.58-.2-.84,0-2.12.1-2.58.2-.75.15-1.48.83-1.64,1.58-.09.45-.19,1.07-.19,1.92,0,.84.09,1.46.19,1.92.17.75.89,1.44,1.64,1.58.25.05.73.11,1.26.14.06.09.12.19.19.3.28.42.49.72.66.93.27.34.66.34.93,0,.16-.21.38-.5.66-.93.07-.11.13-.21.19-.3.53-.04,1.01-.09,1.26-.14.75-.15,1.48-.83,1.64-1.58.09-.45.19-1.07.19-1.92,0-.84-.09-1.46-.19-1.92h0ZM152.46,33.27c-.02.1-.11.2-.22.22-.06.01-.15.02-.27.02-.12,0-.2,0-.27-.02-.1-.02-.2-.11-.22-.22-.01-.06-.02-.15-.02-.27,0-.12,0-.2.02-.27.02-.1.11-.2.22-.22.06-.01.15-.02.27-.02.12,0,.2,0,.27.02.1.02.2.11.22.22.01.06.02.15.02.27,0,.12,0,.2-.02.27ZM153.51,30.45c-.13.19-.33.36-.63.54-.37.23-.43.32-.43.51v.07c0,.26-.21.48-.48.48s-.48-.21-.48-.48v-.17c0-.22.05-.4.17-.57.11-.16.27-.3.51-.44.45-.26.53-.41.53-.61,0-.25-.21-.4-.53-.4-.35,0-.59.16-.69.48-.07.21-.26.35-.48.35-.01,0-.02,0-.04,0-.16-.01-.3-.09-.39-.22-.09-.13-.11-.29-.06-.44.11-.34.32-.62.6-.81.29-.2.65-.3,1.05-.3s.8.13,1.09.37c.29.25.46.59.46.96,0,.26-.07.48-.21.67h0Z\"\/> <\/g> <g id=\"e-3\" data-name=\"e\"> <path d=\"M144.91,28.64c-.1-.44-.52-.84-.96-.92-.27-.05-1.01-.12-1.5-.11-.49,0-1.24.06-1.5.11-.44.08-.86.49-.96.92-.06.27-.11.63-.11,1.12,0,.49.05.85.11,1.12.1.44.52.84.96.92.14.03.43.06.74.08.03.05.07.11.11.17.16.25.29.42.38.54.16.2.39.2.54,0,.09-.12.22-.29.38-.54.04-.06.08-.12.11-.17.31-.02.59-.06.74-.08.44-.08.86-.49.96-.92.06-.27.11-.63.11-1.12,0-.49-.05-.85-.11-1.12h0ZM142.73,31.26c-.01.06-.07.12-.13.13-.04,0-.09.01-.16.01-.07,0-.12,0-.16-.01-.06-.01-.12-.07-.13-.13,0-.04-.01-.09-.01-.16,0-.07,0-.12.01-.16.01-.06.07-.12.13-.13.04,0,.09-.01.16-.01.07,0,.12,0,.16.01.06.01.12.07.13.13,0,.04.01.09.01.16,0,.07,0,.12-.01.16ZM143.34,29.61c-.08.11-.19.21-.37.32-.22.13-.25.19-.25.3v.04c0,.15-.12.28-.28.28s-.28-.12-.28-.28v-.1c0-.13.03-.24.1-.33.06-.09.16-.17.3-.26.26-.15.31-.24.31-.35,0-.14-.12-.24-.31-.24-.21,0-.34.09-.4.28-.04.12-.15.2-.28.2,0,0-.01,0-.02,0-.09,0-.17-.05-.23-.13-.05-.08-.06-.17-.04-.25.06-.2.18-.36.35-.47.17-.11.38-.18.61-.18s.47.08.63.22c.17.14.27.34.27.56,0,.15-.04.28-.12.39h0Z\"\/> <\/g> <\/svg><\/div>\n<h3 style=\"text-align:center;color:#fff;margin:0 0 14px;font-size:1.1em\">Contents<\/h3>\n<ol style=\"margin:0;padding-left:1.2em;line-height:1.5;font-weight:600;color:#fff;font-size:0.9em\">\n<li><a href=\"#how-does-a-web-crawler-work\" style=\"color:#fff;text-decoration:none\">How does a web crawler work?<\/a><\/li>\n<li><a href=\"#different-types-of-crawlers\" style=\"color:#fff;text-decoration:none\">Different types of crawlers<\/a><\/li>\n<li><a href=\"#how-to-control-a-crawler\" style=\"color:#fff;text-decoration:none\">How to control a crawler<\/a><\/li>\n<li><a href=\"#crawlers-and-seo\" style=\"color:#fff;text-decoration:none\">Crawlers and SEO<\/a><\/li>\n<\/ol>\n<\/aside>\n<div class=\"taismo-main\">\n<h1 class=\"brand-claim\">What is a crawler?<\/h1>\n<p class=\"lead-def\">A <strong>crawler<\/strong>, also called a <strong>web crawler<\/strong>, is a software program used above all by search engines to browse the <strong>World Wide Web<\/strong> automatically and build an <strong>index<\/strong>. The name describes the way it works: the program moves from link to link through the web and reads one page after the other.<\/p>\n<figure><img decoding=\"async\" src=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/crawler.png\" width=\"1200\" height=\"675\" alt=\"Crawler explained: a web crawler browses websites automatically and builds the index of a search engine\" loading=\"lazy\"><\/figure>\n<p>While it works through websites, the crawler analyzes and indexes their content so that search engines can read it faster later on. It lays the groundwork for everything a search engine does afterwards.<\/p>\n<h2 id=\"how-does-a-web-crawler-work\">How does a web crawler work?<\/h2>\n<p>A crawler is programmed for repeating actions and therefore works <strong>automatically<\/strong>.<\/p>\n<p>The information it finds while crawling gets assigned to certain categories and is then indexed, which makes it available for retrieval and evaluation. Texts, images and videos are part of that analysis, and so are the links a page contains.<\/p>\n<h2 id=\"different-types-of-crawlers\">Different types of crawlers<\/h2>\n<p>There are many different crawlers. Well known examples are the <strong>Googlebot<\/strong> and the <strong>Bingbot<\/strong>, and Alexa Internet ran a bot of its own called <strong>ia_archiver<\/strong>.<\/p>\n<p>Crawlers can also carry out other tasks, such as <strong>data mining<\/strong>. The web is then searched for specific pieces of information, email addresses for example. <a href=\"https:\/\/taismo.de\/en\/what-is\/black-hat-seo\/\">Black hat SEOs<\/a> use this in part to send spam mail to the addresses they collect.<\/p>\n<p>Another field of work is <strong>plagiarism detection<\/strong>. That function matters above all when copyrighted material is involved.<\/p>\n<h2 id=\"how-to-control-a-crawler\">How to control a crawler<\/h2>\n<p>With <a href=\"https:\/\/taismo.de\/en\/what-is\/meta-title\/\">meta tags<\/a> you give the crawler instructions in the head area of a page. Among other things you control whether the crawler indexes a page or follows a link.<\/p>\n<p>This is what an instruction looks like that keeps the address out of the index and stops the crawler from following the links on it:<\/p>\n<pre><code>&lt;meta name=\"robots\" content=\"noindex,nofollow\"&gt;<\/code><\/pre>\n<p>The <strong>noindex<\/strong> part of it works exactly like the <a href=\"https:\/\/taismo.de\/en\/what-is\/noindex-attribute\/\">noindex attribute<\/a> on any other page. Instructions can be passed to the crawler through the <a href=\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/\">robots.txt<\/a> file as well.<\/p>\n<h2 id=\"crawlers-and-seo\">Crawlers and SEO<\/h2>\n<p>Part of <a href=\"https:\/\/taismo.de\/en\/what-is\/seo\/\">search engine optimization<\/a> is building a website so that the crawler can read it as completely as possible, so that all content ends up in the index of the search engine. A logical <a href=\"https:\/\/taismo.de\/en\/what-is\/url\/\">URL<\/a> structure helps here. On top of that, <a href=\"https:\/\/taismo.de\/en\/what-is\/backlink\/\">backlinks<\/a> and clear <a href=\"https:\/\/taismo.de\/en\/what-is\/internal-link\/\">internal linking<\/a> lead to the crawler visiting your website more often and staying longer.<\/p>\n<p>Whether crawlers actually get through your pages is one of the questions a technical <a href=\"https:\/\/taismo.de\/en\/seo-audit\/\">SEO audit<\/a> answers. The <a href=\"https:\/\/taismo.de\/en\/what-is\/\">SEO glossary<\/a> covers the terms around indexing that go with this one.<\/p>\n<\/p><\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>IKI Contents How does a web crawler work? Different types of crawlers How to control a crawler Crawlers and SEO What is a crawler? A crawler, also called a web crawler, is a software program used above all by search engines to browse the World Wide Web automatically and build an index. The name describes [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":7491,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_taismo_jsonld":"","taismo_jsonld":"{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/crawler\/#webpage\",\"url\":\"https:\/\/taismo.de\/en\/what-is\/crawler\/\",\"name\":\"What is a crawler?\",\"isPartOf\":{\"@id\":\"https:\/\/taismo.de\/#website\"},\"inLanguage\":\"en\",\"primaryImageOfPage\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/crawler\/#primaryimage\"},\"mainEntity\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/crawler\/#definedterm\"},\"breadcrumb\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/crawler\/#breadcrumb\"},\"datePublished\":\"2025-10-07T15:18:01+02:00\",\"dateModified\":\"2026-07-14T12:00:00+02:00\",\"speakable\":{\"@type\":\"SpeakableSpecification\",\"cssSelector\":[\".brand-claim\",\".lead-def\"]},\"author\":{\"@id\":\"https:\/\/taismo.de\/#person-dominik-breitbach\"},\"publisher\":{\"@id\":\"https:\/\/taismo.de\/#organization\"}},{\"@type\":\"ImageObject\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/crawler\/#primaryimage\",\"contentUrl\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/crawler.png\",\"url\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/crawler.png\",\"width\":1200,\"height\":675,\"caption\":\"Crawler: a web crawler browses websites automatically and builds the index of a search engine\"},{\"@type\":\"DefinedTerm\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/crawler\/#definedterm\",\"name\":\"crawler\",\"termCode\":\"crawler\",\"url\":\"https:\/\/taismo.de\/en\/what-is\/crawler\/\",\"inLanguage\":\"en\",\"inDefinedTermSet\":{\"@id\":\"https:\/\/taismo.de\/#seo-wiki\"},\"description\":\"A crawler, also called a web crawler, is a software program used above all by search engines to browse the World Wide Web automatically and build an index. While it works through websites, the crawler analyzes and indexes their content so that search engines can read it faster later on.\",\"author\":{\"@id\":\"https:\/\/taismo.de\/#person-dominik-breitbach\"},\"publisher\":{\"@id\":\"https:\/\/taismo.de\/#organization\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/crawler\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\/\/taismo.de\/en\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"SEO Glossary\",\"item\":\"https:\/\/taismo.de\/en\/what-is\/\"},{\"@type\":\"ListItem\",\"position\":3,\"name\":\"Crawler\"}]}]}","footnotes":""},"categories":[80],"tags":[85],"class_list":["post-7106","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-what-is","tag-c-en"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v28.1 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>What Is a Crawler? How Web Crawlers Work | taismo<\/title>\n<meta name=\"description\" content=\"A crawler reads the web and builds the search index. See how web crawlers work and how you steer them on your own site. Read the definition \ud83d\udd0d\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/taismo.de\/en\/what-is\/crawler\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"What Is a Crawler? How Web Crawlers Work | taismo\" \/>\n<meta property=\"og:description\" content=\"A crawler reads the web and builds the search index. See how web crawlers work and how you steer them on your own site. Read the definition \ud83d\udd0d\" \/>\n<meta property=\"og:url\" content=\"https:\/\/taismo.de\/en\/what-is\/crawler\/\" \/>\n<meta property=\"og:site_name\" content=\"taismo| Agentur f\u00fcr SEO &amp; Webdesign\" \/>\n<meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/TaismoOnlineMarketing\" \/>\n<meta property=\"article:published_time\" content=\"2025-10-07T13:18:01+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2026-07-27T21:14:20+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/crawler_featured.png\" \/>\n\t<meta property=\"og:image:width\" content=\"1200\" \/>\n\t<meta property=\"og:image:height\" content=\"675\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/png\" \/>\n<meta name=\"author\" content=\"Dominik Breitbach\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/\"},\"author\":{\"name\":\"Dominik Breitbach\",\"@id\":\"https:\\\/\\\/taismo.de\\\/#person-dominik-breitbach\"},\"headline\":\"crawler\",\"datePublished\":\"2025-10-07T13:18:01+00:00\",\"dateModified\":\"2026-07-27T21:14:20+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/\"},\"wordCount\":437,\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/07\\\/crawler_featured.png\",\"keywords\":[\"C\"],\"articleSection\":[\"SEO Glossary\"],\"inLanguage\":\"en-US\"},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/\",\"name\":\"What Is a Crawler? How Web Crawlers Work | taismo\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/07\\\/crawler_featured.png\",\"datePublished\":\"2025-10-07T13:18:01+00:00\",\"dateModified\":\"2026-07-27T21:14:20+00:00\",\"description\":\"A crawler reads the web and builds the search index. See how web crawlers work and how you steer them on your own site. Read the definition \ud83d\udd0d\",\"breadcrumb\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/#primaryimage\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/07\\\/crawler_featured.png\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/07\\\/crawler_featured.png\",\"width\":1200,\"height\":675,\"caption\":\"Crawler: A crawler, also called a web crawler, is a software program used above all by search engines to\u2026\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/crawler\\\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Startseite\",\"item\":\"https:\\\/\\\/taismo.de\\\/en\\\/search-marketing\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"crawler\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"name\":\"taismo| Agentur f\u00fcr SEO & Webdesign\",\"description\":\"\",\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"alternateName\":\"taismo\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/taismo.de\\\/en\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Organization\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\",\"name\":\"taismo SEO Agentur M\u00fcnchen\",\"alternateName\":\"taismo\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"logo\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"width\":512,\"height\":512,\"caption\":\"taismo SEO Agentur M\u00fcnchen\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\"},\"sameAs\":[\"https:\\\/\\\/www.facebook.com\\\/TaismoOnlineMarketing\",\"https:\\\/\\\/branchenbuch.portal.muenchen.de\\\/mhp\\\/1380360\\\/\",\"https:\\\/\\\/www.crunchbase.com\\\/organization\\\/taismo-gmbh\",\"https:\\\/\\\/de.trustpilot.com\\\/review\\\/taismo.de\",\"https:\\\/\\\/www.linkedin.com\\\/company\\\/taismo\\\/\",\"https:\\\/\\\/www.instagram.com\\\/taismo.de\",\"https:\\\/\\\/onlinemarketing.de\\\/dienstleister\\\/online-marketing-agentur\\\/kirchheim-bei-muenchen\\\/taismo\",\"https:\\\/\\\/www.agenturtipp.de\\\/agentur\\\/taismo-online-marketing\\\/\",\"https:\\\/\\\/startupvalley.news\\\/de\\\/taismo-online-marketing-agentur\\\/\",\"https:\\\/\\\/www.wlw.de\\\/de\\\/firma\\\/taismo-gmbh-1918135\",\"https:\\\/\\\/www.youtube.com\\\/@taismo8632\",\"https:\\\/\\\/techbehemoths.com\\\/company\\\/taismo-gmbh\",\"https:\\\/\\\/www.kununu.com\\\/de\\\/taismo1\",\"https:\\\/\\\/www.provenexpert.com\\\/taismo1\\\/\",\"https:\\\/\\\/www.provenemployer.com\\\/p\\\/taismo-employer\\\/\",\"https:\\\/\\\/www.trustedshops.de\\\/bewertung\\\/taismo-de\",\"https:\\\/\\\/www.kennstdueinen.de\\\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html\",\"https:\\\/\\\/www.ibusiness.de\\\/dienstleister\\\/jb\\\/5920834fb726.html\",\"https:\\\/\\\/www.seo-united.de\\\/seo-agenturen\\\/taismo-3345\\\/\"]}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"What Is a Crawler? How Web Crawlers Work | taismo","description":"A crawler reads the web and builds the search index. See how web crawlers work and how you steer them on your own site. Read the definition \ud83d\udd0d","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/taismo.de\/en\/what-is\/crawler\/","og_locale":"en_US","og_type":"article","og_title":"What Is a Crawler? How Web Crawlers Work | taismo","og_description":"A crawler reads the web and builds the search index. See how web crawlers work and how you steer them on your own site. Read the definition \ud83d\udd0d","og_url":"https:\/\/taismo.de\/en\/what-is\/crawler\/","og_site_name":"taismo| Agentur f\u00fcr SEO &amp; Webdesign","article_publisher":"https:\/\/www.facebook.com\/TaismoOnlineMarketing","article_published_time":"2025-10-07T13:18:01+00:00","article_modified_time":"2026-07-27T21:14:20+00:00","og_image":[{"width":1200,"height":675,"url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/crawler_featured.png","type":"image\/png"}],"author":"Dominik Breitbach","twitter_card":"summary_large_image","schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/#article","isPartOf":{"@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/"},"author":{"name":"Dominik Breitbach","@id":"https:\/\/taismo.de\/#person-dominik-breitbach"},"headline":"crawler","datePublished":"2025-10-07T13:18:01+00:00","dateModified":"2026-07-27T21:14:20+00:00","mainEntityOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/"},"wordCount":437,"publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/crawler_featured.png","keywords":["C"],"articleSection":["SEO Glossary"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/","url":"https:\/\/taismo.de\/en\/what-is\/crawler\/","name":"What Is a Crawler? How Web Crawlers Work | taismo","isPartOf":{"@id":"https:\/\/taismo.de\/en\/#website"},"primaryImageOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/#primaryimage"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/crawler_featured.png","datePublished":"2025-10-07T13:18:01+00:00","dateModified":"2026-07-27T21:14:20+00:00","description":"A crawler reads the web and builds the search index. See how web crawlers work and how you steer them on your own site. Read the definition \ud83d\udd0d","breadcrumb":{"@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/taismo.de\/en\/what-is\/crawler\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/#primaryimage","url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/crawler_featured.png","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/07\/crawler_featured.png","width":1200,"height":675,"caption":"Crawler: A crawler, also called a web crawler, is a software program used above all by search engines to\u2026"},{"@type":"BreadcrumbList","@id":"https:\/\/taismo.de\/en\/what-is\/crawler\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Startseite","item":"https:\/\/taismo.de\/en\/search-marketing\/"},{"@type":"ListItem","position":2,"name":"crawler"}]},{"@type":"WebSite","@id":"https:\/\/taismo.de\/en\/#website","url":"https:\/\/taismo.de\/en\/","name":"taismo| Agentur f\u00fcr SEO & Webdesign","description":"","publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"alternateName":"taismo","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/taismo.de\/en\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/taismo.de\/en\/#organization","name":"taismo SEO Agentur M\u00fcnchen","alternateName":"taismo","url":"https:\/\/taismo.de\/en\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/","url":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","width":512,"height":512,"caption":"taismo SEO Agentur M\u00fcnchen"},"image":{"@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/TaismoOnlineMarketing","https:\/\/branchenbuch.portal.muenchen.de\/mhp\/1380360\/","https:\/\/www.crunchbase.com\/organization\/taismo-gmbh","https:\/\/de.trustpilot.com\/review\/taismo.de","https:\/\/www.linkedin.com\/company\/taismo\/","https:\/\/www.instagram.com\/taismo.de","https:\/\/onlinemarketing.de\/dienstleister\/online-marketing-agentur\/kirchheim-bei-muenchen\/taismo","https:\/\/www.agenturtipp.de\/agentur\/taismo-online-marketing\/","https:\/\/startupvalley.news\/de\/taismo-online-marketing-agentur\/","https:\/\/www.wlw.de\/de\/firma\/taismo-gmbh-1918135","https:\/\/www.youtube.com\/@taismo8632","https:\/\/techbehemoths.com\/company\/taismo-gmbh","https:\/\/www.kununu.com\/de\/taismo1","https:\/\/www.provenexpert.com\/taismo1\/","https:\/\/www.provenemployer.com\/p\/taismo-employer\/","https:\/\/www.trustedshops.de\/bewertung\/taismo-de","https:\/\/www.kennstdueinen.de\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html","https:\/\/www.ibusiness.de\/dienstleister\/jb\/5920834fb726.html","https:\/\/www.seo-united.de\/seo-agenturen\/taismo-3345\/"]}]}},"_links":{"self":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/7106","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/comments?post=7106"}],"version-history":[{"count":4,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/7106\/revisions"}],"predecessor-version":[{"id":7164,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/7106\/revisions\/7164"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media\/7491"}],"wp:attachment":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media?parent=7106"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/categories?post=7106"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/tags?post=7106"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}