{"id":8031,"date":"2026-10-05T07:30:00","date_gmt":"2026-10-05T05:30:00","guid":{"rendered":"https:\/\/taismo.de\/?p=8031"},"modified":"2026-09-24T14:49:51","modified_gmt":"2026-09-24T12:49:51","slug":"common-crawl","status":"publish","type":"post","link":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/","title":{"rendered":"Common Crawl"},"content":{"rendered":"<style>\n.taismo-art{display:grid;grid-template-columns:320px minmax(0,1fr);gap:36px;align-items:start}\n.taismo-art .toc-sticky{position:sticky;top:80px}\n.taismo-art .toc-sticky ol li{margin-bottom:9px}\n.taismo-art .toc-sticky ol a:hover{color:var(--toc-hover) !important}\n.taismo-art .taismo-main h2{scroll-margin-top:90px;margin-top:38px}\n.taismo-art .lead-def{font-size:1.15em;line-height:1.6;background:#f7f2ec;border-left:4px solid #ED8924;padding:16px 20px;border-radius:8px}\n.taismo-art table{border-collapse:collapse;width:100%;margin:18px 0}\n.taismo-art th,.taismo-art td{border:1px solid #d9e0df;padding:9px 12px;text-align:left;font-size:0.95em}\n.taismo-art th{background:#072328;color:#fff}\n.taismo-art figure{margin:26px 0}\n.taismo-art figcaption{font-size:0.9em;color:#55605f;margin-top:8px}\n.taismo-art pre{background:#072328;color:#f5f5f5;padding:16px 18px;border-radius:10px;overflow-x:auto;font-size:0.86em;line-height:1.5}\n@media(max-width:900px){.taismo-art{grid-template-columns:1fr}.taismo-art .toc-sticky{position:static}}\n<\/style>\n<div class=\"taismo-art\">\n<aside class=\"toc-sticky\" style=\"--toc-hover:#ED8924;background:#072328;border-radius:14px;padding:22px 24px\">\n<div style=\"text-align:center;margin-bottom:14px\"><svg fill=\"#ffffff\" role=\"img\" aria-label=\"taismo Wiki\" style=\"width:200px;max-width:80%;height:auto;display:inline-block\" id=\"Ebene_1\" data-name=\"Ebene 1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 209.04 52.32\"> <g id=\"Layer_2\" data-name=\"Layer 2\"> <g id=\"wikipedia\"> <path d=\"M97.08,27.77h-17.76c-.77,0-1.4.63-1.4,1.4v17.76c0,.77.63,1.4,1.4,1.4h17.76c.77,0,1.4-.63,1.4-1.4v-17.76c0-.77-.63-1.4-1.4-1.4ZM97.22,33.22c0,.25-.21.46-.46.46h-.67c-.07.03-.13.07-.19.11-.19.18-.33.41-.4.66,0,0,0,.01,0,.02-.03.07-2.9,7.29-3.87,9.67v.02c-.09.17-.19.33-.31.48-.15.2-.38.32-.63.33-.56,0-.84-.63-.93-.84-.28-.62-.79-1.83-1.51-3.57-.55,1.24-1.25,2.8-1.64,3.59h0c-.19.36-.49.83-.97.83-.56,0-.82-.63-.9-.84h0c-.49-1.25-1.23-2.99-1.95-4.68-.79-1.83-2.39-5.73-2.43-5.78h-.72c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h4.13c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.82s.03.09.05.14c.59,1.61,2.36,5.75,3.1,7.47l1.28-2.81c-.15-.36-2.12-4.68-2.2-4.8h-.4c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.98c.25,0,.46.21.46.46v.58c0,.25-.21.46-.46.46h-.1c.06.14.94,2.12.94,2.12,0,0,.93-2.06.95-2.12h-.39c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h2.81c.25,0,.46.21.46.46v.58c0,.25-.21.45-.46.46h-.43s-2.06,4.08-2.06,4.08l1.56,3.51s2.9-7.59,2.9-7.6h-.96c-.25,0-.46-.21-.46-.46v-.58c0-.25.21-.45.46-.46h3.78c.25,0,.46.2.46.46v.58Z\"\/> <\/g> <\/g> <text font-family=\"Arial,Helvetica,sans-serif\" font-size=\"27.79\" font-weight=\"800\" transform=\"translate(99.44 47.64) scale(.99 1)\"><tspan x=\"0\" y=\"0\">IKI<\/tspan><\/text> <g id=\"Gruppe_796\" data-name=\"Gruppe 796\"> <path id=\"Pfad_1293\" data-name=\"Pfad 1293\" d=\"M6.92,16.77h-3.56v-6.56h3.66V3.37h8.04v6.84h5.69v6.56h-5.58v8.97c-.08,1.05.13,2.1.6,3.04.4.6,1.24.9,2.51.9.54,0,1.01,0,1.39-.03.41-.02.82-.07,1.23-.13v6.35c-.75.19-1.52.31-2.3.36-1.02.09-1.95.13-2.79.13-1.41.04-2.81-.17-4.15-.6-1.07-.36-2.02-1.01-2.76-1.86-.76-.95-1.29-2.06-1.54-3.25-.34-1.55-.49-3.14-.46-4.73v-9.14h0Z\"\/> <path id=\"Pfad_1294\" data-name=\"Pfad 1294\" d=\"M44.43,35.91h-7.06l-.44-1.97c-.68.72-1.48,1.34-2.35,1.8-.95.46-2.01.69-3.07.66-.99,0-1.98-.1-2.95-.33-.92-.21-1.78-.63-2.51-1.23-.77-.66-1.37-1.49-1.75-2.43-.48-1.28-.71-2.63-.66-3.99v-1.42c-.04-1.17.2-2.33.71-3.39.44-.87,1.09-1.63,1.89-2.19.82-.56,1.74-.96,2.71-1.18,1.06-.24,2.14-.36,3.23-.36.87,0,1.72.06,2.57.22.72.14,1.41.36,2.08.66v-.93c0-.54-.07-1.07-.22-1.59-.16-.48-.46-.91-.87-1.2-.55-.38-1.18-.63-1.83-.77-1-.2-2.01-.29-3.04-.28-.91,0-1.81.09-2.71.25-.97.16-1.8.32-2.49.46v-6.12c.48-.13.96-.22,1.45-.28.57-.07,1.16-.14,1.78-.22.62-.07,1.22-.13,1.78-.16.57-.04,1.04-.05,1.45-.05,2.1-.05,4.18.19,6.21.71,1.4.37,2.69,1.08,3.74,2.08.92.94,1.57,2.13,1.86,3.42.34,1.54.51,3.1.49,4.68v15.15h.01,0ZM33.71,30.83c1.63.12,3.04-1.1,3.16-2.72v-2.58c-.77-.58-1.72-.87-2.68-.82-.85-.04-1.69.16-2.43.57-.6.38-.9,1.16-.9,2.33v.38c-.07.78.16,1.55.66,2.16.6.51,1.39.76,2.19.69\"\/> <rect id=\"Rechteck_760\" data-name=\"Rechteck 760\" x=\"49.46\" y=\"10.21\" width=\"8.26\" height=\"25.71\"\/> <path d=\"M131.36,22.82v-.42c-.06-1.6.12-3.21.52-4.76.35-1.1,1.23-1.64,2.65-1.64.59-.03,1.16.11,1.66.41.43.29.75.73.9,1.23.2.66.32,1.34.36,2.02.05.81.08,1.72.08,2.74v.42h8.38v-1.01c.04-1.92-.23-3.83-.82-5.66-.48-1.43-1.27-2.73-2.33-3.8-1.02-.99-2.27-1.73-3.64-2.13-1.53-.45-3.12-.67-4.71-.66-1.57-.01-3.12.2-4.62.66-1.36.41-2.6,1.14-3.61,2.13-1.06,1.07-1.86,2.37-2.35,3.8-.61,1.82-.89,3.74-.85,5.66v1.01h8.38Z\"\/> <path d=\"M93,22.82v-2.38c-.06-.88.16-1.76.63-2.51.39-.57,1.04-.89,1.72-.87.26,0,.51.03.77.08.26.06.51.19.71.38.25.24.43.54.52.87.15.51.21,1.03.19,1.56v2.88h8.2v-2.7c-.05-.8.16-1.6.6-2.27.4-.52,1.03-.82,1.69-.79.54,0,1.07.17,1.51.49.45.33.69,1.13.69,2.41v2.86h8.2v-4.99c.03-1.32-.17-2.64-.6-3.89-.34-.96-.9-1.83-1.64-2.54-.69-.63-1.51-1.11-2.41-1.39-.92-.3-1.88-.44-2.84-.44-1.16-.02-2.3.21-3.36.69-1.09.55-2.07,1.3-2.87,2.22-.57-.95-1.38-1.72-2.35-2.24-1.16-.49-2.41-.72-3.66-.66-1.18-.04-2.35.22-3.39.76-.99.57-1.88,1.31-2.63,2.19l-.54-2.3h-7.34v12.61h8.2Z\"\/> <path d=\"M73.21,26.15c0-1.84,1.49-3.33,3.33-3.33h2.18c-.87-.85-1.94-1.47-3.12-1.8l-1.75-.6c-.73-.25-1.35-.46-1.86-.63-.43-.13-.85-.3-1.26-.49-.28-.13-.52-.33-.71-.57-.16-.28-.24-.6-.22-.93,0-.22.04-.45.13-.66.13-.25.34-.46.6-.57.43-.2.89-.34,1.36-.41.81-.13,1.62-.18,2.43-.16.92,0,1.85.07,2.76.22.82.11,1.63.28,2.43.49v-6.12c-.48-.13-.95-.22-1.45-.28-.56-.07-1.16-.14-1.78-.22-.61-.07-1.21-.13-1.78-.16s-1.05-.05-1.45-.05c-3.72,0-6.51.75-8.37,2.27-1.78,1.35-2.82,3.48-2.79,5.71-.03,1.07.11,2.13.41,3.15.24.76.64,1.46,1.18,2.05.52.56,1.15,1.01,1.83,1.34.79.38,1.6.72,2.43,1.01l1.26.44c.83.28,1.63.61,2.41,1.01.47.28.75.8.71,1.34.07.72-.35,1.4-1.04,1.64-1.23.31-2.51.45-3.77.38-.95,0-1.9-.09-2.84-.25-1.06-.16-1.93-.31-2.63-.46v6.12c.5.13,1,.22,1.51.28.6.07,1.23.14,1.89.22.66.07,1.3.13,1.92.16.62.03,1.13.05,1.53.05,1.5.03,3.01-.1,4.49-.36v-9.83Z\"\/> <\/g> <g id=\"e\"> <path d=\"M150.81,38.2c-.26-1.16-1.38-2.22-2.54-2.45-.71-.14-2.69-.32-3.99-.3-1.31-.01-3.28.16-3.99.3-1.16.23-2.29,1.29-2.54,2.45-.15.7-.29,1.66-.29,2.97,0,1.31.15,2.27.29,2.97.26,1.16,1.38,2.22,2.54,2.45.38.08,1.14.16,1.95.22.09.15.19.3.3.46.43.66.77,1.11,1.02,1.43.42.53,1.03.53,1.45,0,.25-.32.58-.78,1.02-1.43.11-.16.21-.32.3-.46.82-.06,1.57-.15,1.95-.22,1.16-.23,2.29-1.29,2.54-2.45.15-.7.29-1.66.29-2.97,0-1.31-.15-2.27-.29-2.97h0ZM145.03,45.13c-.03.16-.18.31-.34.34-.1.02-.23.03-.42.03-.18,0-.32-.02-.42-.03-.16-.03-.31-.18-.34-.34-.02-.1-.03-.23-.03-.42,0-.18.02-.32.03-.42.03-.16.18-.31.34-.34.1-.02.23-.03.42-.03.18,0,.32.02.42.03.16.03.31.18.34.34.02.1.03.23.03.42,0,.18-.02.32-.03.42ZM146.65,40.76c-.21.29-.51.56-.97.84-.58.36-.67.5-.67.79v.11c0,.41-.33.74-.74.74s-.74-.33-.74-.74v-.26c0-.34.08-.63.26-.88.17-.24.42-.46.8-.68.69-.41.82-.63.82-.94,0-.38-.32-.63-.83-.63-.55,0-.91.25-1.06.74-.1.32-.4.54-.75.54-.02,0-.04,0-.06,0-.24-.02-.46-.14-.6-.35-.14-.2-.17-.45-.1-.68.17-.52.49-.96.93-1.26.45-.3,1.01-.47,1.63-.47s1.24.21,1.69.58c.45.38.71.91.71,1.49,0,.4-.1.74-.32,1.04h0Z\"\/> <\/g> <g id=\"e-2\" data-name=\"e\"> <path d=\"M156.19,28.79c-.17-.75-.89-1.44-1.64-1.58-.46-.09-1.73-.2-2.58-.2-.84,0-2.12.1-2.58.2-.75.15-1.48.83-1.64,1.58-.09.45-.19,1.07-.19,1.92,0,.84.09,1.46.19,1.92.17.75.89,1.44,1.64,1.58.25.05.73.11,1.26.14.06.09.12.19.19.3.28.42.49.72.66.93.27.34.66.34.93,0,.16-.21.38-.5.66-.93.07-.11.13-.21.19-.3.53-.04,1.01-.09,1.26-.14.75-.15,1.48-.83,1.64-1.58.09-.45.19-1.07.19-1.92,0-.84-.09-1.46-.19-1.92h0ZM152.46,33.27c-.02.1-.11.2-.22.22-.06.01-.15.02-.27.02-.12,0-.2,0-.27-.02-.1-.02-.2-.11-.22-.22-.01-.06-.02-.15-.02-.27,0-.12,0-.2.02-.27.02-.1.11-.2.22-.22.06-.01.15-.02.27-.02.12,0,.2,0,.27.02.1.02.2.11.22.22.01.06.02.15.02.27,0,.12,0,.2-.02.27ZM153.51,30.45c-.13.19-.33.36-.63.54-.37.23-.43.32-.43.51v.07c0,.26-.21.48-.48.48s-.48-.21-.48-.48v-.17c0-.22.05-.4.17-.57.11-.16.27-.3.51-.44.45-.26.53-.41.53-.61,0-.25-.21-.4-.53-.4-.35,0-.59.16-.69.48-.07.21-.26.35-.48.35-.01,0-.02,0-.04,0-.16-.01-.3-.09-.39-.22-.09-.13-.11-.29-.06-.44.11-.34.32-.62.6-.81.29-.2.65-.3,1.05-.3s.8.13,1.09.37c.29.25.46.59.46.96,0,.26-.07.48-.21.67h0Z\"\/> <\/g> <g id=\"e-3\" data-name=\"e\"> <path d=\"M144.91,28.64c-.1-.44-.52-.84-.96-.92-.27-.05-1.01-.12-1.5-.11-.49,0-1.24.06-1.5.11-.44.08-.86.49-.96.92-.06.27-.11.63-.11,1.12,0,.49.05.85.11,1.12.1.44.52.84.96.92.14.03.43.06.74.08.03.05.07.11.11.17.16.25.29.42.38.54.16.2.39.2.54,0,.09-.12.22-.29.38-.54.04-.06.08-.12.11-.17.31-.02.59-.06.74-.08.44-.08.86-.49.96-.92.06-.27.11-.63.11-1.12,0-.49-.05-.85-.11-1.12h0ZM142.73,31.26c-.01.06-.07.12-.13.13-.04,0-.09.01-.16.01-.07,0-.12,0-.16-.01-.06-.01-.12-.07-.13-.13,0-.04-.01-.09-.01-.16,0-.07,0-.12.01-.16.01-.06.07-.12.13-.13.04,0,.09-.01.16-.01.07,0,.12,0,.16.01.06.01.12.07.13.13,0,.04.01.09.01.16,0,.07,0,.12-.01.16ZM143.34,29.61c-.08.11-.19.21-.37.32-.22.13-.25.19-.25.3v.04c0,.15-.12.28-.28.28s-.28-.12-.28-.28v-.1c0-.13.03-.24.1-.33.06-.09.16-.17.3-.26.26-.15.31-.24.31-.35,0-.14-.12-.24-.31-.24-.21,0-.34.09-.4.28-.04.12-.15.2-.28.2,0,0-.01,0-.02,0-.09,0-.17-.05-.23-.13-.05-.08-.06-.17-.04-.25.06-.2.18-.36.35-.47.17-.11.38-.18.61-.18s.47.08.63.22c.17.14.27.34.27.56,0,.15-.04.28-.12.39h0Z\"\/> <\/g> <\/svg><\/div>\n<h3 style=\"text-align:center;color:#fff;margin:0 0 14px;font-size:1.1em\">Contents<\/h3>\n<ol style=\"margin:0;padding-left:1.2em;line-height:1.5;font-weight:600;color:#fff;font-size:0.9em\">\n<li><a href=\"#term-profile\" style=\"color:#fff;text-decoration:none\">Term profile at a glance<\/a><\/li>\n<li><a href=\"#how-common-crawl-works\" style=\"color:#fff;text-decoration:none\">How does Common Crawl work?<\/a><\/li>\n<li><a href=\"#language-models\" style=\"color:#fff;text-decoration:none\">Which language models use Common Crawl?<\/a><\/li>\n<li><a href=\"#old-content\" style=\"color:#fff;text-decoration:none\">Why does old content live on in AI answers through Common Crawl?<\/a><\/li>\n<li><a href=\"#check-your-website\" style=\"color:#fff;text-decoration:none\">How do you check whether your website is included in Common Crawl?<\/a><\/li>\n<li><a href=\"#ccbot-robots-txt\" style=\"color:#fff;text-decoration:none\">How do you control CCBot with robots.txt?<\/a><\/li>\n<li><a href=\"#block-or-allow\" style=\"color:#fff;text-decoration:none\">Should you block or allow CCBot?<\/a><\/li>\n<li><a href=\"#faq\" style=\"color:#fff;text-decoration:none\">Frequently asked questions about Common Crawl<\/a><\/li>\n<\/ol>\n<\/aside>\n<div class=\"taismo-main\">\n<h1 class=\"brand-claim\">What is Common Crawl?<\/h1>\n<p class=\"lead-def\"><strong>Common Crawl<\/strong> is a freely accessible archive of web pages that the nonprofit Common Crawl Foundation has been building since 2008 with its crawler CCBot, adding a new crawl roughly once a month. Each crawl contains raw data, metadata, and extracted text from around two billion pages and is available for download at no cost. This makes Common Crawl one of the most important sources from which large language models draw their knowledge of the web.<\/p>\n<figure>\n      <img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/common-crawl-webarchiv-ccbot-sprachmodelle.webp\" alt=\"Common Crawl explained: the open web archive and the crawler CCBot as a data source for language models, taismo SEO glossary\" title=\"Common Crawl: the open web archive behind many language models\" width=\"1200\" height=\"675\" style=\"width:100%;height:auto;border-radius:12px\" \/><br \/>\n    <\/figure>\n<h2 id=\"term-profile\">Term profile at a glance<\/h2>\n<table>\n<tr>\n<th>Attribute<\/th>\n<th>Details<\/th>\n<\/tr>\n<tr>\n<td>Part of speech<\/td>\n<td>Proper noun, used without an article<\/td>\n<\/tr>\n<tr>\n<td>Pronunciation<\/td>\n<td>\u02c8k\u0252m\u0259n \u02c8kr\u0254\u02d0l<\/td>\n<\/tr>\n<tr>\n<td>Operator<\/td>\n<td>Common Crawl Foundation, a US nonprofit organization founded in 2007<\/td>\n<\/tr>\n<tr>\n<td>Crawler<\/td>\n<td>CCBot, user agent <code>CCBot\/2.0 (https:\/\/commoncrawl.org\/faq\/)<\/code><\/td>\n<\/tr>\n<tr>\n<td>Current crawl<\/td>\n<td>CC-MAIN-2026-39: 2.17 billion pages from 40.4 million hosts, captured from September 4 to 17, 2026<\/td>\n<\/tr>\n<tr>\n<td>File formats<\/td>\n<td>WARC (raw data), WAT (metadata and links), WET (plain text)<\/td>\n<\/tr>\n<tr>\n<td>Related terms<\/td>\n<td>CCBot, AI crawler, training data, robots.txt, web graph<\/td>\n<\/tr>\n<\/table>\n<h2 id=\"how-common-crawl-works\">How does Common Crawl work?<\/h2>\n<p><strong>Common Crawl works in 4 steps: The crawler selects candidates, fetches the pages, archives them in 3 file formats, and publishes the finished crawl as an open data set.<\/strong> A crawl runs for about two weeks; the September 2026 crawl ran from September 4 to 17.<\/p>\n<ol>\n<li><strong>Selecting candidates:<\/strong> Common Crawl computes the harmonic centrality of every domain from its own link graph. The value measures how close a domain sits to all other domains through links. Well-linked domains receive more crawl budget and end up in the archive with more pages.<\/li>\n<li><strong>Fetching pages:<\/strong> The <a href=\"https:\/\/taismo.de\/en\/what-is\/crawler\/\">crawler<\/a> CCBot is based on Apache Nutch. It reads a domain&#8217;s robots.txt first, follows up to four redirects, and uses the sitemap listed there. CCBot does not execute JavaScript and does not set cookies.<\/li>\n<li><strong>Archiving:<\/strong> Every fetched page is stored as a WARC file with the complete HTTP response, as a WAT file with metadata and links, and as a WET file with the plain text. Common Crawl also archives the robots.txt files and the error responses of each crawl.<\/li>\n<li><strong>Publishing:<\/strong> The finished crawl is hosted in the public data sets of Amazon Web Services and at data.commoncrawl.org. Access is free, and a URL index makes individual pages searchable.<\/li>\n<\/ol>\n<p>Common Crawl captures a sample of the web. The foundation itself describes its data set as a randomly selected subset of each website. For website owners, this means: Which pages end up in the archive changes from crawl to crawl.<\/p>\n<p>Content that is loaded later via JavaScript is missing from the archive. This is the same hurdle that <a href=\"https:\/\/taismo.de\/en\/what-is\/javascript-seo\/\">JavaScript SEO<\/a> describes for search engines; with CCBot, there is no rendering service that resolves it later.<\/p>\n<figure>\n      <a href=\"https:\/\/taismo.de\/en\/geo\/\" title=\"GEO and AI visibility by taismo\"><svg xmlns=\"http:\/\/www.w3.org\/2000\/svg\" viewBox=\"0 0 820 320\" role=\"img\" aria-labelledby=\"abb1t abb1d\" style=\"width:100%;height:auto;max-width:820px;font-family:Arial,Helvetica,sans-serif\"><title id=\"abb1t\">How a web page travels through Common Crawl into a language model<\/title><desc id=\"abb1d\">Five boxes in a row show the data path: websites, the crawler CCBot, the Common Crawl archive with the formats WARC, WAT, and WET, filtered data sets such as C4, Pile-CC, and FineWeb, and finally the training of a language model. The CCBot box is highlighted in orange because a website owner&#8217;s robots.txt only takes effect at this point, and only on future crawls.<\/desc><rect x=\"0\" y=\"0\" width=\"820\" height=\"320\" fill=\"#ffffff\"\/><g opacity=\"0.08\" transform=\"translate(668,196) scale(0.115)\"><path fill=\"#072328\" d=\"M1122.08,481.04l-132.61-85.2h-.19c-2.19-1.43-3.97-3.41-5.16-5.74-2.21-4.37-3.05-9.3-2.42-14.16.76-8.35,4.11-36.09,7.27-61.66,1.56-12.75,3.19-25.06,4.24-34.12s1.88-14.92,1.88-14.95c.16-1.49-.91-2.83-2.4-2.99-.82-.09-1.64.2-2.23.79l-125.88,131.4-8.04,8.39-19.55,17.03c-15.82,12.94-32.67,24.58-50.38,34.79-13.56,10.1-30.33,14.95-47.19,13.65-32.06-1.03-63.83-6.46-94.41-16.13-59.13-27.3-123.67-40.89-188.79-39.76l-154,13.68s-38.52,7.46-47.22-19.9c-17.4-52.51-40.3-103.03-68.33-150.72-18.96-28.44-28.95-61.92-28.7-96.1,0,0,6.22-116.55,68.06-133.32,0,0,53.12-19.63,96.05,36.25,0,0,25.03,35.71,80.37,21.79,0,0,111.05-30.87,214.33,25.55,0,0,84.14,40.28,141.09,24.21,0,0,29.25-6.19,42.36-27.71,0,0,17.4-27.71,50.17-18.85,0,0,32.5,6.19,32.23,52.05,0,0,1.61,54.47-38.13,89.39,0,0-14.45,16.6-1.34,32.13l68.6,79.57s10.16,12.04,26.11,12.31l160.06.54s6.26-.27,6.62,3.05c.36,3.32-2.87,4.21-2.87,4.21l-52.7,18.4s-5.72,1.61-5.9,6.01c-.18,4.39,1.79,5.83,3.94,6.9l70.75,33.29s3.22,1.61,2.68,4.75c-.54,3.14-4.66,3.14-4.66,3.14l-116.29,14.83s-12.13,1.7-15.97,10.83c-3.85,9.13-1.7,15.61-1.7,15.61l63.5,142.42s1.79,4.75-1.34,6.19-5.19-.98-5.19-.98l-113.7-128.04s-6.44-8.5-16.06-6.63c-9.62,1.88-11.14,11.55-11.14,11.55l-38.94,168.6s-1.16,5.46-4.66,5.46-4.3-4.03-4.3-4.03l-30.71-165.02s-1.61-11.1-14.72-11.64c-13.11-.54-16.24,9.67-16.24,9.67l-53.06,132.61s-2.06,5.19-5.55,4.12c-3.5-1.07-2.15-6.72-2.15-6.72l16.6-166.36s1.7-12.22-10.34-14.99c-12.04-2.78-17.32,7.17-17.32,7.17l-84.5,126.25s-3.32,4.48-6.72,2.33.36-6.99.36-6.99l53.87-152.24s3.14-10.61-6.9-15.44c-10.03-4.84-16.6,3.14-16.6,3.14l-120.32,103.15s-4.66,3.5-6.63.36c-1.97-3.14,1.7-6.63,1.7-6.63l94.85-136.73s5.28-8.86-3.14-15.71c-8.41-6.85-15.62-1.16-15.62-1.16l-145.64,72.71s-5.1,2.15-6.45-.63c-1.34-2.78,2.51-5.46,2.51-5.46l129.03-107.63s7.35-6.63.72-14.9c-6.63-8.28-15.17-4.66-15.17-4.66l-160.6,44.29s-5.37.81-6.09-2.24,3.14-4.84,3.14-4.84l154.29-79.16s8.15-4.03,4.21-13.29c-3.94-9.27-14.19-8.19-14.19-8.19l-166.45,6.09s-5.19-.36-5.19-3.59,4.48-3.94,4.48-3.94l169.14-42.09s8.86-2.24,7.53-11.91c-1.34-9.67-11.55-11.28-11.55-11.28l-164.03-32.32s-4.84-1.34-4.21-4.48c.63-3.14,5.55-2.51,5.55-2.51l171.29,5.55s9.98.36,11.28-9.4c1.3-9.76-7.98-13.65-7.98-13.65\"\/><\/g><text x=\"8\" y=\"36\" fill=\"#072328\" font-size=\"16\" font-weight=\"bold\">From website to language model<\/text><rect x=\"8\" y=\"60\" width=\"140\" height=\"86\" rx=\"12\" fill=\"#ffffff\" stroke=\"#072328\" stroke-width=\"2\"\/><text x=\"78\" y=\"90\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"15\" font-weight=\"bold\">Websites<\/text><text x=\"78\" y=\"113\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"12\">40.4 million hosts<\/text><text x=\"78\" y=\"130\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"12\">in crawl 2026-39<\/text><path d=\"M151 103 L164 103\" stroke=\"#ED8924\" stroke-width=\"3\"\/><path d=\"M172 103 L161 97 L161 109 Z\" fill=\"#ED8924\"\/><rect x=\"174\" y=\"60\" width=\"140\" height=\"86\" rx=\"12\" fill=\"#ED8924\" stroke=\"#ED8924\" stroke-width=\"2\"\/><text x=\"244\" y=\"90\" text-anchor=\"middle\" fill=\"#ffffff\" font-size=\"15\" font-weight=\"bold\">CCBot<\/text><text x=\"244\" y=\"113\" text-anchor=\"middle\" fill=\"#ffffff\" font-size=\"12\">checks robots.txt,<\/text><text x=\"244\" y=\"130\" text-anchor=\"middle\" fill=\"#ffffff\" font-size=\"12\">no JavaScript<\/text><path d=\"M317 103 L330 103\" stroke=\"#ED8924\" stroke-width=\"3\"\/><path d=\"M338 103 L327 97 L327 109 Z\" fill=\"#ED8924\"\/><rect x=\"340\" y=\"60\" width=\"140\" height=\"86\" rx=\"12\" fill=\"#ffffff\" stroke=\"#072328\" stroke-width=\"2\"\/><text x=\"410\" y=\"90\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"15\" font-weight=\"bold\">Archive<\/text><text x=\"410\" y=\"113\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"12\">WARC \u00b7 WAT \u00b7 WET<\/text><text x=\"410\" y=\"130\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"12\">2.17 billion pages<\/text><path d=\"M483 103 L496 103\" stroke=\"#ED8924\" stroke-width=\"3\"\/><path d=\"M504 103 L493 97 L493 109 Z\" fill=\"#ED8924\"\/><rect x=\"506\" y=\"60\" width=\"140\" height=\"86\" rx=\"12\" fill=\"#ffffff\" stroke=\"#072328\" stroke-width=\"2\"\/><text x=\"576\" y=\"90\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"15\" font-weight=\"bold\">Filtered data sets<\/text><text x=\"576\" y=\"113\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"12\">C4 \u00b7 Pile-CC<\/text><text x=\"576\" y=\"130\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"12\">FineWeb<\/text><path d=\"M649 103 L662 103\" stroke=\"#ED8924\" stroke-width=\"3\"\/><path d=\"M670 103 L659 97 L659 109 Z\" fill=\"#ED8924\"\/><rect x=\"672\" y=\"60\" width=\"140\" height=\"86\" rx=\"12\" fill=\"#ffffff\" stroke=\"#072328\" stroke-width=\"2\"\/><text x=\"742\" y=\"90\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"15\" font-weight=\"bold\">Language model<\/text><text x=\"742\" y=\"113\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"12\">Trained on<\/text><text x=\"742\" y=\"130\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"12\">the snapshot<\/text><path d=\"M244 150 L244 178\" stroke=\"#ED8924\" stroke-width=\"2\"\/><rect x=\"40\" y=\"182\" width=\"620\" height=\"70\" rx=\"12\" fill=\"#fdf1e3\"\/><text x=\"350\" y=\"210\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"14\" font-weight=\"bold\">Your robots.txt only applies when CCBot fetches.<\/text><text x=\"350\" y=\"233\" text-anchor=\"middle\" fill=\"#072328\" font-size=\"13\">Earlier crawls and the data sets built from them remain.<\/text><text x=\"812\" y=\"306\" text-anchor=\"end\" fill=\"#8a9695\" font-size=\"11\">Fig. 1 \u00b7 taismo<\/text><\/svg><\/a><figcaption><strong>Fig. 1: How a web page travels through Common Crawl into a language model. Your robots.txt only applies when CCBot fetches, and only to future crawls.<\/strong><\/figcaption><\/figure>\n<h2 id=\"language-models\">Which language models use Common Crawl?<\/h2>\n<p><strong>At least 30 of 47 large language models released between 2019 and October 2023 were trained on a filtered version of Common Crawl, which is 64 percent.<\/strong> This is shown by the study &#8220;Training Data for the Price of a Sandwich&#8221; published by the Mozilla Foundation in February 2024. For OpenAI&#8217;s GPT-3, more than 80 percent of the training tokens came from Common Crawl.<\/p>\n<p>A <a href=\"https:\/\/taismo.de\/en\/what-is\/llm\/\">large language model<\/a> usually processes Common Crawl through a prepared version. The raw data contains menus, cookie notices, duplicates, and spam; AI developers therefore rely on filtered data sets. The 3 best known are:<\/p>\n<ul>\n<li><strong>C4 (Colossal Clean Crawled Corpus):<\/strong> Google built C4 for the language model T5. The data set is one of the most widely used filtered versions.<\/li>\n<li><strong>Pile-CC:<\/strong> Pile-CC is the Common Crawl portion of the data set &#8220;The Pile&#8221; by EleutherAI.<\/li>\n<li><strong>FineWeb:<\/strong> Hugging Face filtered FineWeb from numerous Common Crawl snapshots and released it openly.<\/li>\n<\/ul>\n<p>Meta combined two filtered versions for the first Llama generation. The filters mainly remove pornography, boilerplate text, and duplicates. They keep the selection of pages that Common Crawl made beforehand unchanged: Content from English-language and well-linked domains is overrepresented.<\/p>\n<h2 id=\"old-content\">Why does old content live on in AI answers through Common Crawl?<\/h2>\n<p><strong>A language model knows a website in the state that the crawl captured at the time of training.<\/strong> If you update a page today, that changes nothing about the knowledge of a model trained on a crawl from 2024. The old version remains in the archive, in the filtered data sets built from it, and in every model trained on them.<\/p>\n<p>Three consequences matter for website owners:<\/p>\n<ol>\n<li><strong>Outdated information remains available:<\/strong> Old prices, former services, or an outdated company name remain available in earlier crawls and can show up in model answers.<\/li>\n<li><strong>New content needs a training cycle:<\/strong> A new page only reaches model knowledge once a later crawl captures it and a provider retrains with it.<\/li>\n<li><strong>Freshness comes through live search:<\/strong> Many AI systems compare their training knowledge with current search results via <a href=\"https:\/\/taismo.de\/en\/what-is\/retrieval-augmented-generation\/\">retrieval augmented generation<\/a>. There, the current state of your page counts.<\/li>\n<\/ol>\n<p>For a brand, this means: Consistent information over the years pays off twice. If you keep your name, services, and core statements stable, you shape training knowledge and live answers with the same information. This turns the company into a clearly recognizable <a href=\"https:\/\/taismo.de\/en\/what-is\/entity\/\">entity<\/a> for models.<\/p>\n<h2 id=\"check-your-website\">How do you check whether your website is included in Common Crawl?<\/h2>\n<p><strong>Whether your website is in Common Crawl is shown by the public URL index at index.commoncrawl.org.<\/strong> A single query per crawl returns all captured addresses of your domain with status code and file type:<\/p>\n<pre><code>https:\/\/index.commoncrawl.org\/CC-MAIN-2026-39-index?url=your-domain.com\/*&amp;output=json<\/code><\/pre>\n<p>The identifier <code>CC-MAIN-2026-39<\/code> stands for the September 2026 crawl; the list of all crawls is available at <code>index.commoncrawl.org\/collinfo.json<\/code>. The index is heavily rate limited. Send your queries one after another and pause between them, otherwise the server blocks your IP address for 24 hours.<\/p>\n<p>We checked this for taismo.de on September 24, 2026. The August crawl CC-MAIN-2026-34 contains 213 retrievable pages of the domain, the September crawl CC-MAIN-2026-39 only 69. The two crawls share just 7 URLs, out of around 450 addresses in the sitemap. The measurement confirms the sampling principle: Each crawl shows a different slice of the same website.<\/p>\n<h2 id=\"ccbot-robots-txt\">How do you control CCBot with robots.txt?<\/h2>\n<p><strong>CCBot follows the <a href=\"https:\/\/taismo.de\/en\/what-is\/robots-txt\/\">robots.txt<\/a> and responds to the user agent <code>CCBot<\/code>.<\/strong> Two lines block the crawler for the entire website:<\/p>\n<pre><code>User-agent: CCBot\nDisallow: \/<\/code><\/pre>\n<p>If CCBot should keep crawling, but more slowly, a crawl delay in seconds is enough. CCBot explicitly honors this directive, while Googlebot ignores it:<\/p>\n<pre><code>User-agent: CCBot\nCrawl-delay: 2<\/code><\/pre>\n<p>Four points are part of controlling CCBot:<\/p>\n<ul>\n<li><strong>The block works going forward:<\/strong> It prevents future fetches. Earlier crawls containing your pages remain published, as do the data sets built from them.<\/li>\n<li><strong>Fake CCBots exist:<\/strong> Common Crawl itself warns about crawlers posing as CCBot. Genuine requests come from fixed IP ranges listed at <code>index.commoncrawl.org\/ccbot.json<\/code> and resolve via reverse DNS to <code>crawl.commoncrawl.org<\/code>.<\/li>\n<li><strong>The sitemap helps when you allow it:<\/strong> CCBot reads every <a href=\"https:\/\/taismo.de\/en\/what-is\/sitemap\/\">sitemap<\/a> listed in the robots.txt. A well-maintained sitemap increases the chance that the important pages are captured.<\/li>\n<li><strong>Google stays unaffected:<\/strong> A CCBot block only affects Common Crawl. Googlebot and your rankings in Google Search do not change as a result.<\/li>\n<\/ul>\n<p>CCBot is one of many bots that collect content for AI systems. How the others differ, and which of them serve training data and which serve live answers, is explained in the article on <a href=\"https:\/\/taismo.de\/en\/what-is\/ai-crawler\/\">AI crawlers<\/a>.<\/p>\n<h2 id=\"block-or-allow\">Should you block or allow CCBot?<\/h2>\n<p><strong>For companies that want to be mentioned in AI answers, allowing CCBot is the better choice in most cases.<\/strong> A block removes the website from the future training data of many open and commercial models. Language models then learn about the brand, its services, and its terminology only from third-party sources.<\/p>\n<table>\n<tr>\n<th>Goal<\/th>\n<th>Recommendation<\/th>\n<th>Reason<\/th>\n<\/tr>\n<tr>\n<td>Become visible in AI answers<\/td>\n<td>Allow CCBot<\/td>\n<td>Your own content flows into future training data and shapes what models know about the brand<\/td>\n<\/tr>\n<tr>\n<td>Protect paid or exclusive content<\/td>\n<td>Block CCBot<\/td>\n<td>The archive is freely accessible, and anyone can download and reuse the texts<\/td>\n<\/tr>\n<tr>\n<td>Limit server load<\/td>\n<td>Set a crawl delay<\/td>\n<td>CCBot stays active and fetches pages at longer intervals<\/td>\n<\/tr>\n<\/table>\n<p>Large publishers such as the New York Times already block CCBot for their content. For a mid-sized company with a service that needs explaining, the same block has a different effect: It makes the company less visible to models without protecting a business model. That is why taismo.de allows CCBot.<\/p>\n<p>Once allowed, you can influence your presence in the archive. Because Common Crawl prioritizes by harmonic centrality, <a href=\"https:\/\/taismo.de\/en\/what-is\/backlink\/\">backlinks<\/a> from well-connected websites increase the chance that more pages of a domain are captured. The <a href=\"https:\/\/taismo.de\/en\/what-is\/llms-txt\/\">llms.txt<\/a> adds a clear entry map for language models. How these levers work together is described on our page about <a href=\"https:\/\/taismo.de\/en\/geo\/\">GEO and AI visibility<\/a>.<\/p>\n<h2 id=\"faq\">Frequently asked questions about Common Crawl<\/h2>\n<p><strong>Is Common Crawl the same as the Internet Archive?<\/strong><br \/>\nCommon Crawl and the Internet Archive are two separate nonprofit projects. The Internet Archive&#8217;s Wayback Machine shows people earlier versions of individual pages in the browser, while Common Crawl provides bulk data for research and data analysis.<\/p>\n<p><strong>Does it cost anything to use Common Crawl?<\/strong><br \/>\nCommon Crawl data is free. The foundation provides it through the public data sets of Amazon Web Services and through data.commoncrawl.org; costs arise at most for your own computing and storage capacity.<\/p>\n<p><strong>Does a CCBot block hurt my Google ranking?<\/strong><br \/>\nA CCBot block has no effect on your Google ranking. Google crawls with its own Googlebot and does not use Common Crawl as a source for search.<\/p>\n<\/p><\/div>\n<\/div>\n","protected":false},"excerpt":{"rendered":"<p>IKI Contents Term profile at a glance How does Common Crawl work? Which language models use Common Crawl? Why does old content live on in AI answers through Common Crawl? How do you check whether your website is included in Common Crawl? How do you control CCBot with robots.txt? Should you block or allow CCBot? [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":8019,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_taismo_jsonld":"","_taismo_h1":"","taismo_jsonld":"{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/\",\"url\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/\",\"name\":\"What Is Common Crawl? Data Set, CCBot, and robots.txt\",\"isPartOf\":{\"@id\":\"https:\/\/taismo.de\/#website\"},\"inLanguage\":\"en-US\",\"primaryImageOfPage\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#primaryimage\"},\"mainEntity\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#definedterm\"},\"breadcrumb\":{\"@id\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#breadcrumb\"},\"datePublished\":\"2026-10-05T07:30:00+02:00\",\"dateModified\":\"2026-10-05T07:30:00+02:00\",\"speakable\":{\"@type\":\"SpeakableSpecification\",\"cssSelector\":[\".brand-claim\",\".lead-def\"]},\"author\":{\"@id\":\"https:\/\/taismo.de\/#person-dominik-breitbach\"},\"publisher\":{\"@id\":\"https:\/\/taismo.de\/#organization\"}},{\"@type\":\"ImageObject\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#primaryimage\",\"contentUrl\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/common-crawl-webarchiv-ccbot-sprachmodelle.webp\",\"url\":\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/common-crawl-webarchiv-ccbot-sprachmodelle.webp\",\"width\":1200,\"height\":675,\"caption\":\"Common Crawl: the open web archive behind many language models\"},{\"@type\":\"DefinedTerm\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#definedterm\",\"name\":\"Common Crawl\",\"alternateName\":\"Common Crawl data set\",\"termCode\":\"common-crawl\",\"url\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/\",\"inLanguage\":\"en-US\",\"inDefinedTermSet\":{\"@id\":\"https:\/\/taismo.de\/#seo-wiki\"},\"description\":\"Common Crawl is a freely accessible archive of web pages that the nonprofit Common Crawl Foundation has been building since 2008 with its crawler CCBot, adding a new crawl roughly once a month. Each crawl contains raw data, metadata, and extracted text from around two billion pages and is available for download at no cost. This makes Common Crawl one of the most important sources from which large language models draw their knowledge of the web.\",\"disambiguatingDescription\":\"In the web and AI context, Common Crawl refers to the open web archive of the nonprofit Common Crawl Foundation and its crawler CCBot; it refers neither to Googlebot nor to a crawler run by an AI provider.\",\"author\":{\"@id\":\"https:\/\/taismo.de\/#person-dominik-breitbach\"},\"publisher\":{\"@id\":\"https:\/\/taismo.de\/#organization\"}},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\/\/taismo.de\/en\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"SEO Glossary\",\"item\":\"https:\/\/taismo.de\/en\/what-is\/\"},{\"@type\":\"ListItem\",\"position\":3,\"name\":\"Common Crawl\"}]},{\"@type\":\"FAQPage\",\"@id\":\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#faqpage\",\"mainEntity\":[{\"@type\":\"Question\",\"name\":\"Is Common Crawl the same as the Internet Archive?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"Common Crawl and the Internet Archive are two separate nonprofit projects. The Internet Archive's Wayback Machine shows people earlier versions of individual pages in the browser, while Common Crawl provides bulk data for research and data analysis.\"}},{\"@type\":\"Question\",\"name\":\"Does it cost anything to use Common Crawl?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"Common Crawl data is free. The foundation provides it through the public data sets of Amazon Web Services and through data.commoncrawl.org; costs arise at most for your own computing and storage capacity.\"}},{\"@type\":\"Question\",\"name\":\"Does a CCBot block hurt my Google ranking?\",\"acceptedAnswer\":{\"@type\":\"Answer\",\"text\":\"A CCBot block has no effect on your Google ranking. Google crawls with its own Googlebot and does not use Common Crawl as a source for search.\"}}]}]}","footnotes":""},"categories":[80],"tags":[85],"class_list":["post-8031","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-what-is","tag-c-en"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v28.1 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>What Is Common Crawl? Data Set, CCBot, robots.txt | taismo<\/title>\n<meta name=\"description\" content=\"Common Crawl is the open web archive many language models learn from. How CCBot works and when to block it in robots.txt. Read on! \ud83d\udd0e\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"What Is Common Crawl? Data Set, CCBot, robots.txt | taismo\" \/>\n<meta property=\"og:description\" content=\"Common Crawl is the open web archive many language models learn from. How CCBot works and when to block it in robots.txt. Read on! \ud83d\udd0e\" \/>\n<meta property=\"og:url\" content=\"https:\/\/taismo.de\/en\/what-is\/common-crawl\/\" \/>\n<meta property=\"og:site_name\" content=\"taismo| Agentur f\u00fcr SEO &amp; Webdesign\" \/>\n<meta property=\"article:publisher\" content=\"https:\/\/www.facebook.com\/TaismoOnlineMarketing\" \/>\n<meta property=\"article:published_time\" content=\"2026-10-05T05:30:00+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/common-crawl-webarchiv-ccbot-sprachmodelle.webp\" \/>\n\t<meta property=\"og:image:width\" content=\"1200\" \/>\n\t<meta property=\"og:image:height\" content=\"675\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/webp\" \/>\n<meta name=\"author\" content=\"Dominik Breitbach\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\\\/\\\/schema.org\",\"@graph\":[{\"@type\":\"Article\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/#article\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/\"},\"author\":{\"name\":\"Dominik Breitbach\",\"@id\":\"https:\\\/\\\/taismo.de\\\/#person-dominik-breitbach\"},\"headline\":\"Common Crawl\",\"datePublished\":\"2026-10-05T05:30:00+00:00\",\"mainEntityOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/\"},\"wordCount\":1746,\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/09\\\/common-crawl-webarchiv-ccbot-sprachmodelle.webp\",\"keywords\":[\"C\"],\"articleSection\":[\"SEO Glossary\"],\"inLanguage\":\"en-US\"},{\"@type\":\"WebPage\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/\",\"name\":\"What Is Common Crawl? Data Set, CCBot, robots.txt | taismo\",\"isPartOf\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\"},\"primaryImageOfPage\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/#primaryimage\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/#primaryimage\"},\"thumbnailUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/09\\\/common-crawl-webarchiv-ccbot-sprachmodelle.webp\",\"datePublished\":\"2026-10-05T05:30:00+00:00\",\"description\":\"Common Crawl is the open web archive many language models learn from. How CCBot works and when to block it in robots.txt. Read on! \ud83d\udd0e\",\"breadcrumb\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/\"]}]},{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/#primaryimage\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/09\\\/common-crawl-webarchiv-ccbot-sprachmodelle.webp\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2026\\\/09\\\/common-crawl-webarchiv-ccbot-sprachmodelle.webp\",\"width\":1200,\"height\":675,\"caption\":\"Common Crawl: Das offene Webarchiv hinter vielen Sprachmodellen\"},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/what-is\\\/common-crawl\\\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Startseite\",\"item\":\"https:\\\/\\\/taismo.de\\\/en\\\/search-marketing\\\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Common Crawl\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#website\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"name\":\"taismo| Agentur f\u00fcr SEO & Webdesign\",\"description\":\"\",\"publisher\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\"},\"alternateName\":\"taismo\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\\\/\\\/taismo.de\\\/en\\\/?s={search_term_string}\"},\"query-input\":{\"@type\":\"PropertyValueSpecification\",\"valueRequired\":true,\"valueName\":\"search_term_string\"}}],\"inLanguage\":\"en-US\"},{\"@type\":\"Organization\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#organization\",\"name\":\"taismo SEO Agentur M\u00fcnchen\",\"alternateName\":\"taismo\",\"url\":\"https:\\\/\\\/taismo.de\\\/en\\\/\",\"logo\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\",\"url\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"contentUrl\":\"https:\\\/\\\/taismo.de\\\/wp-content\\\/uploads\\\/2024\\\/09\\\/cropped-favicon.png\",\"width\":512,\"height\":512,\"caption\":\"taismo SEO Agentur M\u00fcnchen\"},\"image\":{\"@id\":\"https:\\\/\\\/taismo.de\\\/en\\\/#\\\/schema\\\/logo\\\/image\\\/\"},\"sameAs\":[\"https:\\\/\\\/www.facebook.com\\\/TaismoOnlineMarketing\",\"https:\\\/\\\/branchenbuch.portal.muenchen.de\\\/mhp\\\/1380360\\\/\",\"https:\\\/\\\/www.crunchbase.com\\\/organization\\\/taismo-gmbh\",\"https:\\\/\\\/de.trustpilot.com\\\/review\\\/taismo.de\",\"https:\\\/\\\/www.linkedin.com\\\/company\\\/taismo\\\/\",\"https:\\\/\\\/www.instagram.com\\\/taismo.de\",\"https:\\\/\\\/onlinemarketing.de\\\/dienstleister\\\/online-marketing-agentur\\\/kirchheim-bei-muenchen\\\/taismo\",\"https:\\\/\\\/www.agenturtipp.de\\\/agentur\\\/taismo-online-marketing\\\/\",\"https:\\\/\\\/startupvalley.news\\\/de\\\/taismo-online-marketing-agentur\\\/\",\"https:\\\/\\\/www.wlw.de\\\/de\\\/firma\\\/taismo-gmbh-1918135\",\"https:\\\/\\\/www.youtube.com\\\/@taismo8632\",\"https:\\\/\\\/techbehemoths.com\\\/company\\\/taismo-gmbh\",\"https:\\\/\\\/www.kununu.com\\\/de\\\/taismo1\",\"https:\\\/\\\/www.provenexpert.com\\\/taismo1\\\/\",\"https:\\\/\\\/www.provenemployer.com\\\/p\\\/taismo-employer\\\/\",\"https:\\\/\\\/www.trustedshops.de\\\/bewertung\\\/taismo-de\",\"https:\\\/\\\/www.kennstdueinen.de\\\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html\",\"https:\\\/\\\/www.ibusiness.de\\\/dienstleister\\\/jb\\\/5920834fb726.html\",\"https:\\\/\\\/www.seo-united.de\\\/seo-agenturen\\\/taismo-3345\\\/\"]}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"What Is Common Crawl? Data Set, CCBot, robots.txt | taismo","description":"Common Crawl is the open web archive many language models learn from. How CCBot works and when to block it in robots.txt. Read on! \ud83d\udd0e","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/","og_locale":"en_US","og_type":"article","og_title":"What Is Common Crawl? Data Set, CCBot, robots.txt | taismo","og_description":"Common Crawl is the open web archive many language models learn from. How CCBot works and when to block it in robots.txt. Read on! \ud83d\udd0e","og_url":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/","og_site_name":"taismo| Agentur f\u00fcr SEO &amp; Webdesign","article_publisher":"https:\/\/www.facebook.com\/TaismoOnlineMarketing","article_published_time":"2026-10-05T05:30:00+00:00","og_image":[{"width":1200,"height":675,"url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/common-crawl-webarchiv-ccbot-sprachmodelle.webp","type":"image\/webp"}],"author":"Dominik Breitbach","twitter_card":"summary_large_image","schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#article","isPartOf":{"@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/"},"author":{"name":"Dominik Breitbach","@id":"https:\/\/taismo.de\/#person-dominik-breitbach"},"headline":"Common Crawl","datePublished":"2026-10-05T05:30:00+00:00","mainEntityOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/"},"wordCount":1746,"publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/common-crawl-webarchiv-ccbot-sprachmodelle.webp","keywords":["C"],"articleSection":["SEO Glossary"],"inLanguage":"en-US"},{"@type":"WebPage","@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/","url":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/","name":"What Is Common Crawl? Data Set, CCBot, robots.txt | taismo","isPartOf":{"@id":"https:\/\/taismo.de\/en\/#website"},"primaryImageOfPage":{"@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#primaryimage"},"image":{"@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#primaryimage"},"thumbnailUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/common-crawl-webarchiv-ccbot-sprachmodelle.webp","datePublished":"2026-10-05T05:30:00+00:00","description":"Common Crawl is the open web archive many language models learn from. How CCBot works and when to block it in robots.txt. Read on! \ud83d\udd0e","breadcrumb":{"@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/taismo.de\/en\/what-is\/common-crawl\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#primaryimage","url":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/common-crawl-webarchiv-ccbot-sprachmodelle.webp","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2026\/09\/common-crawl-webarchiv-ccbot-sprachmodelle.webp","width":1200,"height":675,"caption":"Common Crawl: Das offene Webarchiv hinter vielen Sprachmodellen"},{"@type":"BreadcrumbList","@id":"https:\/\/taismo.de\/en\/what-is\/common-crawl\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Startseite","item":"https:\/\/taismo.de\/en\/search-marketing\/"},{"@type":"ListItem","position":2,"name":"Common Crawl"}]},{"@type":"WebSite","@id":"https:\/\/taismo.de\/en\/#website","url":"https:\/\/taismo.de\/en\/","name":"taismo| Agentur f\u00fcr SEO & Webdesign","description":"","publisher":{"@id":"https:\/\/taismo.de\/en\/#organization"},"alternateName":"taismo","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/taismo.de\/en\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/taismo.de\/en\/#organization","name":"taismo SEO Agentur M\u00fcnchen","alternateName":"taismo","url":"https:\/\/taismo.de\/en\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/","url":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","contentUrl":"https:\/\/taismo.de\/wp-content\/uploads\/2024\/09\/cropped-favicon.png","width":512,"height":512,"caption":"taismo SEO Agentur M\u00fcnchen"},"image":{"@id":"https:\/\/taismo.de\/en\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/TaismoOnlineMarketing","https:\/\/branchenbuch.portal.muenchen.de\/mhp\/1380360\/","https:\/\/www.crunchbase.com\/organization\/taismo-gmbh","https:\/\/de.trustpilot.com\/review\/taismo.de","https:\/\/www.linkedin.com\/company\/taismo\/","https:\/\/www.instagram.com\/taismo.de","https:\/\/onlinemarketing.de\/dienstleister\/online-marketing-agentur\/kirchheim-bei-muenchen\/taismo","https:\/\/www.agenturtipp.de\/agentur\/taismo-online-marketing\/","https:\/\/startupvalley.news\/de\/taismo-online-marketing-agentur\/","https:\/\/www.wlw.de\/de\/firma\/taismo-gmbh-1918135","https:\/\/www.youtube.com\/@taismo8632","https:\/\/techbehemoths.com\/company\/taismo-gmbh","https:\/\/www.kununu.com\/de\/taismo1","https:\/\/www.provenexpert.com\/taismo1\/","https:\/\/www.provenemployer.com\/p\/taismo-employer\/","https:\/\/www.trustedshops.de\/bewertung\/taismo-de","https:\/\/www.kennstdueinen.de\/online-marketing-kirchheim-bei-muenchen-taismo-gmbh-d2136613.html","https:\/\/www.ibusiness.de\/dienstleister\/jb\/5920834fb726.html","https:\/\/www.seo-united.de\/seo-agenturen\/taismo-3345\/"]}]}},"_links":{"self":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/8031","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/comments?post=8031"}],"version-history":[{"count":1,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/8031\/revisions"}],"predecessor-version":[{"id":8032,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/posts\/8031\/revisions\/8032"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media\/8019"}],"wp:attachment":[{"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/media?parent=8031"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/categories?post=8031"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/taismo.de\/en\/wp-json\/wp\/v2\/tags?post=8031"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}