User-agent: * Content-Signal: search=yes,ai-train=no Allow: / # disallow junk urls indexed for unkown reason @since dec 2021 User-agent: Googlebot Allow: / Disallow: /index.php/forums/ Disallow: /index.php/chiptuning/ Disallow: /index.php/profile.php Disallow: /forums/ User-agent: * Disallow: /post-*.html$ Disallow: /updates-topic.html*$ Disallow: /stop-updates-topic.html*$ Disallow: /ptopic*.html$ Disallow: /ntopic*.html$ Disallow: /ntopic*.html$ Disallow: /link.php Disallow: /link.php Disallow: /out.php Disallow: /lofi/ Disallow: /zece/ User-agent: msnbot Crawl-delay: 10 User-agent: bingbot Crawl-delay: 10 User-Agent: AhrefsBot Crawl-Delay: 5 # AI bots User-agent: Amazonbot Disallow: / User-agent: Applebot Allow: / User-agent: Applebot-Extended Disallow: / User-agent: Bytespider Disallow: / User-agent: CCBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Google-Extended Disallow: / User-agent: GPTBot Disallow: / #User-agent: ChatGPT-User #User-agent: ChatGPT-User/2.0 #Allow: / User-agent: OAI-SearchBot Disallow: / User-agent: anthropic-ai # bulk model training Disallow: / #User-agent: ClaudeBot # chat citation fetch #User-agent: claude-web # web-focused crawl #Allow: / User-agent: PerplexityBot # index builder Disallow: / #User-agent: Perplexity-User # human-triggered visit #Allow: / User-agent: FacebookBot # User-agent: meta-externalagent Disallow: / User-agent: LinkedInBot Disallow: / User-agent: DuckAssistBot Allow: / User-agent: cohere-ai Disallow: / User-agent: BingBot Disallow: / User-agent: tencent User-agent: AliyunSecBot User-agent: AI2Bot User-agent: Ai2Bot-Dolma User-agent: CCBot User-agent: Diffbot User-agent: omgili User agent: OmigiliBot User-agent: Omigili User-agent: TimpiBot User-agent: YouBot Disallow: / # The grub distributed search engine behaves very badly. User-agent: proximic Disallow: / User-agent: grub-client Disallow: / User-Agent: trendictionbot Disallow: / # They totally overwhelm servers with traffic. # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: / # Proprietary German backlinks service. User-agent: SEOkicks-Robot Disallow: / # wget run recusively just *wrecks* server capacity. User-agent: wget Disallow: / # This spider's output isn't public. User-agent: 008 Disallow: / # Entireweb User-agent: Speedy Disallow: / # Foreign-language bot User-agent: Sogou web spider Disallow: / # Poorly behaved bot User-agent: Ezooms Disallow: / # Brandwatch User-agent: magpie-crawler Disallow: / # Russian image search engine User-agent: YandexImages Disallow: / User-agent: YandexCalendar Disallow: / User-agent: YandexBot Disallow: / User-agent: YandexMobileBot Disallow: / # Magestic-12 User-agent: MJ12bot Disallow: / # Yoydao User-agent: YodaoBot Disallow: / # NerdByNature.Net User-agent: NerdByNature.Bot Disallow: / # Discovery Engine User-agent: discobot Disallow: / # No idea. User-agent: knowaboutBot Disallow:/ # Gnip User-agent: UnwindFetchor Disallow: / # These bots are designed to duplicate entire sites. User-agent: sitecheck.internetseer.com Disallow: / User-agent: Zealbot Disallow: / User-agent: MSIECrawler Disallow: / User-agent: WebReaper Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: WebZIP Disallow: / User-agent: linko Disallow: / User-agent: HTTrack Disallow: / User-agent: Microsoft.URL.Control Disallow: / User-agent: Xenu Disallow: / User-agent: larbin Disallow: / User-agent: libwww Disallow: / User-agent: ZyBORG Disallow: / User-agent: Download Ninja Disallow: / User-agent: dotbot Disallow: / User-agent: Exabot Disallow: / User-agent: BotRightHere Disallow: / User-agent: Moreoverbot Disallow: / User-agent: WebZip Disallow: / User-agent: larbin Disallow: / User-agent: b2w/0.1 Disallow: / User-agent: Copernic Disallow: / User-agent: psbot Disallow: / User-agent: Python-urllib Disallow: / User-agent: NetMechanic Disallow: / User-agent: URL_Spider_Pro Disallow: / User-agent: CherryPicker Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: WebBandit Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: Crescent Disallow: / User-agent: SiteSnagger Disallow: / User-agent: ProWebWalker Disallow: / User-agent: CheeseBot Disallow: / User-agent: LNSpiderguy Disallow: / User-agent: Alexibot Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: MIIxpc Disallow: / User-agent: Telesoft Disallow: / User-agent: Website Quester Disallow: / User-agent: WebZip Disallow: / User-agent: moget/2.1 Disallow: / User-agent: WebZip/4.0 Disallow: / User-agent: WebStripper Disallow: / User-agent: WebSauger Disallow: / User-agent: WebCopier Disallow: / User-agent: NetAnts Disallow: / User-agent: Mister PiX Disallow: / User-agent: WebAuto Disallow: / User-agent: TheNomad Disallow: / User-agent: WWW-Collector-E Disallow: / User-agent: RMA Disallow: / User-agent: libWeb/clsHTTP Disallow: / User-agent: asterias Disallow: / User-agent: httplib Disallow: / User-agent: turingos Disallow: / User-agent: spanner Disallow: / User-agent: InfoNaviRobot Disallow: / User-agent: Harvest/1.5 Disallow: / User-agent: Bullseye/1.0 Disallow: / User-agent: Mozilla/4.0 (compatible; BullsEye; Windows 95) Disallow: / User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: / User-agent: CherryPickerSE/1.0 Disallow: / User-agent: grub-client Disallow: / User-agent: grub Disallow: / User-agent: WebZip Disallow: / User-agent: larbin Disallow: / User-agent: b2w/0.1 Disallow: / User-agent: Copernic Disallow: / User-agent: psbot Disallow: / User-agent: Python-urllib Disallow: / User-agent: NetMechanic Disallow: / User-agent: URL_Spider_Pro Disallow: / User-agent: CherryPicker Disallow: / User-agent: EmailCollector Disallow: / User-agent: EmailSiphon Disallow: / User-agent: WebBandit Disallow: / User-agent: EmailWolf Disallow: / User-agent: ExtractorPro Disallow: / User-agent: CopyRightCheck Disallow: / User-agent: Crescent Disallow: / User-agent: SiteSnagger Disallow: / User-agent: ProWebWalker Disallow: / User-agent: CheeseBot Disallow: / User-agent: LNSpiderguy Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: MIIxpc Disallow: / User-agent: Telesoft Disallow: / User-agent: Website Quester Disallow: / User-agent: moget/2.1 Disallow: / User-agent: WebZip/4.0 Disallow: / User-agent: WebStripper Disallow: / User-agent: WebSauger Disallow: / User-agent: WebCopier Disallow: / User-agent: NetAnts Disallow: / User-agent: Mister PiX Disallow: / User-agent: WebAuto Disallow: / User-agent: TheNomad Disallow: / User-agent: WWW-Collector-E Disallow: / User-agent: RMA Disallow: / User-agent: libWeb/clsHTTP Disallow: / User-agent: asterias Disallow: / User-agent: httplib Disallow: / User-agent: turingos Disallow: / User-agent: spanner Disallow: / User-agent: InfoNaviRobot Disallow: / User-agent: Harvest/1.5 Disallow: / User-agent: Bullseye/1.0 Disallow: / User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0 Disallow: / User-agent: CherryPickerSE/1.0 Disallow: / User-agent: CherryPickerElite/1.0 Disallow: / User-agent: WebBandit/3.50 Disallow: / User-agent: NICErsPRO Disallow: / User-agent: Microsoft URL Control - 5.01.4511 Disallow: / User-agent: DittoSpyder Disallow: / User-agent: Foobot Disallow: / User-agent: SpankBot Disallow: / User-agent: BotALot Disallow: / User-agent: lwp-trivial/1.34 Disallow: / User-agent: lwp-trivial Disallow: / User-agent: BunnySlippers Disallow: / User-agent: Microsoft URL Control - 6.00.8169 Disallow: / User-agent: URLy Warning Disallow: / User-agent: Wget/1.6 Disallow: / User-agent: Wget/1.5.3 Disallow: / User-agent: Wget Disallow: / User-agent: LinkWalker Disallow: / User-agent: cosmos Disallow: / User-agent: moget Disallow: / User-agent: hloader Disallow: / User-agent: humanlinks Disallow: / User-agent: LinkextractorPro Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Mata Hari Disallow: / User-agent: LexiBot Disallow: / User-agent: Web Image Collector Disallow: / User-agent: The Intraformant Disallow: / User-agent: True_Robot/1.0 Disallow: / User-agent: True_Robot Disallow: / User-agent: BlowFish/1.0 Disallow: / User-agent: JennyBot Disallow: / User-agent: MIIxpc/4.2 Disallow: / User-agent: BuiltBotTough Disallow: / User-agent: ProPowerBot/2.14 Disallow: / User-agent: BackDoorBot/1.0 Disallow: / User-agent: toCrawl/UrlDispatcher Disallow: / User-agent: WebEnhancer Disallow: / User-agent: suzuran Disallow: / User-agent: VCI WebViewer VCI WebViewer Win32 Disallow: / User-agent: VCI Disallow: / User-agent: Szukacz/1.4 Disallow: / User-agent: QueryN Metasearch Disallow: / User-agent: Openfind data gathere Disallow: / User-agent: Openfind Disallow: / User-agent: Xenu's Link Sleuth 1.1c Disallow: / User-agent: Xenu's Disallow: / User-agent: Zeus Disallow: / User-agent: RepoMonkey Bait & Tackle/v1.01 Disallow: / User-agent: RepoMonkey Disallow: / User-agent: Microsoft URL Control Disallow: / User-agent: Openbot Disallow: / User-agent: URL Control Disallow: / User-agent: Zeus Link Scout Disallow: / User-agent: Zeus 32297 Webster Pro V2.9 Win32 Disallow: / User-agent: Webster Pro Disallow: / User-agent: EroCrawler Disallow: / User-agent: LinkScan/8.1a Unix Disallow: / User-agent: Keyword Density/0.9 Disallow: / User-agent: Kenjin Spider Disallow: / User-agent: Iron33/1.0.2 Disallow: / User-agent: Bookmark search tool Disallow: / User-agent: GetRight/4.2 Disallow: / User-agent: FairAd Client Disallow: / User-agent: Gaisbot Disallow: / User-agent: Aqua_Products Disallow: / User-agent: Radiation Retriever 1.1 Disallow: / User-agent: Flaming AttackBot Disallow: / User-agent: PerMan Disallow: / User-agent: searchpreview Disallow: / User-agent: Baiduspider Disallow: / User-agent: GurujiBot/1.0 Disallow: / User-agent: Yanga WorldSearch Bot v1.1/beta Disallow: / User-agent: archive-ro.com Disallow: / User-agent: sentibot Disallow: / User-agent: BLEXBot Disallow: / User-agent: GPTBot Disallow: / User-agent: Bytespider Disallow: / User-agent: Bytedance Disallow: / User-agent: AddSearchBot User-agent: AI2Bot User-agent: AI2Bot-DeepResearchEval User-agent: Ai2Bot-Dolma User-agent: aiHitBot User-agent: amazon-kendra User-agent: Amazonbot User-agent: AmazonBuyForMe User-agent: Amzn-SearchBot User-agent: Amzn-User User-agent: Andibot User-agent: Anomura User-agent: anthropic-ai User-agent: Applebot User-agent: Applebot-Extended User-agent: atlassian-bot User-agent: Awario User-agent: AzureAI-SearchBot User-agent: bedrockbot User-agent: bigsur.ai User-agent: Bravebot User-agent: Brightbot 1.0 User-agent: BuddyBot User-agent: Bytespider User-agent: CCBot User-agent: Channel3Bot User-agent: ChatGLM-Spider User-agent: ChatGPT Agent User-agent: ChatGPT-User User-agent: Claude-SearchBot User-agent: Claude-User User-agent: Claude-Web User-agent: ClaudeBot User-agent: Cloudflare-AutoRAG User-agent: CloudVertexBot User-agent: cohere-ai User-agent: cohere-training-data-crawler User-agent: Cotoyogi User-agent: Crawl4AI User-agent: Crawlspace User-agent: Datenbank Crawler User-agent: DeepSeekBot User-agent: Devin User-agent: Diffbot User-agent: DuckAssistBot User-agent: Echobot Bot User-agent: EchoboxBot User-agent: FacebookBot User-agent: facebookexternalhit User-agent: Factset_spyderbot User-agent: FirecrawlAgent User-agent: FriendlyCrawler User-agent: Gemini-Deep-Research User-agent: Google-CloudVertexBot User-agent: Google-Extended User-agent: Google-Firebase User-agent: Google-NotebookLM User-agent: GoogleAgent-Mariner User-agent: GoogleOther User-agent: GoogleOther-Image User-agent: GoogleOther-Video User-agent: GPTBot User-agent: iAskBot User-agent: iaskspider User-agent: iaskspider/2.0 User-agent: IbouBot User-agent: ICC-Crawler User-agent: ImagesiftBot User-agent: imageSpider User-agent: img2dataset User-agent: ISSCyberRiskCrawler User-agent: kagi-fetcher User-agent: Kangaroo Bot User-agent: KlaviyoAIBot User-agent: KunatoCrawler User-agent: laion-huggingface-processor User-agent: LAIONDownloader User-agent: LCC User-agent: LinerBot User-agent: Linguee Bot User-agent: LinkupBot User-agent: Manus-User User-agent: meta-externalagent User-agent: Meta-ExternalAgent User-agent: meta-externalfetcher User-agent: Meta-ExternalFetcher User-agent: meta-webindexer User-agent: MistralAI-User User-agent: MistralAI-User/1.0 User-agent: MyCentralAIScraperBot User-agent: netEstate Imprint Crawler User-agent: NotebookLM User-agent: NovaAct User-agent: OAI-SearchBot User-agent: omgili User-agent: omgilibot User-agent: OpenAI User-agent: Operator User-agent: PanguBot User-agent: Panscient User-agent: panscient.com User-agent: Perplexity-User User-agent: PerplexityBot User-agent: PetalBot User-agent: PhindBot User-agent: Poggio-Citations User-agent: Poseidon Research Crawler User-agent: QualifiedBot User-agent: QuillBot User-agent: quillbot.com User-agent: SBIntuitionsBot User-agent: Scrapy User-agent: SemrushBot-OCOB User-agent: SemrushBot-SWA User-agent: ShapBot User-agent: Sidetrade indexer bot User-agent: Spider User-agent: TavilyBot User-agent: TerraCotta User-agent: Thinkbot User-agent: TikTokSpider User-agent: Timpibot User-agent: TwinAgent User-agent: VelenPublicWebCrawler User-agent: WARDBot User-agent: Webzio-Extended User-agent: webzio-extended User-agent: wpbot User-agent: WRTNBot User-agent: YaK User-agent: YandexAdditional User-agent: YandexAdditionalBot User-agent: YouBot User-agent: ZanistaBot User-agent: barkrowler User-agent: SEOkicks Disallow: /