# All content on this website, including text, graphics, images, web design, film, sound
# and other content, is protected by the Swedish Copyright Act (1960:729) (hereinafter
# "the Copyright Act"), both in terms of copyright and related rights. Bonnier News AB and
# its group companies (“Bonnier News”/”us”) reserve all rights to the content, including the
# right to use the content and the press publication for text and data extraction in
# accordance with Section 15a of the Copyright Act and Article 4 of the Copyright Directive.
#
# You may not use, nor may you facilitate or authorize the use of, the websites and/or any
# content for any purpose without prior written permission by Bonnier News, including, but
# not limited to, scraping the content or reproducing, modifying, collecting or extracting
# the content for text and data aggregation, analysis or extraction or commercial use.
# However, permission for website scraping and indexing may be granted for inclusion in a
# search engine provided that such authorization has been made explicitly by us through our
# technical instructions or similar specifications. Any form of scraping and indexing not
# expressly authorized by our technical instructions or specifications is strictly
# prohibited.
# Common Crawl robot, the resulting dataset is the primary training corpus in every LLM.
User-agent: CCBot
Disallow: /
# ChatGPT robot, used to improve the ChatGPT LLM.
User-agent: ChatGPT-User
Disallow: /
# ChatGPT robot, may be used to improve the ChatGPT LLM.
User-agent: GPTBot
Disallow: /
# Robot used to improve Bard and Vertex AI LLMs.
User-agent: Google-Extended
Disallow: /
# Associated with Google Vertex AI agents
User-agent: Google-CloudVertexBot
Disallow: /
# webz.io robot, the resulting dataset can and is purchased to train LLMs.
User-agent: Omgilibot
Disallow: /
# webz.io robot, the resulting dataset can and is purchased to train LLMs.
User-agent: Omgili
Disallow: /
# FacebookBot crawls public web pages to improve LLMs for Facebook's speech recognition technology.
User-agent: FacebookBot
Disallow: /
# Another agent used by Anthropic that is more specifically related to Claude
User-agent: ClaudeBot
Disallow: /
# AI search indexing for Claude search capabilities
User-agent: Claude-SearchBot
Disallow: /
# AI agent for real-time web access when Claude users browse
User-agent: Claude-User
Disallow: /
# Diffbot crawls the web in or others to train their LLMs.
User-agent: Diffbot
Disallow: /
# Uses scraped data on-the-fly to create answers for DuckAssist.
User-agent: DuckAssistBot
Disallow: /
# Used by perplexity.ai. Generates text based on scraped material.
User-agent: PerplexityBot
Disallow: /
# AI agent for real-time browsing when Perplexity users request information
User-agent: Perplexity-User
Disallow: /
# Cohere’s chatbot.
User-agent: cohere-ai
Disallow: /
# Cohere’s chatbot.
User-agent: cohere-training-data-crawler
Disallow: /
# Use cases such as training AI models or improving products by indexing content directly.
User-agent: Meta-ExternalAgent
Disallow: /
# Crawler performs user-initiated fetches of individual links in support of some AI tools.
User-agent: Meta-ExternalFetcher
Disallow: /
# Used to improve Meta AI search.
User-agent: Meta-WebIndexer
Disallow: /
# Used by Timpi to scrape data for training their Large Language Models.
User-agent: Timpibot
Disallow: /
# Used by Webz.io to indicate that your site should not be included those using it to train AI models.
User-agent: Webzio-Extended
Disallow: /
# Crawler behind You.com’s AI search and browser assistant, indexing content for real-time answers.
User-agent: YouBot
Disallow: /
# Amazonbot is used to train Amazon services such as Alexa.
User-agent: Amazonbot
Disallow: /
# Bytespider is ByteDance's bot and may not respect robots.txt.
User-agent: Bytespider
Disallow: /
# Robot used to improve Anthropic AI LLMs.
User-agent: anthropic-ai
Disallow: /
# OpenAI search bot
User-agent: OAI-SearchBot
Disallow: /
# Velen.io/Hunter.io "build business datasets and machine learning models to better understand the web" - seems to focus on collecting email adresses for spam though.
User-agent: VelenPublicWebCrawler
Disallow: /
# AI research agent for Google Gemini’s Deep Research feature
User-agent: Gemini-Deep-Research
Disallow: /
# AI and machine learning data collection
User-agent: ICC-Crawler
Disallow: /
# Mistral’s real-time citation fetcher for “Le Chat” assistant
User-agent: MistralAI-User
Disallow: /
User-Agent: *
Disallow: /sok/
Disallow: /kop/
Disallow: /bn/id/*
Disallow: /foljer
Disallow: /api/*
Sitemap: https://www.st.nu/sitemap.xml
Sitemap: https://www.st.nu/sitemap-news.xml