# robots.txt for b-log.to User-agent: * Allow: / Content-Signal: ai-train=no, search=yes, ai-input=yes # 主要内容页面 Allow: /*/ # 排除舊文章入口(使用新的目录结构) Disallow: /post.html Disallow: /*?search=* # Sitemap Sitemap: https://b-log.to/sitemap.xml # 常见爬虫配置 # 允许所有分类目录 Allow: /ai-analysis/ Allow: /tech-analysis/ Allow: /tech-development/ Allow: /dev-philosophy/ Allow: /business-insights/ Allow: /cultural-insights/ Allow: /life-stories/ Allow: /crossing-field/ Allow: /shirushi/ Allow: /ghostwritten/