# 通用CMS robots.txt
# 百度爬虫规范：允许抓取内容页，禁止抓取后台和缓存目录

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /cache/
Disallow: /includes/
Disallow: /database/
Disallow: /deploy/
Disallow: /uploads/
Disallow: /cli_crawl.php
Disallow: /bulk_crawl.php
Disallow: /crawl_preview.php
Disallow: /router.php
Disallow: /*.json
Disallow: /*.log
Disallow: /*.sql
Disallow: /*.sqlite
Disallow: /*.bak
Crawl-delay: 3

# 百度爬虫
User-agent: Baiduspider
Allow: /
Disallow: /admin/
Disallow: /cache/
Disallow: /includes/
Disallow: /database/
Disallow: /deploy/
Disallow: /uploads/
Disallow: /cli_crawl.php
Disallow: /bulk_crawl.php
Disallow: /crawl_preview.php
Disallow: /router.php
Disallow: /*.json
Disallow: /*.log
Disallow: /*.sql
Disallow: /*.sqlite
Disallow: /*.bak
Crawl-delay: 2

# Sitemap
Sitemap: https://www.boe6.com/sitemap.xml