21 lines
336 B
Plaintext
21 lines
336 B
Plaintext
# SaaS平台robots.txt
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# 禁止访问的路径
|
|
Disallow: /api/
|
|
Disallow: /admin/
|
|
Disallow: /dashboard/private/
|
|
Disallow: /uploads/private/
|
|
|
|
# 允许访问的特定路径
|
|
Allow: /public/
|
|
Allow: /blog/
|
|
Allow: /docs/
|
|
|
|
# 站点地图
|
|
Sitemap: https://yourdomain.com/sitemap.xml
|
|
|
|
# 抓取频率设置
|
|
Crawl-delay: 10
|