# robots.txt for https://rasoulunlimited.ir
# Maintainer: Mohammad Rasoul Sohrabi
# Deployed via GitHub Pages and proxied through Cloudflare
# Contact: rasoul.unlimited@gmail.com
# Updated: 2025-07-03

Sitemap: https://rasoulunlimited.ir/sitemap.xml
Host: rasoulunlimited.ir

# ----- General Crawling Policy -----
User-agent: *
Allow: /

# ----- Preferred Public Content -----
Allow: /en/
Allow: /projects/
Allow: /projects/c-projects/
Allow: /projects/python-projects/
Allow: /contact.html
Allow: /press-kit/
Allow: /press-kit/fa/
Allow: /press.html
Allow: /resume.html
Allow: /en/resume.html
Allow: /about.html
Allow: /media.html
Allow: /proof.html
Allow: /security.html
Allow: /links.txt
Allow: /manifest.json
Allow: /manifest-en.json
Allow: /assets/css/
Allow: /assets/images/
Allow: /assets/js/
Allow: /assets/vendor/
Allow: /faq/
Allow: /en/faq/
Allow: /wiki/

# ----- Identity and Verification Files -----
Allow: /humans.txt
Allow: /foaf.rdf
Allow: /.well-known/
Allow: /.well-known/identity.json
Allow: /.well-known/entity.rdf
Allow: /.well-known/entity.ttl
Allow: /.well-known/security.txt
Allow: /.well-known/entity-profile.json
Allow: /.well-known/webfinger.json
Allow: /.well-known/keybase.txt
Allow: /schema/
Allow: /schema/identity.jsonld

# ----- Blocked Paths -----
Disallow: /admin/
Disallow: /login/
Disallow: /register/
Disallow: /cgi-bin/
Disallow: /private/
Disallow: /tmp/
Disallow: /backup/
Disallow: /404.html
Disallow: /includes/
Disallow: /cloudflare/
Disallow: /gulpfile.js
Disallow: /package.json
Disallow: /package-lock.json
Disallow: /README.md
Disallow: /LICENSE

# ----- Specialized Crawlers -----
User-agent: Googlebot-Image
Allow: /assets/images/

User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: DuckDuckBot
Allow: /


# ----- Reference -----
# FOAF: /foaf.rdf
# humans.txt: /humans.txt
# identity.json: /.well-known/identity.json
# entity.rdf / entity.ttl: /.well-known/
# schema: /schema/identity.jsonld
# security.txt: /.well-known/security.txt
# links.txt: /links.txt
# entity-profile.json: /.well-known/entity-profile.json
# webfinger.json: /.well-known/webfinger.json
# keybase.txt: /.well-known/keybase.txt