# Robots.txt for PJ Cleaners # Updated: October 11, 2026 # Site: https://www.pnjcleaners.com # Default rules for all crawlers User-agent: * Allow: / # Block admin pages (authentication required) Disallow: /admin/ Disallow: /admin # Block client portal (authentication required) Disallow: /portal # Block API routes (no SEO value) Disallow: /api/ # Block development/test pages Disallow: /inner-page-v1 Disallow: /inner-page-v2 Disallow: /inner-page-v3 # Allow all money pages (services, locations, service-locations) Allow: /services/ Allow: /locations/ Allow: /blog/ Allow: /resources/ # Structured summary for AI agents / answer engines Allow: /llms.txt # Allow static pages Allow: /about Allow: /contact Allow: /careers Allow: /reviews Allow: /service-areas Allow: /faq # ============================================ # AI CRAWLER RULES - ALLOW ALL FOR VISIBILITY # ============================================ # OpenAI (ChatGPT, SearchGPT) User-agent: GPTBot Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / # Anthropic (Claude) User-agent: ClaudeBot Allow: / User-agent: anthropic-ai Allow: / # Perplexity AI User-agent: PerplexityBot Allow: / # Google AI (Bard/Gemini training - separate from Googlebot) User-agent: Google-Extended Allow: / User-agent: GoogleOther Allow: / # Common Crawl (used by many AI systems) User-agent: CCBot Allow: / # Meta AI User-agent: FacebookBot Allow: / # Apple Intelligence User-agent: Applebot-Extended Allow: / # ============================================ # SITEMAP DECLARATIONS # ============================================ # One entry: the index lists every sitemap. Sitemap: https://www.pnjcleaners.com/sitemap-index.xml