# robots.txt for https://insignialearning.com # A crawler that has its own group below ignores the "*" group, so the same # Disallow list is repeated in every group. Keep them identical. # --------------------------------------------------------------- # All crawlers (Google, Bing and others) # --------------------------------------------------------------- User-agent: * Allow: / # admin panel and internal endpoints (no SEO value, never meant to be public) Disallow: /login Disallow: /logout Disallow: /dashboard Disallow: /job- Disallow: /blog- Disallow: /user- Disallow: /users- Disallow: /applied-job- Disallow: /get- Disallow: /careers/process Disallow: /captchaImg Disallow: /search Disallow: /uploads/resumes/ Disallow: /phm/ # internal search and tracking parameters (avoid duplicate URLs) Disallow: /*?q= Disallow: /*?test_country= Disallow: /*?test_group= # --------------------------------------------------------------- # AI SEARCH and answer engines: allowed, so the site can be cited # in ChatGPT Search, Perplexity and Google AI features. # (OAI-SearchBot / PerplexityBot decide citability; # the *-User agents fetch pages on behalf of a person asking a question.) # --------------------------------------------------------------- User-agent: OAI-SearchBot User-agent: ChatGPT-User User-agent: PerplexityBot User-agent: Perplexity-User User-agent: Applebot User-agent: DuckAssistBot Allow: / Disallow: /login Disallow: /logout Disallow: /dashboard Disallow: /job- Disallow: /blog- Disallow: /user- Disallow: /users- Disallow: /applied-job- Disallow: /get- Disallow: /careers/process Disallow: /captchaImg Disallow: /search Disallow: /uploads/resumes/ Disallow: /phm/ # --------------------------------------------------------------- # AI TRAINING crawlers: they only decide whether content may be used # to train models. They do NOT affect Google Search, AI Overviews, # or ChatGPT Search. Currently allowed. # To opt out of training, change "Allow: /" to "Disallow: /" in this group. # --------------------------------------------------------------- User-agent: GPTBot User-agent: Google-Extended User-agent: Applebot-Extended User-agent: CCBot User-agent: cohere-ai Allow: / Disallow: /login Disallow: /logout Disallow: /dashboard Disallow: /job- Disallow: /blog- Disallow: /user- Disallow: /users- Disallow: /applied-job- Disallow: /get- Disallow: /careers/process Disallow: /captchaImg Disallow: /search Disallow: /uploads/resumes/ Disallow: /phm/ # --------------------------------------------------------------- # Aggressive scrapers with no search value # --------------------------------------------------------------- User-agent: Bytespider User-agent: AhrefsBot User-agent: SemrushBot User-agent: MJ12bot User-agent: DotBot Disallow: / Sitemap: https://insignialearning.com/sitemap.xml