# Allow all user agents to crawl the site User-agent: * # Specify the sitemap location Sitemap: https://www.smartinfologiks.com/sitemap.xml # Disallow crawling of sensitive or non-essential directories Disallow: /api/ Disallow: /apps/ Disallow: /config/ Disallow: /misc/ Disallow: /plugins/ Disallow: /services/ Disallow: /tmp/ Disallow: /xtras/ Disallow: /cdn-cgi/ # Allow crawling of media files Allow: /media/ # Block crawling of specific file types (optional) Disallow: /*.json$ Disallow: /*.xml$ Disallow: /*.sql$ # Crawl-delay for specific bots (optional, adjust if needed) User-agent: Bingbot Crawl-delay: 10 User-agent: Yandex Crawl-delay: 10 # Block bad bots (optional) User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: /