{"id":371833,"date":"2026-09-24T05:39:47","date_gmt":"2026-09-24T05:39:47","guid":{"rendered":"https:\/\/wordpress.org\/plugins\/wpg-ai-crawler-manager\/"},"modified":"2026-09-24T05:39:23","modified_gmt":"2026-09-24T05:39:23","slug":"wpg-ai-crawler-manager","status":"publish","type":"plugin","link":"https:\/\/wordpress.org\/plugins\/wpg-ai-crawler-manager\/","author":8048773,"comment_status":"closed","ping_status":"closed","template":"","meta":{"version":"1.0.0","stable_tag":"1.0.0","tested":"7.1.2","requires":"6.0","requires_php":"7.4","requires_plugins":null,"header_name":"WPG AI Crawler Manager","header_author":"Rakesh Raja","header_description":"Control how AI bots like GPTBot, ClaudeBot and Google-Extended crawl your site with llms.txt, robots rules and live hit stats. by Rakesh Raja (WPGuru)","assets_banners_color":"1a1b4b","last_updated":"2026-09-24 05:39:23","external_support_url":"","external_repository_url":"","donate_link":"","header_plugin_uri":"","header_author_uri":"https:\/\/rakesh.in","rating":0,"author_block_rating":0,"active_installs":0,"downloads":38,"num_ratings":0,"support_threads":0,"support_threads_resolved":0,"author_block_count":0,"sections":["description","installation","faq","changelog"],"tags":{"1.0.0":{"tag":"1.0.0","author":"rakeshraja","date":"2026-09-24 05:39:23","revision":3710530}},"upgrade_notice":[],"ratings":[],"assets_icons":{"icon-128x128.png":{"filename":"icon-128x128.png","revision":3710530,"resolution":"128x128","location":"assets","locale":"","width":128,"height":128},"icon-256x256.png":{"filename":"icon-256x256.png","revision":3710530,"resolution":"256x256","location":"assets","locale":"","width":256,"height":256}},"assets_banners":{"banner-1544x500.png":{"filename":"banner-1544x500.png","revision":3710530,"resolution":"1544x500","location":"assets","locale":"","width":1544,"height":500},"banner-772x250.png":{"filename":"banner-772x250.png","revision":3710530,"resolution":"772x250","location":"assets","locale":"","width":772,"height":250}},"assets_blueprints":{},"all_blocks":[],"tagged_versions":["1.0.0"],"block_files":[],"assets_screenshots":{"screenshot-1.png":{"filename":"screenshot-1.png","revision":3710530,"resolution":"1","location":"assets","locale":"","width":1208,"height":586},"screenshot-2.png":{"filename":"screenshot-2.png","revision":3710530,"resolution":"2","location":"assets","locale":"","width":1208,"height":541},"screenshot-3.png":{"filename":"screenshot-3.png","revision":3710530,"resolution":"3","location":"assets","locale":"","width":1208,"height":1076},"screenshot-4.png":{"filename":"screenshot-4.png","revision":3710530,"resolution":"4","location":"assets","locale":"","width":1208,"height":279},"screenshot-5.png":{"filename":"screenshot-5.png","revision":3710530,"resolution":"5","location":"assets","locale":"","width":1208,"height":1106},"screenshot-6.png":{"filename":"screenshot-6.png","revision":3710530,"resolution":"6","location":"assets","locale":"","width":1208,"height":334},"screenshot-7.png":{"filename":"screenshot-7.png","revision":3710530,"resolution":"7","location":"assets","locale":"","width":1208,"height":1087},"screenshot-8.png":{"filename":"screenshot-8.png","revision":3710530,"resolution":"8","location":"assets","locale":"","width":1208,"height":1174}},"screenshots":{"1":"Dashboard with AI crawler hit statistics and a 7-day activity chart.","2":"Hits by crawler, with per-bot last-seen timestamps and active rules.","3":"Allow or block any of the 19 tracked AI crawlers from one screen.","4":"Add custom crawler rules for bots that are not in the built-in list.","5":"Configure the generated llms.txt content and choose your key pages.","6":"Live preview of the generated \/llms.txt file.","7":"Statistics retention, reset controls, and Apache\/nginx server snippets.","8":"Reference explaining what each AI crawler does and how to decide."}},"plugin_section":[],"plugin_tags":[2353,216196,216551,239487,6499],"plugin_category":[55],"plugin_contributors":[87221],"plugin_business_model":[],"class_list":["post-371833","plugin","type-plugin","status-publish","hentry","plugin_tags-ai","plugin_tags-chatgpt","plugin_tags-gpt","plugin_tags-llms","plugin_tags-robots","plugin_category-seo-and-marketing","plugin_contributors-rakeshraja","plugin_committers-rakeshraja"],"banners":{"banner":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/banner-772x250.png?rev=3710530","banner_2x":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/banner-1544x500.png?rev=3710530","banner_rtl":false,"banner_2x_rtl":false},"icons":{"svg":false,"icon":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/icon-128x128.png?rev=3710530","icon_2x":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/icon-256x256.png?rev=3710530","generated":false},"screenshots":[{"src":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/screenshot-1.png?rev=3710530","caption":"Dashboard with AI crawler hit statistics and a 7-day activity chart."},{"src":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/screenshot-2.png?rev=3710530","caption":"Hits by crawler, with per-bot last-seen timestamps and active rules."},{"src":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/screenshot-3.png?rev=3710530","caption":"Allow or block any of the 19 tracked AI crawlers from one screen."},{"src":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/screenshot-4.png?rev=3710530","caption":"Add custom crawler rules for bots that are not in the built-in list."},{"src":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/screenshot-5.png?rev=3710530","caption":"Configure the generated llms.txt content and choose your key pages."},{"src":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/screenshot-6.png?rev=3710530","caption":"Live preview of the generated \/llms.txt file."},{"src":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/screenshot-7.png?rev=3710530","caption":"Statistics retention, reset controls, and Apache\/nginx server snippets."},{"src":"https:\/\/ps.w.org\/wpg-ai-crawler-manager\/assets\/screenshot-8.png?rev=3710530","caption":"Reference explaining what each AI crawler does and how to decide."}],"raw_content":"\n

Stop guessing which AI crawlers visit your WordPress site. WPG AI Crawler Manager<\/strong> is the all-in-one control panel for AI bots: serve a clean llms.txt<\/code> file, block ChatGPT \/ GPTBot \/ Claude \/ Perplexity \/ Google-Extended with per-bot robots.txt<\/code> rules (or allow the crawlers you want), and watch live hit statistics \u2014 all stored locally on your own server. No third-party services, no accounts, no data ever leaves your site.<\/p>\n\n

Whether you want to opt out of AI training, welcome AI search traffic, or practice generative engine optimization (GEO), this plugin gives you the switchboard \u2014 with a per-bot \"last seen\" log so you know exactly which AI engines are actually crawling you.<\/p>\n\n

Features<\/h4>\n\n
    \n
  • llms.txt generator<\/strong> \u2014 publish a machine-readable \/llms.txt<\/code> file with your site name, a one-line description, an about paragraph and up to 20 of your best pages, so AI models and LLMs understand what your site is about.<\/li>\n
  • 19 known AI crawlers, tracked<\/strong> \u2014 GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, Claude-User, Google-Extended, PerplexityBot, Perplexity-User, CCBot, Applebot-Extended, Bytespider, Meta-ExternalAgent, Meta-ExternalFetcher, Amazonbot, cohere-ai, Diffbot, ImagesiftBot and PanguBot, each labelled by organization and purpose (Training \/ Search \/ User-triggered \/ Aggregator).<\/li>\n
  • Block AI bots with robots.txt<\/strong> \u2014 a friendly segmented Default \/ Allow \/ Block control per bot, written straight into your robots.txt<\/code>.<\/li>\n
  • Custom crawler rules<\/strong> \u2014 block or allow any AI bot that is not in the known list by entering its user-agent token; it is tracked, counted and written to robots.txt just like the known crawlers.<\/li>\n
  • Server-snippet helper<\/strong> \u2014 copy-paste Apache and nginx blocks for hosts whose robots.txt<\/code> is not served by WordPress.<\/li>\n
  • Local hit statistics<\/strong> \u2014 see all-time totals, a 7-day bar chart and a per-bot breakdown with \"last seen\" timestamps, logged entirely in your own database (no phone-home).<\/li>\n
  • Polished dashboard<\/strong> \u2014 stat tiles, pure-CSS bar chart, status pills, and a reference guide explaining why you might block one crawler and allow another.<\/li>\n<\/ul>\n\n

    All rules are advisory (well-behaved bots obey them); bigger decisions like \"training vs. search\" are explained right in the Help screen so you can decide with confidence. Built and maintained by Rakesh Raja (WPGuru).<\/p>\n\n

    How it works<\/h4>\n\n

    When enabled, the plugin intercepts \/llms.txt<\/code> as a virtual file, appends your per-bot rules to the robots.txt<\/code> output, and watches the user-agent of every front-end visit to log recognized AI crawler hits locally. Every 5 minutes pending hits are merged into the stats, and old days are pruned automatically.<\/p>\n\n

    Who is this for?<\/h4>\n\n
      \n
    • Site owners who want to block AI training crawlers<\/strong> (GPTBot, ClaudeBot, Google-Extended, Bytespider, CCBot) from scraping their content.<\/li>\n
    • Publishers who want AI search visibility<\/strong> and want to allow PerplexityBot, OAI-SearchBot and ChatGPT-User.<\/li>\n
    • SEOs and agencies adding GEO \/ AI visibility<\/strong> to their service stack \u2014 the dashboard gives you client-ready reports of AI crawler activity.<\/li>\n<\/ul>\n\n\n
        \n
      1. Upload the wpg-ai-crawler-manager<\/code> folder to \/wp-content\/plugins\/<\/code>, or use Plugins \u2192 Add New \u2192 Upload Plugin<\/strong>.<\/li>\n
      2. Activate the plugin.<\/li>\n
      3. Open AI Crawlers<\/strong> in your admin menu.<\/li>\n
      4. Set your llms.txt description and key pages, choose which AI crawlers to allow or block, and review the dashboard as hits come in.<\/li>\n<\/ol>\n\n

        No configuration file, no API keys, no external calls \u2014 everything works out of the box.<\/p>\n\n\n

        \n

        How do I block ChatGPT or GPTBot from my WordPress site?<\/h3><\/dt>\n

        Open AI Crawlers \u2192 AI Bots<\/strong>, set GPTBot (OpenAI's training crawler) and\/or ChatGPT-User to Block<\/strong>, and press Save. The rule is written to your robots.txt immediately.<\/p><\/dd>\n

        What is llms.txt?<\/h3><\/dt>\n

        llms.txt is a proposed standard file (like robots.txt) written in Markdown that gives large language models a concise, structured summary of your site: what it is, what it is about, and which pages matter most. This plugin generates and serves it at \/llms.txt<\/code> for you.<\/p><\/dd>\n

        Does this plugin send data anywhere?<\/h3><\/dt>\n

        No. Hit statistics are recorded in your own WordPress database. The plugin never phones home and makes no automatic external HTTP requests.<\/p><\/dd>\n

        Do blocked bots really stop?<\/h3><\/dt>\n

        robots.txt is advisory by design: standards-following bots respect it, but it is not a security mechanism against determined scrapers. For stronger enforcement, use the Apache\/nginx snippets on the Settings screen.<\/p><\/dd>\n

        Will blocking AI crawlers hurt my SEO?<\/h3><\/dt>\n

        Blocking training<\/em> crawlers does not affect search rankings. Blocking search<\/em> crawlers (like OAI-SearchBot or PerplexityBot) can reduce your visibility in AI answer engines \u2014 the Help screen explains each crawler's purpose so you can choose per bot.<\/p><\/dd>\n

        Why is my llms.txt not live?<\/h3><\/dt>\n

        Make sure the publish toggle on the llms.txt page is on and your permalinks are set to something other than \"Plain\". WordPress must be serving pages from the site root for \/llms.txt<\/code> to resolve.<\/p><\/dd>\n

        Which crawlers are tracked?<\/h3><\/dt>\n

        GPTBot, OAI-SearchBot, ChatGPT-User, ClaudeBot, Claude-SearchBot, Claude-User, Google-Extended, PerplexityBot, Perplexity-User, CCBot, Applebot-Extended, Bytespider, Meta-ExternalAgent, Meta-ExternalFetcher, Amazonbot, cohere-ai, Diffbot, ImagesiftBot and PanguBot. Not listed? Add any crawler yourself with a custom rule on the AI Bots page.<\/p><\/dd>\n\n<\/dl>\n\n\n

        1.0.0<\/h4>\n\n
          \n
        • Initial release: llms.txt generator, per-bot robots.txt rules (plus custom crawler rules), Apache\/nginx server snippets, and a local hit-statistics dashboard with per-bot \"last seen\" timestamps.<\/li>\n<\/ul>","raw_excerpt":"Block or allow AI bots like GPTBot, ChatGPT, ClaudeBot and PerplexityBot with an llms.txt generator, robots.txt rules and live AI crawler hit stats.","jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin\/371833","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin"}],"about":[{"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/types\/plugin"}],"replies":[{"embeddable":true,"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/comments?post=371833"}],"author":[{"embeddable":true,"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wporg\/v1\/users\/rakeshraja"}],"wp:attachment":[{"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/media?parent=371833"}],"wp:term":[{"taxonomy":"plugin_section","embeddable":true,"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_section?post=371833"},{"taxonomy":"plugin_tags","embeddable":true,"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_tags?post=371833"},{"taxonomy":"plugin_category","embeddable":true,"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_category?post=371833"},{"taxonomy":"plugin_contributors","embeddable":true,"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_contributors?post=371833"},{"taxonomy":"plugin_business_model","embeddable":true,"href":"https:\/\/wordpress.org\/plugins\/wp-json\/wp\/v2\/plugin_business_model?post=371833"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}