license = $license; if ( $this->crawlers_enabled() ) { add_filter( 'robots_txt', array( $this, 'filter_robots_txt' ), 20, 2 ); } if ( $this->llms_enabled() ) { // Serve /llms.txt on template_redirect: front-end only, and late enough // that WooCommerce has registered the product post type + taxonomies so // the catalogue query returns results (init priority 0 would be too early). add_action( 'template_redirect', array( $this, 'maybe_serve_llms_txt' ) ); } // Rebuild the cached llms.txt when the catalogue changes. add_action( 'save_post_product', array( $this, 'flush_llms_cache' ) ); add_action( 'deleted_post', array( $this, 'flush_llms_cache' ) ); add_action( 'woocommerce_update_product', array( $this, 'flush_llms_cache' ) ); /** * Fires when the agent-access module boots. * * @param ARWC_Agent_Access $module The module instance. */ do_action( 'arwc_agent_access_init', $this ); } /** * Whether the AI-crawler robots rules are enabled (default on). * * @return bool */ public function crawlers_enabled() { return '1' === (string) get_option( self::OPT_CRAWLERS, '1' ); } /** * Whether AgenticTrack serves /llms.txt (default on). * * @return bool */ public function llms_enabled() { return '1' === (string) get_option( self::OPT_LLMS, '1' ); } /** * The AI + shopping-agent crawler user-agents to explicitly allow. * * @return string[] */ public static function crawlers() { $crawlers = array( // OpenAI. 'GPTBot', 'ChatGPT-User', 'OAI-SearchBot', // Anthropic. 'ClaudeBot', 'Claude-User', 'Claude-SearchBot', 'anthropic-ai', // Perplexity. 'PerplexityBot', 'Perplexity-User', // Google / Apple / Amazon AI. 'Google-Extended', 'Applebot-Extended', 'Amazonbot', // Meta AI. 'Meta-ExternalAgent', 'Meta-ExternalFetcher', // Others commonly used by agents / AI search. 'CCBot', 'cohere-ai', 'Diffbot', 'YouBot', 'Bytespider', ); /** * Filter the list of AI/agent crawler user-agents allowed in robots.txt. * * @param string[] $crawlers Crawler user-agent tokens. */ return array_values( array_unique( (array) apply_filters( 'arwc_ai_crawlers', $crawlers ) ) ); } /** * Append explicit Allow blocks for AI/agent crawlers to robots.txt. * * Skipped when the site is set to discourage search engines (respecting the * merchant's own privacy choice). * * @param string $output Existing robots.txt content. * @param bool $public Whether the site is public (blog_public). * @return string */ public function filter_robots_txt( $output, $public ) { if ( ! $public ) { return $output; } return $output . "\n" . $this->robots_snippet() . "\n"; } /** * The exact block AgenticTrack appends to robots.txt. * * @return string */ public function robots_snippet() { $lines = array( '# AgenticTrack — AI & shopping agents allowed' ); foreach ( self::crawlers() as $agent ) { $lines[] = 'User-agent: ' . $agent; $lines[] = 'Allow: /'; $lines[] = ''; } if ( $this->llms_enabled() ) { $lines[] = '# AgenticTrack llms.txt'; $lines[] = 'Llms-txt: ' . esc_url_raw( home_url( '/llms.txt' ) ); } return implode( "\n", $lines ) . "\n"; } /** * Build the effective robots.txt in-process for the in-page preview. * * Reproduces WordPress's own robots.txt generation (the same default lines * do_robots() emits) and runs every registered `robots_txt` filter — including * ours and any other plugin's — so the preview matches what agents actually * get, without an unreliable loopback HTTP request. * * @return string */ public function build_robots_preview() { $public = (int) get_option( 'blog_public' ); if ( $public ) { $site = wp_parse_url( site_url() ); $path = ! empty( $site['path'] ) ? $site['path'] : ''; $output = "User-agent: *\n"; $output .= 'Disallow: ' . $path . "/wp-admin/\n"; $output .= 'Allow: ' . $path . "/wp-admin/admin-ajax.php\n"; } else { $output = "User-agent: *\nDisallow: /\n"; } /** This filter is documented in wp-includes/functions.php (do_robots). */ // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound -- Re-running WordPress core's own robots_txt filter to render an in-page preview; not a new hook. return (string) apply_filters( 'robots_txt', $output, $public ); } /** * If the current request is for /llms.txt, output it and stop. * * @return void */ public function maybe_serve_llms_txt() { if ( isset( $_SERVER['REQUEST_METHOD'] ) && 'GET' !== strtoupper( sanitize_text_field( wp_unslash( $_SERVER['REQUEST_METHOD'] ) ) ) ) { return; } $uri = isset( $_SERVER['REQUEST_URI'] ) ? esc_url_raw( wp_unslash( $_SERVER['REQUEST_URI'] ) ) : ''; $path = (string) wp_parse_url( $uri, PHP_URL_PATH ); if ( '/llms.txt' !== untrailingslashit( $path ) ) { return; } nocache_headers(); if ( ! headers_sent() ) { header( 'Content-Type: text/plain; charset=utf-8' ); header( 'X-Robots-Tag: noindex' ); } status_header( 200 ); echo wp_kses_post( $this->get_llms_txt() ); // Plain text; kses is a belt-and-braces guard. exit; } /** * Get the llms.txt body (cached). * * @return string */ public function get_llms_txt() { $cached = get_transient( self::LLMS_TRANSIENT ); if ( is_string( $cached ) && '' !== $cached ) { return $cached; } $body = $this->build_llms_txt(); set_transient( self::LLMS_TRANSIENT, $body, DAY_IN_SECONDS ); return $body; } /** * Clear the cached llms.txt so it rebuilds on next request. * * @return void */ public function flush_llms_cache() { delete_transient( self::LLMS_TRANSIENT ); } /** * Whether the served/last-built llms.txt actually lists products. * * @return bool */ public function llms_has_products() { return false !== strpos( $this->get_llms_txt(), "\n## Products" ); } /** * Build a WooCommerce-aware llms.txt: store intro, product categories, and a * flat list of published products with a one-line factual summary each. * * @return string */ public function build_llms_txt() { $name = wp_strip_all_tags( (string) get_bloginfo( 'name' ) ); $tagline = wp_strip_all_tags( (string) get_bloginfo( 'description' ) ); $out = array(); $out[] = '# ' . $name; $out[] = ''; if ( '' !== $tagline ) { $out[] = '> ' . $tagline; $out[] = ''; } $out[] = 'This is an AI-agent guide to the ' . $name . ' store. It lists products agents can surface, compare and (where supported) purchase.'; $out[] = ''; $out[] = '- Store: ' . esc_url_raw( home_url( '/' ) ); $out[] = '- Generated: ' . gmdate( 'Y-m-d' ); // Stable signature line — lets the plugin verify the LIVE /llms.txt is the // one it serves (vs a static file or another plugin shadowing it). $out[] = '- Source: ' . self::LLMS_SIGNATURE; $out[] = ''; // Categories. $terms = get_terms( array( 'taxonomy' => 'product_cat', 'hide_empty' => true, 'number' => 100, ) ); if ( is_array( $terms ) && ! empty( $terms ) ) { $out[] = '## Product Categories'; $out[] = ''; foreach ( $terms as $term ) { if ( ! $term instanceof WP_Term ) { continue; } $link = get_term_link( $term ); $url = is_wp_error( $link ) ? '' : $link; $out[] = '- [' . wp_strip_all_tags( $term->name ) . '](' . esc_url_raw( $url ) . ') — ' . (int) $term->count . ' products'; } $out[] = ''; } // Products. /** * Filter the maximum number of products listed in llms.txt. * * @param int $limit Max products (default 1000). */ $limit = (int) apply_filters( 'arwc_llms_product_limit', 1000 ); $limit = max( 1, min( 5000, $limit ) ); $products = wc_get_products( array( 'status' => 'publish', 'limit' => $limit, 'orderby' => 'title', 'order' => 'ASC', ) ); if ( ! empty( $products ) ) { $out[] = '## Products'; $out[] = ''; foreach ( $products as $product ) { if ( ! $product instanceof WC_Product ) { continue; } $out[] = $this->product_line( $product ); } $out[] = ''; } $out[] = ''; return implode( "\n", $out ) . "\n"; } /** * One markdown line describing a product for agents: name (linked), price, a * short factual summary, and identifiers when present. * * @param WC_Product $product Product. * @return string */ protected function product_line( WC_Product $product ) { $name = wp_strip_all_tags( $product->get_name() ); $url = get_permalink( $product->get_id() ); $summary = wp_strip_all_tags( $product->get_short_description() ); if ( '' === trim( $summary ) ) { $summary = wp_strip_all_tags( $product->get_description() ); } $summary = trim( preg_replace( '/\s+/', ' ', $summary ) ); if ( '' !== $summary ) { $summary = wp_trim_words( $summary, 30, '…' ); } $facts = array(); $price = wc_get_price_to_display( $product ); if ( '' !== (string) $product->get_price() && $price > 0 ) { $facts[] = html_entity_decode( wp_strip_all_tags( wc_price( $price ) ), ENT_QUOTES, 'UTF-8' ); } $sku = $product->get_sku(); if ( '' !== $sku ) { $facts[] = 'SKU ' . wp_strip_all_tags( $sku ); } if ( is_callable( array( $product, 'get_global_unique_id' ) ) ) { $gtin = (string) $product->get_global_unique_id(); if ( '' !== $gtin ) { $facts[] = 'GTIN ' . preg_replace( '/\D+/', '', $gtin ); } } $line = '- [' . $name . '](' . esc_url_raw( $url ) . ')'; if ( ! empty( $facts ) ) { $line .= ' (' . implode( ', ', $facts ) . ')'; } if ( '' !== $summary ) { $line .= ': ' . $summary; } return $line; } /** * Build a status snapshot for the Agent Access admin screen. * * @return array */ public function status() { $robots = $this->fetch_robots_txt(); $crawlers = self::crawlers(); $allowed = array(); $blocked = array(); foreach ( $crawlers as $bot ) { if ( $this->robots_allows( $robots, $bot ) ) { $allowed[] = $bot; } else { $blocked[] = $bot; } } return array( 'crawlers_enabled' => $this->crawlers_enabled(), 'llms_enabled' => $this->llms_enabled(), 'crawlers_total' => count( $crawlers ), 'crawlers_allowed' => $allowed, 'crawlers_blocked' => $blocked, 'robots_fetched' => ( '' !== $robots ), 'static_robots' => file_exists( ABSPATH . 'robots.txt' ), 'static_llms' => file_exists( ABSPATH . 'llms.txt' ), 'llms_url' => home_url( '/llms.txt' ), 'llms_has_products' => $this->llms_has_products(), 'llms_live' => $this->llms_enabled() ? $this->llms_served_by_us() : 'unknown', 'conflicts' => $this->detect_conflicts(), 'mcp' => $this->mcp_status(), ); } /** * Fetch the site's effective robots.txt over HTTP (loopback). * * @return string */ protected function fetch_robots_txt() { $res = wp_remote_get( home_url( '/robots.txt' ), array( 'timeout' => 8, 'sslverify' => false, ) ); if ( is_wp_error( $res ) || 200 !== (int) wp_remote_retrieve_response_code( $res ) ) { return ''; } return (string) wp_remote_retrieve_body( $res ); } /** * Whether a robots.txt body allows a given crawler (i.e. does not Disallow: /). * * Simple heuristic: if the bot has its own group, honour it; otherwise fall * back to the wildcard group. A group with "Disallow: /" (and no re-allow) * counts as blocked. * * @param string $robots Robots.txt body. * @param string $bot Crawler user-agent token. * @return bool */ protected function robots_allows( $robots, $bot ) { if ( '' === $robots ) { return false; } // A per-bot "Allow: /" block is a clear allow. if ( preg_match( '/User-agent:\s*' . preg_quote( $bot, '/' ) . '\s*\n\s*Allow:\s*\//i', $robots ) ) { return true; } // A per-bot "Disallow: /" block is a clear block. if ( preg_match( '/User-agent:\s*' . preg_quote( $bot, '/' ) . '\s*\n\s*Disallow:\s*\/\s*(\n|$)/i', $robots ) ) { return false; } // Fall back to the wildcard group: blocked only if it Disallows everything. if ( preg_match( '/User-agent:\s*\*\s*\n(?:.*\n)*?\s*Disallow:\s*\/\s*(\n|$)/i', $robots ) ) { return false; } return true; } /** * Known plugins that manage the same agent-access surface (robots/llms.txt). * * @return array plugin file => human label. */ protected function known_conflicts() { return array( 'llmagnet-llm-txt-generator/llmagnet-ai-seo-optimizer.php' => 'LLMagnet (also serves /llms.txt)', 'agentic-commerce-for-woocommerce/agentic-commerce-for-woocommerce.php' => 'Agentic Commerce by xpay (also serves /llms.txt and adds AI-crawler rules)', ); } /** * Detect other active plugins that manage the same surface, to warn about. * * @return string[] Human labels of conflicting plugins. */ protected function detect_conflicts() { if ( ! function_exists( 'is_plugin_active' ) ) { require_once ABSPATH . 'wp-admin/includes/plugin.php'; } $conflicts = array(); foreach ( $this->known_conflicts() as $file => $label ) { if ( is_plugin_active( $file ) ) { $conflicts[] = $label; } } return $conflicts; } /** * Plugin files of the currently-active conflicting plugins. * * @return string[] Active conflicting plugin files. */ public function active_conflict_files() { if ( ! function_exists( 'is_plugin_active' ) ) { require_once ABSPATH . 'wp-admin/includes/plugin.php'; } $files = array(); foreach ( $this->known_conflicts() as $file => $label ) { if ( is_plugin_active( $file ) ) { $files[] = $file; } } return $files; } /** * Whether a static llms.txt exists at the site root (shadowing our dynamic one). * * @return bool */ public function has_static_llms() { return file_exists( ABSPATH . 'llms.txt' ); } /** * Verify — over the live URL — whether /llms.txt is actually the map WE serve. * * A static file OR another plugin (or a rewrite rule) can shadow our dynamic * output even after we delete the root static file, so file-existence alone is * not proof. We loopback-fetch the real URL and look for our signature line. * * @return string 'yes' (ours), 'no' (something else serves it), or 'unknown' * (loopback failed — cannot tell; do not cry wolf). */ public function llms_served_by_us() { $res = wp_remote_get( home_url( '/llms.txt' ), array( 'timeout' => 8, 'sslverify' => false, // Cache-bust so we read the current file, not a proxy/CDN copy. 'headers' => array( 'Cache-Control' => 'no-cache' ), ) ); if ( is_wp_error( $res ) ) { return 'unknown'; } $code = (int) wp_remote_retrieve_response_code( $res ); if ( 200 !== $code ) { // A 404 means nothing is serving /llms.txt at all — our template_redirect // handler should answer, so treat a non-200 as inconclusive. return 'unknown'; } $body = (string) wp_remote_retrieve_body( $res ); if ( '' === trim( $body ) ) { return 'unknown'; } // Match our signature OR the long-standing intro line, so a cache built // before the signature existed (still ours) is not mistaken for a shadow. $ours = ( false !== strpos( $body, self::LLMS_SIGNATURE ) ) || ( false !== strpos( $body, 'This is an AI-agent guide to the' ) ); return $ours ? 'yes' : 'no'; } /** * Delete the static llms.txt file at the site root so our dynamic, product-aware * map takes over. Uses the WordPress filesystem API (never a raw unlink) and only * ever targets the single root-level llms.txt. Returns a status string: * 'deleted' | 'absent' | 'failed'. * * @return string */ public function delete_static_llms() { $file = ABSPATH . 'llms.txt'; if ( ! file_exists( $file ) ) { return 'absent'; } if ( ! function_exists( 'WP_Filesystem' ) ) { require_once ABSPATH . 'wp-admin/includes/file.php'; } // Request direct (no-credentials) access. If the host needs FTP credentials // we fail cleanly rather than prompting mid-redirect. global $wp_filesystem; if ( ! WP_Filesystem() || ! is_object( $wp_filesystem ) ) { return 'failed'; } $deleted = $wp_filesystem->delete( $file, false, 'f' ); if ( ! $deleted ) { return 'failed'; } // Rebuild our dynamic map so /llms.txt now serves products immediately. $this->flush_llms_cache(); return 'deleted'; } /** * Deactivate the conflicting plugins so AgenticTrack owns robots.txt/llms.txt. * * Also clears any rewrite rule they registered for /llms.txt and our own * cached body, so the takeover is immediate. * * @return int Number of plugins deactivated. */ public function deactivate_conflicts() { require_once ABSPATH . 'wp-admin/includes/plugin.php'; $files = $this->active_conflict_files(); if ( ! empty( $files ) ) { deactivate_plugins( $files ); } $this->flush_llms_cache(); flush_rewrite_rules( false ); return count( $files ); } /** * Best-effort status of the WooCommerce MCP / Abilities REST surface. * * @return array */ protected function mcp_status() { $routes = array(); if ( function_exists( 'rest_get_server' ) ) { $server = rest_get_server(); if ( $server ) { $routes = array_keys( $server->get_routes() ); } } $mcp = array_values( array_filter( $routes, static function ( $r ) { return false !== stripos( $r, 'mcp' ) || false !== stripos( $r, 'abilities' ); } ) ); return array( 'available' => ! empty( $mcp ), 'routes' => array_slice( $mcp, 0, 5 ), 'wc_version' => defined( 'WC_VERSION' ) ? WC_VERSION : '', ); } }