[ 'owned' => bool, 'conflict' => bool ], ... ]. */ const OPT_PHYSICAL_STATE = 'adg_aim_llms_physical'; /** * Register hooks. */ public function hooks() { add_action( 'init', array( __CLASS__, 'register_rewrite_rules' ) ); add_filter( 'query_vars', array( $this, 'add_query_vars' ) ); add_action( 'template_redirect', array( $this, 'maybe_serve' ), 0 ); // Clear the short llms.txt cache when content changes. add_action( 'save_post', array( $this, 'clear_cache' ) ); add_action( 'trashed_post', array( $this, 'clear_cache' ) ); add_action( 'deleted_post', array( $this, 'clear_cache' ) ); } /** * Rewrite rules for the virtual files. */ public static function register_rewrite_rules() { add_rewrite_rule( '^llms\.txt$', 'index.php?adg_llms_txt=1', 'top' ); add_rewrite_rule( '^llms-full\.txt$', 'index.php?adg_llms_full=1', 'top' ); } /** * Query vars. * * @param array $vars Existing. * @return array */ public function add_query_vars( $vars ) { $vars[] = 'adg_llms_txt'; $vars[] = 'adg_llms_full'; return $vars; } /** * Clear the cached llms.txt. * * @param int $post_id Post ID. */ public function clear_cache( $post_id ) { if ( wp_is_post_revision( $post_id ) || wp_is_post_autosave( $post_id ) ) { return; } delete_transient( self::TRANSIENT_LLMS ); delete_transient( 'adg_aim_llms_full_physical_synced' ); self::purge_page_caches(); } /** * Ask known page-cache plugins to drop their cached copy of llms.txt / * llms-full.txt, so a full-page cache doesn't keep serving a stale 404 * (or stale content) after the source changes. No-ops for any plugin * that isn't active; never purges more than these two URLs. */ private static function purge_page_caches() { $urls = array( home_url( '/llms.txt' ), home_url( '/llms-full.txt' ) ); if ( function_exists( 'rocket_clean_files' ) ) { rocket_clean_files( $urls ); } if ( has_action( 'litespeed_purge_url' ) ) { foreach ( $urls as $url ) { do_action( 'litespeed_purge_url', $url ); } } if ( function_exists( 'w3tc_flush_url' ) ) { foreach ( $urls as $url ) { w3tc_flush_url( $url ); } } } /** * Public "regenerate now" method (button in the admin). */ public function force_regenerate() { delete_transient( self::TRANSIENT_LLMS ); delete_transient( 'adg_aim_llms_full_physical_synced' ); $this->build_llms_txt(); self::purge_page_caches(); } /** * Serve the file if requested. */ public function maybe_serve() { if ( ! ADG_AIM_Plugin::get_setting( 'llms_enabled', true ) ) { return; } if ( get_query_var( 'adg_llms_txt' ) ) { $this->output( $this->get_llms_txt() ); } if ( get_query_var( 'adg_llms_full' ) ) { $this->output( $this->build_llms_full_txt() ); } } /** * Output the markdown text and exit. * * @param string $content Content. */ private function output( $content ) { nocache_headers(); status_header( 200 ); header( 'Content-Type: text/markdown; charset=UTF-8' ); header( 'X-Robots-Tag: noindex' ); header( 'X-ADG-LLMS: 1' ); // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped -- Markdown served as text/markdown; HTML escaping would corrupt it. echo $content; exit; } /** * Get llms.txt (lazily from cache, otherwise build). * * @return string */ public function get_llms_txt() { $cached = get_transient( self::TRANSIENT_LLMS ); if ( false !== $cached ) { return $cached; } return $this->build_llms_txt(); } /** * Build llms.txt and cache it. * * @return string */ private function build_llms_txt() { $site_name = get_bloginfo( 'name' ); $desc = ADG_AIM_Plugin::get_setting( 'llms_site_desc', '' ); if ( '' === $desc ) { $desc = get_bloginfo( 'description' ); // Tagline. } $out = '# ' . $site_name . "\n\n"; if ( '' !== $desc ) { $out .= '> ' . $desc . "\n\n"; } $types = (array) ADG_AIM_Plugin::get_setting( 'post_types', array( 'post', 'page' ) ); foreach ( $types as $type ) { $obj = get_post_type_object( $type ); if ( ! $obj ) { continue; } $section = $obj->labels->name ? $obj->labels->name : $type; $posts = $this->query_public_posts( $type, 500 ); if ( empty( $posts ) ) { continue; } $out .= '## ' . $section . "\n\n"; foreach ( $posts as $post ) { if ( ADG_AIM_Plugin::is_hidden_from_ai( $post ) ) { continue; } $title = get_the_title( $post ); $url = get_permalink( $post ); $note = $this->get_description( $post ); $line = '- [' . $title . '](' . $url . ')'; if ( '' !== $note ) { $line .= ': ' . $note; } $out .= $line . "\n"; } $out .= "\n"; } $out = rtrim( $out ) . "\n"; set_transient( self::TRANSIENT_LLMS, $out, WEEK_IN_SECONDS ); $this->sync_physical( 'llms.txt', $out ); return $out; } /** * Build llms-full.txt on demand: concatenation of the pages' Markdown with a size cap. * * @return string */ private function build_llms_full_txt() { $site_name = get_bloginfo( 'name' ); $header = '# ' . $site_name . " — Full Content Export\n\n"; $types = (array) ADG_AIM_Plugin::get_setting( 'llms_full_types', array( 'post', 'page' ) ); $all = array(); foreach ( $types as $type ) { $all = array_merge( $all, $this->query_public_posts( $type, 1000 ) ); } usort( $all, static function ( $a, $b ) { return strcmp( $b->post_modified_gmt, $a->post_modified_gmt ); } ); $out = $header; $size = strlen( $out ); foreach ( $all as $post ) { if ( ADG_AIM_Plugin::is_hidden_from_ai( $post ) ) { continue; } // Read the shared Markdown cache; on a miss convert on the fly without writing cache here. $body = ADG_AIM_Markdown_Cache::get( $post, false ); $chunk = "\n\n---\n\n"; $chunk .= ADG_AIM_Frontmatter::build_for_post( $post ); $chunk .= '# ' . get_the_title( $post ) . "\n\n"; $chunk .= $body; $chunk_size = strlen( $chunk ); if ( $size + $chunk_size > self::MAX_FULL_BYTES ) { break; } $out .= $chunk; $size += $chunk_size; } // Full export is rebuilt on every request (not cached); rate-limit the // disk write so a burst of bot requests doesn't hammer the filesystem. if ( false === get_transient( 'adg_aim_llms_full_physical_synced' ) ) { $this->sync_physical( 'llms-full.txt', $out ); set_transient( 'adg_aim_llms_full_physical_synced', 1, HOUR_IN_SECONDS ); } return $out; } /** * Query public published posts of a type. * * @param string $type Post type. * @param int $limit Limit. * @return WP_Post[] */ private function query_public_posts( $type, $limit ) { $q = new WP_Query( array( 'post_type' => $type, 'post_status' => 'publish', 'posts_per_page' => $limit, 'has_password' => false, 'orderby' => 'modified', 'order' => 'DESC', 'no_found_rows' => true, 'update_post_meta_cache' => false, 'update_post_term_cache' => false, 'ignore_sticky_posts' => true, ) ); return $q->posts; } /** * Get a page description: Yoast/RankMath meta → excerpt → content trim. * * @param WP_Post $post Post. * @return string */ private function get_description( WP_Post $post ) { $yoast = get_post_meta( $post->ID, '_yoast_wpseo_metadesc', true ); if ( ! empty( $yoast ) ) { return $this->one_line( $yoast ); } $rank = get_post_meta( $post->ID, 'rank_math_description', true ); if ( ! empty( $rank ) ) { return $this->one_line( $rank ); } if ( ! empty( $post->post_excerpt ) ) { return $this->one_line( $post->post_excerpt ); } $text = wp_strip_all_tags( strip_shortcodes( $post->post_content ) ); $text = $this->one_line( $text ); if ( function_exists( 'mb_strlen' ) && mb_strlen( $text ) > 150 ) { $text = mb_substr( $text, 0, 150 ) . '…'; } return $text; } /** * Collapse to a single line. * * @param string $text Text. * @return string */ private function one_line( $text ) { return trim( preg_replace( '/\s+/u', ' ', (string) $text ) ); } /* --------------------------------------------------------------------- * * Physical (on-disk) fallback. * * Some hosts (managed WordPress edge proxies, WAF rules) block or * intercept requests before WordPress ever boots, so the virtual * rewrite → template_redirect route above never runs. Writing a real * file to the site root lets the webserver serve it directly, the same * way /robots.txt works when a physical copy exists. * --------------------------------------------------------------------- */ /** * Absolute path to the directory a physical llms.txt should live in * (mirrors how WordPress core locates a physical robots.txt). * * @return string */ private function physical_dir() { if ( ! function_exists( 'get_home_path' ) ) { // Core has moved this function before (misc.php → file.php); requiring // both is harmless (require_once) and resilient to either location. require_once ABSPATH . 'wp-admin/includes/file.php'; require_once ABSPATH . 'wp-admin/includes/misc.php'; } return get_home_path(); } /** * Write $filename to the site root and persist ownership/conflict state. * * @param string $filename e.g. 'llms.txt' or 'llms-full.txt'. * @param string $content File content. */ private function sync_physical( $filename, $content ) { $state = get_option( self::OPT_PHYSICAL_STATE, array() ); $owned = ! empty( $state[ $filename ]['owned'] ); $result = ADG_AIM_LLMS_Physical_Writer::write( $this->physical_dir(), $filename, $content, $owned ); $state[ $filename ] = array( 'owned' => $result['owned'], 'conflict' => ( 'conflict' === $result['reason'] ), ); update_option( self::OPT_PHYSICAL_STATE, $state, false ); } /** * Whether the plugin owns (last wrote) the on-disk copy of $filename. * * @param string $filename e.g. 'llms.txt'. * @return bool */ public static function owns_physical( $filename ) { $state = get_option( self::OPT_PHYSICAL_STATE, array() ); return ! empty( $state[ $filename ]['owned'] ); } /** * Filenames currently blocked from being written because a foreign file * occupies that path (for the admin "conflicting file" notice). * * @return string[] */ public static function get_physical_conflicts() { $state = get_option( self::OPT_PHYSICAL_STATE, array() ); $conflicts = array(); foreach ( $state as $filename => $info ) { if ( ! empty( $info['conflict'] ) ) { $conflicts[] = $filename; } } return $conflicts; } /** * Remove physical files the plugin owns (uninstall) and clear the state option. */ public static function cleanup_physical_files() { $instance = new self(); $state = get_option( self::OPT_PHYSICAL_STATE, array() ); foreach ( array( 'llms.txt', 'llms-full.txt' ) as $filename ) { $owned = ! empty( $state[ $filename ]['owned'] ); ADG_AIM_LLMS_Physical_Writer::delete_if_owned( $instance->physical_dir(), $filename, $owned ); } delete_option( self::OPT_PHYSICAL_STATE ); } /** * Run a self-test: fetch /llms.txt and check for the X-ADG-LLMS header. * Stores result in the llms_selftest option. * * @return array [ 'status' => 'ok'|'blocked', 'checked_at' => timestamp, 'details' => string ] */ public static function run_selftest() { if ( ! ADG_AIM_Plugin::get_setting( 'llms_enabled', true ) ) { // Feature is disabled — no point testing. return array( 'status' => 'ok', 'checked_at' => time(), 'details' => 'llms.txt is disabled in settings', ); } $url = home_url( '/llms.txt' ); $response = wp_remote_get( $url, array( 'timeout' => 8, 'sslverify' => apply_filters( 'https_local_ssl_verify', false ), ) ); $result = array( 'status' => 'ok', 'checked_at' => time(), 'details' => '', 'host' => ADG_AIM_Host_Detector::detect(), ); if ( is_wp_error( $response ) ) { $result['status'] = 'blocked'; $result['details'] = 'Request failed: ' . $response->get_error_message(); update_option( ADG_AIM_OPT_LLMS_SELFTEST, $result ); return $result; } $code = (int) wp_remote_retrieve_response_code( $response ); if ( 200 !== $code ) { $result['status'] = 'blocked'; $result['details'] = 'HTTP ' . $code; update_option( ADG_AIM_OPT_LLMS_SELFTEST, $result ); return $result; } // Check for the marker header that proves the plugin's own PHP code was // reached. Its absence is not necessarily a problem any more: if the // on-disk fallback file is doing the serving (the webserver returning // it directly, without ever running WordPress), there is no header to // find, but the content is correct and the site is working. $headers = wp_remote_retrieve_headers( $response ); if ( ! isset( $headers['x-adg-llms'] ) ) { $body = (string) wp_remote_retrieve_body( $response ); if ( self::owns_physical( 'llms.txt' ) && '' !== trim( $body ) ) { $result['details'] = 'Served via the on-disk fallback file — the virtual route is still blocked on this host, but that no longer matters.'; update_option( ADG_AIM_OPT_LLMS_SELFTEST, $result ); return $result; } $result['status'] = 'blocked'; $result['details'] = 'HTTP 200 but X-ADG-LLMS header missing (server block detected)'; update_option( ADG_AIM_OPT_LLMS_SELFTEST, $result ); return $result; } // Success: plugin's code was reached. update_option( ADG_AIM_OPT_LLMS_SELFTEST, $result ); return $result; } }