telemetry = $telemetry; $this->converter = new Markdown_Converter(); } /** * Hook everything up. */ public function register_hooks() { add_action( 'init', array( __CLASS__, 'add_rewrite_rules' ) ); add_filter( 'query_vars', array( $this, 'register_query_vars' ) ); add_action( 'parse_request', array( $this, 'maybe_serve' ), 0 ); add_action( 'wp_update_nav_menu', array( $this, 'invalidate' ) ); add_action( 'ael42a_daily', array( $this, 'regenerate' ) ); } /** * Rewrite rules for both files. Static for activation-time flushing. * Skipped per file when the site already provides it from another * source, so a competing handler's rule wins the pattern. */ public static function add_rewrite_rules() { if ( ! Existing_Content::is_external( 'llms_txt' ) ) { add_rewrite_rule( '^llms\.txt$', 'index.php?' . self::QUERY_VAR . '=txt', 'top' ); } if ( ! Existing_Content::is_external( 'llms_full_txt' ) ) { add_rewrite_rule( '^llms-full\.txt$', 'index.php?' . self::QUERY_VAR . '=full', 'top' ); } } /** * Register the query var. * * @param string[] $vars Public query vars. * @return string[] */ public function register_query_vars( $vars ) { $vars[] = self::QUERY_VAR; return $vars; } /** * Serve at parse_request time - covers both the rewrite-matched case and * a raw REQUEST_URI fallback for sites with broken rewrites. * * @param \WP $wp The WP environment instance. */ public function maybe_serve( $wp ) { // Detection probes must see what the site would serve WITHOUT us. if ( Existing_Content::is_probe_request() ) { return; } $which = isset( $wp->query_vars[ self::QUERY_VAR ] ) ? $wp->query_vars[ self::QUERY_VAR ] : ''; if ( '' === $which ) { $path = $this->request_path(); if ( '/llms.txt' === $path ) { $which = 'txt'; } elseif ( '/llms-full.txt' === $path ) { $which = 'full'; } } if ( 'txt' !== $which && 'full' !== $which ) { return; } // The site already provides this file from another source (physical // file or another plugin) - never override it. if ( Existing_Content::is_external( 'txt' === $which ? 'llms_txt' : 'llms_full_txt' ) ) { return; } $body = ( 'txt' === $which ) ? $this->get_llms_txt() : $this->get_llms_full_txt(); $this->telemetry->increment( 'llms_txt_hits' ); $this->telemetry->record_bot_hit_from_request(); status_header( 200 ); header( 'Content-Type: text/plain; charset=utf-8' ); header( 'X-42A-AEL: ' . AEL42A_VERSION ); echo $body; // phpcs:ignore WordPress.Security.EscapeOutput -- plain text body, not HTML. exit; } /** * Current request path (no query string), relative to the site path. * * @return string */ private function request_path() { $uri = isset( $_SERVER['REQUEST_URI'] ) ? wp_unslash( $_SERVER['REQUEST_URI'] ) : ''; // phpcs:ignore WordPress.Security.ValidatedSanitizedInput -- compared against fixed strings only. $path = (string) wp_parse_url( $uri, PHP_URL_PATH ); // Subdirectory installs: strip the home path prefix. $home_path = rtrim( (string) wp_parse_url( home_url( '/' ), PHP_URL_PATH ), '/' ); if ( '' !== $home_path && 0 === strpos( $path, $home_path . '/' ) ) { $path = substr( $path, strlen( $home_path ) ); } return '/' . ltrim( $path, '/' ); } /** * Get llms.txt content (cached). * * @return string */ public function get_llms_txt() { $cached = get_transient( Cache::LLMS_TXT ); if ( is_string( $cached ) && '' !== $cached ) { return $cached; } $body = $this->build_llms_txt(); set_transient( Cache::LLMS_TXT, $body, self::TTL ); return $body; } /** * Get llms-full.txt content (cached). * * @return string */ public function get_llms_full_txt() { $cached = get_transient( Cache::LLMS_FULL ); if ( is_string( $cached ) && '' !== $cached ) { return $cached; } $body = $this->build_llms_full_txt(); set_transient( Cache::LLMS_FULL, $body, self::TTL ); return $body; } /** * Build llms.txt. * * @return string */ public function build_llms_txt() { $site_name = get_bloginfo( 'name' ); $tagline = get_bloginfo( 'description' ); $sections = array(); // Main pages: front page + primary nav (fallback: published pages). $main_links = array(); $front = $this->front_page_link(); if ( $front ) { $main_links[] = $front; } foreach ( $this->nav_pages( 20 ) as $post ) { if ( $front && (int) $front['id'] === (int) $post->ID ) { continue; } $main_links[] = $this->post_link( $post ); } if ( ! empty( $main_links ) ) { $sections[] = array( 'title' => __( 'Main Pages', 'ael-42a' ), 'links' => $main_links, ); } // Recent posts. $recent_links = array(); foreach ( $this->recent_posts( 'post', 10 ) as $post ) { $recent_links[] = $this->post_link( $post ); } if ( ! empty( $recent_links ) ) { $sections[] = array( 'title' => __( 'Recent Posts', 'ael-42a' ), 'links' => $recent_links, ); } // WooCommerce product categories. if ( class_exists( 'WooCommerce' ) && taxonomy_exists( 'product_cat' ) ) { $product_links = array(); $terms = get_terms( array( 'taxonomy' => 'product_cat', 'hide_empty' => true, 'number' => 15, ) ); if ( is_array( $terms ) ) { foreach ( $terms as $term ) { $term_url = get_term_link( $term ); if ( is_wp_error( $term_url ) ) { continue; } $product_links[] = array( 'id' => 0, 'title' => $term->name, 'url' => $term_url, 'description' => $this->trim_description( $term->description ? $term->description : sprintf( /* translators: %d: number of products. */ __( '%d products', 'ael-42a' ), (int) $term->count ) ), ); } } if ( ! empty( $product_links ) ) { $sections[] = array( 'title' => __( 'Products', 'ael-42a' ), 'links' => $product_links, ); } } // Optional resources. $sections[] = array( 'title' => __( 'Optional', 'ael-42a' ), 'links' => array( array( 'id' => 0, 'title' => 'llms-full.txt', 'url' => home_url( '/llms-full.txt' ), 'description' => __( 'Full markdown content of the main pages of this site in one file', 'ael-42a' ), ), array( 'id' => 0, 'title' => __( 'MCP server', 'ael-42a' ), 'url' => rest_url( '42a/v1/mcp' ), 'description' => __( 'Model Context Protocol endpoint for structured access to this site', 'ael-42a' ), ), array( 'id' => 0, 'title' => __( 'MCP discovery', 'ael-42a' ), 'url' => home_url( '/.well-known/mcp.json' ), 'description' => __( 'MCP server discovery document', 'ael-42a' ), ), ), ); /** * Filter the llms.txt sections. * * @param array $sections Array of { title, links: [ { id, title, url, description } ] }. */ $sections = apply_filters( 'ael42a_llms_txt_sections', $sections ); $lines = array(); $lines[] = '# ' . $site_name; $lines[] = ''; if ( '' !== trim( (string) $tagline ) ) { $lines[] = '> ' . $this->one_line( $tagline ); $lines[] = ''; } $lines[] = sprintf( /* translators: 1: site name. */ __( 'This file lists the key pages of %1$s for AI agents and LLMs. Every page on this site is also available as markdown: append ?format=markdown to any URL. A machine-readable MCP server is available for structured access.', 'ael-42a' ), $site_name ); foreach ( $sections as $section ) { if ( empty( $section['links'] ) ) { continue; } $lines[] = ''; $lines[] = '## ' . $this->one_line( $section['title'] ); $lines[] = ''; foreach ( $section['links'] as $link ) { $entry = '- [' . $this->one_line( $link['title'] ) . '](' . $link['url'] . ')'; if ( ! empty( $link['description'] ) ) { $entry .= ': ' . $this->one_line( $link['description'] ); } $lines[] = $entry; } } return implode( "\n", $lines ) . "\n"; } /** * Build llms-full.txt: header + concatenated markdown documents. * * @return string */ public function build_llms_full_txt() { $site_name = get_bloginfo( 'name' ); $tagline = get_bloginfo( 'description' ); $limit = (int) get_option( 'ael42a_full_limit', self::FULL_ITEM_LIMIT ); if ( $limit < 1 || $limit > self::FULL_ITEM_LIMIT ) { $limit = self::FULL_ITEM_LIMIT; } $header = array(); $header[] = '# ' . $site_name . ' - llms-full.txt'; $header[] = ''; if ( '' !== trim( (string) $tagline ) ) { $header[] = '> ' . $this->one_line( $tagline ); $header[] = ''; } $header[] = sprintf( /* translators: 1: generator id, 2: date. */ __( 'Concatenated markdown of the main pages of this site. Generated by %1$s on %2$s. Individual pages: append ?format=markdown to any URL.', 'ael-42a' ), 'ael-42a/' . AEL42A_VERSION, gmdate( 'Y-m-d\TH:i:s\Z' ) ); $out = implode( "\n", $header ); $bytes = strlen( $out ); $seen = array(); $posts = array(); $front_id = (int) get_option( 'page_on_front' ); if ( $front_id > 0 ) { $front = get_post( $front_id ); if ( $front instanceof \WP_Post && 'publish' === $front->post_status ) { $posts[] = $front; } } foreach ( $this->nav_pages( $limit ) as $page ) { $posts[] = $page; } foreach ( $this->recent_posts( 'post', $limit ) as $post ) { $posts[] = $post; } $count = 0; foreach ( $posts as $post ) { if ( $count >= $limit ) { break; } if ( isset( $seen[ $post->ID ] ) || post_password_required( $post ) ) { continue; } $seen[ $post->ID ] = true; $doc = $this->converter->document( $post ); $piece = "\n\n---\n\n" . rtrim( $doc ); if ( $bytes + strlen( $piece ) > self::FULL_BYTE_CAP ) { break; } $out .= $piece; $bytes += strlen( $piece ); $count++; } return $out . "\n"; } /** * Invalidate both cached files (save_post is covered by Cache hooks). */ public function invalidate() { delete_transient( Cache::LLMS_TXT ); delete_transient( Cache::LLMS_FULL ); } /** * Daily cron: rebuild both transients so cold hits stay fast. */ public function regenerate() { $this->invalidate(); $this->get_llms_txt(); $this->get_llms_full_txt(); } /** * Front page link row, if a static front page is configured. * * @return array|null */ private function front_page_link() { $front_id = (int) get_option( 'page_on_front' ); if ( $front_id > 0 ) { $front = get_post( $front_id ); if ( $front instanceof \WP_Post && 'publish' === $front->post_status ) { return $this->post_link( $front ); } return null; } return array( 'id' => 0, 'title' => __( 'Home', 'ael-42a' ), 'url' => home_url( '/' ), 'description' => $this->trim_description( get_bloginfo( 'description' ) ), ); } /** * Pages from the primary nav menu; fallback to all published pages. * * @param int $limit Max pages. * @return \WP_Post[] */ private function nav_pages( $limit ) { $pages = array(); $locations = get_nav_menu_locations(); $menu_id = 0; foreach ( array( 'primary', 'main', 'header', 'menu-1', 'top' ) as $slot ) { if ( ! empty( $locations[ $slot ] ) ) { $menu_id = (int) $locations[ $slot ]; break; } } if ( 0 === $menu_id && ! empty( $locations ) ) { $menu_id = (int) reset( $locations ); } if ( $menu_id > 0 ) { $items = wp_get_nav_menu_items( $menu_id ); if ( is_array( $items ) ) { foreach ( $items as $item ) { if ( count( $pages ) >= $limit ) { break; } if ( 'post_type' !== $item->type ) { continue; } $post = get_post( (int) $item->object_id ); if ( $post instanceof \WP_Post && 'publish' === $post->post_status && is_post_publicly_viewable( $post ) ) { $pages[ $post->ID ] = $post; } } } } if ( empty( $pages ) ) { $all = get_pages( array( 'sort_column' => 'menu_order,post_title', 'number' => $limit, ) ); if ( is_array( $all ) ) { foreach ( $all as $page ) { $pages[ $page->ID ] = $page; } } } return array_values( $pages ); } /** * Recent published posts of a type. * * @param string $post_type Post type. * @param int $limit Max posts. * @return \WP_Post[] */ private function recent_posts( $post_type, $limit ) { $query = new \WP_Query( array( 'post_type' => $post_type, 'post_status' => 'publish', 'posts_per_page' => $limit, 'orderby' => 'date', 'order' => 'DESC', 'no_found_rows' => true, 'ignore_sticky_posts' => true, ) ); return $query->posts; } /** * Link row for a post: markdown URL + short description. * * @param \WP_Post $post Post. * @return array */ private function post_link( $post ) { return array( 'id' => $post->ID, 'title' => get_the_title( $post ), 'url' => Markdown_Controller::markdown_url( $post ), 'description' => $this->post_description( $post ), ); } /** * Short plain-text description of a post. * * @param \WP_Post $post Post. * @return string */ private function post_description( $post ) { $text = $post->post_excerpt; if ( '' === trim( (string) $text ) ) { $text = $post->post_content; } return $this->trim_description( $text ); } /** * Strip tags/shortcodes and trim to ~25 words. * * @param string $text Raw text. * @return string */ private function trim_description( $text ) { $text = wp_strip_all_tags( strip_shortcodes( (string) $text ) ); $text = preg_replace( '/\s+/u', ' ', $text ); return wp_trim_words( trim( (string) $text ), 25, '...' ); } /** * Force a string onto one line. * * @param string $text Text. * @return string */ private function one_line( $text ) { return trim( preg_replace( '/\s+/u', ' ', (string) $text ) ); } }