settings = $settings; $this->chunker = new MYBOT_Chunker(); } /** * Register the queue handler. * * @param array $handlers Existing handler map. * @return array */ public function register_handlers( $handlers ) { $handlers[ self::JOB_TYPE ] = array( $this, 'handle_job' ); return $handlers; } /** * Register hooks that track content changes. * * @return void */ public function register_hooks() { add_action( 'save_post', array( $this, 'on_save_post' ), 20, 3 ); add_action( 'before_delete_post', array( $this, 'on_delete_post' ) ); add_action( 'wp_trash_post', array( $this, 'on_delete_post' ) ); } /** * Mark a post for re-indexing when its content changes. * * @param int $post_id Post ID. * @param WP_Post $post Post object. * @param bool $update Whether this is an update or a new post. * @return void */ public function on_save_post( $post_id, $post, $update ) { if ( wp_is_post_revision( $post_id ) || wp_is_post_autosave( $post_id ) ) { return; } if ( ! $post instanceof WP_Post || 'publish' !== $post->post_status ) { return; } if ( ! $this->is_indexable( $post ) ) { return; } if ( ! $this->settings->rag_available() ) { return; } unset( $update ); mybot()->queue->push( self::JOB_TYPE, array( 'post_ids' => array( (int) $post_id ) ), 30 ); } /** * Delete vectors when a post is deleted. * * @param int $post_id Post ID. * @return void */ public function on_delete_post( $post_id ) { $store = MYBOT_Vector_Factory::make( $this->settings ); if ( is_wp_error( $store ) ) { return; } $store->delete_by_source( (int) $post_id ); } /** * Whether this post should be indexed. * * Knowledge imported from files is always indexed, regardless of the * "Content types to index" selection in the settings. * * @param WP_Post $post The post. * @return bool */ protected function is_indexable( $post ) { if ( MYBOT_Knowledge_CPT::is_knowledge( $post->post_type ) ) { return true; } return in_array( $post->post_type, $this->indexable_types(), true ); } /** * List of post types to index, always including the knowledge repository. * * @return array */ protected function indexable_types() { $types = (array) $this->settings->get( 'index_post_types', array( 'post', 'page' ) ); $types[] = MYBOT_Knowledge_CPT::POST_TYPE; /** * Filters the post types that indexing walks. * * An add-on that knows about content living outside WordPress — WooCommerce product variations in their * own tables, a CRM mirror, a documentation site — needs the indexer to see those as indexable types, * or it has to reimplement chunking, embedding and the fingerprint itself. This is the seam that lets * it use the pipeline instead (docs/13 muc 5). * * `post` and `page` stay in by default, so a site with no add-on walks exactly what it walked before. * * @param array $types Post type names. */ return array_values( array_unique( apply_filters( 'mybot_register_knowledge_sources', $types ) ) ); } /** * Start indexing all knowledge. * * @param bool $force Re-index even posts that have not changed. * @return array|WP_Error Statistics on the posts that were queued. */ public function start_full_index( $force = false ) { if ( ! $this->settings->rag_available() ) { return new WP_Error( 'mybot_rag_unavailable', __( 'RAG is disabled, or the embedding and vector store keys are not fully configured.', 'aihouse' ) ); } $types = $this->indexable_types(); $post_ids = get_posts( array( 'post_type' => $types, 'post_status' => 'publish', 'numberposts' => -1, 'fields' => 'ids', 'suppress_filters' => false, 'orderby' => 'ID', 'order' => 'ASC', ) ); if ( empty( $post_ids ) ) { return new WP_Error( 'mybot_nothing_to_index', __( 'No content found to index.', 'aihouse' ) ); } $queued = 0; /* * The admin controls this with "Items per batch". The constant is only the fallback: * a hard-coded value here meant the setting was saved, shown, and never read, so * changing it did nothing at all. */ $per_job = max( 1, (int) $this->settings->get( 'index_batch_size', self::POSTS_PER_JOB ) ); $batches = array_chunk( array_map( 'intval', $post_ids ), $per_job ); foreach ( $batches as $batch ) { if ( ! $force ) { $batch = array_values( array_filter( $batch, array( $this, 'needs_indexing' ) ) ); } if ( empty( $batch ) ) { continue; } $pushed = mybot()->queue->push( self::JOB_TYPE, array( 'post_ids' => $batch ) ); if ( $pushed ) { $queued += count( $batch ); } } /* * How many items this run decided NOT to touch, because their content is unchanged since * the last run. It is stored rather than only returned, because the card that has to * explain the progress figure is usually read after a page reload, when the REST reply of * the run is long gone. Two counters described by the same run must be kept together: * `processed` never passes `queued`, and `skipped` is the rest of `total` * (docs/08 muc 6.44). */ $total = count( $post_ids ); $skipped = max( 0, $total - $queued ); update_option( MYBOT_INDEX_STATE_OPTION, array( 'started_at' => time(), 'queued' => $queued, 'total' => $total, 'skipped' => $skipped, 'force' => (bool) $force, /* * Reset on every run, and it must be: `update_progress()` ADDS to this counter, while * `total` above only ever describes the run that just started. Without this line the * second run of the same content reported "84 / 42" — a ratio that cannot exist and * that made the progress bar jump to 100% before any work was done (docs/08 muc * 3.37(h), measured on a real site on 06/10/2026). */ 'processed' => 0, /* * Re-armed on every run: the write-then-read-back check below runs once per * indexing run, so a fresh run is the moment to ask the question again. */ self::SELF_CHECK_KEY => array(), ), false ); return array( 'queued' => $queued, 'skipped' => $skipped, 'total' => $total, ); } /** * Whether this post needs to be re-indexed. * * @param int $post_id Post ID. * @return bool */ public function needs_indexing( $post_id ) { $post = get_post( $post_id ); if ( ! $post instanceof WP_Post || 'publish' !== $post->post_status ) { return false; } $stored = (string) get_post_meta( $post_id, self::HASH_META, true ); if ( '' === $stored ) { return true; } return $stored !== $this->fingerprint( $post ); } /** * Content fingerprint; changes when the post is edited or the chunking settings change. * * @param WP_Post $post The post. * @return string */ protected function fingerprint( $post ) { return md5( implode( '|', array( $post->ID, $post->post_modified_gmt, (string) $this->settings->get( 'chunk_size', 1200 ), (string) $this->settings->get( 'chunk_overlap', 200 ), (string) $this->settings->get( 'embedding_provider', 'openai' ), $this->settings->embedding_model(), ) ) ); } /** * Handle an indexing job. * * @param array $payload Job data. * @return void * @throws Exception When the time budget runs out, so the queue retries the remainder. */ public function handle_job( $payload ) { $post_ids = isset( $payload['post_ids'] ) && is_array( $payload['post_ids'] ) ? array_map( 'intval', $payload['post_ids'] ) : array(); if ( empty( $post_ids ) ) { return; } $store = MYBOT_Vector_Factory::make( $this->settings ); if ( is_wp_error( $store ) ) { // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- an exception message is not HTML output: it becomes a WP_Error or a notice, and it is escaped where it is displayed throw new Exception( $store->get_error_message() ); } $embedder = MYBOT_LLM_Factory::embedding_provider( $this->settings ); if ( is_wp_error( $embedder ) ) { // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- an exception message is not HTML output: it becomes a WP_Error or a notice, and it is escaped where it is displayed throw new Exception( $embedder->get_error_message() ); } $started = time(); $done = 0; foreach ( $post_ids as $index => $post_id ) { if ( ( time() - $started ) > self::TIME_BUDGET ) { // Push the remainder back onto the queue and stop. $remaining = array_slice( $post_ids, $index ); if ( ! empty( $remaining ) ) { mybot()->queue->push( self::JOB_TYPE, array( 'post_ids' => $remaining ), 10 ); } break; } $this->index_post( $post_id, $store, $embedder ); ++$done; } $this->update_progress( $done ); } /** * Index a single post. * * @param int $post_id Post ID. * @param MYBOT_Vector_Store $store Vector store. * @param MYBOT_LLM_Provider $embedder Embedding provider. * @return void */ protected function index_post( $post_id, $store, $embedder ) { $post = get_post( $post_id ); if ( ! $post instanceof WP_Post || 'publish' !== $post->post_status || ! $this->is_indexable( $post ) ) { return; } $text = MYBOT_Chunker::to_text( $post->post_content ); $text = $post->post_title . "\n\n" . $text; if ( MYBOT_Chunker::length( trim( $text ) ) < 40 ) { return; } $chunks = $this->chunker->split( $text, (int) $this->settings->get( 'chunk_size', 1200 ), (int) $this->settings->get( 'chunk_overlap', 200 ) ); if ( empty( $chunks ) ) { return; } // Delete old vectors before writing new ones to avoid stale data. $store->delete_by_source( $post_id ); $model = $this->settings->embedding_model(); $records = array(); foreach ( array_chunk( $chunks, self::EMBED_BATCH ) as $offset => $batch ) { $vectors = $embedder->embed( $batch, $model ); if ( is_wp_error( $vectors ) ) { // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- an exception message is not HTML output: it becomes a WP_Error or a notice, and it is escaped where it is displayed throw new Exception( $vectors->get_error_message() ); } /* * A short answer is a broken answer, not a partial one. Skipping the chunks that came * back without a vector used to store the rest of the post under its fingerprint anyway, * so "Index missing items" would never come back for the missing chunks and no counter * would ever show a problem — the same silent failure shape as 6.22. */ if ( ! is_array( $vectors ) || count( $vectors ) !== count( $batch ) ) { $mybot_short_answer = sprintf( /* translators: 1: number of chunks sent, 2: number of vectors returned, 3: post ID */ __( 'The embedding provider returned %2$d vectors for %1$d chunks (post %3$d). Nothing was written for this post: everything is re-sent on the next indexing run.', 'aihouse' ), count( $batch ), is_array( $vectors ) ? count( $vectors ) : 0, (int) $post_id ); // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- an exception message is not HTML output: it becomes a WP_Error or a notice, and it is escaped where it is displayed. throw new Exception( $mybot_short_answer ); } foreach ( $batch as $position => $chunk ) { if ( ! isset( $vectors[ $position ] ) ) { continue; } $chunk_index = ( $offset * self::EMBED_BATCH ) + $position; $records[] = array( 'id' => $post_id . '-' . $chunk_index, 'values' => $vectors[ $position ], 'content' => $chunk, 'metadata' => array( 'source_id' => (int) $post_id, 'source_type' => $post->post_type, 'title' => $post->post_title, 'url' => MYBOT_Citations::source_url( $post_id ), 'chunk_index' => (int) $chunk_index, 'modified' => $post->post_modified_gmt, ), ); } } if ( empty( $records ) ) { return; } $result = $store->upsert( $records ); if ( is_wp_error( $result ) ) { // phpcs:ignore WordPress.Security.EscapeOutput.ExceptionNotEscaped -- an exception message is not HTML output: it becomes a WP_Error or a notice, and it is escaped where it is displayed throw new Exception( $result->get_error_message() ); } /* * One extra read per indexing run, not per post: the store is asked to find a vector it * has just accepted. Every counter the plugin had until now measured the write ("26/26, * Error = 0") and none of them could see that the vector stored under a chunk belonged to * a different chunk or a different model (6.22, 6.23). This is the measurement that can. */ if ( $this->self_check_pending() ) { $this->self_check( $store, $records[0], $post_id ); } update_post_meta( $post_id, self::HASH_META, $this->fingerprint( $post ) ); /** * Fires after one source has been written to the vector store. * * This is the only place that knows BOTH how many chunks a source produced and which source it was, and * that is exactly what a cost meter needs: embedding is billed per token per source, and until this * action existed nothing could say "these chunks cost money because of that product page" * (docs/13 muc 7.1). * * It does NOT fire for content that produced no chunks — the early returns above mean a post too short * to index is never reported as an indexing cost, which is correct: nothing was sent to the provider. * * It fires AFTER the fingerprint is written, so a listener that itself fails cannot make the post look * un-indexed and have it re-sent on every run. * * @param int $post_id Source post ID. * @param string $type Post type. * @param int $chunks Number of chunks written. */ do_action( 'mybot_knowledge_ingested', (int) $post_id, (string) $post->post_type, count( $records ) ); } /** * Whether the write-then-read-back check still has to run in this indexing run. * * @return bool */ protected function self_check_pending() { $state = get_option( MYBOT_INDEX_STATE_OPTION, array() ); if ( ! is_array( $state ) ) { return true; } return empty( $state[ self::SELF_CHECK_KEY ] ); } /** * Asks the store to find back a vector it has just been given. * * Deliberately never throws and never blocks the write: the chunks are in the store either * way, and failing the job here would leave the post without a fingerprint and retry forever. * The result is kept in the index state, where the Status card and the progress API read it, * and where a wrong embedding model becomes a number instead of a mystery. * * @param MYBOT_Vector_Store $store Vector store. * @param array $record First record that was just written. * @param int $post_id Post the record came from. * @return void */ protected function self_check( $store, $record, $post_id ) { $id = isset( $record['id'] ) ? (string) $record['id'] : ''; $vector = isset( $record['values'] ) && is_array( $record['values'] ) ? $record['values'] : array(); if ( '' === $id || empty( $vector ) ) { return; } // The threshold is the same one the "Look this up" round trip uses, so the two // measurements can never disagree about what "the store recognises its own text" means. $results = $store->query( $vector, 1, 0.0 ); /* * Pinecone-style stores make a fresh write visible a moment later, so an empty answer on * the first try is not yet evidence of anything. One second and one more question keeps * the check honest without turning eventually-consistent stores into false alarms; this * runs in a background job, once per indexing run. */ if ( ! is_wp_error( $results ) && empty( $results ) ) { usleep( 1000000 ); $results = $store->query( $vector, 1, 0.0 ); } if ( is_wp_error( $results ) ) { $this->record_self_check( array( 'ok' => false, 'post_id' => (int) $post_id, 'id' => $id, 'score' => null, 'message' => sprintf( /* translators: %s: error message from the vector store */ __( 'The vector was written, but reading it back failed: %s', 'aihouse' ), $results->get_error_message() ), ) ); return; } if ( empty( $results ) || ! is_array( $results[0] ) ) { $this->record_self_check( array( 'ok' => false, 'post_id' => (int) $post_id, 'id' => $id, 'score' => null, 'message' => __( 'The vector store accepted a chunk and then returned no rows when asked for that exact vector, so indexing writes into a table it cannot read back. Check the API key: with Supabase the store needs the service_role key, because row level security is on with no policy.', 'aihouse' ), ) ); return; } $top = $results[0]; $score = isset( $top['score'] ) ? (float) $top['score'] : 0.0; $found = isset( $top['id'] ) ? (string) $top['id'] : ''; $ok = $score >= MYBOT_Retriever::SELF_MATCH_MIN; if ( $ok ) { $message = sprintf( /* translators: 1: score, 2: record ID */ __( 'The store found the chunk it had just been given, at %1$s (stored as %2$s): the vectors match the embedding model in use.', 'aihouse' ), MYBOT_Retriever::format_score( $score ), $id ); } else { $message = sprintf( /* translators: 1: score, 2: record ID the store answered with */ __( 'The store answered with %2$s at %1$s for a vector it had just accepted, so the stored vectors were made by a different embedding model and every similarity score is meaningless. Re-index everything with the model now selected before trusting any citation.', 'aihouse' ), MYBOT_Retriever::format_score( $score ), '' !== $found ? $found : __( 'another chunk', 'aihouse' ) ); } $this->record_self_check( array( 'ok' => $ok, 'post_id' => (int) $post_id, 'id' => $id, 'found' => $found, 'score' => $score, 'message' => $message, ) ); } /** * Stores the result of the write-then-read-back check. * * @param array $result Result data. * @return void */ protected function record_self_check( $result ) { $state = get_option( MYBOT_INDEX_STATE_OPTION, array() ); if ( ! is_array( $state ) ) { $state = array(); } $result['at'] = time(); $state[ self::SELF_CHECK_KEY ] = $result; update_option( MYBOT_INDEX_STATE_OPTION, $state, false ); } /** * Update progress so the administrator can follow along. * * @param int $processed Number of posts just processed. * @return void */ protected function update_progress( $processed ) { $state = get_option( MYBOT_INDEX_STATE_OPTION, array() ); if ( ! is_array( $state ) ) { $state = array(); } $state['processed'] = ( isset( $state['processed'] ) ? (int) $state['processed'] : 0 ) + (int) $processed; $state['updated_at'] = time(); update_option( MYBOT_INDEX_STATE_OPTION, $state, false ); } /** * How much of the site carries an indexing fingerprint. * * The "Get started" screen needs one sentence a new administrator can act on: "12 of 42 items have been * indexed". Counting `processed` would be wrong — that counter describes ONE run and resets on the next * one, so on a site that was indexed last month it reads zero — and reading the whole manifest into * memory to count it would be worse on a site with ten thousand posts. So this asks the database for * two numbers, and says so when it stopped early rather than reporting a partial count as a total * (docs/08 muc 6.31: a number with no way of knowing it is incomplete). * * @param int $limit Cap on the posts examined. * @return array{indexed:int,published:int,complete:bool} */ public function fingerprint_coverage( $limit = 2000 ) { $limit = max( 1, (int) $limit ); $published = (int) wp_count_posts( 'post' )->publish + (int) wp_count_posts( 'page' )->publish; foreach ( $this->indexable_types() as $type ) { if ( in_array( $type, array( 'post', 'page' ), true ) ) { continue; } $count = wp_count_posts( $type ); if ( $count && isset( $count->publish ) ) { $published += (int) $count->publish; } } $indexed = get_posts( array( 'post_type' => $this->indexable_types(), 'post_status' => 'publish', 'numberposts' => $limit, 'fields' => 'ids', 'suppress_filters' => false, 'meta_key' => self::HASH_META, 'meta_compare' => 'EXISTS', ) ); return array( 'indexed' => count( $indexed ), 'published' => $published, 'complete' => count( $indexed ) < $limit, ); } /** * Indexing status for the administrator. * * @return array */ public function get_progress() { $state = get_option( MYBOT_INDEX_STATE_OPTION, array() ); $state = is_array( $state ) ? $state : array(); $pending = mybot()->queue->pending_count( self::JOB_TYPE ); return array( 'started_at' => isset( $state['started_at'] ) ? (int) $state['started_at'] : 0, 'total' => isset( $state['total'] ) ? (int) $state['total'] : 0, 'queued' => isset( $state['queued'] ) ? (int) $state['queued'] : 0, 'skipped' => isset( $state['skipped'] ) ? (int) $state['skipped'] : 0, 'processed' => isset( $state['processed'] ) ? (int) $state['processed'] : 0, 'pending' => $pending, 'running' => $pending > 0, 'self_check' => ( isset( $state[ self::SELF_CHECK_KEY ] ) && is_array( $state[ self::SELF_CHECK_KEY ] ) ) ? $state[ self::SELF_CHECK_KEY ] : array(), ); } /** * Clear the indexed markers for all content, forcing a full re-index. * * @return int Number of posts whose markers were removed. */ public function reset_manifest() { global $wpdb; // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery, WordPress.DB.DirectDatabaseQuery.NoCaching return (int) $wpdb->delete( $wpdb->postmeta, array( 'meta_key' => self::HASH_META ) ); } }