get_var( $wpdb->prepare( "SELECT id FROM %i WHERE file_path = %s", $doc_table, $file_path ) ); $data = array( 'title' => $title, 'filename' => $filename, 'mime_type' => $mime_type, 'file_path' => $file_path, 'status' => 'pending', 'created_at' => current_time( 'mysql' ), 'updated_at' => current_time( 'mysql' ), ); if ( $existing_id > 0 ) { $wpdb->update( $doc_table, array( 'status' => 'pending', 'updated_at' => current_time( 'mysql' ) ), array( 'id' => $existing_id ) ); $document_id = $existing_id; } else { $wpdb->insert( $doc_table, $data ); $document_id = (int) $wpdb->insert_id; } if ( $document_id > 0 ) { Logger::info( "Document registered successfully. Starting indexing.", array( 'document_id' => $document_id ) ); self::index_document( $document_id ); return $document_id; } return 0; } /** * Process text extraction, chunking, and vector embedding uploads. * * @param int $document_id Document database ID. * @return bool True on success, false on failure. */ public static function index_document( int $document_id ): bool { global $wpdb; $doc_table = DatabaseManager::get_documents_table(); $chunks_table = DatabaseManager::get_chunks_table(); // Get document record. $doc = $wpdb->get_row( $wpdb->prepare( "SELECT * FROM %i WHERE id = %d", $doc_table, $document_id ) ); if ( ! $doc ) { Logger::error( "Document ID $document_id not found during indexing." ); return false; } // Update status to indexing. $wpdb->update( $doc_table, array( 'status' => 'indexing', 'updated_at' => current_time( 'mysql' ) ), array( 'id' => $document_id ) ); try { // Get settings. $settings = Settings::get_instance(); $chunk_size = $settings->get( 'chunk_size', 700 ); $chunk_overlap = $settings->get( 'chunk_overlap', 100 ); $vector_store = EmbeddingService::get_vector_store(); // 1. Clean up existing chunks in MySQL and Vector Store. $vector_store->delete_document( $document_id ); $wpdb->delete( $chunks_table, array( 'document_id' => $document_id ) ); // 2. Extract Text. $extractor = ExtractorFactory::get_extractor( $doc->mime_type, $doc->filename ); if ( ! $extractor ) { throw new \Exception( 'Unsupported file type format: ' . $doc->mime_type ); } $text = $extractor->extract( $doc->file_path ); if ( empty( trim( $text ) ) ) { if ( 'application/pdf' === $doc->mime_type || 'pdf' === strtolower( pathinfo( $doc->filename, PATHINFO_EXTENSION ) ) ) { throw new \Exception( 'Extracted PDF text content is empty. This document may be scanned or image-only; please ensure it contains selectable text.' ); } throw new \Exception( 'Extracted document text content is empty.' ); } // 3. Chunk Text. $chunks = ChunkingService::chunk( $text, $chunk_size, $chunk_overlap ); if ( empty( $chunks ) ) { throw new \Exception( 'Failed to partition document text into chunks.' ); } Logger::info( sprintf( 'Document chunked into %d elements.', count( $chunks ) ), array( 'document_id' => $document_id ) ); // 4. Batch Embedding Generation and Storage. // Group chunks for batch processing to minimize API requests. $batch_size = 16; $chunks_count = count( $chunks ); for ( $offset = 0; $offset < $chunks_count; $offset += $batch_size ) { $batch = array_slice( $chunks, $offset, $batch_size ); foreach ( $batch as $chunk_key => $chunk ) { $chunk_text = $chunk['chunk_text']; // Generate vector embedding. $vector = EmbeddingService::generate( $chunk_text ); if ( empty( $vector ) ) { throw new \Exception( "Failed to generate vector embedding for chunk #{$chunk['chunk_number']}." ); } // Upsert to local MySQL cache AND selected vector database. $meta = array( 'page_number' => $chunk['page_number'], 'token_count' => $chunk['token_count'], 'filename' => $doc->filename, ); // Local database cache insert. $local_saved = $vector_store->upsert( $document_id, $chunk['chunk_number'], $vector, $chunk_text, $meta ); if ( ! $local_saved ) { throw new \Exception( "Failed to save vector embedding in store for chunk #{$chunk['chunk_number']}." ); } } } // Complete index status. $wpdb->update( $doc_table, array( 'status' => 'completed', 'error_message' => null, 'updated_at' => current_time( 'mysql' ) ), array( 'id' => $document_id ) ); delete_transient( 'ai_support_assistant_statistics' ); // Invalidate statistics cache. Logger::info( "Document ID $document_id indexed successfully." ); return true; } catch ( \Exception $e ) { Logger::error( "Indexing failed for Document ID $document_id: " . $e->getMessage() ); $wpdb->update( $doc_table, array( 'status' => 'failed', 'error_message' => $e->getMessage(), 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $document_id ) ); throw $e; } } /** * Crawl a webpage URL and index it. * * @param string $url Webpage URL. * @return int Document ID on success, 0 on failure. */ public static function crawl_and_index_url( string $url ): int { global $wpdb; $doc_table = DatabaseManager::get_documents_table(); $chunks_table = DatabaseManager::get_chunks_table(); // Check if already registered. $existing_id = (int) $wpdb->get_var( $wpdb->prepare( "SELECT id FROM %i WHERE file_path = %s", $doc_table, $url ) ); // Crawl the webpage. try { $crawled = UrlCrawler::crawl( $url ); } catch ( \Exception $e ) { Logger::error( "Crawl failed for URL $url: " . $e->getMessage() ); // If already registered, set status to failed. if ( $existing_id > 0 ) { $wpdb->update( $doc_table, array( 'status' => 'failed', 'error_message' => $e->getMessage(), 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $existing_id ) ); } throw $e; } $title = $crawled['title']; $text = $crawled['text']; $filename = wp_parse_url( $url, PHP_URL_HOST ) . wp_parse_url( $url, PHP_URL_PATH ); $filename = trim( $filename, '/' ); $data = array( 'title' => $title, 'filename' => $filename, 'mime_type' => 'text/html', 'file_path' => $url, 'status' => 'indexing', 'created_at' => current_time( 'mysql' ), 'updated_at' => current_time( 'mysql' ), ); if ( $existing_id > 0 ) { $wpdb->update( $doc_table, array( 'title' => $title, 'status' => 'indexing', 'error_message' => null, 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $existing_id ) ); $document_id = $existing_id; } else { $wpdb->insert( $doc_table, $data ); $document_id = (int) $wpdb->insert_id; } if ( $document_id <= 0 ) { return 0; } try { // Get settings. $settings = Settings::get_instance(); $chunk_size = $settings->get( 'chunk_size', 700 ); $chunk_overlap = $settings->get( 'chunk_overlap', 100 ); $vector_store = EmbeddingService::get_vector_store(); // 1. Clean up existing chunks in MySQL and Vector Store. $vector_store->delete_document( $document_id ); $wpdb->delete( $chunks_table, array( 'document_id' => $document_id ) ); // 2. Chunk text content. $chunks = ChunkingService::chunk( $text, $chunk_size, $chunk_overlap ); if ( empty( $chunks ) ) { throw new \Exception( 'Failed to partition webpage text into chunks.' ); } Logger::info( sprintf( 'URL %s chunked into %d elements.', $url, count( $chunks ) ), array( 'document_id' => $document_id ) ); // 3. Batch Embedding Generation and Storage. $batch_size = 16; $chunks_count = count( $chunks ); for ( $offset = 0; $offset < $chunks_count; $offset += $batch_size ) { $batch = array_slice( $chunks, $offset, $batch_size ); foreach ( $batch as $chunk ) { $chunk_text = $chunk['chunk_text']; // Generate vector embedding. $vector = EmbeddingService::generate( $chunk_text ); if ( empty( $vector ) ) { throw new \Exception( "Failed to generate vector embedding for chunk #{$chunk['chunk_number']}." ); } $meta = array( 'page_number' => $chunk['page_number'], 'token_count' => $chunk['token_count'], 'filename' => $filename, ); // Local database cache insert. $local_saved = $vector_store->upsert( $document_id, $chunk['chunk_number'], $vector, $chunk_text, $meta ); if ( ! $local_saved ) { throw new \Exception( "Failed to save vector embedding in store for chunk #{$chunk['chunk_number']}." ); } } } // Complete index status. $wpdb->update( $doc_table, array( 'status' => 'completed', 'error_message' => null, 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $document_id ) ); delete_transient( 'ai_support_assistant_statistics' ); // Invalidate statistics cache. Logger::info( "URL ID $document_id indexed successfully." ); return $document_id; } catch ( \Exception $e ) { Logger::error( "Indexing failed for URL ID $document_id: " . $e->getMessage() ); $wpdb->update( $doc_table, array( 'status' => 'failed', 'error_message' => $e->getMessage(), 'updated_at' => current_time( 'mysql' ), ), array( 'id' => $document_id ) ); throw $e; } } /** * Delete a document and all associated text chunks and vector embeddings. * * @param int $document_id Document database ID. * @return bool True on success. */ public static function delete_document( int $document_id ): bool { global $wpdb; $doc_table = DatabaseManager::get_documents_table(); // Delete vectors from active store. $vector_store = EmbeddingService::get_vector_store(); $vector_store->delete_document( $document_id ); // Delete from local cache tables. $wpdb->delete( DatabaseManager::get_chunks_table(), array( 'document_id' => $document_id ) ); $result = $wpdb->delete( $doc_table, array( 'id' => $document_id ) ); delete_transient( 'ai_support_assistant_statistics' ); Logger::info( "Document ID $document_id and all its chunks deleted." ); return false !== $result; } }