shard 0, next 10K -> shard 1. */ public const MAX_PRODUCTS_PER_SHARD = 10000; /** * Maximum compressed shard size in bytes (5 MB). * Triggers new shard creation during build when exceeded. */ public const MAX_SHARD_SIZE_BYTES = 5 * 1024 * 1024; /** * Estimated compression ratio for TOON+gzip. * * Conservative value of 4 chosen to prevent shard overflow. * Actual ratio varies 5-7x depending on data, but 4x provides * safe margin for worst-case scenarios (short text, lots of numbers). */ private const ESTIMATED_COMPRESSION_RATIO = 4; /** * Safety threshold for shard size (80% of max). * * Triggers shard split at 80% to leave buffer for: * - Compression ratio variance * - Category metadata overhead * - Final encoding overhead */ private const SIZE_SAFETY_THRESHOLD = 0.80; /** * Index directory relative to uploads. */ private const INDEX_DIR = 'ai-product-tools/ai-index'; /** * Option holding the per-site token in the map file name (autoload off). */ public const MAP_TOKEN_OPTION = 'aipt_store_index_map_token'; /** * Profile ID. * * @var int */ private $profile_id; /** * ToonEncoder instance. * * @var ToonEncoder */ private $encoder; /** * ToonDecoder instance. * * @var ToonDecoder */ private $decoder; /** * Shard cache for read operations. * * @var ShardCache */ private $shard_cache; /** * Loaded map data (cached after first read). * * @var array|null */ private $map = null; /** * Current build state (only used during BUILD operations). * * @var array{ * shards: array, * current_shard_id: int, * category_distribution: array * }|null */ private $build_state = null; /** * Last delete/cleanup result (set by deleteAllShards() and * cleanupOrphanTempFiles()). * * Shape: ['ok' => bool, 'deleted' => string[], 'failed' => array] * * Kept on the instance rather than returned directly so the existing * bool return signatures stay stable for callers that don't need * partial-failure detail. Inspect via {@see self::getLastDeleteResult()}. * * @var array|null */ private $last_delete_result = null; /** * Last structured load error (set by loadMap()/loadShard() on failure). * * Shape: * - kind: 'missing_file' | 'read_failed' | 'gzdecode_failed' * | 'decoder_non_array' | 'version_too_new' * - path: absolute path of the file (when relevant) * - found: int (only when kind=version_too_new — observed map_version) * - expected: int (only when kind=version_too_new — MAP_VERSION_READER) * * Cleared at the start of each load attempt so a successful read leaves * a clean slate. * * @var array|null */ private $last_load_error = null; /** * Constructor. * * @param int $profile_id Profile ID. */ public function __construct( int $profile_id ) { $this->profile_id = $profile_id; $this->encoder = new ToonEncoder(); $this->decoder = new ToonDecoder(); $this->shard_cache = new ShardCache(); } /** * Initialize build state for a new sharded index build. * * Called at the start of IndexBuilder batch processing. * Resets all shard state and prepares for sequential product assignment. * * @return void */ public function initBuild(): void { $this->build_state = array( 'shards' => array(), 'current_shard_id' => 0, 'category_distribution' => array(), 'total_products' => 0, ); $this->build_state['shards'][0] = array( 'products' => array(), 'categories' => array(), 'estimated_size' => 0, ); } /** * Add a product to the appropriate shard during build. * * Automatically creates new shards when limits are reached. * Uses incremental size tracking (O(1)) to prevent shard overflow. * Tracks category distribution for map file. * * @param array $product Product profile data with 'id', 'categories', etc. * @return int Shard ID where product was added. * @throws \RuntimeException If build not initialized. */ public function addProduct( array $product ): int { if ( $this->build_state === null ) { throw new \RuntimeException( 'Build not initialized. Call initBuild() first.' ); } $shard_id = $this->build_state['current_shard_id']; $current_shard = &$this->build_state['shards'][ $shard_id ]; $product_estimated_size = $this->estimateProductSize( $product ); $size_threshold = (int) ( self::MAX_SHARD_SIZE_BYTES * self::SIZE_SAFETY_THRESHOLD ); $current_estimated_size = $current_shard['estimated_size'] ?? 0; $needs_new_shard = false; if ( count( $current_shard['products'] ) >= self::MAX_PRODUCTS_PER_SHARD ) { $needs_new_shard = true; } if ( ! $needs_new_shard && ( $current_estimated_size + $product_estimated_size ) > $size_threshold ) { if ( count( $current_shard['products'] ) > 0 ) { $needs_new_shard = true; } } if ( $needs_new_shard ) { ++$this->build_state['current_shard_id']; $shard_id = $this->build_state['current_shard_id']; $this->build_state['shards'][ $shard_id ] = array( 'products' => array(), 'categories' => array(), 'estimated_size' => 0, ); $current_shard = &$this->build_state['shards'][ $shard_id ]; } $product_id = (string) $product['id']; $current_shard['products'][ $product_id ] = $product; $current_shard['estimated_size'] = ( $current_shard['estimated_size'] ?? 0 ) + $product_estimated_size; if ( ! empty( $product['categories'] ) && is_array( $product['categories'] ) ) { foreach ( $product['categories'] as $cat_id ) { $cat_key = (string) $cat_id; if ( ! isset( $current_shard['categories'][ $cat_key ] ) ) { $current_shard['categories'][ $cat_key ] = array(); } $current_shard['categories'][ $cat_key ][] = (int) $product['id']; if ( ! isset( $this->build_state['category_distribution'][ $cat_id ] ) ) { $this->build_state['category_distribution'][ $cat_id ] = array(); } if ( ! isset( $this->build_state['category_distribution'][ $cat_id ][ $shard_id ] ) ) { $this->build_state['category_distribution'][ $cat_id ][ $shard_id ] = 0; } ++$this->build_state['category_distribution'][ $cat_id ][ $shard_id ]; } } ++$this->build_state['total_products']; return $shard_id; } /** * Get current build state for saving to TempStorage. * * @return array|null Build state or null if not building. */ public function getBuildState(): ?array { return $this->build_state; } /** * Get minimal build state for manifest storage. * * Returns only counters and shard references, not the full product data. * This prevents memory exhaustion and timeout when saving large builds. * The full product data is written directly to temp shard files. * * @return array|null Minimal build state or null if not building. */ public function getMinimalBuildState(): ?array { if ( $this->build_state === null ) { return null; } $shard_summaries = array(); $total_products = 0; foreach ( $this->build_state['shards'] as $shard_id => $shard_data ) { $product_count = count( $shard_data['products'] ?? array() ); $total_products += $product_count; $shard_summaries[ $shard_id ] = array( 'product_count' => $product_count, 'category_count' => count( $shard_data['categories'] ?? array() ), 'estimated_size' => $shard_data['estimated_size'] ?? 0, ); } return array( 'current_shard_id' => $this->build_state['current_shard_id'], 'shard_summaries' => $shard_summaries, 'total_products' => $total_products, 'category_distribution' => $this->build_state['category_distribution'], ); } /** * Restore build state from TempStorage. * * Ensures backward compatibility by recalculating estimated_size * for shards that don't have it (from older builds). * * @param array $state Previously saved build state. * @return void */ public function restoreBuildState( array $state ): void { $this->build_state = $state; if ( isset( $this->build_state['shards'] ) ) { foreach ( $this->build_state['shards'] as $shard_id => &$shard_data ) { if ( ! isset( $shard_data['estimated_size'] ) ) { $shard_data['estimated_size'] = $this->recalculateShardEstimatedSize( $shard_data ); } } unset( $shard_data ); } if ( ! isset( $this->build_state['total_products'] ) ) { $total = 0; foreach ( $this->build_state['shards'] as $shard ) { $total += count( $shard['products'] ?? array() ); } $this->build_state['total_products'] = $total; } } /** * Recalculate estimated_size for a shard from its products. * * Used for backward compatibility when restoring state without estimated_size. * * @param array $shard_data Shard data with products. * @return int Total estimated compressed size in bytes. */ private function recalculateShardEstimatedSize( array $shard_data ): int { $total_size = 0; if ( ! empty( $shard_data['products'] ) ) { foreach ( $shard_data['products'] as $product ) { $total_size += $this->estimateProductSize( $product ); } } return $total_size; } /** * Get current build statistics. * * @return array{shard_count: int, total_products: int, current_shard_products: int}|null */ public function getBuildStats(): ?array { if ( $this->build_state === null ) { return null; } $total_products = 0; foreach ( $this->build_state['shards'] as $shard ) { $total_products += count( $shard['products'] ); } $current_shard_id = $this->build_state['current_shard_id']; $current_shard_products = count( $this->build_state['shards'][ $current_shard_id ]['products'] ?? array() ); return array( 'shard_count' => count( $this->build_state['shards'] ), 'total_products' => $total_products, 'current_shard_products' => $current_shard_products, ); } /** * Get total products via cached counter — O(1) for both build and read paths. * * P-06: During build, reads the in-memory counter incremented by addProduct(). * After build, reads map['meta']['total_products'] written by finalizeBuild(). * Use recomputeTotalProducts() for a full shard-scan when needed for diagnostics. * * @return int Total product count. */ public function getTotalProducts(): int { if ( $this->build_state !== null ) { return (int) ( $this->build_state['total_products'] ?? 0 ); } $map = $this->loadMap(); if ( $map !== null ) { return (int) ( $map['meta']['total_products'] ?? 0 ); } return 0; } /** * Recompute total products by scanning all shards. * * P-06: Full-scan fallback for admin diagnostics or counter reconciliation. * Prefer getTotalProducts() for normal use. * * @return int Actual total product count from shard scan. */ public function recomputeTotalProducts(): int { if ( $this->build_state !== null ) { $total = 0; foreach ( $this->build_state['shards'] as $shard ) { $total += count( $shard['products'] ?? array() ); } return $total; } $map = $this->loadMap(); if ( $map === null ) { return 0; } $total = 0; $shard_count = (int) ( $map['meta']['total_shards'] ?? 0 ); for ( $i = 0; $i < $shard_count; $i++ ) { $shard = $this->loadShard( $i ); if ( $shard !== null ) { $total += count( $shard['products'] ?? array() ); } } return $total; } /** * Finalize build: write all shards and create map file. * * Called at the end of IndexBuilder batch processing. * Writes shard files atomically and generates the map. * * @param array $global_meta Global index metadata (version, built_at, etc.). * @return array Map data structure with total_size and shard_count. * @throws \RuntimeException If write operations fail. */ public function finalizeBuild( array $global_meta ): array { if ( $this->build_state === null ) { throw new \RuntimeException( 'No build state to finalize.' ); } $this->ensureDirectory(); $this->cleanupOrphanTempFiles(); $shard_metadata = array(); $total_products = 0; $total_categories = 0; $total_size = 0; $temp_files = array(); $backup_files = array(); $is_first_build = false; $category_meta = array(); $shard_files = array(); $shard_final_paths = array(); try { foreach ( $this->build_state['shards'] as $shard_id => $shard_data ) { if ( empty( $shard_data['products'] ) ) { continue; } $enriched_categories = $this->enrichCategoryMetadata( $shard_data['categories'] ); foreach ( $enriched_categories as $cat_id => $cat_data ) { $cat_key = (string) $cat_id; if ( ! isset( $category_meta[ $cat_key ] ) ) { $category_meta[ $cat_key ] = array( 'cat_id' => (int) $cat_id, 'name' => $cat_data['name'] ?? '', 'slug' => $cat_data['slug'] ?? '', ); } } $shard_index = array( 'meta' => array_merge( $global_meta, array( 'shard_id' => $shard_id, 'product_count' => count( $shard_data['products'] ), 'category_count' => count( $enriched_categories ), ) ), 'products' => $shard_data['products'], 'categories' => $enriched_categories, ); $suffix = bin2hex( random_bytes( 6 ) ); $shard_final_paths[ $shard_id ] = $this->getShardFilePath( $shard_id, $suffix ); $shard_files[ $shard_id ] = basename( $shard_final_paths[ $shard_id ] ); $temp_path = $this->writeShardToTemp( $shard_id, $shard_index, $suffix ); $temp_files[ $shard_id ] = $temp_path; $file_size_result = filesize( $temp_path ); if ( $file_size_result === false ) { error_log( sprintf( '[AIPT] ShardManager: Failed to get file size for shard %d at %s', $shard_id, $temp_path ) ); $file_size = 0; } else { $file_size = (int) $file_size_result; } $product_ids_in_shard = array_map( 'intval', array_keys( $shard_data['products'] ) ); $shard_metadata[ $shard_id ] = array( 'shard_id' => $shard_id, 'product_count' => count( $shard_data['products'] ), 'category_count' => count( $enriched_categories ), 'file_size' => $file_size, 'min_product_id' => ! empty( $product_ids_in_shard ) ? min( $product_ids_in_shard ) : 0, 'max_product_id' => ! empty( $product_ids_in_shard ) ? max( $product_ids_in_shard ) : 0, ); $total_products += count( $shard_data['products'] ); $total_categories += count( $enriched_categories ); $total_size += $file_size; } $category_map = $this->buildCategoryMap(); $map_data = array( 'meta' => array_merge( $global_meta, array( 'total_shards' => count( $shard_metadata ), 'total_products' => $total_products, 'total_categories' => $total_categories, 'shards' => $shard_metadata, 'map_version' => 3, 'shard_files' => $shard_files, 'category_meta' => $category_meta, ) ), 'products' => array(), 'categories' => $category_map, 'category_meta' => $category_meta, 'shard_files' => $shard_files, ); $map_temp_path = $this->writeMapToTemp( $map_data ); $temp_files['map'] = $map_temp_path; $map_size_result = filesize( $map_temp_path ); if ( $map_size_result !== false ) { $total_size += (int) $map_size_result; } $map_data['meta']['total_size'] = $total_size; $this->writeMapToTemp( $map_data, $map_temp_path ); $pattern = $this->getDirectory() . '/products-ai-index-' . $this->profile_id . '-*.toon.gz'; $existing = glob( $pattern ) ?: array(); $is_first_build = empty( $existing ); foreach ( $existing as $file ) { $backup = $file . '.bak'; if ( @rename( $file, $backup ) ) { $backup_files[] = $backup; } else { foreach ( $backup_files as $created_backup ) { $original = preg_replace( '/\.bak$/', '', $created_backup ); @rename( $created_backup, $original ); } foreach ( $temp_files as $temp_path ) { if ( file_exists( $temp_path ) ) { @unlink( $temp_path ); } } throw new \RuntimeException( sprintf( 'Failed to create backup for %s. Aborting finalization to prevent data loss.', $file ) ); } } foreach ( $temp_files as $id => $temp_path ) { if ( $id === 'map' ) { $final_path = $this->getMapFilePath(); } else { $final_path = $shard_final_paths[ (int) $id ]; } if ( ! $this->atomicRename( $temp_path, $final_path ) ) { foreach ( $backup_files as $backup ) { $original = preg_replace( '/\.bak$/', '', $backup ); @rename( $backup, $original ); } throw new \RuntimeException( "Failed to finalize file: {$id}" ); } } foreach ( $backup_files as $backup ) { @unlink( $backup ); } } catch ( \Exception $e ) { foreach ( $temp_files as $temp_path ) { if ( file_exists( $temp_path ) ) { @unlink( $temp_path ); } } if ( ! empty( $backup_files ) ) { foreach ( $backup_files as $backup ) { $original = preg_replace( '/\.bak$/', '', $backup ); @rename( $backup, $original ); } } elseif ( $is_first_build ) { $this->deleteAllShards(); } else { error_log( '[AIPT] ShardManager: Finalize failed and backup restoration not possible. ' . 'Live index preserved but may be in inconsistent state. Manual cleanup may be required.' ); } throw $e; } $this->build_state = null; $this->clearCache(); $this->invalidateMapCache(); return array( 'total_size' => $total_size, 'shard_count' => count( $shard_metadata ), 'total_products' => $total_products, 'total_categories' => $total_categories, 'map_data' => $map_data, ); } /** * Load the map file. * * Cached after first read. Provides shard metadata and category hints. * * @return array|null Map data or null if unavailable. */ public function loadMap(): ?array { if ( $this->map !== null ) { return $this->map; } $this->last_load_error = null; $map_path = $this->getReadableMapFilePath(); if ( ! file_exists( $map_path ) ) { $this->last_load_error = array( 'kind' => 'missing_file', 'path' => $map_path ); return null; } $mtime = @filemtime( $map_path ); $cache_key = sprintf( 'aipt_storeindex_map_%d_%d', $this->profile_id, $mtime ?: 0 ); $cached = function_exists( 'wp_cache_get' ) ? wp_cache_get( $cache_key, 'aipt_storeindex' ) : false; if ( is_array( $cached ) ) { $this->map = $cached; return $this->map; } try { $compressed = file_get_contents( $map_path ); if ( $compressed === false ) { $this->last_load_error = array( 'kind' => 'read_failed', 'path' => $map_path ); return null; } $toon = @gzdecode( $compressed ); if ( $toon === false ) { $this->last_load_error = array( 'kind' => 'gzdecode_failed', 'path' => $map_path ); return null; } $decoded = $this->decoder->decode( $toon ); if ( ! is_array( $decoded ) ) { $this->last_load_error = array( 'kind' => 'decoder_non_array', 'path' => $map_path ); return null; } $found_version = isset( $decoded['meta']['map_version'] ) ? (int) $decoded['meta']['map_version'] : 1; if ( $found_version > self::MAP_VERSION_READER ) { $this->last_load_error = array( 'kind' => 'version_too_new', 'path' => $map_path, 'found' => $found_version, 'expected' => self::MAP_VERSION_READER, ); error_log( sprintf( '[AIPT] ShardManager: map_version=%d on disk exceeds reader=%d (%s). Refusing to decode.', $found_version, self::MAP_VERSION_READER, $map_path ) ); return null; } $this->map = $decoded; if ( function_exists( 'wp_cache_set' ) ) { wp_cache_set( $cache_key, $decoded, 'aipt_storeindex', HOUR_IN_SECONDS ); } return $this->map; } catch ( \Exception $e ) { $this->last_load_error = array( 'kind' => 'read_failed', 'path' => $map_path ); return null; } } /** * Best-effort cache invalidation for the current profile's map entry. * * Cache keys are mtime-scoped so finalizeBuild()'s atomic rename * naturally invalidates the previous generation, but explicit delete * keeps the working set tight on object-cache backends that retain * unreferenced keys until eviction. * * @return void */ private function invalidateMapCache(): void { $map_path = $this->getReadableMapFilePath(); if ( ! file_exists( $map_path ) ) { return; } $mtime = @filemtime( $map_path ); if ( $mtime === false ) { return; } if ( function_exists( 'wp_cache_delete' ) ) { wp_cache_delete( sprintf( 'aipt_storeindex_map_%d_%d', $this->profile_id, $mtime ), 'aipt_storeindex' ); } } /** * Return the structured reason for the most recent loadMap/loadShard * failure, or null when the last load succeeded (or no load was attempted * yet). * * Shape: * - kind: 'missing_file' | 'read_failed' | 'gzdecode_failed' * | 'decoder_non_array' | 'version_too_new' * - path: absolute path of the file that failed (when relevant) * - found / expected: only set when kind=version_too_new * * Consumed by IndexSettings::getStatus()'s `meta.index_health` field and * by the REST `/status` envelope so the admin UI can distinguish a * missing index from a corrupt or future-version one. * * @return array|null */ public function getLastLoadError(): ?array { return $this->last_load_error; } /** * Check if sharded index is available. * * @return bool True if map file exists and is readable. */ public function isAvailable(): bool { $map = $this->loadMap(); return $map !== null && ! empty( $map['meta']['shards'] ); } /** * Get total shard count from map. * * @return int Number of shards or 0 if unavailable. */ public function getShardCount(): int { $map = $this->loadMap(); return $map['meta']['total_shards'] ?? 0; } /** * Load a shard with LRU caching. * * @param int $shard_id Shard ID to load. * @return array|null Shard data or null if unavailable. */ public function loadShard( int $shard_id ): ?array { $cached = $this->shard_cache->get( $shard_id ); if ( $cached !== null ) { return $cached; } $this->last_load_error = null; $shard_path = $this->resolveShardFilePath( $shard_id ); if ( ! file_exists( $shard_path ) ) { $this->last_load_error = array( 'kind' => 'missing_file', 'path' => $shard_path, 'shard_id' => $shard_id, ); return null; } try { $compressed = file_get_contents( $shard_path ); if ( $compressed === false ) { $this->last_load_error = array( 'kind' => 'read_failed', 'path' => $shard_path, 'shard_id' => $shard_id, ); return null; } $toon = @gzdecode( $compressed ); if ( $toon === false ) { $this->last_load_error = array( 'kind' => 'gzdecode_failed', 'path' => $shard_path, 'shard_id' => $shard_id, ); return null; } $shard_data = $this->decoder->decode( $toon ); if ( ! is_array( $shard_data ) ) { $this->last_load_error = array( 'kind' => 'decoder_non_array', 'path' => $shard_path, 'shard_id' => $shard_id, ); return null; } $this->shard_cache->put( $shard_id, $shard_data ); return $shard_data; } catch ( \Exception $e ) { $this->last_load_error = array( 'kind' => 'read_failed', 'path' => $shard_path, 'shard_id' => $shard_id, ); return null; } } /** * Get shards that contain products from specified categories. * * Uses the category distribution map for intelligent shard selection. * * @param array $category_ids Category term IDs to find. * @return array Array of shard IDs sorted by relevance (most products first). */ public function getShardsForCategories( array $category_ids ): array { $map = $this->loadMap(); if ( $map === null ) { return array(); } $shard_scores = array(); foreach ( $category_ids as $cat_id ) { $cat_key = (string) $cat_id; if ( isset( $map['categories'][ $cat_key ] ) ) { $distribution = $map['categories'][ $cat_key ]; foreach ( $distribution as $shard_id => $count ) { if ( ! isset( $shard_scores[ $shard_id ] ) ) { $shard_scores[ $shard_id ] = 0; } $shard_scores[ $shard_id ] += $count; } } } arsort( $shard_scores ); return array_keys( $shard_scores ); } /** * Get default shards to load when no category hint is available. * * Returns a randomized selection from the top shards (by product count) * to provide variety while still preferring larger shards. * * @param int $limit Maximum shards to return. * @return array Array of shard IDs. */ public function getDefaultShards( int $limit = 3 ): array { $map = $this->loadMap(); if ( $map === null || empty( $map['meta']['shards'] ) ) { return array(); } $shards = $map['meta']['shards']; uasort( $shards, function ( $a, $b ) { return ( $b['product_count'] ?? 0 ) - ( $a['product_count'] ?? 0 ); } ); $shard_keys = array_keys( $shards ); $pool_size = min( $limit * 2, count( $shard_keys ) ); $shard_pool = array_slice( $shard_keys, 0, $pool_size ); shuffle( $shard_pool ); return array_slice( $shard_pool, 0, $limit ); } /** * Analyze a search query and return relevant shard IDs. * * Extracts category hints from the query and maps to shards. * * @param string $query Search query. * @return array{shard_ids: int[], confidence: float} */ public function getShardsForQuery( string $query ): array { $map = $this->loadMap(); if ( $map === null ) { return array( 'shard_ids' => array(), 'confidence' => 0.0, ); } $words = QueryTokenizer::tokenize( $query ); if ( empty( $words ) ) { return array( 'shard_ids' => $this->getDefaultShards(), 'confidence' => 0.0, ); } $matched_categories = array(); $categories = $map['categories'] ?? array(); $category_meta_raw = $map['meta']['category_meta'] ?? ( $map['category_meta'] ?? array() ); $category_meta = array(); foreach ( $category_meta_raw as $maybe_cat_id => $cat_data ) { if ( ! is_array( $cat_data ) ) { continue; } if ( ! isset( $cat_data['cat_id'] ) && isset( $cat_data[0], $cat_data[1], $cat_data[2] ) ) { $cat_data = array( 'cat_id' => $cat_data[0], 'name' => $cat_data[1], 'slug' => $cat_data[2], ); } $cat_id = isset( $cat_data['cat_id'] ) ? (int) $cat_data['cat_id'] : (int) $maybe_cat_id; if ( ! empty( $cat_data['name'] ) ) { $category_meta[ $cat_id ] = QueryTokenizer::normalize( $cat_data['name'] ); } if ( ! empty( $cat_data['slug'] ) ) { $category_meta[ $cat_id . '_slug' ] = QueryTokenizer::normalize( $cat_data['slug'] ); } } foreach ( $words as $word ) { foreach ( $category_meta as $key => $name ) { if ( strpos( $name, $word ) !== false || strpos( $word, $name ) !== false ) { $cat_id = (int) preg_replace( '/_slug$/', '', $key ); if ( isset( $categories[ (string) $cat_id ] ) ) { $matched_categories[ $cat_id ] = true; } } } } if ( empty( $matched_categories ) ) { return array( 'shard_ids' => $this->getDefaultShards(), 'confidence' => 0.0, ); } $shard_ids = $this->getShardsForCategories( array_keys( $matched_categories ) ); $confidence = min( 1.0, count( $matched_categories ) * 0.3 ); return array( 'shard_ids' => $shard_ids, 'confidence' => $confidence, ); } /** * Get products by category with intelligent shard selection. * * @param int $category_id Category term ID. * @return array{product_ids: int[], products: array} */ public function getProductsByCategory( int $category_id ): array { $map = $this->loadMap(); if ( $map === null ) { return array( 'product_ids' => array(), 'products' => array(), ); } $cat_key = (string) $category_id; $product_ids = array(); $products = array(); $shard_hints = $map['categories'][ $cat_key ] ?? null; if ( $shard_hints !== null ) { foreach ( array_keys( $shard_hints ) as $shard_id ) { $shard_data = $this->loadShard( (int) $shard_id ); if ( $shard_data === null ) { continue; } if ( isset( $shard_data['categories'][ $cat_key ]['productIds'] ) ) { $ids = $shard_data['categories'][ $cat_key ]['productIds']; $product_ids = array_merge( $product_ids, $ids ); foreach ( $ids as $pid ) { $pid_key = (string) $pid; if ( isset( $shard_data['products'][ $pid_key ] ) ) { $products[ $pid ] = $shard_data['products'][ $pid_key ]; } } } } } else { $shard_count = $map['meta']['total_shards'] ?? 0; for ( $i = 0; $i < $shard_count; $i++ ) { $shard_data = $this->loadShard( $i ); if ( $shard_data === null ) { continue; } if ( isset( $shard_data['categories'][ $cat_key ]['productIds'] ) ) { $ids = $shard_data['categories'][ $cat_key ]['productIds']; $product_ids = array_merge( $product_ids, $ids ); foreach ( $ids as $pid ) { $pid_key = (string) $pid; if ( isset( $shard_data['products'][ $pid_key ] ) ) { $products[ $pid ] = $shard_data['products'][ $pid_key ]; } } } } } return array( 'product_ids' => $product_ids, 'products' => $products, ); } /** * Get products by IDs with intelligent shard selection. * * Uses min/max product ID metadata per shard to skip shards that * cannot contain any of the requested IDs. Products are stored in * ID-sorted order, so each shard contains a contiguous ID range. * * @param int[] $product_ids Product IDs to retrieve. * @return array Map of product_id => profile. */ public function getProductsByIds( array $product_ids ): array { $map = $this->loadMap(); if ( $map === null || empty( $product_ids ) ) { return array(); } $results = array(); $shard_metadata = $map['meta']['shards'] ?? array(); $product_ids_int = array_map( 'intval', $product_ids ); $min_requested = min( $product_ids_int ); $max_requested = max( $product_ids_int ); foreach ( $shard_metadata as $shard_id => $meta ) { if ( isset( $meta['min_product_id'], $meta['max_product_id'] ) ) { $shard_min = (int) $meta['min_product_id']; $shard_max = (int) $meta['max_product_id']; if ( $shard_max < $min_requested || $shard_min > $max_requested ) { continue; } } $shard_data = $this->loadShard( (int) $shard_id ); if ( $shard_data === null ) { continue; } foreach ( $product_ids as $product_id ) { $product_key = (string) $product_id; if ( isset( $shard_data['products'][ $product_key ] ) ) { $results[ $product_id ] = $shard_data['products'][ $product_key ]; } } if ( count( $results ) === count( $product_ids ) ) { break; } } return $results; } /** * Load multiple shards and merge their products. * * @param array $shard_ids Shard IDs to load. * @return array Merged products array. */ public function loadAndMergeShards( array $shard_ids ): array { $merged = array( 'products' => array(), 'categories' => array(), ); foreach ( $shard_ids as $shard_id ) { $shard_data = $this->loadShard( (int) $shard_id ); if ( $shard_data === null ) { continue; } foreach ( $shard_data['products'] ?? array() as $pid => $product ) { $merged['products'][ $pid ] = $product; } foreach ( $shard_data['categories'] ?? array() as $cat_id => $cat_data ) { if ( ! isset( $merged['categories'][ $cat_id ] ) ) { $merged['categories'][ $cat_id ] = $cat_data; } else { if ( isset( $cat_data['productIds'] ) ) { $merged['categories'][ $cat_id ]['productIds'] = array_merge( $merged['categories'][ $cat_id ]['productIds'] ?? array(), $cat_data['productIds'] ); } } } } return $merged; } /** * Delete all shard files and map file for this profile. * * Uses DirectoryIterator for memory-efficient file listing instead of glob(). * This prevents memory exhaustion on sites with many shard files. * * @return bool True if all files deleted or didn't exist. */ public function deleteAllShards(): bool { $this->invalidateMapCache(); $deleted = array(); $failed = array(); $dir = $this->getDirectory(); if ( ! is_dir( $dir ) ) { $this->last_delete_result = array( 'ok' => true, 'deleted' => $deleted, 'failed' => $failed ); return true; } $prefix = 'products-ai-index-' . $this->profile_id . '-'; try { $iterator = new \DirectoryIterator( $dir ); foreach ( $iterator as $file ) { if ( $file->isDot() || ! $file->isFile() ) { continue; } $filename = $file->getFilename(); if ( strpos( $filename, $prefix ) !== 0 ) { continue; } if ( substr( $filename, -8 ) === '.toon.gz' || strpos( $filename, '.tmp.' ) !== false ) { $path = $file->getPathname(); if ( @unlink( $path ) ) { $deleted[] = $path; } else { $failed[] = array( 'path' => $path, 'reason' => 'unlink_failed' ); } } } } catch ( \Exception $e ) { $patterns = array( $dir . '/' . $prefix . '*.toon.gz', $dir . '/' . $prefix . '*.tmp.*', ); foreach ( $patterns as $pattern ) { $files = glob( $pattern ); if ( ! $files ) { continue; } foreach ( $files as $path ) { if ( @unlink( $path ) ) { $deleted[] = $path; } else { $failed[] = array( 'path' => $path, 'reason' => 'unlink_failed' ); } } } } $this->last_delete_result = array( 'ok' => empty( $failed ), 'deleted' => $deleted, 'failed' => $failed, ); return empty( $failed ); } /** * Return the per-file outcome of the most recent * deleteAllShards() / cleanupOrphanTempFiles() call. * * Shape: ['ok' => bool, 'deleted' => string[], 'failed' => array]. * Returns null when no delete has run yet on this instance. * * Lets controllers surface partial-failure warnings on `/reset` * without breaking the historical bool return contract. * * @return array|null */ public function getLastDeleteResult(): ?array { return $this->last_delete_result; } /** * Clear all caches (map and shard cache). * * @return void */ public function clearCache(): void { $this->map = null; $this->shard_cache->clear(); } /** * Get cache statistics. * * @return array Cache stats. */ public function getCacheStats(): array { return $this->shard_cache->getStats(); } /** * Check if the storage directory is writable. * * @return bool True if directory is writable. */ public function isWritable(): bool { $dir = $this->getDirectory(); if ( ! is_dir( $dir ) ) { $upload_dir = wp_upload_dir(); return \wp_is_writable( $upload_dir['basedir'] ); } return \wp_is_writable( $dir ); } /** * Get the base directory for index files. * * @return string Absolute path to index directory. */ private function getDirectory(): string { $upload_dir = wp_upload_dir(); return $upload_dir['basedir'] . '/' . self::INDEX_DIR; } /** * Get the shard directory path for this profile. * * Returns the directory where all shard files for this profile are stored. * Profile isolation is achieved via filename prefix (products-ai-index-{profile_id}-*), * not via subdirectory, to maintain backward compatibility with existing installations. * * @return string Absolute path to shard directory. */ public function getShardDirectory(): string { return $this->getDirectory(); } /** * Get shard file path. * * @param int $shard_id Shard ID. * @param string $suffix Optional 12-char hex suffix for S-14 secure naming (empty = legacy). * @return string Absolute path. */ private function getShardFilePath( int $shard_id, string $suffix = '' ): string { $base = $this->getDirectory() . '/products-ai-index-' . $this->profile_id . '-shard-' . $shard_id; if ( $suffix !== '' ) { return $base . '-' . $suffix . '.toon.gz'; } return $base . '.toon.gz'; } /** * Resolve shard file path from map metadata. * * S-14: Prefers the filename recorded in map.toon.gz (new format with random suffix) * to prevent enumeration of predictable shard filenames under wp-content/uploads. * Falls back to the legacy predictable filename for backward compatibility with * indexes built before this fix. * * @param int $shard_id Shard ID. * @return string Absolute path to shard file. */ private function resolveShardFilePath( int $shard_id ): string { if ( $this->map === null ) { $this->loadMap(); } if ( is_array( $this->map ) ) { if ( isset( $this->map['meta']['shard_files'][ $shard_id ] ) ) { return $this->getDirectory() . '/' . $this->map['meta']['shard_files'][ $shard_id ]; } if ( isset( $this->map['shard_files'][ $shard_id ] ) ) { return $this->getDirectory() . '/' . $this->map['shard_files'][ $shard_id ]; } } return $this->getShardFilePath( $shard_id ); } /** * Get the per-site token that makes the map file name unguessable. * * The map lists the random shard file names, so on hosts that ignore * .htaccess a predictable map name would expose every shard. The token is * stored in an option (not derived from salts, which security plugins * rotate) and created once on the first write. * * @param bool $create Create and store a token when none exists. * @return string Token, or an empty string when none exists and $create is false. */ private function getMapToken( bool $create ): string { $token = get_option( self::MAP_TOKEN_OPTION, '' ); if ( is_string( $token ) && preg_match( '/^[A-Za-z0-9]{32}$/', $token ) ) { return $token; } if ( ! $create ) { return ''; } $token = wp_generate_password( 32, false ); if ( ! add_option( self::MAP_TOKEN_OPTION, $token, '', false ) ) { wp_cache_delete( 'notoptions', 'options' ); wp_cache_delete( self::MAP_TOKEN_OPTION, 'options' ); $stored = get_option( self::MAP_TOKEN_OPTION, '' ); if ( is_string( $stored ) && preg_match( '/^[A-Za-z0-9]{32}$/', $stored ) ) { return $stored; } update_option( self::MAP_TOKEN_OPTION, $token, false ); } return $token; } /** * Get the legacy, predictable map file path (indexes built before the token). * * @return string Absolute path. */ private function getLegacyMapFilePath(): string { return $this->getDirectory() . '/products-ai-index-' . $this->profile_id . '-map.toon.gz'; } /** * Get the path a new map file is written to (token-named). * * @return string Absolute path. */ private function getMapFilePath(): string { return $this->getDirectory() . '/products-ai-index-' . $this->profile_id . '-map-' . $this->getMapToken( true ) . '.toon.gz'; } /** * Get the path the map is read from: the token-named file when present, * else the legacy file so pre-upgrade indexes keep working until rebuilt. * * @return string Absolute path (the token-named one when neither exists). */ private function getReadableMapFilePath(): string { $token = $this->getMapToken( false ); if ( $token !== '' ) { $tokened = $this->getDirectory() . '/products-ai-index-' . $this->profile_id . '-map-' . $token . '.toon.gz'; if ( file_exists( $tokened ) ) { return $tokened; } } $legacy = $this->getLegacyMapFilePath(); if ( file_exists( $legacy ) ) { return $legacy; } return $token !== '' ? $this->getDirectory() . '/products-ai-index-' . $this->profile_id . '-map-' . $token . '.toon.gz' : $legacy; } /** * Ensure the storage directory exists. * * @return bool True if directory exists or was created. */ private function ensureDirectory(): bool { $dir = $this->getDirectory(); if ( ! is_dir( $dir ) && ! wp_mkdir_p( $dir ) ) { return false; } $this->writeProtectionFiles( $dir ); return \wp_is_writable( $dir ); } /** * Write the directory listing and Apache deny files when they are missing. * * Web servers other than Apache ignore .htaccess, so the random shard file * names (S-14) remain the protection there. * * @param string $dir Directory path. * @return void */ private function writeProtectionFiles( string $dir ): void { $index_file = $dir . '/index.php'; if ( ! file_exists( $index_file ) ) { file_put_contents( $index_file, ' Require all denied Order deny,allow Deny from all " ); } } /** * Estimate a single product's compressed size contribution. * * Uses lightweight JSON length estimation (O(1) vs full TOON encode). * Conservative multiplier accounts for TOON overhead and compression variance. * * @param array $product Product profile data. * @return int Estimated compressed size in bytes. */ private function estimateProductSize( array $product ): int { $json = wp_json_encode( $product, JSON_UNESCAPED_UNICODE ); if ( $json === false ) { return 2048; } $json_length = strlen( $json ); $toon_estimate = (int) ( $json_length * 0.9 ); $compressed_estimate = (int) ceil( ( $toon_estimate / self::ESTIMATED_COMPRESSION_RATIO ) * 1.1 ); return max( 100, $compressed_estimate ); } /** * Estimate shard size (TOON + gzip). * * Returns a conservative high estimate on error to fail-safe * (trigger shard split rather than allowing oversized shards). * * @param array $shard_data Shard data with products and categories. * @return int Estimated bytes (returns MAX_SHARD_SIZE_BYTES on error). */ private function estimateShardSize( array $shard_data ): int { try { $toon = $this->encoder->encode( array( 'meta' => array(), 'products' => $shard_data['products'], 'categories' => $shard_data['categories'], ) ); return (int) ceil( strlen( $toon ) / self::ESTIMATED_COMPRESSION_RATIO ); } catch ( \Exception $e ) { error_log( sprintf( '[AIPT] ShardManager: Failed to estimate shard size (products: %d). Error: %s', count( $shard_data['products'] ?? array() ), $e->getMessage() ) ); return self::MAX_SHARD_SIZE_BYTES; } } /** * Write shard to temp file. * * @param int $shard_id Shard ID. * @param array $shard_data Shard index data. * @param string $suffix Optional 12-char hex suffix for S-14 secure naming (empty = legacy). * @return string Temp file path. * @throws \RuntimeException If write fails. */ private function writeShardToTemp( int $shard_id, array $shard_data, string $suffix = '' ): string { $temp_path = $this->getShardFilePath( $shard_id, $suffix ) . '.tmp.' . wp_generate_password( 8, false ); $toon = $this->encoder->encode( $shard_data ); $compressed = gzencode( $toon, 9 ); if ( $compressed === false ) { throw new \RuntimeException( 'Failed to compress shard ' . $shard_id ); } if ( strlen( $compressed ) > self::MAX_SHARD_SIZE_BYTES ) { throw new \RuntimeException( sprintf( 'Shard %d exceeds size limit: %d bytes (max: %d bytes)', $shard_id, strlen( $compressed ), self::MAX_SHARD_SIZE_BYTES ) ); } $written = file_put_contents( $temp_path, $compressed ); if ( $written === false ) { throw new \RuntimeException( 'Failed to write shard temp file: ' . $shard_id ); } return $temp_path; } /** * Write map file to temp location. * * @param array $map_data Map data structure. * @param string|null $reuse_path Optional pre-existing temp path to write * INTO. When provided, the contents at this * path are replaced instead of allocating a * fresh random `.tmp.` file. Used by * finalizeBuild()'s two-pass map write so * that the second write does NOT create an * orphan temp file (F4). Pass null on the * first write to allocate a new temp path. * @return string Temp file path written to (equals $reuse_path when given). * @throws \RuntimeException If write fails. */ private function writeMapToTemp( array $map_data, ?string $reuse_path = null ): string { $temp_path = $reuse_path ?? ( $this->getMapFilePath() . '.tmp.' . wp_generate_password( 8, false ) ); $toon = $this->encoder->encode( $map_data ); $compressed = gzencode( $toon, 9 ); if ( $compressed === false ) { throw new \RuntimeException( 'Failed to compress map file' ); } $written = file_put_contents( $temp_path, $compressed ); if ( $written === false ) { throw new \RuntimeException( 'Failed to write map temp file' ); } return $temp_path; } /** * Remove orphaned temp files for THIS profile from the index directory. * * Orphans accumulate from: * - Pre-2.6.0 builds, where finalizeBuild() leaked one map `.tmp.*` per * successful build because the second writeMapToTemp() call allocated * a new random suffix instead of reusing the first temp path (F4). * - Failed/crashed builds that crashed BETWEEN write and rename. * * Profile isolation is enforced by the filename prefix; a concurrent build * of the SAME profile cannot exist because IndexBuilder holds an exclusive * batch lock keyed by job_id, so we can unconditionally unlink without an * mtime guard. * * @return int Number of files removed. */ private function cleanupOrphanTempFiles(): int { $dir = $this->getDirectory(); if ( ! is_dir( $dir ) ) { return 0; } $prefix = 'products-ai-index-' . $this->profile_id . '-'; $removed = 0; try { $iterator = new \DirectoryIterator( $dir ); foreach ( $iterator as $file ) { if ( $file->isDot() || ! $file->isFile() ) { continue; } $filename = $file->getFilename(); if ( strpos( $filename, $prefix ) !== 0 ) { continue; } if ( strpos( $filename, '.tmp.' ) === false ) { continue; } if ( @unlink( $file->getPathname() ) ) { ++$removed; } } } catch ( \Exception $e ) { $pattern = $dir . '/' . $prefix . '*.tmp.*'; $files = glob( $pattern ); if ( $files ) { foreach ( $files as $file ) { if ( @unlink( $file ) ) { ++$removed; } } } } return $removed; } /** * Build category→shard distribution map. * * @return array Category ID => [shard_id => product_count], sorted by count desc. */ private function buildCategoryMap(): array { if ( $this->build_state === null ) { return array(); } $map = array(); foreach ( $this->build_state['category_distribution'] as $cat_id => $shard_counts ) { arsort( $shard_counts ); $map[ (string) $cat_id ] = $shard_counts; } return $map; } /** * Enrich category metadata with term names and slugs. * * @param array $categories Category map with product ID arrays. * @return array Enriched category map. */ private function enrichCategoryMetadata( array $categories ): array { if ( empty( $categories ) ) { return array(); } $category_ids = array_map( 'intval', array_keys( $categories ) ); $terms = get_terms( array( 'taxonomy' => 'product_cat', 'include' => $category_ids, 'hide_empty' => false, ) ); $term_map = array(); if ( ! is_wp_error( $terms ) ) { foreach ( $terms as $term ) { $term_map[ $term->term_id ] = array( 'name' => $term->name, 'slug' => $term->slug, ); } } $enriched = array(); foreach ( $categories as $cat_id => $product_ids ) { $int_id = (int) $cat_id; $enriched[ (string) $cat_id ] = array( 'id' => $int_id, 'name' => $term_map[ $int_id ]['name'] ?? null, 'slug' => $term_map[ $int_id ]['slug'] ?? null, 'productIds' => $product_ids, ); } return $enriched; } /** * Atomic rename with Windows compatibility. * * @param string $source Source file path. * @param string $target Target file path. * @return bool True on success. */ private function atomicRename( string $source, string $target ): bool { if ( @rename( $source, $target ) ) { return true; } if ( file_exists( $target ) ) { $max_retries = 3; $retry_delay_ms = 100; for ( $i = 0; $i < $max_retries; $i++ ) { if ( @unlink( $target ) ) { break; } if ( $i < $max_retries - 1 ) { usleep( $retry_delay_ms * 1000 ); } } } return @rename( $source, $target ); } }