post_ids = $post_ids; $this->stop_words = $stop_words; $this->exclude_links = $exclude_links; } public function get_single_keywords($limit = 5000) { $global_word_counts = []; foreach ($this->post_ids as $pid) { $sentences = $this->get_tokens_for_post($pid); foreach ($sentences as $words) { foreach ($words as $word) { $word_lower = mb_strtolower($word, 'UTF-8'); $clean_word = str_replace("'", "", $word_lower); if (mb_strlen($clean_word, 'UTF-8') < 3 || is_numeric($word_lower)) { continue; } if (in_array($word_lower, $this->stop_words, true)) { continue; } if (!isset($global_word_counts[$word_lower])) { $global_word_counts[$word_lower] = 0; } $global_word_counts[$word_lower]++; } } } arsort($global_word_counts); return array_slice($global_word_counts, 0, $limit, true); } /** * Extracts tokens grouped by sentences. * * @param int $post_id * @return array Array of sentences (each sentence is an array of words). */ private function get_tokens_for_post($post_id) { $content = get_post_field('post_content', $post_id); if (empty($content)) { return []; } // Remove shortcodes before processing text $content = strip_shortcodes($content); // USE SMART PARSER if (class_exists('AILM_DOM_Helper')) { $dom_helper = new AILM_DOM_Helper($content); $text = $dom_helper->get_virtual_text(); } else { // Fallback $content = preg_replace('#]*?>.*?|]*>.*?#si', '', $content); $text = wp_strip_all_tags($content); } // Decode HTML entities (& -> &) $text = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'); // Exclude words that are already occupied by our FROZEN links // DEPRECATED v2.0: _ailm_frozen_map meta is replaced by wp_ailm_link_rules. if ($this->exclude_links) { $frozen_rules = []; if (class_exists('AILM_Link_Repository')) { $frozen_rules = AILM_Link_Repository::get_rules_for_donor($post_id); } if (!empty($frozen_rules)) { usort($frozen_rules, function ($a, $b) { return mb_strlen($b->anchor, 'UTF-8') - mb_strlen($a->anchor, 'UTF-8'); }); foreach ($frozen_rules as $rule) { if (!empty($rule->anchor)) { $word_to_remove = preg_quote($rule->anchor, '/'); $text = preg_replace('/(?post_ids as $pid) { $sentences = $this->get_tokens_for_post($pid); foreach ($sentences as $words) { if (empty($words)) continue; foreach ($sizes as $n) { if (count($words) < $n) continue; for ($i = 0; $i <= count($words) - $n; $i++) { $phrase_words = array_slice($words, $i, $n); $phrase_lower_words = array_map(function ($word) { return mb_strtolower($word, 'UTF-8'); }, $phrase_words); $first_word = $phrase_lower_words[0]; $last_word = end($phrase_lower_words); if (in_array($first_word, $this->stop_words, true) || in_array($last_word, $this->stop_words, true)) { continue; } $phrase = implode(' ', $phrase_lower_words); if (!isset($global_phrase_counts[$phrase])) { $global_phrase_counts[$phrase] = 0; } $global_phrase_counts[$phrase]++; } } } } arsort($global_phrase_counts); return array_slice($global_phrase_counts, 0, $limit, true); } public function get_deep_context($phrase, $around_window = 2, $inside_gap = 1) { $phrase = str_replace(['’', '‘', '`', '´'], "'", $phrase); $phrase_words = explode(' ', mb_strtolower($phrase, 'UTF-8')); $phrase_len = count($phrase_words); $around_counts = []; $inside_counts = []; foreach ($this->post_ids as $pid) { $sentences = $this->get_tokens_for_post($pid); foreach ($sentences as $tokens) { $total_tokens = count($tokens); if ($total_tokens < $phrase_len) continue; for ($i = 0; $i < $total_tokens; $i++) { $token_lower = mb_strtolower($tokens[$i], 'UTF-8'); if ($token_lower === $phrase_words[0]) { $match_end_idx = $this->match_phrase_with_gap($tokens, $i, $phrase_words, $inside_gap); if ($match_end_idx !== false) { // 1. INSIDE $actual_len = $match_end_idx - $i + 1; if ($actual_len > $phrase_len) { $inside_phrase = implode(' ', array_slice($tokens, $i, $actual_len)); $this->add_phrase_to_count($inside_counts, $inside_phrase, $pid); } // 2. AROUND for ($l = 0; $l <= $around_window; $l++) { for ($r = 0; $r <= $around_window; $r++) { if ($l === 0 && $r === 0) continue; $start_ext = $i - $l; $end_ext = $match_end_idx + $r; if ($start_ext < 0 || $end_ext >= $total_tokens) continue; $first_word = mb_strtolower($tokens[$start_ext], 'UTF-8'); $last_word = mb_strtolower($tokens[$end_ext], 'UTF-8'); if ($this->is_valid_boundary($first_word) && $this->is_valid_boundary($last_word)) { $ext_phrase = implode(' ', array_slice($tokens, $start_ext, $end_ext - $start_ext + 1)); $this->add_phrase_to_count($around_counts, $ext_phrase, $pid); } } } } } } } } $sort_cb = function ($a, $b) { return $b['count'] <=> $a['count']; }; uasort($around_counts, $sort_cb); uasort($inside_counts, $sort_cb); return [ 'around' => array_slice($around_counts, 0, 50, true), 'inside' => array_slice($inside_counts, 0, 50, true), ]; } private function match_phrase_with_gap($tokens, $start_idx, $phrase_words, $max_gap) { $current_idx = $start_idx; $total_tokens = count($tokens); for ($p = 1; $p < count($phrase_words); $p++) { $target_word = $phrase_words[$p]; $found = false; for ($g = 1; $g <= $max_gap + 1; $g++) { $check_idx = $current_idx + $g; if ($check_idx >= $total_tokens) break; if (mb_strtolower($tokens[$check_idx], 'UTF-8') === $target_word) { $found = true; $current_idx = $check_idx; break; } } if (!$found) return false; } return $current_idx; } private function is_valid_boundary($word) { if (is_numeric($word)) return false; if (in_array($word, $this->stop_words, true)) return false; return true; } private function add_phrase_to_count(&$array, $phrase, $pid) { $p = mb_strtolower($phrase, 'UTF-8'); if (!isset($array[$p])) { $array[$p] = ['count' => 0, 'posts' => []]; } $array[$p]['count']++; $array[$p]['posts'][$pid] = true; } }