<?php
if (!defined('ABSPATH')) exit;

/**
 * Consolidated Content Processor - Combines content extraction and background processing
 * Replaces: background-processor.php, content-extractor.php
 */

/**
 * Content extraction strategies for different websites
 */
class AIRSREFE_ContentExtractor {
    private $performanceTracker;
    private $extractionCache;
    
    public function __construct() {
        try {
            $this->extractionCache = [];
            $this->performanceTracker = new AIRSREFE_PerformanceTracker();
            airsrefe_append_log("✅ Content extractor initialized successfully");
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error initializing content extractor: " . $e->getMessage());
            $this->extractionCache = [];
            $this->performanceTracker = null;
        }
    }
    
    /**
     * Extract content from URL using appropriate strategy
     */
    public function extractContent($url, $fallback_content = '') {
        try {
            if (empty($url) || !filter_var($url, FILTER_VALIDATE_URL)) {
                airsrefe_append_log("❌ Invalid URL for content extraction: " . ($url ?: 'empty'));
                return $fallback_content;
            }
            
            $cache_key = 'airsrefe_extract_' . md5($url);
            $cached = wp_cache_get($cache_key, 'airsrefe_content');
            
            if ($cached !== false) {
                airsrefe_append_log("💾 Using cached content for: " . $url);
                return $cached;
            }
            
            $strategy = $this->selectExtractionStrategy($url);
            airsrefe_append_log("🎯 Using extraction strategy: " . $strategy . " for: " . $url);
            
            $content = $this->executeExtractionStrategy($url, $strategy);
            
            if (empty($content)) {
                airsrefe_append_log("⚠️ Content extraction failed for: " . $url . ", using fallback");
                $content = $fallback_content;
            } else {
                $content_length = strlen(wp_strip_all_tags($content));
                airsrefe_append_log("✅ Content extraction successful for: " . $url . " - " . $content_length . " characters");
            }
            
            // Cache for 1 hour
            $cache_result = wp_cache_set($cache_key, $content, 'airsrefe_content', HOUR_IN_SECONDS);
            if ($cache_result) {
                airsrefe_append_log("💾 Content cached successfully for: " . $url);
            } else {
                airsrefe_append_log("⚠️ Failed to cache content for: " . $url);
            }
            
            return $content;
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error in extractContent for " . $url . ": " . $e->getMessage());
            return $fallback_content;
        }
    }
    
    /**
     * Select best extraction strategy - Always use Readability for best quality
     */
    private function selectExtractionStrategy($url) {
        // Always use readability method for best content quality
        return 'readability';
    }
    
    /**
     * Execute content extraction using specified strategy
     */
    private function executeExtractionStrategy($url, $strategy) {
        try {
            if (empty($strategy)) {
                airsrefe_append_log("⚠️ No strategy specified for content extraction");
                return '';
            }
            
            $content = $this->extractWithStrategy($url, $strategy);
            
            if (!empty($content)) {
                // Only record success if we have a performance tracker
                if ($this->performanceTracker) {
                    $this->recordStrategySuccess($url, $strategy);
                } else {
                    airsrefe_append_log("⚠️ Performance tracker not available, skipping strategy recording");
                }
                return $content;
            }
            
            airsrefe_append_log("⚠️ Strategy " . $strategy . " failed to extract content from: " . $url);
            return '';
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error executing extraction strategy " . $strategy . " for " . $url . ": " . $e->getMessage());
            return '';
        }
    }
    
    /**
     * Extract content using specific strategy
     */
    private function extractWithStrategy($url, $strategy) {
        try {
            if (empty($strategy)) {
                airsrefe_append_log("⚠️ No strategy specified for content extraction");
                return '';
            }
            
            $html = $this->fetchContent($url);
            if (empty($html)) {
                airsrefe_append_log("⚠️ No HTML content fetched for: " . $url);
                return '';
            }
            
            airsrefe_append_log("🔍 Extracting content using strategy: " . $strategy . " for: " . $url);
            
            $content = '';
            
            switch ($strategy) {
                case 'readability':
                    $content = $this->extractWithReadability($html);
                    break;
                case 'dom':
                    $content = $this->extractWithDOM($html);
                    break;
                case 'meta':
                    $content = $this->extractWithMeta($html);
                    break;
                case 'fallback':
                    $content = $this->extractWithFallback($html);
                    break;
                default:
                    airsrefe_append_log("⚠️ Unknown strategy: " . $strategy . ", using readability");
                    $content = $this->extractWithReadability($html);
            }
            
            if (!empty($content)) {
                $content_length = strlen(wp_strip_all_tags($content));
                airsrefe_append_log("✅ Strategy " . $strategy . " extracted " . $content_length . " characters");
            } else {
                airsrefe_append_log("⚠️ Strategy " . $strategy . " failed to extract content");
            }
            
            return $content;
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error in strategy " . $strategy . " for " . $url . ": " . $e->getMessage());
            return '';
        }
    }
    
    /**
     * DOM-based content extraction
     */
    private function extractWithDOM($html) {
        if (!class_exists('DOMDocument')) return '';
        
        $dom = new DOMDocument();
        
        // Safe libxml error handling
        $old_setting = libxml_use_internal_errors(true);
        libxml_clear_errors();
        
        $load_result = $dom->loadHTML('<?xml encoding="UTF-8">' . $html, LIBXML_NOERROR | LIBXML_NOWARNING | LIBXML_HTML_NOIMPLIED);
        
        // Clear any errors and restore previous setting
        libxml_clear_errors();
        libxml_use_internal_errors($old_setting);
        
        if (!$load_result) return '';
        
        $xpath = new DOMXPath($dom);
        
        // Common content selectors
        $selectors = [
            'article',
            '[role="main"]',
            '.content',
            '.post-content',
            '.entry-content',
            '.article-content',
            '.story-content',
            '.post-body',
            '.entry-body'
        ];
        
        foreach ($selectors as $selector) {
            $nodes = $xpath->query($selector);
            if ($nodes->length > 0) {
                $content = $this->extractTextFromNodes($nodes);
                if (strlen($content) > 200) {
                    return $content;
                }
            }
        }
        
        // Fallback to paragraph extraction
        return $this->extractParagraphs($xpath);
    }
    
    /**
     * Readability-based extraction - Enhanced algorithm for best quality
     */
    private function extractWithReadability($html) {
        if (!class_exists('DOMDocument')) {
            airsrefe_append_log("❌ DOMDocument class not available");
            return '';
        }
        
        try {
            $dom = new DOMDocument();
            // Safe libxml error handling
            $old_setting = libxml_use_internal_errors(true);
            libxml_clear_errors();
            
            $load_result = $dom->loadHTML('<?xml encoding="UTF-8">' . $html, LIBXML_NOERROR | LIBXML_NOWARNING | LIBXML_HTML_NOIMPLIED);
            
            // Clear any errors and restore previous setting
            libxml_clear_errors();
            libxml_use_internal_errors($old_setting);
            
            if (!$load_result) {
                airsrefe_append_log("⚠️ Failed to load HTML into DOMDocument");
                return $this->extractWithFallback($html);
            }
            
            $xpath = new DOMXPath($dom);
            airsrefe_append_log("✅ DOM loaded successfully, starting content extraction");
            
            // Remove unwanted elements first
            $this->removeUnwantedElements($xpath);
            
            // Score-based content extraction
            $content_elements = $this->scoreContentElements($xpath);
            
            airsrefe_append_log("📊 Found " . count($content_elements) . " potential content elements");
            
            if (empty($content_elements)) {
                airsrefe_append_log("⚠️ No content elements found, using fallback extraction");
                return $this->extractWithFallback($html);
            }
            
            // Sort by score and extract top content
            usort($content_elements, function($a, $b) {
                return $b['score'] - $a['score'];
            });
            
            $content = '';
            $total_length = 0;
            $max_length = 10000; // Maximum content length
            $elements_used = 0;
            
            foreach ($content_elements as $element) {
                if ($total_length >= $max_length) break;
                
                $text = trim($element['element']->textContent);
                $text_length = strlen($text);
                
                if ($text_length > 50 && $element['score'] > 0.3) {
                    $formatted_content = $this->formatElementContent($element['element']);
                    if (!empty($formatted_content)) {
                        $content .= $formatted_content . "\n\n";
                        $total_length += $text_length;
                        $elements_used++;
                    }
                }
            }
            
            airsrefe_append_log("✅ Readability extraction completed: " . $elements_used . " elements, " . $total_length . " characters");
            
            return trim($content);
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error in readability extraction: " . $e->getMessage());
            airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
            return '';
        }
    }
    
    /**
     * Remove unwanted elements from DOM
     */
    private function removeUnwantedElements($xpath) {
        try {
            $unwanted_selectors = [
                '//script', '//style', '//nav', '//header', '//footer', '//aside',
                '//form', '//iframe', '//noscript', '//meta', '//link',
                '//div[contains(@class, "ad")]', '//div[contains(@class, "advertisement")]',
                '//div[contains(@class, "banner")]', '//div[contains(@class, "sidebar")]',
                '//div[contains(@class, "widget")]', '//div[contains(@class, "comment")]',
                '//div[contains(@class, "social")]', '//div[contains(@class, "share")]',
                '//div[contains(@class, "related")]', '//div[contains(@class, "recommended")]',
                '//div[contains(@id, "ad")]', '//div[contains(@id, "banner")]',
                '//div[contains(@id, "sidebar")]', '//div[contains(@id, "widget")]'
            ];
            
            $removed_count = 0;
            $errors_count = 0;
            
            foreach ($unwanted_selectors as $selector) {
                try {
                    $nodes = $xpath->query($selector);
                    if ($nodes) {
                        foreach ($nodes as $node) {
                            if ($node && $node->parentNode) {
                                $node->parentNode->removeChild($node);
                                $removed_count++;
                            }
                        }
                    }
                } catch (Exception $e) {
                    $errors_count++;
                    airsrefe_append_log("⚠️ Error removing elements with selector " . $selector . ": " . $e->getMessage());
                }
            }
            
            if ($removed_count > 0) {
                airsrefe_append_log("🧹 Removed " . $removed_count . " unwanted elements from DOM");
            }
            
            if ($errors_count > 0) {
                airsrefe_append_log("⚠️ Encountered " . $errors_count . " errors while removing unwanted elements");
            }
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error in removeUnwantedElements: " . $e->getMessage());
        }
    }
    
    /**
     * Score content elements based on readability algorithm
     */
    private function scoreContentElements($xpath) {
        $content_elements = [];
        
        try {
            // Get all potential content elements
            $elements = $xpath->query('//p | //div | //article | //section | //main | //h1 | //h2 | //h3 | //h4 | //h5 | //h6');
            
            if (!$elements) {
                airsrefe_append_log("⚠️ No elements found with XPath query");
                return [];
            }
            
            airsrefe_append_log("📊 Found " . $elements->length . " potential content elements");
            
            $scored_count = 0;
            $error_count = 0;
            
            foreach ($elements as $element) {
                try {
                    if (!$element) {
                        continue;
                    }
                    
                    $score = $this->calculateElementScore($element, $xpath);
                    
                    if ($score > 0) {
                        $content_elements[] = [
                            'element' => $element,
                            'score' => $score
                        ];
                        $scored_count++;
                    }
                } catch (Exception $e) {
                    $error_count++;
                    airsrefe_append_log("⚠️ Error scoring element: " . $e->getMessage());
                    continue;
                }
            }
            
            airsrefe_append_log("✅ Successfully scored " . $scored_count . " content elements");
            
            if ($error_count > 0) {
                airsrefe_append_log("⚠️ Encountered " . $error_count . " errors while scoring elements");
            }
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error in scoreContentElements: " . $e->getMessage());
        }
        
        return $content_elements;
    }
    
    /**
     * Calculate score for a content element
     */
    private function calculateElementScore($element, $xpath) {
        try {
            if (!$element) {
                return 0;
            }
            
            $score = 0;
            $text = trim($element->textContent);
            $text_length = strlen($text);
            
            // Skip very short or very long text
            if ($text_length < 20 || $text_length > 2000) return 0;
            
            // Base score based on text length (optimal: 100-500 chars)
            if ($text_length >= 100 && $text_length <= 500) {
                $score += 10;
            } elseif ($text_length > 500) {
                $score += 8;
            } else {
                $score += 5;
            }
            
            // Bonus for article-specific elements
            $tag_name = strtolower($element->nodeName);
            if ($tag_name === 'article') $score += 15;
            elseif ($tag_name === 'main') $score += 12;
            elseif ($tag_name === 'section') $score += 8;
            elseif (strpos($tag_name, 'h') === 0) $score += 6;
            
            // Bonus for content-specific classes
            $class = $element->getAttribute('class');
            if ($class) {
                $content_classes = ['content', 'post', 'article', 'entry', 'story', 'text', 'body'];
                foreach ($content_classes as $content_class) {
                    if (stripos($class, $content_class) !== false) {
                        $score += 8;
                        break;
                    }
                }
            }
            
            // Penalty for navigation-like content
            if ($this->isNavigationOrFooter($element)) {
                $score -= 20;
            }
            
            // Bonus for good sentence structure
            $sentences = preg_split('/[.!?]+/', $text);
            $avg_sentence_length = $text_length / max(1, count($sentences));
            if ($avg_sentence_length >= 20 && $avg_sentence_length <= 150) {
                $score += 5;
            }
            
            // Bonus for paragraph density (good content areas have more paragraphs)
            $parent = $element->parentNode;
            if ($parent) {
                try {
                    $siblings = $xpath->query('p', $parent);
                    if ($siblings && $siblings->length > 2) {
                        $score += 3;
                    }
                } catch (Exception $e) {
                    // Ignore XPath errors for siblings
                }
            }
            
            return max(0, $score);
            
        } catch (Exception $e) {
            airsrefe_append_log("⚠️ Error calculating element score: " . $e->getMessage());
            return 0;
        }
    }
    
            /**
     * Format element content with proper HTML structure
     */
    private function formatElementContent($element) {
        try {
            if (!$element) {
                return '';
            }
            
            $tag_name = strtolower($element->nodeName);
            $text = trim($element->textContent);
            
            // Skip empty text
            if (empty($text)) {
                return '';
            }
            
            // Preserve headings
            if (strpos($tag_name, 'h') === 0) {
                return "<{$tag_name}>" . $text . "</{$tag_name}>";
            }
            
            // Convert divs to paragraphs if they contain substantial text
            if ($tag_name === 'div' && strlen($text) > 100) {
                return "<p>" . $text . "</p>";
            }
            
            // Keep paragraphs as is
            if ($tag_name === 'p') {
                return "<p>" . $text . "</p>";
            }
            
            // Default to paragraph
            return "<p>" . $text . "</p>";
            
        } catch (Exception $e) {
            airsrefe_append_log("⚠️ Error formatting element content: " . $e->getMessage());
            return '';
        }
    }
    
    /**
     * Meta-based extraction
     */
    private function extractWithMeta($html) {
        preg_match_all('/<meta[^>]+name=["\'](description|og:description|twitter:description)["\'][^>]+content=["\']([^"\']+)["\']/', $html, $matches);
        
        if (!empty($matches[2])) {
            return implode(' ', $matches[2]);
        }
        
        return '';
    }
    
    /**
     * Fallback extraction
     */
    private function extractWithFallback($html) {
        // Remove scripts, styles, and other non-content elements
        $html = preg_replace('/<script[^>]*>.*?<\/script>/is', '', $html);
        $html = preg_replace('/<style[^>]*>.*?<\/style>/is', '', $html);
        $html = preg_replace('/<nav[^>]*>.*?<\/nav>/is', '', $html);
        $html = preg_replace('/<header[^>]*>.*?<\/header>/is', '', $html);
        $html = preg_replace('/<footer[^>]*>.*?<\/footer>/is', '', $html);
        
        // Extract text content
        $text = wp_strip_all_tags($html);
        $text = preg_replace('/\s+/', ' ', $text);
        
        return trim($text);
    }
    
    /**
     * Extract text from DOM nodes
     */
    private function extractTextFromNodes($nodes) {
        $content = '';
        foreach ($nodes as $node) {
            $content .= $node->textContent . "\n\n";
        }
        return trim($content);
    }
    
    /**
     * Extract paragraphs from DOM
     */
    private function extractParagraphs($xpath) {
        $paragraphs = $xpath->query('//p[string-length(text()) > 50]');
        $content = '';
        
        foreach ($paragraphs as $p) {
            $text = trim($p->textContent);
            if (!$this->isNavigationOrFooter($p)) {
                $content .= $text . "\n\n";
            }
        }
        
        return trim($content);
    }
    
    /**
     * Check if element is navigation or footer
     */
    private function isNavigationOrFooter($element) {
        try {
            if (!$element) {
                return false;
            }
            
            $text = strtolower($element->textContent);
            $navigation_words = ['menu', 'navigation', 'nav', 'home', 'about', 'contact', 'privacy', 'terms', 'cookie', 'sitemap', 'rss', 'feed', 'subscribe', 'newsletter'];
            
            foreach ($navigation_words as $word) {
                if (strpos($text, $word) !== false) {
                    return true;
                }
            }
            
            // Check for navigation-like classes and IDs
            $class = $element->getAttribute('class');
            $id = $element->getAttribute('id');
            
            if ($class) {
                $nav_classes = ['nav', 'navigation', 'menu', 'sidebar', 'footer', 'header'];
                foreach ($nav_classes as $nav_class) {
                    if (stripos($class, $nav_class) !== false) {
                        return true;
                    }
                }
            }
            
            if ($id) {
                $nav_ids = ['nav', 'navigation', 'menu', 'sidebar', 'footer', 'header'];
                foreach ($nav_ids as $nav_id) {
                    if (stripos($id, $nav_id) !== false) {
                        return true;
                    }
                }
            }
            
            return false;
            
        } catch (Exception $e) {
            airsrefe_append_log("⚠️ Error checking navigation/footer: " . $e->getMessage());
            return false;
        }
    }
    
    /**
     * Fetch content from URL
     */
    private function fetchContent($url) {
        airsrefe_append_log("Fetching content from: " . $url);
        
        $response = wp_remote_get($url, [
            'timeout' => 30,
            'user-agent' => 'Mozilla/5.0 (compatible; AI RSS Rewriter Bot)',
            'headers' => [
                'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language' => 'en-US,en;q=0.5',
                'Accept-Encoding' => 'gzip, deflate',
                'Connection' => 'keep-alive',
            ]
        ]);
        
        if (is_wp_error($response)) {
            airsrefe_append_log("Error fetching content: " . $response->get_error_message());
            return '';
        }
        
        $response_code = wp_remote_retrieve_response_code($response);
        if ($response_code !== 200) {
            airsrefe_append_log("HTTP error " . $response_code . " when fetching content from: " . $url);
            return '';
        }
        
        $body = wp_remote_retrieve_body($response);
        $body_length = strlen($body);
        
        if ($body_length < 100) {
            airsrefe_append_log("Content too short (" . $body_length . " chars), likely not HTML");
            return '';
        }
        
        airsrefe_append_log("Successfully fetched " . $body_length . " characters from: " . $url);
        return $body;
    }
    
    /**
     * Record successful strategy for domain
     */
    private function recordStrategySuccess($url, $strategy) {
        try {
            if (!$this->performanceTracker) {
                airsrefe_append_log("⚠️ Performance tracker not available, skipping strategy recording");
                return;
            }
            
            $domain = wp_parse_url($url, PHP_URL_HOST);
            if ($domain) {
                $this->performanceTracker->recordStrategySuccess($domain, $strategy);
                airsrefe_append_log("✅ Recorded successful strategy " . $strategy . " for domain: " . $domain);
            } else {
                airsrefe_append_log("⚠️ Could not parse domain from URL: " . $url);
            }
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error recording strategy success: " . $e->getMessage());
        }
    }
}

/**
 * Performance tracking for content extraction
 */
class AIRSREFE_PerformanceTracker {
    private $stats;
    
    public function __construct() {
        try {
            $this->stats = get_option('airsrefe_extraction_stats', []);
            
            if (!is_array($this->stats)) {
                $this->stats = [];
                airsrefe_append_log("⚠️ Reset invalid extraction stats to empty array");
            }
            
            airsrefe_append_log("📊 Performance tracker initialized with " . count($this->stats) . " domain stats");
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error initializing performance tracker: " . $e->getMessage());
            $this->stats = [];
        }
    }
    
    /**
     * Get domain statistics
     */
    public function getDomainStats($domain) {
        try {
            if (empty($domain)) {
                airsrefe_append_log("⚠️ Empty domain requested for stats");
                return null;
            }
            
            $stats = $this->stats[$domain] ?? null;
            
            if ($stats) {
                airsrefe_append_log("📊 Retrieved stats for domain: " . $domain . " (strategies: " . count($stats['strategies'] ?? []) . ")");
            } else {
                airsrefe_append_log("📊 No stats found for domain: " . $domain);
            }
            
            return $stats;
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error getting domain stats for " . $domain . ": " . $e->getMessage());
            return null;
        }
    }
    
    /**
     * Record successful strategy
     */
    public function recordStrategySuccess($domain, $strategy) {
        try {
            if (empty($domain) || empty($strategy)) {
                airsrefe_append_log("⚠️ Invalid domain or strategy for recording: " . ($domain ?: 'empty') . ", " . ($strategy ?: 'empty'));
                return;
            }
            
            if (!isset($this->stats[$domain])) {
                $this->stats[$domain] = [
                    'strategies' => [],
                    'total_attempts' => 0,
                    'successful_extractions' => 0
                ];
            }
            
            if (!isset($this->stats[$domain]['strategies'][$strategy])) {
                $this->stats[$domain]['strategies'][$strategy] = [
                    'attempts' => 0,
                    'successes' => 0
                ];
            }
            
            $this->stats[$domain]['strategies'][$strategy]['attempts']++;
            $this->stats[$domain]['strategies'][$strategy]['successes']++;
            $this->stats[$domain]['total_attempts']++;
            $this->stats[$domain]['successful_extractions']++;
            
            // Update best strategy
            $this->updateBestStrategy($domain);
            
            // Save stats
            $update_result = update_option('airsrefe_extraction_stats', $this->stats);
            
            if ($update_result) {
                airsrefe_append_log("✅ Updated stats for domain " . $domain . " - strategy " . $strategy . " success recorded");
            } else {
                airsrefe_append_log("⚠️ Failed to update extraction stats option");
            }
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error recording strategy success: " . $e->getMessage());
        }
    }
    
    /**
     * Update best strategy for domain
     */
    private function updateBestStrategy($domain) {
        try {
            if (!isset($this->stats[$domain]) || !isset($this->stats[$domain]['strategies'])) {
                return;
            }
            
            $best_strategy = null;
            $best_ratio = 0;
            $total_strategies = count($this->stats[$domain]['strategies']);
            
            foreach ($this->stats[$domain]['strategies'] as $strategy => $data) {
                if ($data['attempts'] > 5) { // Minimum attempts threshold
                    $ratio = $data['successes'] / $data['attempts'];
                    if ($ratio > $best_ratio) {
                        $best_ratio = $ratio;
                        $best_strategy = $strategy;
                    }
                }
            }
            
            if ($best_strategy) {
                $this->stats[$domain]['best_strategy'] = $best_strategy;
                airsrefe_append_log("✅ Updated best strategy for domain " . $domain . " to: " . $best_strategy . " (ratio: " . round($best_ratio, 2) . ")");
            } else {
                airsrefe_append_log("⚠️ No strategy meets minimum threshold for domain " . $domain . " (total strategies: " . $total_strategies . ")");
            }
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error updating best strategy for domain " . $domain . ": " . $e->getMessage());
        }
    }
}

/**
 * Background content processing
 */
class AIRSREFE_BackgroundProcessor {
    private $queue;
    private $max_concurrent;
    private $processing_delay;
    
    public function __construct($max_concurrent = 3, $processing_delay = 5) {
        try {
            $this->max_concurrent = max(1, min(10, $max_concurrent)); // Ensure reasonable limits
            $this->processing_delay = max(1, min(60, $processing_delay)); // Ensure reasonable limits
            $this->queue = get_option('airsrefe_content_queue', []);
            
            if (!is_array($this->queue)) {
                $this->queue = [];
                airsrefe_append_log("⚠️ Reset invalid queue data to empty array");
            }
            
            // Only log initialization if there are queued items
            $queue_count = count($this->queue);
            if ($queue_count > 0) {
                airsrefe_append_log("🚀 Background processor initialized with " . $queue_count . " queued items");
            }
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error initializing background processor: " . $e->getMessage());
            $this->max_concurrent = 3;
            $this->processing_delay = 5;
            $this->queue = [];
        }
    }
    
    /**
     * Add content to processing queue
     */
    public function addToQueue($post_id, $url, $priority = 'normal') {
        try {
            if (empty($post_id) || empty($url)) {
                airsrefe_append_log("❌ Invalid parameters for queue: post_id=" . $post_id . ", url=" . ($url ?: 'empty'));
                return false;
            }
            
            $queue_item = [
                'post_id' => $post_id,
                'url' => $url,
                'priority' => $priority,
                'status' => 'pending',
                'created_at' => current_time('mysql'),
                'attempts' => 0,
                'max_attempts' => 3
            ];
            
            $this->queue[] = $queue_item;
            $update_result = update_option('airsrefe_content_queue', $this->queue);
            
            if (!$update_result) {
                airsrefe_append_log("❌ Failed to update content queue option");
                return false;
            }
            
            // Schedule processing if not already scheduled
            if (!wp_next_scheduled('airsrefe_process_content_queue')) {
                $scheduled = wp_schedule_single_event(time() + $this->processing_delay, 'airsrefe_process_content_queue');
                if ($scheduled) {
                    airsrefe_append_log("✅ Scheduled content queue processing for " . $this->processing_delay . " seconds from now");
                } else {
                    airsrefe_append_log("❌ Failed to schedule content queue processing");
                }
            }
            
            airsrefe_append_log("✅ Added post " . $post_id . " to content queue with priority: " . $priority);
            return true;
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error adding to content queue: " . $e->getMessage());
            return false;
        }
    }
    
    /**
     * Process content queue
     */
    public function processQueue() {
        try {
            if (empty($this->queue)) {
                airsrefe_append_log("📋 Content queue is empty, nothing to process");
                return;
            }
            
            airsrefe_append_log("🚀 Processing content queue with " . count($this->queue) . " items");
            
            // Sort by priority
            usort($this->queue, function($a, $b) {
                $priorities = ['high' => 3, 'normal' => 2, 'low' => 1];
                return $priorities[$b['priority']] - $priorities[$a['priority']];
            });
            
            $processing = array_slice($this->queue, 0, $this->max_concurrent);
            $remaining = array_slice($this->queue, $this->max_concurrent);
            
            airsrefe_append_log("⚡ Processing " . count($processing) . " items concurrently, " . count($remaining) . " remaining");
            
            foreach ($processing as $item) {
                $this->processQueueItem($item);
            }
            
            // Update queue
            $this->queue = $remaining;
            $update_result = update_option('airsrefe_content_queue', $this->queue);
            
            if (!$update_result) {
                airsrefe_append_log("⚠️ Failed to update content queue after processing");
            }
            
            // Schedule next processing if items remain
            if (!empty($remaining)) {
                $scheduled = wp_schedule_single_event(time() + $this->processing_delay, 'airsrefe_process_content_queue');
                if ($scheduled) {
                    airsrefe_append_log("⏰ Scheduled next content queue processing for " . $this->processing_delay . " seconds from now");
                } else {
                    airsrefe_append_log("❌ Failed to schedule next content queue processing");
                }
            } else {
                airsrefe_append_log("✅ Content queue processing completed, no more items");
            }
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error processing content queue: " . $e->getMessage());
        }
    }
    
    /**
     * Process individual queue item
     */
    private function processQueueItem($item) {
        try {
            if ($item['attempts'] >= $item['max_attempts']) {
                airsrefe_append_log("❌ Failed to process content for post {$item['post_id']} after {$item['attempts']} attempts");
                return false;
            }
            
            $item['attempts']++;
            airsrefe_append_log("🔄 Processing queue item for post {$item['post_id']} (attempt {$item['attempts']}/{$item['max_attempts']})");
            
            $extractor = new AIRSREFE_ContentExtractor();
            
            if (!$extractor) {
                airsrefe_append_log("❌ Failed to create content extractor for post {$item['post_id']}");
                return false;
            }
            
            $content = $extractor->extractContent($item['url']);
            
            if (!empty($content)) {
                // Update post with extracted content
                $update_result = $this->updatePostContent($item['post_id'], $content);
                
                if ($update_result) {
                    airsrefe_append_log("✅ Content extracted and updated for post {$item['post_id']}");
                    return true;
                } else {
                    airsrefe_append_log("⚠️ Content extracted but failed to update post {$item['post_id']}");
                }
            } else {
                airsrefe_append_log("⚠️ Content extraction failed for post {$item['post_id']}");
            }
            
            // Retry if content extraction failed
            if ($item['attempts'] < $item['max_attempts']) {
                $this->queue[] = $item;
                airsrefe_append_log("🔄 Re-queued post {$item['post_id']} for retry (attempt {$item['attempts']}/{$item['max_attempts']})");
            } else {
                airsrefe_append_log("❌ Post {$item['post_id']} exceeded maximum attempts, removing from queue");
            }
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error processing content for post {$item['post_id']}: " . $e->getMessage());
            
            if ($item['attempts'] < $item['max_attempts']) {
                $this->queue[] = $item;
                airsrefe_append_log("🔄 Re-queued post {$item['post_id']} after error for retry");
            }
        }
        
        return false;
    }
    
    /**
     * Update post with extracted content
     */
    private function updatePostContent($post_id, $content) {
        try {
            if (empty($post_id) || empty($content)) {
                airsrefe_append_log("❌ Invalid parameters for updatePostContent: post_id=" . $post_id . ", content_length=" . strlen($content));
                return false;
            }
            
            $post = get_post($post_id);
            if (!$post) {
                airsrefe_append_log("❌ Post not found for ID: " . $post_id);
                return false;
            }
            
            $original_content = $post->post_content;
            $updated_content = $original_content;
            
            if (!empty($original_content)) {
                $updated_content .= "\n\n" . $content;
                airsrefe_append_log("📝 Appending extracted content to existing post content for post {$post_id}");
            } else {
                $updated_content = $content;
                airsrefe_append_log("📝 Setting extracted content as main content for post {$post_id}");
            }
            
            $update_result = wp_update_post([
                'ID' => $post_id,
                'post_content' => $updated_content
            ]);
            
            if ($update_result) {
                // Mark as processed
                $meta_result = update_post_meta($post_id, '_airsrefe_content_extracted', current_time('mysql'));
                
                if ($meta_result) {
                    airsrefe_append_log("✅ Successfully updated post {$post_id} with extracted content and meta");
                } else {
                    airsrefe_append_log("⚠️ Post updated but meta update failed for post {$post_id}");
                }
                
                return true;
            } else {
                airsrefe_append_log("❌ Failed to update post {$post_id} with extracted content");
                return false;
            }
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error updating post content for post {$post_id}: " . $e->getMessage());
            return false;
        }
    }
    
    /**
     * Get queue status
     */
    public function getQueueStatus() {
        try {
            $total_items = count($this->queue);
            $pending = count(array_filter($this->queue, function($item) {
                return $item['status'] === 'pending';
            }));
            $processing = count(array_filter($this->queue, function($item) {
                return $item['status'] === 'processing';
            }));
            
            $status = [
                'total_items' => $total_items,
                'pending' => $pending,
                'processing' => $processing
            ];
            
            airsrefe_append_log("📊 Queue status: " . $total_items . " total, " . $pending . " pending, " . $processing . " processing");
            
            return $status;
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error getting queue status: " . $e->getMessage());
            return [
                'total_items' => 0,
                'pending' => 0,
                'processing' => 0
            ];
        }
    }
    
    /**
     * Clear queue
     */
    public function clearQueue() {
        try {
            $queue_size = count($this->queue);
            
            if ($queue_size === 0) {
                airsrefe_append_log("📋 Content queue is already empty");
                return true;
            }
            
            $this->queue = [];
            $update_result = update_option('airsrefe_content_queue', []);
            $clear_result = wp_clear_scheduled_hook('airsrefe_process_content_queue');
            
            if ($update_result && $clear_result) {
                airsrefe_append_log("✅ Successfully cleared content queue with " . $queue_size . " items");
                return true;
            } else {
                airsrefe_append_log("⚠️ Partially cleared content queue - some operations failed");
                return false;
            }
            
        } catch (Exception $e) {
            airsrefe_append_log("❌ Error clearing content queue: " . $e->getMessage());
            return false;
        }
    }
}

/**
 * Background Processor Singleton
 * Manages the global background processor instance
 */
class AIRSREFE_Background_Processor_Singleton {
    
    private static $instance = null;
    
    /**
     * Get the singleton instance
     *
     * @return AIRSREFE_BackgroundProcessor
     */
    public static function get_instance() {
        if (self::$instance === null) {
            self::$instance = new AIRSREFE_BackgroundProcessor();
        }
        return self::$instance;
    }
}

// Hook for background processing using singleton
add_action('airsrefe_process_content_queue', [AIRSREFE_Background_Processor_Singleton::class, 'get_instance']);

// Helper function for content extraction
function airsrefe_extract_content($url, $fallback_content = '') {
    try {
        if (empty($url) || !filter_var($url, FILTER_VALIDATE_URL)) {
            airsrefe_append_log("❌ Invalid URL for content extraction: " . ($url ?: 'empty'));
            return $fallback_content;
        }
        
        $extractor = new AIRSREFE_ContentExtractor();
        
        if (!$extractor) {
            airsrefe_append_log("❌ Failed to create content extractor");
            return $fallback_content;
        }
        
        $content = $extractor->extractContent($url, $fallback_content);
        
        if (!empty($content)) {
            $content_length = strlen(wp_strip_all_tags($content));
            airsrefe_append_log("✅ Content extraction completed: " . $content_length . " characters from: " . $url);
        } else {
            airsrefe_append_log("⚠️ Content extraction failed for: " . $url);
        }
        
        return $content;
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error in airsrefe_extract_content: " . $e->getMessage());
        return $fallback_content;
    }
}

/**
 * Process content in background queue
 *
 * @param int    $post_id Post ID to process
 * @param string $url     URL to extract content from
 * @param string $priority Processing priority (normal, high, low)
 * @return bool Success status
 */
function airsrefe_process_content_background($post_id, $url, $priority = 'normal') {
    try {
        if (empty($post_id) || empty($url)) {
            airsrefe_append_log("❌ Invalid parameters for background processing: post_id=" . $post_id . ", url=" . ($url ?: 'empty'));
            return false;
        }
        
        $processor = AIRSREFE_Background_Processor_Singleton::get_instance();
        $result = $processor->addToQueue($post_id, $url, $priority);
        
        if ($result) {
            airsrefe_append_log("✅ Added post " . $post_id . " to background processing queue with priority: " . $priority);
        } else {
            airsrefe_append_log("❌ Failed to add post " . $post_id . " to background processing queue");
        }
        
        return $result;
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error in airsrefe_process_content_background: " . $e->getMessage());
        return false;
    }
}

/**
 * Extract full content from RSS item using the best extraction method
 * This function is called from RSS parser to enhance content
 */
function airsrefe_extract_full_content($item, $fallback_content = '') {
    if (!$item || !method_exists($item, 'get_link')) {
        airsrefe_append_log("❌ Invalid RSS item for content extraction");
        return $fallback_content;
    }
    
    $url = $item->get_link();
    if (empty($url) || !filter_var($url, FILTER_VALIDATE_URL)) {
        airsrefe_append_log("❌ Invalid URL for content extraction: " . ($url ?: 'empty'));
        return $fallback_content;
    }
    
    // Get extraction method from settings (should always be readability now)
    $extraction_method = get_option('airsrefe_content_extraction_method', 'readability');
    
    airsrefe_append_log("🚀 Starting content extraction from: " . $url . " using method: " . $extraction_method);
    
    try {
        $extractor = new AIRSREFE_ContentExtractor();
        
        if (!$extractor) {
            airsrefe_append_log("❌ Failed to create content extractor");
            return $fallback_content;
        }
        
        // Force readability method for best quality
        $content = $extractor->extractContent($url, $fallback_content);
        
        if (!empty($content) && $content !== $fallback_content) {
            $content_length = strlen(wp_strip_all_tags($content));
            $fallback_length = strlen(wp_strip_all_tags($fallback_content));
            $improvement = $content_length - $fallback_length;
            
            airsrefe_append_log("✅ Content extraction successful! Extracted " . $content_length . " characters (improvement: +" . $improvement . " chars)");
            
            // Clean and format the extracted content
            $cleaned_content = airsrefe_clean_extracted_content($content);
            $cleaned_length = strlen(wp_strip_all_tags($cleaned_content));
            
            airsrefe_append_log("🧹 Content cleaned: " . $cleaned_length . " characters after cleaning");
            
            return $cleaned_content;
        } else {
            airsrefe_append_log("⚠️ Content extraction returned empty or same as fallback content");
            return $fallback_content;
        }
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error during content extraction: " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
        return $fallback_content;
    }
}

/**
 * Clean and format extracted content
 */
function airsrefe_clean_extracted_content($content) {
    if (empty($content)) return '';
    
    try {
        $original_length = strlen(wp_strip_all_tags($content));
        
        // Remove common unwanted elements
        $unwanted_patterns = [
            '/<script[^>]*>.*?<\/script>/is',
            '/<style[^>]*>.*?<\/style>/is',
            '/<nav[^>]*>.*?<\/nav>/is',
            '/<header[^>]*>.*?<\/header>/is',
            '/<footer[^>]*>.*?<\/footer>/is',
            '/<aside[^>]*>.*?<\/aside>/is',
            '/<form[^>]*>.*?<\/form>/is',
            '/<div[^>]*class=["\'][^"\']*(?:ad|advertisement|banner|sidebar|widget|comment|social|share|related|recommended)[^"\']*["\'][^>]*>.*?<\/div>/is',
            '/<div[^>]*id=["\'][^"\']*(?:ad|advertisement|banner|sidebar|widget|comment|social|share|related|recommended)[^"\']*["\'][^>]*>.*?<\/div>/is'
        ];
        
        $removed_count = 0;
        foreach ($unwanted_patterns as $pattern) {
            $before = strlen($content);
            $content = preg_replace($pattern, '', $content);
            $after = strlen($content);
            if ($before > $after) {
                $removed_count++;
            }
        }
        
        // Clean up whitespace and formatting
        $content = preg_replace('/\s+/', ' ', $content);
        $content = preg_replace('/\n{3,}/', "\n\n", $content);
        $content = preg_replace('/<p[^>]*>\s*<\/p>/i', '', $content);
        
        // Ensure proper paragraph structure
        $content = preg_replace('/<\/p>\s*<p[^>]*>/', "\n\n", $content);
        
        // Remove any remaining empty tags
        $content = preg_replace('/<[^>]*>\s*<\/[^>]*>/i', '', $content);
        
        $cleaned_length = strlen(wp_strip_all_tags($content));
        $improvement = $original_length - $cleaned_length;
        
        if ($improvement > 0) {
            airsrefe_append_log("🧹 Content cleaned: " . $cleaned_length . " characters remaining (removed " . $improvement . " chars, " . $removed_count . " patterns applied)");
        } else {
            airsrefe_append_log("🧹 Content cleaned: " . $cleaned_length . " characters remaining (no significant cleaning needed)");
        }
        
        return trim($content);
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error cleaning extracted content: " . $e->getMessage());
        return $content; // Return original content if cleaning fails
    }
}