<?php

if (!defined('ABSPATH')) exit;

/**
 * Memory-efficient RSS parsing with XMLReader streaming
 * 
 * Processes RSS feeds without loading entire content into memory using XMLReader
 * for better performance and memory usage on large feeds.
 * 
 * @param string $feed_url The RSS feed URL to process
 * @param int $limit Maximum number of items to process (default: 10)
 * @return bool|array Returns false on failure, array with processing results on success
 */
function airsrefe_process_feed_streaming($feed_url, $limit = 10) {
    try {
        if (empty($feed_url) || !airsrefe_validate_feed_url($feed_url)) {
            airsrefe_append_log("❌ Invalid feed URL for streaming: " . ($feed_url ?: 'empty'));
            return false;
        }
        
        // PERFORMANCE FIX: Enhanced memory management
        $memory_start = memory_get_usage();
        $memory_limit = ini_get('memory_limit');
        $memory_limit_bytes = wp_convert_hr_to_bytes($memory_limit);
        $max_memory_usage = $memory_limit_bytes * 0.8; // Use max 80% of available memory
        
        $items_processed = 0;
        $items_created = 0;
        
        airsrefe_append_log("🚀 Starting streaming RSS processing for: {$feed_url}");
        
        // Initialize XMLReader for streaming
        $reader = new XMLReader();
        
        if (!$reader->open($feed_url)) {
            airsrefe_append_log("❌ Failed to open RSS feed for streaming: {$feed_url}");
            return false;
        }
        
        $items = [];
        $current_item = null;
        $inside_item = false;
        
        // Stream through RSS content
        while ($reader->read() && $items_processed < $limit) {
            try {
                switch ($reader->nodeType) {
                    case XMLReader::ELEMENT:
                        if ($reader->localName === 'item' || $reader->localName === 'entry') {
                            $inside_item = true;
                            $current_item = [];
                        } elseif ($inside_item) {
                            $element_name = $reader->localName;
                            $reader->read();
                            if ($reader->nodeType === XMLReader::TEXT || $reader->nodeType === XMLReader::CDATA) {
                                $current_item[$element_name] = $reader->value;
                            }
                        }
                        break;
                        
                    case XMLReader::END_ELEMENT:
                        if ($reader->localName === 'item' || $reader->localName === 'entry') {
                            $inside_item = false;
                            
                            if (!empty($current_item)) {
                                // Process item immediately instead of storing in array
                                $processed = airsrefe_process_single_rss_item($current_item);
                                if ($processed) {
                                    $items_created++;
                                }
                                $items_processed++;
                                
                                // PERFORMANCE FIX: Enhanced memory management
                                if ($items_processed % 5 === 0) {
                                    $memory_current = memory_get_usage();
                                    $memory_diff = $memory_current - $memory_start;
                                    
                                    // Check against dynamic memory limit
                                    if ($memory_current > $max_memory_usage) {
                                        airsrefe_append_log("⚠️ Memory limit reached (" . size_format($memory_current) . "), stopping processing");
                                        break 2;
                                    }
                                    
                                    // Force garbage collection every 10 items
                                    if ($items_processed % 10 === 0) {
                                        wp_cache_flush();
                                        if (function_exists('gc_collect_cycles')) {
                                            gc_collect_cycles();
                                        }
                                    }
                                }
                            }
                            $current_item = null;
                        }
                        break;
                }
            } catch (Exception $e) {
                airsrefe_append_log("⚠️ Error processing RSS element: " . $e->getMessage());
                continue; // Continue with next element
            }
        }
        
        $reader->close();
        
        $memory_end = memory_get_usage();
        $memory_used = $memory_end - $memory_start;
        
        airsrefe_append_log(sprintf("✅ Streaming RSS complete: %d items processed, %d posts created, %s memory used", 
            $items_processed, $items_created, size_format($memory_used)));
        
        return $items_created;
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error during streaming RSS processing: " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
        return false;
    }
}

/**
 * Process single RSS item with quality filtering and AI enhancement
 * 
 * Takes RSS item data and processes it through quality checks, duplicate detection,
 * and AI enhancement if enabled. Creates WordPress post if item passes all filters.
 * 
 * @param array $item_data RSS item data containing title, description, link, etc.
 * @return bool|int Returns false on failure, post ID on success
 */
function airsrefe_process_single_rss_item($item_data) {
    try {
        if (empty($item_data) || !is_array($item_data)) {
            airsrefe_append_log("❌ Invalid item data for RSS processing");
            return false;
        }
        
        // Extract basic data
        $title = $item_data['title'] ?? '';
        $link = $item_data['link'] ?? $item_data['guid'] ?? '';
        $content = $item_data['description'] ?? $item_data['content'] ?? '';
        $pub_date = $item_data['pubDate'] ?? $item_data['published'] ?? '';
        
        if (empty($title) || empty($link)) {
            airsrefe_append_log("⚠️ Missing title or link for RSS item: " . ($title ?: 'empty title') . " | " . ($link ?: 'empty link'));
            return false;
        }
        
        airsrefe_append_log("🔍 Processing RSS item: " . $title);
        
        // Quick duplicate check
        if (airsrefe_check_duplicate_post($link)) {
            airsrefe_append_log("⚠️ Duplicate found, skipping: {$title}");
            return false;
        }
        
        // Enhanced content quality filtering
        if (!airsrefe_should_process_with_ai_enhanced($title, $content)) {
            airsrefe_append_log("⚠️ Content filtered out: {$title}");
            return false;
        }
        
        // Check daily limits and budget
        if (airsrefe_check_daily_parsing_limit()) {
            airsrefe_append_log("⚠️ Daily limit reached, stopping processing");
            return false;
        }
        
        $daily_budget = get_option('airsrefe_daily_budget_limit', 10.00);
        $today_cost = airsrefe_get_today_costs();
        
        if ($today_cost >= $daily_budget * 0.95) {
            airsrefe_append_log("⚠️ Budget nearly exhausted, stopping processing");
            return false;
        }
        
        // Create post with optimized processing
        $result = airsrefe_create_post_from_rss_item($title, $content, $link, $pub_date);
        
        if ($result !== false) {
            airsrefe_append_log("✅ Successfully processed RSS item: " . $title);
        } else {
            airsrefe_append_log("❌ Failed to create post from RSS item: " . $title);
        }
        
        return $result !== false;
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error processing RSS item: " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
        return false;
    }
}

/**
 * Get cached AI result with enhanced multi-level caching
 * 
 * Retrieves AI processing results from multi-level cache system (memory, transient, database)
 * for improved performance and reduced API calls.
 * 
 * @param string $content_hash Hash of the content being processed
 * @param string $task_type Type of AI task (rewrite, summarize, etc.)
 * @param string $model AI model used for processing
 * @return mixed Cached result or false if not found
 */
function airsrefe_get_cached_ai_result_enhanced($content_hash, $task_type, $model) {
    // Level 1: Memory cache (fastest)
    $memory_key = "airsrefe_ai_{$task_type}_{$model}_" . substr($content_hash, 0, 12);
    $cached = wp_cache_get($memory_key, 'airsrefe_ai_results');
    
    if ($cached !== false) {
        airsrefe_track_cache_hit();
        return $cached;
    }
    
    // Level 2: Object cache (medium speed)
    $object_key = "airsrefe_ai_obj_{$content_hash}_{$task_type}_{$model}";
    $cached = wp_cache_get($object_key, 'airsrefe_ai_persistent');
    
    if ($cached !== false) {
        // Warm memory cache for faster future access
        wp_cache_set($memory_key, $cached, 'airsrefe_ai_results', 300); // 5 min in memory
        airsrefe_track_cache_hit();
        return $cached;
    }
    
    airsrefe_track_cache_miss();
    return false;
}

/**
 * OPTIMIZATION: Cache AI results with adaptive TTL based on cost and quality
 */
function airsrefe_set_cached_ai_result_enhanced($content_hash, $task_type, $model, $result, $cost = 0) {
    $memory_key = "airsrefe_ai_{$task_type}_{$model}_" . substr($content_hash, 0, 12);
    $object_key = "airsrefe_ai_obj_{$content_hash}_{$task_type}_{$model}";
    
    // Adaptive TTL based on cost and model
    $ttl = airsrefe_calculate_ai_cache_ttl($cost, $model, $task_type);
    
    // Store in both cache levels
    wp_cache_set($memory_key, $result, 'airsrefe_ai_results', min(300, $ttl)); // Max 5min in memory
    wp_cache_set($object_key, $result, 'airsrefe_ai_persistent', $ttl);
    
    airsrefe_append_log("AI result cached for {$ttl}s - cost: $" . number_format($cost, 4));
}

/**
 * OPTIMIZATION: Calculate optimal cache TTL based on processing cost
 */
function airsrefe_calculate_ai_cache_ttl($cost, $model, $task_type) {
    $base_ttl = 3600; // 1 hour base
    
    // Cost-based multiplier
    if ($cost > 0.05) {
        $multiplier = 48; // 48 hours for expensive requests
    } elseif ($cost > 0.01) {
        $multiplier = 24; // 24 hours for medium cost
    } elseif ($cost > 0.001) {
        $multiplier = 12; // 12 hours for low cost
    } else {
        $multiplier = 6; // 6 hours for very cheap requests
    }
    
    // Model-based adjustment
    $model_multipliers = [
        'gpt-5' => 2.0,       // Cache expensive model results longer
        'gpt-5-mini' => 1.5,  // Medium caching
        'gpt-5-nano' => 1.0   // Standard caching
    ];
    
    $model_multiplier = $model_multipliers[$model] ?? 1.0;
    
    // Task-based adjustment
    $task_multipliers = [
        'content' => 2.0,  // Content generation cached longest
        'title' => 1.5,    // Titles cached medium time
        'tags' => 1.0,     // Tags cached standard time
        'seo' => 1.2       // SEO descriptions cached a bit longer
    ];
    
    $task_multiplier = $task_multipliers[$task_type] ?? 1.0;
    
    $final_ttl = $base_ttl * $multiplier * $model_multiplier * $task_multiplier;
    
    // Cap maximum cache time at 7 days
    return min($final_ttl, 7 * 24 * 3600);
}

/**
 * OPTIMIZATION: Smart cache prewarming for popular feeds
 */
function airsrefe_prewarm_feed_cache($feed_urls) {
    if (empty($feed_urls)) {
        return;
    }
    
    $prewarmed = 0;
    $max_prewarm = 5; // Limit to avoid overload
    
    foreach (array_slice($feed_urls, 0, $max_prewarm) as $feed_url) {
        $cache_key = 'airsrefe_feed_' . substr(md5($feed_url), 0, 16);
        $cached = wp_cache_get($cache_key, 'airsrefe_rss_feeds');
        
        if ($cached === false) {
            // Asynchronously warm the cache
            wp_schedule_single_event(time() + ($prewarmed * 30), 'airsrefe_warm_feed_cache', [$feed_url]);
            $prewarmed++;
        }
    }
    
    if ($prewarmed > 0) {
        airsrefe_append_log("Scheduled cache warming for {$prewarmed} feeds");
    }
}

function airsrefe_check_daily_parsing_limit() {
$processing_mode = get_option('airsrefe_processing_mode', 'ai');

if ($processing_mode !== 'ai') {
return false;
}

$daily_limit = get_option('airsrefe_daily_parsing_limit', 50);

if ($daily_limit >= 1000) {
return false;
}

$cache_key = 'airsrefe_daily_count_' . current_time('Y-m-d');
$today_count = wp_cache_get($cache_key, 'airsrefe_daily_limits');

if (false === $today_count) {
$today_count = airsrefe_get_cached_daily_count();
wp_cache_set($cache_key, $today_count, 'airsrefe_daily_limits', 300);
}

if ($today_count >= $daily_limit) {
airsrefe_append_log(sprintf("Daily parsing limit reached: %d/%d posts processed today", $today_count, $daily_limit));
return true;
}

airsrefe_append_log(sprintf("Daily parsing usage: %d/%d posts (%d remaining)", $today_count, $daily_limit, $daily_limit - $today_count));
return false;
}

/**
 * Get cached daily count of processed posts for a specific date
 *
 * @since 1.0.0
 * @param string|null $date Date in Y-m-d format. Defaults to current date.
 * @return int Number of posts processed on the given date.
 */
function airsrefe_get_cached_daily_count($date = null) {
    if (!$date) {
        $date = current_time('Y-m-d');
    }

    $cache_key = 'airsrefe_daily_count_' . $date;
    $count = wp_cache_get($cache_key, 'airsrefe_daily_limits');

    if (false === $count) {
        global $wpdb;
        $date_obj = date_create($date, wp_timezone());
        
        if (!$date_obj) {
            return 0;
        }
        
        // SECURITY FIX: Use proper table names with proper escaping
        // OPTIMIZATION: Use direct SQL query instead of get_posts + get_post_meta loop
        
        // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery -- Performance optimization for daily count calculation
        $count = (int) $wpdb->get_var($wpdb->prepare("
            SELECT COUNT(DISTINCT p.ID) 
            FROM {$wpdb->posts} p 
            INNER JOIN {$wpdb->postmeta} pm ON p.ID = pm.post_id 
            WHERE p.post_type = %s
            AND p.post_status != %s
            AND YEAR(p.post_date) = %d
            AND MONTH(p.post_date) = %d  
            AND DAY(p.post_date) = %d
            AND pm.meta_key = %s
            AND pm.meta_value != %s
        ", 
            'post',
            'trash',
            (int) $date_obj->format('Y'),
            (int) $date_obj->format('m'),
            (int) $date_obj->format('d'),
            '_airsrefe_processed_date',
            ''
        ));
        
        // Cache for 5 minutes
        wp_cache_set($cache_key, $count, 'airsrefe_daily_limits', 300);
    }

    return (int) $count;
}

function airsrefe_invalidate_daily_count_cache($date = null) {
    if (!$date) {
        $date = current_time('Y-m-d');
    }
    
    $cache_key = 'airsrefe_daily_count_' . $date;
    wp_cache_delete($cache_key, 'airsrefe_daily_limits');
}

/**
 * OPTIMIZED: Batch duplicate checking - eliminates N+1 problem
 * Checks multiple URLs at once instead of individual queries
 */
function airsrefe_check_duplicates_batch($links) {
    if (empty($links)) {
        return [];
    }
    
    $cache_results = [];
    $uncached_links = [];
    
    // OPTIMIZATION: Check cache with performance tracking
    foreach ($links as $link) {
        $cache_key = 'airsrefe_dup_' . substr(md5($link), 0, 16); // Shorter cache keys
        $cached_result = wp_cache_get($cache_key, 'airsrefe_duplicates');
        
        if (false !== $cached_result) {
            $cache_results[$link] = $cached_result;
            airsrefe_track_cache_hit();
        } else {
            $uncached_links[] = $link;
            airsrefe_track_cache_miss();
        }
    }
    
    $db_results = [];
    
    if (!empty($uncached_links)) {
        global $wpdb;

        // PERFORMANCE FIX: Use single query instead of N+1 queries
        if (count($uncached_links) > 0) {
            // SECURITY FIX: Use proper table names and prepared statement
            // Create placeholders for IN clause dynamically
            $placeholders = implode(',', array_fill(0, count($uncached_links), '%s'));
            
            // Prepare parameters array
            $params = array_merge(
                ['_airsrefe_source_url'],
                $uncached_links,
                ['post', 'publish', 'draft', 'private', 'pending']
            );
            
            // phpcs:ignore WordPress.DB.DirectDatabaseQuery.DirectQuery,WordPress.DB.PreparedSQLPlaceholders.ReplacementsWrongNumber -- Performance optimization for batch duplicate checking
            $results = $wpdb->get_results($wpdb->prepare("
                SELECT DISTINCT pm.post_id, pm.meta_value as source_url
                FROM {$wpdb->postmeta} pm
                INNER JOIN {$wpdb->posts} p ON pm.post_id = p.ID
                WHERE pm.meta_key = %s
                AND pm.meta_value IN (" . implode(',', array_fill(0, count($uncached_links), '%s')) . ")
                AND p.post_type = %s
                AND p.post_status IN (%s, %s, %s, %s)
                ORDER BY p.post_date DESC
            ", $params));
        } else {
            $results = [];
        }
        
        // Build duplicates array
        $duplicates = [];
        foreach ($results as $result) {
            $duplicates[] = (object) [
                'source_url' => $result->source_url,
                'ID' => (int) $result->ID
            ];
        }
        
        foreach ($duplicates as $duplicate) {
            $db_results[$duplicate->source_url] = (int) $duplicate->ID;
        }
        
        foreach ($uncached_links as $link) {
            $post_id = isset($db_results[$link]) ? $db_results[$link] : 0;
            $cache_key = 'airsrefe_dup_' . substr(md5($link), 0, 16);

            $ttl = $post_id ? 7200 : 1800;
            wp_cache_set($cache_key, $post_id, 'airsrefe_duplicates', $ttl);
            $db_results[$link] = $post_id;
        }
    }
    
    return array_merge($cache_results, $db_results);
}

function airsrefe_check_duplicate_post($link, $title) {
    $batch_result = airsrefe_check_duplicates_batch([$link]);
    $existing_post_id = isset($batch_result[$link]) ? $batch_result[$link] : 0;
    
    if ($existing_post_id) {
        return $existing_post_id;
    }
    
    return post_exists($title);
}

function airsrefe_process_feed($feed) {
    try {
        if (empty($feed) || !is_array($feed)) {
            airsrefe_append_log("❌ Invalid feed data for processing");
            return;
        }
        
        if (!isset($feed['url']) || empty($feed['url'])) {
            airsrefe_append_log("❌ Feed URL is missing or empty");
            return;
        }
        
        if (get_option('airsrefe_block_parsing') == 1) {
            airsrefe_append_log("⚠️ Parsing blocked due to API issues or budget limits");
            return;
        }

        if (isset($feed['enabled']) && !$feed['enabled']) {
            airsrefe_append_log("⚠️ Feed disabled: " . $feed['url']);
            return;
        }

        if (airsrefe_check_daily_parsing_limit()) {
            airsrefe_append_log("⚠️ Daily parsing limit reached - skipping feed processing");
            return;
        }

$priority = airsrefe_get_processing_priority();

$daily_budget = get_option('airsrefe_daily_budget_limit', 10.00);
$today_cost = airsrefe_get_today_costs();

if ($daily_budget > 0) {
$remaining_budget = $daily_budget - $today_cost;
$budget_percent = ($today_cost / $daily_budget) * 100;

airsrefe_append_log(sprintf("Budget: $%.2f used / $%.2f budget (%.1f%% used, $%.2f remaining)", 
$today_cost, $daily_budget, $budget_percent, $remaining_budget));

if ($remaining_budget < 0.50) {
airsrefe_append_log("Low budget - switching to economy mode");
$priority = 'economy';
}
}

if (!function_exists('post_exists')) {
require_once ABSPATH . 'wp-admin/includes/post.php';
}

airsrefe_append_log("Starting feed processing: " . $feed['url']);

$rss = airsrefe_get_feed_with_smart_cache($feed['url']);

if (is_wp_error($rss)) {
airsrefe_append_log("RSS loading error: " . $rss->get_error_message());
return;
}

$limit = $feed['limit'] ?? 5;
$user_id = $feed['user_id'] ?? get_current_user_id();
$use_original_date = get_option('airsrefe_use_original_date', 1);
$processing_mode = get_option('airsrefe_processing_mode', 'ai');
$processing_level = get_option('airsrefe_default_processing_level', 'full');
$image_source = get_option('airsrefe_image_source', 'auto');

if (get_option('airsrefe_per_feed_ai_enabled', 0) && isset($feed['ai_settings'])) {
$processing_level = $feed['ai_settings']['processing_level'] ?? $processing_level;
$priority = $feed['ai_settings']['priority'] ?? $priority;
}

$items = $rss->get_items(0, $limit);
if (empty($items)) {
airsrefe_append_log("RSS feed empty or unavailable");
return;
}

airsrefe_append_log(sprintf("Found %d items (level: %s, priority: %s)", 
count($items), $processing_level, $priority));

$processed_count = 0;
$total_cost_this_session = 0.00;
$daily_limit = get_option('airsrefe_daily_parsing_limit', 50);

$item_data = [];
$all_links = [];

foreach ($items as $item) {
$title = $item->get_title();
$content = $item->get_content();
$link = $item->get_link();

if (empty($title) || empty($content) || empty($link)) {
continue;
}

$item_data[] = [
'item' => $item,
'title' => $title,
'content' => $content,
'link' => $link
];

$all_links[] = $link;
}

$duplicates_map = airsrefe_check_duplicates_batch($all_links);
airsrefe_append_log(sprintf("Batch duplicate check completed for %d items", count($all_links)));

foreach ($item_data as $data) {
try {
if ($processing_mode === 'ai' && $daily_limit < 1000) {
$current_today_count = airsrefe_get_cached_daily_count();

if ($current_today_count >= $daily_limit) {
airsrefe_append_log(sprintf("Daily parsing limit (%d) reached during processing - stopping", $daily_limit));
break;
}
}

if ($daily_budget > 0 && airsrefe_get_today_costs() >= $daily_budget) {
airsrefe_append_log("Daily budget reached - stopping processing");
break;
}

$item = $data['item'];
$title = $data['title'];
$content = $data['content'];
$link = $data['link'];

$enhanced_content = airsrefe_enhance_content_with_extraction($item, $content);

if (!empty($enhanced_content) && $enhanced_content !== $content) {
$content = $enhanced_content;
airsrefe_append_log("Full content extracted from source page");
}

if ($use_original_date) {
$pub_date = $item->get_date('Y-m-d H:i:s');
$pub_date_gmt = $item->get_date('Y-m-d H:i:s', true);
} else {
$pub_date = current_time('mysql');
$pub_date_gmt = current_time('mysql', 1);
}

if (isset($duplicates_map[$link]) && $duplicates_map[$link] > 0) {
airsrefe_append_log("Skipped duplicate: " . $title);
continue;
}

if (post_exists($title)) {
airsrefe_append_log("Skipped duplicate by title: " . $title);
continue;
}

$content = airsrefe_clean_content($content);
$featured_image_url = airsrefe_extract_image_from_content($content, $item, $link);

$start_cost = airsrefe_get_today_costs();

if ($processing_mode === 'ai') {
$ai_result = airsrefe_process_with_ai($title, $content, $processing_level, $priority);
} else {
$ai_result = ['title' => $title, 'content' => $content];
}

$post_categories = airsrefe_determine_categories_for_feed($feed, $item, $ai_result['content']);

$tags = [];
$seo_description = '';

if ($processing_mode === 'ai' && in_array($processing_level, ['basic', 'full', 'premium'])) {
if (get_option('airsrefe_generate_tags', 1)) {
$tags = airsrefe_generate_tags_enhanced($ai_result['title'], $ai_result['content'], $priority);
}

$seo_description = airsrefe_generate_description_enhanced($ai_result['content'], $priority);
}

// Title length is enforced at generation time by AI prompts (≤ configured limit). No truncation here.

$post_id = wp_insert_post([
'post_title'=> $ai_result['title'],
'post_content'  => $ai_result['content'],
'post_status'   => 'publish',
'post_category' => $post_categories,
'post_date' => $pub_date ?: current_time('mysql'),
'post_author'   => $user_id,
'post_date_gmt' => $pub_date_gmt ?: current_time('mysql', 1)
]);

if (is_wp_error($post_id)) {
airsrefe_append_log("Post creation error: " . $post_id->get_error_message());
continue;
}

airsrefe_add_post_metadata($post_id, $link, $processing_mode, $processing_level, $priority, $use_original_date, isset($feed['url']) ? $feed['url'] : '');

airsrefe_invalidate_daily_count_cache();

$cache_key = 'airsrefe_duplicate_url_' . md5($link);
wp_cache_delete($cache_key, 'airsrefe_duplicates');

if (!empty($tags)) {
wp_set_post_tags($post_id, $tags);
}

if (!empty($seo_description)) {
airsrefe_add_seo_metadata($post_id, $seo_description, $ai_result['title']);
}

$image_set = airsrefe_handle_post_image($post_id, $ai_result['title'], $ai_result['content'], $featured_image_url, $image_source, $processing_mode, $link);

$end_cost = airsrefe_get_today_costs();
$post_cost = $end_cost - $start_cost;
$total_cost_this_session += $post_cost;

$processed_count++;
$category_names = array_map('get_cat_name', $post_categories);
$date_info = $use_original_date ? 'original date' : 'parsing time';

airsrefe_append_log(sprintf("Created: '%s' | Level: %s | Cost: $%.4f | Categories: %s | Date: %s (%s) | Image: %s", 
$ai_result['title'], 
$processing_level,
$post_cost,
implode(', ', $category_names), 
$pub_date, 
$date_info,
$image_set ? 'Set' : 'None'
));

if ($processed_count < count($items)) {
$delay = airsrefe_calculate_smart_delay($priority, $post_cost);
if ($delay > 0) {
airsrefe_append_log("Smart delay: {$delay}s");
sleep($delay);
}
}

} catch (Exception $e) {
airsrefe_append_log("Post processing error: " . $e->getMessage());
continue;
}
}

$remaining_daily_limit = '';
if ($processing_mode === 'ai' && $daily_limit < 1000) {
$current_today_count = airsrefe_get_cached_daily_count();
$remaining_daily_limit = sprintf(" | Daily: %d/%d posts", $current_today_count, $daily_limit);
}

airsrefe_append_log(sprintf("✅ Processing completed. Processed: %d posts | Total cost: $%.4f | Remaining budget: $%.2f%s", 
$processed_count, 
$total_cost_this_session,
max(0, $daily_budget - airsrefe_get_today_costs()),
$remaining_daily_limit
));
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error during feed processing: " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
    }
}

function airsrefe_enhance_content_with_extraction($item, $original_content) {
    try {
        if (!get_option('airsrefe_extract_full_content', 1)) {
            airsrefe_append_log("Content extraction disabled in settings");
            return $original_content;
        }

        if (!$item || !method_exists($item, 'get_link')) {
            airsrefe_append_log("❌ Invalid RSS item for content enhancement");
            return $original_content;
        }

        $link = $item->get_link();
        if (empty($link) || !filter_var($link, FILTER_VALIDATE_URL)) {
            airsrefe_append_log("❌ Invalid or missing URL for content extraction: " . ($link ?: 'empty'));
            return $original_content;
        }

        $min_content_length = apply_filters('airsrefe_min_content_length', 300);
        $clean_content = wp_strip_all_tags($original_content);
        $content_length = strlen($clean_content);

        airsrefe_append_log("🔍 Content extraction check - RSS content length: " . $content_length . " chars, minimum required: " . $min_content_length . " chars");

        if ($content_length > $min_content_length) {
            airsrefe_append_log("✅ RSS content sufficient (" . $content_length . " chars), skipping extraction");
            return $original_content;
        }

        airsrefe_append_log("📄 RSS content too short (" . $content_length . " chars), visiting original article: " . $link);

        if (function_exists('airsrefe_extract_full_content')) {
            $extracted_content = airsrefe_extract_full_content($item, $original_content);

            if (!empty($extracted_content) && $extracted_content !== $original_content) {
                $extracted_length = strlen(wp_strip_all_tags($extracted_content));
                $improvement = $extracted_length - $content_length;
                airsrefe_append_log("✅ Content extraction successful! Extracted " . $extracted_length . " chars (improvement: +" . $improvement . " chars)");
                return $extracted_content;
            } else {
                airsrefe_append_log("⚠️ Content extraction returned same or empty content, using RSS content");
            }
        } else {
            airsrefe_append_log("❌ airsrefe_extract_full_content function not found");
        }

        airsrefe_append_log("⚠️ Content extraction failed, using RSS content");
        return $original_content;
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error during content enhancement: " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
        return $original_content;
    }
}

function airsrefe_process_with_ai($title, $content, $processing_level, $priority) {
    try {
        if (empty($title) || empty($content)) {
            airsrefe_append_log("❌ Invalid parameters for AI processing: title=" . ($title ?: 'empty') . ", content_length=" . strlen($content));
            return ['title' => $title ?: '', 'content' => $content ?: ''];
        }
        
        if (empty($processing_level)) {
            airsrefe_append_log("⚠️ No processing level specified, using 'full' as default");
            $processing_level = 'full';
        }
        
        airsrefe_append_log("🤖 Processing with AI: level=" . $processing_level . ", priority=" . $priority);
        
        switch ($processing_level) {
            case 'minimal':
                $new_title = airsrefe_generate_title_only($title, $content, $priority);
                airsrefe_log_processing_result($title, $new_title, 'minimal', !empty($new_title));
                return [
                    'title' => $new_title ?: $title,
                    'content' => $content
                ];

            case 'basic':
                $result = airsrefe_generate_ai_post_enhanced($title, $content, 'content', $priority);
                $result['content'] = $content;
                return $result;

            case 'full':
            case 'premium':
            default:
                return airsrefe_generate_ai_post_enhanced($title, $content, 'content', $priority);
        }
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error during AI processing: " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
        return ['title' => $title, 'content' => $content];
    }
}

function airsrefe_generate_title_only($title, $content, $priority) {
    try {
        if (empty($title) || empty($content)) {
            airsrefe_append_log("❌ Invalid parameters for title generation: title=" . ($title ?: 'empty') . ", content_length=" . strlen($content));
            return $title;
        }
        
        if (empty($priority)) {
            airsrefe_append_log("⚠️ No priority specified for title generation, using 'normal' as default");
            $priority = 'normal';
        }
        
        airsrefe_append_log("📝 Generating AI title for: " . substr($title, 0, 50) . "...");
        
        $model = airsrefe_get_optimal_model('title', strlen($content), $priority);

        $max_title_length = absint(get_option('airsrefe_max_title_length', 120));
        $prompt = "Create a compelling, unique headline for this article. Keep it under {$max_title_length} characters without trailing ellipsis. Respond only with the title:

Original title: \"{$title}\"
Article content: \"" . substr($content, 0, 300) . "...\"";

        $response = airsrefe_make_api_request_with_retry($model, $prompt);

        if ($response && isset($response['choices'][0]['message']['content'])) {
            $new_title = trim($response['choices'][0]['message']['content']);
            $new_title = airsrefe_clean_ai_text($new_title);

            $input_tokens = airsrefe_estimate_tokens($prompt);
            $output_tokens = airsrefe_estimate_tokens($new_title);
            $cost = airsrefe_calculate_request_cost($model, $input_tokens, $output_tokens);
            airsrefe_record_cost($cost, $model, 'title');

            airsrefe_append_log("✅ AI title generated successfully: " . substr($new_title, 0, 50) . "...");
            return $new_title;
        } else {
            airsrefe_append_log("⚠️ AI title generation failed, using original title");
        }

        return $title;
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error during title generation: " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
        return $title;
    }
}

function airsrefe_extract_image_from_content($content, $item, $source_url = '') {
    try {
        if (empty($content)) {
            airsrefe_append_log("⚠️ Empty content for image extraction");
            return '';
        }
        
        if (!$item) {
            airsrefe_append_log("⚠️ Invalid RSS item for image extraction");
            return '';
        }
        
        airsrefe_append_log("🖼️ Starting image extraction from content");
        $image_url = '';

        // Extract from content
        if (preg_match('/<img[^>]+src=["\']([^"\']+)["\'][^>]*>/i', $content, $img_match)) {
            $image_url = $img_match[1];

            if ($source_url && !filter_var($image_url, FILTER_VALIDATE_URL)) {
                $image_url = airsrefe_resolve_relative_url($image_url, $source_url);
            }

            airsrefe_append_log("✅ Found image in content: " . $image_url);
        }

        // Extract from enclosures
        if (empty($image_url)) {
            try {
                $enclosures = $item->get_enclosures();
                if (!empty($enclosures)) {
                    foreach ($enclosures as $enclosure) {
                        if (strpos($enclosure->get_type(), 'image/') === 0) {
                            $image_url = $enclosure->get_link();
                            airsrefe_append_log("✅ Found image in enclosure: " . $image_url);
                            break;
                        }
                    }
                }
            } catch (Exception $e) {
                airsrefe_append_log("⚠️ Error getting enclosures: " . $e->getMessage());
            }
        }

        // Extract from media:thumbnail
        if (empty($image_url)) {
            try {
                $raw_data = $item->get_item_tags('http://search.yahoo.com/mrss/', 'thumbnail');
                if (!empty($raw_data[0]['attribs']['']['url'])) {
                    $image_url = $raw_data[0]['attribs']['']['url'];
                    airsrefe_append_log("✅ Found image in media:thumbnail: " . $image_url);
                }
            } catch (Exception $e) {
                airsrefe_append_log("⚠️ Error getting media:thumbnail: " . $e->getMessage());
            }
        }

        // Extract from source page
        if (empty($image_url) && !empty($source_url) && function_exists('airsrefe_extract_featured_image')) {
            try {
                $featured_image_data = airsrefe_extract_featured_image($source_url);
                if ($featured_image_data && !empty($featured_image_data['url'])) {
                    $image_url = $featured_image_data['url'];
                    airsrefe_append_log("✅ Found featured image from source page: " . $image_url);
                }
            } catch (Exception $e) {
                airsrefe_append_log("⚠️ Error extracting featured image from source: " . $e->getMessage());
            }
        }

        // Validate and clean URL
        if (!empty($image_url)) {
            $image_url = esc_url_raw($image_url);
            if (!filter_var($image_url, FILTER_VALIDATE_URL)) {
                $image_url = '';
                airsrefe_append_log("⚠️ Invalid image URL found after cleaning");
            } else {
                airsrefe_append_log("✅ Image extraction completed successfully: " . $image_url);
            }
        } else {
            airsrefe_append_log("⚠️ No image found during extraction");
        }

        return $image_url;
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error during image extraction: " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
        return '';
    }
}

function airsrefe_handle_post_image($post_id, $title, $content, $rss_image_url, $image_source, $processing_mode, $source_url = '') {
    try {
        if (empty($post_id)) {
            airsrefe_append_log("❌ Invalid post ID for image handling");
            return false;
        }
        
        if (empty($title)) {
            airsrefe_append_log("⚠️ Empty title for image handling, post ID: " . $post_id);
        }
        
        if (empty($image_source)) {
            airsrefe_append_log("⚠️ No image source specified, using 'auto' as default");
            $image_source = 'auto';
        }
        
        airsrefe_append_log("🖼️ Handling post image: post_id=" . $post_id . ", source=" . $image_source . ", mode=" . $processing_mode);
        
        switch ($image_source) {
            case 'rss':
                if (!empty($rss_image_url)) {
                    $result = airsrefe_set_featured_image($rss_image_url, $post_id, $title);
                    if ($result) {
                        airsrefe_append_log("✅ RSS image set successfully for post " . $post_id);
                    } else {
                        airsrefe_append_log("⚠️ Failed to set RSS image for post " . $post_id);
                    }
                    return $result;
                } else {
                    airsrefe_append_log("⚠️ No RSS image URL provided for post " . $post_id);
                    return false;
                }

            case 'ai':
                if ($processing_mode === 'ai') {
                    $result = airsrefe_generate_and_set_ai_image($post_id, $title, $content);
                    if ($result) {
                        airsrefe_append_log("✅ AI image generated and set successfully for post " . $post_id);
                    } else {
                        airsrefe_append_log("⚠️ Failed to generate/set AI image for post " . $post_id);
                    }
                    return $result;
                } else {
                    airsrefe_append_log("⚠️ AI mode not enabled, cannot generate AI image for post " . $post_id);
                    return false;
                }

            case 'auto':
                $image_set = false;

                // Try RSS image first
                if (!empty($rss_image_url)) {
                    airsrefe_append_log("🔄 Trying RSS image for post " . $post_id);
                    $image_set = airsrefe_set_featured_image($rss_image_url, $post_id, $title);
                    if ($image_set) {
                        airsrefe_append_log("✅ RSS image set successfully for post " . $post_id);
                    }
                }

                // Try source page extraction
                if (!$image_set && !empty($source_url) && function_exists('airsrefe_extract_featured_image')) {
                    airsrefe_append_log("🔄 Trying to extract featured image from source page for post " . $post_id);
                    try {
                        $featured_image_data = airsrefe_extract_featured_image($source_url);
                        if ($featured_image_data && !empty($featured_image_data['url'])) {
                            $image_set = airsrefe_set_featured_image($featured_image_data['url'], $post_id, $title);
                            if ($image_set) {
                                airsrefe_append_log("✅ Featured image extracted and set from source page for post " . $post_id);
                            }
                        }
                    } catch (Exception $e) {
                        airsrefe_append_log("⚠️ Error extracting featured image from source: " . $e->getMessage());
                    }
                }

                // Try AI generation as fallback
                if (!$image_set && $processing_mode === 'ai') {
                    airsrefe_append_log("🔄 Trying AI image generation as fallback for post " . $post_id);
                    $image_set = airsrefe_generate_and_set_ai_image($post_id, $title, $content);
                    if ($image_set) {
                        airsrefe_append_log("✅ AI image generated and set as fallback for post " . $post_id);
                    }
                }

                if (!$image_set) {
                    airsrefe_append_log("⚠️ No image could be set for post " . $post_id);
                }

                return $image_set;

            case 'none':
            default:
                airsrefe_append_log("ℹ️ Image processing disabled for post " . $post_id);
                return false;
        }
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error handling post image for post " . $post_id . ": " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
        return false;
    }
}

function airsrefe_resolve_relative_url($relative_url, $base_url) {
    try {
        if (empty($relative_url)) {
            airsrefe_append_log("⚠️ Empty relative URL for resolution");
            return '';
        }
        
        if (empty($base_url)) {
            airsrefe_append_log("⚠️ Empty base URL for relative URL resolution");
            return $relative_url;
        }
        
        // If already absolute, return as is
        if (strpos($relative_url, 'http') === 0) {
            return $relative_url;
        }

        $parsed_base = wp_parse_url($base_url);
        
        if (!$parsed_base || !isset($parsed_base['scheme']) || !isset($parsed_base['host'])) {
            airsrefe_append_log("⚠️ Invalid base URL for relative URL resolution: " . $base_url);
            return $relative_url;
        }

        // Protocol-relative URL
        if (strpos($relative_url, '//') === 0) {
            $resolved = $parsed_base['scheme'] . ':' . $relative_url;
            airsrefe_append_log("✅ Resolved protocol-relative URL: " . $resolved);
            return $resolved;
        }

        // Absolute path
        if (strpos($relative_url, '/') === 0) {
            $resolved = $parsed_base['scheme'] . '://' . $parsed_base['host'] . $relative_url;
            airsrefe_append_log("✅ Resolved absolute path URL: " . $resolved);
            return $resolved;
        }

        // Relative path
        $base_path = dirname($parsed_base['path'] ?? '/');
        if ($base_path === '.') {
            $base_path = '/';
        }

        $resolved = $parsed_base['scheme'] . '://' . $parsed_base['host'] . $base_path . '/' . $relative_url;
        airsrefe_append_log("✅ Resolved relative path URL: " . $resolved);
        return $resolved;
        
    } catch (Exception $e) {
        airsrefe_append_log("❌ Error resolving relative URL: " . $e->getMessage());
        airsrefe_append_log("Stack trace: " . $e->getTraceAsString());
        return $relative_url;
    }
}

function airsrefe_set_featured_image($image_url, $post_id, $title = '') {
if (empty($image_url)) return false;

if (!function_exists('media_sideload_image')) {
require_once ABSPATH . 'wp-admin/includes/image.php';
require_once ABSPATH . 'wp-admin/includes/file.php';
require_once ABSPATH . 'wp-admin/includes/media.php';
}

airsrefe_append_log("Downloading image: " . $image_url);

try {
$media = media_sideload_image($image_url, $post_id, $title, 'id');

if (is_wp_error($media)) {
airsrefe_append_log("Image loading error: " . $media->get_error_message());
return false;
}

if (is_numeric($media) && set_post_thumbnail($post_id, $media)) {
airsrefe_append_log("Image set as featured (ID: $media)");
return true;
}
} catch (Exception $e) {
airsrefe_append_log("Exception downloading image: " . $e->getMessage());
}

return false;
}

function airsrefe_generate_and_set_ai_image($post_id, $title, $content) {
    airsrefe_append_log("🖼️ Starting AI image generation for post {$post_id}");
    
    if (!function_exists('airsrefe_can_generate_image')) {
        airsrefe_append_log("❌ airsrefe_can_generate_image function not found");
        return false;
    }
    
    if (!airsrefe_can_generate_image()) {
        airsrefe_append_log("❌ Image generation not available or disabled");
        return false;
    }

    if (!function_exists('airsrefe_generate_image')) {
        airsrefe_append_log("❌ airsrefe_generate_image function not found");
        return false;
    }

    airsrefe_append_log("🎨 Generating AI image with title: " . substr($title, 0, 50) . "...");

    // Clean content for image generation (remove HTML tags)
    $clean_content = wp_strip_all_tags($content);
    $clean_content = wp_trim_words($clean_content, 50); // Limit to 50 words
    
    // Create a proper prompt for image generation
    $image_prompt = "Create a professional, modern image for an article titled: " . $title;
    if (!empty($clean_content)) {
        $image_prompt .= ". Article content: " . $clean_content;
    }
    
    airsrefe_append_log("🎨 Image prompt: " . substr($image_prompt, 0, 100) . "...");

    $generated_image_url = airsrefe_generate_image($image_prompt);

    if (is_wp_error($generated_image_url)) {
        airsrefe_append_log("❌ AI image generation failed: " . $generated_image_url->get_error_message());
        return false;
    }

    if (!$generated_image_url) {
        airsrefe_append_log("❌ AI image generation returned empty result");
        return false;
    }

    airsrefe_append_log("✅ AI image generated successfully: " . $generated_image_url);

    if (!function_exists('airsrefe_save_generated_image')) {
        airsrefe_append_log("❌ airsrefe_save_generated_image function not found");
        return false;
    }

    $saved_image_id = airsrefe_save_generated_image($generated_image_url, $post_id, $title);
    if ($saved_image_id) {
        airsrefe_append_log("✅ AI image generated and set successfully (ID: $saved_image_id)");
        return true;
    }

    airsrefe_append_log("❌ Failed to save generated AI image");
    return false;
}

function airsrefe_determine_categories_for_feed($feed, $item, $content) {
$categories = $feed['categories'] ?? [$feed['category'] ?? 0];
$final_categories = [];

foreach ($categories as $cat_id) {
if ($cat_id == 0) {
$auto_cats = airsrefe_get_post_categories($item, $content);
$final_categories = array_merge($final_categories, $auto_cats);
} else {
$final_categories[] = $cat_id;
}
}

return !empty(array_unique($final_categories)) ? array_unique($final_categories) : [1];
}

function airsrefe_get_post_categories($item, $content) {
$categories = [];

$site_categories = wp_cache_get('airsrefe_site_categories', 'airsrefe_categories');
if (false === $site_categories) {
$site_categories = get_categories(['hide_empty' => false]);
wp_cache_set('airsrefe_site_categories', $site_categories, 'airsrefe_categories', 1800);
}

$site_cat_names = [];
foreach ($site_categories as $cat) {
$site_cat_names[strtolower(trim($cat->name))] = $cat->term_id;
}

$rss_categories = $item->get_categories();
if (!empty($rss_categories)) {
foreach ($rss_categories as $cat_obj) {
$cat_name = strtolower(trim($cat_obj->get_label()));
if (isset($site_cat_names[$cat_name])) {
$categories[] = $site_cat_names[$cat_name];
}
}

if (!empty($categories)) {
airsrefe_append_log("Categories from RSS: " . implode(', ', array_map('get_cat_name', $categories)));
return array_unique($categories);
}
}

if (function_exists('airsrefe_classify_with_ai_enhanced')) {
$categories = airsrefe_classify_with_ai_enhanced($content);
}

return !empty($categories) ? array_unique($categories) : [1];
}

function airsrefe_clean_content($content) {
if (empty($content)) return '';

// Preserve important media and embed elements before cleaning
$preserved_elements = airsrefe_extract_media_elements($content);

// Remove problematic links but keep media
$content = preg_replace('/<a[^>]*href=["\']?([^"\'>]+)["\']?[^>]*>((?!<img|<iframe).)*?<\/a>/is', '$2', $content);

// Remove source attribution patterns but preserve the content structure
$source_patterns = [
'/<p[^>]*>\s*(The original article.*?)<\/p>/is',
'/<p[^>]*>\s*(This article.*?originally appeared on.*?)<\/p>/is',
'/<p[^>]*>\s*(Originally published on.*?)<\/p>/is',
'/<p[^>]*>\s*(Read more:.*?)<\/p>/is',
'/<p[^>]*>\s*(Source:.*?)<\/p>/is',
'/\[Continue reading.*?\]/is',
'/Read the full article.*$/is',
];

foreach ($source_patterns as $pattern) {
$content = preg_replace($pattern, '', $content);
}

// Remove empty paragraphs but preserve structure
$content = preg_replace('/<p[^>]*>\s*<\/p>/i', '', $content);

// Restore preserved media elements
$content = airsrefe_restore_media_elements($content, $preserved_elements);

// Clean up whitespace but preserve line breaks for structure
$content = preg_replace('/[ \t]+/', ' ', $content);
$content = preg_replace('/\n{3,}/', "\n\n", $content);

return trim($content);
}
function airsrefe_extract_media_elements($content) {
$preserved = [];
$counter = 0;

// Patterns for media elements to preserve
$media_patterns = [
// Images with various attributes
'/<img[^>]*>/i',
// iframes (YouTube, Twitter, etc.)
'/<iframe[^>]*>.*?<\/iframe>/is',
// Embedded tweets
'/<blockquote[^>]*class[^>]*twitter[^>]*>.*?<\/blockquote>/is',
// Twitter script tags
'/<script[^>]*platform\.twitter\.com.*?<\/script>/is',
// Instagram embeds
'/<blockquote[^>]*class[^>]*instagram[^>]*>.*?<\/blockquote>/is',
// YouTube embeds
'/<div[^>]*class[^>]*youtube[^>]*>.*?<\/div>/is',
// Facebook embeds
'/<div[^>]*class[^>]*fb-[^>]*>.*?<\/div>/is',
// Generic embed containers
'/<div[^>]*class[^>]*embed[^>]*>.*?<\/div>/is',
// Figure elements with media
'/<figure[^>]*>.*?<\/figure>/is',
];

foreach ($media_patterns as $pattern) {
$content = preg_replace_callback($pattern, function($matches) use (&$preserved, &$counter) {
$placeholder = "{{MEDIA_PLACEHOLDER_{$counter}}}";
$preserved[$placeholder] = $matches[0];
$counter++;
return $placeholder;
}, $content);
}

return ['content' => $content, 'elements' => $preserved];
}
function airsrefe_restore_media_elements($content, $preserved_data) {
if (empty($preserved_data['elements'])) {
return $content;
}

foreach ($preserved_data['elements'] as $placeholder => $original_element) {
$content = str_replace($placeholder, $original_element, $content);
}

return $content;
}


// Intelligently split text while preserving meaning
function airsrefe_split_text_intelligently($text) {
// First split by double line breaks (paragraph indicators)
$paragraphs = preg_split('/\n\s*\n/', trim($text), -1, PREG_SPLIT_NO_EMPTY);

if (count($paragraphs) > 1) {
return $paragraphs;
}

// If no paragraph breaks, split by sentences but keep related sentences together
$sentences = preg_split('/(?<=[.!?])\s+/', trim($text), -1, PREG_SPLIT_NO_EMPTY);

if (count($sentences) <= 3) {
return [$text]; // Keep short content as single block
}

// Group sentences into logical paragraphs
$segments = [];
$current_segment = '';
$sentence_count = 0;

foreach ($sentences as $sentence) {
$current_segment .= $sentence . ' ';
$sentence_count++;

// End segment if we have 2-4 sentences or if next sentence seems to start new topic
if ($sentence_count >= 2 && (
$sentence_count >= 4 || 
(isset($sentences[array_search($sentence, $sentences) + 1]) && 
 airsrefe_seems_new_topic($sentence, $sentences[array_search($sentence, $sentences) + 1]))
)) {
$segments[] = trim($current_segment);
$current_segment = '';
$sentence_count = 0;
}
}

// Add any remaining content
if (!empty($current_segment)) {
$segments[] = trim($current_segment);
}

return $segments;
}

// Check if next sentence seems to start a new topic
function airsrefe_seems_new_topic($current_sentence, $next_sentence) {
$topic_starters = [
'However', 'Meanwhile', 'Additionally', 'Furthermore', 'Moreover',
'In contrast', 'On the other hand', 'Similarly', 'Nevertheless',
'Consequently', 'Therefore', 'Thus', 'As a result'
];

foreach ($topic_starters as $starter) {
if (stripos(trim($next_sentence), $starter) === 0) {
return true;
}
}

return false;
}

// Parse HTML structure from original content
function airsrefe_parse_html_structure($html_content) {
$structure = [];

// Use DOMDocument to parse structure safely
$dom = new DOMDocument();
        // Safe libxml error handling
        $old_setting = libxml_use_internal_errors(true);
        libxml_clear_errors();
        
        $load_result = $dom->loadHTML('<?xml encoding="UTF-8">' . $html_content, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD | LIBXML_NOERROR);
        
        // Clear any errors and restore previous setting
        libxml_clear_errors();
        libxml_use_internal_errors($old_setting);
        
        if (!$load_result) return $html_content;

$xpath = new DOMXPath($dom);

// Find all content-bearing elements
$elements = $xpath->query('//p | //h1 | //h2 | //h3 | //h4 | //h5 | //h6 | //li | //blockquote | //div[normalize-space(text())]');

foreach ($elements as $element) {
$text_content = trim($element->textContent);

if (strlen($text_content) > 10) { // Only meaningful content
$structure[] = [
'tag' => $element->nodeName,
'class' => $element->getAttribute('class'),
'length' => strlen($text_content),
'text_snippet' => substr($text_content, 0, 50)
];
}
}

return $structure;
}

// Map new content segments to original structure
function airsrefe_map_content_to_structure($segments, $structure) {
if (empty($structure)) {
// Fallback: create simple paragraph structure
$content = '';
foreach ($segments as $segment) {
$content .= "<p>" . trim($segment) . "</p>\n\n";
}
return $content;
}

$mapped_content = '';
$segment_index = 0;

foreach ($structure as $element) {
if ($segment_index >= count($segments)) {
break;
}

$tag = $element['tag'];
$content_segment = trim($segments[$segment_index]);

// Determine how many segments this element should consume based on length
$segments_needed = max(1, round($element['length'] / 200));
$combined_content = $content_segment;

// Combine segments if needed
for ($i = 1; $i < $segments_needed && ($segment_index + $i) < count($segments); $i++) {
$combined_content .= ' ' . trim($segments[$segment_index + $i]);
}

$segment_index += $segments_needed;

// Apply appropriate HTML tag
switch ($tag) {
case 'h1':
case 'h2':
case 'h3':
case 'h4':
case 'h5':
case 'h6':
// For headings, use first sentence only
$heading_text = explode('.', $combined_content)[0];
$mapped_content .= "<{$tag}>" . trim($heading_text) . "</{$tag}>\n\n";

// Use rest of content for next paragraph if any
$remaining = trim(str_replace($heading_text . '.', '', $combined_content));
if (!empty($remaining)) {
$mapped_content .= "<p>" . $remaining . "</p>\n\n";
}
break;

case 'li':
// Handle list items
if (strpos($mapped_content, '<ul>') === false && strpos($mapped_content, '<ol>') === false) {
$mapped_content .= "<ul>\n";
}
$mapped_content .= "<li>" . $combined_content . "</li>\n";
break;

case 'blockquote':
$mapped_content .= "<blockquote>" . $combined_content . "</blockquote>\n\n";
break;

default:
$mapped_content .= "<p>" . $combined_content . "</p>\n\n";
}
}

// Close any open lists
if (strpos($mapped_content, '<ul>') !== false && strpos($mapped_content, '</ul>') === false) {
$mapped_content .= "</ul>\n\n";
}
if (strpos($mapped_content, '<ol>') !== false && strpos($mapped_content, '</ol>') === false) {
$mapped_content .= "</ol>\n\n";
}

// Add any remaining segments as paragraphs
while ($segment_index < count($segments)) {
$remaining_content = trim($segments[$segment_index]);
if (!empty($remaining_content)) {
$mapped_content .= "<p>" . $remaining_content . "</p>\n\n";
}
$segment_index++;
}

return trim($mapped_content);
}
/**
 * Add post metadata for AI RSS Rewriter processing
 *
 * @param int    $post_id           Post ID to add metadata to
 * @param string $link              Original source URL
 * @param string $processing_mode   Processing mode (ai/simple)
 * @param string $processing_level  AI processing level
 * @param string $priority          Processing priority
 * @param bool   $use_original_date Whether to use original RSS date
 * @param string $feed_url          RSS feed URL
 * @return void
 */
function airsrefe_add_post_metadata($post_id, $link, $processing_mode, $processing_level, $priority, $use_original_date, $feed_url = '') {
$metadata = [
'original_url' => $link,
'_airsrefe_source_url' => $link,
'_airsrefe_feed_url' => $feed_url,
'_airsrefe_processed_date' => current_time('mysql'),
'_airsrefe_processing_mode' => $processing_mode,
'_airsrefe_processing_level' => $processing_level,
'_airsrefe_priority' => $priority,
'_airsrefe_date_mode' => $use_original_date ? 'original' : 'parsing'
];

foreach ($metadata as $key => $value) {
update_post_meta($post_id, $key, $value);
}
}

/**
 * Add SEO metadata for post optimization
 *
 * @param int    $post_id     Post ID to add SEO metadata to
 * @param string $description Meta description for SEO
 * @param string $title       Post title for focus keyword generation
 * @return void
 */
function airsrefe_add_seo_metadata($post_id, $description, $title) {
update_post_meta($post_id, 'rank_math_description', $description);
update_post_meta($post_id, '_yoast_wpseo_metadesc', $description);

$focus_keyword = airsrefe_generate_focus_keyword($title);
if (!empty($focus_keyword)) {
update_post_meta($post_id, 'rank_math_focus_keyword', $focus_keyword);
update_post_meta($post_id, '_yoast_wpseo_focuskw', $focus_keyword);
}
}

/**
 * Calculate smart delay based on priority and cost
 *
 * @param string $priority          Processing priority (speed/economy/quality/balanced)
 * @param float  $last_request_cost Cost of the last API request
 * @return int Delay in seconds
 */
function airsrefe_calculate_smart_delay($priority, $last_request_cost) {
$delays = [
'speed' => 1,
'economy' => 3,
'quality' => 2,
'balanced' => 2
];

$base_delay = $delays[$priority] ?? 2;

if ($last_request_cost > 0.10) {
$base_delay += 2;
} elseif ($last_request_cost > 0.05) {
$base_delay += 1;
}

return $base_delay;
}

if (!function_exists('airsrefe_generate_focus_keyword')) {
/**
 * Generate focus keyword from post title
 *
 * @param string $title Post title
 * @return string Focus keyword (first two words)
 */
function airsrefe_generate_focus_keyword($title) {
$words = explode(' ', $title);
return implode(' ', array_slice($words, 0, 2));
}
}

if (!function_exists('airsrefe_classify_with_ai_enhanced')) {
/**
 * Enhanced AI classification for content categorization
 *
 * @param string $content Content to classify
 * @return array Array of category IDs
 */
function airsrefe_classify_with_ai_enhanced($content) {
return [1];
}
}

if (!function_exists('airsrefe_get_today_costs')) {
/**
 * Get today's AI processing costs
 *
 * @return float Today's costs in dollars
 */
function airsrefe_get_today_costs() {
$costs = get_option('airsrefe_daily_costs', []);
return $costs[current_time('Y-m-d')] ?? 0.00;
}
}

/**
 * OPTIMIZATION: Smart RSS feed caching with adaptive TTL
 */
/**
 * OPTIMIZATION: Enhanced RSS feed caching with intelligent adaptation
 */
function airsrefe_get_feed_with_smart_cache($feed_url) {
    // OPTIMIZATION: Shorter cache keys for better performance
    $cache_key = 'airsrefe_feed_' . substr(md5($feed_url), 0, 16);
    $feed_data = wp_cache_get($cache_key, 'airsrefe_rss_feeds');
    
    if (false !== $feed_data) {
        airsrefe_append_log("RSS feed loaded from cache: " . $feed_url);
        airsrefe_track_cache_hit();
        return $feed_data;
    }
    
    airsrefe_track_cache_miss();
    
    $feed_meta_key = 'airsrefe_feed_meta_' . md5($feed_url);
$feed_meta = wp_cache_get($feed_meta_key, 'airsrefe_rss_feeds');
    
    if (false === $feed_meta) {
        $feed_meta = [
            'last_update' => 0,
            'update_frequency' => 3600,
'consecutive_fails' => 0,
'last_item_count' => 0,
'avg_update_interval' => 3600
        ];
    }
    
    $base_cache_time = get_option('airsrefe_cache_time', 30) * 60;
$adaptive_cache_time = airsrefe_calculate_adaptive_cache_time($feed_meta, $base_cache_time);

add_filter('wp_feed_cache_transient_lifetime', function() use ($adaptive_cache_time) { 
return $adaptive_cache_time; 
});
    
    airsrefe_append_log(sprintf("Fetching RSS feed with adaptive cache time: %d seconds", $adaptive_cache_time));
    
    $rss = fetch_feed($feed_url);

if (is_wp_error($rss)) {
$feed_meta['consecutive_fails']++;
$feed_meta['last_update'] = time();
wp_cache_set($feed_meta_key, $feed_meta, 'airsrefe_rss_feeds', 86400);

return $rss;
}
    
    $items = $rss->get_items();
$current_time = time();
$item_count = count($items);

if ($feed_meta['last_update'] > 0) {
$time_diff = $current_time - $feed_meta['last_update'];
$feed_meta['avg_update_interval'] = ($feed_meta['avg_update_interval'] + $time_diff) / 2;
}
    
    $feed_meta['last_update'] = $current_time;
    $feed_meta['consecutive_fails'] = 0;
    $feed_meta['last_item_count'] = $item_count;
    
    if ($item_count > 10) {
$feed_meta['update_frequency'] = 1800;
} elseif ($item_count > 5) {
$feed_meta['update_frequency'] = 3600;
} else {
$feed_meta['update_frequency'] = 7200;
}
    
    wp_cache_set($feed_meta_key, $feed_meta, 'airsrefe_rss_feeds', 86400);
wp_cache_set($cache_key, $rss, 'airsrefe_rss_feeds', $adaptive_cache_time);
    
    airsrefe_append_log(sprintf("RSS feed cached: %d items, next check in %d seconds", $item_count, $adaptive_cache_time));
    
    return $rss;
}

function airsrefe_calculate_adaptive_cache_time($feed_meta, $base_cache_time) {
$adaptive_time = $base_cache_time;

if ($feed_meta['consecutive_fails'] > 0) {
$adaptive_time *= (1 + $feed_meta['consecutive_fails'] * 0.5);
}
    
    if ($feed_meta['avg_update_interval'] > 0) {
$feed_frequency = $feed_meta['avg_update_interval'];

$max_cache_time = max($feed_frequency / 2, 300);
$adaptive_time = min($adaptive_time, $max_cache_time);
}

return max(300, min($adaptive_time, 14400));
}

function airsrefe_get_cached_ai_result($content_hash, $task_type, $model) {
    $cache_key = "airsrefe_ai_{$task_type}_{$model}_" . substr($content_hash, 0, 16);
    $cached_result = wp_cache_get($cache_key, 'airsrefe_ai_results');
    
    if (false !== $cached_result) {
        airsrefe_append_log("AI result loaded from cache for {$task_type} using {$model}");
        airsrefe_track_cache_hit();
        return $cached_result;
    }
    
    airsrefe_track_cache_miss();
    return false;
}

/**
 * OPTIMIZATION: Caching AI results
 */
function airsrefe_cache_ai_result($content_hash, $task_type, $model, $result, $cost = 0) {
    $cache_key = "airsrefe_ai_{$task_type}_{$model}_" . substr($content_hash, 0, 16);
    
    $cache_time = 3600;

if ($cost > 0.01) {
$cache_time = 86400;
} elseif ($cost > 0.005) {
$cache_time = 43200;
}
    
    wp_cache_set($cache_key, $result, 'airsrefe_ai_results', $cache_time);
    
    airsrefe_append_log(sprintf("AI result cached for %d seconds (cost: $%.4f)", $cache_time, $cost));
}

function airsrefe_process_feed_async($feed) {
$batch_size = 5;

$rss = airsrefe_get_feed_with_smart_cache($feed['url']);
    
    if (is_wp_error($rss)) {
        airsrefe_append_log("RSS loading error for async processing: " . $rss->get_error_message());
        return;
    }
    
    $items = $rss->get_items();
    $limit = $feed['limit'] ?? 5;
    $items = array_slice($items, 0, $limit);
    
    $batches = array_chunk($items, $batch_size);

airsrefe_append_log(sprintf("Async processing: %d items in %d batches", count($items), count($batches)));

foreach ($batches as $batch_index => $batch) {
$delay = $batch_index * 30;
        
        wp_schedule_single_event(
            time() + $delay, 
            'airsrefe_process_batch', 
            [$feed, $batch, $batch_index]
        );
        
        airsrefe_append_log(sprintf("Scheduled batch %d with %d items (delay: %ds)", 
            $batch_index, count($batch), $delay));
    }
}

function airsrefe_handle_batch_processing($feed, $batch_items, $batch_index) {
airsrefe_append_log(sprintf("Processing batch %d with %d items", $batch_index, count($batch_items)));

if (airsrefe_check_daily_parsing_limit()) {
airsrefe_append_log("Daily limit reached - stopping batch processing");
return;
}
    
    $daily_budget = get_option('airsrefe_daily_budget_limit', 10.00);
    if ($daily_budget > 0 && airsrefe_get_today_costs() >= $daily_budget) {
        airsrefe_append_log("Budget limit reached - stopping batch processing");
        return;
    }
    
    $processing_mode = get_option('airsrefe_processing_mode', 'ai');
    $use_original_date = get_option('airsrefe_use_original_date', 1);
    $user_id = $feed['user_id'] ?? get_current_user_id();
    
    $batch_links = [];
$batch_data = [];

foreach ($batch_items as $item) {
$title = $item->get_title();
$content = $item->get_content();
$link = $item->get_link();

if (empty($title) || empty($content) || empty($link)) {
continue;
}

$batch_data[] = [
'item' => $item,
'title' => $title,
'content' => $content,
'link' => $link
];

$batch_links[] = $link;
}

$duplicates_map = airsrefe_check_duplicates_batch($batch_links);
    
    $processed_in_batch = 0;
    $batch_cost = 0.00;
    
    foreach ($batch_data as $data) {
if ($daily_budget > 0 && airsrefe_get_today_costs() + $batch_cost >= $daily_budget) {
airsrefe_append_log("Budget would be exceeded - stopping batch");
break;
}

if (isset($duplicates_map[$data['link']]) && $duplicates_map[$data['link']] > 0) {
airsrefe_append_log("Skipped duplicate: " . $data['title']);
continue;
}

if (post_exists($data['title'])) {
airsrefe_append_log("Skipped duplicate by title: " . $data['title']);
continue;
}

$enhanced_content = airsrefe_enhance_content_with_extraction($data['item'], $data['content']);
        if (!empty($enhanced_content) && $enhanced_content !== $data['content']) {
            $data['content'] = $enhanced_content;
        }
        
        $data['content'] = airsrefe_clean_content($data['content']);
        $featured_image_url = airsrefe_extract_image_from_content($data['content'], $data['item'], $data['link']);
        
        if ($use_original_date) {
$pub_date = $data['item']->get_date('Y-m-d H:i:s');
$pub_date_gmt = gmdate('Y-m-d H:i:s', strtotime($pub_date));
} else {
$pub_date = current_time('mysql');
$pub_date_gmt = current_time('mysql', 1);
}
        
        $start_cost = airsrefe_get_today_costs();
        
        if ($processing_mode === 'ai') {
$ai_result = airsrefe_generate_ai_post_enhanced($data['title'], $data['content']);

if ($ai_result && isset($ai_result['title'], $ai_result['content'])) {
$final_title = $ai_result['title'];
$final_content = $ai_result['content'];
} else {
$final_title = $data['title'];
$final_content = $data['content'];
}
} else {
$final_title = $data['title'];
$final_content = $data['content'];
}
        
        $item_cost = airsrefe_get_today_costs() - $start_cost;
        $batch_cost += $item_cost;
        
        $post_data = [
'post_title' => $final_title,
'post_content' => $final_content,
'post_status' => 'draft',
'post_date' => $pub_date,
'post_date_gmt' => $pub_date_gmt,
'post_author' => $user_id,
'meta_input' => [
'_airsrefe_source_url' => $data['link'],
'_airsrefe_processed_date' => current_time('mysql'),
'_airsrefe_batch_index' => $batch_index,
'_airsrefe_processing_cost' => $item_cost
]
];
        
        $post_id = wp_insert_post($post_data);
        
        if (!is_wp_error($post_id) && $post_id > 0) {
airsrefe_add_post_metadata($post_id, $data['link'], $processing_mode, 'batch', 'normal', $use_original_date, $feed['url']);

if (!empty($featured_image_url)) {
airsrefe_set_featured_image_from_url($post_id, $featured_image_url);
}

if (!empty($feed['categories'])) {
wp_set_post_categories($post_id, $feed['categories']);
}

if ($processing_mode === 'ai' && get_option('airsrefe_generate_tags', 1)) {
$tags = airsrefe_generate_tags($final_title, $final_content);
if (!empty($tags)) {
wp_set_post_tags($post_id, $tags);
}
}
            
            $processed_in_batch++;
            airsrefe_append_log(sprintf("Post created in batch %d: %s (cost: $%.4f)", 
                $batch_index, $final_title, $item_cost));
        } else {
            airsrefe_append_log("Error creating post in batch: " . $data['title']);
        }
    }
    
    airsrefe_append_log(sprintf("Batch %d completed: %d posts processed, total cost: $%.4f", 
        $batch_index, $processed_in_batch, $batch_cost));
}

function airsrefe_init_background_queue() {
add_action('airsrefe_process_queue_batch', 'airsrefe_process_queue_batch');
add_action('airsrefe_queue_maintenance', 'airsrefe_queue_maintenance');

if (!wp_next_scheduled('airsrefe_queue_maintenance')) {
wp_schedule_event(time(), 'hourly', 'airsrefe_queue_maintenance');
}
}

function airsrefe_add_to_queue($item_data, $priority = 'normal') {
$queue_key = 'airsrefe_background_queue';
$queue = get_option($queue_key, []);

$queue_item = [
'id' => uniqid('airsrefe_', true),
'data' => $item_data,
'priority' => $priority,
'attempts' => 0,
'max_attempts' => 3,
'created' => time(),
'status' => 'pending'
];

if ($priority === 'high') {
array_unshift($queue, $queue_item);
} else {
$queue[] = $queue_item;
}

if (count($queue) > 500) {
$queue = array_slice($queue, 0, 500);
airsrefe_append_log("Queue size limited to 500 items");
}
    
    update_option($queue_key, $queue);
    
    if (count($queue) === 1) {
wp_schedule_single_event(time() + 30, 'airsrefe_process_queue_batch');
}
    
    airsrefe_append_log("Item added to queue (priority: {$priority}, queue size: " . count($queue) . ")");
}

/**
 * OPTIMIZATION: Process queue batch with memory and time limits
 */
function airsrefe_process_queue_batch() {
    $memory_start = memory_get_usage();
    $time_start = time();
    $time_limit = 60; // 1 minute processing limit
    $memory_limit = 100 * 1024 * 1024; // 100MB limit
    
    $queue_key = 'airsrefe_background_queue';
    $queue = get_option($queue_key, []);
    
    if (empty($queue)) {
        return;
    }
    
    $processed = 0;
    $successful = 0;
    $failed = 0;
    
    airsrefe_append_log("Starting queue processing: " . count($queue) . " items");
    
    foreach ($queue as $index => $item) {
        // Check time and memory limits
        if ((time() - $time_start) > $time_limit) {
            airsrefe_append_log("Time limit reached, stopping batch processing");
            break;
        }
        
        $memory_current = memory_get_usage();
        if (($memory_current - $memory_start) > $memory_limit) {
            airsrefe_append_log("Memory limit reached, stopping batch processing");
            break;
        }
        
        if ($item['status'] !== 'pending') {
            continue;
        }
        
        // Process item with retry logic
        $result = airsrefe_process_queue_item($item);
        
        if ($result['success']) {
            unset($queue[$index]);
            $successful++;
        } else {
            $queue[$index]['attempts']++;
            
            if ($queue[$index]['attempts'] >= $item['max_attempts']) {
                $queue[$index]['status'] = 'failed';
                $queue[$index]['error'] = $result['error'] ?? 'Max attempts reached';
                $failed++;
            } else {
                // Exponential backoff for retry
                $delay = pow(2, $queue[$index]['attempts']) * 60; // 2, 4, 8 minutes
                $queue[$index]['next_retry'] = time() + $delay;
                $queue[$index]['status'] = 'retry';
            }
        }
        
        $processed++;
        
        // Process max 10 items per batch to avoid timeouts
        if ($processed >= 10) {
            break;
        }
    }
    
    // Re-index array and update queue
    $queue = array_values($queue);
    update_option($queue_key, $queue);
    
    $memory_used = memory_get_usage() - $memory_start;
    $time_used = time() - $time_start;
    
    airsrefe_append_log(sprintf("Queue batch complete: %d processed, %d successful, %d failed (Time: %ds, Memory: %s)", 
        $processed, $successful, $failed, $time_used, size_format($memory_used)));
    
    // Schedule next batch if queue not empty
    if (!empty($queue)) {
        wp_schedule_single_event(time() + 120, 'airsrefe_process_queue_batch'); // 2 minutes delay
    }
}

/**
 * OPTIMIZATION: Process individual queue item
 */
function airsrefe_process_queue_item($item) {
    try {
        $data = $item['data'];
        
        // Check if this is RSS item processing
        if (isset($data['type']) && $data['type'] === 'rss_item') {
            $result = airsrefe_process_single_rss_item($data);
            return ['success' => $result !== false];
        }
        
        // Check if this is feed processing
        if (isset($data['type']) && $data['type'] === 'feed_process') {
            $result = airsrefe_process_feed($data['feed']);
            return ['success' => true];
        }
        
        return ['success' => false, 'error' => 'Unknown item type'];
        
    } catch (Exception $e) {
        airsrefe_append_log("Queue item processing error: " . $e->getMessage());
        return ['success' => false, 'error' => $e->getMessage()];
    }
}

/**
 * Enhanced URL validation for RSS feeds
 */
function airsrefe_validate_feed_url($url) {
    if (empty($url)) {
        return false;
    }
    
    // Basic URL validation
    if (!filter_var($url, FILTER_VALIDATE_URL)) {
        return false;
    }
    
    $parsed = wp_parse_url($url);
    if (!$parsed || !isset($parsed['scheme']) || !isset($parsed['host'])) {
        return false;
    }
    
    // Only allow HTTP and HTTPS
    if (!in_array($parsed['scheme'], ['http', 'https'])) {
        return false;
    }
    
    // Block dangerous protocols
    $dangerous_protocols = ['javascript:', 'data:', 'file:', 'ftp:', 'gopher:', 'ldap:'];
    foreach ($dangerous_protocols as $protocol) {
        if (stripos($url, $protocol) === 0) {
            return false;
        }
    }
    
    // Block localhost and private IPs (unless explicitly allowed)
    $allowed_local = apply_filters('airsrefe_allow_local_feeds', false);
    if (!$allowed_local) {
        $host = $parsed['host'];
        if (in_array($host, ['localhost', '127.0.0.1', '::1']) || 
            preg_match('/^192\.168\./', $host) ||
            preg_match('/^10\./', $host) ||
            preg_match('/^172\.(1[6-9]|2[0-9]|3[0-1])\./', $host)) {
            return false;
        }
    }
    
    // Check for suspicious patterns
    $suspicious_patterns = [
        '/\.\./',  // Directory traversal
        '/\0/',    // Null bytes
        '/\x00/',  // Null bytes
    ];
    
    foreach ($suspicious_patterns as $pattern) {
        if (preg_match($pattern, $url)) {
            return false;
        }
    }
    
    return true;
}