findByUuid( $job_uuid ); if ( null === $row ) { ChatbotLogger::warn( 'job_worker_missing_row', [ 'job_uuid' => $job_uuid ] ); return; } if ( 'pending' !== (string) ( $row['status'] ?? '' ) ) { ChatbotLogger::info( 'job_worker_skipped_status', [ 'job_uuid' => $job_uuid, 'status' => (string) ( $row['status'] ?? '' ), ] ); return; } $claim_token = $repository->claimAtomic( $job_uuid, ChatbotConstants::claimTtlSeconds() ); if ( null === $claim_token ) { ChatbotLogger::info( 'job_worker_claim_skipped', [ 'job_uuid' => $job_uuid ] ); return; } $context = self::getIdempotencyContext(); try { $payload = self::decodePayload( $row ); $provider = self::resolveProvider( $payload ); $cooldown = self::getCooldown(); if ( $cooldown->isProviderCoolingDown( $provider ) && ! $cooldown->attemptRecoveryProbe( $provider ) ) { $backoff_seconds = max( 1, $cooldown->getProviderCooldownRetryAfter( $provider ) ); $repository->markRequiresUserRetry( $job_uuid, $claim_token, 'provider_cooldown', $backoff_seconds ); ChatbotLogger::warn( 'job_worker_provider_cooldown', [ 'job_uuid' => $job_uuid, 'provider' => $provider, 'retry_after' => $backoff_seconds, 'session_id' => self::getSessionId( $row, $payload ), 'request_id' => self::getClientRequestId( $row, $payload ), ] ); return; } self::setWorkerTimeLimit(); $client_request_id = self::getClientRequestId( $row, $payload ); $idempotency_key = '' === $client_request_id ? $job_uuid : hash( 'sha256', $job_uuid . '|' . $client_request_id ); IdempotencyKeyContext::setIdempotencyKey( $idempotency_key ); $service = self::getService( $context ); $response = $service->processMessage( self::getPayloadString( $payload, 'message' ), self::getSessionId( $row, $payload ), self::getPayloadArray( $payload, 'history' ), self::getPayloadString( $payload, 'user_name' ), self::getPayloadString( $payload, 'user_email' ), self::getPayloadBool( $payload, 'consent_given' ), self::getPayloadNullableInt( $payload, 'consent_timestamp' ) ); if ( is_array( $response ) && true === ( $response['success'] ?? false ) && empty( $response['data']['response'] ) && ( ! empty( $response['data']['queued'] ) || ! empty( $response['data']['message'] ) ) ) { $queued_message = isset( $response['data']['message'] ) && is_string( $response['data']['message'] ) ? self::sanitizeErrorDetail( $response['data']['message'] ) : null; $queued_error_code = isset( $response['data']['error_code'] ) && is_string( $response['data']['error_code'] ) && '' !== $response['data']['error_code'] ? \sanitize_key( $response['data']['error_code'] ) : 'worker_response_queued_no_reply'; $repository->markFailed( $job_uuid, $claim_token, $queued_error_code, $queued_message ); if ( self::shouldActivateCooldownForErrorCode( $queued_error_code ) ) { $cooldown_instance = self::getCooldown(); $cooldown_instance->activateForProvider( $provider, $cooldown_instance->getCooldownSeconds() ); } ChatbotLogger::warn( 'job_worker_queued_no_reply', [ 'job_uuid' => $job_uuid, 'provider' => $provider, 'error_code' => $queued_error_code, 'session_id' => self::getSessionId( $row, $payload ), 'request_id' => self::getClientRequestId( $row, $payload ), ] ); return; } if ( is_array( $response ) && isset( $response['success'] ) && false === $response['success'] ) { $error_code = isset( $response['error'] ) && is_string( $response['error'] ) && '' !== $response['error'] ? \sanitize_key( $response['error'] ) : 'worker_service_error'; $error_detail = isset( $response['message'] ) && is_string( $response['message'] ) ? self::sanitizeErrorDetail( $response['message'] ) : null; $repository->markFailed( $job_uuid, $claim_token, $error_code, $error_detail ); if ( self::shouldActivateCooldownForErrorCode( $error_code ) ) { $cooldown_instance = self::getCooldown(); $cooldown_instance->activateForProvider( $provider, $cooldown_instance->getCooldownSeconds() ); } ChatbotLogger::error( 'job_worker_service_failed', [ 'job_uuid' => $job_uuid, 'provider' => $provider, 'error_code' => $error_code, ] ); return; } $repository->markCompleted( $job_uuid, $claim_token, $response ); ChatbotLogger::info( 'job_worker_completed', [ 'job_uuid' => $job_uuid, 'provider' => $provider, 'session_id' => self::getSessionId( $row, $payload ), 'request_id' => self::getClientRequestId( $row, $payload ), ] ); } catch ( \Throwable $throwable ) { $safe_error_detail = self::sanitizeErrorDetail( $throwable->getMessage() ); $repository->markFailed( $job_uuid, $claim_token, 'worker_unhandled', $safe_error_detail ); ChatbotLogger::error( 'job_worker_unhandled', [ 'job_uuid' => $job_uuid, 'error_detail' => $safe_error_detail, ] ); } finally { IdempotencyKeyContext::clear(); } } public static function reclaimStaleClaims(): void { $reclaimed = self::getRepository()->reclaimStaleClaims( ChatbotConstants::STALE_RECOVERY_GRACE_SECONDS ); if ( empty( $reclaimed ) ) { return; } foreach ( $reclaimed as $job_uuid ) { AsyncRunner::scheduleJob( (string) $job_uuid ); } } public static function scheduleStaleRecovery(): void { if ( ! \function_exists( 'as_next_scheduled_action' ) || ! \function_exists( 'as_schedule_recurring_action' ) ) { return; } try { $group = 'aipt-chatbot'; if ( \as_next_scheduled_action( ChatbotConstants::STALE_RECOVERY_HOOK, [], $group ) ) { return; } \as_schedule_recurring_action( \time() + ChatbotConstants::STALE_RECOVERY_INTERVAL_SECONDS, ChatbotConstants::STALE_RECOVERY_INTERVAL_SECONDS, ChatbotConstants::STALE_RECOVERY_HOOK, [], $group ); } catch ( \Throwable $throwable ) { ChatbotLogger::warn( 'job_worker_stale_recovery_schedule_failed', [ 'error' => $throwable->getMessage() ] ); } } /** * Self-heal entry point for admin_init. Activation can run before Action * Scheduler exists and auto-updates skip activation, so the recurring * stale-claim recovery is re-armed here when it has gone missing. The * check is throttled to once an hour and only counts once Action * Scheduler is actually available. */ public static function ensureStaleRecoveryScheduled(): void { if ( ! \function_exists( 'as_next_scheduled_action' ) || ! \function_exists( 'as_schedule_recurring_action' ) ) { return; } if ( \get_transient( self::STALE_RECOVERY_CHECK_TRANSIENT ) ) { return; } \set_transient( self::STALE_RECOVERY_CHECK_TRANSIENT, 1, \HOUR_IN_SECONDS ); self::scheduleStaleRecovery(); } public static function unscheduleStaleRecovery(): void { if ( ! \function_exists( 'as_unschedule_all_actions' ) ) { return; } try { \as_unschedule_all_actions( ChatbotConstants::STALE_RECOVERY_HOOK, [], 'aipt-chatbot' ); } catch ( \Throwable $throwable ) { ChatbotLogger::warn( 'job_worker_stale_recovery_unschedule_failed', [ 'error' => $throwable->getMessage() ] ); } } /** * Cancel every pending single-action enqueue for the chatbot job * worker. Each enqueued chat job creates an * `as_schedule_single_action( ACTION_SCHEDULER_HOOK, [job_uuid], 'aipt-chatbot' )` * row that AS consumes on run. If the plugin is deactivated with * jobs still queued, those rows become orphaned schedules in the AS * UI and execute on reactivation against a possibly-different * schema. Deactivator calls this to wipe the queue cleanly. */ public static function unscheduleProcessJobActions(): void { if ( ! \function_exists( 'as_unschedule_all_actions' ) ) { return; } try { \as_unschedule_all_actions( ChatbotConstants::ACTION_SCHEDULER_HOOK, [], 'aipt-chatbot' ); } catch ( \Throwable $throwable ) { ChatbotLogger::warn( 'job_worker_process_job_unschedule_failed', [ 'error' => $throwable->getMessage() ] ); } } private static function getRepository(): ChatbotJobRepository { if ( self::$testRepository instanceof ChatbotJobRepository ) { return self::$testRepository; } $repository = \apply_filters( 'aipt_chatbot_job_worker_repository', null ); return $repository instanceof ChatbotJobRepository ? $repository : new ChatbotJobRepository(); } private static function getService( IdempotencyKeyContext $context ) { if ( is_object( self::$testService ) ) { return self::$testService; } $service = \apply_filters( 'aipt_chatbot_job_worker_service', null ); return is_object( $service ) ? $service : new ChatbotService( null, null, new RestAPIClientFactory( $context ) ); } private static function getCooldown(): ProviderCooldown { if ( self::$testCooldown instanceof ProviderCooldown ) { return self::$testCooldown; } $cooldown = \apply_filters( 'aipt_chatbot_job_worker_cooldown', null ); return $cooldown instanceof ProviderCooldown ? $cooldown : new ProviderCooldown(); } private static function getIdempotencyContext(): IdempotencyKeyContext { if ( self::$testIdempotencyContext instanceof IdempotencyKeyContext ) { return self::$testIdempotencyContext; } $context = \apply_filters( 'aipt_chatbot_job_worker_idempotency_context', null ); return $context instanceof IdempotencyKeyContext ? $context : new IdempotencyKeyContext(); } private static function decodePayload( array $row ): array { $payload = isset( $row['payload'] ) && is_string( $row['payload'] ) ? json_decode( $row['payload'], true ) : []; return is_array( $payload ) ? $payload : []; } private static function resolveProvider( array $payload ): string { $provider = self::getPayloadString( $payload, 'provider' ); if ( '' === $provider ) { $settings = \apply_filters( 'aipt_chatbot_job_worker_settings', null ); $settings = $settings instanceof ChatbotSettings ? $settings : new ChatbotSettings(); $provider = (string) $settings->get( 'behavior.provider', 'openai' ); } $provider = \sanitize_key( $provider ); return '' !== $provider ? $provider : 'openai'; } private static function getSessionId( array $row, array $payload ): string { $session_id = self::getPayloadString( $payload, 'session_id' ); return '' !== $session_id ? $session_id : (string) ( $row['session_id'] ?? '' ); } private static function getClientRequestId( array $row, array $payload ): string { $request_id = self::getPayloadString( $payload, 'client_request_id' ); return '' !== $request_id ? $request_id : (string) ( $row['client_request_id'] ?? '' ); } private static function getPayloadString( array $payload, string $key ): string { return isset( $payload[ $key ] ) && is_scalar( $payload[ $key ] ) ? (string) $payload[ $key ] : ''; } private static function getPayloadArray( array $payload, string $key ): array { return isset( $payload[ $key ] ) && is_array( $payload[ $key ] ) ? $payload[ $key ] : []; } private static function getPayloadBool( array $payload, string $key ): bool { return isset( $payload[ $key ] ) ? (bool) $payload[ $key ] : false; } private static function getPayloadNullableInt( array $payload, string $key ): ?int { return isset( $payload[ $key ] ) && is_numeric( $payload[ $key ] ) ? (int) $payload[ $key ] : null; } private static function sanitizeErrorDetail( string $detail ): string { return ErrorHandler::sanitize_error_for_logging( $detail ); } /** * Decide whether a worker error_code warrants activating the provider * cooldown. Only failures that recover by waiting qualify — rate limits, * transient outages, provider-side back-pressure. Auth and queue-internal * errors are intentionally excluded; cooldown-gating them would lock the * chatbot for everyone while the operator still hasn't fixed the root * cause (e.g. a wrong API key). * * Public since 2.8.0: ChatbotController's sync /chat route shares this * exact allowlist for its own (previously dead) cooldown activation, so * the sync and async paths agree on which failures recover by waiting. */ public static function shouldActivateCooldownForErrorCode( string $error_code ): bool { if ( '' === $error_code ) { return false; } $exact = [ 'chatbot_unavailable', 'provider_unavailable', 'provider_busy', 'provider_overloaded', 'service_unavailable', 'rate_limited', 'network_timeout', 'network_error', ]; if ( in_array( $error_code, $exact, true ) ) { return true; } return false !== strpos( $error_code, 'rate_limit' ) || false !== strpos( $error_code, 'unavailable' ); } private static function setWorkerTimeLimit(): void { if ( \function_exists( 'set_time_limit' ) ) { set_time_limit( ChatbotConstants::workerTimeoutSeconds() ); } } }