<?php
/**
 * Registry of known AI crawler / assistant user-agent signatures.
 *
 * Each entry maps a unique slug to a [label, needle, group] signature. The needle is
 * matched case-insensitively as a substring of the request's User-Agent
 * header. Site owners can add further custom signatures from the Settings
 * screen without editing this file.
 */

if ( ! defined( 'ABSPATH' ) ) {
	exit;
}

class AI_O_Optimiser_Bots {

	/**
	 * Built-in signatures. Slug => [ 'label' => ..., 'needle' => ..., 'group' => ... ].
	 *
	 * @return array
	 */
	public static function defaults() {
		return array(
			'claudebot'        => array(
				'label'  => 'Anthropic – ClaudeBot (training crawler)',
				'needle' => 'ClaudeBot',
				'group'  => 'training',
			),
			'claude-searchbot' => array(
				'label'  => 'Anthropic – Claude-SearchBot (search/index crawler)',
				'needle' => 'Claude-SearchBot',
				'group'  => 'training',
			),
			'claude-user'      => array(
				'label'  => 'Anthropic – Claude-User (live fetch for a chat)',
				'needle' => 'Claude-User',
				'group'  => 'user',
			),
			'anthropic-ai'     => array(
				'label'  => 'Anthropic – other (anthropic-ai)',
				'needle' => 'anthropic-ai',
				'group'  => 'training',
			),
			'gptbot'           => array(
				'label'  => 'OpenAI – GPTBot (training crawler)',
				'needle' => 'GPTBot',
				'group'  => 'training',
			),
			'chatgpt-user'     => array(
				'label'  => 'OpenAI – ChatGPT-User (live fetch for a chat)',
				'needle' => 'ChatGPT-User',
				'group'  => 'user',
			),
			'oai-searchbot'    => array(
				'label'  => 'OpenAI – OAI-SearchBot (ChatGPT search crawler)',
				'needle' => 'OAI-SearchBot',
				'group'  => 'training',
			),
			'oai-adsbot'       => array(
				'label'  => 'OpenAI – OAI-AdsBot (ChatGPT ads validation)',
				'needle' => 'OAI-AdsBot',
				'group'  => 'other',
			),
			'perplexitybot'    => array(
				'label'  => 'Perplexity – PerplexityBot (search/index crawler)',
				'needle' => 'PerplexityBot',
				'group'  => 'training',
			),
			'perplexity-user'  => array(
				'label'  => 'Perplexity – Perplexity-User (live fetch)',
				'needle' => 'Perplexity-User',
				'group'  => 'user',
			),
			'ccbot'            => array(
				'label'    => 'Common Crawl – CCBot (used to train many LLMs)',
				'needle'   => 'CCBot',
				'group'    => 'training',
				// Five letters, and always sent as its own token ("CCBot/2.0").
				'boundary' => true,
			),
			'bytespider'       => array(
				'label'  => 'ByteDance – Bytespider',
				'needle' => 'Bytespider',
				'group'  => 'training',
			),
			'amazonbot'        => array(
				'label'  => 'Amazon – Amazonbot',
				'needle' => 'Amazonbot',
				'group'  => 'training',
			),
			'applebot'         => array(
				'label'  => 'Apple – Applebot (Siri, Spotlight and Apple Intelligence search)',
				'needle' => 'Applebot',
				'group'  => 'training',
			),
			'meta-extagent'    => array(
				'label'  => 'Meta – Meta-ExternalAgent (automated AI crawler)',
				'needle' => 'Meta-ExternalAgent',
				'group'  => 'training',
			),
			'meta-extfetcher'  => array(
				'label'  => 'Meta – Meta-ExternalFetcher (live fetch)',
				'needle' => 'Meta-ExternalFetcher',
				'group'  => 'user',
			),
			'cohere-ai'        => array(
				'label'  => 'Cohere – cohere-ai',
				'needle' => 'cohere-ai',
				'group'  => 'training',
			),
			'duckassistbot'    => array(
				'label'  => 'DuckDuckGo – DuckAssistBot',
				'needle' => 'DuckAssistBot',
				'group'  => 'training',
			),
			'youbot'           => array(
				'label'  => 'You.com – YouBot',
				'needle' => 'YouBot',
				'group'  => 'training',
			),
			'diffbot'          => array(
				'label'  => 'Diffbot',
				'needle' => 'Diffbot',
				'group'  => 'training',
			),
			'google-agent'     => array(
				'label'  => 'Google – Google-Agent (AI agent live browsing, e.g. Project Mariner)',
				'needle' => 'Google-Agent',
				'group'  => 'user',
			),
			'google-gemininotebook' => array(
				'label'  => 'Google – Google-GeminiNotebook (user-requested Gemini Notebook source fetch)',
				'needle' => 'Google-GeminiNotebook',
				'group'  => 'user',
			),
			'google-notebooklm' => array(
				'label'  => 'Google – Google-NotebookLM (legacy Gemini Notebook source fetch)',
				'needle' => 'Google-NotebookLM',
				'group'  => 'user',
			),
			'google-cloudvertexbot' => array(
				'label'  => 'Google – Google-CloudVertexBot (Vertex AI Agent crawl requested by a site owner)',
				'needle' => 'Google-CloudVertexBot',
				'group'  => 'other',
			),
			'mistralai-user'   => array(
				'label'  => 'Mistral AI – MistralAI-User (live fetch for Le Chat)',
				'needle' => 'MistralAI-User',
				'group'  => 'user',
			),
			'deepseekbot'      => array(
				'label'  => 'DeepSeek – DeepSeekBot (training crawler)',
				'needle' => 'DeepSeekBot',
				'group'  => 'training',
			),
			'ai2bot'           => array(
				'label'  => 'Allen Institute for AI – ai2bot (Dolma training corpus)',
				'needle' => 'ai2bot',
				'group'  => 'training',
			),
			'xai-grok'         => array(
				'label'    => 'xAI – Grok crawler (unofficial: xAI publishes no vendor documentation for this bot, and its user-agent string is not confirmed stable)',
				'needle'   => 'Grok',
				'group'    => 'training',
				// Four letters is short enough to appear inside an unrelated
				// agent -- Grokipedia, a product name, a library version. The
				// needle has to stand as its own word.
				'boundary' => true,
			),
			// Google's general-purpose crawler, used by internal teams including
			// AI research. Grouped as other rather than training: Google does not
			// say what any one fetch was for, and claiming it as AI discovery
			// would assert more than the user agent supports. The two media
			// variants are listed first, since the plain needle is a prefix of
			// both and would otherwise absorb them.
			'googleother-image' => array(
				'label'  => 'Google – GoogleOther-Image (general-purpose image fetch)',
				'needle' => 'GoogleOther-Image',
				'group'  => 'other',
			),
			'googleother-video' => array(
				'label'  => 'Google – GoogleOther-Video (general-purpose video fetch)',
				'needle' => 'GoogleOther-Video',
				'group'  => 'other',
			),
			'googleother'      => array(
				'label'  => 'Google – GoogleOther (general-purpose crawl, including AI research)',
				'needle' => 'GoogleOther',
				'group'  => 'other',
			),
			'petalbot'         => array(
				'label'  => 'Huawei – PetalBot',
				'needle' => 'PetalBot',
				'group'  => 'training',
			),
			'pangubot'         => array(
				'label'  => 'Huawei – PanguBot (PanGu model crawler)',
				'needle' => 'PanguBot',
				'group'  => 'training',
			),
			'tiktokspider'     => array(
				'label'  => 'ByteDance – TikTokSpider',
				'needle' => 'TikTokSpider',
				'group'  => 'training',
			),
			'timpibot'         => array(
				'label'  => 'Timpi – Timpibot',
				'needle' => 'Timpibot',
				'group'  => 'training',
			),
			'imagesiftbot'     => array(
				'label'  => 'Hive AI – ImagesiftBot',
				'needle' => 'ImagesiftBot',
				'group'  => 'training',
			),
			'omgili'           => array(
				'label'  => 'Webz.io – omgili (web data resold for model training)',
				'needle' => 'omgili',
				'group'  => 'training',
			),
			'webzio'           => array(
				'label'  => 'Webz.io – Webzio-Extended',
				'needle' => 'Webzio',
				'group'  => 'training',
			),
			'icc-crawler'      => array(
				'label'  => 'NICT – ICC-Crawler (Japanese research corpus)',
				'needle' => 'ICC-Crawler',
				'group'  => 'training',
			),
			'kangaroo-bot'     => array(
				'label'  => 'Kangaroo Bot',
				'needle' => 'Kangaroo Bot',
				'group'  => 'training',
			),
		);
	}

	/**
	 * Robots.txt policy tokens that do not appear as standalone HTTP User-Agent strings.
	 *
	 * These are useful for System Check guidance, but they are deliberately excluded
	 * from runtime User-Agent matching.
	 *
	 * @return array
	 */
	public static function robots_policy_tokens() {
		return array(
			'googlebot-search' => array(
				'label'  => 'Google – Googlebot (Search + AI Search crawl eligibility)',
				'needle' => 'Googlebot',
				'group'  => 'policy',
			),
			'google-extended' => array(
				'label'  => 'Google – Google-Extended (Gemini / Vertex policy token)',
				'needle' => 'Google-Extended',
				'group'  => 'policy',
			),
			// Apple documents Applebot-Extended as a robots.txt control for
			// generative-AI training, not as a crawler: no request ever arrives
			// with it as a User-Agent. It was matched at runtime here, where it
			// could never fire, while Applebot itself -- the agent that actually
			// fetches for Apple Intelligence -- went unmatched. Both are now on
			// the side that fits them.
			'applebot-extended' => array(
				'label'  => 'Apple – Applebot-Extended (Apple Intelligence training policy token)',
				'needle' => 'Applebot-Extended',
				'group'  => 'policy',
			),
			// Bingbot grounds Copilot answers, but it is also ordinary Bing
			// Search. Counting it as AI activity would inflate every total on
			// the dashboard with search traffic, exactly as Googlebot would.
			'bingbot'         => array(
				'label'  => 'Microsoft – Bingbot (Bing Search + Copilot grounding eligibility)',
				'needle' => 'Bingbot',
				'group'  => 'policy',
			),
		);
	}

	/**
	 * Merge built-in signatures with any custom ones saved in settings.
	 *
	 * @return array
	 */
	public static function all() {
		$settings = get_option( 'ai_o_optimiser_settings', array() );
		$custom   = isset( $settings['custom_bots'] ) ? $settings['custom_bots'] : array();

		return array_merge( self::defaults(), $custom );
	}

	/**
	 * Human-readable bot group labels.
	 *
	 * Discovery/index crawlers are useful for understanding model discovery, search
	 * index activity and training-related access where providers use dedicated crawlers. User-triggered fetchers usually indicate a person
	 * asked an AI assistant to inspect or cite a page.
	 *
	 * @return array
	 */
	public static function group_definitions() {
		return array(
			'training' => __( 'Discovery / index crawlers', 'ai-o-optimiser' ),
			'user'     => __( 'User-triggered fetchers', 'ai-o-optimiser' ),
			'other'    => __( 'Other / validation / custom bots', 'ai-o-optimiser' ),
		);
	}

	/**
	 * Resolve a bot group's display label.
	 *
	 * @param string $group Group key.
	 * @return string
	 */
	public static function group_label( $group ) {
		$groups = self::group_definitions();
		return isset( $groups[ $group ] ) ? $groups[ $group ] : $groups['other'];
	}

	/**
	 * Resolve the group for a bot slug.
	 *
	 * @param string $slug Bot slug.
	 * @return string
	 */
	public static function group_for_slug( $slug ) {
		$bots = self::all();
		if ( isset( $bots[ $slug ]['group'] ) ) {
			$group  = $bots[ $slug ]['group'];
			$groups = self::group_definitions();
			return isset( $groups[ $group ] ) ? $group : 'other';
		}

		return 'other';
	}

	/**
	 * Return all bot slugs that belong to a group.
	 *
	 * @param string $group Group key.
	 * @return array
	 */
	public static function slugs_for_group( $group ) {
		$slugs  = array();
		$groups = self::group_definitions();

		if ( ! isset( $groups[ $group ] ) ) {
			return $slugs;
		}

		foreach ( self::all() as $slug => $bot ) {
			$bot_group = isset( $bot['group'] ) ? $bot['group'] : 'other';
			if ( $group === $bot_group ) {
				$slugs[] = $slug;
			}
		}

		return $slugs;
	}

	/**
	 * Identify which (if any) known bot matches a given User-Agent string.
	 *
	 * @param string $user_agent Raw User-Agent header value.
	 * @return array|null [ 'slug' => ..., 'label' => ..., 'group' => ... ] or null if no match.
	 */
	public static function match( $user_agent ) {
		if ( empty( $user_agent ) ) {
			return null;
		}

		$settings     = wp_parse_args( get_option( 'ai_o_optimiser_settings', array() ), AI_O_Optimiser_Settings::defaults() );
		$tracked_bots = isset( $settings['tracked_bots'] ) && is_array( $settings['tracked_bots'] ) ? $settings['tracked_bots'] : array();

		foreach ( self::all() as $slug => $bot ) {
			// Skip bots the site owner has explicitly disabled. Missing keys fall back to enabled for compatibility.
			if ( array_key_exists( $slug, $tracked_bots ) && empty( $tracked_bots[ $slug ] ) ) {
				continue;
			}

			if ( self::needle_matches( $user_agent, $bot ) ) {
				return array(
					'slug'  => $slug,
					'label' => $bot['label'],
					'group' => isset( $bot['group'] ) ? $bot['group'] : 'other',
				);
			}
		}

		return null;
	}

	/**
	 * Whether one signature matches a User-Agent.
	 *
	 * Substring matching is right for the long, distinctive tokens most vendors
	 * publish. It is wrong for a short one: a signature carrying 'boundary' has
	 * to stand as its own word, so that a four-letter needle cannot match inside
	 * an unrelated product name.
	 *
	 * @param string $user_agent Raw User-Agent header value.
	 * @param array  $bot        Signature definition.
	 * @return bool
	 */
	private static function needle_matches( $user_agent, $bot ) {
		$needle = isset( $bot['needle'] ) ? (string) $bot['needle'] : '';
		if ( '' === $needle ) {
			return false;
		}

		if ( empty( $bot['boundary'] ) ) {
			return false !== stripos( $user_agent, $needle );
		}

		return 1 === preg_match( '/(?<![A-Za-z0-9])' . preg_quote( $needle, '/' ) . '(?![A-Za-z0-9])/i', $user_agent );
	}

	const NEAR_MISS_OPTION  = 'ai_o_optimiser_near_miss_uas';
	const NEAR_MISS_MAX     = 40;
	const NEAR_MISS_TTL_DAYS = 30;

	/**
	 * AI-brand keywords used to flag a request worth a second look, once it has
	 * already failed to match every known signature above. Deliberately brand
	 * names, not generic terms like "bot" or "ai" — those would also catch
	 * ordinary search-engine crawlers and match inside common words, drowning
	 * out the signal this exists to surface.
	 *
	 * @return array
	 */
	public static function near_miss_keywords() {
		return (array) apply_filters(
			'ai_o_optimiser_near_miss_keywords',
			array(
				'google-agent', 'google-gemini', 'google-notebook', 'cloudvertex', 'gemini', 'bard', 'vertex',
				'claude', 'anthropic',
				'openai', 'chatgpt', 'gpt',
				'perplexity', 'copilot', 'bing',
				'meta-external', 'metaai', 'llama',
				'mistral', 'cohere', 'you.com', 'youbot',
				'deepseek', 'grok', 'character.ai',
			)
		);
	}

	/**
	 * Ordinary crawlers that a near-miss keyword would otherwise catch.
	 *
	 * The capture holds forty entries and exists to surface an AI agent the
	 * signature list does not know yet. Bingbot alone would fill it: 'bing' is a
	 * keyword, Bingbot is deliberately not a runtime signature because it is
	 * also ordinary Bing Search, so every Bingbot request qualified as a
	 * near-miss and pushed out the one unknown agent the site owner needed to
	 * see. Search, SEO and social-preview crawlers are recognised here and left
	 * alone -- their identity is not in question.
	 *
	 * @return array
	 */
	public static function near_miss_exclusions() {
		return (array) apply_filters(
			'ai_o_optimiser_near_miss_exclusions',
			array(
				// Search engines.
				'googlebot', 'bingbot', 'msnbot', 'bingpreview', 'duckduckbot', 'yandex',
				'baiduspider', 'sogou', 'exabot', 'seznambot', 'qwantify', 'petalbot',
				'slurp', 'applebot',
				// SEO and marketing crawlers.
				'ahrefsbot', 'semrushbot', 'mj12bot', 'dotbot', 'rogerbot', 'screaming frog',
				'sitebulb', 'dataforseo', 'barkrowler', 'blexbot', 'serpstatbot',
				// Social and messaging link previews.
				'facebookexternalhit', 'twitterbot', 'linkedinbot', 'slackbot',
				'whatsapp', 'telegrambot', 'discordbot', 'pinterest', 'redditbot',
				// Uptime and infrastructure.
				'uptimerobot', 'pingdom', 'statuscake', 'site24x7', 'newrelicpinger',
			)
		);
	}

	/**
	 * Whether a User-Agent that has already failed to match a known signature
	 * loosely resembles an AI platform anyway, and is therefore worth capturing
	 * for a site owner to inspect.
	 *
	 * @param string $user_agent Raw User-Agent header value.
	 * @return bool
	 */
	public static function looks_ai_like( $user_agent ) {
		$user_agent = (string) $user_agent;
		if ( '' === $user_agent ) {
			return false;
		}

		// Checked first: a known crawler is not an unidentified one, whichever
		// keyword its name happens to contain.
		foreach ( self::near_miss_exclusions() as $known ) {
			if ( false !== stripos( $user_agent, $known ) ) {
				return false;
			}
		}

		foreach ( self::near_miss_keywords() as $keyword ) {
			if ( false !== stripos( $user_agent, $keyword ) ) {
				return true;
			}
		}
		return false;
	}

	/**
	 * Record one near-miss request for later review under Settings.
	 *
	 * Deduplicated by the exact User-Agent string, capped, and self-expiring —
	 * this is a short-lived diagnostic capture, not a permanent traffic log.
	 * Deliberately does not store an IP address, hashed or otherwise: the raw
	 * User-Agent is the only thing needed to add a new signature, and this
	 * capture is opt-in specifically to keep it minimal.
	 *
	 * @param string $user_agent Raw User-Agent header value.
	 * @param string $request_uri Request path, for context only.
	 */
	public static function record_near_miss( $user_agent, $request_uri ) {
		$user_agent = trim( (string) $user_agent );
		if ( '' === $user_agent ) {
			return;
		}
		$user_agent = function_exists( 'mb_substr' ) ? mb_substr( $user_agent, 0, 300 ) : substr( $user_agent, 0, 300 );
		$key        = md5( $user_agent );
		$now        = time();
		$entries    = self::near_misses_raw();

		if ( isset( $entries[ $key ] ) ) {
			$entries[ $key ]['count']++;
			$entries[ $key ]['last_seen'] = $now;
		} else {
			$entries[ $key ] = array(
				'user_agent'  => $user_agent,
				'request_uri' => (string) $request_uri,
				'count'       => 1,
				'first_seen'  => $now,
				'last_seen'   => $now,
			);
		}

		// Expire old entries before capping, so a stale one-off does not
		// permanently occupy a slot a genuinely recurring signature needs.
		$cutoff = $now - ( self::NEAR_MISS_TTL_DAYS * DAY_IN_SECONDS );
		foreach ( $entries as $entry_key => $entry ) {
			if ( $entry['last_seen'] < $cutoff ) {
				unset( $entries[ $entry_key ] );
			}
		}

		if ( count( $entries ) > self::NEAR_MISS_MAX ) {
			uasort( $entries, function( $a, $b ) { return $b['last_seen'] <=> $a['last_seen']; } );
			$entries = array_slice( $entries, 0, self::NEAR_MISS_MAX, true );
		}

		update_option( self::NEAR_MISS_OPTION, $entries, false );
	}

	/**
	 * @return array Raw stored entries, keyed by md5(user_agent).
	 */
	private static function near_misses_raw() {
		$entries = get_option( self::NEAR_MISS_OPTION, array() );
		return is_array( $entries ) ? $entries : array();
	}

	/**
	 * Captured near-miss entries, most recently seen first.
	 *
	 * @return array
	 */
	public static function get_near_misses() {
		$entries = self::near_misses_raw();
		uasort( $entries, function( $a, $b ) { return $b['last_seen'] <=> $a['last_seen']; } );
		return $entries;
	}

	/**
	 * Remove one captured entry (e.g. once added as a signature, or dismissed).
	 *
	 * @param string $key md5(user_agent) key.
	 */
	public static function dismiss_near_miss( $key ) {
		$entries = self::near_misses_raw();
		unset( $entries[ sanitize_text_field( $key ) ] );
		update_option( self::NEAR_MISS_OPTION, $entries, false );
	}

	/**
	 * Clear every captured entry.
	 */
	public static function clear_near_misses() {
		delete_option( self::NEAR_MISS_OPTION );
	}
}