""" Meta_Manager.py Manages the Meta.db database which stores global player vehicle statistics. Fetches vehicle data for all players from sq_battles.db and stores it in a structured format. """ # Standard Library Imports import asyncio import hashlib import json import logging import time from typing import Any, Dict, List, Tuple, Optional # Third-Party Library Imports import aiosqlite # Local Module Imports from .data_parser import LangTableReader, normalize_name from .game_api import obtain_clan_new_points, obtain_player_data_api from .utils import STORAGE_DIR, SQUADRONS_DB_PATH, compress_json, decompress_json STORAGE_DIR.mkdir(parents=True, exist_ok=True) # Database paths SQ_BATTLES_DB_PATH = STORAGE_DIR / "sq_battles.db" META_DB_PATH = STORAGE_DIR / "Meta.db" # Language translator for vehicle names translate = LangTableReader("English") async def init_meta_db(): """Initialize Meta.db with the Players_Global, Guilds, and Guild_Metas tables. Also runs schema migrations (e.g. adding the ``password_hint`` column to Guilds if missing). Idempotent — safe to call on every startup. """ async with aiosqlite.connect(META_DB_PATH) as db: # Create Players_Global table - ONE ROW PER PLAYER with vehicles as JSON await db.execute(""" CREATE TABLE IF NOT EXISTS Players_Global ( userID TEXT PRIMARY KEY, nick TEXT NOT NULL, clanTag TEXT, clanName TEXT, clanID TEXT, lastDay INTEGER, vehicles TEXT NOT NULL, updated_unix INTEGER NOT NULL ) """) # Create index for efficient queries await db.execute(""" CREATE INDEX IF NOT EXISTS idx_players_global_updated ON Players_Global(updated_unix) """) # Create Guilds table for server-specific meta data access settings await db.execute(""" CREATE TABLE IF NOT EXISTS Guilds ( guild_id TEXT PRIMARY KEY, squadron_clanID TEXT NOT NULL, squadron_name TEXT NOT NULL, access_password TEXT NOT NULL, require_password BOOLEAN DEFAULT 0, lock_squadron_data BOOLEAN DEFAULT 0, allow_public_meta BOOLEAN DEFAULT 0, created_unix INTEGER NOT NULL, updated_unix INTEGER NOT NULL ) """) # Create Guild_Metas table to store which players are in each guild's meta await db.execute(""" CREATE TABLE IF NOT EXISTS Guild_Metas ( id INTEGER PRIMARY KEY AUTOINCREMENT, guild_id TEXT NOT NULL, userID TEXT NOT NULL, nick TEXT NOT NULL, clanTag TEXT, clanName TEXT, clanID TEXT, added_unix INTEGER NOT NULL, UNIQUE(guild_id, userID), FOREIGN KEY(guild_id) REFERENCES Guilds(guild_id) ON DELETE CASCADE ) """) # Create index for efficient queries await db.execute(""" CREATE INDEX IF NOT EXISTS idx_guild_metas_guild ON Guild_Metas(guild_id) """) await db.execute(""" CREATE INDEX IF NOT EXISTS idx_guild_metas_userid ON Guild_Metas(userID) """) # Migration: add password_hint column if it doesn't exist cursor = await db.execute("PRAGMA table_info(Guilds)") columns = [row[1] for row in await cursor.fetchall()] if "password_hint" not in columns: await db.execute("ALTER TABLE Guilds ADD COLUMN password_hint TEXT DEFAULT NULL") await db.commit() logging.info("✅ Meta.db initialized with Players_Global, Guilds, and Guild_Metas tables") def generate_password_from_clanid(clan_id: str) -> str: """ Generate a secure password from clanID. Args: clan_id: The squadron's clan ID Returns: Generated password string """ # Create a hash of the clan_id and take first 12 characters # Add some salt for uniqueness salt = "SREBOT_META_ACCESS" hash_input = f"{clan_id}_{salt}".encode('utf-8') hash_digest = hashlib.sha256(hash_input).hexdigest() # Take first 12 chars and format nicely password = f"{hash_digest[:4]}-{hash_digest[4:8]}-{hash_digest[8:12]}" result = password.upper() logging.info(f"[PASSWORD GEN] clanID: '{clan_id}' -> password generated") return result async def get_guild_settings(guild_id: str) -> Optional[dict]: """ Get guild settings from Meta.db. Args: guild_id: Discord guild ID Returns: Dict with guild settings or None if not found """ async with aiosqlite.connect(META_DB_PATH) as db: cursor = await db.execute(""" SELECT guild_id, squadron_clanID, squadron_name, access_password, require_password, lock_squadron_data, allow_public_meta, created_unix, updated_unix, password_hint FROM Guilds WHERE guild_id = ? """, (guild_id,)) row = await cursor.fetchone() if not row: return None return { "guild_id": row[0], "squadron_clanID": row[1], "squadron_name": row[2], "access_password": row[3], "require_password": bool(row[4]), "lock_squadron_data": bool(row[5]), "allow_public_meta": bool(row[6]), "created_unix": row[7], "updated_unix": row[8], "password_hint": row[9] or "" } async def get_squadron_owner(squadron_clanID: str) -> Optional[dict]: """ Check if this squadron is already claimed by any server. Args: squadron_clanID: Squadron clan ID Returns: Dict with owner guild settings or None if squadron not claimed """ async with aiosqlite.connect(META_DB_PATH) as db: cursor = await db.execute(""" SELECT guild_id, squadron_clanID, squadron_name, access_password, require_password, lock_squadron_data, allow_public_meta, created_unix, updated_unix, password_hint FROM Guilds WHERE squadron_clanID = ? ORDER BY created_unix ASC LIMIT 1 """, (squadron_clanID,)) row = await cursor.fetchone() if not row: return None return { "guild_id": row[0], "squadron_clanID": row[1], "squadron_name": row[2], "access_password": row[3], "require_password": bool(row[4]), "lock_squadron_data": bool(row[5]), "allow_public_meta": bool(row[6]), "created_unix": row[7], "updated_unix": row[8], "password_hint": row[9] or "" } async def create_or_update_guild(guild_id: str, squadron_clanID: str, squadron_name: str, require_password: bool = False, lock_squadron_data: bool = False) -> str: """ Create or update guild settings in Meta.db. Args: guild_id: Discord guild ID squadron_clanID: Squadron clan ID squadron_name: Squadron name require_password: Whether to require password for meta access lock_squadron_data: Whether to lock squadron data to this server Returns: Access password for the guild """ # Generate password from clanID password = generate_password_from_clanid(squadron_clanID) current_time = int(time.time()) async with aiosqlite.connect(META_DB_PATH) as db: # Check if guild exists existing = await get_guild_settings(guild_id) if existing: # Update existing (including new password for new squadron) logging.info(f"[CREATE_OR_UPDATE] UPDATING guild {guild_id}: squadron_clanID {existing['squadron_clanID']} -> {squadron_clanID}") await db.execute(""" UPDATE Guilds SET squadron_clanID = ?, squadron_name = ?, access_password = ?, require_password = ?, lock_squadron_data = ?, updated_unix = ? WHERE guild_id = ? """, (squadron_clanID, squadron_name, password, int(require_password), int(lock_squadron_data), current_time, guild_id)) else: # Create new logging.info(f"[CREATE_OR_UPDATE] CREATING guild {guild_id}: squadron_clanID {squadron_clanID}") await db.execute(""" INSERT INTO Guilds (guild_id, squadron_clanID, squadron_name, access_password, require_password, lock_squadron_data, created_unix, updated_unix) VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, (guild_id, squadron_clanID, squadron_name, password, int(require_password), int(lock_squadron_data), current_time, current_time)) await db.commit() return password async def validate_squadron_password(squadron_clanID: str, provided_password: str) -> bool: """ Validate if provided password matches the squadron's password. Args: squadron_clanID: Squadron clan ID provided_password: Password to validate Returns: True if password matches, False otherwise """ owner = await get_squadron_owner(squadron_clanID) if not owner: logging.warning(f"[PASSWORD VALIDATE] No owner found for squadron {squadron_clanID}") return False stored_password = owner["access_password"].upper() provided_upper = provided_password.upper() logging.info(f"[PASSWORD VALIDATE] squadron_clanID: {squadron_clanID}, match: {stored_password == provided_upper}") return stored_password == provided_upper async def update_squadron_password(squadron_clanID: str, new_password: str, password_hint: Optional[str] = None) -> bool: """ Update the squadron's access password and optionally the password hint. Args: squadron_clanID: Squadron clan ID new_password: The new password to set password_hint: Optional hint to store (None = no change, "" = clear hint) Returns: True if successful, False otherwise """ async with aiosqlite.connect(META_DB_PATH) as db: updates = ["access_password = ?", "updated_unix = ?"] values = [new_password, int(time.time())] if password_hint is not None: updates.append("password_hint = ?") values.append(password_hint if password_hint else None) values.append(squadron_clanID) await db.execute(f""" UPDATE Guilds SET {', '.join(updates)} WHERE squadron_clanID = ? """, tuple(values)) await db.commit() logging.info(f"[PASSWORD UPDATE] Password updated for squadron {squadron_clanID}") return True async def transfer_squadron_to_guild(squadron_clanID: str, new_guild_id: str, squadron_name: str) -> bool: """ Transfer squadron ownership to a new guild (when they move servers). Args: squadron_clanID: Squadron clan ID new_guild_id: New guild ID to transfer to squadron_name: Squadron name Returns: True if successful, False otherwise """ async with aiosqlite.connect(META_DB_PATH) as db: # Update the guild_id for this squadron await db.execute(""" UPDATE Guilds SET guild_id = ?, squadron_name = ?, updated_unix = ? WHERE squadron_clanID = ? """, (new_guild_id, squadron_name, int(time.time()), squadron_clanID)) await db.commit() return True async def update_guild_settings(guild_id: str, require_password: Optional[bool] = None, lock_squadron_data: Optional[bool] = None, allow_public_meta: Optional[bool] = None) -> bool: """ Update specific guild settings. Args: guild_id: Discord guild ID require_password: Whether to require password (None = no change) lock_squadron_data: Whether to lock data (None = no change) allow_public_meta: Whether to allow non-admins to use /meta (None = no change) Returns: True if successful, False otherwise """ async with aiosqlite.connect(META_DB_PATH) as db: updates = [] values = [] if require_password is not None: updates.append("require_password = ?") values.append(int(require_password)) if lock_squadron_data is not None: updates.append("lock_squadron_data = ?") values.append(int(lock_squadron_data)) if allow_public_meta is not None: updates.append("allow_public_meta = ?") values.append(int(allow_public_meta)) if not updates: return False updates.append("updated_unix = ?") values.append(int(time.time())) values.append(guild_id) await db.execute(f""" UPDATE Guilds SET {', '.join(updates)} WHERE guild_id = ? """, tuple(values)) await db.commit() return True async def add_player_to_guild_meta(guild_id: str, user_id: str, squadron_clanID: str) -> tuple[bool, str]: """ Add a player to a guild's meta roster. Args: guild_id: Discord guild ID user_id: Player UID squadron_clanID: Squadron clan ID to validate against Returns: Tuple of (success: bool, message: str) """ # Get player data from Players_Global async with aiosqlite.connect(META_DB_PATH) as db: cursor = await db.execute(""" SELECT DISTINCT userID, nick, clanTag, clanName, clanID FROM Players_Global WHERE userID = ? LIMIT 1 """, (user_id,)) row = await cursor.fetchone() if not row: return False, f"Player `{user_id}` not found in Players_Global database." player_uid, nick, clan_tag, clan_name, player_clan_id = row # Validate that player belongs to the squadron if player_clan_id != squadron_clanID: return False, f"Player `{nick}` ({user_id}) does not belong to this squadron." # Add to Guild_Metas try: await db.execute(""" INSERT INTO Guild_Metas (guild_id, userID, nick, clanTag, clanName, clanID, added_unix) VALUES (?, ?, ?, ?, ?, ?, ?) """, (guild_id, player_uid, nick, clan_tag, clan_name, player_clan_id, int(time.time()))) await db.commit() return True, f"Successfully added **{nick}** to guild meta." except Exception as e: error_msg = str(e) if "UNIQUE constraint" in error_msg: return False, f"Player **{nick}** is already in your guild meta." return False, f"Failed to add player: {error_msg}" async def bulk_add_squadron_players_to_guild_meta( guild_id: str, squadron_clanID: str, squadron_name: str, clan_tag: str, members: Dict[str, Dict[str, Any]], ) -> Tuple[int, int, int]: """ Sync all squadron players to a guild's meta roster. Adds new members and removes players who are no longer in the squadron. Args: guild_id: Discord guild ID squadron_clanID: Squadron clan ID squadron_name: Squadron long name clan_tag: Squadron tag name members: Dict from obtain_clan_new_points(): ``{uid: {"nick": str, ...}}``. Only the ``"nick"`` key is accessed per member. Returns: Tuple of (added_count, removed_count, skipped_count) """ now = int(time.time()) added = 0 removed = 0 skipped = 0 current_uids = set(members.keys()) async with aiosqlite.connect(META_DB_PATH) as db: # Insert new members for uid, info in members.items(): nick = info.get("nick", "Unknown") cursor = await db.execute(""" INSERT OR IGNORE INTO Guild_Metas (guild_id, userID, nick, clanTag, clanName, clanID, added_unix) VALUES (?, ?, ?, ?, ?, ?, ?) """, (guild_id, uid, nick, clan_tag, squadron_name, squadron_clanID, now)) if cursor.rowcount > 0: added += 1 else: skipped += 1 # Remove members who are no longer in the squadron cursor = await db.execute( "SELECT userID FROM Guild_Metas WHERE guild_id = ?", (guild_id,) ) existing_uids = {row[0] for row in await cursor.fetchall()} departed_uids = existing_uids - current_uids for uid in departed_uids: await db.execute( "DELETE FROM Guild_Metas WHERE guild_id = ? AND userID = ?", (guild_id, uid) ) removed += 1 await db.commit() return added, removed, skipped async def refresh_guild_player_vehicles(guild_id: str, max_concurrent: int = 5): """ Re-fetch vehicle data from the Game API for all players in a guild's meta roster. Skips players updated within 2 days. Args: guild_id: Discord guild ID max_concurrent: Maximum concurrent API requests Returns: None. Players_Global rows are updated in-place. """ async with aiosqlite.connect(META_DB_PATH) as db: cursor = await db.execute( "SELECT userID, nick FROM Guild_Metas WHERE guild_id = ?", (guild_id,) ) players = [(row[0], row[1]) for row in await cursor.fetchall()] if not players: return semaphore = asyncio.Semaphore(max_concurrent) async with aiosqlite.connect(META_DB_PATH) as meta_db: for i in range(0, len(players), 20): batch = players[i:i + 20] await process_player_batch(batch, meta_db, skip_existing=True, semaphore=semaphore) await meta_db.commit() logging.info(f"[META] Refreshed vehicle data for {len(players)} players in guild {guild_id}") async def sync_all_guild_metas() -> Dict[str, Any]: """ Daily sync: for every guild in the Guilds table, fetch the current squadron roster from the Game API and reconcile Guild_Metas (add new, remove departed). Returns: Summary dict with total added/removed counts and per-guild results. """ summary = {"guilds_processed": 0, "total_added": 0, "total_removed": 0, "errors": []} async with aiosqlite.connect(META_DB_PATH) as db: db.row_factory = aiosqlite.Row cursor = await db.execute( "SELECT guild_id, squadron_clanID, squadron_name FROM Guilds" ) guilds = await cursor.fetchall() for guild in guilds: guild_id = guild["guild_id"] clan_id = guild["squadron_clanID"] squadron_name = guild["squadron_name"] try: members, _total_points = await obtain_clan_new_points(squadron_name) except Exception as e: logging.warning(f"[META-SYNC] Failed to fetch members for guild {guild_id} ({squadron_name}): {e}") summary["errors"].append(guild_id) continue if not members: logging.warning(f"[META-SYNC] No members returned for guild {guild_id} ({squadron_name}), skipping.") continue # Resolve clan_tag from squadrons.db clan_tag = "" try: async with aiosqlite.connect(SQUADRONS_DB_PATH) as sq_db: sq_cursor = await sq_db.execute( "SELECT tag_name FROM squadrons_data WHERE clan_id = ? LIMIT 1", (clan_id,) ) row = await sq_cursor.fetchone() if row: clan_tag = row[0] or "" except Exception: pass try: added, removed, _skipped = await bulk_add_squadron_players_to_guild_meta( guild_id, clan_id, squadron_name, clan_tag, members ) summary["guilds_processed"] += 1 summary["total_added"] += added summary["total_removed"] += removed if added or removed: logging.info(f"[META-SYNC] Guild {guild_id} ({squadron_name}): +{added} added, -{removed} removed") except Exception as e: logging.warning(f"[META-SYNC] Failed to sync guild {guild_id}: {e}") summary["errors"].append(guild_id) logging.info( f"[META-SYNC] Done. {summary['guilds_processed']} guilds, " f"+{summary['total_added']} added, -{summary['total_removed']} removed, " f"{len(summary['errors'])} errors." ) return summary async def remove_player_from_guild_meta(guild_id: str, user_id: str) -> tuple[bool, str]: """ Remove a player from a guild's meta roster. Args: guild_id: Discord guild ID user_id: Player UID Returns: Tuple of (success: bool, message: str) """ async with aiosqlite.connect(META_DB_PATH) as db: # Get player nick first cursor = await db.execute(""" SELECT nick FROM Guild_Metas WHERE guild_id = ? AND userID = ? """, (guild_id, user_id)) row = await cursor.fetchone() if not row: return False, f"Player `{user_id}` not found in your guild meta." nick = row[0] # Remove player await db.execute(""" DELETE FROM Guild_Metas WHERE guild_id = ? AND userID = ? """, (guild_id, user_id)) await db.commit() return True, f"Successfully removed **{nick}** from guild meta." async def get_guild_meta_players(guild_id: str) -> List[dict]: """ Get all players in a guild's meta roster. Args: guild_id: Discord guild ID Returns: List of dicts, each with keys: ``userID``, ``nick``, ``clanTag``, ``clanName``, ``clanID``, ``added_unix``. """ async with aiosqlite.connect(META_DB_PATH) as db: cursor = await db.execute(""" SELECT userID, nick, clanTag, clanName, clanID, added_unix FROM Guild_Metas WHERE guild_id = ? ORDER BY nick ASC """, (guild_id,)) rows = await cursor.fetchall() return [ { "userID": row[0], "nick": row[1], "clanTag": row[2], "clanName": row[3], "clanID": row[4], "added_unix": row[5] } for row in rows ] def _fuzzy_match(search_terms: List[str], target: str) -> bool: """ Check if all search terms appear in the target string. This allows searches like "leopard 2a7" to match "Leopard 2A7V". Args: search_terms: List of lowercase search terms target: Target string to match against Returns: True if all terms found in target """ target_lower = target.lower() return all(term in target_lower for term in search_terms) def _score_match(search_terms: List[str], target: str) -> int: """ Score how well search terms match a target. Higher score = better match. Args: search_terms: List of lowercase search terms target: Target string to match against Returns: Match score (0 = no match) """ target_lower = target.lower() score = 0 for term in search_terms: if term in target_lower: score += 10 # Bonus for exact word match if term in target_lower.split(): score += 5 # Bonus for start of word match if target_lower.startswith(term) or f" {term}" in target_lower or f"-{term}" in target_lower: score += 3 return score async def get_guild_meta_vehicles(guild_id: str) -> List[dict]: """ Get all unique vehicles from players in a guild's meta roster. Used for autocomplete suggestions. Deduplicates by human-readable name to avoid showing the same display name multiple times (e.g., different nation variants like germ_leopard_2a4 and sw_leopard_2a4 both display as "Leopard 2A4"). Args: guild_id: Discord guild ID Returns: List of dicts with vehicle_name (internal) and vehicle_human (readable) """ # Use dict keyed by human name to dedupe display duplicates # Store first internal name encountered for each human name vehicles_by_human = {} async with aiosqlite.connect(META_DB_PATH) as db: # Get all players in this guild's meta cursor = await db.execute(""" SELECT userID FROM Guild_Metas WHERE guild_id = ? """, (guild_id,)) guild_players = await cursor.fetchall() for (user_id,) in guild_players: cursor = await db.execute(""" SELECT vehicles FROM Players_Global WHERE userID = ? """, (user_id,)) row = await cursor.fetchone() if not row or not row[0]: continue try: vehicles_array = decompress_json(row[0]) except (json.JSONDecodeError, TypeError, OSError): continue for vehicle in vehicles_array: internal = vehicle.get("vehicle_name", "") human = vehicle.get("vehicle_human", "") or internal # Dedupe by human name - keep first internal name encountered if human and human not in vehicles_by_human: vehicles_by_human[human] = internal return [{"vehicle_name": v, "vehicle_human": k} for k, v in vehicles_by_human.items()] async def search_guild_meta_by_vehicle(guild_id: str, vehicle_name: str, exact_only: bool = False) -> List[dict]: """ Search for players in guild meta who have a specific vehicle. Searches within the vehicles JSON array for each player. Supports fuzzy matching on human-readable names: - "KVT" matches "M1 KVT" - "leopard 2a7" matches "Leopard 2A7V" - "f16" matches "F-16C" Args: guild_id: Discord guild ID. vehicle_name: Vehicle name (partial/fuzzy match supported). exact_only: If True, only return exact internal-name matches. Returns: List of dicts with player info and vehicle stats. """ async with aiosqlite.connect(META_DB_PATH) as db: # Get all players in this guild's meta cursor = await db.execute(""" SELECT userID, nick, clanTag, clanName FROM Guild_Metas WHERE guild_id = ? """, (guild_id,)) guild_players = await cursor.fetchall() results = [] # Prepare search terms (split by spaces, remove empty) search_input = vehicle_name.strip().lower() search_terms = [t for t in search_input.split() if t] # For each player, get their vehicles JSON from Players_Global # First pass: try exact internal name match (from autocomplete) # Second pass: fall back to fuzzy match on human-readable name only exact_results = [] fuzzy_results = [] for user_id, nick, clan_tag, clan_name in guild_players: cursor = await db.execute(""" SELECT vehicles FROM Players_Global WHERE userID = ? """, (user_id,)) row = await cursor.fetchone() if not row or not row[0]: continue # Parse vehicles array (may be gzip-compressed BLOB or legacy TEXT) try: vehicles_array = decompress_json(row[0]) except (json.JSONDecodeError, TypeError, OSError): continue # Search through vehicles array for matching vehicles for vehicle in vehicles_array: vehicle_name_internal = vehicle.get("vehicle_name", "") vehicle_human = vehicle.get("vehicle_human", "") or "" # Exact internal name match (highest priority) if vehicle_name_internal.lower() == search_input: match_score = 100 exact_results.append((user_id, nick, clan_tag, clan_name, vehicle, match_score)) continue # Fuzzy matching on human-readable name only (not internal name) # This prevents "so_4050_vautour_2n" from matching "so_4050_vautour_2n_late" human_lower = vehicle_human.lower() match_score = 0 if search_terms: if _fuzzy_match(search_terms, human_lower): match_score = _score_match(search_terms, human_lower) if search_input in human_lower: match_score = max(match_score, 15) if match_score > 0: fuzzy_results.append((user_id, nick, clan_tag, clan_name, vehicle, match_score)) # Use exact results if found, otherwise fall back to fuzzy (unless exact_only) if exact_results: matched = exact_results elif exact_only: matched = [] else: matched = fuzzy_results for user_id, nick, clan_tag, clan_name, vehicle, match_score in matched: stats = vehicle.get("stats", {}) results.append({ "userID": user_id, "nick": nick, "clanTag": clan_tag, "clanName": clan_name, "intname": vehicle.get("vehicle_name", ""), "vehicle_human": vehicle.get("vehicle_human", "") or "", "mode": vehicle.get("mode", ""), "flyouts": vehicle.get("flyouts", 0), "was_in_session": vehicle.get("was_in_session", 0), "deaths": stats.get("deaths", 0), "ground_kills": stats.get("ground_kills", 0), "air_kills": stats.get("air_kills", 0), "_match_score": match_score }) # Sort by match score (best matches first), then by player name results.sort(key=lambda x: (-x.get("_match_score", 0), x.get("nick", "").lower())) # Remove the internal match score before returning for r in results: r.pop("_match_score", None) return results async def get_all_player_uids() -> List[Tuple[str, str]]: """ Get all unique player UIDs and nicknames from sq_battles.db. Returns: List of tuples (UID, nick) """ if not SQ_BATTLES_DB_PATH.exists(): logging.error(f"❌ {SQ_BATTLES_DB_PATH} does not exist!") return [] async with aiosqlite.connect(SQ_BATTLES_DB_PATH) as db: cursor = await db.execute(""" SELECT DISTINCT UID, nick FROM player_games_hist WHERE UID IS NOT NULL AND UID != '' ORDER BY UID """) rows = await cursor.fetchall() result = [(str(row[0]), str(row[1])) for row in rows] logging.info(f"📊 Found {len(result)} unique players in sq_battles.db") return result def translate_vehicle_name(vehicle_cdk: str) -> Optional[str]: """ Translate vehicle CDK (code name) to human-readable name. Args: vehicle_cdk: Internal vehicle code name Returns: Human-readable vehicle name or None if not found """ try: raw = translate.get_translate(vehicle_cdk) if raw: return normalize_name(raw) return None except Exception: return None async def fetch_player_vehicles(uid: str, max_retries: int = 3) -> Optional[dict]: """ Fetch player vehicle data from Game API with retry logic. Args: uid: Player UID max_retries: Maximum number of retry attempts for 503 errors Returns: Cleaned player data dict or None on failure """ last_error = None for attempt in range(max_retries): try: data = await obtain_player_data_api(uid, raw=False) # Check if token expired (empty dict returned) if not data or not isinstance(data, dict): return None return data except Exception as e: last_error = e error_msg = str(e) # Check if it's a rate limit error (503) if "503" in error_msg or "RETRY" in error_msg or "Network request failed" in error_msg: if attempt < max_retries - 1: wait_time = (attempt + 1) * 2 # Exponential backoff: 2s, 4s, 6s # Silently retry - don't spam logs await asyncio.sleep(wait_time) continue else: # Only log after all retries exhausted logging.info(f"⚠️ Rate limited UID {uid} after {max_retries} attempts") return None else: # Non-rate-limit error - log it logging.error(f"❌ Failed to fetch UID {uid}: {error_msg}") return None return None async def insert_player_vehicles(db, player_data: dict, updated_unix: int): """ Insert or update player vehicle data in Meta.db. Stores ONE ROW PER PLAYER with all vehicles as JSON array. Args: db: Database connection player_data: Cleaned player data from Game API updated_unix: Unix timestamp of when this data was fetched Returns: None. The row is upserted into Players_Global. """ user_id = player_data.get("userID", "") nick = player_data.get("nick", "") clan_tag = player_data.get("clanTag", "") clan_name = player_data.get("clanName", "") clan_id = player_data.get("clanID", "") last_day = player_data.get("lastDay", 0) vehicles_raw = player_data.get("vehicles", []) if not user_id: logging.warning("⚠️ Skipping player with no userID") return # Build vehicles array with translated names vehicles_array = [] for vehicle in vehicles_raw: vehicle_name = vehicle.get("name", "") mode = vehicle.get("mode", "") flyouts = vehicle.get("flyouts", 0) was_in_session = vehicle.get("was_in_session", 0) if not vehicle_name or not mode: continue # Translate vehicle name to human-readable vehicle_human = translate_vehicle_name(vehicle_name) # Extract stats stats_dict = {k: v for k, v in vehicle.items() if k not in ["name", "mode", "flyouts", "was_in_session"]} vehicles_array.append({ "vehicle_name": vehicle_name, "vehicle_human": vehicle_human, "mode": mode, "flyouts": flyouts, "was_in_session": was_in_session, "stats": stats_dict }) # Compress vehicles array for BLOB storage vehicles_json = compress_json(vehicles_array) # Insert or update - ONE ROW PER PLAYER await db.execute(""" INSERT INTO Players_Global (userID, nick, clanTag, clanName, clanID, lastDay, vehicles, updated_unix) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(userID) DO UPDATE SET nick = excluded.nick, clanTag = excluded.clanTag, clanName = excluded.clanName, clanID = excluded.clanID, lastDay = excluded.lastDay, vehicles = excluded.vehicles, updated_unix = excluded.updated_unix """, (user_id, nick, clan_tag, clan_name, clan_id, last_day, vehicles_json, updated_unix)) async def process_player_batch(players_batch: List[Tuple[str, str]], meta_db, skip_existing: bool, semaphore): """ Process a batch of players concurrently. Args: players_batch: List of (uid, nick) tuples meta_db: Database connection skip_existing: If True, skip players with recent data semaphore: Asyncio semaphore for rate limiting Returns: Dict with keys ``"success"``, ``"errors"``, and ``"skipped"`` containing integer counts for each outcome. """ results = {"success": 0, "errors": 0, "skipped": 0} async def process_single_player(uid: str, nick: str): async with semaphore: try: # Check if player was recently updated (within 2 days) if skip_existing: cursor = await meta_db.execute(""" SELECT updated_unix FROM Players_Global WHERE userID = ? """, (uid,)) row = await cursor.fetchone() if row and row[0]: days_old = (time.time() - row[0]) / 86400 if days_old < 2: return "skipped" # Fetch player data player_data = await fetch_player_vehicles(uid) if player_data: # Insert vehicle data updated_unix = int(time.time()) await insert_player_vehicles(meta_db, player_data, updated_unix) return "success" else: return "error" except Exception as e: logging.error(f"❌ Error processing UID {uid}: {e}") return "error" # Process all players in batch concurrently tasks = [process_single_player(uid, nick) for uid, nick in players_batch] batch_results = await asyncio.gather(*tasks, return_exceptions=True) # Count results for result in batch_results: if isinstance(result, Exception): results["errors"] += 1 elif result == "success": results["success"] += 1 elif result == "skipped": results["skipped"] += 1 elif result == "error": results["errors"] += 1 return results async def process_all_players(limit: Optional[int] = None, skip_existing: bool = True, batch_size: int = 20, max_concurrent: int = 5): """ Process all players and fetch their vehicle data. Args: limit: Optional limit on number of players to process skip_existing: If True, skip players already in Meta.db with recent data (within 2 days) batch_size: Number of players to process in each batch before committing max_concurrent: Maximum number of concurrent API requests (default 5 to avoid rate limits) Returns: None. Results are committed to Meta.db and logged. """ # Initialize Meta.db await init_meta_db() # Get all player UIDs players = await get_all_player_uids() if limit: players = players[:limit] logging.info(f"📌 Processing limited to {limit} players") total = len(players) processed = 0 success = 0 errors = 0 skipped = 0 # Create semaphore for rate limiting semaphore = asyncio.Semaphore(max_concurrent) async with aiosqlite.connect(META_DB_PATH) as meta_db: # Process in batches for i in range(0, total, batch_size): batch = players[i:i + batch_size] batch_results = await process_player_batch(batch, meta_db, skip_existing, semaphore) # Update counters success += batch_results["success"] errors += batch_results["errors"] skipped += batch_results["skipped"] processed += len(batch) # Commit after each batch await meta_db.commit() # Small delay between batches to avoid overwhelming the API await asyncio.sleep(1) # Final commit await meta_db.commit() async def main(): """CLI entry point for meta_manager. Fetches and stores vehicle data for all players in sq_battles.db, skipping recently-updated entries. """ # Process all players with concurrent batch processing # max_concurrent=5 means up to 5 API requests running at once (reduced to avoid 503 errors) # batch_size=50 means commit to DB every 50 players await process_all_players( limit=None, # No limit - process all players skip_existing=True, # Skip players updated within 2 days batch_size=50, # Commit every 50 players max_concurrent=5 # 5 concurrent API requests (conservative to avoid rate limits) ) if __name__ == "__main__": asyncio.run(main())