Files
SREBOT/BOT/meta_manager.py
NotSoToothless ff420e131f fix relative .data_parser imports in BOT/* after SHARED move (#1224)
PR #1223 + fixup moved data_parser into BOTS/SHARED, but five BOT modules
(analytics, autologging, botscript, lux_apis, meta_manager) still used
`from .data_parser import ...`. That relative form looks inside the BOT
package, which no longer contains data_parser, so the bot crashed at
startup with ModuleNotFoundError.

Add BOT/__init__.py to put BOTS/SHARED on sys.path at package import,
then switch all five files to absolute `from data_parser import ...`.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-13 23:30:15 -07:00

1181 lines
40 KiB
Python

"""
Meta_Manager.py
Manages the Meta.db database which stores global player vehicle statistics.
Fetches vehicle data for all players from sq_battles.db and stores it in a structured format.
"""
# Standard Library Imports
import asyncio
import hashlib
import json
import logging
import time
from typing import Any, Dict, List, Tuple, Optional
# Third-Party Library Imports
import aiosqlite
# Local Module Imports
from data_parser import LangTableReader, normalize_name
from .game_api import obtain_clan_new_points, obtain_player_data_api
from .utils import STORAGE_DIR, SQUADRONS_DB_PATH, compress_json, decompress_json
STORAGE_DIR.mkdir(parents=True, exist_ok=True)
# Database paths
SQ_BATTLES_DB_PATH = STORAGE_DIR / "sq_battles.db"
META_DB_PATH = STORAGE_DIR / "Meta.db"
# Language translator for vehicle names
translate = LangTableReader("English")
async def init_meta_db():
"""Initialize Meta.db with the Players_Global, Guilds, and Guild_Metas tables.
Also runs schema migrations (e.g. adding the ``password_hint`` column
to Guilds if missing). Idempotent — safe to call on every startup.
"""
async with aiosqlite.connect(META_DB_PATH) as db:
# Create Players_Global table - ONE ROW PER PLAYER with vehicles as JSON
await db.execute("""
CREATE TABLE IF NOT EXISTS Players_Global (
userID TEXT PRIMARY KEY,
nick TEXT NOT NULL,
clanTag TEXT,
clanName TEXT,
clanID TEXT,
lastDay INTEGER,
vehicles TEXT NOT NULL,
updated_unix INTEGER NOT NULL
)
""")
# Create index for efficient queries
await db.execute("""
CREATE INDEX IF NOT EXISTS idx_players_global_updated
ON Players_Global(updated_unix)
""")
# Create Guilds table for server-specific meta data access settings
await db.execute("""
CREATE TABLE IF NOT EXISTS Guilds (
guild_id TEXT PRIMARY KEY,
squadron_clanID TEXT NOT NULL,
squadron_name TEXT NOT NULL,
access_password TEXT NOT NULL,
require_password BOOLEAN DEFAULT 0,
lock_squadron_data BOOLEAN DEFAULT 0,
allow_public_meta BOOLEAN DEFAULT 0,
created_unix INTEGER NOT NULL,
updated_unix INTEGER NOT NULL
)
""")
# Create Guild_Metas table to store which players are in each guild's meta
await db.execute("""
CREATE TABLE IF NOT EXISTS Guild_Metas (
id INTEGER PRIMARY KEY AUTOINCREMENT,
guild_id TEXT NOT NULL,
userID TEXT NOT NULL,
nick TEXT NOT NULL,
clanTag TEXT,
clanName TEXT,
clanID TEXT,
added_unix INTEGER NOT NULL,
UNIQUE(guild_id, userID),
FOREIGN KEY(guild_id) REFERENCES Guilds(guild_id) ON DELETE CASCADE
)
""")
# Create index for efficient queries
await db.execute("""
CREATE INDEX IF NOT EXISTS idx_guild_metas_guild
ON Guild_Metas(guild_id)
""")
await db.execute("""
CREATE INDEX IF NOT EXISTS idx_guild_metas_userid
ON Guild_Metas(userID)
""")
# Migration: add password_hint column if it doesn't exist
cursor = await db.execute("PRAGMA table_info(Guilds)")
columns = [row[1] for row in await cursor.fetchall()]
if "password_hint" not in columns:
await db.execute("ALTER TABLE Guilds ADD COLUMN password_hint TEXT DEFAULT NULL")
await db.commit()
logging.info("✅ Meta.db initialized with Players_Global, Guilds, and Guild_Metas tables")
def generate_password_from_clanid(clan_id: str) -> str:
"""
Generate a secure password from clanID.
Args:
clan_id: The squadron's clan ID
Returns:
Generated password string
"""
# Create a hash of the clan_id and take first 12 characters
# Add some salt for uniqueness
salt = "SREBOT_META_ACCESS"
hash_input = f"{clan_id}_{salt}".encode('utf-8')
hash_digest = hashlib.sha256(hash_input).hexdigest()
# Take first 12 chars and format nicely
password = f"{hash_digest[:4]}-{hash_digest[4:8]}-{hash_digest[8:12]}"
result = password.upper()
logging.info(f"[PASSWORD GEN] clanID: '{clan_id}' -> password generated")
return result
async def get_guild_settings(guild_id: str) -> Optional[dict]:
"""
Get guild settings from Meta.db.
Args:
guild_id: Discord guild ID
Returns:
Dict with guild settings or None if not found
"""
async with aiosqlite.connect(META_DB_PATH) as db:
cursor = await db.execute("""
SELECT guild_id, squadron_clanID, squadron_name, access_password,
require_password, lock_squadron_data, allow_public_meta, created_unix, updated_unix,
password_hint
FROM Guilds
WHERE guild_id = ?
""", (guild_id,))
row = await cursor.fetchone()
if not row:
return None
return {
"guild_id": row[0],
"squadron_clanID": row[1],
"squadron_name": row[2],
"access_password": row[3],
"require_password": bool(row[4]),
"lock_squadron_data": bool(row[5]),
"allow_public_meta": bool(row[6]),
"created_unix": row[7],
"updated_unix": row[8],
"password_hint": row[9] or ""
}
async def get_squadron_owner(squadron_clanID: str) -> Optional[dict]:
"""
Check if this squadron is already claimed by any server.
Args:
squadron_clanID: Squadron clan ID
Returns:
Dict with owner guild settings or None if squadron not claimed
"""
async with aiosqlite.connect(META_DB_PATH) as db:
cursor = await db.execute("""
SELECT guild_id, squadron_clanID, squadron_name, access_password,
require_password, lock_squadron_data, allow_public_meta, created_unix, updated_unix,
password_hint
FROM Guilds
WHERE squadron_clanID = ?
ORDER BY created_unix ASC
LIMIT 1
""", (squadron_clanID,))
row = await cursor.fetchone()
if not row:
return None
return {
"guild_id": row[0],
"squadron_clanID": row[1],
"squadron_name": row[2],
"access_password": row[3],
"require_password": bool(row[4]),
"lock_squadron_data": bool(row[5]),
"allow_public_meta": bool(row[6]),
"created_unix": row[7],
"updated_unix": row[8],
"password_hint": row[9] or ""
}
async def create_or_update_guild(guild_id: str, squadron_clanID: str, squadron_name: str,
require_password: bool = False, lock_squadron_data: bool = False) -> str:
"""
Create or update guild settings in Meta.db.
Args:
guild_id: Discord guild ID
squadron_clanID: Squadron clan ID
squadron_name: Squadron name
require_password: Whether to require password for meta access
lock_squadron_data: Whether to lock squadron data to this server
Returns:
Access password for the guild
"""
# Generate password from clanID
password = generate_password_from_clanid(squadron_clanID)
current_time = int(time.time())
async with aiosqlite.connect(META_DB_PATH) as db:
# Check if guild exists
existing = await get_guild_settings(guild_id)
if existing:
# Update existing (including new password for new squadron)
logging.info(f"[CREATE_OR_UPDATE] UPDATING guild {guild_id}: squadron_clanID {existing['squadron_clanID']} -> {squadron_clanID}")
await db.execute("""
UPDATE Guilds
SET squadron_clanID = ?,
squadron_name = ?,
access_password = ?,
require_password = ?,
lock_squadron_data = ?,
updated_unix = ?
WHERE guild_id = ?
""", (squadron_clanID, squadron_name, password, int(require_password),
int(lock_squadron_data), current_time, guild_id))
else:
# Create new
logging.info(f"[CREATE_OR_UPDATE] CREATING guild {guild_id}: squadron_clanID {squadron_clanID}")
await db.execute("""
INSERT INTO Guilds
(guild_id, squadron_clanID, squadron_name, access_password,
require_password, lock_squadron_data, created_unix, updated_unix)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (guild_id, squadron_clanID, squadron_name, password,
int(require_password), int(lock_squadron_data), current_time, current_time))
await db.commit()
return password
async def validate_squadron_password(squadron_clanID: str, provided_password: str) -> bool:
"""
Validate if provided password matches the squadron's password.
Args:
squadron_clanID: Squadron clan ID
provided_password: Password to validate
Returns:
True if password matches, False otherwise
"""
owner = await get_squadron_owner(squadron_clanID)
if not owner:
logging.warning(f"[PASSWORD VALIDATE] No owner found for squadron {squadron_clanID}")
return False
stored_password = owner["access_password"].upper()
provided_upper = provided_password.upper()
logging.info(f"[PASSWORD VALIDATE] squadron_clanID: {squadron_clanID}, match: {stored_password == provided_upper}")
return stored_password == provided_upper
async def update_squadron_password(squadron_clanID: str, new_password: str,
password_hint: Optional[str] = None) -> bool:
"""
Update the squadron's access password and optionally the password hint.
Args:
squadron_clanID: Squadron clan ID
new_password: The new password to set
password_hint: Optional hint to store (None = no change, "" = clear hint)
Returns:
True if successful, False otherwise
"""
async with aiosqlite.connect(META_DB_PATH) as db:
updates = ["access_password = ?", "updated_unix = ?"]
values = [new_password, int(time.time())]
if password_hint is not None:
updates.append("password_hint = ?")
values.append(password_hint if password_hint else None)
values.append(squadron_clanID)
await db.execute(f"""
UPDATE Guilds
SET {', '.join(updates)}
WHERE squadron_clanID = ?
""", tuple(values))
await db.commit()
logging.info(f"[PASSWORD UPDATE] Password updated for squadron {squadron_clanID}")
return True
async def transfer_squadron_to_guild(squadron_clanID: str, new_guild_id: str, squadron_name: str) -> bool:
"""
Transfer squadron ownership to a new guild (when they move servers).
Args:
squadron_clanID: Squadron clan ID
new_guild_id: New guild ID to transfer to
squadron_name: Squadron name
Returns:
True if successful, False otherwise
"""
async with aiosqlite.connect(META_DB_PATH) as db:
# Update the guild_id for this squadron
await db.execute("""
UPDATE Guilds
SET guild_id = ?,
squadron_name = ?,
updated_unix = ?
WHERE squadron_clanID = ?
""", (new_guild_id, squadron_name, int(time.time()), squadron_clanID))
await db.commit()
return True
async def update_guild_settings(guild_id: str, require_password: Optional[bool] = None,
lock_squadron_data: Optional[bool] = None,
allow_public_meta: Optional[bool] = None) -> bool:
"""
Update specific guild settings.
Args:
guild_id: Discord guild ID
require_password: Whether to require password (None = no change)
lock_squadron_data: Whether to lock data (None = no change)
allow_public_meta: Whether to allow non-admins to use /meta (None = no change)
Returns:
True if successful, False otherwise
"""
async with aiosqlite.connect(META_DB_PATH) as db:
updates = []
values = []
if require_password is not None:
updates.append("require_password = ?")
values.append(int(require_password))
if lock_squadron_data is not None:
updates.append("lock_squadron_data = ?")
values.append(int(lock_squadron_data))
if allow_public_meta is not None:
updates.append("allow_public_meta = ?")
values.append(int(allow_public_meta))
if not updates:
return False
updates.append("updated_unix = ?")
values.append(int(time.time()))
values.append(guild_id)
await db.execute(f"""
UPDATE Guilds
SET {', '.join(updates)}
WHERE guild_id = ?
""", tuple(values))
await db.commit()
return True
async def add_player_to_guild_meta(guild_id: str, user_id: str, squadron_clanID: str) -> tuple[bool, str]:
"""
Add a player to a guild's meta roster.
Args:
guild_id: Discord guild ID
user_id: Player UID
squadron_clanID: Squadron clan ID to validate against
Returns:
Tuple of (success: bool, message: str)
"""
# Get player data from Players_Global
async with aiosqlite.connect(META_DB_PATH) as db:
cursor = await db.execute("""
SELECT DISTINCT userID, nick, clanTag, clanName, clanID
FROM Players_Global
WHERE userID = ?
LIMIT 1
""", (user_id,))
row = await cursor.fetchone()
if not row:
return False, f"Player `{user_id}` not found in Players_Global database."
player_uid, nick, clan_tag, clan_name, player_clan_id = row
# Validate that player belongs to the squadron
if player_clan_id != squadron_clanID:
return False, f"Player `{nick}` ({user_id}) does not belong to this squadron."
# Add to Guild_Metas
try:
await db.execute("""
INSERT INTO Guild_Metas (guild_id, userID, nick, clanTag, clanName, clanID, added_unix)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (guild_id, player_uid, nick, clan_tag, clan_name, player_clan_id, int(time.time())))
await db.commit()
return True, f"Successfully added **{nick}** to guild meta."
except Exception as e:
error_msg = str(e)
if "UNIQUE constraint" in error_msg:
return False, f"Player **{nick}** is already in your guild meta."
return False, f"Failed to add player: {error_msg}"
async def bulk_add_squadron_players_to_guild_meta(
guild_id: str,
squadron_clanID: str,
squadron_name: str,
clan_tag: str,
members: Dict[str, Dict[str, Any]],
) -> Tuple[int, int, int]:
"""
Sync all squadron players to a guild's meta roster.
Adds new members and removes players who are no longer in the squadron.
Args:
guild_id: Discord guild ID
squadron_clanID: Squadron clan ID
squadron_name: Squadron long name
clan_tag: Squadron tag name
members: Dict from obtain_clan_new_points(): ``{uid: {"nick": str, ...}}``.
Only the ``"nick"`` key is accessed per member.
Returns:
Tuple of (added_count, removed_count, skipped_count)
"""
now = int(time.time())
added = 0
removed = 0
skipped = 0
current_uids = set(members.keys())
async with aiosqlite.connect(META_DB_PATH) as db:
# Insert new members
for uid, info in members.items():
nick = info.get("nick", "Unknown")
cursor = await db.execute("""
INSERT OR IGNORE INTO Guild_Metas
(guild_id, userID, nick, clanTag, clanName, clanID, added_unix)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (guild_id, uid, nick, clan_tag, squadron_name, squadron_clanID, now))
if cursor.rowcount > 0:
added += 1
else:
skipped += 1
# Remove members who are no longer in the squadron
cursor = await db.execute(
"SELECT userID FROM Guild_Metas WHERE guild_id = ?", (guild_id,)
)
existing_uids = {row[0] for row in await cursor.fetchall()}
departed_uids = existing_uids - current_uids
for uid in departed_uids:
await db.execute(
"DELETE FROM Guild_Metas WHERE guild_id = ? AND userID = ?",
(guild_id, uid)
)
removed += 1
await db.commit()
return added, removed, skipped
async def refresh_guild_player_vehicles(guild_id: str, max_concurrent: int = 5):
"""
Re-fetch vehicle data from the Game API for all players in a guild's meta roster.
Skips players updated within 2 days.
Args:
guild_id: Discord guild ID
max_concurrent: Maximum concurrent API requests
Returns:
None. Players_Global rows are updated in-place.
"""
async with aiosqlite.connect(META_DB_PATH) as db:
cursor = await db.execute(
"SELECT userID, nick FROM Guild_Metas WHERE guild_id = ?", (guild_id,)
)
players = [(row[0], row[1]) for row in await cursor.fetchall()]
if not players:
return
semaphore = asyncio.Semaphore(max_concurrent)
async with aiosqlite.connect(META_DB_PATH) as meta_db:
for i in range(0, len(players), 20):
batch = players[i:i + 20]
await process_player_batch(batch, meta_db, skip_existing=True, semaphore=semaphore)
await meta_db.commit()
logging.info(f"[META] Refreshed vehicle data for {len(players)} players in guild {guild_id}")
async def sync_all_guild_metas() -> Dict[str, Any]:
"""
Daily sync: for every guild in the Guilds table, fetch the current squadron
roster from the Game API and reconcile Guild_Metas (add new, remove departed).
Returns:
Summary dict with total added/removed counts and per-guild results.
"""
summary = {"guilds_processed": 0, "total_added": 0, "total_removed": 0, "errors": []}
async with aiosqlite.connect(META_DB_PATH) as db:
db.row_factory = aiosqlite.Row
cursor = await db.execute(
"SELECT guild_id, squadron_clanID, squadron_name FROM Guilds"
)
guilds = await cursor.fetchall()
for guild in guilds:
guild_id = guild["guild_id"]
clan_id = guild["squadron_clanID"]
squadron_name = guild["squadron_name"]
try:
members, _total_points = await obtain_clan_new_points(squadron_name)
except Exception as e:
logging.warning(f"[META-SYNC] Failed to fetch members for guild {guild_id} ({squadron_name}): {e}")
summary["errors"].append(guild_id)
continue
if not members:
logging.warning(f"[META-SYNC] No members returned for guild {guild_id} ({squadron_name}), skipping.")
continue
# Resolve clan_tag from squadrons.db
clan_tag = ""
try:
async with aiosqlite.connect(SQUADRONS_DB_PATH) as sq_db:
sq_cursor = await sq_db.execute(
"SELECT tag_name FROM squadrons_data WHERE clan_id = ? LIMIT 1",
(clan_id,)
)
row = await sq_cursor.fetchone()
if row:
clan_tag = row[0] or ""
except Exception:
pass
try:
added, removed, _skipped = await bulk_add_squadron_players_to_guild_meta(
guild_id, clan_id, squadron_name, clan_tag, members
)
summary["guilds_processed"] += 1
summary["total_added"] += added
summary["total_removed"] += removed
if added or removed:
logging.info(f"[META-SYNC] Guild {guild_id} ({squadron_name}): +{added} added, -{removed} removed")
except Exception as e:
logging.warning(f"[META-SYNC] Failed to sync guild {guild_id}: {e}")
summary["errors"].append(guild_id)
logging.info(
f"[META-SYNC] Done. {summary['guilds_processed']} guilds, "
f"+{summary['total_added']} added, -{summary['total_removed']} removed, "
f"{len(summary['errors'])} errors."
)
return summary
async def remove_player_from_guild_meta(guild_id: str, user_id: str) -> tuple[bool, str]:
"""
Remove a player from a guild's meta roster.
Args:
guild_id: Discord guild ID
user_id: Player UID
Returns:
Tuple of (success: bool, message: str)
"""
async with aiosqlite.connect(META_DB_PATH) as db:
# Get player nick first
cursor = await db.execute("""
SELECT nick FROM Guild_Metas
WHERE guild_id = ? AND userID = ?
""", (guild_id, user_id))
row = await cursor.fetchone()
if not row:
return False, f"Player `{user_id}` not found in your guild meta."
nick = row[0]
# Remove player
await db.execute("""
DELETE FROM Guild_Metas
WHERE guild_id = ? AND userID = ?
""", (guild_id, user_id))
await db.commit()
return True, f"Successfully removed **{nick}** from guild meta."
async def get_guild_meta_players(guild_id: str) -> List[dict]:
"""
Get all players in a guild's meta roster.
Args:
guild_id: Discord guild ID
Returns:
List of dicts, each with keys: ``userID``, ``nick``, ``clanTag``,
``clanName``, ``clanID``, ``added_unix``.
"""
async with aiosqlite.connect(META_DB_PATH) as db:
cursor = await db.execute("""
SELECT userID, nick, clanTag, clanName, clanID, added_unix
FROM Guild_Metas
WHERE guild_id = ?
ORDER BY nick ASC
""", (guild_id,))
rows = await cursor.fetchall()
return [
{
"userID": row[0],
"nick": row[1],
"clanTag": row[2],
"clanName": row[3],
"clanID": row[4],
"added_unix": row[5]
}
for row in rows
]
def _fuzzy_match(search_terms: List[str], target: str) -> bool:
"""
Check if all search terms appear in the target string.
This allows searches like "leopard 2a7" to match "Leopard 2A7V".
Args:
search_terms: List of lowercase search terms
target: Target string to match against
Returns:
True if all terms found in target
"""
target_lower = target.lower()
return all(term in target_lower for term in search_terms)
def _score_match(search_terms: List[str], target: str) -> int:
"""
Score how well search terms match a target.
Higher score = better match.
Args:
search_terms: List of lowercase search terms
target: Target string to match against
Returns:
Match score (0 = no match)
"""
target_lower = target.lower()
score = 0
for term in search_terms:
if term in target_lower:
score += 10
# Bonus for exact word match
if term in target_lower.split():
score += 5
# Bonus for start of word match
if target_lower.startswith(term) or f" {term}" in target_lower or f"-{term}" in target_lower:
score += 3
return score
async def get_guild_meta_vehicles(guild_id: str) -> List[dict]:
"""
Get all unique vehicles from players in a guild's meta roster.
Used for autocomplete suggestions.
Deduplicates by human-readable name to avoid showing the same display name
multiple times (e.g., different nation variants like germ_leopard_2a4 and
sw_leopard_2a4 both display as "Leopard 2A4").
Args:
guild_id: Discord guild ID
Returns:
List of dicts with vehicle_name (internal) and vehicle_human (readable)
"""
# Use dict keyed by human name to dedupe display duplicates
# Store first internal name encountered for each human name
vehicles_by_human = {}
async with aiosqlite.connect(META_DB_PATH) as db:
# Get all players in this guild's meta
cursor = await db.execute("""
SELECT userID FROM Guild_Metas WHERE guild_id = ?
""", (guild_id,))
guild_players = await cursor.fetchall()
for (user_id,) in guild_players:
cursor = await db.execute("""
SELECT vehicles FROM Players_Global WHERE userID = ?
""", (user_id,))
row = await cursor.fetchone()
if not row or not row[0]:
continue
try:
vehicles_array = decompress_json(row[0])
except (json.JSONDecodeError, TypeError, OSError):
continue
for vehicle in vehicles_array:
internal = vehicle.get("vehicle_name", "")
human = vehicle.get("vehicle_human", "") or internal
# Dedupe by human name - keep first internal name encountered
if human and human not in vehicles_by_human:
vehicles_by_human[human] = internal
return [{"vehicle_name": v, "vehicle_human": k} for k, v in vehicles_by_human.items()]
async def search_guild_meta_by_vehicle(guild_id: str, vehicle_name: str, exact_only: bool = False) -> List[dict]:
"""
Search for players in guild meta who have a specific vehicle.
Searches within the vehicles JSON array for each player.
Supports fuzzy matching on human-readable names:
- "KVT" matches "M1 KVT"
- "leopard 2a7" matches "Leopard 2A7V"
- "f16" matches "F-16C"
Args:
guild_id: Discord guild ID.
vehicle_name: Vehicle name (partial/fuzzy match supported).
exact_only: If True, only return exact internal-name matches.
Returns:
List of dicts with player info and vehicle stats.
"""
async with aiosqlite.connect(META_DB_PATH) as db:
# Get all players in this guild's meta
cursor = await db.execute("""
SELECT userID, nick, clanTag, clanName
FROM Guild_Metas
WHERE guild_id = ?
""", (guild_id,))
guild_players = await cursor.fetchall()
results = []
# Prepare search terms (split by spaces, remove empty)
search_input = vehicle_name.strip().lower()
search_terms = [t for t in search_input.split() if t]
# For each player, get their vehicles JSON from Players_Global
# First pass: try exact internal name match (from autocomplete)
# Second pass: fall back to fuzzy match on human-readable name only
exact_results = []
fuzzy_results = []
for user_id, nick, clan_tag, clan_name in guild_players:
cursor = await db.execute("""
SELECT vehicles
FROM Players_Global
WHERE userID = ?
""", (user_id,))
row = await cursor.fetchone()
if not row or not row[0]:
continue
# Parse vehicles array (may be gzip-compressed BLOB or legacy TEXT)
try:
vehicles_array = decompress_json(row[0])
except (json.JSONDecodeError, TypeError, OSError):
continue
# Search through vehicles array for matching vehicles
for vehicle in vehicles_array:
vehicle_name_internal = vehicle.get("vehicle_name", "")
vehicle_human = vehicle.get("vehicle_human", "") or ""
# Exact internal name match (highest priority)
if vehicle_name_internal.lower() == search_input:
match_score = 100
exact_results.append((user_id, nick, clan_tag, clan_name, vehicle, match_score))
continue
# Fuzzy matching on human-readable name only (not internal name)
# This prevents "so_4050_vautour_2n" from matching "so_4050_vautour_2n_late"
human_lower = vehicle_human.lower()
match_score = 0
if search_terms:
if _fuzzy_match(search_terms, human_lower):
match_score = _score_match(search_terms, human_lower)
if search_input in human_lower:
match_score = max(match_score, 15)
if match_score > 0:
fuzzy_results.append((user_id, nick, clan_tag, clan_name, vehicle, match_score))
# Use exact results if found, otherwise fall back to fuzzy (unless exact_only)
if exact_results:
matched = exact_results
elif exact_only:
matched = []
else:
matched = fuzzy_results
for user_id, nick, clan_tag, clan_name, vehicle, match_score in matched:
stats = vehicle.get("stats", {})
results.append({
"userID": user_id,
"nick": nick,
"clanTag": clan_tag,
"clanName": clan_name,
"intname": vehicle.get("vehicle_name", ""),
"vehicle_human": vehicle.get("vehicle_human", "") or "",
"mode": vehicle.get("mode", ""),
"flyouts": vehicle.get("flyouts", 0),
"was_in_session": vehicle.get("was_in_session", 0),
"deaths": stats.get("deaths", 0),
"ground_kills": stats.get("ground_kills", 0),
"air_kills": stats.get("air_kills", 0),
"_match_score": match_score
})
# Sort by match score (best matches first), then by player name
results.sort(key=lambda x: (-x.get("_match_score", 0), x.get("nick", "").lower()))
# Remove the internal match score before returning
for r in results:
r.pop("_match_score", None)
return results
async def get_all_player_uids() -> List[Tuple[str, str]]:
"""
Get all unique player UIDs and nicknames from sq_battles.db.
Returns:
List of tuples (UID, nick)
"""
if not SQ_BATTLES_DB_PATH.exists():
logging.error(f"❌ {SQ_BATTLES_DB_PATH} does not exist!")
return []
async with aiosqlite.connect(SQ_BATTLES_DB_PATH) as db:
cursor = await db.execute("""
SELECT DISTINCT UID, nick
FROM player_games_hist
WHERE UID IS NOT NULL AND UID != ''
ORDER BY UID
""")
rows = await cursor.fetchall()
result = [(str(row[0]), str(row[1])) for row in rows]
logging.info(f"📊 Found {len(result)} unique players in sq_battles.db")
return result
def translate_vehicle_name(vehicle_cdk: str) -> Optional[str]:
"""
Translate vehicle CDK (code name) to human-readable name.
Args:
vehicle_cdk: Internal vehicle code name
Returns:
Human-readable vehicle name or None if not found
"""
try:
raw = translate.get_translate(vehicle_cdk)
if raw:
return normalize_name(raw)
return None
except Exception:
return None
async def fetch_player_vehicles(uid: str, max_retries: int = 3) -> Optional[dict]:
"""
Fetch player vehicle data from Game API with retry logic.
Args:
uid: Player UID
max_retries: Maximum number of retry attempts for 503 errors
Returns:
Cleaned player data dict or None on failure
"""
last_error = None
for attempt in range(max_retries):
try:
data = await obtain_player_data_api(uid, raw=False)
# Check if token expired (empty dict returned)
if not data or not isinstance(data, dict):
return None
return data
except Exception as e:
last_error = e
error_msg = str(e)
# Check if it's a rate limit error (503)
if "503" in error_msg or "RETRY" in error_msg or "Network request failed" in error_msg:
if attempt < max_retries - 1:
wait_time = (attempt + 1) * 2 # Exponential backoff: 2s, 4s, 6s
# Silently retry - don't spam logs
await asyncio.sleep(wait_time)
continue
else:
# Only log after all retries exhausted
logging.info(f"⚠️ Rate limited UID {uid} after {max_retries} attempts")
return None
else:
# Non-rate-limit error - log it
logging.error(f"❌ Failed to fetch UID {uid}: {error_msg}")
return None
return None
async def insert_player_vehicles(db, player_data: dict, updated_unix: int):
"""
Insert or update player vehicle data in Meta.db.
Stores ONE ROW PER PLAYER with all vehicles as JSON array.
Args:
db: Database connection
player_data: Cleaned player data from Game API
updated_unix: Unix timestamp of when this data was fetched
Returns:
None. The row is upserted into Players_Global.
"""
user_id = player_data.get("userID", "")
nick = player_data.get("nick", "")
clan_tag = player_data.get("clanTag", "")
clan_name = player_data.get("clanName", "")
clan_id = player_data.get("clanID", "")
last_day = player_data.get("lastDay", 0)
vehicles_raw = player_data.get("vehicles", [])
if not user_id:
logging.warning("⚠️ Skipping player with no userID")
return
# Build vehicles array with translated names
vehicles_array = []
for vehicle in vehicles_raw:
vehicle_name = vehicle.get("name", "")
mode = vehicle.get("mode", "")
flyouts = vehicle.get("flyouts", 0)
was_in_session = vehicle.get("was_in_session", 0)
if not vehicle_name or not mode:
continue
# Translate vehicle name to human-readable
vehicle_human = translate_vehicle_name(vehicle_name)
# Extract stats
stats_dict = {k: v for k, v in vehicle.items()
if k not in ["name", "mode", "flyouts", "was_in_session"]}
vehicles_array.append({
"vehicle_name": vehicle_name,
"vehicle_human": vehicle_human,
"mode": mode,
"flyouts": flyouts,
"was_in_session": was_in_session,
"stats": stats_dict
})
# Compress vehicles array for BLOB storage
vehicles_json = compress_json(vehicles_array)
# Insert or update - ONE ROW PER PLAYER
await db.execute("""
INSERT INTO Players_Global
(userID, nick, clanTag, clanName, clanID, lastDay, vehicles, updated_unix)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(userID) DO UPDATE SET
nick = excluded.nick,
clanTag = excluded.clanTag,
clanName = excluded.clanName,
clanID = excluded.clanID,
lastDay = excluded.lastDay,
vehicles = excluded.vehicles,
updated_unix = excluded.updated_unix
""", (user_id, nick, clan_tag, clan_name, clan_id, last_day, vehicles_json, updated_unix))
async def process_player_batch(players_batch: List[Tuple[str, str]], meta_db, skip_existing: bool, semaphore):
"""
Process a batch of players concurrently.
Args:
players_batch: List of (uid, nick) tuples
meta_db: Database connection
skip_existing: If True, skip players with recent data
semaphore: Asyncio semaphore for rate limiting
Returns:
Dict with keys ``"success"``, ``"errors"``, and ``"skipped"``
containing integer counts for each outcome.
"""
results = {"success": 0, "errors": 0, "skipped": 0}
async def process_single_player(uid: str, nick: str):
async with semaphore:
try:
# Check if player was recently updated (within 2 days)
if skip_existing:
cursor = await meta_db.execute("""
SELECT updated_unix FROM Players_Global WHERE userID = ?
""", (uid,))
row = await cursor.fetchone()
if row and row[0]:
days_old = (time.time() - row[0]) / 86400
if days_old < 2:
return "skipped"
# Fetch player data
player_data = await fetch_player_vehicles(uid)
if player_data:
# Insert vehicle data
updated_unix = int(time.time())
await insert_player_vehicles(meta_db, player_data, updated_unix)
return "success"
else:
return "error"
except Exception as e:
logging.error(f"❌ Error processing UID {uid}: {e}")
return "error"
# Process all players in batch concurrently
tasks = [process_single_player(uid, nick) for uid, nick in players_batch]
batch_results = await asyncio.gather(*tasks, return_exceptions=True)
# Count results
for result in batch_results:
if isinstance(result, Exception):
results["errors"] += 1
elif result == "success":
results["success"] += 1
elif result == "skipped":
results["skipped"] += 1
elif result == "error":
results["errors"] += 1
return results
async def process_all_players(limit: Optional[int] = None, skip_existing: bool = True, batch_size: int = 20, max_concurrent: int = 5):
"""
Process all players and fetch their vehicle data.
Args:
limit: Optional limit on number of players to process
skip_existing: If True, skip players already in Meta.db with recent data (within 2 days)
batch_size: Number of players to process in each batch before committing
max_concurrent: Maximum number of concurrent API requests (default 5 to avoid rate limits)
Returns:
None. Results are committed to Meta.db and logged.
"""
# Initialize Meta.db
await init_meta_db()
# Get all player UIDs
players = await get_all_player_uids()
if limit:
players = players[:limit]
logging.info(f"📌 Processing limited to {limit} players")
total = len(players)
processed = 0
success = 0
errors = 0
skipped = 0
# Create semaphore for rate limiting
semaphore = asyncio.Semaphore(max_concurrent)
async with aiosqlite.connect(META_DB_PATH) as meta_db:
# Process in batches
for i in range(0, total, batch_size):
batch = players[i:i + batch_size]
batch_results = await process_player_batch(batch, meta_db, skip_existing, semaphore)
# Update counters
success += batch_results["success"]
errors += batch_results["errors"]
skipped += batch_results["skipped"]
processed += len(batch)
# Commit after each batch
await meta_db.commit()
# Small delay between batches to avoid overwhelming the API
await asyncio.sleep(1)
# Final commit
await meta_db.commit()
async def main():
"""CLI entry point for meta_manager.
Fetches and stores vehicle data for all players in sq_battles.db,
skipping recently-updated entries.
"""
# Process all players with concurrent batch processing
# max_concurrent=5 means up to 5 API requests running at once (reduced to avoid 503 errors)
# batch_size=50 means commit to DB every 50 players
await process_all_players(
limit=None, # No limit - process all players
skip_existing=True, # Skip players updated within 2 days
batch_size=50, # Commit every 50 players
max_concurrent=5 # 5 concurrent API requests (conservative to avoid rate limits)
)
if __name__ == "__main__":
asyncio.run(main())