ff420e131f
PR #1223 + fixup moved data_parser into BOTS/SHARED, but five BOT modules (analytics, autologging, botscript, lux_apis, meta_manager) still used `from .data_parser import ...`. That relative form looks inside the BOT package, which no longer contains data_parser, so the bot crashed at startup with ModuleNotFoundError. Add BOT/__init__.py to put BOTS/SHARED on sys.path at package import, then switch all five files to absolute `from data_parser import ...`. Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1181 lines
40 KiB
Python
1181 lines
40 KiB
Python
"""
|
|
Meta_Manager.py
|
|
|
|
Manages the Meta.db database which stores global player vehicle statistics.
|
|
Fetches vehicle data for all players from sq_battles.db and stores it in a structured format.
|
|
"""
|
|
|
|
# Standard Library Imports
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import time
|
|
from typing import Any, Dict, List, Tuple, Optional
|
|
|
|
# Third-Party Library Imports
|
|
import aiosqlite
|
|
|
|
# Local Module Imports
|
|
from data_parser import LangTableReader, normalize_name
|
|
from .game_api import obtain_clan_new_points, obtain_player_data_api
|
|
from .utils import STORAGE_DIR, SQUADRONS_DB_PATH, compress_json, decompress_json
|
|
|
|
|
|
STORAGE_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Database paths
|
|
SQ_BATTLES_DB_PATH = STORAGE_DIR / "sq_battles.db"
|
|
META_DB_PATH = STORAGE_DIR / "Meta.db"
|
|
|
|
# Language translator for vehicle names
|
|
translate = LangTableReader("English")
|
|
|
|
|
|
async def init_meta_db():
|
|
"""Initialize Meta.db with the Players_Global, Guilds, and Guild_Metas tables.
|
|
|
|
Also runs schema migrations (e.g. adding the ``password_hint`` column
|
|
to Guilds if missing). Idempotent — safe to call on every startup.
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
# Create Players_Global table - ONE ROW PER PLAYER with vehicles as JSON
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS Players_Global (
|
|
userID TEXT PRIMARY KEY,
|
|
nick TEXT NOT NULL,
|
|
clanTag TEXT,
|
|
clanName TEXT,
|
|
clanID TEXT,
|
|
lastDay INTEGER,
|
|
vehicles TEXT NOT NULL,
|
|
updated_unix INTEGER NOT NULL
|
|
)
|
|
""")
|
|
|
|
# Create index for efficient queries
|
|
await db.execute("""
|
|
CREATE INDEX IF NOT EXISTS idx_players_global_updated
|
|
ON Players_Global(updated_unix)
|
|
""")
|
|
|
|
# Create Guilds table for server-specific meta data access settings
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS Guilds (
|
|
guild_id TEXT PRIMARY KEY,
|
|
squadron_clanID TEXT NOT NULL,
|
|
squadron_name TEXT NOT NULL,
|
|
access_password TEXT NOT NULL,
|
|
require_password BOOLEAN DEFAULT 0,
|
|
lock_squadron_data BOOLEAN DEFAULT 0,
|
|
allow_public_meta BOOLEAN DEFAULT 0,
|
|
created_unix INTEGER NOT NULL,
|
|
updated_unix INTEGER NOT NULL
|
|
)
|
|
""")
|
|
|
|
# Create Guild_Metas table to store which players are in each guild's meta
|
|
await db.execute("""
|
|
CREATE TABLE IF NOT EXISTS Guild_Metas (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
guild_id TEXT NOT NULL,
|
|
userID TEXT NOT NULL,
|
|
nick TEXT NOT NULL,
|
|
clanTag TEXT,
|
|
clanName TEXT,
|
|
clanID TEXT,
|
|
added_unix INTEGER NOT NULL,
|
|
|
|
UNIQUE(guild_id, userID),
|
|
FOREIGN KEY(guild_id) REFERENCES Guilds(guild_id) ON DELETE CASCADE
|
|
)
|
|
""")
|
|
|
|
# Create index for efficient queries
|
|
await db.execute("""
|
|
CREATE INDEX IF NOT EXISTS idx_guild_metas_guild
|
|
ON Guild_Metas(guild_id)
|
|
""")
|
|
|
|
await db.execute("""
|
|
CREATE INDEX IF NOT EXISTS idx_guild_metas_userid
|
|
ON Guild_Metas(userID)
|
|
""")
|
|
|
|
# Migration: add password_hint column if it doesn't exist
|
|
cursor = await db.execute("PRAGMA table_info(Guilds)")
|
|
columns = [row[1] for row in await cursor.fetchall()]
|
|
if "password_hint" not in columns:
|
|
await db.execute("ALTER TABLE Guilds ADD COLUMN password_hint TEXT DEFAULT NULL")
|
|
|
|
await db.commit()
|
|
logging.info("✅ Meta.db initialized with Players_Global, Guilds, and Guild_Metas tables")
|
|
|
|
|
|
def generate_password_from_clanid(clan_id: str) -> str:
|
|
"""
|
|
Generate a secure password from clanID.
|
|
|
|
Args:
|
|
clan_id: The squadron's clan ID
|
|
|
|
Returns:
|
|
Generated password string
|
|
"""
|
|
# Create a hash of the clan_id and take first 12 characters
|
|
# Add some salt for uniqueness
|
|
salt = "SREBOT_META_ACCESS"
|
|
hash_input = f"{clan_id}_{salt}".encode('utf-8')
|
|
hash_digest = hashlib.sha256(hash_input).hexdigest()
|
|
# Take first 12 chars and format nicely
|
|
password = f"{hash_digest[:4]}-{hash_digest[4:8]}-{hash_digest[8:12]}"
|
|
result = password.upper()
|
|
logging.info(f"[PASSWORD GEN] clanID: '{clan_id}' -> password generated")
|
|
return result
|
|
|
|
|
|
async def get_guild_settings(guild_id: str) -> Optional[dict]:
|
|
"""
|
|
Get guild settings from Meta.db.
|
|
|
|
Args:
|
|
guild_id: Discord guild ID
|
|
|
|
Returns:
|
|
Dict with guild settings or None if not found
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
cursor = await db.execute("""
|
|
SELECT guild_id, squadron_clanID, squadron_name, access_password,
|
|
require_password, lock_squadron_data, allow_public_meta, created_unix, updated_unix,
|
|
password_hint
|
|
FROM Guilds
|
|
WHERE guild_id = ?
|
|
""", (guild_id,))
|
|
row = await cursor.fetchone()
|
|
|
|
if not row:
|
|
return None
|
|
|
|
return {
|
|
"guild_id": row[0],
|
|
"squadron_clanID": row[1],
|
|
"squadron_name": row[2],
|
|
"access_password": row[3],
|
|
"require_password": bool(row[4]),
|
|
"lock_squadron_data": bool(row[5]),
|
|
"allow_public_meta": bool(row[6]),
|
|
"created_unix": row[7],
|
|
"updated_unix": row[8],
|
|
"password_hint": row[9] or ""
|
|
}
|
|
|
|
|
|
async def get_squadron_owner(squadron_clanID: str) -> Optional[dict]:
|
|
"""
|
|
Check if this squadron is already claimed by any server.
|
|
|
|
Args:
|
|
squadron_clanID: Squadron clan ID
|
|
|
|
Returns:
|
|
Dict with owner guild settings or None if squadron not claimed
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
cursor = await db.execute("""
|
|
SELECT guild_id, squadron_clanID, squadron_name, access_password,
|
|
require_password, lock_squadron_data, allow_public_meta, created_unix, updated_unix,
|
|
password_hint
|
|
FROM Guilds
|
|
WHERE squadron_clanID = ?
|
|
ORDER BY created_unix ASC
|
|
LIMIT 1
|
|
""", (squadron_clanID,))
|
|
row = await cursor.fetchone()
|
|
|
|
if not row:
|
|
return None
|
|
|
|
return {
|
|
"guild_id": row[0],
|
|
"squadron_clanID": row[1],
|
|
"squadron_name": row[2],
|
|
"access_password": row[3],
|
|
"require_password": bool(row[4]),
|
|
"lock_squadron_data": bool(row[5]),
|
|
"allow_public_meta": bool(row[6]),
|
|
"created_unix": row[7],
|
|
"updated_unix": row[8],
|
|
"password_hint": row[9] or ""
|
|
}
|
|
|
|
|
|
async def create_or_update_guild(guild_id: str, squadron_clanID: str, squadron_name: str,
|
|
require_password: bool = False, lock_squadron_data: bool = False) -> str:
|
|
"""
|
|
Create or update guild settings in Meta.db.
|
|
|
|
Args:
|
|
guild_id: Discord guild ID
|
|
squadron_clanID: Squadron clan ID
|
|
squadron_name: Squadron name
|
|
require_password: Whether to require password for meta access
|
|
lock_squadron_data: Whether to lock squadron data to this server
|
|
|
|
Returns:
|
|
Access password for the guild
|
|
"""
|
|
# Generate password from clanID
|
|
password = generate_password_from_clanid(squadron_clanID)
|
|
current_time = int(time.time())
|
|
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
# Check if guild exists
|
|
existing = await get_guild_settings(guild_id)
|
|
|
|
if existing:
|
|
# Update existing (including new password for new squadron)
|
|
logging.info(f"[CREATE_OR_UPDATE] UPDATING guild {guild_id}: squadron_clanID {existing['squadron_clanID']} -> {squadron_clanID}")
|
|
await db.execute("""
|
|
UPDATE Guilds
|
|
SET squadron_clanID = ?,
|
|
squadron_name = ?,
|
|
access_password = ?,
|
|
require_password = ?,
|
|
lock_squadron_data = ?,
|
|
updated_unix = ?
|
|
WHERE guild_id = ?
|
|
""", (squadron_clanID, squadron_name, password, int(require_password),
|
|
int(lock_squadron_data), current_time, guild_id))
|
|
else:
|
|
# Create new
|
|
logging.info(f"[CREATE_OR_UPDATE] CREATING guild {guild_id}: squadron_clanID {squadron_clanID}")
|
|
await db.execute("""
|
|
INSERT INTO Guilds
|
|
(guild_id, squadron_clanID, squadron_name, access_password,
|
|
require_password, lock_squadron_data, created_unix, updated_unix)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (guild_id, squadron_clanID, squadron_name, password,
|
|
int(require_password), int(lock_squadron_data), current_time, current_time))
|
|
|
|
await db.commit()
|
|
|
|
return password
|
|
|
|
|
|
async def validate_squadron_password(squadron_clanID: str, provided_password: str) -> bool:
|
|
"""
|
|
Validate if provided password matches the squadron's password.
|
|
|
|
Args:
|
|
squadron_clanID: Squadron clan ID
|
|
provided_password: Password to validate
|
|
|
|
Returns:
|
|
True if password matches, False otherwise
|
|
"""
|
|
owner = await get_squadron_owner(squadron_clanID)
|
|
if not owner:
|
|
logging.warning(f"[PASSWORD VALIDATE] No owner found for squadron {squadron_clanID}")
|
|
return False
|
|
|
|
stored_password = owner["access_password"].upper()
|
|
provided_upper = provided_password.upper()
|
|
|
|
logging.info(f"[PASSWORD VALIDATE] squadron_clanID: {squadron_clanID}, match: {stored_password == provided_upper}")
|
|
|
|
return stored_password == provided_upper
|
|
|
|
|
|
async def update_squadron_password(squadron_clanID: str, new_password: str,
|
|
password_hint: Optional[str] = None) -> bool:
|
|
"""
|
|
Update the squadron's access password and optionally the password hint.
|
|
|
|
Args:
|
|
squadron_clanID: Squadron clan ID
|
|
new_password: The new password to set
|
|
password_hint: Optional hint to store (None = no change, "" = clear hint)
|
|
|
|
Returns:
|
|
True if successful, False otherwise
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
updates = ["access_password = ?", "updated_unix = ?"]
|
|
values = [new_password, int(time.time())]
|
|
|
|
if password_hint is not None:
|
|
updates.append("password_hint = ?")
|
|
values.append(password_hint if password_hint else None)
|
|
|
|
values.append(squadron_clanID)
|
|
|
|
await db.execute(f"""
|
|
UPDATE Guilds
|
|
SET {', '.join(updates)}
|
|
WHERE squadron_clanID = ?
|
|
""", tuple(values))
|
|
await db.commit()
|
|
|
|
logging.info(f"[PASSWORD UPDATE] Password updated for squadron {squadron_clanID}")
|
|
return True
|
|
|
|
|
|
async def transfer_squadron_to_guild(squadron_clanID: str, new_guild_id: str, squadron_name: str) -> bool:
|
|
"""
|
|
Transfer squadron ownership to a new guild (when they move servers).
|
|
|
|
Args:
|
|
squadron_clanID: Squadron clan ID
|
|
new_guild_id: New guild ID to transfer to
|
|
squadron_name: Squadron name
|
|
|
|
Returns:
|
|
True if successful, False otherwise
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
# Update the guild_id for this squadron
|
|
await db.execute("""
|
|
UPDATE Guilds
|
|
SET guild_id = ?,
|
|
squadron_name = ?,
|
|
updated_unix = ?
|
|
WHERE squadron_clanID = ?
|
|
""", (new_guild_id, squadron_name, int(time.time()), squadron_clanID))
|
|
await db.commit()
|
|
|
|
return True
|
|
|
|
|
|
async def update_guild_settings(guild_id: str, require_password: Optional[bool] = None,
|
|
lock_squadron_data: Optional[bool] = None,
|
|
allow_public_meta: Optional[bool] = None) -> bool:
|
|
"""
|
|
Update specific guild settings.
|
|
|
|
Args:
|
|
guild_id: Discord guild ID
|
|
require_password: Whether to require password (None = no change)
|
|
lock_squadron_data: Whether to lock data (None = no change)
|
|
allow_public_meta: Whether to allow non-admins to use /meta (None = no change)
|
|
|
|
Returns:
|
|
True if successful, False otherwise
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
updates = []
|
|
values = []
|
|
|
|
if require_password is not None:
|
|
updates.append("require_password = ?")
|
|
values.append(int(require_password))
|
|
|
|
if lock_squadron_data is not None:
|
|
updates.append("lock_squadron_data = ?")
|
|
values.append(int(lock_squadron_data))
|
|
|
|
if allow_public_meta is not None:
|
|
updates.append("allow_public_meta = ?")
|
|
values.append(int(allow_public_meta))
|
|
|
|
if not updates:
|
|
return False
|
|
|
|
updates.append("updated_unix = ?")
|
|
values.append(int(time.time()))
|
|
values.append(guild_id)
|
|
|
|
await db.execute(f"""
|
|
UPDATE Guilds
|
|
SET {', '.join(updates)}
|
|
WHERE guild_id = ?
|
|
""", tuple(values))
|
|
|
|
await db.commit()
|
|
|
|
return True
|
|
|
|
|
|
async def add_player_to_guild_meta(guild_id: str, user_id: str, squadron_clanID: str) -> tuple[bool, str]:
|
|
"""
|
|
Add a player to a guild's meta roster.
|
|
|
|
Args:
|
|
guild_id: Discord guild ID
|
|
user_id: Player UID
|
|
squadron_clanID: Squadron clan ID to validate against
|
|
|
|
Returns:
|
|
Tuple of (success: bool, message: str)
|
|
"""
|
|
# Get player data from Players_Global
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
cursor = await db.execute("""
|
|
SELECT DISTINCT userID, nick, clanTag, clanName, clanID
|
|
FROM Players_Global
|
|
WHERE userID = ?
|
|
LIMIT 1
|
|
""", (user_id,))
|
|
row = await cursor.fetchone()
|
|
|
|
if not row:
|
|
return False, f"Player `{user_id}` not found in Players_Global database."
|
|
|
|
player_uid, nick, clan_tag, clan_name, player_clan_id = row
|
|
|
|
# Validate that player belongs to the squadron
|
|
if player_clan_id != squadron_clanID:
|
|
return False, f"Player `{nick}` ({user_id}) does not belong to this squadron."
|
|
|
|
# Add to Guild_Metas
|
|
try:
|
|
await db.execute("""
|
|
INSERT INTO Guild_Metas (guild_id, userID, nick, clanTag, clanName, clanID, added_unix)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""", (guild_id, player_uid, nick, clan_tag, clan_name, player_clan_id, int(time.time())))
|
|
await db.commit()
|
|
return True, f"Successfully added **{nick}** to guild meta."
|
|
except Exception as e:
|
|
error_msg = str(e)
|
|
if "UNIQUE constraint" in error_msg:
|
|
return False, f"Player **{nick}** is already in your guild meta."
|
|
return False, f"Failed to add player: {error_msg}"
|
|
|
|
|
|
async def bulk_add_squadron_players_to_guild_meta(
|
|
guild_id: str,
|
|
squadron_clanID: str,
|
|
squadron_name: str,
|
|
clan_tag: str,
|
|
members: Dict[str, Dict[str, Any]],
|
|
) -> Tuple[int, int, int]:
|
|
"""
|
|
Sync all squadron players to a guild's meta roster.
|
|
Adds new members and removes players who are no longer in the squadron.
|
|
|
|
Args:
|
|
guild_id: Discord guild ID
|
|
squadron_clanID: Squadron clan ID
|
|
squadron_name: Squadron long name
|
|
clan_tag: Squadron tag name
|
|
members: Dict from obtain_clan_new_points(): ``{uid: {"nick": str, ...}}``.
|
|
Only the ``"nick"`` key is accessed per member.
|
|
|
|
Returns:
|
|
Tuple of (added_count, removed_count, skipped_count)
|
|
"""
|
|
now = int(time.time())
|
|
added = 0
|
|
removed = 0
|
|
skipped = 0
|
|
current_uids = set(members.keys())
|
|
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
# Insert new members
|
|
for uid, info in members.items():
|
|
nick = info.get("nick", "Unknown")
|
|
cursor = await db.execute("""
|
|
INSERT OR IGNORE INTO Guild_Metas
|
|
(guild_id, userID, nick, clanTag, clanName, clanID, added_unix)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""", (guild_id, uid, nick, clan_tag, squadron_name, squadron_clanID, now))
|
|
if cursor.rowcount > 0:
|
|
added += 1
|
|
else:
|
|
skipped += 1
|
|
|
|
# Remove members who are no longer in the squadron
|
|
cursor = await db.execute(
|
|
"SELECT userID FROM Guild_Metas WHERE guild_id = ?", (guild_id,)
|
|
)
|
|
existing_uids = {row[0] for row in await cursor.fetchall()}
|
|
departed_uids = existing_uids - current_uids
|
|
for uid in departed_uids:
|
|
await db.execute(
|
|
"DELETE FROM Guild_Metas WHERE guild_id = ? AND userID = ?",
|
|
(guild_id, uid)
|
|
)
|
|
removed += 1
|
|
|
|
await db.commit()
|
|
|
|
return added, removed, skipped
|
|
|
|
|
|
async def refresh_guild_player_vehicles(guild_id: str, max_concurrent: int = 5):
|
|
"""
|
|
Re-fetch vehicle data from the Game API for all players in a guild's meta roster.
|
|
Skips players updated within 2 days.
|
|
|
|
Args:
|
|
guild_id: Discord guild ID
|
|
max_concurrent: Maximum concurrent API requests
|
|
|
|
Returns:
|
|
None. Players_Global rows are updated in-place.
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
cursor = await db.execute(
|
|
"SELECT userID, nick FROM Guild_Metas WHERE guild_id = ?", (guild_id,)
|
|
)
|
|
players = [(row[0], row[1]) for row in await cursor.fetchall()]
|
|
|
|
if not players:
|
|
return
|
|
|
|
semaphore = asyncio.Semaphore(max_concurrent)
|
|
|
|
async with aiosqlite.connect(META_DB_PATH) as meta_db:
|
|
for i in range(0, len(players), 20):
|
|
batch = players[i:i + 20]
|
|
await process_player_batch(batch, meta_db, skip_existing=True, semaphore=semaphore)
|
|
await meta_db.commit()
|
|
|
|
logging.info(f"[META] Refreshed vehicle data for {len(players)} players in guild {guild_id}")
|
|
|
|
|
|
async def sync_all_guild_metas() -> Dict[str, Any]:
|
|
"""
|
|
Daily sync: for every guild in the Guilds table, fetch the current squadron
|
|
roster from the Game API and reconcile Guild_Metas (add new, remove departed).
|
|
|
|
Returns:
|
|
Summary dict with total added/removed counts and per-guild results.
|
|
"""
|
|
summary = {"guilds_processed": 0, "total_added": 0, "total_removed": 0, "errors": []}
|
|
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
cursor = await db.execute(
|
|
"SELECT guild_id, squadron_clanID, squadron_name FROM Guilds"
|
|
)
|
|
guilds = await cursor.fetchall()
|
|
|
|
for guild in guilds:
|
|
guild_id = guild["guild_id"]
|
|
clan_id = guild["squadron_clanID"]
|
|
squadron_name = guild["squadron_name"]
|
|
|
|
try:
|
|
members, _total_points = await obtain_clan_new_points(squadron_name)
|
|
except Exception as e:
|
|
logging.warning(f"[META-SYNC] Failed to fetch members for guild {guild_id} ({squadron_name}): {e}")
|
|
summary["errors"].append(guild_id)
|
|
continue
|
|
|
|
if not members:
|
|
logging.warning(f"[META-SYNC] No members returned for guild {guild_id} ({squadron_name}), skipping.")
|
|
continue
|
|
|
|
# Resolve clan_tag from squadrons.db
|
|
clan_tag = ""
|
|
try:
|
|
async with aiosqlite.connect(SQUADRONS_DB_PATH) as sq_db:
|
|
sq_cursor = await sq_db.execute(
|
|
"SELECT tag_name FROM squadrons_data WHERE clan_id = ? LIMIT 1",
|
|
(clan_id,)
|
|
)
|
|
row = await sq_cursor.fetchone()
|
|
if row:
|
|
clan_tag = row[0] or ""
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
added, removed, _skipped = await bulk_add_squadron_players_to_guild_meta(
|
|
guild_id, clan_id, squadron_name, clan_tag, members
|
|
)
|
|
summary["guilds_processed"] += 1
|
|
summary["total_added"] += added
|
|
summary["total_removed"] += removed
|
|
if added or removed:
|
|
logging.info(f"[META-SYNC] Guild {guild_id} ({squadron_name}): +{added} added, -{removed} removed")
|
|
except Exception as e:
|
|
logging.warning(f"[META-SYNC] Failed to sync guild {guild_id}: {e}")
|
|
summary["errors"].append(guild_id)
|
|
|
|
logging.info(
|
|
f"[META-SYNC] Done. {summary['guilds_processed']} guilds, "
|
|
f"+{summary['total_added']} added, -{summary['total_removed']} removed, "
|
|
f"{len(summary['errors'])} errors."
|
|
)
|
|
return summary
|
|
|
|
|
|
async def remove_player_from_guild_meta(guild_id: str, user_id: str) -> tuple[bool, str]:
|
|
"""
|
|
Remove a player from a guild's meta roster.
|
|
|
|
Args:
|
|
guild_id: Discord guild ID
|
|
user_id: Player UID
|
|
|
|
Returns:
|
|
Tuple of (success: bool, message: str)
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
# Get player nick first
|
|
cursor = await db.execute("""
|
|
SELECT nick FROM Guild_Metas
|
|
WHERE guild_id = ? AND userID = ?
|
|
""", (guild_id, user_id))
|
|
row = await cursor.fetchone()
|
|
|
|
if not row:
|
|
return False, f"Player `{user_id}` not found in your guild meta."
|
|
|
|
nick = row[0]
|
|
|
|
# Remove player
|
|
await db.execute("""
|
|
DELETE FROM Guild_Metas
|
|
WHERE guild_id = ? AND userID = ?
|
|
""", (guild_id, user_id))
|
|
await db.commit()
|
|
|
|
return True, f"Successfully removed **{nick}** from guild meta."
|
|
|
|
|
|
async def get_guild_meta_players(guild_id: str) -> List[dict]:
|
|
"""
|
|
Get all players in a guild's meta roster.
|
|
|
|
Args:
|
|
guild_id: Discord guild ID
|
|
|
|
Returns:
|
|
List of dicts, each with keys: ``userID``, ``nick``, ``clanTag``,
|
|
``clanName``, ``clanID``, ``added_unix``.
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
cursor = await db.execute("""
|
|
SELECT userID, nick, clanTag, clanName, clanID, added_unix
|
|
FROM Guild_Metas
|
|
WHERE guild_id = ?
|
|
ORDER BY nick ASC
|
|
""", (guild_id,))
|
|
rows = await cursor.fetchall()
|
|
|
|
return [
|
|
{
|
|
"userID": row[0],
|
|
"nick": row[1],
|
|
"clanTag": row[2],
|
|
"clanName": row[3],
|
|
"clanID": row[4],
|
|
"added_unix": row[5]
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
|
|
def _fuzzy_match(search_terms: List[str], target: str) -> bool:
|
|
"""
|
|
Check if all search terms appear in the target string.
|
|
This allows searches like "leopard 2a7" to match "Leopard 2A7V".
|
|
|
|
Args:
|
|
search_terms: List of lowercase search terms
|
|
target: Target string to match against
|
|
|
|
Returns:
|
|
True if all terms found in target
|
|
"""
|
|
target_lower = target.lower()
|
|
return all(term in target_lower for term in search_terms)
|
|
|
|
|
|
def _score_match(search_terms: List[str], target: str) -> int:
|
|
"""
|
|
Score how well search terms match a target.
|
|
Higher score = better match.
|
|
|
|
Args:
|
|
search_terms: List of lowercase search terms
|
|
target: Target string to match against
|
|
|
|
Returns:
|
|
Match score (0 = no match)
|
|
"""
|
|
target_lower = target.lower()
|
|
score = 0
|
|
|
|
for term in search_terms:
|
|
if term in target_lower:
|
|
score += 10
|
|
# Bonus for exact word match
|
|
if term in target_lower.split():
|
|
score += 5
|
|
# Bonus for start of word match
|
|
if target_lower.startswith(term) or f" {term}" in target_lower or f"-{term}" in target_lower:
|
|
score += 3
|
|
|
|
return score
|
|
|
|
|
|
async def get_guild_meta_vehicles(guild_id: str) -> List[dict]:
|
|
"""
|
|
Get all unique vehicles from players in a guild's meta roster.
|
|
Used for autocomplete suggestions.
|
|
|
|
Deduplicates by human-readable name to avoid showing the same display name
|
|
multiple times (e.g., different nation variants like germ_leopard_2a4 and
|
|
sw_leopard_2a4 both display as "Leopard 2A4").
|
|
|
|
Args:
|
|
guild_id: Discord guild ID
|
|
|
|
Returns:
|
|
List of dicts with vehicle_name (internal) and vehicle_human (readable)
|
|
"""
|
|
# Use dict keyed by human name to dedupe display duplicates
|
|
# Store first internal name encountered for each human name
|
|
vehicles_by_human = {}
|
|
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
# Get all players in this guild's meta
|
|
cursor = await db.execute("""
|
|
SELECT userID FROM Guild_Metas WHERE guild_id = ?
|
|
""", (guild_id,))
|
|
guild_players = await cursor.fetchall()
|
|
|
|
for (user_id,) in guild_players:
|
|
cursor = await db.execute("""
|
|
SELECT vehicles FROM Players_Global WHERE userID = ?
|
|
""", (user_id,))
|
|
row = await cursor.fetchone()
|
|
|
|
if not row or not row[0]:
|
|
continue
|
|
|
|
try:
|
|
vehicles_array = decompress_json(row[0])
|
|
except (json.JSONDecodeError, TypeError, OSError):
|
|
continue
|
|
|
|
for vehicle in vehicles_array:
|
|
internal = vehicle.get("vehicle_name", "")
|
|
human = vehicle.get("vehicle_human", "") or internal
|
|
# Dedupe by human name - keep first internal name encountered
|
|
if human and human not in vehicles_by_human:
|
|
vehicles_by_human[human] = internal
|
|
|
|
return [{"vehicle_name": v, "vehicle_human": k} for k, v in vehicles_by_human.items()]
|
|
|
|
|
|
async def search_guild_meta_by_vehicle(guild_id: str, vehicle_name: str, exact_only: bool = False) -> List[dict]:
|
|
"""
|
|
Search for players in guild meta who have a specific vehicle.
|
|
Searches within the vehicles JSON array for each player.
|
|
|
|
Supports fuzzy matching on human-readable names:
|
|
- "KVT" matches "M1 KVT"
|
|
- "leopard 2a7" matches "Leopard 2A7V"
|
|
- "f16" matches "F-16C"
|
|
|
|
Args:
|
|
guild_id: Discord guild ID.
|
|
vehicle_name: Vehicle name (partial/fuzzy match supported).
|
|
exact_only: If True, only return exact internal-name matches.
|
|
|
|
Returns:
|
|
List of dicts with player info and vehicle stats.
|
|
"""
|
|
async with aiosqlite.connect(META_DB_PATH) as db:
|
|
# Get all players in this guild's meta
|
|
cursor = await db.execute("""
|
|
SELECT userID, nick, clanTag, clanName
|
|
FROM Guild_Metas
|
|
WHERE guild_id = ?
|
|
""", (guild_id,))
|
|
guild_players = await cursor.fetchall()
|
|
|
|
results = []
|
|
|
|
# Prepare search terms (split by spaces, remove empty)
|
|
search_input = vehicle_name.strip().lower()
|
|
search_terms = [t for t in search_input.split() if t]
|
|
|
|
# For each player, get their vehicles JSON from Players_Global
|
|
# First pass: try exact internal name match (from autocomplete)
|
|
# Second pass: fall back to fuzzy match on human-readable name only
|
|
exact_results = []
|
|
fuzzy_results = []
|
|
|
|
for user_id, nick, clan_tag, clan_name in guild_players:
|
|
cursor = await db.execute("""
|
|
SELECT vehicles
|
|
FROM Players_Global
|
|
WHERE userID = ?
|
|
""", (user_id,))
|
|
row = await cursor.fetchone()
|
|
|
|
if not row or not row[0]:
|
|
continue
|
|
|
|
# Parse vehicles array (may be gzip-compressed BLOB or legacy TEXT)
|
|
try:
|
|
vehicles_array = decompress_json(row[0])
|
|
except (json.JSONDecodeError, TypeError, OSError):
|
|
continue
|
|
|
|
# Search through vehicles array for matching vehicles
|
|
for vehicle in vehicles_array:
|
|
vehicle_name_internal = vehicle.get("vehicle_name", "")
|
|
vehicle_human = vehicle.get("vehicle_human", "") or ""
|
|
|
|
# Exact internal name match (highest priority)
|
|
if vehicle_name_internal.lower() == search_input:
|
|
match_score = 100
|
|
exact_results.append((user_id, nick, clan_tag, clan_name, vehicle, match_score))
|
|
continue
|
|
|
|
# Fuzzy matching on human-readable name only (not internal name)
|
|
# This prevents "so_4050_vautour_2n" from matching "so_4050_vautour_2n_late"
|
|
human_lower = vehicle_human.lower()
|
|
|
|
match_score = 0
|
|
if search_terms:
|
|
if _fuzzy_match(search_terms, human_lower):
|
|
match_score = _score_match(search_terms, human_lower)
|
|
|
|
if search_input in human_lower:
|
|
match_score = max(match_score, 15)
|
|
|
|
if match_score > 0:
|
|
fuzzy_results.append((user_id, nick, clan_tag, clan_name, vehicle, match_score))
|
|
|
|
# Use exact results if found, otherwise fall back to fuzzy (unless exact_only)
|
|
if exact_results:
|
|
matched = exact_results
|
|
elif exact_only:
|
|
matched = []
|
|
else:
|
|
matched = fuzzy_results
|
|
|
|
for user_id, nick, clan_tag, clan_name, vehicle, match_score in matched:
|
|
stats = vehicle.get("stats", {})
|
|
|
|
results.append({
|
|
"userID": user_id,
|
|
"nick": nick,
|
|
"clanTag": clan_tag,
|
|
"clanName": clan_name,
|
|
"intname": vehicle.get("vehicle_name", ""),
|
|
"vehicle_human": vehicle.get("vehicle_human", "") or "",
|
|
"mode": vehicle.get("mode", ""),
|
|
"flyouts": vehicle.get("flyouts", 0),
|
|
"was_in_session": vehicle.get("was_in_session", 0),
|
|
"deaths": stats.get("deaths", 0),
|
|
"ground_kills": stats.get("ground_kills", 0),
|
|
"air_kills": stats.get("air_kills", 0),
|
|
"_match_score": match_score
|
|
})
|
|
|
|
# Sort by match score (best matches first), then by player name
|
|
results.sort(key=lambda x: (-x.get("_match_score", 0), x.get("nick", "").lower()))
|
|
|
|
# Remove the internal match score before returning
|
|
for r in results:
|
|
r.pop("_match_score", None)
|
|
|
|
return results
|
|
|
|
|
|
async def get_all_player_uids() -> List[Tuple[str, str]]:
|
|
"""
|
|
Get all unique player UIDs and nicknames from sq_battles.db.
|
|
|
|
Returns:
|
|
List of tuples (UID, nick)
|
|
"""
|
|
if not SQ_BATTLES_DB_PATH.exists():
|
|
logging.error(f"❌ {SQ_BATTLES_DB_PATH} does not exist!")
|
|
return []
|
|
|
|
async with aiosqlite.connect(SQ_BATTLES_DB_PATH) as db:
|
|
cursor = await db.execute("""
|
|
SELECT DISTINCT UID, nick
|
|
FROM player_games_hist
|
|
WHERE UID IS NOT NULL AND UID != ''
|
|
ORDER BY UID
|
|
""")
|
|
rows = await cursor.fetchall()
|
|
result = [(str(row[0]), str(row[1])) for row in rows]
|
|
logging.info(f"📊 Found {len(result)} unique players in sq_battles.db")
|
|
return result
|
|
|
|
|
|
def translate_vehicle_name(vehicle_cdk: str) -> Optional[str]:
|
|
"""
|
|
Translate vehicle CDK (code name) to human-readable name.
|
|
|
|
Args:
|
|
vehicle_cdk: Internal vehicle code name
|
|
|
|
Returns:
|
|
Human-readable vehicle name or None if not found
|
|
"""
|
|
try:
|
|
raw = translate.get_translate(vehicle_cdk)
|
|
if raw:
|
|
return normalize_name(raw)
|
|
return None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
async def fetch_player_vehicles(uid: str, max_retries: int = 3) -> Optional[dict]:
|
|
"""
|
|
Fetch player vehicle data from Game API with retry logic.
|
|
|
|
Args:
|
|
uid: Player UID
|
|
max_retries: Maximum number of retry attempts for 503 errors
|
|
|
|
Returns:
|
|
Cleaned player data dict or None on failure
|
|
"""
|
|
last_error = None
|
|
|
|
for attempt in range(max_retries):
|
|
try:
|
|
data = await obtain_player_data_api(uid, raw=False)
|
|
|
|
# Check if token expired (empty dict returned)
|
|
if not data or not isinstance(data, dict):
|
|
return None
|
|
|
|
return data
|
|
except Exception as e:
|
|
last_error = e
|
|
error_msg = str(e)
|
|
# Check if it's a rate limit error (503)
|
|
if "503" in error_msg or "RETRY" in error_msg or "Network request failed" in error_msg:
|
|
if attempt < max_retries - 1:
|
|
wait_time = (attempt + 1) * 2 # Exponential backoff: 2s, 4s, 6s
|
|
# Silently retry - don't spam logs
|
|
await asyncio.sleep(wait_time)
|
|
continue
|
|
else:
|
|
# Only log after all retries exhausted
|
|
logging.info(f"⚠️ Rate limited UID {uid} after {max_retries} attempts")
|
|
return None
|
|
else:
|
|
# Non-rate-limit error - log it
|
|
logging.error(f"❌ Failed to fetch UID {uid}: {error_msg}")
|
|
return None
|
|
|
|
return None
|
|
|
|
|
|
async def insert_player_vehicles(db, player_data: dict, updated_unix: int):
|
|
"""
|
|
Insert or update player vehicle data in Meta.db.
|
|
Stores ONE ROW PER PLAYER with all vehicles as JSON array.
|
|
|
|
Args:
|
|
db: Database connection
|
|
player_data: Cleaned player data from Game API
|
|
updated_unix: Unix timestamp of when this data was fetched
|
|
|
|
Returns:
|
|
None. The row is upserted into Players_Global.
|
|
"""
|
|
user_id = player_data.get("userID", "")
|
|
nick = player_data.get("nick", "")
|
|
clan_tag = player_data.get("clanTag", "")
|
|
clan_name = player_data.get("clanName", "")
|
|
clan_id = player_data.get("clanID", "")
|
|
last_day = player_data.get("lastDay", 0)
|
|
vehicles_raw = player_data.get("vehicles", [])
|
|
|
|
if not user_id:
|
|
logging.warning("⚠️ Skipping player with no userID")
|
|
return
|
|
|
|
# Build vehicles array with translated names
|
|
vehicles_array = []
|
|
for vehicle in vehicles_raw:
|
|
vehicle_name = vehicle.get("name", "")
|
|
mode = vehicle.get("mode", "")
|
|
flyouts = vehicle.get("flyouts", 0)
|
|
was_in_session = vehicle.get("was_in_session", 0)
|
|
|
|
if not vehicle_name or not mode:
|
|
continue
|
|
|
|
# Translate vehicle name to human-readable
|
|
vehicle_human = translate_vehicle_name(vehicle_name)
|
|
|
|
# Extract stats
|
|
stats_dict = {k: v for k, v in vehicle.items()
|
|
if k not in ["name", "mode", "flyouts", "was_in_session"]}
|
|
|
|
vehicles_array.append({
|
|
"vehicle_name": vehicle_name,
|
|
"vehicle_human": vehicle_human,
|
|
"mode": mode,
|
|
"flyouts": flyouts,
|
|
"was_in_session": was_in_session,
|
|
"stats": stats_dict
|
|
})
|
|
|
|
# Compress vehicles array for BLOB storage
|
|
vehicles_json = compress_json(vehicles_array)
|
|
|
|
# Insert or update - ONE ROW PER PLAYER
|
|
await db.execute("""
|
|
INSERT INTO Players_Global
|
|
(userID, nick, clanTag, clanName, clanID, lastDay, vehicles, updated_unix)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(userID) DO UPDATE SET
|
|
nick = excluded.nick,
|
|
clanTag = excluded.clanTag,
|
|
clanName = excluded.clanName,
|
|
clanID = excluded.clanID,
|
|
lastDay = excluded.lastDay,
|
|
vehicles = excluded.vehicles,
|
|
updated_unix = excluded.updated_unix
|
|
""", (user_id, nick, clan_tag, clan_name, clan_id, last_day, vehicles_json, updated_unix))
|
|
|
|
|
|
async def process_player_batch(players_batch: List[Tuple[str, str]], meta_db, skip_existing: bool, semaphore):
|
|
"""
|
|
Process a batch of players concurrently.
|
|
|
|
Args:
|
|
players_batch: List of (uid, nick) tuples
|
|
meta_db: Database connection
|
|
skip_existing: If True, skip players with recent data
|
|
semaphore: Asyncio semaphore for rate limiting
|
|
|
|
Returns:
|
|
Dict with keys ``"success"``, ``"errors"``, and ``"skipped"``
|
|
containing integer counts for each outcome.
|
|
"""
|
|
results = {"success": 0, "errors": 0, "skipped": 0}
|
|
|
|
async def process_single_player(uid: str, nick: str):
|
|
async with semaphore:
|
|
try:
|
|
# Check if player was recently updated (within 2 days)
|
|
if skip_existing:
|
|
cursor = await meta_db.execute("""
|
|
SELECT updated_unix FROM Players_Global WHERE userID = ?
|
|
""", (uid,))
|
|
row = await cursor.fetchone()
|
|
if row and row[0]:
|
|
days_old = (time.time() - row[0]) / 86400
|
|
if days_old < 2:
|
|
return "skipped"
|
|
|
|
# Fetch player data
|
|
player_data = await fetch_player_vehicles(uid)
|
|
|
|
if player_data:
|
|
# Insert vehicle data
|
|
updated_unix = int(time.time())
|
|
await insert_player_vehicles(meta_db, player_data, updated_unix)
|
|
return "success"
|
|
else:
|
|
return "error"
|
|
|
|
except Exception as e:
|
|
logging.error(f"❌ Error processing UID {uid}: {e}")
|
|
return "error"
|
|
|
|
# Process all players in batch concurrently
|
|
tasks = [process_single_player(uid, nick) for uid, nick in players_batch]
|
|
batch_results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
# Count results
|
|
for result in batch_results:
|
|
if isinstance(result, Exception):
|
|
results["errors"] += 1
|
|
elif result == "success":
|
|
results["success"] += 1
|
|
elif result == "skipped":
|
|
results["skipped"] += 1
|
|
elif result == "error":
|
|
results["errors"] += 1
|
|
|
|
return results
|
|
|
|
|
|
async def process_all_players(limit: Optional[int] = None, skip_existing: bool = True, batch_size: int = 20, max_concurrent: int = 5):
|
|
"""
|
|
Process all players and fetch their vehicle data.
|
|
|
|
Args:
|
|
limit: Optional limit on number of players to process
|
|
skip_existing: If True, skip players already in Meta.db with recent data (within 2 days)
|
|
batch_size: Number of players to process in each batch before committing
|
|
max_concurrent: Maximum number of concurrent API requests (default 5 to avoid rate limits)
|
|
|
|
Returns:
|
|
None. Results are committed to Meta.db and logged.
|
|
"""
|
|
# Initialize Meta.db
|
|
await init_meta_db()
|
|
|
|
# Get all player UIDs
|
|
players = await get_all_player_uids()
|
|
|
|
if limit:
|
|
players = players[:limit]
|
|
logging.info(f"📌 Processing limited to {limit} players")
|
|
|
|
total = len(players)
|
|
processed = 0
|
|
success = 0
|
|
errors = 0
|
|
skipped = 0
|
|
|
|
# Create semaphore for rate limiting
|
|
semaphore = asyncio.Semaphore(max_concurrent)
|
|
|
|
async with aiosqlite.connect(META_DB_PATH) as meta_db:
|
|
# Process in batches
|
|
for i in range(0, total, batch_size):
|
|
batch = players[i:i + batch_size]
|
|
batch_results = await process_player_batch(batch, meta_db, skip_existing, semaphore)
|
|
|
|
# Update counters
|
|
success += batch_results["success"]
|
|
errors += batch_results["errors"]
|
|
skipped += batch_results["skipped"]
|
|
processed += len(batch)
|
|
|
|
# Commit after each batch
|
|
await meta_db.commit()
|
|
|
|
|
|
# Small delay between batches to avoid overwhelming the API
|
|
await asyncio.sleep(1)
|
|
|
|
# Final commit
|
|
await meta_db.commit()
|
|
|
|
|
|
|
|
async def main():
|
|
"""CLI entry point for meta_manager.
|
|
|
|
Fetches and stores vehicle data for all players in sq_battles.db,
|
|
skipping recently-updated entries.
|
|
"""
|
|
# Process all players with concurrent batch processing
|
|
# max_concurrent=5 means up to 5 API requests running at once (reduced to avoid 503 errors)
|
|
# batch_size=50 means commit to DB every 50 players
|
|
await process_all_players(
|
|
limit=None, # No limit - process all players
|
|
skip_existing=True, # Skip players updated within 2 days
|
|
batch_size=50, # Commit every 50 players
|
|
max_concurrent=5 # 5 concurrent API requests (conservative to avoid rate limits)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|