add SREBOT, SHARED, TSSBOT contents (fixup for #1223)
PR #1223 only staged the deletions of the old paths because the new top-level directories were still untracked when the commit was authored. This commit adds the actual restructured tree: SREBOT/ (existing bot), SHARED/ (vromfs, data_parser, ICONS/MAPS/FONTS, DAGOR_FILES, update_game_files), and TSSBOT/ (skeleton). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
FURRO404
@@ -0,0 +1,170 @@
|
||||
"""
|
||||
Diagnostic: query Discord's entitlements API directly and report what it returns
|
||||
right now. Compare against the local entitlements.db cache.
|
||||
|
||||
Usage (on the server):
|
||||
cd ~/GitHub/SREBOT_MEOW
|
||||
source .venv/bin/activate
|
||||
python scripts/diag_entitlements.py
|
||||
|
||||
Targeted guild: 1379510072815779961 (bot owner's max-tier server).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
APPLICATION_ID = "1254679514466877540"
|
||||
TARGET_GUILD = "1379510072815779961"
|
||||
TOKEN = os.environ.get("DISCORD_KEY", "")
|
||||
_storage_env = os.environ.get("SREBOT_STORAGE_VOL_PATH", "").strip()
|
||||
if not _storage_env:
|
||||
raise RuntimeError("SREBOT_STORAGE_VOL_PATH must be set")
|
||||
STORAGE = Path(_storage_env)
|
||||
DB_PATH = STORAGE / "entitlements.db"
|
||||
|
||||
|
||||
async def fetch_all_entitlements() -> tuple[list[dict], list[tuple[int, int, str, str]]]:
|
||||
"""
|
||||
Returns (entitlements, page_logs).
|
||||
page_logs: list of (page_num, count, status, after_cursor)
|
||||
"""
|
||||
if not TOKEN:
|
||||
print("ERROR: DISCORD_KEY env var not set", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
headers = {"Authorization": f"Bot {TOKEN}"}
|
||||
url = f"https://discord.com/api/v10/applications/{APPLICATION_ID}/entitlements"
|
||||
params: dict[str, str] = {"exclude_ended": "true", "limit": "100"}
|
||||
|
||||
all_ents: list[dict] = []
|
||||
page_logs: list[tuple[int, int, str, str]] = []
|
||||
page = 0
|
||||
after: str | None = None
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=60)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
while True:
|
||||
page += 1
|
||||
q = dict(params)
|
||||
if after:
|
||||
q["after"] = after
|
||||
t0 = time.monotonic()
|
||||
async with session.get(url, headers=headers, params=q) as resp:
|
||||
elapsed = time.monotonic() - t0
|
||||
status = f"{resp.status} ({elapsed:.2f}s)"
|
||||
if resp.status != 200:
|
||||
body = await resp.text()
|
||||
page_logs.append((page, 0, status, after or ""))
|
||||
print(f"[!] page {page} failed: {status}\n {body[:300]}")
|
||||
break
|
||||
batch = await resp.json()
|
||||
page_logs.append((page, len(batch), status, after or ""))
|
||||
all_ents.extend(batch)
|
||||
if len(batch) < 100:
|
||||
break
|
||||
# paginate forward by id
|
||||
after = batch[-1]["id"]
|
||||
# safety break
|
||||
if page > 50:
|
||||
print("[!] hit 50-page safety limit, stopping")
|
||||
break
|
||||
|
||||
return all_ents, page_logs
|
||||
|
||||
|
||||
def read_db_state() -> dict[str, int]:
|
||||
if not DB_PATH.exists():
|
||||
return {"db_missing": 1}
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
out: dict[str, int] = {}
|
||||
out["guild_entitlements_total"] = cur.execute("SELECT COUNT(*) FROM guild_entitlements").fetchone()[0]
|
||||
out["guild_entitlements_active"] = cur.execute("SELECT COUNT(*) FROM guild_entitlements WHERE status='active'").fetchone()[0]
|
||||
out["manual_entitlements_total"] = cur.execute("SELECT COUNT(*) FROM manual_entitlements").fetchone()[0]
|
||||
out["discord_entitlements_total"] = cur.execute("SELECT COUNT(*) FROM discord_entitlements").fetchone()[0]
|
||||
row = cur.execute("SELECT * FROM discord_entitlements WHERE guild_id=?", (TARGET_GUILD,)).fetchone()
|
||||
out["target_in_discord_table"] = 1 if row else 0
|
||||
return out
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
print("=" * 70)
|
||||
print(f"Discord Entitlements Diagnostic — {time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime())}")
|
||||
print("=" * 70)
|
||||
|
||||
print(f"\n[1] Local DB state ({DB_PATH}):")
|
||||
for k, v in read_db_state().items():
|
||||
print(f" {k}: {v}")
|
||||
|
||||
print("\n[2] Discord API: GET /applications/{app_id}/entitlements?exclude_ended=true")
|
||||
ents, pages = await fetch_all_entitlements()
|
||||
|
||||
print("\n Page log:")
|
||||
for p, n, status, after in pages:
|
||||
print(f" page {p:>2} count={n:>3} status={status:<18} after={after}")
|
||||
|
||||
print(f"\n Total entitlements returned: {len(ents)}")
|
||||
|
||||
# Group by guild
|
||||
by_guild: dict[str, list[dict]] = {}
|
||||
for e in ents:
|
||||
gid = e.get("guild_id")
|
||||
if gid:
|
||||
by_guild.setdefault(str(gid), []).append(e)
|
||||
|
||||
print(f" Unique guild_ids: {len(by_guild)}")
|
||||
print(f" Entitlements with NULL guild_id: {sum(1 for e in ents if not e.get('guild_id'))}")
|
||||
|
||||
# Status breakdown
|
||||
deleted = sum(1 for e in ents if e.get("deleted"))
|
||||
consumed = sum(1 for e in ents if e.get("consumed"))
|
||||
starts_in_future = sum(1 for e in ents if e.get("starts_at") and e["starts_at"] > time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()))
|
||||
ends_in_past = sum(1 for e in ents if e.get("ends_at") and e["ends_at"] < time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()))
|
||||
print(f" deleted={deleted} consumed={consumed} starts_in_future={starts_in_future} ends_in_past={ends_in_past}")
|
||||
|
||||
print(f"\n[3] Target guild {TARGET_GUILD}:")
|
||||
target_ents = by_guild.get(TARGET_GUILD, [])
|
||||
if not target_ents:
|
||||
print(f" *** NOT FOUND in Discord API response ***")
|
||||
# Show sample of what we DID get to confirm we're not just paginating wrong
|
||||
sample_guilds = list(by_guild.keys())[:5]
|
||||
print(f" Sample of guild_ids returned: {sample_guilds}")
|
||||
else:
|
||||
for e in target_ents:
|
||||
print(f" id={e.get('id')} sku_id={e.get('sku_id')} type={e.get('type')} "
|
||||
f"deleted={e.get('deleted')} starts_at={e.get('starts_at')} ends_at={e.get('ends_at')}")
|
||||
|
||||
print("\n[4] SKU breakdown across all returned entitlements:")
|
||||
sku_counts: dict[str, int] = {}
|
||||
for e in ents:
|
||||
sku = str(e.get("sku_id") or "")
|
||||
sku_counts[sku] = sku_counts.get(sku, 0) + 1
|
||||
for sku, c in sorted(sku_counts.items(), key=lambda x: -x[1]):
|
||||
print(f" {sku}: {c}")
|
||||
|
||||
print("\n[5] Recommendation:")
|
||||
if not target_ents:
|
||||
print(" Discord is NOT returning the target guild's entitlement.")
|
||||
print(" Either Discord's API is degraded right now, OR the entitlement no longer exists.")
|
||||
print(" Re-run this script in a few minutes to see if results change.")
|
||||
else:
|
||||
print(" Discord IS returning the target guild's entitlement.")
|
||||
print(" The bot's cache must be stale or wholesale-replaced by an empty result.")
|
||||
print(" Force a refresh: restart bot OR call refresh_entitled_guilds(force=True).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
Diagnostic: figure out why /meta shows no points per player.
|
||||
|
||||
Compares Guild_Metas userIDs against the uid keys returned by
|
||||
obtain_clan_new_points() for the guild's bound squadron, and reports each
|
||||
plausible failure mode (missing Guilds row, empty API response, uid format
|
||||
mismatch, squadron-name mismatch).
|
||||
|
||||
Usage (on the server):
|
||||
cd ~/SREBOT/SREBOT_MEOW
|
||||
source .venv/bin/activate
|
||||
python scripts/diag_meta_points.py 1378960248118841507
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import aiosqlite
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from BOT.game_api import ClanInfoError, obtain_clan_new_points
|
||||
from BOT.utils import STORAGE_DIR
|
||||
|
||||
|
||||
async def diag(guild_id: str) -> None:
|
||||
meta_db = STORAGE_DIR / "Meta.db"
|
||||
print(f"== Meta.db: {meta_db} (exists={meta_db.exists()})")
|
||||
|
||||
if not meta_db.exists():
|
||||
print("Meta.db missing — wrong STORAGE_DIR?")
|
||||
return
|
||||
|
||||
async with aiosqlite.connect(meta_db) as db:
|
||||
cur = await db.execute(
|
||||
"SELECT squadron_name, squadron_clanID FROM Guilds WHERE guild_id=?",
|
||||
(guild_id,),
|
||||
)
|
||||
guilds_row = await cur.fetchone()
|
||||
print(f"== Guilds row for {guild_id}: {guilds_row}")
|
||||
|
||||
cur = await db.execute(
|
||||
"SELECT COUNT(*) FROM Guild_Metas WHERE guild_id=?", (guild_id,)
|
||||
)
|
||||
gm_count_row = await cur.fetchone()
|
||||
gm_count = gm_count_row[0] if gm_count_row else 0
|
||||
print(f"== Guild_Metas count for {guild_id}: {gm_count}")
|
||||
|
||||
cur = await db.execute(
|
||||
"SELECT userID, nick, clanName FROM Guild_Metas "
|
||||
"WHERE guild_id=? ORDER BY nick LIMIT 5",
|
||||
(guild_id,),
|
||||
)
|
||||
gm_rows = await cur.fetchall()
|
||||
print("== Guild_Metas sample:")
|
||||
for uid, nick, clan in gm_rows:
|
||||
print(f" uid={uid!r} type={type(uid).__name__} nick={nick!r} clan={clan!r}")
|
||||
|
||||
if not guilds_row:
|
||||
print("\nNo Guilds row — /meta-management was never run for this guild.")
|
||||
return
|
||||
|
||||
sq_name = guilds_row[0]
|
||||
print(f"\n== Calling obtain_clan_new_points({sq_name!r})...")
|
||||
try:
|
||||
members, total = await obtain_clan_new_points(sq_name)
|
||||
except ClanInfoError as e:
|
||||
print(f" ClanInfoError: {e}")
|
||||
return
|
||||
except Exception as e:
|
||||
print(f" {type(e).__name__}: {e}")
|
||||
return
|
||||
|
||||
print(f"== API returned: {len(members)} members, total_score={total}")
|
||||
if not members:
|
||||
print(" Empty members dict — JWT was likely refreshed; rerun the script.")
|
||||
return
|
||||
|
||||
sample = list(members.items())[:3]
|
||||
print("== API sample:")
|
||||
for uid, info in sample:
|
||||
print(f" uid={uid!r} type={type(uid).__name__} info={info}")
|
||||
|
||||
gm_rows_list = list(gm_rows)
|
||||
if gm_rows_list:
|
||||
gm_uid = gm_rows_list[0][0]
|
||||
print(
|
||||
f"\n== Membership test for first Guild_Metas uid {gm_uid!r}:\n"
|
||||
f" raw in api_map -> {gm_uid in members}\n"
|
||||
f" str(uid) in api_map -> {str(gm_uid) in members}"
|
||||
)
|
||||
|
||||
api_uids = set(members.keys())
|
||||
gm_uids_all = set()
|
||||
async with aiosqlite.connect(meta_db) as db:
|
||||
cur = await db.execute(
|
||||
"SELECT userID FROM Guild_Metas WHERE guild_id=?", (guild_id,)
|
||||
)
|
||||
gm_uids_all = {str(r[0]) for r in await cur.fetchall()}
|
||||
api_uids_str = {str(u) for u in api_uids}
|
||||
overlap = gm_uids_all & api_uids_str
|
||||
only_gm = gm_uids_all - api_uids_str
|
||||
only_api = api_uids_str - gm_uids_all
|
||||
print(
|
||||
f"\n== Set overlap (str-cast both sides):\n"
|
||||
f" in both: {len(overlap)}\n"
|
||||
f" only in Guild_Metas: {len(only_gm)} (left squadron / wrong sq_name?)\n"
|
||||
f" only in API: {len(only_api)} (never bulk-added)"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: python scripts/diag_meta_points.py <guild_id>")
|
||||
sys.exit(2)
|
||||
asyncio.run(diag(sys.argv[1]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe the War Thunder clan leaderboard for a specific squadron.
|
||||
|
||||
This mirrors the bot's ``obtain_clans_leaderboard()`` flow:
|
||||
1. Load the JWT from the storage auth file.
|
||||
2. Call the clan leaderboard endpoint with ``action=cln_clan_get_leaderboard``.
|
||||
3. Decode the BLK payload.
|
||||
4. Print leaderboard counts and the first matching squadron entry.
|
||||
|
||||
Usage:
|
||||
python3 scripts/leaderboard_score_probe.py
|
||||
python3 scripts/leaderboard_score_probe.py --needle SCORE --count 1000
|
||||
python3 scripts/leaderboard_score_probe.py --auth-file /mnt/.../STORAGE/auth_JWT.json --char-url "$WT_CHAR_URL"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
def _infer_storage_root(auth_file: Path) -> Path:
|
||||
if auth_file.parent.name == "AUTH":
|
||||
return auth_file.parent.parent
|
||||
return auth_file.parent
|
||||
|
||||
|
||||
def _prepare_env(storage_root: Path) -> None:
|
||||
os.environ["SREBOT_STORAGE_VOL_PATH"] = str(storage_root)
|
||||
|
||||
|
||||
def _score_hit(clan: dict[str, Any], needle: str) -> bool:
|
||||
needle_u = needle.upper()
|
||||
values = (
|
||||
str(clan.get("short_name", "")).upper(),
|
||||
str(clan.get("tag", "")).upper(),
|
||||
str(clan.get("long_name", "")).upper(),
|
||||
)
|
||||
return any(v == needle_u or needle_u in v for v in values)
|
||||
|
||||
|
||||
async def _main() -> int:
|
||||
repo_root = Path(__file__).resolve().parents[1]
|
||||
env_path = repo_root / ".env"
|
||||
if str(repo_root) not in sys.path:
|
||||
sys.path.insert(0, str(repo_root))
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--needle", default="SCORE", help="Squadron name to search for")
|
||||
parser.add_argument("--start", type=int, default=0, help="Leaderboard offset")
|
||||
parser.add_argument("--count", type=int, default=1000, help="Leaderboard page size")
|
||||
parser.add_argument(
|
||||
"--auth-file",
|
||||
default="",
|
||||
help="Path to the JWT auth JSON file. If omitted, the script will use a temp file under the storage root.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--char-url",
|
||||
default="",
|
||||
help="War Thunder char API URL (defaults to WT_CHAR_URL env var)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.auth_file:
|
||||
auth_file = Path(args.auth_file).expanduser().resolve()
|
||||
storage_root = _infer_storage_root(auth_file)
|
||||
else:
|
||||
storage_root_env = os.environ.get("SREBOT_STORAGE_VOL_PATH", "").strip()
|
||||
if not storage_root_env:
|
||||
print(
|
||||
"SREBOT_STORAGE_VOL_PATH is not set. Export it or pass --auth-file.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
storage_root = Path(storage_root_env)
|
||||
auth_file = storage_root / "AUTH" / "auth_JWT.json"
|
||||
|
||||
_prepare_env(storage_root)
|
||||
auth_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
load_dotenv(dotenv_path=env_path)
|
||||
if not args.char_url:
|
||||
args.char_url = os.environ.get("WT_CHAR_URL", "")
|
||||
|
||||
if args.char_url:
|
||||
os.environ["WT_CHAR_URL"] = args.char_url
|
||||
elif not os.environ.get("WT_CHAR_URL"):
|
||||
print(
|
||||
"WT_CHAR_URL is not set. Pass --char-url or export WT_CHAR_URL first.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
# Import after env setup so BOT.game_api picks up the correct paths.
|
||||
from BOT import game_api # type: ignore
|
||||
import aiohttp
|
||||
|
||||
game_api.AUTH_FILE = auth_file
|
||||
bin_blk_to_json = game_api.bin_blk_to_json
|
||||
|
||||
if not auth_file.exists():
|
||||
await game_api.get_JWT()
|
||||
|
||||
with auth_file.open("r", encoding="utf-8") as f:
|
||||
jwt = json.load(f)["jwt"]
|
||||
|
||||
headers = {
|
||||
"action": "cln_clan_get_leaderboard",
|
||||
"token": jwt,
|
||||
"start": str(args.start),
|
||||
"count": str(args.count),
|
||||
"sortField": "dr_era5_hist",
|
||||
"shortMode": "off",
|
||||
}
|
||||
|
||||
url = os.environ["WT_CHAR_URL"]
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(url, headers=headers) as res:
|
||||
raw = await res.read()
|
||||
print(f"http_status={res.status}")
|
||||
if res.status != 200:
|
||||
print(raw[:1000].decode("utf-8", "replace"), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
data = await bin_blk_to_json(raw)
|
||||
if "root" in data:
|
||||
data = data["root"]
|
||||
|
||||
clans = data.get("clan") or []
|
||||
print(f"returned_clans={len(clans)}")
|
||||
positive = sum(1 for clan in clans if int((clan.get("astat") or {}).get("dr_era5_hist") or 0) > 0)
|
||||
print(f"positive_clans={positive}")
|
||||
|
||||
matches = []
|
||||
for idx, clan in enumerate(clans, start=1):
|
||||
if _score_hit(clan, args.needle):
|
||||
astat = clan.get("astat") or {}
|
||||
matches.append(
|
||||
{
|
||||
"index": idx,
|
||||
"clan_id": clan.get("_id"),
|
||||
"tag": clan.get("tag"),
|
||||
"name": clan.get("name"),
|
||||
"short_name": clan.get("tag", "")[1:-1] if clan.get("tag") else "",
|
||||
"clanrating": astat.get("dr_era5_hist"),
|
||||
"members": clan.get("members_cnt"),
|
||||
"position": clan.get("pos"),
|
||||
}
|
||||
)
|
||||
|
||||
if not matches:
|
||||
print(f"needle={args.needle!r} not found")
|
||||
return 0
|
||||
|
||||
print("matches=" + json.dumps(matches[:10], ensure_ascii=False))
|
||||
first = matches[0]["index"]
|
||||
lo = max(1, first - 3)
|
||||
hi = min(len(clans), first + 3)
|
||||
window = []
|
||||
for idx in range(lo, hi + 1):
|
||||
clan = clans[idx - 1]
|
||||
astat = clan.get("astat") or {}
|
||||
window.append(
|
||||
{
|
||||
"index": idx,
|
||||
"tag": clan.get("tag"),
|
||||
"name": clan.get("name"),
|
||||
"clanrating": astat.get("dr_era5_hist"),
|
||||
"position": clan.get("pos"),
|
||||
}
|
||||
)
|
||||
print("window=" + json.dumps(window, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(_main()))
|
||||
@@ -0,0 +1,739 @@
|
||||
"""
|
||||
One-shot migration: switch SREBOT storage to use clan_id (numeric squadron UID)
|
||||
as the canonical identifier across DB tables and preference JSON files.
|
||||
|
||||
Goals
|
||||
-----
|
||||
1. Add `clan_id` to historical tables (player_games_hist, match_summary).
|
||||
2. Rebuild points.db tables (profile_member_points, profile_totals,
|
||||
game_cache) so their PKs include `clan_id` instead of squadron text.
|
||||
3. Rebuild wl.db tables to use clan_id (currently empty so trivial).
|
||||
4. Add `squadron_name_history` table to squadrons.db (for old-name → clan_id
|
||||
redirects when a squadron renames). Seed with current squadrons_data.
|
||||
5. Re-key every PREFERENCES/<guild_id>-preferences.json so squadron entries
|
||||
are keyed by `str(clan_id)` instead of long_name. Special wildcard keys
|
||||
("Global", "everything", "all", "*") are preserved.
|
||||
6. Backups: write a tarball of STORAGE/ before any change. Each preferences
|
||||
file gets copied to PREFERENCES_BACKUP_<timestamp>/.
|
||||
|
||||
Run with `--dry-run` first. The script prints exactly what it would do.
|
||||
|
||||
Usage
|
||||
-----
|
||||
source .venv/bin/activate
|
||||
python scripts/migrate_clan_id.py --dry-run
|
||||
python scripts/migrate_clan_id.py --apply
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
import tarfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
)
|
||||
log = logging.getLogger("migrate_clan_id")
|
||||
|
||||
|
||||
# Auto-load the repo's .env so SREBOT_STORAGE_VOL_PATH and friends resolve when
|
||||
# the script is run directly (PM2/services already get them from .env).
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
_here = Path(__file__).resolve()
|
||||
for candidate in (_here.parent / ".env", _here.parent.parent / ".env"):
|
||||
if candidate.exists():
|
||||
load_dotenv(candidate)
|
||||
break
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
_storage_env = os.environ.get("SREBOT_STORAGE_VOL_PATH", "").strip()
|
||||
if not _storage_env:
|
||||
raise RuntimeError("SREBOT_STORAGE_VOL_PATH must be set")
|
||||
STORAGE_DIR = Path(_storage_env)
|
||||
|
||||
SQ_BATTLES_DB = STORAGE_DIR / "sq_battles.db"
|
||||
SQUADRONS_DB = STORAGE_DIR / "squadrons.db"
|
||||
POINTS_DB = STORAGE_DIR / "points.db"
|
||||
WL_DB = STORAGE_DIR / "wl.db"
|
||||
PREFS_DIR = STORAGE_DIR / "PREFERENCES"
|
||||
|
||||
WILDCARD_KEYS = {"global", "everything", "all", "*"}
|
||||
PGH_BATCH = 50_000
|
||||
|
||||
|
||||
def backup_storage() -> Path:
|
||||
ts = time.strftime("%Y%m%d-%H%M%S")
|
||||
backup_path = STORAGE_DIR.parent / f"STORAGE_BACKUP_{ts}.tar.gz"
|
||||
log.info("Creating tarball backup at %s (this may take a while)...", backup_path)
|
||||
with tarfile.open(backup_path, "w:gz") as tar:
|
||||
for db in (SQ_BATTLES_DB, SQUADRONS_DB, POINTS_DB, WL_DB):
|
||||
if db.exists():
|
||||
tar.add(db, arcname=db.name)
|
||||
if PREFS_DIR.exists():
|
||||
tar.add(PREFS_DIR, arcname=PREFS_DIR.name)
|
||||
log.info("Backup complete: %s", backup_path)
|
||||
return backup_path
|
||||
|
||||
|
||||
def load_squadron_index() -> tuple[dict[str, int], dict[str, int], dict[int, dict[str, Any]]]:
|
||||
"""Return (long_name_lower -> clan_id, short_name_lower -> clan_id, clan_id -> row)."""
|
||||
long_to_id: dict[str, int] = {}
|
||||
short_to_id: dict[str, int] = {}
|
||||
by_id: dict[int, dict[str, Any]] = {}
|
||||
con = sqlite3.connect(SQUADRONS_DB)
|
||||
try:
|
||||
con.row_factory = sqlite3.Row
|
||||
for row in con.execute(
|
||||
"SELECT clan_id, long_name, short_name, tag_name FROM squadrons_data"
|
||||
):
|
||||
cid = int(row["clan_id"])
|
||||
by_id[cid] = dict(row)
|
||||
if row["long_name"]:
|
||||
long_to_id[row["long_name"].lower()] = cid
|
||||
if row["short_name"]:
|
||||
short_to_id[row["short_name"].lower()] = cid
|
||||
finally:
|
||||
con.close()
|
||||
log.info("Loaded %d squadrons from squadrons_data", len(by_id))
|
||||
return long_to_id, short_to_id, by_id
|
||||
|
||||
|
||||
def ensure_squadron_name_history(apply: bool, by_id: dict[int, dict[str, Any]]) -> None:
|
||||
"""Create squadron_name_history table and seed from current squadrons_data.
|
||||
|
||||
Schema:
|
||||
clan_id INTEGER NOT NULL
|
||||
long_name TEXT NOT NULL
|
||||
first_seen INTEGER NOT NULL (unix)
|
||||
last_seen INTEGER NOT NULL (unix)
|
||||
PRIMARY KEY (clan_id, long_name)
|
||||
"""
|
||||
con = sqlite3.connect(SQUADRONS_DB)
|
||||
try:
|
||||
existing = con.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='squadron_name_history'"
|
||||
).fetchone()
|
||||
if existing:
|
||||
log.info("squadron_name_history already exists; skipping create")
|
||||
else:
|
||||
log.info("Creating squadron_name_history table")
|
||||
if apply:
|
||||
con.executescript(
|
||||
"""
|
||||
CREATE TABLE squadron_name_history (
|
||||
clan_id INTEGER NOT NULL,
|
||||
long_name TEXT NOT NULL,
|
||||
first_seen INTEGER NOT NULL,
|
||||
last_seen INTEGER NOT NULL,
|
||||
PRIMARY KEY (clan_id, long_name)
|
||||
);
|
||||
CREATE INDEX idx_snh_long_name ON squadron_name_history(long_name COLLATE NOCASE);
|
||||
CREATE INDEX idx_snh_clan_id ON squadron_name_history(clan_id);
|
||||
"""
|
||||
)
|
||||
|
||||
now = int(time.time())
|
||||
rows = [
|
||||
(cid, row["long_name"], now, now)
|
||||
for cid, row in by_id.items()
|
||||
if row.get("long_name")
|
||||
]
|
||||
log.info("Seeding squadron_name_history with %d (clan_id, long_name) pairs", len(rows))
|
||||
if apply and rows:
|
||||
con.executemany(
|
||||
"""
|
||||
INSERT INTO squadron_name_history (clan_id, long_name, first_seen, last_seen)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(clan_id, long_name) DO UPDATE SET last_seen = excluded.last_seen
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def add_squadrons_points_index(apply: bool) -> None:
|
||||
con = sqlite3.connect(SQUADRONS_DB)
|
||||
try:
|
||||
log.info("Adding clan_id index on squadrons_points")
|
||||
if apply:
|
||||
con.executescript(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_squadrons_points_clanid_time
|
||||
ON squadrons_points(clan_id, unix_time);
|
||||
"""
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def migrate_player_games_hist(
|
||||
apply: bool, long_to_id: dict[str, int]
|
||||
) -> None:
|
||||
"""Add clan_id INTEGER column + backfill via long_name lookup. Add index."""
|
||||
con = sqlite3.connect(SQ_BATTLES_DB)
|
||||
try:
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(player_games_hist)").fetchall()]
|
||||
if "clan_id" not in cols:
|
||||
log.info("Adding clan_id column to player_games_hist")
|
||||
if apply:
|
||||
con.execute("ALTER TABLE player_games_hist ADD COLUMN clan_id INTEGER")
|
||||
con.commit()
|
||||
else:
|
||||
log.info("player_games_hist.clan_id already present")
|
||||
|
||||
# Refresh after the conditional ALTER.
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(player_games_hist)").fetchall()]
|
||||
total = con.execute("SELECT COUNT(*) FROM player_games_hist").fetchone()[0]
|
||||
if "clan_id" in cols:
|
||||
unbackfilled = con.execute(
|
||||
"SELECT COUNT(*) FROM player_games_hist WHERE clan_id IS NULL"
|
||||
).fetchone()[0]
|
||||
else:
|
||||
unbackfilled = total
|
||||
log.info(
|
||||
"player_games_hist: %d total rows, %d need backfill",
|
||||
total,
|
||||
unbackfilled,
|
||||
)
|
||||
|
||||
if apply and unbackfilled:
|
||||
# Faster path: pre-build a name → clan_id map in Python and run one
|
||||
# indexed UPDATE per distinct squadron_name. Each UPDATE hits the
|
||||
# idx_pgh_squadron_name index. ~1500 small ops vs one giant
|
||||
# correlated subquery. squadron_name is the WT short_name (after
|
||||
# alphanum strip), but very rarely a long_name leaks through, so
|
||||
# we accept matches against either.
|
||||
log.info("Building name → clan_id map in Python...")
|
||||
sq_con = sqlite3.connect(f"file:{SQUADRONS_DB}?mode=ro", uri=True)
|
||||
try:
|
||||
sq_con.row_factory = sqlite3.Row
|
||||
name_to_id: dict[str, int] = {}
|
||||
for row in sq_con.execute(
|
||||
"SELECT clan_id, long_name, short_name FROM squadrons_data"
|
||||
):
|
||||
cid = int(row["clan_id"])
|
||||
if row["long_name"]:
|
||||
name_to_id.setdefault(row["long_name"].lower(), cid)
|
||||
if row["short_name"]:
|
||||
name_to_id.setdefault(row["short_name"].lower(), cid)
|
||||
finally:
|
||||
sq_con.close()
|
||||
|
||||
log.info("Fetching distinct squadron_name values from player_games_hist...")
|
||||
distinct_names = [
|
||||
r[0] for r in con.execute(
|
||||
"SELECT DISTINCT squadron_name FROM player_games_hist WHERE clan_id IS NULL"
|
||||
).fetchall()
|
||||
if r[0]
|
||||
]
|
||||
log.info("Distinct unbackfilled squadron_names: %d", len(distinct_names))
|
||||
|
||||
updates = [
|
||||
(name_to_id[n.lower()], n)
|
||||
for n in distinct_names
|
||||
if n.lower() in name_to_id
|
||||
]
|
||||
log.info("Will UPDATE %d distinct names that resolve to clan_ids", len(updates))
|
||||
if updates:
|
||||
cur = con.executemany(
|
||||
"UPDATE player_games_hist SET clan_id = ? "
|
||||
"WHERE squadron_name = ? AND clan_id IS NULL",
|
||||
updates,
|
||||
)
|
||||
con.commit()
|
||||
log.info("Backfill: %s row-updates committed", cur.rowcount)
|
||||
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(player_games_hist)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
still_null = con.execute(
|
||||
"SELECT COUNT(*) FROM player_games_hist WHERE clan_id IS NULL"
|
||||
).fetchone()[0]
|
||||
log.info("player_games_hist orphans (clan_id NULL after backfill): %d", still_null)
|
||||
else:
|
||||
log.info("player_games_hist orphans: n/a (column absent in dry-run)")
|
||||
|
||||
log.info("Ensuring index on player_games_hist(clan_id, endtime_unix)")
|
||||
if apply:
|
||||
con.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_pgh_clanid_endtime "
|
||||
"ON player_games_hist(clan_id, endtime_unix)"
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def migrate_match_summary(apply: bool) -> None:
|
||||
"""Add winning_clan_id + losing_clan_id, backfill via squadrons_data."""
|
||||
con = sqlite3.connect(SQ_BATTLES_DB)
|
||||
try:
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(match_summary)").fetchall()]
|
||||
for new_col in ("winning_clan_id", "losing_clan_id"):
|
||||
if new_col not in cols:
|
||||
log.info("Adding %s column to match_summary", new_col)
|
||||
if apply:
|
||||
con.execute(f"ALTER TABLE match_summary ADD COLUMN {new_col} INTEGER")
|
||||
con.commit()
|
||||
else:
|
||||
log.info("match_summary.%s already present", new_col)
|
||||
|
||||
# Refresh after potential ALTERs.
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(match_summary)").fetchall()]
|
||||
total = con.execute("SELECT COUNT(*) FROM match_summary").fetchone()[0]
|
||||
if "winning_clan_id" in cols and "losing_clan_id" in cols:
|
||||
unbackfilled = con.execute(
|
||||
"SELECT COUNT(*) FROM match_summary "
|
||||
"WHERE winning_clan_id IS NULL OR losing_clan_id IS NULL"
|
||||
).fetchone()[0]
|
||||
else:
|
||||
# Dry-run path: columns don't exist yet, so every row is "to be backfilled".
|
||||
unbackfilled = total
|
||||
log.info(
|
||||
"match_summary: %d total rows, %d need backfill",
|
||||
total,
|
||||
unbackfilled,
|
||||
)
|
||||
|
||||
if apply and unbackfilled:
|
||||
con.execute(f"ATTACH DATABASE '{SQUADRONS_DB}' AS sq")
|
||||
try:
|
||||
# winning_sq / losing_sq can be either short_name or tag_name
|
||||
# depending on replay metadata. Try short_name first since that's
|
||||
# what the autologger writes most often.
|
||||
for col_in, col_out in (
|
||||
("winning_sq", "winning_clan_id"),
|
||||
("losing_sq", "losing_clan_id"),
|
||||
):
|
||||
log.info("Backfilling %s ← %s via short_name then long_name", col_out, col_in)
|
||||
con.execute(
|
||||
f"""
|
||||
UPDATE match_summary
|
||||
SET {col_out} = (
|
||||
SELECT clan_id FROM sq.squadrons_data
|
||||
WHERE LOWER(squadrons_data.short_name) = LOWER(match_summary.{col_in})
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE {col_out} IS NULL AND {col_in} IS NOT NULL
|
||||
"""
|
||||
)
|
||||
con.execute(
|
||||
f"""
|
||||
UPDATE match_summary
|
||||
SET {col_out} = (
|
||||
SELECT clan_id FROM sq.squadrons_data
|
||||
WHERE LOWER(squadrons_data.long_name) = LOWER(match_summary.{col_in})
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE {col_out} IS NULL AND {col_in} IS NOT NULL
|
||||
"""
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.execute("DETACH DATABASE sq")
|
||||
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(match_summary)").fetchall()]
|
||||
for col, label in (("winning_clan_id", "winning"), ("losing_clan_id", "losing")):
|
||||
if col not in cols:
|
||||
log.info("match_summary orphans (%s NULL): n/a (column absent in dry-run)", label)
|
||||
continue
|
||||
n = con.execute(
|
||||
f"SELECT COUNT(*) FROM match_summary WHERE {col} IS NULL"
|
||||
).fetchone()[0]
|
||||
log.info("match_summary orphans (%s NULL): %d", label, n)
|
||||
|
||||
log.info("Ensuring indexes on match_summary clan_id columns")
|
||||
if apply:
|
||||
con.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_ms_winning_clanid ON match_summary(winning_clan_id)"
|
||||
)
|
||||
con.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_ms_losing_clanid ON match_summary(losing_clan_id)"
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def _resolve_squadron_to_clan_id(name: Optional[str], name_to_id: dict[str, int]) -> int:
|
||||
if not name:
|
||||
return -1
|
||||
return name_to_id.get(name.lower(), -1)
|
||||
|
||||
|
||||
def rebuild_points_db(apply: bool, long_to_id: dict[str, int]) -> None:
|
||||
"""Rebuild profile_member_points, profile_totals, game_cache with clan_id columns.
|
||||
|
||||
The squadron text column is preserved for reference but the new tables key
|
||||
off clan_id for primary keys. Resolution happens in Python so we don't have
|
||||
to ATTACH squadrons.db (avoids the cross-db lock issue we hit earlier).
|
||||
"""
|
||||
# Build name → clan_id map once (long_name AND short_name).
|
||||
sq_con = sqlite3.connect(f"file:{SQUADRONS_DB}?mode=ro", uri=True)
|
||||
try:
|
||||
sq_con.row_factory = sqlite3.Row
|
||||
name_to_id: dict[str, int] = {}
|
||||
for row in sq_con.execute(
|
||||
"SELECT clan_id, long_name, short_name FROM squadrons_data"
|
||||
):
|
||||
cid = int(row["clan_id"])
|
||||
if row["long_name"]:
|
||||
name_to_id.setdefault(row["long_name"].lower(), cid)
|
||||
if row["short_name"]:
|
||||
name_to_id.setdefault(row["short_name"].lower(), cid)
|
||||
finally:
|
||||
sq_con.close()
|
||||
|
||||
con = sqlite3.connect(POINTS_DB)
|
||||
try:
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
|
||||
# profile_member_points -------------------------------------------------
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(profile_member_points)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
log.info("profile_member_points already migrated; skipping rebuild")
|
||||
else:
|
||||
log.info("Rebuilding profile_member_points with clan_id PK")
|
||||
if apply:
|
||||
# Idempotency: a previous half-finished run may have left _new behind.
|
||||
con.execute("DROP TABLE IF EXISTS profile_member_points_new")
|
||||
con.execute(
|
||||
"""CREATE TABLE profile_member_points_new (
|
||||
clan_id INTEGER NOT NULL,
|
||||
squadron TEXT NOT NULL,
|
||||
uid TEXT NOT NULL,
|
||||
points INTEGER NOT NULL,
|
||||
PRIMARY KEY (clan_id, uid)
|
||||
)"""
|
||||
)
|
||||
src = con.execute(
|
||||
"SELECT squadron, uid, points FROM profile_member_points"
|
||||
).fetchall()
|
||||
rows = [
|
||||
(_resolve_squadron_to_clan_id(s, name_to_id), s, u, p)
|
||||
for (s, u, p) in src
|
||||
]
|
||||
con.executemany(
|
||||
"INSERT OR IGNORE INTO profile_member_points_new "
|
||||
"(clan_id, squadron, uid, points) VALUES (?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
con.commit()
|
||||
log.info("profile_member_points: copied %d rows", len(rows))
|
||||
con.execute("DROP TABLE profile_member_points")
|
||||
con.execute("ALTER TABLE profile_member_points_new RENAME TO profile_member_points")
|
||||
con.execute("CREATE INDEX IF NOT EXISTS idx_pmp_squadron ON profile_member_points(squadron)")
|
||||
con.commit()
|
||||
|
||||
# profile_totals --------------------------------------------------------
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(profile_totals)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
log.info("profile_totals already migrated; skipping rebuild")
|
||||
else:
|
||||
log.info("Rebuilding profile_totals with clan_id PK")
|
||||
if apply:
|
||||
con.execute("DROP TABLE IF EXISTS profile_totals_new")
|
||||
con.execute(
|
||||
"""CREATE TABLE profile_totals_new (
|
||||
clan_id INTEGER PRIMARY KEY,
|
||||
squadron TEXT NOT NULL,
|
||||
total INTEGER NOT NULL
|
||||
)"""
|
||||
)
|
||||
src = con.execute("SELECT squadron, total FROM profile_totals").fetchall()
|
||||
rows = [
|
||||
(_resolve_squadron_to_clan_id(s, name_to_id), s, t)
|
||||
for (s, t) in src
|
||||
]
|
||||
con.executemany(
|
||||
"INSERT OR IGNORE INTO profile_totals_new (clan_id, squadron, total) VALUES (?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
con.commit()
|
||||
log.info("profile_totals: copied %d rows", len(rows))
|
||||
con.execute("DROP TABLE profile_totals")
|
||||
con.execute("ALTER TABLE profile_totals_new RENAME TO profile_totals")
|
||||
con.commit()
|
||||
|
||||
# game_cache ------------------------------------------------------------
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(game_cache)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
log.info("game_cache already migrated; skipping rebuild")
|
||||
else:
|
||||
log.info("Rebuilding game_cache with clan_id in PK")
|
||||
if apply:
|
||||
con.execute("DROP TABLE IF EXISTS game_cache_new")
|
||||
con.execute(
|
||||
"""CREATE TABLE game_cache_new (
|
||||
game_id TEXT NOT NULL,
|
||||
clan_id INTEGER NOT NULL,
|
||||
squadron TEXT NOT NULL,
|
||||
diffs_json TEXT NOT NULL,
|
||||
diff_total INTEGER NOT NULL,
|
||||
updated_json TEXT NOT NULL,
|
||||
created_at REAL NOT NULL DEFAULT (strftime('%s','now')),
|
||||
PRIMARY KEY (game_id, clan_id)
|
||||
)"""
|
||||
)
|
||||
src = con.execute(
|
||||
"SELECT game_id, squadron, diffs_json, diff_total, updated_json, created_at FROM game_cache"
|
||||
).fetchall()
|
||||
rows = [
|
||||
(
|
||||
gid,
|
||||
_resolve_squadron_to_clan_id(s, name_to_id),
|
||||
s,
|
||||
dj,
|
||||
dt,
|
||||
uj,
|
||||
ca,
|
||||
)
|
||||
for (gid, s, dj, dt, uj, ca) in src
|
||||
]
|
||||
con.executemany(
|
||||
"INSERT OR IGNORE INTO game_cache_new "
|
||||
"(game_id, clan_id, squadron, diffs_json, diff_total, updated_json, created_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
con.commit()
|
||||
log.info("game_cache: copied %d rows", len(rows))
|
||||
con.execute("DROP TABLE game_cache")
|
||||
con.execute("ALTER TABLE game_cache_new RENAME TO game_cache")
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def rebuild_wl_db(apply: bool) -> None:
|
||||
"""wl_events / wl_standings are empty - swap schemas to clan_id keyed tables."""
|
||||
con = sqlite3.connect(WL_DB)
|
||||
try:
|
||||
# wl_standings
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(wl_standings)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
log.info("wl_standings already migrated")
|
||||
else:
|
||||
log.info("Recreating wl_standings keyed by clan_id (was empty)")
|
||||
if apply:
|
||||
con.executescript(
|
||||
"""
|
||||
DROP TABLE IF EXISTS wl_standings;
|
||||
CREATE TABLE wl_standings (
|
||||
clan_id INTEGER PRIMARY KEY,
|
||||
squadron TEXT NOT NULL,
|
||||
wins INTEGER NOT NULL DEFAULT 0,
|
||||
losses INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
"""
|
||||
)
|
||||
con.commit()
|
||||
|
||||
# wl_events: store winner_clan_id alongside winner text
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(wl_events)").fetchall()]
|
||||
if "winner_clan_id" in cols:
|
||||
log.info("wl_events already migrated")
|
||||
else:
|
||||
log.info("Adding winner_clan_id column to wl_events (table is empty)")
|
||||
if apply:
|
||||
con.execute("ALTER TABLE wl_events ADD COLUMN winner_clan_id INTEGER")
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def migrate_preferences(
|
||||
apply: bool,
|
||||
long_to_id: dict[str, int],
|
||||
short_to_id: dict[str, int],
|
||||
by_id: dict[int, dict[str, Any]],
|
||||
) -> None:
|
||||
"""Re-key every PREFERENCES/<guild_id>-preferences.json from long_name → str(clan_id).
|
||||
|
||||
Special wildcard keys (Global, everything, all, *) are preserved as-is.
|
||||
Each migrated entry gets a `Long` field with the squadron's current
|
||||
long_name so display fallback works if squadrons_data is unavailable.
|
||||
Original files are copied to PREFERENCES_BACKUP_<timestamp>/.
|
||||
"""
|
||||
if not PREFS_DIR.exists():
|
||||
log.warning("PREFERENCES dir missing at %s; skipping prefs migration", PREFS_DIR)
|
||||
return
|
||||
|
||||
ts = time.strftime("%Y%m%d-%H%M%S")
|
||||
backup_dir = STORAGE_DIR / f"PREFERENCES_BACKUP_{ts}"
|
||||
if apply:
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pref_files = sorted(PREFS_DIR.glob("*-preferences.json"))
|
||||
log.info("Found %d preference files to migrate", len(pref_files))
|
||||
|
||||
migrated_count = 0
|
||||
orphan_count = 0
|
||||
skipped_count = 0
|
||||
files_changed = 0
|
||||
|
||||
for pref_file in pref_files:
|
||||
try:
|
||||
data = json.loads(pref_file.read_text(encoding="utf-8"))
|
||||
except Exception as e:
|
||||
log.warning("Could not read %s: %s", pref_file.name, e)
|
||||
continue
|
||||
|
||||
if not isinstance(data, dict):
|
||||
log.warning("Skipping %s: not an object", pref_file.name)
|
||||
continue
|
||||
|
||||
new_data: dict[str, Any] = {}
|
||||
any_changes = False
|
||||
|
||||
for key, value in data.items():
|
||||
if not isinstance(value, dict):
|
||||
new_data[key] = value
|
||||
continue
|
||||
|
||||
key_lower = str(key).lower()
|
||||
|
||||
# Preserve wildcards / Global
|
||||
if key_lower in WILDCARD_KEYS:
|
||||
new_data[key] = value
|
||||
continue
|
||||
|
||||
# Already a numeric clan_id (running migration twice)
|
||||
if str(key).isdigit():
|
||||
new_data[key] = value
|
||||
continue
|
||||
|
||||
# Try long_name then short_name
|
||||
cid = long_to_id.get(key_lower) or short_to_id.get(key_lower)
|
||||
|
||||
# Maybe entry has a "Short" hint we can use as fallback
|
||||
if cid is None and value.get("Short"):
|
||||
cid = short_to_id.get(str(value["Short"]).lower())
|
||||
|
||||
if cid is None:
|
||||
log.warning(
|
||||
" [%s] orphan key %r — squadron not found in squadrons_data",
|
||||
pref_file.name,
|
||||
key,
|
||||
)
|
||||
# Keep the original key so the user doesn't lose their data;
|
||||
# a future load_guild_preferences will surface it for cleanup.
|
||||
new_data[key] = value
|
||||
orphan_count += 1
|
||||
continue
|
||||
|
||||
new_key = str(cid)
|
||||
row = by_id.get(cid, {})
|
||||
merged = dict(value)
|
||||
# Preserve display fields - the bot uses them when squadrons_data is stale
|
||||
merged.setdefault("Long", row.get("long_name") or key)
|
||||
if row.get("short_name"):
|
||||
merged["Short"] = row["short_name"]
|
||||
|
||||
if new_key in new_data:
|
||||
# Two prefs entries for the same squadron (rare). Merge,
|
||||
# preferring values from this iteration.
|
||||
existing = new_data[new_key]
|
||||
if isinstance(existing, dict):
|
||||
existing.update(merged)
|
||||
new_data[new_key] = existing
|
||||
else:
|
||||
new_data[new_key] = merged
|
||||
else:
|
||||
new_data[new_key] = merged
|
||||
|
||||
if new_key != key:
|
||||
any_changes = True
|
||||
migrated_count += 1
|
||||
|
||||
if not any_changes:
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
if apply:
|
||||
backup_path = backup_dir / pref_file.name
|
||||
shutil.copy2(pref_file, backup_path)
|
||||
tmp = pref_file.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(new_data, ensure_ascii=False), encoding="utf-8")
|
||||
os.replace(tmp, pref_file)
|
||||
files_changed += 1
|
||||
|
||||
log.info(
|
||||
"Preferences migration: %d files changed, %d unchanged, %d entries migrated, %d orphans",
|
||||
files_changed,
|
||||
skipped_count,
|
||||
migrated_count,
|
||||
orphan_count,
|
||||
)
|
||||
if apply:
|
||||
log.info("Preference backups saved to %s", backup_dir)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true", help="Actually apply changes")
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print what would happen without writing anything (default)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-backup",
|
||||
action="store_true",
|
||||
help="Skip the tarball backup step (use only if you already have one)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.apply and args.dry_run:
|
||||
log.error("--apply and --dry-run are mutually exclusive")
|
||||
return 2
|
||||
|
||||
apply = args.apply
|
||||
|
||||
log.info("STORAGE_DIR=%s", STORAGE_DIR)
|
||||
log.info("apply=%s", apply)
|
||||
|
||||
for db in (SQ_BATTLES_DB, SQUADRONS_DB, POINTS_DB, WL_DB):
|
||||
if not db.exists():
|
||||
log.error("Required DB missing: %s", db)
|
||||
return 1
|
||||
|
||||
if apply and not args.skip_backup:
|
||||
backup_storage()
|
||||
|
||||
long_to_id, short_to_id, by_id = load_squadron_index()
|
||||
|
||||
ensure_squadron_name_history(apply, by_id)
|
||||
add_squadrons_points_index(apply)
|
||||
migrate_player_games_hist(apply, long_to_id)
|
||||
migrate_match_summary(apply)
|
||||
rebuild_points_db(apply, long_to_id)
|
||||
rebuild_wl_db(apply)
|
||||
migrate_preferences(apply, long_to_id, short_to_id, by_id)
|
||||
|
||||
if not apply:
|
||||
log.info("DRY RUN complete. Re-run with --apply to actually write changes.")
|
||||
else:
|
||||
log.info("Migration applied successfully.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Move legacy repo-root replays into STORAGE/REPLAYS.
|
||||
|
||||
Legacy directories were named replays/0<hex_session_id>. The new canonical
|
||||
layout is STORAGE/REPLAYS/<hex_session_id>.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_SOURCE = REPO_ROOT / "replays"
|
||||
_storage_env = os.environ.get("SREBOT_STORAGE_VOL_PATH", "").strip()
|
||||
if not _storage_env:
|
||||
raise RuntimeError("SREBOT_STORAGE_VOL_PATH must be set")
|
||||
DEFAULT_STORAGE = Path(_storage_env)
|
||||
|
||||
LEGACY_REPLAY_DIR = re.compile(r"^0([0-9a-fA-F]+)$")
|
||||
HEX_REPLAY_DIR = re.compile(r"^[0-9a-fA-F]+$")
|
||||
|
||||
|
||||
def canonical_name(name: str) -> str | None:
|
||||
legacy = LEGACY_REPLAY_DIR.fullmatch(name)
|
||||
if legacy:
|
||||
return legacy.group(1).lower()
|
||||
if HEX_REPLAY_DIR.fullmatch(name):
|
||||
return name.lower()
|
||||
return None
|
||||
|
||||
|
||||
def iter_files(root: Path) -> list[Path]:
|
||||
return [p for p in root.rglob("*") if p.is_file()]
|
||||
|
||||
|
||||
def copy_replay_dir(source: Path, dest: Path, dry_run: bool) -> tuple[int, int]:
|
||||
copied = 0
|
||||
skipped = 0
|
||||
for src_file in iter_files(source):
|
||||
rel = src_file.relative_to(source)
|
||||
dst_file = dest / rel
|
||||
if dst_file.exists() and dst_file.stat().st_size == src_file.stat().st_size:
|
||||
skipped += 1
|
||||
continue
|
||||
copied += 1
|
||||
if dry_run:
|
||||
continue
|
||||
dst_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src_file, dst_file)
|
||||
return copied, skipped
|
||||
|
||||
|
||||
def copied_completely(source: Path, dest: Path) -> bool:
|
||||
for src_file in iter_files(source):
|
||||
dst_file = dest / src_file.relative_to(source)
|
||||
if not dst_file.exists() or dst_file.stat().st_size != src_file.stat().st_size:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE, help="Legacy replays directory.")
|
||||
parser.add_argument(
|
||||
"--storage-dir",
|
||||
type=Path,
|
||||
default=DEFAULT_STORAGE,
|
||||
help="Storage root. Destination defaults to <storage-dir>/REPLAYS.",
|
||||
)
|
||||
parser.add_argument("--dest", type=Path, help="Destination replay directory.")
|
||||
parser.add_argument("--execute", action="store_true", help="Actually copy files. Default is dry-run.")
|
||||
parser.add_argument("--move", action="store_true", help="Remove source dirs after a successful executed copy.")
|
||||
args = parser.parse_args()
|
||||
|
||||
source = args.source.expanduser().resolve()
|
||||
dest_root = (args.dest or (args.storage_dir / "REPLAYS")).expanduser().resolve()
|
||||
dry_run = not args.execute
|
||||
|
||||
if args.move and dry_run:
|
||||
parser.error("--move requires --execute")
|
||||
if not source.exists():
|
||||
raise SystemExit(f"Source does not exist: {source}")
|
||||
|
||||
planned = 0
|
||||
copied_total = 0
|
||||
skipped_total = 0
|
||||
|
||||
print(f"Source: {source}")
|
||||
print(f"Destination: {dest_root}")
|
||||
print(f"Mode: {'dry-run' if dry_run else 'execute'}")
|
||||
|
||||
for entry in sorted(source.iterdir()):
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
new_name = canonical_name(entry.name)
|
||||
if new_name is None:
|
||||
print(f"skip non-replay dir: {entry.name}")
|
||||
continue
|
||||
|
||||
planned += 1
|
||||
dest = dest_root / new_name
|
||||
copied, skipped = copy_replay_dir(entry, dest, dry_run)
|
||||
copied_total += copied
|
||||
skipped_total += skipped
|
||||
print(f"{entry.name} -> {new_name}: copy {copied}, skip {skipped}")
|
||||
|
||||
if args.move and copied_completely(entry, dest):
|
||||
shutil.rmtree(entry)
|
||||
print(f"removed source dir: {entry}")
|
||||
|
||||
print(f"Replay dirs: {planned}; files to copy: {copied_total}; already present: {skipped_total}")
|
||||
if dry_run:
|
||||
print("Dry-run only. Re-run with --execute to copy files.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
# Restarts srebot-api + srebot-web and times the recovery milestones.
|
||||
# Usage: bash scripts/restart-test.sh
|
||||
set -u
|
||||
T0=$(date +%s.%N)
|
||||
echo "=== T0: $(date +%H:%M:%S.%3N) restarting srebot-api + srebot-web ==="
|
||||
pm2 restart srebot-api srebot-web >/dev/null
|
||||
|
||||
echo "--- polling /health (every 0.5s until ready:true) ---"
|
||||
for i in $(seq 1 120); do
|
||||
R=$(curl -sS --max-time 2 http://127.0.0.1:6000/health 2>/dev/null || true)
|
||||
if echo "$R" | grep -q '"ready":true'; then
|
||||
echo "ready at +$(echo "$(date +%s.%N) - $T0" | bc)s : $R"
|
||||
break
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo "--- /api/squadrons/1066957 (first cold hit, separate connection) ---"
|
||||
curl -sS -o /dev/null -w " HTTP=%{http_code} time=%{time_total}s\n" "http://127.0.0.1:6000/api/squadrons/1066957"
|
||||
|
||||
echo "--- 4 leaderboards in parallel (heavyDb concurrent reads) ---"
|
||||
for ep in stats squadrons players vehicles; do
|
||||
(curl -sS -o /dev/null -w " $ep: HTTP=%{http_code} time=%{time_total}s\n" "http://127.0.0.1:6000/api/leaderboard/$ep") &
|
||||
done
|
||||
wait
|
||||
|
||||
echo "=== full warm at +$(echo "$(date +%s.%N) - $T0" | bc)s ==="
|
||||
echo "--- pm2 status ---"
|
||||
pm2 info srebot-api | grep -E "uptime|restart|status" | head -5
|
||||
Reference in New Issue
Block a user