add SREBOT, SHARED, TSSBOT contents (fixup for #1223)
PR #1223 only staged the deletions of the old paths because the new top-level directories were still untracked when the commit was authored. This commit adds the actual restructured tree: SREBOT/ (existing bot), SHARED/ (vromfs, data_parser, ICONS/MAPS/FONTS, DAGOR_FILES, update_game_files), and TSSBOT/ (skeleton). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
FURRO404
@@ -0,0 +1,739 @@
|
||||
"""
|
||||
One-shot migration: switch SREBOT storage to use clan_id (numeric squadron UID)
|
||||
as the canonical identifier across DB tables and preference JSON files.
|
||||
|
||||
Goals
|
||||
-----
|
||||
1. Add `clan_id` to historical tables (player_games_hist, match_summary).
|
||||
2. Rebuild points.db tables (profile_member_points, profile_totals,
|
||||
game_cache) so their PKs include `clan_id` instead of squadron text.
|
||||
3. Rebuild wl.db tables to use clan_id (currently empty so trivial).
|
||||
4. Add `squadron_name_history` table to squadrons.db (for old-name → clan_id
|
||||
redirects when a squadron renames). Seed with current squadrons_data.
|
||||
5. Re-key every PREFERENCES/<guild_id>-preferences.json so squadron entries
|
||||
are keyed by `str(clan_id)` instead of long_name. Special wildcard keys
|
||||
("Global", "everything", "all", "*") are preserved.
|
||||
6. Backups: write a tarball of STORAGE/ before any change. Each preferences
|
||||
file gets copied to PREFERENCES_BACKUP_<timestamp>/.
|
||||
|
||||
Run with `--dry-run` first. The script prints exactly what it would do.
|
||||
|
||||
Usage
|
||||
-----
|
||||
source .venv/bin/activate
|
||||
python scripts/migrate_clan_id.py --dry-run
|
||||
python scripts/migrate_clan_id.py --apply
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import sys
|
||||
import tarfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
)
|
||||
log = logging.getLogger("migrate_clan_id")
|
||||
|
||||
|
||||
# Auto-load the repo's .env so SREBOT_STORAGE_VOL_PATH and friends resolve when
|
||||
# the script is run directly (PM2/services already get them from .env).
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
|
||||
_here = Path(__file__).resolve()
|
||||
for candidate in (_here.parent / ".env", _here.parent.parent / ".env"):
|
||||
if candidate.exists():
|
||||
load_dotenv(candidate)
|
||||
break
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
_storage_env = os.environ.get("SREBOT_STORAGE_VOL_PATH", "").strip()
|
||||
if not _storage_env:
|
||||
raise RuntimeError("SREBOT_STORAGE_VOL_PATH must be set")
|
||||
STORAGE_DIR = Path(_storage_env)
|
||||
|
||||
SQ_BATTLES_DB = STORAGE_DIR / "sq_battles.db"
|
||||
SQUADRONS_DB = STORAGE_DIR / "squadrons.db"
|
||||
POINTS_DB = STORAGE_DIR / "points.db"
|
||||
WL_DB = STORAGE_DIR / "wl.db"
|
||||
PREFS_DIR = STORAGE_DIR / "PREFERENCES"
|
||||
|
||||
WILDCARD_KEYS = {"global", "everything", "all", "*"}
|
||||
PGH_BATCH = 50_000
|
||||
|
||||
|
||||
def backup_storage() -> Path:
|
||||
ts = time.strftime("%Y%m%d-%H%M%S")
|
||||
backup_path = STORAGE_DIR.parent / f"STORAGE_BACKUP_{ts}.tar.gz"
|
||||
log.info("Creating tarball backup at %s (this may take a while)...", backup_path)
|
||||
with tarfile.open(backup_path, "w:gz") as tar:
|
||||
for db in (SQ_BATTLES_DB, SQUADRONS_DB, POINTS_DB, WL_DB):
|
||||
if db.exists():
|
||||
tar.add(db, arcname=db.name)
|
||||
if PREFS_DIR.exists():
|
||||
tar.add(PREFS_DIR, arcname=PREFS_DIR.name)
|
||||
log.info("Backup complete: %s", backup_path)
|
||||
return backup_path
|
||||
|
||||
|
||||
def load_squadron_index() -> tuple[dict[str, int], dict[str, int], dict[int, dict[str, Any]]]:
|
||||
"""Return (long_name_lower -> clan_id, short_name_lower -> clan_id, clan_id -> row)."""
|
||||
long_to_id: dict[str, int] = {}
|
||||
short_to_id: dict[str, int] = {}
|
||||
by_id: dict[int, dict[str, Any]] = {}
|
||||
con = sqlite3.connect(SQUADRONS_DB)
|
||||
try:
|
||||
con.row_factory = sqlite3.Row
|
||||
for row in con.execute(
|
||||
"SELECT clan_id, long_name, short_name, tag_name FROM squadrons_data"
|
||||
):
|
||||
cid = int(row["clan_id"])
|
||||
by_id[cid] = dict(row)
|
||||
if row["long_name"]:
|
||||
long_to_id[row["long_name"].lower()] = cid
|
||||
if row["short_name"]:
|
||||
short_to_id[row["short_name"].lower()] = cid
|
||||
finally:
|
||||
con.close()
|
||||
log.info("Loaded %d squadrons from squadrons_data", len(by_id))
|
||||
return long_to_id, short_to_id, by_id
|
||||
|
||||
|
||||
def ensure_squadron_name_history(apply: bool, by_id: dict[int, dict[str, Any]]) -> None:
|
||||
"""Create squadron_name_history table and seed from current squadrons_data.
|
||||
|
||||
Schema:
|
||||
clan_id INTEGER NOT NULL
|
||||
long_name TEXT NOT NULL
|
||||
first_seen INTEGER NOT NULL (unix)
|
||||
last_seen INTEGER NOT NULL (unix)
|
||||
PRIMARY KEY (clan_id, long_name)
|
||||
"""
|
||||
con = sqlite3.connect(SQUADRONS_DB)
|
||||
try:
|
||||
existing = con.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='squadron_name_history'"
|
||||
).fetchone()
|
||||
if existing:
|
||||
log.info("squadron_name_history already exists; skipping create")
|
||||
else:
|
||||
log.info("Creating squadron_name_history table")
|
||||
if apply:
|
||||
con.executescript(
|
||||
"""
|
||||
CREATE TABLE squadron_name_history (
|
||||
clan_id INTEGER NOT NULL,
|
||||
long_name TEXT NOT NULL,
|
||||
first_seen INTEGER NOT NULL,
|
||||
last_seen INTEGER NOT NULL,
|
||||
PRIMARY KEY (clan_id, long_name)
|
||||
);
|
||||
CREATE INDEX idx_snh_long_name ON squadron_name_history(long_name COLLATE NOCASE);
|
||||
CREATE INDEX idx_snh_clan_id ON squadron_name_history(clan_id);
|
||||
"""
|
||||
)
|
||||
|
||||
now = int(time.time())
|
||||
rows = [
|
||||
(cid, row["long_name"], now, now)
|
||||
for cid, row in by_id.items()
|
||||
if row.get("long_name")
|
||||
]
|
||||
log.info("Seeding squadron_name_history with %d (clan_id, long_name) pairs", len(rows))
|
||||
if apply and rows:
|
||||
con.executemany(
|
||||
"""
|
||||
INSERT INTO squadron_name_history (clan_id, long_name, first_seen, last_seen)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(clan_id, long_name) DO UPDATE SET last_seen = excluded.last_seen
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def add_squadrons_points_index(apply: bool) -> None:
|
||||
con = sqlite3.connect(SQUADRONS_DB)
|
||||
try:
|
||||
log.info("Adding clan_id index on squadrons_points")
|
||||
if apply:
|
||||
con.executescript(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_squadrons_points_clanid_time
|
||||
ON squadrons_points(clan_id, unix_time);
|
||||
"""
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def migrate_player_games_hist(
|
||||
apply: bool, long_to_id: dict[str, int]
|
||||
) -> None:
|
||||
"""Add clan_id INTEGER column + backfill via long_name lookup. Add index."""
|
||||
con = sqlite3.connect(SQ_BATTLES_DB)
|
||||
try:
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(player_games_hist)").fetchall()]
|
||||
if "clan_id" not in cols:
|
||||
log.info("Adding clan_id column to player_games_hist")
|
||||
if apply:
|
||||
con.execute("ALTER TABLE player_games_hist ADD COLUMN clan_id INTEGER")
|
||||
con.commit()
|
||||
else:
|
||||
log.info("player_games_hist.clan_id already present")
|
||||
|
||||
# Refresh after the conditional ALTER.
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(player_games_hist)").fetchall()]
|
||||
total = con.execute("SELECT COUNT(*) FROM player_games_hist").fetchone()[0]
|
||||
if "clan_id" in cols:
|
||||
unbackfilled = con.execute(
|
||||
"SELECT COUNT(*) FROM player_games_hist WHERE clan_id IS NULL"
|
||||
).fetchone()[0]
|
||||
else:
|
||||
unbackfilled = total
|
||||
log.info(
|
||||
"player_games_hist: %d total rows, %d need backfill",
|
||||
total,
|
||||
unbackfilled,
|
||||
)
|
||||
|
||||
if apply and unbackfilled:
|
||||
# Faster path: pre-build a name → clan_id map in Python and run one
|
||||
# indexed UPDATE per distinct squadron_name. Each UPDATE hits the
|
||||
# idx_pgh_squadron_name index. ~1500 small ops vs one giant
|
||||
# correlated subquery. squadron_name is the WT short_name (after
|
||||
# alphanum strip), but very rarely a long_name leaks through, so
|
||||
# we accept matches against either.
|
||||
log.info("Building name → clan_id map in Python...")
|
||||
sq_con = sqlite3.connect(f"file:{SQUADRONS_DB}?mode=ro", uri=True)
|
||||
try:
|
||||
sq_con.row_factory = sqlite3.Row
|
||||
name_to_id: dict[str, int] = {}
|
||||
for row in sq_con.execute(
|
||||
"SELECT clan_id, long_name, short_name FROM squadrons_data"
|
||||
):
|
||||
cid = int(row["clan_id"])
|
||||
if row["long_name"]:
|
||||
name_to_id.setdefault(row["long_name"].lower(), cid)
|
||||
if row["short_name"]:
|
||||
name_to_id.setdefault(row["short_name"].lower(), cid)
|
||||
finally:
|
||||
sq_con.close()
|
||||
|
||||
log.info("Fetching distinct squadron_name values from player_games_hist...")
|
||||
distinct_names = [
|
||||
r[0] for r in con.execute(
|
||||
"SELECT DISTINCT squadron_name FROM player_games_hist WHERE clan_id IS NULL"
|
||||
).fetchall()
|
||||
if r[0]
|
||||
]
|
||||
log.info("Distinct unbackfilled squadron_names: %d", len(distinct_names))
|
||||
|
||||
updates = [
|
||||
(name_to_id[n.lower()], n)
|
||||
for n in distinct_names
|
||||
if n.lower() in name_to_id
|
||||
]
|
||||
log.info("Will UPDATE %d distinct names that resolve to clan_ids", len(updates))
|
||||
if updates:
|
||||
cur = con.executemany(
|
||||
"UPDATE player_games_hist SET clan_id = ? "
|
||||
"WHERE squadron_name = ? AND clan_id IS NULL",
|
||||
updates,
|
||||
)
|
||||
con.commit()
|
||||
log.info("Backfill: %s row-updates committed", cur.rowcount)
|
||||
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(player_games_hist)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
still_null = con.execute(
|
||||
"SELECT COUNT(*) FROM player_games_hist WHERE clan_id IS NULL"
|
||||
).fetchone()[0]
|
||||
log.info("player_games_hist orphans (clan_id NULL after backfill): %d", still_null)
|
||||
else:
|
||||
log.info("player_games_hist orphans: n/a (column absent in dry-run)")
|
||||
|
||||
log.info("Ensuring index on player_games_hist(clan_id, endtime_unix)")
|
||||
if apply:
|
||||
con.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_pgh_clanid_endtime "
|
||||
"ON player_games_hist(clan_id, endtime_unix)"
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def migrate_match_summary(apply: bool) -> None:
|
||||
"""Add winning_clan_id + losing_clan_id, backfill via squadrons_data."""
|
||||
con = sqlite3.connect(SQ_BATTLES_DB)
|
||||
try:
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(match_summary)").fetchall()]
|
||||
for new_col in ("winning_clan_id", "losing_clan_id"):
|
||||
if new_col not in cols:
|
||||
log.info("Adding %s column to match_summary", new_col)
|
||||
if apply:
|
||||
con.execute(f"ALTER TABLE match_summary ADD COLUMN {new_col} INTEGER")
|
||||
con.commit()
|
||||
else:
|
||||
log.info("match_summary.%s already present", new_col)
|
||||
|
||||
# Refresh after potential ALTERs.
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(match_summary)").fetchall()]
|
||||
total = con.execute("SELECT COUNT(*) FROM match_summary").fetchone()[0]
|
||||
if "winning_clan_id" in cols and "losing_clan_id" in cols:
|
||||
unbackfilled = con.execute(
|
||||
"SELECT COUNT(*) FROM match_summary "
|
||||
"WHERE winning_clan_id IS NULL OR losing_clan_id IS NULL"
|
||||
).fetchone()[0]
|
||||
else:
|
||||
# Dry-run path: columns don't exist yet, so every row is "to be backfilled".
|
||||
unbackfilled = total
|
||||
log.info(
|
||||
"match_summary: %d total rows, %d need backfill",
|
||||
total,
|
||||
unbackfilled,
|
||||
)
|
||||
|
||||
if apply and unbackfilled:
|
||||
con.execute(f"ATTACH DATABASE '{SQUADRONS_DB}' AS sq")
|
||||
try:
|
||||
# winning_sq / losing_sq can be either short_name or tag_name
|
||||
# depending on replay metadata. Try short_name first since that's
|
||||
# what the autologger writes most often.
|
||||
for col_in, col_out in (
|
||||
("winning_sq", "winning_clan_id"),
|
||||
("losing_sq", "losing_clan_id"),
|
||||
):
|
||||
log.info("Backfilling %s ← %s via short_name then long_name", col_out, col_in)
|
||||
con.execute(
|
||||
f"""
|
||||
UPDATE match_summary
|
||||
SET {col_out} = (
|
||||
SELECT clan_id FROM sq.squadrons_data
|
||||
WHERE LOWER(squadrons_data.short_name) = LOWER(match_summary.{col_in})
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE {col_out} IS NULL AND {col_in} IS NOT NULL
|
||||
"""
|
||||
)
|
||||
con.execute(
|
||||
f"""
|
||||
UPDATE match_summary
|
||||
SET {col_out} = (
|
||||
SELECT clan_id FROM sq.squadrons_data
|
||||
WHERE LOWER(squadrons_data.long_name) = LOWER(match_summary.{col_in})
|
||||
LIMIT 1
|
||||
)
|
||||
WHERE {col_out} IS NULL AND {col_in} IS NOT NULL
|
||||
"""
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.execute("DETACH DATABASE sq")
|
||||
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(match_summary)").fetchall()]
|
||||
for col, label in (("winning_clan_id", "winning"), ("losing_clan_id", "losing")):
|
||||
if col not in cols:
|
||||
log.info("match_summary orphans (%s NULL): n/a (column absent in dry-run)", label)
|
||||
continue
|
||||
n = con.execute(
|
||||
f"SELECT COUNT(*) FROM match_summary WHERE {col} IS NULL"
|
||||
).fetchone()[0]
|
||||
log.info("match_summary orphans (%s NULL): %d", label, n)
|
||||
|
||||
log.info("Ensuring indexes on match_summary clan_id columns")
|
||||
if apply:
|
||||
con.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_ms_winning_clanid ON match_summary(winning_clan_id)"
|
||||
)
|
||||
con.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_ms_losing_clanid ON match_summary(losing_clan_id)"
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def _resolve_squadron_to_clan_id(name: Optional[str], name_to_id: dict[str, int]) -> int:
|
||||
if not name:
|
||||
return -1
|
||||
return name_to_id.get(name.lower(), -1)
|
||||
|
||||
|
||||
def rebuild_points_db(apply: bool, long_to_id: dict[str, int]) -> None:
|
||||
"""Rebuild profile_member_points, profile_totals, game_cache with clan_id columns.
|
||||
|
||||
The squadron text column is preserved for reference but the new tables key
|
||||
off clan_id for primary keys. Resolution happens in Python so we don't have
|
||||
to ATTACH squadrons.db (avoids the cross-db lock issue we hit earlier).
|
||||
"""
|
||||
# Build name → clan_id map once (long_name AND short_name).
|
||||
sq_con = sqlite3.connect(f"file:{SQUADRONS_DB}?mode=ro", uri=True)
|
||||
try:
|
||||
sq_con.row_factory = sqlite3.Row
|
||||
name_to_id: dict[str, int] = {}
|
||||
for row in sq_con.execute(
|
||||
"SELECT clan_id, long_name, short_name FROM squadrons_data"
|
||||
):
|
||||
cid = int(row["clan_id"])
|
||||
if row["long_name"]:
|
||||
name_to_id.setdefault(row["long_name"].lower(), cid)
|
||||
if row["short_name"]:
|
||||
name_to_id.setdefault(row["short_name"].lower(), cid)
|
||||
finally:
|
||||
sq_con.close()
|
||||
|
||||
con = sqlite3.connect(POINTS_DB)
|
||||
try:
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
|
||||
# profile_member_points -------------------------------------------------
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(profile_member_points)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
log.info("profile_member_points already migrated; skipping rebuild")
|
||||
else:
|
||||
log.info("Rebuilding profile_member_points with clan_id PK")
|
||||
if apply:
|
||||
# Idempotency: a previous half-finished run may have left _new behind.
|
||||
con.execute("DROP TABLE IF EXISTS profile_member_points_new")
|
||||
con.execute(
|
||||
"""CREATE TABLE profile_member_points_new (
|
||||
clan_id INTEGER NOT NULL,
|
||||
squadron TEXT NOT NULL,
|
||||
uid TEXT NOT NULL,
|
||||
points INTEGER NOT NULL,
|
||||
PRIMARY KEY (clan_id, uid)
|
||||
)"""
|
||||
)
|
||||
src = con.execute(
|
||||
"SELECT squadron, uid, points FROM profile_member_points"
|
||||
).fetchall()
|
||||
rows = [
|
||||
(_resolve_squadron_to_clan_id(s, name_to_id), s, u, p)
|
||||
for (s, u, p) in src
|
||||
]
|
||||
con.executemany(
|
||||
"INSERT OR IGNORE INTO profile_member_points_new "
|
||||
"(clan_id, squadron, uid, points) VALUES (?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
con.commit()
|
||||
log.info("profile_member_points: copied %d rows", len(rows))
|
||||
con.execute("DROP TABLE profile_member_points")
|
||||
con.execute("ALTER TABLE profile_member_points_new RENAME TO profile_member_points")
|
||||
con.execute("CREATE INDEX IF NOT EXISTS idx_pmp_squadron ON profile_member_points(squadron)")
|
||||
con.commit()
|
||||
|
||||
# profile_totals --------------------------------------------------------
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(profile_totals)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
log.info("profile_totals already migrated; skipping rebuild")
|
||||
else:
|
||||
log.info("Rebuilding profile_totals with clan_id PK")
|
||||
if apply:
|
||||
con.execute("DROP TABLE IF EXISTS profile_totals_new")
|
||||
con.execute(
|
||||
"""CREATE TABLE profile_totals_new (
|
||||
clan_id INTEGER PRIMARY KEY,
|
||||
squadron TEXT NOT NULL,
|
||||
total INTEGER NOT NULL
|
||||
)"""
|
||||
)
|
||||
src = con.execute("SELECT squadron, total FROM profile_totals").fetchall()
|
||||
rows = [
|
||||
(_resolve_squadron_to_clan_id(s, name_to_id), s, t)
|
||||
for (s, t) in src
|
||||
]
|
||||
con.executemany(
|
||||
"INSERT OR IGNORE INTO profile_totals_new (clan_id, squadron, total) VALUES (?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
con.commit()
|
||||
log.info("profile_totals: copied %d rows", len(rows))
|
||||
con.execute("DROP TABLE profile_totals")
|
||||
con.execute("ALTER TABLE profile_totals_new RENAME TO profile_totals")
|
||||
con.commit()
|
||||
|
||||
# game_cache ------------------------------------------------------------
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(game_cache)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
log.info("game_cache already migrated; skipping rebuild")
|
||||
else:
|
||||
log.info("Rebuilding game_cache with clan_id in PK")
|
||||
if apply:
|
||||
con.execute("DROP TABLE IF EXISTS game_cache_new")
|
||||
con.execute(
|
||||
"""CREATE TABLE game_cache_new (
|
||||
game_id TEXT NOT NULL,
|
||||
clan_id INTEGER NOT NULL,
|
||||
squadron TEXT NOT NULL,
|
||||
diffs_json TEXT NOT NULL,
|
||||
diff_total INTEGER NOT NULL,
|
||||
updated_json TEXT NOT NULL,
|
||||
created_at REAL NOT NULL DEFAULT (strftime('%s','now')),
|
||||
PRIMARY KEY (game_id, clan_id)
|
||||
)"""
|
||||
)
|
||||
src = con.execute(
|
||||
"SELECT game_id, squadron, diffs_json, diff_total, updated_json, created_at FROM game_cache"
|
||||
).fetchall()
|
||||
rows = [
|
||||
(
|
||||
gid,
|
||||
_resolve_squadron_to_clan_id(s, name_to_id),
|
||||
s,
|
||||
dj,
|
||||
dt,
|
||||
uj,
|
||||
ca,
|
||||
)
|
||||
for (gid, s, dj, dt, uj, ca) in src
|
||||
]
|
||||
con.executemany(
|
||||
"INSERT OR IGNORE INTO game_cache_new "
|
||||
"(game_id, clan_id, squadron, diffs_json, diff_total, updated_json, created_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
rows,
|
||||
)
|
||||
con.commit()
|
||||
log.info("game_cache: copied %d rows", len(rows))
|
||||
con.execute("DROP TABLE game_cache")
|
||||
con.execute("ALTER TABLE game_cache_new RENAME TO game_cache")
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def rebuild_wl_db(apply: bool) -> None:
|
||||
"""wl_events / wl_standings are empty - swap schemas to clan_id keyed tables."""
|
||||
con = sqlite3.connect(WL_DB)
|
||||
try:
|
||||
# wl_standings
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(wl_standings)").fetchall()]
|
||||
if "clan_id" in cols:
|
||||
log.info("wl_standings already migrated")
|
||||
else:
|
||||
log.info("Recreating wl_standings keyed by clan_id (was empty)")
|
||||
if apply:
|
||||
con.executescript(
|
||||
"""
|
||||
DROP TABLE IF EXISTS wl_standings;
|
||||
CREATE TABLE wl_standings (
|
||||
clan_id INTEGER PRIMARY KEY,
|
||||
squadron TEXT NOT NULL,
|
||||
wins INTEGER NOT NULL DEFAULT 0,
|
||||
losses INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
"""
|
||||
)
|
||||
con.commit()
|
||||
|
||||
# wl_events: store winner_clan_id alongside winner text
|
||||
cols = [r[1] for r in con.execute("PRAGMA table_info(wl_events)").fetchall()]
|
||||
if "winner_clan_id" in cols:
|
||||
log.info("wl_events already migrated")
|
||||
else:
|
||||
log.info("Adding winner_clan_id column to wl_events (table is empty)")
|
||||
if apply:
|
||||
con.execute("ALTER TABLE wl_events ADD COLUMN winner_clan_id INTEGER")
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def migrate_preferences(
|
||||
apply: bool,
|
||||
long_to_id: dict[str, int],
|
||||
short_to_id: dict[str, int],
|
||||
by_id: dict[int, dict[str, Any]],
|
||||
) -> None:
|
||||
"""Re-key every PREFERENCES/<guild_id>-preferences.json from long_name → str(clan_id).
|
||||
|
||||
Special wildcard keys (Global, everything, all, *) are preserved as-is.
|
||||
Each migrated entry gets a `Long` field with the squadron's current
|
||||
long_name so display fallback works if squadrons_data is unavailable.
|
||||
Original files are copied to PREFERENCES_BACKUP_<timestamp>/.
|
||||
"""
|
||||
if not PREFS_DIR.exists():
|
||||
log.warning("PREFERENCES dir missing at %s; skipping prefs migration", PREFS_DIR)
|
||||
return
|
||||
|
||||
ts = time.strftime("%Y%m%d-%H%M%S")
|
||||
backup_dir = STORAGE_DIR / f"PREFERENCES_BACKUP_{ts}"
|
||||
if apply:
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pref_files = sorted(PREFS_DIR.glob("*-preferences.json"))
|
||||
log.info("Found %d preference files to migrate", len(pref_files))
|
||||
|
||||
migrated_count = 0
|
||||
orphan_count = 0
|
||||
skipped_count = 0
|
||||
files_changed = 0
|
||||
|
||||
for pref_file in pref_files:
|
||||
try:
|
||||
data = json.loads(pref_file.read_text(encoding="utf-8"))
|
||||
except Exception as e:
|
||||
log.warning("Could not read %s: %s", pref_file.name, e)
|
||||
continue
|
||||
|
||||
if not isinstance(data, dict):
|
||||
log.warning("Skipping %s: not an object", pref_file.name)
|
||||
continue
|
||||
|
||||
new_data: dict[str, Any] = {}
|
||||
any_changes = False
|
||||
|
||||
for key, value in data.items():
|
||||
if not isinstance(value, dict):
|
||||
new_data[key] = value
|
||||
continue
|
||||
|
||||
key_lower = str(key).lower()
|
||||
|
||||
# Preserve wildcards / Global
|
||||
if key_lower in WILDCARD_KEYS:
|
||||
new_data[key] = value
|
||||
continue
|
||||
|
||||
# Already a numeric clan_id (running migration twice)
|
||||
if str(key).isdigit():
|
||||
new_data[key] = value
|
||||
continue
|
||||
|
||||
# Try long_name then short_name
|
||||
cid = long_to_id.get(key_lower) or short_to_id.get(key_lower)
|
||||
|
||||
# Maybe entry has a "Short" hint we can use as fallback
|
||||
if cid is None and value.get("Short"):
|
||||
cid = short_to_id.get(str(value["Short"]).lower())
|
||||
|
||||
if cid is None:
|
||||
log.warning(
|
||||
" [%s] orphan key %r — squadron not found in squadrons_data",
|
||||
pref_file.name,
|
||||
key,
|
||||
)
|
||||
# Keep the original key so the user doesn't lose their data;
|
||||
# a future load_guild_preferences will surface it for cleanup.
|
||||
new_data[key] = value
|
||||
orphan_count += 1
|
||||
continue
|
||||
|
||||
new_key = str(cid)
|
||||
row = by_id.get(cid, {})
|
||||
merged = dict(value)
|
||||
# Preserve display fields - the bot uses them when squadrons_data is stale
|
||||
merged.setdefault("Long", row.get("long_name") or key)
|
||||
if row.get("short_name"):
|
||||
merged["Short"] = row["short_name"]
|
||||
|
||||
if new_key in new_data:
|
||||
# Two prefs entries for the same squadron (rare). Merge,
|
||||
# preferring values from this iteration.
|
||||
existing = new_data[new_key]
|
||||
if isinstance(existing, dict):
|
||||
existing.update(merged)
|
||||
new_data[new_key] = existing
|
||||
else:
|
||||
new_data[new_key] = merged
|
||||
else:
|
||||
new_data[new_key] = merged
|
||||
|
||||
if new_key != key:
|
||||
any_changes = True
|
||||
migrated_count += 1
|
||||
|
||||
if not any_changes:
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
if apply:
|
||||
backup_path = backup_dir / pref_file.name
|
||||
shutil.copy2(pref_file, backup_path)
|
||||
tmp = pref_file.with_suffix(".json.tmp")
|
||||
tmp.write_text(json.dumps(new_data, ensure_ascii=False), encoding="utf-8")
|
||||
os.replace(tmp, pref_file)
|
||||
files_changed += 1
|
||||
|
||||
log.info(
|
||||
"Preferences migration: %d files changed, %d unchanged, %d entries migrated, %d orphans",
|
||||
files_changed,
|
||||
skipped_count,
|
||||
migrated_count,
|
||||
orphan_count,
|
||||
)
|
||||
if apply:
|
||||
log.info("Preference backups saved to %s", backup_dir)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true", help="Actually apply changes")
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Print what would happen without writing anything (default)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-backup",
|
||||
action="store_true",
|
||||
help="Skip the tarball backup step (use only if you already have one)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.apply and args.dry_run:
|
||||
log.error("--apply and --dry-run are mutually exclusive")
|
||||
return 2
|
||||
|
||||
apply = args.apply
|
||||
|
||||
log.info("STORAGE_DIR=%s", STORAGE_DIR)
|
||||
log.info("apply=%s", apply)
|
||||
|
||||
for db in (SQ_BATTLES_DB, SQUADRONS_DB, POINTS_DB, WL_DB):
|
||||
if not db.exists():
|
||||
log.error("Required DB missing: %s", db)
|
||||
return 1
|
||||
|
||||
if apply and not args.skip_backup:
|
||||
backup_storage()
|
||||
|
||||
long_to_id, short_to_id, by_id = load_squadron_index()
|
||||
|
||||
ensure_squadron_name_history(apply, by_id)
|
||||
add_squadrons_points_index(apply)
|
||||
migrate_player_games_hist(apply, long_to_id)
|
||||
migrate_match_summary(apply)
|
||||
rebuild_points_db(apply, long_to_id)
|
||||
rebuild_wl_db(apply)
|
||||
migrate_preferences(apply, long_to_id, short_to_id, by_id)
|
||||
|
||||
if not apply:
|
||||
log.info("DRY RUN complete. Re-run with --apply to actually write changes.")
|
||||
else:
|
||||
log.info("Migration applied successfully.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user