From c0214eaaae15553b0c09e5de9b9267f953d15060 Mon Sep 17 00:00:00 2001 From: deploy Date: Wed, 1 Jul 2026 19:20:26 +0000 Subject: [PATCH] fix(recap): speed up /card lookup, fix stale completed-season cache, add Place Finished MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /card player lookup was a leading-wildcard substring match with a Python ulower() UDF over 6.16M player_games_hist rows — a full scan measured at 27s live before the ~2s render. Add a prefix fast-path (nick LIKE 'name%' COLLATE NOCASE) that uses the existing NOCASE index (~1ms), falling back to the ulower substring scan only when the prefix finds nothing. Same fix applied to _resolve_player_uids_batch (compare/stats). Lookup: 27,205ms -> 1ms. - Completed-season recap cache served ANY cached PNG forever, including files rendered mid-season (frozen at whatever point they were last viewed). Only serve from cache when the file's mtime is after season end; otherwise re-render. Fixed in both BOT/utils.py and web/server.js (shared cache). - Replace squadron card "Rating change" with "Place finished" (#rank / total), derived by ranking clans by final total_score among those active in-season. - Add (CARD)/(RECAP) timing logs for prod observability. Co-Authored-By: Claude Opus 4.8 --- BOT/botscript.py | 66 ++++++++++++++++++++++++++++++++------------- BOT/render_recap.py | 43 ++++++++++++++++++++++++++++- BOT/utils.py | 16 ++++++++++- web/locales/en.json | 1 + web/server.js | 10 +++++-- 5 files changed, 114 insertions(+), 22 deletions(-) diff --git a/BOT/botscript.py b/BOT/botscript.py index 203b4e2..e0d0617 100644 --- a/BOT/botscript.py +++ b/BOT/botscript.py @@ -3574,21 +3574,41 @@ async def card( ) try: + _t_lookup = time_module.monotonic() async with aiosqlite.connect(SQ_BATTLES_DB_PATH) as db: db.row_factory = aiosqlite.Row - await db.create_function("ulower", 1, str.lower) - async with db.execute( - """ - SELECT UID, MIN(nick) AS nick - FROM player_games_hist - WHERE ulower(nick) LIKE ulower(?) - GROUP BY UID - ORDER BY nick - LIMIT 25 - """, - (f"%{player}%",), - ) as cursor: - results = list(await cursor.fetchall()) + + async def _lookup(where: str, pattern: str) -> list: + async with db.execute( + f""" + SELECT UID, MIN(nick) AS nick + FROM player_games_hist + WHERE {where} + GROUP BY UID + ORDER BY nick + LIMIT 25 + """, + (pattern,), + ) as cursor: + return list(await cursor.fetchall()) + + # Fast path: a prefix match uses the `nick COLLATE NOCASE` index + # (~ms), covering the common case of typing the start of a name. + # This avoids a full scan of the multi-million-row games table. + results = await _lookup("nick LIKE ? COLLATE NOCASE", f"{player}%") + lookup_path = "prefix" + + # Fallback: only if the prefix found nothing do we pay for a full + # substring scan. ulower() gives full Unicode case-folding (e.g. + # Cyrillic) that the ASCII-only NOCASE collation can't. + if not results: + await db.create_function("ulower", 1, str.lower) + results = await _lookup("ulower(nick) LIKE ulower(?)", f"%{player}%") + lookup_path = "substring" + logging.info( + "(CARD) nick lookup query=%r path=%s matches=%d ms=%d", + player, lookup_path, len(results), (time_module.monotonic() - _t_lookup) * 1000, + ) except Exception as e: error_str = str(e)[:1800] return await interaction.followup.send( @@ -5024,19 +5044,29 @@ _LOWER_IS_BETTER = {"deaths", "losses"} async def _resolve_player_uids_batch(db, usernames: List[str], lang: str = "en"): """Resolve multiple usernames in one pass. Returns (uid_list, error_msg) or (None, error_msg).""" uids: List[str] = [] - for name in usernames: + + async def _resolve_one(where: str, pattern: str) -> list: async with db.execute( - """ + f""" SELECT UID, nick FROM ( SELECT UID, nick, ROW_NUMBER() OVER (PARTITION BY UID ORDER BY endtime_unix DESC) as rn FROM player_games_hist - WHERE ulower(nick) LIKE ulower(?) + WHERE {where} ) WHERE rn = 1 ORDER BY nick LIMIT 25 """, - (f"%{name}%",), + (pattern,), ) as cursor: - results = list(await cursor.fetchall()) + return list(await cursor.fetchall()) + + for name in usernames: + # Fast path: prefix match uses the nick COLLATE NOCASE index (~ms). + # Only fall back to a full substring scan (ulower for Unicode + # case-folding) when the prefix finds nothing. + results = await _resolve_one("nick LIKE ? COLLATE NOCASE", f"{name}%") + if not results: + await db.create_function("ulower", 1, str.lower) + results = await _resolve_one("ulower(nick) LIKE ulower(?)", f"%{name}%") if not results: return None, t(lang, "compare.no_players_found", name=name) if len(results) > 1: diff --git a/BOT/render_recap.py b/BOT/render_recap.py index c77f10a..fda9086 100644 --- a/BOT/render_recap.py +++ b/BOT/render_recap.py @@ -369,6 +369,43 @@ def gather_squadron_rating(conn_sq: sqlite3.Connection, clan_id: int, first=first, change=final - first) +def gather_squadron_place(conn_sq: sqlite3.Connection, clan_id: int, + season_start: int, season_end: int + ) -> Optional[Tuple[int, int]]: + """Finishing leaderboard place for the season. + + We don't store a per-season position, but ``total_score`` in + squadrons_points IS the squadron rating that the in-game leaderboard ranks + by. So rank clans by their final score and read off this clan's rank. + + Only clans with at least one snapshot *within the season window* are ranked + — a clan whose last data predates the season never appeared on that season's + ladder (dead/inactive clans drop off in-game), so counting them would inflate + the denominator. Each ranked clan uses its last in-season snapshot. + + Returns (place, total_ranked) or None if this clan wasn't active in-season. + Works for in-progress seasons too (season_end in the future → last snapshot + so far = current standing). + """ + cur = conn_sq.execute( + "SELECT p.clan_id, p.total_score " + "FROM squadrons_points p " + "JOIN (SELECT clan_id, MAX(unix_time) AS mt FROM squadrons_points " + " WHERE unix_time BETWEEN ? AND ? GROUP BY clan_id) m " + " ON m.clan_id = p.clan_id AND m.mt = p.unix_time", + (season_start, season_end), + ) + board = [(cid, sc) for cid, sc in cur.fetchall() if sc is not None] + if not board: + return None + board.sort(key=lambda r: r[1], reverse=True) + total = len(board) + for i, (cid, _sc) in enumerate(board): + if cid == clan_id: + return i + 1, total + return None + + @dataclass class MatchRow: session_id: str @@ -1070,6 +1107,7 @@ def render_squadron_card(out_path: Path, ident: SquadronIdent, season: str, rolling_battles: List[Tuple[int, int]], season_start: int, season_end: int, week_boundaries: List[int], + place: Optional[Tuple[int, int]] = None, theme: str = "light", lang: str = DEFAULT_LANG) -> None: pal = theme_palette(theme) @@ -1219,7 +1257,7 @@ def render_squadron_card(out_path: Path, ident: SquadronIdent, season: str, # Short rows render in 2 columns; wide rows span full width. short_rows: List[Tuple[str, str]] = [ (t("imgStatPeakRating"), f"{_fmt_int(rating.peak)} ({_fmt_date(rating.peak_ts)})"), - (t("imgStatRatingChange"), f"{'+' if (rating.change or 0) >= 0 else ''}{_fmt_int(rating.change)}"), + (t("imgStatPlaceFinished"), f"#{_fmt_int(place[0])} / {_fmt_int(place[1])}" if place else "—"), (t("imgStatTotalKills"), f"{_fmt_int(players.total_kills)} ({_fmt_int(players.ground_kills)} {t('imgGroundShort')} / {_fmt_int(players.air_kills)} {t('imgAirShort')})"), (t("imgStatTotalDeaths"), _fmt_int(players.deaths)), (t("imgStatAssistsCaptures"), f"{_fmt_int(players.assists)} / {_fmt_int(players.captures)}"), @@ -1677,6 +1715,8 @@ def run_squadron(args: Args) -> int: logging.info(f"resolved {ident.short_name} / {ident.long_name}") rating = gather_squadron_rating(conn_sq, args.clan_id, args.season_start, args.season_end) logging.info(f"rating: series={len(rating.series)} final={rating.final} peak={rating.peak} change={rating.change}") + place = gather_squadron_place(conn_sq, args.clan_id, args.season_start, args.season_end) + logging.info(f"place: {place}") with _open_ro(SQ_BATTLES_DB) as conn_b: stream = gather_squadron_match_stream(conn_b, ident.short_name, args.season_start, args.season_end) @@ -1707,6 +1747,7 @@ def run_squadron(args: Args) -> int: rolling_wr, rolling_kd, rolling_battles, args.season_start, args.season_end, args.week_boundaries, + place=place, theme=args.theme, lang=args.lang, ) diff --git a/BOT/utils.py b/BOT/utils.py index 8c1ba72..8af2152 100644 --- a/BOT/utils.py +++ b/BOT/utils.py @@ -2161,14 +2161,19 @@ async def _get_recap( try: stat = cache_path.stat() if season_range["status"] == "completed": - serve_from_cache = True + # A completed season's card is only final if it was rendered AFTER + # the season ended. Files rendered mid-season are stale snapshots + # frozen at whatever point they were last viewed — re-render those. + serve_from_cache = stat.st_mtime > season_range["end"] elif (time.time() - stat.st_mtime) < RECAP_TTL_SECONDS: serve_from_cache = True except FileNotFoundError: pass + ident = clan_id if mode == "squadron" else uid if not serve_from_cache: cache_dir.mkdir(parents=True, exist_ok=True) + t0 = time.monotonic() await _spawn_recap_render( mode, clan_id=clan_id, @@ -2181,6 +2186,15 @@ async def _get_recap( theme=theme, lang=lang, ) + logging.info( + "(RECAP) %s id=%s season=%s theme=%s lang=%s result=fresh ms=%d", + mode, ident, season, theme, lang, (time.monotonic() - t0) * 1000, + ) + else: + logging.info( + "(RECAP) %s id=%s season=%s theme=%s lang=%s result=cache", + mode, ident, season, theme, lang, + ) if not cache_path.exists(): raise RecapError("recap file missing after render") diff --git a/web/locales/en.json b/web/locales/en.json index cd442a5..13bc906 100644 --- a/web/locales/en.json +++ b/web/locales/en.json @@ -702,6 +702,7 @@ "imgAxisWinRate": "Win Rate (%)", "imgStatPeakRating": "Peak rating", "imgStatRatingChange": "Rating change", + "imgStatPlaceFinished": "Place finished", "imgStatTotalKills": "Total kills", "imgStatTotalDeaths": "Total deaths", "imgStatAssistsCaptures": "Assists / captures", diff --git a/web/server.js b/web/server.js index 2871771..2075aa4 100644 --- a/web/server.js +++ b/web/server.js @@ -1044,7 +1044,10 @@ app.get('/squadron/:clan_id/recap/:season.png', async (req, res) => { try { const stat = await fsp.stat(cachePath); if (range.status === 'completed') { - serveFromCache = true; + // Only trust a completed-season cache if it was rendered after the + // season ended; mid-season files are stale snapshots (range.end is + // epoch seconds, mtimeMs is ms). + serveFromCache = stat.mtimeMs > range.end * 1000; } else if (Date.now() - stat.mtimeMs < RECAP_TTL_MS) { serveFromCache = true; } @@ -1112,7 +1115,10 @@ app.get('/players/:uid/recap/:season.png', async (req, res) => { try { const stat = await fsp.stat(cachePath); if (range.status === 'completed') { - serveFromCache = true; + // Only trust a completed-season cache if it was rendered after the + // season ended; mid-season files are stale snapshots (range.end is + // epoch seconds, mtimeMs is ms). + serveFromCache = stat.mtimeMs > range.end * 1000; } else if (Date.now() - stat.mtimeMs < RECAP_TTL_MS) { serveFromCache = true; }