Auto merge dev → main (#1332)

* feat(tssbot): build_match_logs + match_logs persistence

* feat(tssbot): create match_logs table and write logs at ingest

* feat(tssbot): one-time match_logs backfill script

* feat(srebot): persist chat/battle logs to match_logs (parity, no backfill)

* feat(tssbot): Battle/Chat Log buttons on Discord scoreboards

BOT/autologging.py

@@ -11,7 +11,9 @@ only receives one scoreboard for a given game.
 from __future__ import annotations
 
 import asyncio
+import json
 import logging
+import sqlite3
 from pathlib import Path
 from typing import Any, Optional
 
@@ -98,6 +100,53 @@ def _bar_color(game: dict[str, Any], guild_id: int) -> str:
 # Scoreboard view + render/send
 # ---------------------------------------------------------------------------
 
+def _load_match_logs(session_id: str) -> tuple[list[str], list[str]]:
+    """Read (chat_log, battle_log) for a session from match_logs; empty on miss."""
+    from .storage import TSS_BATTLES_DB_PATH
+    try:
+        conn = sqlite3.connect(TSS_BATTLES_DB_PATH)
+        try:
+            row = conn.execute(
+                "SELECT chat_log_json, battle_log_json FROM match_logs WHERE session_id = ?",
+                (str(session_id),),
+            ).fetchone()
+        finally:
+            conn.close()
+    except Exception:
+        return [], []
+    if not row:
+        return [], []
+    chat = json.loads(row[0]) if row[0] else []
+    battle = json.loads(row[1]) if row[1] else []
+    return chat, battle
+
+
+async def _send_log(interaction: discord.Interaction, lines: list[str], title: str) -> None:
+    """Send a log as ephemeral diff-formatted message(s), chunked under Discord's limit."""
+    await interaction.response.defer(thinking=True, ephemeral=True)
+    if not lines:
+        await interaction.followup.send("No log available for this match.", ephemeral=True)
+        return
+    chunks: list[str] = []
+    chunk: list[str] = []
+    length = 0
+    for line in lines:
+        if length + len(line) + 1 > 1800:
+            chunks.append("\n".join(chunk))
+            chunk = [line]
+            length = len(line) + 1
+        else:
+            chunk.append(line)
+            length += len(line) + 1
+    if chunk:
+        chunks.append("\n".join(chunk))
+    first = True
+    for c in chunks:
+        content = (f"**{title}**\n" if first else "") + f"```diff\n{c}\n```"
+        await interaction.followup.send(content, ephemeral=True)
+        first = False
+
+
 def build_tss_scoreboard_view(session_id: str) -> discord.ui.View:
     """Link buttons under a scoreboard: in-game replay + the TSS website."""
     view = discord.ui.View(timeout=None)
@@ -110,6 +159,28 @@ def build_tss_scoreboard_view(session_id: str) -> discord.ui.View:
         label="View on Website", style=discord.ButtonStyle.link,
         url=f"https://tss.pawjob.us/games/{session_id}", emoji="🌐",
     ))
+
+    chat_log, battle_log = _load_match_logs(session_id)
+
+    battle_btn = discord.ui.Button(label="Battle Log", style=discord.ButtonStyle.green)
+
+    async def _battle_cb(interaction: discord.Interaction) -> None:
+        _, b = _load_match_logs(session_id)
+        await _send_log(interaction, b, f"Battle Log · {session_id}")
+
+    battle_btn.callback = _battle_cb
+    view.add_item(battle_btn)
+
+    if chat_log:
+        chat_btn = discord.ui.Button(label="Chat Log", style=discord.ButtonStyle.green)
+
+        async def _chat_cb(interaction: discord.Interaction) -> None:
+            c, _ = _load_match_logs(session_id)
+            await _send_log(interaction, c, f"Chat Log · {session_id}")
+
+        chat_btn.callback = _chat_cb
+        view.add_item(chat_btn)
+
     return view
 
 

BOT/match_logs.py

@@ -0,0 +1,184 @@
+"""Chat + battle log builder and persistence for TSS matches.
+
+Ports SREBOT/BOT/utils.py log formatting so both sites render identically.
+Logs are stored in match_logs (tss_battles.db) as JSON string arrays.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import logging
+from typing import Any
+
+import aiosqlite
+
+try:
+    from data_parser import LangTableReader, apply_vehicle_name_filters  # type: ignore
+except Exception:  # pragma: no cover
+    LangTableReader = None  # type: ignore
+
+    def apply_vehicle_name_filters(name, strip_decorations=True):  # type: ignore
+        return name
+
+log = logging.getLogger("tssbot.match_logs")
+
+MATCH_LOGS_SQL = """
+CREATE TABLE IF NOT EXISTS match_logs (
+    session_id      TEXT PRIMARY KEY,
+    chat_log_json   TEXT,
+    battle_log_json TEXT,
+    built_unix      INTEGER
+)
+"""
+
+
+def _fmt_time(ms: int) -> str:
+    total_s = int(ms) // 1000
+    return f"{total_s // 60:02d}:{total_s % 60:02d}"
+
+
+def _decompress_events(raw_events: Any) -> dict:
+    if isinstance(raw_events, str):
+        try:
+            import zstandard
+            compressed = base64.b85decode(raw_events)
+            return json.loads(zstandard.decompress(compressed).decode("utf-8"))
+        except Exception as exc:  # pragma: no cover - corrupt payloads
+            log.error("Failed to decompress events: %s", exc)
+            return {}
+    return raw_events or {}
+
+
+def _strip_tag(tag: str) -> str:
+    s = (tag or "").strip()
+    if len(s) >= 3 and not s[0].isalnum() and not s[-1].isalnum():
+        return s[1:-1]
+    return s
+
+
+def build_match_logs(game: dict[str, Any]) -> tuple[list[str], list[str]]:
+    """Return (chat_log, battle_log) as pre-formatted string lists."""
+    players = game.get("players", {}) or {}
+    winner_winged = str(game.get("winner") or "")
+    loser_winged = str(game.get("loser") or "")
+    winner_sq = _strip_tag(winner_winged)
+    loser_sq = _strip_tag(loser_winged)
+
+    uid_lookup: dict[str, dict[str, str]] = {}
+    for uid_str, p in players.items():
+        tag = p.get("tag", "") or ""
+        uid_lookup[str(uid_str)] = {
+            "name": p.get("name", "") or "",
+            "tag_stripped": tag[1:-1] if tag else "",
+        }
+
+    translate = None
+    if LangTableReader is not None:
+        try:
+            translate = LangTableReader("English")
+        except Exception:  # pragma: no cover
+            translate = None
+
+    def _veh(cdk: str | None) -> str:
+        if not cdk:
+            return "Unknown"
+        if translate is not None:
+            t = translate.get_translate(cdk)
+            if t:
+                return apply_vehicle_name_filters(t)
+        return cdk
+
+    def _player(uid: str | None) -> tuple[str, str]:
+        if uid is None:
+            return "Unknown", ""
+        info = uid_lookup.get(str(uid))
+        if info:
+            return info["name"], info["tag_stripped"]
+        return f"Player#{uid}", ""
+
+    def _prefix(sq: str) -> str:
+        if sq == winner_sq:
+            return "+"
+        if sq == loser_sq:
+            return "-"
+        return " "
+
+    chat_log: list[str] = []
+    for c in game.get("chat", []) or []:
+        info = uid_lookup.get(
+            str(c.get("uid", "")), {"name": "Unknown", "tag_stripped": "???"}
+        )
+        chat_log.append(
+            f"[{_fmt_time(c.get('time', 0))}] [{c.get('type', 'ALL')}] "
+            f"[{info['tag_stripped']}] `{info['name']}`: {c.get('message', '')}"
+        )
+
+    events = _decompress_events(game.get("events", {}))
+    merged: list[dict[str, Any]] = []
+    for k in events.get("kills", []) or []:
+        merged.append({
+            "kind": "kill", "time": k.get("time", 0),
+            "off_uid": str(k["offender_uid"]) if k.get("offender_uid") is not None else None,
+            "off_unit": k.get("offender_unit"),
+            "vic_uid": str(k["offended_uid"]) if k.get("offended_uid") is not None else None,
+            "vic_unit": k.get("offended_unit"),
+            "crashed": k.get("crashed", False), "afire": False,
+        })
+    for d in events.get("damage", []) or []:
+        merged.append({
+            "kind": "damage", "time": d.get("time", 0),
+            "off_uid": str(d["offender_uid"]) if d.get("offender_uid") is not None else None,
+            "off_unit": d.get("offender_unit"),
+            "vic_uid": str(d["offended_uid"]) if d.get("offended_uid") is not None else None,
+            "vic_unit": d.get("offended_unit"),
+            "crashed": False, "afire": d.get("afire", False),
+        })
+    merged.sort(key=lambda e: e.get("time", 0))
+
+    battle_log: list[str] = []
+    for ev in merged:
+        ts = _fmt_time(ev["time"])
+        vic_name, vic_sq = _player(ev["vic_uid"])
+        vic_veh = _veh(ev["vic_unit"])
+        if ev["kind"] == "kill":
+            if ev["off_uid"] is None or ev["crashed"]:
+                battle_log.append(
+                    f"{_prefix(vic_sq)}[{ts}] {f'[{vic_sq}]':<7} "
+                    f"{vic_name} ({vic_veh}) crashed"
+                )
+            else:
+                name, sq = _player(ev["off_uid"])
+                battle_log.append(
+                    f"{_prefix(sq)}[{ts}] {f'[{sq}]':<7} {name} ({_veh(ev['off_unit'])}) "
+                    f"destroyed {vic_name} ({vic_veh})"
+                )
+        else:
+            if ev["off_uid"] is None:
+                continue
+            name, sq = _player(ev["off_uid"])
+            afire = "(FIRE) " if ev["afire"] else ""
+            battle_log.append(
+                f"{_prefix(sq)}[{ts}] {f'[{sq}]':<7} {name} ({_veh(ev['off_unit'])}) "
+                f"damaged {afire}{vic_name} ({vic_veh})"
+            )
+
+    return chat_log, battle_log
+
+
+async def upsert_match_logs(
+    db_path, session_id: str, chat_log: list[str], battle_log: list[str]
+) -> None:
+    import time
+    async with aiosqlite.connect(db_path) as conn:
+        await conn.execute(MATCH_LOGS_SQL)
+        await conn.execute(
+            """INSERT INTO match_logs (session_id, chat_log_json, battle_log_json, built_unix)
+               VALUES (?, ?, ?, ?)
+               ON CONFLICT(session_id) DO UPDATE SET
+                   chat_log_json=excluded.chat_log_json,
+                   battle_log_json=excluded.battle_log_json,
+                   built_unix=excluded.built_unix""",
+            (str(session_id), json.dumps(chat_log, ensure_ascii=False),
+             json.dumps(battle_log, ensure_ascii=False), int(time.time())),
+        )
+        await conn.commit()

BOT/storage.py

@@ -222,6 +222,8 @@ async def _init_battles_db() -> None:
             await conn.execute(sql)
         await conn.execute(_MATCH_SUMMARY_SQL)
         await conn.execute(_PLAYER_GAMES_SQL)
+        from BOT.match_logs import MATCH_LOGS_SQL
+        await conn.execute(MATCH_LOGS_SQL)
         await _rebuild_table(conn, "match_summary", _MATCH_SUMMARY_SQL, [
             "session_id", "mission_mode", "mission_name", "level_path",
             "mission_path", "difficulty",

scripts/backfill_match_logs.py

@@ -0,0 +1,49 @@
+"""Backfill match_logs from on-disk TSS replays (one-time).
+
+Usage: python TSSBOT/scripts/backfill_match_logs.py [--dry-run]
+"""
+import asyncio
+import gzip
+import json
+import os
+import sys
+import pathlib
+
+ROOT = pathlib.Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT.parent / "SHARED"))
+
+from BOT.match_logs import build_match_logs, upsert_match_logs, MATCH_LOGS_SQL
+from BOT.storage import TSS_BATTLES_DB_PATH
+
+REPLAYS = pathlib.Path(os.environ["STORAGE_VOL_PATH"]) / "REPLAYS" / "TSS"
+
+
+async def main(dry_run: bool) -> None:
+    import aiosqlite
+    files = sorted(REPLAYS.glob("*/replay_data.json.gz"))
+    print(f"Found {len(files)} replays under {REPLAYS}")
+    async with aiosqlite.connect(TSS_BATTLES_DB_PATH) as conn:
+        await conn.execute(MATCH_LOGS_SQL)
+        await conn.commit()
+    done = 0
+    for f in files:
+        sid = f.parent.name
+        try:
+            with gzip.open(f, "rb") as fh:
+                game = json.loads(fh.read().decode("utf-8"))
+        except Exception as exc:
+            print(f"  skip {sid}: {exc}")
+            continue
+        chat, battle = build_match_logs(game)
+        if dry_run:
+            print(f"  {sid}: chat={len(chat)} battle={len(battle)}")
+        else:
+            await upsert_match_logs(TSS_BATTLES_DB_PATH, sid, chat, battle)
+            done += 1
+    print(f"{'Would backfill' if dry_run else 'Backfilled'} "
+          f"{len(files) if dry_run else done} sessions")
+
+
+if __name__ == "__main__":
+    asyncio.run(main("--dry-run" in sys.argv))

tests/test_match_logs.py

@@ -0,0 +1,60 @@
+import sys
+import pathlib
+
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
+sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2] / "SHARED"))
+
+from BOT.match_logs import build_match_logs
+
+
+def _game():
+    return {
+        "_id": "abc",
+        "winner": "[WIN]", "loser": "[LOS]",
+        "players": {
+            "1": {"name": "alice", "tag": "[WIN]", "team": "1",
+                  "units": [{"unit": "ussr_t_34", "used": True}],
+                  "air_kills": 0, "ground_kills": 1, "assists": 0,
+                  "deaths": 0, "captures": 0, "score": 100},
+            "2": {"name": "bob", "tag": "[LOS]", "team": "2",
+                  "units": [{"unit": "germ_pz_iv", "used": True}],
+                  "air_kills": 0, "ground_kills": 0, "assists": 0,
+                  "deaths": 1, "captures": 0, "score": 10},
+        },
+        "chat": [{"uid": "1", "type": "ALL", "message": "gg", "time": 65000}],
+        "events": {"kills": [{"time": 60000, "offender_uid": 1,
+                              "offender_unit": "ussr_t_34",
+                              "offended_uid": 2, "offended_unit": "germ_pz_iv",
+                              "crashed": False}]},
+    }
+
+
+def test_chat_log_format():
+    chat, _ = build_match_logs(_game())
+    assert chat == ["[01:05] [ALL] [WIN] `alice`: gg"]
+
+
+def test_battle_log_kill_prefix_and_text():
+    _, battle = build_match_logs(_game())
+    assert len(battle) == 1
+    line = battle[0]
+    assert line.startswith("+[01:00] [WIN]")
+    assert "alice" in line and "destroyed" in line and "bob" in line
+
+
+def test_events_base85_zstd_compressed():
+    import base64
+    import json
+    import zstandard
+    g = _game()
+    raw = json.dumps(g["events"]).encode()
+    g["events"] = base64.b85encode(zstandard.ZstdCompressor().compress(raw)).decode()
+    _, battle = build_match_logs(g)
+    assert battle and "destroyed" in battle[0]
+
+
+if __name__ == "__main__":
+    test_chat_log_format()
+    test_battle_log_kill_prefix_and_text()
+    test_events_base85_zstd_compressed()
+    print("ALL TESTS PASSED")

tss_ws.py

@@ -164,6 +164,10 @@ async def _handle_game(game: Dict[str, Any]) -> None:
         await insert_match(game)
         await insert_player_games(game)
         await upsert_tss_teams(game)
+        from BOT.match_logs import build_match_logs, upsert_match_logs
+        from BOT.storage import TSS_BATTLES_DB_PATH
+        chat_log, battle_log = build_match_logs(game)
+        await upsert_match_logs(TSS_BATTLES_DB_PATH, sid, chat_log, battle_log)
         log.info("Stored game %s in DB", sid)
     except Exception as exc:
         log.error("DB insert failed for %s: %s", sid, exc)