add tss tournament stuff (#1348)

scripts/backfill_tournaments.py

@@ -8,6 +8,7 @@ import asyncio
 import pathlib
 import sqlite3
 import sys
+import time
 from typing import List, Optional, Tuple
 
 ROOT = pathlib.Path(__file__).resolve().parents[1]
@@ -23,7 +24,12 @@ except Exception:
     pass
 
 from BOT.storage import TSS_BATTLES_DB_PATH  # noqa: E402
-from BOT.tss_tournaments import init_tss_tournaments_db, scan_and_store  # noqa: E402
+from BOT.tss_tournaments import (  # noqa: E402
+    TSS_TOURNAMENTS_DB_PATH,
+    build_scan_sync,
+    init_tss_tournaments_db,
+    store_scan,
+)
 
 
 async def tournament_ids(limit: Optional[int]) -> List[Tuple[int, Optional[str]]]:
@@ -45,24 +51,59 @@ async def tournament_ids(limit: Optional[int]) -> List[Tuple[int, Optional[str]]
     return [(int(row[0]), row[1]) for row in rows]
 
 
+def scanned_tournament_ids() -> set[int]:
+    if not TSS_TOURNAMENTS_DB_PATH.exists():
+        return set()
+    with sqlite3.connect(TSS_TOURNAMENTS_DB_PATH) as conn:
+        try:
+            rows = conn.execute(
+                "SELECT tournament_id FROM tournaments WHERE scanned_unix IS NOT NULL"
+            ).fetchall()
+        except sqlite3.OperationalError:
+            return set()
+    return {int(row[0]) for row in rows}
+
+
 async def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--limit", type=int, default=None)
     parser.add_argument("--sleep", type=float, default=1.0)
+    parser.add_argument("--battle-workers", type=int, default=8)
+    parser.add_argument("--rescan", action="store_true", help="rescan tournaments already present in tss_tournaments.db")
     args = parser.parse_args()
 
     rows = await tournament_ids(args.limit)
     print(f"Found {len(rows)} tournament ids in {TSS_BATTLES_DB_PATH}")
+    await init_tss_tournaments_db()
+    if not args.rescan:
+        done = scanned_tournament_ids()
+        if done:
+            before = len(rows)
+            rows = [(tid, name) for tid, name in rows if tid not in done]
+            print(f"Skipping {before - len(rows)} already-scanned tournaments ({len(rows)} remaining)")
     if args.dry_run:
         for tid, name in rows:
             print(f"  {tid}: {name or 'Tournament ' + str(tid)}")
         return
 
-    await init_tss_tournaments_db()
     for index, (tid, name) in enumerate(rows, start=1):
-        print(f"[{index}/{len(rows)}] scanning tournament {tid}")
-        await scan_and_store(tid, fallback_name=name)
+        started = time.monotonic()
+        print(f"[{index}/{len(rows)}] scanning tournament {tid}", flush=True)
+        scan = build_scan_sync(
+            tid,
+            fallback_name=name,
+            battle_workers=args.battle_workers,
+            progress=lambda msg, tid=tid: print(f"  {tid}: {msg}", flush=True),
+        )
+        await store_scan(scan)
+        elapsed = time.monotonic() - started
+        print(
+            f"  stored {tid}: {scan['match_count']} matches, "
+            f"{len(scan['battles'])} battles, {len(scan['standings'])} standings "
+            f"({scan['status']}) in {elapsed:.1f}s",
+            flush=True,
+        )
         if args.sleep and index < len(rows):
             await asyncio.sleep(args.sleep)