compress the mf (#1267)

BOT/autologging.py

@@ -8,6 +8,7 @@ scoreboards, and posts results to configured Discord channels.
 
 # Standard Library Imports
 import asyncio
+import gzip
 import json
 import logging
 import os
@@ -389,10 +390,10 @@ def minutes_ago(ts, now=None):
 
 
 def load_replay_data_from_disk(session_id: str):
-    """Load replay_data.json from disk for a session."""
+    """Load replay_data.json.gz from disk for a session."""
     path = replay_data_path(session_id)
     if path.is_file():
-        with open(path, "r", encoding="utf-8") as f:
+        with gzip.open(path, "rt", encoding="utf-8") as f:
             return json.load(f)
     return None
 
@@ -477,13 +478,14 @@ async def process_ws_replays(replays: list[dict]):
         # Replay is valid - save to disk
         replay_dir = replay_session_dir(hex_id)
         replay_dir.mkdir(parents=True, exist_ok=True)
-        replay_file = replay_dir / "replay_data.json"
+        replay_file = replay_dir / "replay_data.json.gz"
 
         try:
-            async with aiofiles.open(replay_file, 'w', encoding='utf-8') as f:
-                content = json.dumps(local_data, indent=4, ensure_ascii=False)
-                await f.write(content)
-            logging.info(f"[WSS] Saved {hex_id} ({len(content)} bytes)")
+            raw = json.dumps(local_data, ensure_ascii=False).encode("utf-8")
+            compressed = await asyncio.to_thread(gzip.compress, raw)
+            async with aiofiles.open(replay_file, "wb") as f:
+                await f.write(compressed)
+            logging.info(f"[WSS] Saved {hex_id} ({len(compressed)} bytes compressed)")
         except Exception as e:
             logging.error(f"[WSS] Failed to save replay {hex_id}: {e}")
             continue
@@ -697,8 +699,8 @@ async def build_hex_plus_guild(
         replay_path = replay_data_path(sid)
 
         try:
-            async with aiofiles.open(replay_path, "r", encoding="utf-8") as fp:
-                replay_data = json.loads(await fp.read())
+            raw = await asyncio.to_thread(replay_path.read_bytes)
+            replay_data = json.loads(gzip.decompress(raw))
         except Exception:
             logging.error(f"SESSION HEX {sid} FAILED TO GET REPLAY DATA")
             mapping[sid] = (g, [])
@@ -958,15 +960,15 @@ async def process_session(
     """
     # Load replay JSON
     base_dir = replay_session_dir(session_id)
-    replay_path = base_dir / "replay_data.json"
+    replay_path = replay_data_path(session_id)
     try:
-        async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
-            replay_data = json.loads(await f.read())
+        raw = await asyncio.to_thread(replay_path.read_bytes)
+        replay_data = json.loads(gzip.decompress(raw))
     except FileNotFoundError:
         logging.error(f"Replay file not found for session ID {session_id}")
         return
-    except json.JSONDecodeError:
-        logging.error(f"Replay file for session ID {session_id} is invalid JSON")
+    except (OSError, json.JSONDecodeError) as e:
+        logging.error(f"Replay file for session ID {session_id} is invalid: {e}")
         return
 
     # Extract winner/loser/draw
@@ -1711,17 +1713,16 @@ async def process_comps(new_games):
         session_id = game.get('sessionIdHex')
         endtime_raw = game.get('endTime')
 
-        base_dir = replay_session_dir(session_id)
-        replay_path = base_dir / "replay_data.json"
+        replay_path = replay_data_path(session_id)
 
         try:
-            async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
-                replay_data = json.loads(await f.read())
+            raw = await asyncio.to_thread(replay_path.read_bytes)
+            replay_data = json.loads(gzip.decompress(raw))
         except FileNotFoundError:
             logging.warning(f"(COMP-WRITE) Replay file not found: {replay_path}")
             continue
-        except json.JSONDecodeError:
-            logging.warning(f"(COMP-WRITE) Invalid JSON in replay: {replay_path}")
+        except (OSError, json.JSONDecodeError) as e:
+            logging.warning(f"(COMP-WRITE) Invalid replay: {replay_path}: {e}")
             continue
 
         # Translate vehicle names
@@ -1870,15 +1871,12 @@ async def process_stats(new_games):
         for sid in sessions_to_process:
             endtime_raw = end_times.get(sid, 0)
 
-            base_dir = replay_session_dir(sid)
-            replay_path = base_dir / "replay_data.json"
+            replay_path = replay_data_path(sid)
 
             try:
-                async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
-                    replay_data = json.loads(await f.read())
-            except FileNotFoundError:
-                continue
-            except json.JSONDecodeError:
+                raw = await asyncio.to_thread(replay_path.read_bytes)
+                replay_data = json.loads(gzip.decompress(raw))
+            except (FileNotFoundError, OSError, json.JSONDecodeError):
                 continue
 
             winning_team = replay_data.get("winning_team_squadron")