compress the mf (#1267)

BOT/autologging.py

@@ -8,6 +8,7 @@ scoreboards, and posts results to configured Discord channels.
 
 # Standard Library Imports
 import asyncio
+import gzip
 import json
 import logging
 import os
@@ -389,10 +390,10 @@ def minutes_ago(ts, now=None):
 
 
 def load_replay_data_from_disk(session_id: str):
-    """Load replay_data.json from disk for a session."""
+    """Load replay_data.json.gz from disk for a session."""
     path = replay_data_path(session_id)
     if path.is_file():
-        with open(path, "r", encoding="utf-8") as f:
+        with gzip.open(path, "rt", encoding="utf-8") as f:
             return json.load(f)
     return None
 
@@ -477,13 +478,14 @@ async def process_ws_replays(replays: list[dict]):
         # Replay is valid - save to disk
         replay_dir = replay_session_dir(hex_id)
         replay_dir.mkdir(parents=True, exist_ok=True)
-        replay_file = replay_dir / "replay_data.json"
+        replay_file = replay_dir / "replay_data.json.gz"
 
         try:
-            async with aiofiles.open(replay_file, 'w', encoding='utf-8') as f:
+            raw = json.dumps(local_data, ensure_ascii=False).encode("utf-8")
-                content = json.dumps(local_data, indent=4, ensure_ascii=False)
+            compressed = await asyncio.to_thread(gzip.compress, raw)
-                await f.write(content)
+            async with aiofiles.open(replay_file, "wb") as f:
-            logging.info(f"[WSS] Saved {hex_id} ({len(content)} bytes)")
+                await f.write(compressed)
+            logging.info(f"[WSS] Saved {hex_id} ({len(compressed)} bytes compressed)")
         except Exception as e:
             logging.error(f"[WSS] Failed to save replay {hex_id}: {e}")
             continue
@@ -697,8 +699,8 @@ async def build_hex_plus_guild(
         replay_path = replay_data_path(sid)
 
         try:
-            async with aiofiles.open(replay_path, "r", encoding="utf-8") as fp:
+            raw = await asyncio.to_thread(replay_path.read_bytes)
-                replay_data = json.loads(await fp.read())
+            replay_data = json.loads(gzip.decompress(raw))
         except Exception:
             logging.error(f"SESSION HEX {sid} FAILED TO GET REPLAY DATA")
             mapping[sid] = (g, [])
@@ -958,15 +960,15 @@ async def process_session(
     """
     # Load replay JSON
     base_dir = replay_session_dir(session_id)
-    replay_path = base_dir / "replay_data.json"
+    replay_path = replay_data_path(session_id)
     try:
-        async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
+        raw = await asyncio.to_thread(replay_path.read_bytes)
-            replay_data = json.loads(await f.read())
+        replay_data = json.loads(gzip.decompress(raw))
     except FileNotFoundError:
         logging.error(f"Replay file not found for session ID {session_id}")
         return
-    except json.JSONDecodeError:
+    except (OSError, json.JSONDecodeError) as e:
-        logging.error(f"Replay file for session ID {session_id} is invalid JSON")
+        logging.error(f"Replay file for session ID {session_id} is invalid: {e}")
         return
 
     # Extract winner/loser/draw
@@ -1711,17 +1713,16 @@ async def process_comps(new_games):
         session_id = game.get('sessionIdHex')
         endtime_raw = game.get('endTime')
 
-        base_dir = replay_session_dir(session_id)
+        replay_path = replay_data_path(session_id)
-        replay_path = base_dir / "replay_data.json"
 
         try:
-            async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
+            raw = await asyncio.to_thread(replay_path.read_bytes)
-                replay_data = json.loads(await f.read())
+            replay_data = json.loads(gzip.decompress(raw))
         except FileNotFoundError:
             logging.warning(f"(COMP-WRITE) Replay file not found: {replay_path}")
             continue
-        except json.JSONDecodeError:
+        except (OSError, json.JSONDecodeError) as e:
-            logging.warning(f"(COMP-WRITE) Invalid JSON in replay: {replay_path}")
+            logging.warning(f"(COMP-WRITE) Invalid replay: {replay_path}: {e}")
             continue
 
         # Translate vehicle names
@@ -1870,15 +1871,12 @@ async def process_stats(new_games):
         for sid in sessions_to_process:
             endtime_raw = end_times.get(sid, 0)
 
-            base_dir = replay_session_dir(sid)
+            replay_path = replay_data_path(sid)
-            replay_path = base_dir / "replay_data.json"
 
             try:
-                async with aiofiles.open(replay_path, "r", encoding="utf-8") as f:
+                raw = await asyncio.to_thread(replay_path.read_bytes)
-                    replay_data = json.loads(await f.read())
+                replay_data = json.loads(gzip.decompress(raw))
-            except FileNotFoundError:
+            except (FileNotFoundError, OSError, json.JSONDecodeError):
-                continue
-            except json.JSONDecodeError:
                 continue
 
             winning_team = replay_data.get("winning_team_squadron")

BOT/botscript.py

@@ -8,6 +8,7 @@ translations, and administrative functions with interactive UI components.
 
 # Standard Library Imports
 import asyncio
+import gzip
 import json
 import logging
 import math
@@ -4412,8 +4413,10 @@ async def _send_view_match_scoreboard(interaction: discord.Interaction, session_
         # Save to disk for buttons (Chat Log, Battle Log read from disk)
         replay_dir = replay_session_dir(session_id)
         replay_dir.mkdir(parents=True, exist_ok=True)
-        async with aiofiles.open(replay_dir / "replay_data.json", "w", encoding="utf-8") as f:
+        raw = json.dumps(replay_data, ensure_ascii=False).encode("utf-8")
-            await f.write(json.dumps(replay_data, indent=4, ensure_ascii=False))
+        compressed = await asyncio.to_thread(gzip.compress, raw)
+        async with aiofiles.open(replay_dir / "replay_data.json.gz", "wb") as f:
+            await f.write(compressed)
 
     # 3. Translate vehicles
     translate = LangTableReader("English")

BOT/render_replay.py

@@ -532,7 +532,11 @@ def _clean_map_key(raw: str) -> str:
 
 def _load_json_file(path: Path) -> dict | None:
     try:
-        return json.loads(path.read_text())
+        import gzip as _gzip
+        raw = path.read_bytes()
+        if path.suffix == ".gz":
+            raw = _gzip.decompress(raw)
+        return json.loads(raw)
     except Exception as e:
         print(f"  Failed to parse {path.name}: {e}")
         return None
@@ -3279,8 +3283,12 @@ def _convert_local_replay_to_render_dict(replay: dict[str, Any]) -> dict[str, An
 
 
 def load_gob_file(replay_path: Path) -> dict[str, Any]:
-    """Load a replay .json file and normalize it for render/export routines."""
+    """Load a replay .json or .json.gz file and normalize it for render/export routines."""
-    data = json.loads(replay_path.read_text(encoding="utf-8"))
+    import gzip as _gzip
+    raw = replay_path.read_bytes()
+    if replay_path.suffix == ".gz":
+        raw = _gzip.decompress(raw)
+    data = json.loads(raw.decode("utf-8"))
     if isinstance(data, dict) and {"Players", "Entities", "Mission"}.issubset(data.keys()):
         return data
     if isinstance(data, dict):
@@ -3585,7 +3593,7 @@ def main():
     if args.replay:
         replay_path = Path(args.replay)
     else:
-        candidates = sorted(REPLAYS_DIR.glob("*/replay_data.json"))
+        candidates = sorted(REPLAYS_DIR.glob("*/replay_data.json.gz"))
         if not candidates:
             candidates = sorted(REPLAYS_DIR.glob("*.json"))
         if not candidates:

BOT/task_executors.py

@@ -292,7 +292,7 @@ async def cleanup_replays():
     whenever files inside are added or removed (including by this cleanup), which
     would otherwise keep dirs perpetually "fresh".
     """
-    KEEP_FILES = {"replay_data.json"}
+    KEEP_FILES = {"replay_data.json.gz"}
 
     def _sync_cleanup_replays():
         """Synchronous helper that walks replay dirs and deletes stale files."""
@@ -312,7 +312,7 @@ async def cleanup_replays():
             if not entry_path.is_dir():
                 continue
 
-            json_path = entry_path / "replay_data.json"
+            json_path = entry_path / "replay_data.json.gz"
             if json_path.exists():
                 entry_mtime = json_path.stat().st_mtime
             else:

BOT/utils.py

@@ -89,7 +89,7 @@ def replay_session_dir(session_id: str | int) -> Path:
 
 
 def replay_data_path(session_id: str | int) -> Path:
-    return replay_session_dir(session_id) / "replay_data.json"
+    return replay_session_dir(session_id) / "replay_data.json.gz"
 
 # Dev team Discord user IDs (bot owner + trusted devs)
 DEV_DISCORD_IDS: set[int] = {

server.js

@@ -27,7 +27,7 @@ const TSS_BATTLES_DB_PATH = path.join(STORAGE_ROOT, 'tss_battles.db');
 fs.mkdirSync(REPLAYS_PATH, { recursive: true });
 
 function replayDataPath(sessionId) {
-    return path.join(REPLAYS_PATH, String(sessionId).toLowerCase(), 'replay_data.json');
+    return path.join(REPLAYS_PATH, String(sessionId).toLowerCase(), 'replay_data.json.gz');
 }
 
 const app = express();
@@ -1443,7 +1443,7 @@ function readReplayJson(sessionId) {
         return null;
     }
     try {
-        return JSON.parse(fs.readFileSync(replayPath, 'utf-8'));
+        return JSON.parse(zlib.gunzipSync(fs.readFileSync(replayPath)));
     } catch (err) {
         log.warn('Failed to read replay JSON', { sessionId, error: err.message });
         return null;
@@ -2753,7 +2753,7 @@ app.get('/api/match/:sessionId/replay', (req, res) => {
     }
 
     try {
-        const data = JSON.parse(fs.readFileSync(replayPath, 'utf-8'));
+        const data = JSON.parse(zlib.gunzipSync(fs.readFileSync(replayPath)));
         res.json({
             available: true,
             session_id: sessionId,

web/server.js

@@ -1828,7 +1828,7 @@ app.get('/api/match/:sessionId/video', async (req, res) => {
 
     const sessionDir = resolveReplaySessionDir(sessionId);
     const videoPath = path.join(sessionDir, 'replay_video.mp4');
-    const replayPath = path.join(sessionDir, 'replay_data.json');
+    const replayPath = path.join(sessionDir, 'replay_data.json.gz');
 
     // 1. Serve from disk if cached
     if (fs.existsSync(videoPath)) {
@@ -1891,7 +1891,7 @@ app.get('/api/match/:sessionId/replay-canvas', async (req, res) => {
 
     const sessionDir = resolveReplaySessionDir(sessionId);
     const jsonPath = path.join(sessionDir, 'replay_canvas.json');
-    const replayPath = path.join(sessionDir, 'replay_data.json');
+    const replayPath = path.join(sessionDir, 'replay_data.json.gz');
 
     // 1. Serve from disk if cached
     if (fs.existsSync(jsonPath)) {